Overhaul gradle inclusion/exclusion logic (#620)

      with:

        java-version: ${{ env.java_version }}

    - name: generate a check all services

      run: ./gradlew :aws:sdk:cargoCheck

      env:

        GENERATE_ALL_SERVICES: 'true'

      run: ./gradlew -Paws.fullsdk=true :aws:sdk:cargoCheck

    - name: Generate a name for the SDK

      id: gen-name

      run: echo "name=${GITHUB_REF##*/}" >> $GITHUB_ENV

## vNext (Month Day Year)

**New This Week**

- :bug: Correctly encode HTTP Checksums using base64 instead of hex. Fixes aws-sdk-rust#164. (#615)

- Update SDK gradle build logic to use gradle properties (#620)

- (When complete) Add profile file provider for region (#594, #xyz)

- Overhaul serialization/deserialization of numeric/boolean types. This resolves issues around serialization of NaN/Infinity and should also reduce the number of allocations required during serialization. (#618)

- Update SQS example to clarify usage of FIFO vs. standard queues (#622, @trevorrobertsjr)

Run an SDK example:

`./gradlew :aws:sdk:runExample --example dynamo-helloworld`

## Controlling service generation

You can use gradle properties to opt/out of generating specific services:

```bash

# generate only s3,ec2,sts

./gradlew -Paws.services=+s3,+ec2,+sts :aws:sdk:assemble

# generate a complete SDK for release

./gradlew -Paws.fullsdk=true :aws:sdk:assemble

```

The generation logic is as follows:

1. If `aws.services` is specified, generate an SDK based on the inclusion/exclusion list.

2. Otherwise, if `aws.fullsdk` is specified generate an SDK based on `aws.services.fullsdk`.

3. Otherwise, generate an SDK based on `aws.services.tier1`

import software.amazon.smithy.model.shapes.ServiceShape

import software.amazon.smithy.aws.traits.ServiceTrait

import kotlin.streams.toList

import java.util.*

extra["displayName"] = "Smithy :: Rust :: AWS-SDK"

extra["moduleName"] = "software.amazon.smithy.rust.awssdk"

    implementation("software.amazon.smithy:smithy-aws-cloudformation-traits:$smithyVersion")

}

// Tier 1 Services have examples and tests

val tier1Services = setOf(

    "apigateway",

    "applicationautoscaling",

    "autoscaling",

    "autoscalingplans",

    "batch",

    "cloudformation",

    "cloudwatch",

    "cloudwatch",

    "cloudwatchlogs",

    "cognitoidentity",

    "cognitoidentityprovider",

    "cognitosync",

    "config",

    "dynamodb",

    "ebs",

    "ec2",

    "ecr",

    "ecs",

    "eks",

    "iam",

    "kinesis",

    "kms",

    "lambda",

    "medialive",

    "mediapackage",

    "polly",

    "qldb",

    "qldbsession",

    "rds",

    "rdsdata",

    "route53",

    "s3",

    "sagemaker",

    "sagemakera2iruntime",

    "sagemakeredge",

    "sagemakerfeaturestoreruntime",

    "secretsmanager",

    "sesv2",

    "snowball",

    "sns",

    "sqs",

    "ssm",

    "sts"

)

// get a project property by name if it exists (including from local.properties)

fun getProperty(name: String): String? {

    if (project.hasProperty(name)) {

        return project.properties[name].toString()

    }

private val disableServices = setOf(

    // transcribe streaming contains exclusively EventStream operations which are not supported

    "transcribestreaming",

    // Glacier requires a customization which is not currently supported:

    // https://github.com/awslabs/smithy-rs/issues/137

    "glacier",

    // https://github.com/awslabs/smithy-rs/issues/606

    "iotdataplane",

    // timestream requires endpoint discovery

    // https://github.com/awslabs/aws-sdk-rust/issues/114

    "timestreamwrite",

    "timestreamquery"

)

    val localProperties = Properties()

    val propertiesFile: File = rootProject.file("local.properties")

    if (propertiesFile.exists()) {

        propertiesFile.inputStream().use { localProperties.load(it) }

        if (localProperties.containsKey(name)) {

            return localProperties[name].toString()

        }

    }

    return null

}

// Class and functions for service and protocol membership for SDK generation

data class Membership(val inclusions: Set<String> = emptySet(), val exclusions: Set<String> = emptySet())

fun Membership.isMember(member: String): Boolean = when {

    exclusions.contains(member) -> false

    inclusions.contains(member) -> true

    inclusions.isEmpty() -> true

    else -> false

}

fun parseMembership(rawList: String): Membership {

    val inclusions = mutableSetOf<String>()

    val exclusions = mutableSetOf<String>()

    rawList.split(",").map { it.trim() }.forEach { item ->

        when {

            item.startsWith('-') -> exclusions.add(item.substring(1))

            item.startsWith('+') -> inclusions.add(item.substring(1))

            else -> error("Must specify inclusion (+) or exclusion (-) prefix character to $item.")

        }

    }

    val conflictingMembers = inclusions.intersect(exclusions)

    require(conflictingMembers.isEmpty()) { "$conflictingMembers specified both for inclusion and exclusion in $rawList" }

    return Membership(inclusions, exclusions)

}

data class AwsService(

    val service: String,

    fun files(): List<File> = listOf(modelFile) + extraFiles

}

val generateAllServices =

    project.providers.environmentVariable("GENERATE_ALL_SERVICES").forUseAtConfigurationTime().orElse("")

val awsServices: List<AwsService> by lazy { discoverServices() }

val generateOnly: Provider<Set<String>> =

    project.providers.environmentVariable("GENERATE_ONLY")

        .forUseAtConfigurationTime()

        .map { envVar ->

            envVar.split(",").filter { service -> service.trim().isNotBlank() }

fun loadServiceMembership(): Membership {

    val membershipOverride = getProperty("aws.services")?.let { parseMembership(it) }

    println(membershipOverride)

    val fullSdk = parseMembership(getProperty("aws.services.fullsdk") ?: throw kotlin.Exception("never list missing"))

    val tier1 = parseMembership(getProperty("aws.services.tier1") ?: throw kotlin.Exception("tier1 list missing"))

    return membershipOverride ?: if ((getProperty("aws.fullsdk") ?: "") == "true") {

        fullSdk

    } else {

        tier1

    }

        .orElse(listOf())

        .map { it.toSet() }

val awsServices: Provider<List<AwsService>> = generateAllServices.zip(generateOnly) { v, only ->

    discoverServices(v.toLowerCase() == "true", only)

}

/**

 *

 * Do not invoke this function directly. Use the `awsServices` provider.

 */

fun discoverServices(allServices: Boolean, generateOnly: Set<String>): List<AwsService> {

fun discoverServices(): List<AwsService> {

    val models = project.file("aws-models")

    val serviceMembership = loadServiceMembership()

    val baseServices = fileTree(models)

        .sortedBy { file -> file.name }

        .mapNotNull { file ->

            }

        }

    val baseModules = baseServices.map { it.module }.toSet()

    disableServices.forEach{ disabledService ->

    // validate the full exclusion list hits

    serviceMembership.exclusions.forEach { disabledService ->

        check(baseModules.contains(disabledService)) {

            "Service $disabledService was explicitly disabled but no service was generated with that name. Generated:\n ${baseModules.joinToString("\n ")}"

        }

            "Service $disabledService was explicitly disabled but no service was generated with that name. Generated:\n ${

                baseModules.joinToString(

                    "\n "

                )

            }"

        }

    val services = baseServices.filterNot {

        disableServices.contains(it.module)

    }.filter {

        val inGenerateOnly = generateOnly.isNotEmpty() && generateOnly.contains(it.module)

        val inTier1 = generateOnly.isEmpty() && tier1Services.contains(it.module)

        allServices || inGenerateOnly || inTier1

    }

    if (generateOnly.isEmpty()) {

        val modules = services.map { it.module }.toSet()

        tier1Services.forEach { service ->

            check(modules.contains(service)) { "Service $service was in list of tier 1 services but not generated! ($generateOnly)" }

    // validate inclusion list hits

    serviceMembership.inclusions.forEach { service ->

        check(baseModules.contains(service)) { "Service $service was in explicit inclusion list but not generated!" }

    }

    return baseServices.filter {

        serviceMembership.isMember(it.module)

    }

    return services

}

fun generateSmithyBuild(tests: List<AwsService>): String {

task("generateSmithyBuild") {

    description = "generate smithy-build.json"

    dependsOn(awsServices)

    doFirst {

        projectDir.resolve("smithy-build.json").writeText(generateSmithyBuild(awsServices.get()))

        projectDir.resolve("smithy-build.json").writeText(generateSmithyBuild(awsServices))

    }

    inputs.property("servicelist", awsServices.sortedBy { it.module }.toString())

    inputs.dir(projectDir.resolve("aws-models"))

    outputs.file(projectDir.resolve("smithy-build.json"))

}

task("relocateServices") {

    description = "relocate AWS services to their final destination"

    doLast {

        awsServices.get().forEach {

        awsServices.forEach {

            logger.info("Relocating ${it.module}...")

            copy {

                from("$buildDir/smithyprojections/sdk/${it.module}/rust-codegen")

    description = "generate Cargo.toml workspace file"

    doFirst {

        sdkOutputDir.mkdirs()

        sdkOutputDir.resolve("Cargo.toml").writeText(generateCargoWorkspace(awsServices.get()))

        sdkOutputDir.resolve("Cargo.toml").writeText(generateCargoWorkspace(awsServices))

    }

    dependsOn(awsServices)

    inputs.property("servicelist", awsServices.sortedBy { it.module }.toString())

    inputs.dir(projectDir.resolve("examples"))

    outputs.file(sdkOutputDir.resolve("Cargo.toml"))

    outputs.upToDateWhen { false }

tasks["smithyBuildJar"].inputs.file(projectDir.resolve("smithy-build.json"))

tasks["smithyBuildJar"].inputs.dir(projectDir.resolve("aws-models"))

tasks["smithyBuildJar"].dependsOn("generateSmithyBuild")

tasks["smithyBuildJar"].dependsOn(awsServices)

tasks["smithyBuildJar"].dependsOn("generateCargoWorkspace")

tasks["smithyBuildJar"].outputs.upToDateWhen { false }

tasks["assemble"].dependsOn("smithyBuildJar")

#

# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.

# SPDX-License-Identifier: Apache-2.0.

#

# several services are broken pending custom work:

# transcribe streaming contains exclusively EventStream operations which are not supported

# Glacier requires a customization which is not currently supported:

# https://github.com/awslabs/smithy-rs/issues/137

# IOT data plane requires a signing customization https://github.com/awslabs/smithy-rs/issues/606

# timestream requires endpoint discovery: https://github.com/awslabs/aws-sdk-rust/issues/114

aws.services.fullsdk=-transcribestreaming,-glacier,-iotdataplane,-timestreamwrite,-timestreamquery

# Generate an entire sdk vs. aws.services.tier1

aws.fullsdk=false

# base set of services that are generated unless other options are specified:

aws.services.tier1=+apigateway,\

    +applicationautoscaling,\

    +autoscaling,\

    +autoscalingplans,\

    +batch,\

    +cloudformation,\

    +cloudwatch,\

    +cloudwatch,\

    +cloudwatchlogs,\

    +cognitoidentity,\

    +cognitoidentityprovider,\

    +cognitosync,\

    +config,\

    +dynamodb,\

    +ebs,\

    +ec2,\

    +ecr,\

    +ecs,\

    +eks,\

    +iam,\

    +kinesis,\

    +kms,\

    +lambda,\

    +medialive,\

    +mediapackage,\

    +polly,\

    +qldb,\

    +qldbsession,\

    +rds,\

    +rdsdata,\

    +route53,\

    +s3,\

    +sagemaker,\

    +sagemakera2iruntime,\

    +sagemakeredge,\

    +sagemakerfeaturestoreruntime,\

    +secretsmanager,\

    +sesv2,\

    +snowball,\

    +sns,\

    +sqs,\

    +ssm,\

    +sts