diff --git a/aws/sdk/build.gradle.kts b/aws/sdk/build.gradle.kts index a8946018090e5a04b77aa5caf587ab54dc436be6..31a407964bcc4e091804de506d030188524643bd 100644 --- a/aws/sdk/build.gradle.kts +++ b/aws/sdk/build.gradle.kts @@ -5,6 +5,7 @@ import aws.sdk.AwsServices import aws.sdk.Membership +import aws.sdk.RootTest import aws.sdk.discoverServices import aws.sdk.docsLandingPage import aws.sdk.parseMembership @@ -198,6 +199,28 @@ tasks.register("relocateExamples") { outputs.dir(outputDir) } +tasks.register("relocateTests") { + description = "relocate the root integration tests and rewrite path dependencies" + doLast { + if (awsServices.rootTests.isNotEmpty()) { + copy { + val testDir = projectDir.resolve("integration-tests") + from(testDir) + awsServices.rootTests.forEach { test -> + include(test.path.toRelativeString(testDir) + "/**") + } + into(outputDir.resolve("tests")) + exclude("**/target") + filter { line -> line.replace("build/aws-sdk/sdk/", "sdk/") } + } + } + } + for (test in awsServices.rootTests) { + inputs.dir(test.path) + } + outputs.dir(outputDir) +} + tasks.register("fixExampleManifests") { description = "Adds dependency path and corrects version number of examples after relocation" enabled = awsServices.examples.isNotEmpty() @@ -272,6 +295,8 @@ tasks.register("relocateChangelog") { fun generateCargoWorkspace(services: AwsServices): String { return """ |[workspace] + |exclude = [${"\n"}${services.rootTests.map(RootTest::manifestName).joinToString(",\n") { "| \"$it\"" }} + |] |members = [${"\n"}${services.allModules.joinToString(",\n") { "| \"$it\"" }} |] """.trimMargin() @@ -287,6 +312,9 @@ tasks.register("generateCargoWorkspace") { if (awsServices.examples.isNotEmpty()) { inputs.dir(projectDir.resolve("examples")) } + for (test in awsServices.rootTests) { + inputs.dir(test.path) + } outputs.file(outputDir.resolve("Cargo.toml")) outputs.upToDateWhen { false } } @@ -310,6 +338,7 @@ tasks.register("fixManifests") { dependsOn("relocateRuntime") dependsOn("relocateAwsRuntime") dependsOn("relocateExamples") + dependsOn("relocateTests") } tasks.register("hydrateReadme") { @@ -371,6 +400,7 @@ tasks.register("finalizeSdk") { "relocateRuntime", "relocateAwsRuntime", "relocateExamples", + "relocateTests", "generateIndexMd", "fixManifests", "generateVersionManifest", diff --git a/aws/sdk/integration-tests/dynamodb/Cargo.toml b/aws/sdk/integration-tests/dynamodb/Cargo.toml index da5539583884993c013c3cbbe49afe7e96eceb2c..bf47eddcca55f917e7e562f4021df6f6aaee563d 100644 --- a/aws/sdk/integration-tests/dynamodb/Cargo.toml +++ b/aws/sdk/integration-tests/dynamodb/Cargo.toml @@ -4,6 +4,9 @@ name = "dynamo-tests" version = "0.1.0" authors = ["AWS Rust SDK Team ", "Russell Cohen "] edition = "2021" +license = "Apache-2.0" +repository = "https://github.com/awslabs/smithy-rs" +publish = false # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html diff --git a/aws/sdk/integration-tests/ec2/Cargo.toml b/aws/sdk/integration-tests/ec2/Cargo.toml index ee6c83b09477d78d3aaa09c351fc0278ddb1dda2..d7580d6b8968372c08fa1aced6090a5a43718795 100644 --- a/aws/sdk/integration-tests/ec2/Cargo.toml +++ b/aws/sdk/integration-tests/ec2/Cargo.toml @@ -2,6 +2,9 @@ name = "ec2-tests" version = "0.1.0" edition = "2021" +license = "Apache-2.0" +repository = "https://github.com/awslabs/smithy-rs" +publish = false [dev-dependencies] aws-credential-types = { path = "../../build/aws-sdk/sdk/aws-credential-types", features = ["test-util"] } diff --git a/aws/sdk/integration-tests/glacier/Cargo.toml b/aws/sdk/integration-tests/glacier/Cargo.toml index ea1d13b69526a0ffc7c2eecc104d2b5441b25bea..88e20751d2a4b08662e1444c873805d0c863786d 100644 --- a/aws/sdk/integration-tests/glacier/Cargo.toml +++ b/aws/sdk/integration-tests/glacier/Cargo.toml @@ -4,6 +4,9 @@ name = "glacier-tests" version = "0.1.0" authors = ["AWS Rust SDK Team "] edition = "2021" +license = "Apache-2.0" +repository = "https://github.com/awslabs/smithy-rs" +publish = false # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html diff --git a/aws/sdk/integration-tests/iam/Cargo.toml b/aws/sdk/integration-tests/iam/Cargo.toml index b9061aa85ffa861cb725f4987618d76d2be6675b..ea3ff36ea023ce555a91de675cd8bf9aef33fe59 100644 --- a/aws/sdk/integration-tests/iam/Cargo.toml +++ b/aws/sdk/integration-tests/iam/Cargo.toml @@ -4,6 +4,9 @@ name = "iam-tests" version = "0.1.0" authors = ["AWS Rust SDK Team "] edition = "2021" +license = "Apache-2.0" +repository = "https://github.com/awslabs/smithy-rs" +publish = false # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html diff --git a/aws/sdk/integration-tests/kms/Cargo.toml b/aws/sdk/integration-tests/kms/Cargo.toml index 4e6611d4a4482a7780a6444eae298a1ac05eddd3..bfe5ed5cd973d8f90619ae15409d0fc83766c8fd 100644 --- a/aws/sdk/integration-tests/kms/Cargo.toml +++ b/aws/sdk/integration-tests/kms/Cargo.toml @@ -4,6 +4,9 @@ name = "kms-tests" version = "0.1.0" authors = ["Russell Cohen "] edition = "2021" +license = "Apache-2.0" +repository = "https://github.com/awslabs/smithy-rs" +publish = false # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html diff --git a/aws/sdk/integration-tests/lambda/Cargo.toml b/aws/sdk/integration-tests/lambda/Cargo.toml index 830e3eaf756a29fd9291bb54f6ce479831dbddc9..5492fbc68a1f2ee8907a9b66071bc7d3015da4cd 100644 --- a/aws/sdk/integration-tests/lambda/Cargo.toml +++ b/aws/sdk/integration-tests/lambda/Cargo.toml @@ -3,6 +3,9 @@ name = "lambda" version = "0.1.0" authors = ["AWS Rust SDK Team ", "Zelda Hessler "] edition = "2021" +license = "Apache-2.0" +repository = "https://github.com/awslabs/smithy-rs" +publish = false [dev-dependencies] async-stream = "0.3.0" diff --git a/aws/sdk/integration-tests/no-default-features/Cargo.toml b/aws/sdk/integration-tests/no-default-features/Cargo.toml index 45980e2b90b4f22baccc07ee3ec8b3e2d4622b6b..bb0ba9d02d63ce8ebe5e9bcc0cdf6ae7440ac0eb 100644 --- a/aws/sdk/integration-tests/no-default-features/Cargo.toml +++ b/aws/sdk/integration-tests/no-default-features/Cargo.toml @@ -1,4 +1,3 @@ -# This Cargo.toml is unused in generated code. It exists solely to enable these tests to compile in-situ [package] name = "no-default-features" version = "0.1.0" @@ -8,6 +7,9 @@ These tests ensure that things will fail (or not fail) as expected when default features are disabled for all SDK and runtime crates. """ edition = "2021" +license = "Apache-2.0" +repository = "https://github.com/awslabs/smithy-rs" +publish = false # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html diff --git a/aws/sdk/integration-tests/polly/Cargo.toml b/aws/sdk/integration-tests/polly/Cargo.toml index c5ea81ebecc204ccd82921943e425d4ebb4fe8b0..b7b1a0b6f86ed9f23f4c013c8b40d2722214eed3 100644 --- a/aws/sdk/integration-tests/polly/Cargo.toml +++ b/aws/sdk/integration-tests/polly/Cargo.toml @@ -4,6 +4,9 @@ name = "polly-tests" version = "0.1.0" authors = ["John DiSanti "] edition = "2021" +license = "Apache-2.0" +repository = "https://github.com/awslabs/smithy-rs" +publish = false # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html diff --git a/aws/sdk/integration-tests/qldbsession/Cargo.toml b/aws/sdk/integration-tests/qldbsession/Cargo.toml index fe28618f59dbd45538680a8ba3d2864a5bef840d..f6710cb575c887eb61cf91a68750660a1c755da2 100644 --- a/aws/sdk/integration-tests/qldbsession/Cargo.toml +++ b/aws/sdk/integration-tests/qldbsession/Cargo.toml @@ -4,6 +4,9 @@ name = "qldb-tests" version = "0.1.0" authors = ["Russell Cohen ", "Shing Lyu "] edition = "2021" +license = "Apache-2.0" +repository = "https://github.com/awslabs/smithy-rs" +publish = false # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html diff --git a/aws/sdk/integration-tests/s3control/Cargo.toml b/aws/sdk/integration-tests/s3control/Cargo.toml index 53b4c48c73dcf8876a3020d228a30ed8bd7fe4ff..ada6d95b415b7147a6c730c379d809aafde3b7f1 100644 --- a/aws/sdk/integration-tests/s3control/Cargo.toml +++ b/aws/sdk/integration-tests/s3control/Cargo.toml @@ -4,6 +4,9 @@ name = "s3control" version = "0.1.0" authors = ["AWS Rust SDK Team "] edition = "2021" +license = "Apache-2.0" +repository = "https://github.com/awslabs/smithy-rs" +publish = false # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html diff --git a/aws/sdk/integration-tests/sts/Cargo.toml b/aws/sdk/integration-tests/sts/Cargo.toml index 874ce80c894e13923f18d7725ffb7a2c4c223ac3..63c91c463927cf39f9cba2c56e20c2e7a55fdd82 100644 --- a/aws/sdk/integration-tests/sts/Cargo.toml +++ b/aws/sdk/integration-tests/sts/Cargo.toml @@ -4,6 +4,9 @@ name = "sts-tests" version = "0.1.0" authors = ["Russell Cohen "] edition = "2021" +license = "Apache-2.0" +repository = "https://github.com/awslabs/smithy-rs" +publish = false # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html diff --git a/aws/sdk/integration-tests/transcribestreaming/Cargo.toml b/aws/sdk/integration-tests/transcribestreaming/Cargo.toml index b398e3554de252f77eb048725785542de67f6832..1c19e73f9f1c0f7f1acdbf4c932540d7e826cb06 100644 --- a/aws/sdk/integration-tests/transcribestreaming/Cargo.toml +++ b/aws/sdk/integration-tests/transcribestreaming/Cargo.toml @@ -1,8 +1,12 @@ +# This Cargo.toml is unused in generated code. It exists solely to enable these tests to compile in-situ [package] name = "transcribestreaming" version = "0.1.0" authors = ["AWS Rust SDK Team ", "John DiSanti "] edition = "2021" +license = "Apache-2.0" +repository = "https://github.com/awslabs/smithy-rs" +publish = false [dev-dependencies] async-stream = "0.3.0" diff --git a/buildSrc/src/main/kotlin/aws/sdk/ServiceLoader.kt b/buildSrc/src/main/kotlin/aws/sdk/ServiceLoader.kt index bb99ff02515dd410c46b67d670dd8844a84be8de..fdc6448b0c58a592fd42eac981bf2a6ab834e86e 100644 --- a/buildSrc/src/main/kotlin/aws/sdk/ServiceLoader.kt +++ b/buildSrc/src/main/kotlin/aws/sdk/ServiceLoader.kt @@ -14,6 +14,11 @@ import software.amazon.smithy.model.traits.TitleTrait import java.io.File import kotlin.streams.toList +data class RootTest( + val path: File, + val manifestName: String, +) + class AwsServices( private val project: Project, services: List, @@ -23,37 +28,56 @@ class AwsServices( val services: List val moduleNames: Set by lazy { services.map { it.module }.toSortedSet() } + init { + this.services = services.sortedBy { it.module } + } + val allModules: Set by lazy { ( services.map(AwsService::module).map { "sdk/$it" } + CrateSet.AWS_SDK_SMITHY_RUNTIME.map { "sdk/$it" } + CrateSet.AWS_SDK_RUNTIME.map { "sdk/$it" } + examples + // Root tests should not be included since they can't be part of the root Cargo workspace + // in order to test differences in Cargo features. ).toSortedSet() } val examples: List by lazy { project.projectDir.resolve("examples") .listFiles { file -> !file.name.startsWith(".") }.orEmpty().toList() - .filter { file -> - val cargoToml = File(file, "Cargo.toml") - if (cargoToml.exists()) { - val usedModules = cargoToml.readLines() - .map { line -> line.substringBefore('=').trim() } - .filter { line -> line.startsWith("aws-sdk-") } - .map { line -> line.substringAfter("aws-sdk-") } - .toSet() - moduleNames.containsAll(usedModules) - } else { - false - } - } + .filter { file -> manifestCompatibleWithGeneratedServices(file) } .map { "examples/${it.name}" } } - init { - this.services = services.sortedBy { it.module } + /** + * Tests in `aws/sdk/integration-tests` that are not named after a service module, and therefore, + * are not included in a service's `tests/` directory. These are to be included at the SDK root + * `tests/` directory for inclusion in CI. + */ + val rootTests: List by lazy { + project.projectDir.resolve("integration-tests") + .listFiles { file -> !file.name.startsWith(".") }.orEmpty().toList() + .filter { file -> !moduleNames.contains(file.name) && manifestCompatibleWithGeneratedServices(file) } + .map { file -> RootTest(file, "tests/${file.name}") } } + + /** + * Returns true if the Cargo manifest in the given path is compatible with the set of generated services. + */ + private fun manifestCompatibleWithGeneratedServices(path: File) = + File(path, "Cargo.toml").let { cargoToml -> + if (cargoToml.exists()) { + val usedModules = cargoToml.readLines() + .map { line -> line.substringBefore('=').trim() } + .filter { line -> line.startsWith("aws-sdk-") } + .map { line -> line.substringAfter("aws-sdk-") } + .toSet() + moduleNames.containsAll(usedModules) + } else { + false + } + } } /** diff --git a/tools/ci-build/scripts/check-aws-sdk-services b/tools/ci-build/scripts/check-aws-sdk-services index 9d25ba0e29001d7f886d2ef0a34b17be9bfa6550..a440c98f8396c738f191eeae8dd77f048288b18e 100755 --- a/tools/ci-build/scripts/check-aws-sdk-services +++ b/tools/ci-build/scripts/check-aws-sdk-services @@ -11,3 +11,10 @@ cd aws-sdk sed -i '/"examples\//d' Cargo.toml cargo test --all-features + +for test_dir in tests/*; do + if [ -f "${test_dir}/Cargo.toml" ]; then + echo "#### Testing ${test_dir}..." + cargo test --all-features --manifest-path "${test_dir}/Cargo.toml" + fi +done diff --git a/tools/ci-build/scripts/check-only-aws-sdk-services b/tools/ci-build/scripts/check-only-aws-sdk-services index 494f47f13c949b05fa4ec2942862910b2723f512..27c93ea479ef41ab313fa7b22234784c532b640c 100755 --- a/tools/ci-build/scripts/check-only-aws-sdk-services +++ b/tools/ci-build/scripts/check-only-aws-sdk-services @@ -13,3 +13,10 @@ cd aws-sdk sed -i '/"examples\//d' Cargo.toml cargo check --all-targets --all-features + +for test_dir in tests/*; do + if [ -f "${test_dir}/Cargo.toml" ]; then + echo "#### Checking ${test_dir}..." + cargo check --all-targets --all-features --manifest-path "${test_dir}/Cargo.toml" + fi +done