Unverified Commit 3a9e64e5 authored by John DiSanti's avatar John DiSanti Committed by GitHub
Browse files

Test the orchestrator against the SDK in CI (#2715) 

## Motivation and Context
This PR adds the SDK smoke test with the orchestrator implementation to
CI, and excludes services that don't yet pass tests.

----

_By submitting this pull request, I confirm that you can use, modify,
copy, and redistribute this contribution, under the terms of your
choice._
parent 9bfe936f

.github/workflows/ci.yml

+3 −0
Original line number Diff line number Diff line
@@ -84,6 +84,9 @@ jobs: 
        test:

        - action: check-aws-sdk-adhoc-tests

          runner: ubuntu-latest

        # TODO(enableNewSmithyRuntime): Remove `check-aws-sdk-orchestrator-impl` when cleaning up middleware

        - action: check-aws-sdk-orchestrator-impl

          runner: smithy_ubuntu-latest_8-core

        - action: check-client-codegen-integration-tests

          runner: smithy_ubuntu-latest_8-core

        - action: check-client-codegen-unit-tests

aws/sdk/README.md

+1 −1
Original line number Diff line number Diff line
@@ -12,7 +12,7 @@ Generate an SDK: 
`./gradlew :aws:sdk:assemble`



Generate, compile, and test an SDK:

`./gradlew :aws:sdk:build`

`./gradlew :aws:sdk:check`



Run an SDK example:

`./gradlew :aws:sdk:runExample --example dynamo-helloworld`

aws/sdk/build.gradle.kts

+27 −23
Original line number Diff line number Diff line
@@ -151,6 +151,8 @@ tasks.register("generateSmithyBuild") { 
}



tasks.register("generateIndexMd") {

    dependsOn("smithyBuildJar")



    inputs.property("servicelist", awsServices.services.toString())

    val indexMd = outputDir.resolve("index.md")

    outputs.file(indexMd)
@@ -161,6 +163,8 @@ tasks.register("generateIndexMd") { 


tasks.register("relocateServices") {

    description = "relocate AWS services to their final destination"

    dependsOn("smithyBuildJar")



    doLast {

        awsServices.services.forEach {

            logger.info("Relocating ${it.module}...")
@@ -186,6 +190,8 @@ tasks.register("relocateServices") { 


tasks.register("relocateExamples") {

    description = "relocate the examples folder & rewrite path dependencies"

    dependsOn("smithyBuildJar")



    doLast {

        if (awsServices.examples.isNotEmpty()) {

            copy {
@@ -207,6 +213,8 @@ tasks.register("relocateExamples") { 


tasks.register("relocateTests") {

    description = "relocate the root integration tests and rewrite path dependencies"

    dependsOn("smithyBuildJar")



    doLast {

        if (awsServices.rootTests.isNotEmpty()) {

            copy {
@@ -230,6 +238,7 @@ tasks.register("relocateTests") { 
tasks.register<ExecRustBuildTool>("fixExampleManifests") {

    description = "Adds dependency path and corrects version number of examples after relocation"

    enabled = awsServices.examples.isNotEmpty()

    dependsOn("relocateExamples")



    toolPath = sdkVersionerToolPath

    binaryName = "sdk-versioner"
@@ -257,6 +266,7 @@ fun rewritePathDependency(line: String): String { 
}



tasks.register<Copy>("copyAllRuntimes") {

    dependsOn("smithyBuildJar")

    from("$rootDir/aws/rust-runtime") {

        CrateSet.AWS_SDK_RUNTIME.forEach { include("$it/**") }

    }
@@ -293,6 +303,7 @@ tasks.register("relocateRuntime") { 
}



tasks.register<Copy>("relocateChangelog") {

    dependsOn("smithyBuildJar")

    from("$rootDir/aws")

    include("SDK_CHANGELOG.md")

    into(outputDir)
@@ -328,6 +339,11 @@ tasks.register("generateCargoWorkspace") { 


tasks.register<ExecRustBuildTool>("fixManifests") {

    description = "Run the publisher tool's `fix-manifests` sub-command on the generated services"

    dependsOn("relocateServices")

    dependsOn("relocateRuntime")

    dependsOn("relocateAwsRuntime")

    dependsOn("relocateExamples")

    dependsOn("relocateTests")



    inputs.dir(publisherToolPath)

    outputs.dir(outputDir)
@@ -339,18 +355,10 @@ tasks.register<ExecRustBuildTool>("fixManifests") { 
            add("--disable-version-number-validation")

        }

    }



    dependsOn("assemble")

    dependsOn("relocateServices")

    dependsOn("relocateRuntime")

    dependsOn("relocateAwsRuntime")

    dependsOn("relocateExamples")

    dependsOn("relocateTests")

}



tasks.register<ExecRustBuildTool>("hydrateReadme") {

    description = "Run the publisher tool's `hydrate-readme` sub-command to create the final AWS Rust SDK README file"



    dependsOn("generateVersionManifest")



    inputs.dir(publisherToolPath)
@@ -401,10 +409,17 @@ tasks.register<ExecRustBuildTool>("generateVersionManifest") { 
    }

}



tasks.register("finalizeSdk") {

    dependsOn("assemble")

tasks["smithyBuildJar"].apply {

    inputs.file(buildDir.resolve("smithy-build.json"))

    inputs.dir(projectDir.resolve("aws-models"))

    dependsOn("generateSmithyBuild")

    dependsOn("generateCargoWorkspace")

    outputs.upToDateWhen { false }

    finalizedBy(

}

tasks["assemble"].apply {

    dependsOn(

        "deleteSdk",

        "smithyBuildJar",

        "relocateServices",

        "relocateRuntime",

        "relocateAwsRuntime",
@@ -417,24 +432,13 @@ tasks.register("finalizeSdk") { 
        "hydrateReadme",

        "relocateChangelog",

    )

}



tasks["smithyBuildJar"].apply {

    inputs.file(buildDir.resolve("smithy-build.json"))

    inputs.dir(projectDir.resolve("aws-models"))

    dependsOn("generateSmithyBuild")

    dependsOn("generateCargoWorkspace")

    outputs.upToDateWhen { false }

}

tasks["assemble"].apply {

    dependsOn("deleteSdk")

    dependsOn("smithyBuildJar")

    finalizedBy("finalizeSdk")

}



project.registerCargoCommandsTasks(outputDir, defaultRustDocFlags)

project.registerGenerateCargoConfigTomlTask(outputDir)



tasks["test"].dependsOn("assemble")

tasks["test"].finalizedBy(Cargo.CLIPPY.toString, Cargo.TEST.toString, Cargo.DOCS.toString)



tasks.register<Delete>("deleteSdk") {

tools/ci-scripts/check-aws-sdk-orchestrator-impl

0 → 100755
+59 −0
Original line number Diff line number Diff line 
#!/bin/bash

#

# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.

# SPDX-License-Identifier: Apache-2.0

#



# This script tests the SDK smoke test services against the orchestrator implementation



C_YELLOW='\033[1;33m'

C_RESET='\033[0m'



set -eu

cd smithy-rs



# TODO(enableNewSmithyRuntime): Move these into `services_that_compile` as more progress is made

services_that_fail_compile=(\

    "s3"\

    "s3control"\

    "transcribestreaming"\

)



# TODO(enableNewSmithyRuntime): Move these into `services_that_pass_tests` as more progress is made

services_that_compile=(\

    "dynamodb"\

    "glacier"\

    "route53"\

    "sts"\

)



services_that_pass_tests=(\

    "config"\

    "ec2"\

    "ecs"\

    "iam"\

    "kms"\

    "lambda"\

    "polly"\

    "qldbsession"\

    "sso"\

)



./gradlew aws:sdk:assemble -Psmithy.runtime.mode=orchestrator



cd aws/sdk/build/aws-sdk/sdk

for service in "${services_that_compile[@]}"; do

    pushd "${service}"

    echo -e "${C_YELLOW}# Running 'cargo check --all-features' on '${service}'${C_RESET}"

    cargo check --all-features

    popd

done



for service in "${services_that_pass_tests[@]}"; do

    pushd "${service}"

    echo -e "${C_YELLOW}# Running 'cargo test --all-features' on '${service}'${C_RESET}"

    cargo test --all-features --no-fail-fast

    popd

done



echo "SUCCESS"