Unverified Commit 6e1d1f6c authored by John DiSanti's avatar John DiSanti Committed by GitHub
Browse files

Fix adhoc SDK codegen and refactor SDK CI (#1624) 

* Fix `aws:sdk-codegen-test` build
* Add config for SDK endpoint/readme generation
* Rename `sdk-codegen-test` to `sdk-adhoc-test` and update README
* Add `sdk-adhoc-tests` to CI
* Refactor CI scripts

The `check-rust-runtime-and-tools` script was getting beyond 20
minutes to run, so this commit splits it into `check-aws-config`,
`check-rust-runtimes`, and `check-tools`.

This also enables most of the checks to run before the smoke test SDK is
generated, which should deliver more immediate feedback on PRs.

* Give the canary its own CI step
parent 70a62193

.github/workflows/ci.yml

+9 −6
Original line number Diff line number Diff line
@@ -59,16 +59,19 @@ jobs: 
      matrix:

        # These correspond to scripts in tools/ci-build/scripts that will be run in the Docker build image

        test:

        - action: check-aws-sdk-adhoc-tests

        - action: check-client-codegen-integration-tests

        - action: check-client-codegen-unit-tests

        - action: check-rust-runtimes

        - action: check-sdk-codegen-unit-tests

        - action: check-server-codegen-integration-tests

        - action: check-server-codegen-unit-tests

        - action: check-server-codegen-integration-tests-python

        - action: check-server-codegen-unit-tests

        - action: check-server-codegen-unit-tests-python

        - action: check-server-e2e-test

        - action: check-server-python-e2e-test

        - action: check-style-and-lints

        - action: check-tools

    steps:

    - uses: actions/checkout@v3

      with:
@@ -80,8 +83,8 @@ jobs: 


  # Test all the things that require generated code. Note: the Rust runtimes require codegen

  # to be checked since `aws-config` depends on the generated STS client.

  test-runtimes-tools-and-sdk:

    name: Test Rust Runtimes, Tools, and SDK

  test-sdk:

    name: Test the SDK

    needs: generate

    runs-on: ubuntu-latest

    # To avoid repeating setup boilerplate, we have the actual test commands
@@ -91,12 +94,12 @@ jobs: 
      matrix:

        # These correspond to scripts in tools/ci-build/scripts that will be run in the Docker build image

        test:

        - action: check-aws-config

        - action: check-aws-sdk-canary

        - action: check-aws-sdk-services

        - action: check-aws-sdk-smoketest-additional-checks

        - action: check-aws-sdk-smoketest-docs-clippy-udeps

        - action: check-aws-sdk-smoketest-unit-tests

        - action: check-aws-sdk-standalone-integration-tests

        - action: check-rust-runtimes-and-tools

    steps:

    - uses: actions/checkout@v3

      with:
@@ -159,7 +162,7 @@ jobs: 
    needs:

    - generate

    - test-codegen

    - test-runtimes-tools-and-sdk

    - test-sdk

    - test-rust-windows

    # Run this job even if its dependency jobs fail

    if: always()

aws/rust-runtime/aws-config/additional-ci

deleted100755 → 0
+0 −24
Original line number Diff line number Diff line 
#!/bin/bash

#

# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.

# SPDX-License-Identifier: Apache-2.0

#



# This script contains additional CI checks to run for this specific package



set -e



echo "### Checking for external types in public API"

cargo "+${RUST_NIGHTLY_VERSION:-nightly}" api-linter --all-features --config api-linter.toml



echo "### Checking for duplicate dependency versions in the normal dependency graph with all features enabled"

cargo tree -d --edges normal --all-features



# `aws-config` is not part of the `rust-runtime` workspace. As a result it is not run through

# `check-rust-runtimes-and-tools` and does not get `cargo test --all-features` prior to this point. As a result we do

# not apply `--exclude-all-features` here.

echo "### Testing every combination of features"

cargo hack test --feature-powerset



echo "### Checking that compiling with the minimal versions succeeds"

cargo "+${RUST_NIGHTLY_VERSION:-nightly}" minimal-versions check --all-features

aws/sdk-adhoc-test/README.md

0 → 100644
+19 −0
Original line number Diff line number Diff line 
AWS SDK Adhoc Codegen Test

==========================



This module tests that adhoc SDKs can be generated without the rest of the

release automation machinery used to make the official SDK releases.

The `build.gradle.kts` generates a `smithy-build.json` file as part of

the build, and the Smithy build plugin then invokes the SDK codegen

to generate a client.



This module also exists to code generate and execute service specific protocol tests such as

[ApiGateway](https://github.com/awslabs/smithy/blob/main/smithy-aws-protocol-tests/model/restJson1/services/apigateway.smithy).



Usage

-----



```

# From repo root:

./gradlew :aws:sdk-adhoc-test:test

```

aws/sdk-codegen-test/build.gradle.kts→aws/sdk-adhoc-test/build.gradle.kts

+32 −3
Original line number Diff line number Diff line
@@ -8,14 +8,28 @@ extra["moduleName"] = "software.amazon.smithy.kotlin.codegen.test" 


tasks["jar"].enabled = false



plugins { id("software.amazon.smithy").version("0.5.3") }

plugins {

    val smithyGradlePluginVersion: String by project

    id("software.amazon.smithy").version(smithyGradlePluginVersion)

}



val smithyVersion: String by project

val defaultRustDocFlags: String by project

val properties = PropertyRetriever(rootProject, project)



val pluginName = "rust-codegen"

val workingDirUnderBuildDir = "smithyprojections/sdk-codegen-test/"

val workingDirUnderBuildDir = "smithyprojections/sdk-adhoc-test/"



configure<software.amazon.smithy.gradle.SmithyExtension> {

    outputDirectory = file("$buildDir/$workingDirUnderBuildDir")

}



buildscript {

    val smithyVersion: String by project

    dependencies {

        classpath("software.amazon.smithy:smithy-cli:$smithyVersion")

    }

}



dependencies {

    implementation(project(":aws:sdk-codegen"))
@@ -25,11 +39,26 @@ dependencies { 
}



val allCodegenTests = listOf(

    CodegenTest("com.amazonaws.apigateway#BackplaneControlService", "apigateway"),

    CodegenTest(

        "com.amazonaws.apigateway#BackplaneControlService",

        "apigateway",

        extraConfig = """

            ,

            "codegen": {

                "includeFluentClient": false

            },

            "customizationConfig": {

                "awsSdk": {

                    "generateReadme": false

                }

            }

        """,

    ),

)



project.registerGenerateSmithyBuildTask(rootProject, pluginName, allCodegenTests)

project.registerGenerateCargoWorkspaceTask(rootProject, pluginName, allCodegenTests, workingDirUnderBuildDir)

project.registerGenerateCargoConfigTomlTask(buildDir.resolve(workingDirUnderBuildDir))



tasks["smithyBuildJar"].dependsOn("generateSmithyBuild")

tasks["assemble"].finalizedBy("generateCargoWorkspace")