Unverified Commit 3b54f124 authored by John DiSanti's avatar John DiSanti Committed by GitHub
Browse files

Reorganize CI and add per-crate CI checks (#942)

* Add ability to have per-crate CI checks

* Split SDK CI into a separate workflow file

* Combine SDK tests into one matrix job

* Rename the codegen diff workflow

* Combine codegen tests into one matrix job
parent cb67d800
Loading
Loading
Loading
Loading
+210 −0
Original line number Diff line number Diff line
on:
  push:
    branches: [main]
    tags:
    - '*'
  pull_request:

name: AWS SDK CI

env:
  rust_version: 1.54.0
  rust_toolchain_components: clippy,rustfmt
  java_version: 11

jobs:
  generate-smoke-test:
    name: Smoke Test - Generate
    runs-on: ubuntu-latest
    steps:
    - uses: actions/checkout@v2
    - uses: actions/cache@v2
      name: Gradle Cache
      with:
        path: |
          ~/.gradle/caches
          ~/.gradle/wrapper
        key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle*') }}
        restore-keys: |
          ${{ runner.os }}-gradle-
    - uses: actions-rs/toolchain@v1
      with:
        toolchain: ${{ env.rust_version }}
        components: ${{ env.rust_toolchain_components }}
        default: true
    - name: Set up JDK
      uses: actions/setup-java@v1
      with:
        java-version: ${{ env.java_version }}
    - name: Generate the SDK
      run: ./gradlew :aws:sdk:assemble
    - name: Generate a list of services with tests
      run: python aws/sdk/test-services.py > aws/sdk/build/aws-sdk/services-with-tests
    - name: Generate a name for the SDK
      id: gen-name
      run: echo "name=${GITHUB_REF##*/}" >> $GITHUB_ENV
    - name: Tar the SDK
      run: tar -cvf sdk.tar -C aws/sdk/build/aws-sdk/ .
    - uses: actions/upload-artifact@v2
      name: Upload SDK Artifact
      with:
        name: aws-sdk-${{ env.name }}-smoketest-${{ github.sha }}
        path: sdk.tar

  smoke-test:
    name: Smoke Test
    needs: generate-smoke-test
    runs-on: ubuntu-latest
    strategy:
      fail-fast: false
      matrix:
        test:
        - name: Unit Tests
          run: cargo test $(cat service-with-tests)
          working-directory: aws-sdk
        - name: Docs
          run: cargo doc --no-deps --document-private-items
          working-directory: aws-sdk
        - name: Clippy
          run: cargo clippy
          working-directory: aws-sdk
    env:
      # Disable incremental compilation to reduce disk space use
      CARGO_INCREMENTAL: 0
      RUSTFLAGS: -D warnings
      # Note: the .cargo/config.toml is lost because we untar the SDK rather than checking out the repo,
      # so we have to manually restore the target directory override
      CARGO_TARGET_DIR: ../target
    steps:
    - uses: actions-rs/toolchain@v1
      with:
        toolchain: ${{ env.rust_version }}
        components: ${{ env.rust_toolchain_components }}
        default: true
    - name: Generate a name for the SDK
      id: gen-name
      run: echo "name=${GITHUB_REF##*/}" >> $GITHUB_ENV
    - uses: actions/download-artifact@v2
      name: Download SDK Artifact
      with:
        name: aws-sdk-${{ env.name }}-smoketest-${{ github.sha }}
        path: artifact
    - name: untar
      run: mkdir aws-sdk && cd aws-sdk && tar -xvf ../artifact/sdk.tar
      # Pinned to the commit hash of v1.3.0
    - uses: Swatinem/rust-cache@842ef286fff290e445b90b4002cc9807c3669641
      with:
        sharedKey: ${{ runner.os }}-${{ env.rust_version }}-${{ github.job }}
        target-dir: ../target
    - name: ${{ matrix.test.name }}
      run: ${{ matrix.test.run }}
      working-directory: ${{ matrix.test.working-directory }}

  unused-sdk-dependencies:
    name: Smoke Test - Unused dependencies
    needs: generate-smoke-test
    runs-on: ubuntu-latest
    steps:
      # Pinned to the commit hash of v1.3.0
    - uses: Swatinem/rust-cache@842ef286fff290e445b90b4002cc9807c3669641
      with:
        sharedKey: ${{ runner.os }}-${{ github.job }}
        target-dir: ../target
    - uses: actions-rs/toolchain@v1
      with:
          # Cargo udeps requires nightly
        toolchain: nightly
        default: true
    - name: Generate a name for the SDK
      id: gen-name
      run: echo "name=${GITHUB_REF##*/}" >> $GITHUB_ENV
    - uses: actions/download-artifact@v2
      name: Download SDK Artifact
      with:
        name: aws-sdk-${{ env.name }}-smoketest-${{ github.sha }}
        path: artifact
    - name: untar
      run: mkdir aws-sdk && cd aws-sdk && tar -xvf ../artifact/sdk.tar
    - name: Install `cargo udeps`
      run: cargo install cargo-udeps
    - name: Check for unused dependencies with default features
      run: cargo udeps
      working-directory: aws-sdk
    - name: Check for unused dependencies with `--all-features`
      run: cargo udeps --all-features
      working-directory: aws-sdk
    env:
      # Note: the .cargo/config.toml is lost because we untar the SDK rather than checking out the repo,
      # so we have to manually restore the target directory override
      CARGO_TARGET_DIR: ../target

  standalone-integration-tests-check:
    name: Standalone Integration Tests - cargo check
    needs: generate-smoke-test
    runs-on: ubuntu-latest
    steps:
    - uses: actions/checkout@v2
      # Pinned to the commit hash of v1.3.0
    - uses: Swatinem/rust-cache@842ef286fff290e445b90b4002cc9807c3669641
      with:
        working-directory: aws/sdk/integration-tests
        sharedKey: ${{ runner.os }}-${{ env.rust_version }}-${{ github.job }}
        target-dir: ../../../target
    - uses: actions-rs/toolchain@v1
      with:
        toolchain: ${{ env.rust_version }}
        components: ${{ env.rust_toolchain_components }}
        default: true
    # The integration tests path-depend on crates in the build/ path, so we have to download a generated SDK
    - name: Generate a name for the SDK
      id: gen-name
      run: echo "name=${GITHUB_REF##*/}" >> $GITHUB_ENV
    - uses: actions/download-artifact@v2
      name: Download SDK Artifact
      with:
        name: aws-sdk-${{ env.name }}-smoketest-${{ github.sha }}
        path: artifact
    - name: untar
      run: mkdir -p aws/sdk/build/aws-sdk && cd aws/sdk/build/aws-sdk && tar -xvf ../../../../artifact/sdk.tar
    - name: Check integration tests
      run: cargo check
      working-directory: aws/sdk/integration-tests
      env:
        RUSTC_FORCE_INCREMENTAL: 1
        RUSTFLAGS: -D warnings

  all-services-check:
    name: Full SDK - Generate and cargo check
    runs-on: ubuntu-latest
    steps:
    - uses: actions/checkout@v2
    - uses: actions/cache@v2
      name: Gradle Cache
      with:
        path: |
          ~/.gradle/caches
          ~/.gradle/wrapper
        key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle*') }}
        restore-keys: |
          ${{ runner.os }}-gradle-
    - uses: actions-rs/toolchain@v1
      with:
        toolchain: ${{ env.rust_version }}
        components: ${{ env.rust_toolchain_components }}
        default: true
    - name: Set up JDK
      uses: actions/setup-java@v1
      with:
        java-version: ${{ env.java_version }}
    - name: Generate and check all services
      run: ./gradlew -Paws.fullsdk=true :aws:sdk:cargoCheck
    - name: Generate a name for the SDK
      id: gen-name
      run: echo "name=${GITHUB_REF##*/}" >> $GITHUB_ENV
    - name: Tar the SDK
      run: tar -cvf sdk.tar -C aws/sdk/build/aws-sdk/ .
    - uses: actions/upload-artifact@v2
      name: Upload SDK Artifact
      with:
        name: aws-sdk-${{ env.name }}-${{ github.sha }}
        path: sdk.tar
+29 −342
Original line number Diff line number Diff line
@@ -14,7 +14,7 @@ env:

jobs:
  repo-lint:
    name: smithy-rs custom repo lints
    name: Repo Lints
    runs-on: ubuntu-latest
    steps:
    - uses: actions/checkout@v2
@@ -29,65 +29,26 @@ jobs:
    - run: cargo run -- fix --all
      name: run fixes
      working-directory: tools/sdk-lints
  style:
    name: Kotlin style checks
    runs-on: ubuntu-latest
    steps:
    - uses: actions/checkout@v2
    - name: Set up JDK
      uses: actions/setup-java@v1
      with:
        java-version: ${{ env.java_version }}
    - uses: actions/cache@v2
      with:
        path: |
          ~/.gradle/caches
          ~/.gradle/wrapper
        key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle*') }}
        restore-keys: |
          ${{ runner.os }}-gradle-
    - name: ktlint
      run: ./gradlew ktlint

  unit-tests:
    name: Codegen unit tests
  codegen-tests:
    name: Codegen Tests
    runs-on: ubuntu-latest
    steps:
    - uses: actions/checkout@v2
    - uses: actions/cache@v2
      with:
        path: |
          ~/.gradle/caches
          ~/.gradle/wrapper
        key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle*') }}
        restore-keys: |
          ${{ runner.os }}-gradle-
    - uses: actions/cache@v2
      with:
        path: |
          ~/.cargo/registry
          ~/.cargo/git
          target
        key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }}
    - uses: actions-rs/toolchain@v1
      with:
        toolchain: ${{ env.rust_version }}
        components: ${{ env.rust_toolchain_components }}
        default: true
    - name: Set up JDK
      uses: actions/setup-java@v1
      with:
        java-version: ${{ env.java_version }}
    - name: client tests
    strategy:
      fail-fast: false
      matrix:
        test:
        - name: Kotlin Style
          run: ./gradlew ktlint
        - name: Client Unit Tests
          run: ./gradlew :codegen:test
    - name: aws tests
        - name: SDK Unit Tests
          run: ./gradlew :aws:sdk-codegen:test
    - name: server tests
        - name: Server Unit Tests
          run: ./gradlew :codegen-server:test

  integration-tests:
    name: Codegen integration tests
    runs-on: ubuntu-latest
        - name: Client Integration Tests
          run: ./gradlew :codegen-test:test
        - name: Server Integration Tests
          run: ./gradlew :codegen-server-test:test
    steps:
    - uses: actions/checkout@v2
    - uses: actions/cache@v2
@@ -99,14 +60,11 @@ jobs:
        key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle*') }}
        restore-keys: |
          ${{ runner.os }}-gradle-
    - uses: actions/cache@v2
      name: Cargo Cache
      # Pinned to the commit hash of v1.3.0
    - uses: Swatinem/rust-cache@842ef286fff290e445b90b4002cc9807c3669641
      with:
        path: |
          ~/.cargo/registry
          ~/.cargo/git
          target
        key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }}
        sharedKey: ${{ runner.os }}-${{ env.rust_version }}-${{ github.job }}
        target-dir: ./target
    - uses: actions-rs/toolchain@v1
      with:
        toolchain: ${{ env.rust_version }}
@@ -116,31 +74,11 @@ jobs:
      uses: actions/setup-java@v1
      with:
        java-version: ${{ env.java_version }}
    - name: client integration-tests
      run: ./gradlew :codegen-test:test
    - uses: actions/upload-artifact@v2
      name: Upload Client Codegen Output for inspection
        # Always upload the output even if the tests failed
      if: ${{ always() }}
      with:
        name: client-codegen-output
        path: |
          codegen-test/build/smithyprojections/codegen-test/*/rust-codegen/
          codegen-test/build/smithyprojections/codegen-test/Cargo.toml
    - name: server integration-tests
      run: ./gradlew :codegen-server-test:test
    - uses: actions/upload-artifact@v2
      name: Upload Server Codegen Output for inspection
        # Always upload the output even if the tests failed
      if: ${{ always() }}
      with:
        name: server-codegen-output
        path: |
          codegen-server-test/build/smithyprojections/codegen-server-test/*/rust-server-codegen/
          codegen-server-test/build/smithyprojections/codegen-server-test/Cargo.toml
    - name: ${{ matrix.test.name }}
      run: ${{ matrix.test.run }}

  runtime-tests:
    name: Rust runtime tests
    name: Rust Runtime Tests
    strategy:
      matrix:
        os: [ubuntu-latest, windows-latest]
@@ -151,9 +89,8 @@ jobs:
    # Pinned to the commit hash of v1.3.0
    - uses: Swatinem/rust-cache@842ef286fff290e445b90b4002cc9807c3669641
      with:
        working-directory: ${{ matrix.runtime }}/rust-runtime/
        sharedKey: ${{ runner.os }}-${{ env.rust_version }}-${{ github.job }}
        target-dir: ../target
        target-dir: ./target
    - uses: actions-rs/toolchain@v1
      with:
        toolchain: ${{ env.rust_version }}
@@ -176,255 +113,5 @@ jobs:
      working-directory: ${{ matrix.runtime }}/rust-runtime/
      env:
        RUSTDOCFLAGS: -D warnings

  generate-sdk:
    name: AWS Rust SDK Tier 1 - Generate
    runs-on: ubuntu-latest
    steps:
    - uses: actions/checkout@v2
    - uses: actions/cache@v2
      name: Gradle Cache
      with:
        path: |
          ~/.gradle/caches
          ~/.gradle/wrapper
        key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle*') }}
        restore-keys: |
          ${{ runner.os }}-gradle-
    - uses: actions-rs/toolchain@v1
      with:
        toolchain: ${{ env.rust_version }}
        components: ${{ env.rust_toolchain_components }}
        default: true
    - name: Set up JDK
      uses: actions/setup-java@v1
      with:
        java-version: ${{ env.java_version }}
    - name: Generate the SDK
      run: ./gradlew :aws:sdk:assemble
    - name: Generate a list of services with tests
      run: python aws/sdk/test-services.py > aws/sdk/build/aws-sdk/services-with-tests
    - name: Generate a name for the SDK
      id: gen-name
      run: echo "name=${GITHUB_REF##*/}" >> $GITHUB_ENV
    - name: Tar the SDK
      run: tar -cvf sdk.tar -C aws/sdk/build/aws-sdk/ .
    - uses: actions/upload-artifact@v2
      name: Upload SDK Artifact
      with:
        name: aws-sdk-${{ env.name }}-tier1-${{ github.sha }}
        path: sdk.tar

  test-sdk:
    name: AWS Rust SDK Tier 1 - cargo test
    needs: generate-sdk
    runs-on: ubuntu-latest
    steps:
    - uses: actions-rs/toolchain@v1
      with:
        toolchain: ${{ env.rust_version }}
        components: ${{ env.rust_toolchain_components }}
        default: true
    - name: Generate a name for the SDK
      id: gen-name
      run: echo "name=${GITHUB_REF##*/}" >> $GITHUB_ENV
    - uses: actions/download-artifact@v2
      name: Download SDK Artifact
      with:
        name: aws-sdk-${{ env.name }}-tier1-${{ github.sha }}
        path: artifact
    - name: untar
      run: mkdir aws-sdk && cd aws-sdk && tar -xvf ../artifact/sdk.tar
    # Pinned to the commit hash of v1.3.0
    - uses: Swatinem/rust-cache@842ef286fff290e445b90b4002cc9807c3669641
      with:
        sharedKey: ${{ runner.os }}-${{ env.rust_version }}-${{ github.job }}
        target-dir: ../target
    - name: Cargo Test
      run: cargo test $(cat service-with-tests)
      working-directory: aws-sdk
      env:
        # Disable incremental compilation to reduce disk space use
        CARGO_INCREMENTAL: 0
        RUSTFLAGS: -D warnings
        # Note: the .cargo/config.toml is lost because we untar the SDK rather than checking out the repo,
        # so we have to manually restore the target directory override
        CARGO_TARGET_DIR: ../target

  docs-sdk:
    name: AWS Rust SDK Tier 1 - cargo docs
    needs: generate-sdk
    runs-on: ubuntu-latest
    steps:
    # Pinned to the commit hash of v1.3.0
    - uses: Swatinem/rust-cache@842ef286fff290e445b90b4002cc9807c3669641
      with:
        sharedKey: ${{ runner.os }}-${{ env.rust_version }}-${{ github.job }}
        target-dir: ../target
    - uses: actions-rs/toolchain@v1
      with:
        toolchain: ${{ env.rust_version }}
        components: ${{ env.rust_toolchain_components }}
        default: true
    - name: Generate a name for the SDK
      id: gen-name
      run: echo "name=${GITHUB_REF##*/}" >> $GITHUB_ENV
    - uses: actions/download-artifact@v2
      name: Download SDK Artifact
      with:
        name: aws-sdk-${{ env.name }}-tier1-${{ github.sha }}
        path: artifact
    - name: untar
      run: mkdir aws-sdk && cd aws-sdk && tar -xvf ../artifact/sdk.tar
    - name: Cargo Docs
      run: cargo doc --no-deps --document-private-items
      working-directory: aws-sdk
      env:
        RUSTDOCFLAGS: -D warnings
        # Note: the .cargo/config.toml is lost because we untar the SDK rather than checking out the repo,
        # so we have to manually restore the target directory override
        CARGO_TARGET_DIR: ../target

  clippy-sdk:
    name: AWS Rust SDK Tier 1 - cargo clippy
    needs: generate-sdk
    runs-on: ubuntu-latest
    steps:
    # Pinned to the commit hash of v1.3.0
    - uses: Swatinem/rust-cache@842ef286fff290e445b90b4002cc9807c3669641
      with:
        sharedKey: ${{ runner.os }}-${{ env.rust_version }}-${{ github.job }}
        target-dir: ../target
    - uses: actions-rs/toolchain@v1
      with:
        toolchain: ${{ env.rust_version }}
        components: ${{ env.rust_toolchain_components }}
        default: true
    - name: Generate a name for the SDK
      id: gen-name
      run: echo "name=${GITHUB_REF##*/}" >> $GITHUB_ENV
    - uses: actions/download-artifact@v2
      name: Download SDK Artifact
      with:
        name: aws-sdk-${{ env.name }}-tier1-${{ github.sha }}
        path: artifact
    - name: untar
      run: mkdir aws-sdk && cd aws-sdk && tar -xvf ../artifact/sdk.tar
    - name: Cargo Clippy
      run: cargo clippy -- -D warnings
      working-directory: aws-sdk
      env:
        # Note: the .cargo/config.toml is lost because we untar the SDK rather than checking out the repo,
        # so we have to manually restore the target directory override
        CARGO_TARGET_DIR: ../target

  unused-sdk-dependencies:
    name: AWS Rust SDK Tier 1 - Unused dependencies
    needs: generate-sdk
    runs-on: ubuntu-latest
    steps:
    # Pinned to the commit hash of v1.3.0
    - uses: Swatinem/rust-cache@842ef286fff290e445b90b4002cc9807c3669641
      with:
        sharedKey: ${{ runner.os }}-${{ github.job }}
        target-dir: ../target
    - uses: actions-rs/toolchain@v1
      with:
        # Cargo udeps requires nightly
        toolchain: nightly
        default: true
    - name: Generate a name for the SDK
      id: gen-name
      run: echo "name=${GITHUB_REF##*/}" >> $GITHUB_ENV
    - uses: actions/download-artifact@v2
      name: Download SDK Artifact
      with:
        name: aws-sdk-${{ env.name }}-tier1-${{ github.sha }}
        path: artifact
    - name: untar
      run: mkdir aws-sdk && cd aws-sdk && tar -xvf ../artifact/sdk.tar
    - name: Install `cargo udeps`
      run: cargo install cargo-udeps
    - name: Check for unused dependencies with default features
      run: cargo udeps
      working-directory: aws-sdk
    - name: Check for unused dependencies with `--all-features`
      run: cargo udeps --all-features
      working-directory: aws-sdk
    env:
      # Note: the .cargo/config.toml is lost because we untar the SDK rather than checking out the repo,
      # so we have to manually restore the target directory override
      CARGO_TARGET_DIR: ../target

  standalone-integration-tests-check:
    name: AWS Rust SDK Standalone Integration Tests - cargo check
    needs: generate-sdk
    runs-on: ubuntu-latest
    steps:
    - uses: actions/checkout@v2
    - uses: actions/cache@v2
      name: Gradle Cache
      with:
        path: |
          ~/.gradle/caches
          ~/.gradle/wrapper
        key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle*') }}
        restore-keys: |
          ${{ runner.os }}-gradle-
    - uses: actions-rs/toolchain@v1
      with:
        toolchain: ${{ env.rust_version }}
        components: ${{ env.rust_toolchain_components }}
        default: true
    - name: Generate a name for the SDK
      id: gen-name
      run: echo "name=${GITHUB_REF##*/}" >> $GITHUB_ENV
    - uses: actions/download-artifact@v2
      name: Download SDK Artifact
      with:
        name: aws-sdk-${{ env.name }}-tier1-${{ github.sha }}
        path: artifact
    - name: untar
      run: mkdir -p aws/sdk/build/aws-sdk && cd aws/sdk/build/aws-sdk && tar -xvf ../../../../artifact/sdk.tar
    - name: Check integration tests
      run: cargo check
      working-directory: aws/sdk/integration-tests
      env:
        RUSTC_FORCE_INCREMENTAL: 1
        RUSTFLAGS: -D warnings

  all-services-check:
    name: AWS Rust SDK Tier 2 - Generate and cargo check
    runs-on: ubuntu-latest
    steps:
    - uses: actions/checkout@v2
    - uses: actions/cache@v2
      name: Gradle Cache
      with:
        path: |
          ~/.gradle/caches
          ~/.gradle/wrapper
        key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle*') }}
        restore-keys: |
          ${{ runner.os }}-gradle-
    - uses: actions-rs/toolchain@v1
      with:
        toolchain: ${{ env.rust_version }}
        components: ${{ env.rust_toolchain_components }}
        default: true
    - name: Set up JDK
      uses: actions/setup-java@v1
      with:
        java-version: ${{ env.java_version }}
    - name: Generate and check all services
      run: ./gradlew -Paws.fullsdk=true :aws:sdk:cargoCheck
    - name: Generate a name for the SDK
      id: gen-name
      run: echo "name=${GITHUB_REF##*/}" >> $GITHUB_ENV
    - name: Tar the SDK
      run: tar -cvf sdk.tar -C aws/sdk/build/aws-sdk/ .
    - uses: actions/upload-artifact@v2
      name: Upload SDK Artifact
      with:
        name: aws-sdk-${{ env.name }}-${{ github.sha }}
        path: sdk.tar
    - name: Additional per-crate checks
      run: ./tools/additional-per-crate-checks.sh ${{ matrix.runtime }}/rust-runtime/
+1 −1
Original line number Diff line number Diff line
name: codegen diff preview
name: PR Bot
# This job will generate a codegen diff, upload it to S3, and link to it in a comment on the PR.
on:
  pull_request:
+20 −0
Original line number Diff line number Diff line
#!/bin/bash
#
# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
# SPDX-License-Identifier: Apache-2.0.
#

# This script contains additional CI checks to run for this specific package

set -e

echo "### Checking for duplicate dependency versions in the normal dependency graph with all features enabled"
cargo tree -d --edges normal --all-features

FEATURES=(
    "hardcoded-credentials"
)
for feature in "${FEATURES[@]}"; do
    echo "### Checking feature '${feature}'..."
    cargo test --no-default-features --features "${feature}"
done
+12 −0
Original line number Diff line number Diff line
#!/bin/bash
#
# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
# SPDX-License-Identifier: Apache-2.0.
#

# This script contains additional CI checks to run for this specific package

set -e

echo "### Checking for duplicate dependency versions in the normal dependency graph with all features enabled"
cargo tree -d --edges normal --all-features
Loading