Unverified Commit 7245df1c authored by John DiSanti's avatar John DiSanti Committed by GitHub
Browse files

Generate a `versions.toml` with the SDK (#1311) 

* Implement `crate-hasher` utility to hash crate source files
* Add `generate-version-manifest` subcommand to publisher tool
* Incorporate `generate-version-manifest` into the build system
* Start making the PR bot run through the Docker build image
* Use nightly clippy in tools CI when necessary
* Fix transient failure when acquiring the build image
* Include examples repo revision in `versions.toml`
* Fix nightly clippy lint in `cargo-api-linter`
* Track examples revision in sync tool
* Make it easier to run Docker CI locally
parent b7a8eb58

.github/actions/docker-build/action.yml

+4 −1
Original line number Diff line number Diff line
@@ -9,6 +9,9 @@ inputs: 
  action:

    description: What action to run in the Docker build

    required: true

  action-arguments:

    description: Arguments to pass to the action

    required: false

runs:

  using: composite

  steps:
@@ -53,7 +56,7 @@ runs: 
  - name: Run ${{ inputs.action }}

    shell: bash

    run: |

      ./smithy-rs/tools/ci-build/ci-action ${{ inputs.action }}

      ./smithy-rs/tools/ci-build/ci-action ${{ inputs.action }} ${{ inputs.action-arguments}}

      tar cfz artifacts-${{ inputs.action }}.tar.gz -C artifacts .

  - name: Upload artifacts

    uses: actions/upload-artifact@v3

.github/workflows/ci.yml

+11 −0
Original line number Diff line number Diff line
@@ -36,6 +36,17 @@ jobs: 
        path: smithy-rs-base-image

        retention-days: 1



  # The PR bot requires a Docker build image, so make it depend on the `acquire-base-image` job.

  pr_bot:

    name: PR Bot

    needs: acquire-base-image

    # Only run this job on pull requests (not directly on main)

    if: ${{ github.head_ref }}

    uses: ./.github/workflows/pull-request-bot.yml

    secrets:

      SMITHY_RS_PULL_REQUEST_CDN_S3_BUCKET_NAME: ${{ secrets.SMITHY_RS_PULL_REQUEST_CDN_S3_BUCKET_NAME }}

      SMITHY_RS_PULL_REQUEST_CDN_ROLE_ARN: ${{ secrets.SMITHY_RS_PULL_REQUEST_CDN_ROLE_ARN }}



  # The `generate` job runs scripts that produce artifacts that are required by the `test` job,

  # and also runs some checks/lints so that those are run sooner rather than later.

  generate:

.github/workflows/pull-request-bot.yml

+25 −33
Original line number Diff line number Diff line 
name: PR Bot

# This job will generate a codegen diff, upload it to S3, and link to it in a comment on the PR.

name: PR Bot

on:

  pull_request:

    types:

    - opened

    - reopened

    - synchronize

  workflow_call:

    secrets:

      SMITHY_RS_PULL_REQUEST_CDN_S3_BUCKET_NAME:

        required: true

      SMITHY_RS_PULL_REQUEST_CDN_ROLE_ARN:

        required: true



# Allow one instance of this workflow per pull request, and cancel older runs when new changes are pushed

concurrency:
@@ -33,32 +34,13 @@ jobs: 
      bot-message: ${{ steps.generate-diff.outputs.bot-message }}

    steps:

    - uses: actions/checkout@v2

    - uses: actions/cache@v2

      name: Gradle Cache

      with:

        path: |

          ~/.gradle/caches

          ~/.gradle/wrapper

        key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle*') }}

        restore-keys: |

          ${{ runner.os }}-gradle-

    # JDK is needed to generate code

    - name: Set up JDK

      uses: actions/setup-java@v1

      with:

        java-version: ${{ env.java_version }}

    # Node is needed to run diff2html

    - name: Set up NodeJS

      uses: actions/setup-node@v2

      with:

        node-version: '16'

    - name: Install diff2html-cli

      run: npm install -g diff2html-cli@5.1.11

        path: smithy-rs

    - name: Generate diff

      id: generate-diff

      run: |

        ./tools/codegen-diff-revisions.py . ${{ github.event.pull_request.base.sha }}

        echo "::set-output name=bot-message::$(cat tmp-codegen-diff/bot-message)"

      uses: ./smithy-rs/.github/actions/docker-build

      with:

        action: generate-codegen-diff

        action-arguments: ${{ github.event.pull_request.base.sha }}

    - uses: aws-actions/configure-aws-credentials@v1

      name: Acquire credentials for uploading to S3

      with:
@@ -67,8 +49,8 @@ jobs: 
        aws-region: us-west-2

    - name: Upload diff to S3

      run: |

        if [[ -d tmp-codegen-diff/${{ github.event.pull_request.base.sha }} ]]; then

            aws s3 cp tmp-codegen-diff/${{ github.event.pull_request.base.sha }} \

        if [[ -d artifacts/codegen-diff/${{ github.event.pull_request.base.sha }} ]]; then

            aws s3 cp artifacts/codegen-diff/${{ github.event.pull_request.base.sha }} \

                "s3://${S3_BUCKET_NAME}/codegen-diff/${{ github.event.pull_request.base.sha }}" --recursive

        fi
@@ -224,6 +206,16 @@ jobs: 
      contents: read

      pull-requests: write

    steps:

    - uses: actions/checkout@v3

      with:

        path: smithy-rs

    - name: Download all artifacts

      uses: ./smithy-rs/.github/actions/download-all-artifacts

    - name: Set bot message outputs

      id: bot-messages

      run: |

        set -eux

        echo ::set-output name=codegen-diff::"$(cat ./bot-message-codegen-diff)"

    - name: Post bot comment

      uses: actions/github-script@v5

      if: ${{ github.head_ref != null }}
@@ -233,7 +225,7 @@ jobs: 
            issue_number: context.issue.number,

            owner: context.repo.owner,

            repo: context.repo.repo,

            body: '${{ needs.generate-diff.outputs.bot-message }}\n\n' +

            body: '${{ steps.bot-messages.outputs.codegen-diff }}\n\n' +

              '${{ needs.generate-doc-preview.outputs.bot-message }}\n\n' +

              '${{ needs.generate-server-benchmark.outputs.bot-message }}\n\n'

          })

.gitignore

+0 −1
Original line number Diff line number Diff line
@@ -18,7 +18,6 @@ 
*.nar

*.ear

*.zip

*.tar.gz

*.rar



# virtual machine crash logs, see http://www.java.com/en/download/help/error_hotspot.xml

aws/sdk/build.gradle.kts

+29 −1
Original line number Diff line number Diff line
@@ -23,6 +23,7 @@ val defaultRustFlags: String by project 
val defaultRustDocFlags: String by project

val properties = PropertyRetriever(rootProject, project)



val crateHasherToolPath = rootProject.projectDir.resolve("tools/crate-hasher")

val publisherToolPath = rootProject.projectDir.resolve("tools/publisher")

val sdkVersionerToolPath = rootProject.projectDir.resolve("tools/sdk-versioner")

val outputDir = buildDir.resolve("aws-sdk")
@@ -314,6 +315,32 @@ tasks.register<ExecRustBuildTool>("hydrateReadme") { 
    )

}



tasks.register<RequireRustBuildTool>("requireCrateHasher") {

    description = "Ensures the crate-hasher tool is available"

    inputs.dir(crateHasherToolPath)

    toolPath = crateHasherToolPath

}



tasks.register<ExecRustBuildTool>("generateVersionManifest") {

    description = "Generate the SDK version.toml file"

    dependsOn("requireCrateHasher")

    dependsOn("fixManifests")



    inputs.dir(publisherToolPath)



    toolPath = publisherToolPath

    binaryName = "publisher"

    arguments = listOf(

        "generate-version-manifest",

        "--location",

        outputDir.absolutePath,

        "--smithy-build",

        outputDir.resolve("../../smithy-build.json").normalize().absolutePath,

        "--examples-revision",

        properties.get("aws.sdk.examples.revision") ?: "missing"

    )

}



task("finalizeSdk") {

    dependsOn("assemble")

    outputs.upToDateWhen { false }
@@ -325,7 +352,8 @@ task("finalizeSdk") { 
        "generateIndexMd",

        "fixManifests",

        "hydrateReadme",

        "relocateChangelog"

        "relocateChangelog",

        "generateVersionManifest"

    )

}