From 52c1174ac3694bdda938dd91c39af291a2d91e01 Mon Sep 17 00:00:00 2001 From: John DiSanti Date: Mon, 27 Jun 2022 12:16:48 -0700 Subject: [PATCH] Decouple `aws-sdk-rust` release from `smithy-rs` release (#1495) * Remove smoketest/fullsdk model classifications * Generate full SDK for CI using models in aws-sdk-rust * Move the changelog renderer out of `sdk-lints` and add changelog splitter * Save revision information into split off SDK changelog * Replace common git impl with `sdk-sync`'s impl * Filter SDK changelog entries on changelog render * Add smithy-rs release workflow * Add initial next SDK changelog * Update `sdk-sync` to understand new models location --- .github/workflows/ci.yml | 8 +- .../release-scripts/create-release.js | 83 ++ .github/workflows/release.yml | 95 ++ aws/SDK_CHANGELOG.md | 1107 ----------------- aws/SDK_CHANGELOG.next.json | 49 + aws/sdk/README.md | 19 +- aws/sdk/aws-models/README.md | 16 + aws/sdk/build.gradle.kts | 12 +- aws/sdk/gradle.properties | 32 +- .../src/main/kotlin/aws/sdk/ServiceLoader.kt | 30 +- ci.mk | 4 +- tools/Dockerfile | 1 + tools/api-linter/Cargo.lock | 1 + tools/changelogger/Cargo.lock | 522 ++++++++ tools/changelogger/Cargo.toml | 29 + .../smithy-rs-maintainers.txt | 0 tools/changelogger/src/entry.rs | 112 ++ tools/changelogger/src/lib.rs | 8 + tools/changelogger/src/main.rs | 152 +++ tools/changelogger/src/render.rs | 576 +++++++++ tools/changelogger/src/split.rs | 93 ++ tools/changelogger/tests/e2e_test.rs | 349 ++++++ tools/ci-build/ci-create-workspace | 1 + tools/ci-build/scripts/generate-aws-sdk | 22 +- .../scripts/generate-aws-sdk-smoketest | 4 +- .../scripts/generate-smithy-rs-release | 46 + .../scripts/generate-smithy-rs-runtime-bundle | 11 - tools/ci-build/scripts/sanity-test | 1 + tools/ci-cdk/canary-runner/Cargo.lock | 1 + tools/ci-cdk/canary-runner/src/run.rs | 29 +- tools/codegen-diff-revisions.py | 4 +- tools/publisher/Cargo.lock | 1 + tools/publisher/src/main.rs | 1 - tools/publisher/src/repo.rs | 42 - .../subcommand/generate_version_manifest.rs | 12 +- .../src/subcommand/hydrate_readme.rs | 6 +- tools/publisher/src/subcommand/publish.rs | 40 +- tools/sdk-lints/Cargo.lock | 158 +-- tools/sdk-lints/Cargo.toml | 7 +- tools/sdk-lints/src/changelog.rs | 614 +-------- tools/sdk-lints/src/main.rs | 144 --- tools/sdk-sync/Cargo.lock | 24 + tools/sdk-sync/fake-cli/git-squash-merge | 19 - tools/sdk-sync/src/git.rs | 613 --------- tools/sdk-sync/src/lib.rs | 1 - tools/sdk-sync/src/main.rs | 1 + tools/sdk-sync/src/sync.rs | 99 +- tools/sdk-sync/src/sync/gen.rs | 25 +- tools/sdk-sync/src/versions.rs | 2 +- tools/sdk-sync/tests/create-test-workspace | 38 +- tools/sdk-sync/tests/e2e_test.rs | 60 +- tools/sdk-sync/tests/fake-sdk-assemble | 17 +- tools/sdk-versioner/Cargo.lock | 24 + tools/smithy-rs-tool-common/Cargo.toml | 1 + .../fake-cli/git-changed-files | 0 .../fake-cli/git-changed-files-empty | 0 .../fake-cli/git-clone | 0 .../fake-cli/git-commit | 0 .../fake-cli/git-commit-on-behalf | 0 .../fake-cli/git-create-branch | 0 .../fake-cli/git-current-branch-name | 0 .../fake-cli/git-delete-branch | 0 .../fake-cli/git-extract-commit-info | 0 .../fake-cli/git-get-head-revision | 0 .../fake-cli/git-reset-hard | 0 .../fake-cli/git-rev-list | 0 .../fake-cli/git-rev-list-path | 0 .../fake-cli/git-show | 0 .../fake-cli/git-squash-merge | 13 + .../fake-cli/git-stage | 0 .../fake-cli/git-untracked-files | 0 .../fake-cli/git-untracked-files-empty | 0 .../fake_git/git_describe_tags | 6 - .../smithy-rs-tool-common/fake_git/git_fails | 4 - .../smithy-rs-tool-common/fake_git/git_reset | 5 - .../fake_git/git_revparse_head | 6 - .../fake_git/git_revparse_show_toplevel | 6 - tools/smithy-rs-tool-common/src/changelog.rs | 243 ++++ tools/smithy-rs-tool-common/src/git.rs | 615 ++++++++- .../src/git/get_repo_root.rs | 73 -- tools/smithy-rs-tool-common/src/lib.rs | 1 + 81 files changed, 3376 insertions(+), 2962 deletions(-) create mode 100644 .github/workflows/release-scripts/create-release.js create mode 100644 .github/workflows/release.yml delete mode 100644 aws/SDK_CHANGELOG.md create mode 100644 aws/SDK_CHANGELOG.next.json create mode 100644 aws/sdk/aws-models/README.md create mode 100644 tools/changelogger/Cargo.lock create mode 100644 tools/changelogger/Cargo.toml rename tools/{sdk-lints => changelogger}/smithy-rs-maintainers.txt (100%) create mode 100644 tools/changelogger/src/entry.rs create mode 100644 tools/changelogger/src/lib.rs create mode 100644 tools/changelogger/src/main.rs create mode 100644 tools/changelogger/src/render.rs create mode 100644 tools/changelogger/src/split.rs create mode 100644 tools/changelogger/tests/e2e_test.rs create mode 100755 tools/ci-build/scripts/generate-smithy-rs-release delete mode 100755 tools/ci-build/scripts/generate-smithy-rs-runtime-bundle delete mode 100644 tools/publisher/src/repo.rs delete mode 100755 tools/sdk-sync/fake-cli/git-squash-merge delete mode 100644 tools/sdk-sync/src/git.rs rename tools/{sdk-sync => smithy-rs-tool-common}/fake-cli/git-changed-files (100%) rename tools/{sdk-sync => smithy-rs-tool-common}/fake-cli/git-changed-files-empty (100%) rename tools/{sdk-sync => smithy-rs-tool-common}/fake-cli/git-clone (100%) rename tools/{sdk-sync => smithy-rs-tool-common}/fake-cli/git-commit (100%) rename tools/{sdk-sync => smithy-rs-tool-common}/fake-cli/git-commit-on-behalf (100%) rename tools/{sdk-sync => smithy-rs-tool-common}/fake-cli/git-create-branch (100%) rename tools/{sdk-sync => smithy-rs-tool-common}/fake-cli/git-current-branch-name (100%) rename tools/{sdk-sync => smithy-rs-tool-common}/fake-cli/git-delete-branch (100%) rename tools/{sdk-sync => smithy-rs-tool-common}/fake-cli/git-extract-commit-info (100%) rename tools/{sdk-sync => smithy-rs-tool-common}/fake-cli/git-get-head-revision (100%) rename tools/{sdk-sync => smithy-rs-tool-common}/fake-cli/git-reset-hard (100%) rename tools/{sdk-sync => smithy-rs-tool-common}/fake-cli/git-rev-list (100%) rename tools/{sdk-sync => smithy-rs-tool-common}/fake-cli/git-rev-list-path (100%) rename tools/{sdk-sync => smithy-rs-tool-common}/fake-cli/git-show (100%) create mode 100755 tools/smithy-rs-tool-common/fake-cli/git-squash-merge rename tools/{sdk-sync => smithy-rs-tool-common}/fake-cli/git-stage (100%) rename tools/{sdk-sync => smithy-rs-tool-common}/fake-cli/git-untracked-files (100%) rename tools/{sdk-sync => smithy-rs-tool-common}/fake-cli/git-untracked-files-empty (100%) delete mode 100755 tools/smithy-rs-tool-common/fake_git/git_describe_tags delete mode 100755 tools/smithy-rs-tool-common/fake_git/git_fails delete mode 100755 tools/smithy-rs-tool-common/fake_git/git_reset delete mode 100755 tools/smithy-rs-tool-common/fake_git/git_revparse_head delete mode 100755 tools/smithy-rs-tool-common/fake_git/git_revparse_show_toplevel create mode 100644 tools/smithy-rs-tool-common/src/changelog.rs delete mode 100644 tools/smithy-rs-tool-common/src/git/get_repo_root.rs diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 00a725106..849b76410 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -27,11 +27,17 @@ jobs: actions: - action: generate-aws-sdk - action: generate-aws-sdk-smoketest - - action: generate-smithy-rs-runtime-bundle + - action: generate-smithy-rs-release steps: - uses: actions/checkout@v3 with: path: smithy-rs + # The models from aws-sdk-rust are needed to generate the full SDK for CI + - uses: actions/checkout@v3 + with: + repository: awslabs/aws-sdk-rust + path: aws-sdk-rust + # The examples from aws-doc-sdk-examples are needed to see if smithy-rs changes break examples - uses: actions/checkout@v3 with: repository: awsdocs/aws-doc-sdk-examples diff --git a/.github/workflows/release-scripts/create-release.js b/.github/workflows/release-scripts/create-release.js new file mode 100644 index 000000000..ad4f328e8 --- /dev/null +++ b/.github/workflows/release-scripts/create-release.js @@ -0,0 +1,83 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +// This script is used by the release.yml GitHub Actions workflow to idempotently +// create a tagged GitHub release from the generated release manifest file. + +const assert = require("assert"); +const fs = require("fs"); + +const smithy_rs_repo = { + owner: "awslabs", + repo: "smithy-rs", +}; + +async function getExistingRelease(github, tag) { + try { + const response = await github.rest.repos.getReleaseByTag({ ...smithy_rs_repo, tag }); + return { name: response.data.name, body: response.data.body }; + } catch (error) { + if (error.status === 404) { + console.info(`No existing release found with tag '${tag}'`); + return null; + } + throw error; + } +} + +function loadReleaseManifest(path) { + const releaseManifest = JSON.parse(fs.readFileSync(path)); + console.info("Release manifest: ", releaseManifest); + assert(releaseManifest.tagName !== undefined, "release manifest must have a `tagName` field"); + assert(releaseManifest.name !== undefined, "release manifest must have a `name` field"); + assert(releaseManifest.body !== undefined, "release manifest must have a `body` field"); + assert(releaseManifest.prerelease !== undefined, "release manifest must have a `prerelease` field"); + return releaseManifest; +} + +module.exports = async ({ + // GitHub API (Octokit) + github, + // Boolean indicating if this is a dry-run release or not + isDryRun, + // Release manifest file path + releaseManifestPath, +}) => { + assert(github !== undefined, "The `github` argument is required"); + assert(isDryRun !== undefined, "The `isDryRun` argument is required"); + assert(releaseManifestPath !== undefined, "The `releaseManifestPath` argument is required"); + + console.info(`Starting GitHub release creation with isDryRun: ${isDryRun}, and releaseManifestPath: '${releaseManifestPath}'`); + + // Load the release manifest generated by the `changelogger` tool during build + const releaseManifest = loadReleaseManifest(releaseManifestPath); + + // Idempotency check: Look up an existing GitHub release for the tag in the release manifest + const existingRelease = await getExistingRelease(github, releaseManifest.tagName); + if (existingRelease) { + console.info(`Found an existing release with tag '${releaseManifest.tagName}': `, existingRelease); + if (existingRelease.name !== releaseManifest.name || existingRelease.body !== releaseManifest.body) { + throw Error("FATAL: Existing release does not match the release manifest!"); + } else { + console.info("SUCCESS: Existing release matches details in the release manifest. No work needs to be done!"); + } + } else { + // Explicitly comparing against `false` to avoid accidental publish in the event that + // the `isDryRun` argument wasn't passed in correctly from the GitHub Actions workflow. + if (isDryRun === false) { + console.info("Not a dry-run; creating a new release..."); + const response = await github.rest.repos.createRelease({ + ...smithy_rs_repo, + tag_name: releaseManifest.tagName, + name: releaseManifest.name, + body: releaseManifest.body, + prerelease: releaseManifest.prerelease, + }); + console.info(`SUCCESS: Created release with ID: ${response.data.id}, URL: ${response.data.html_url} `); + } else { + console.info("SUCCESS: Exiting early since this is a dry-run release."); + } + } +}; diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 000000000..65eadab80 --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,95 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 + +# This workflow performs a release of smithy-rs. It is manually +# kicked off via GitHub Actions workflow dispatch. + +# Allow only one release to run at a time +concurrency: + group: release-smithy-rs + cancel-in-progress: true + +env: + rust_version: 1.58.1 + +name: Release smithy-rs +on: + workflow_dispatch: + inputs: + dry_run: + description: Dry runs will only produce release artifacts, but will not cut a release tag in GitHub nor publish to crates.io + required: true + type: boolean + default: true + +jobs: + release-ci: + name: Prerelease checks + uses: ./.github/workflows/ci.yml + + release: + name: Release + needs: + - release-ci + runs-on: ubuntu-latest + steps: + - name: Install Rust + uses: actions-rs/toolchain@v1 + with: + toolchain: ${{ env.rust_version }} + default: true + - name: Checkout smithy-rs + uses: actions/checkout@v3 + with: + path: smithy-rs + token: ${{ secrets.RELEASE_AUTOMATION_BOT_PAT }} + - name: Generate release artifacts + uses: ./smithy-rs/.github/actions/docker-build + with: + action: generate-smithy-rs-release + - name: Download all artifacts + uses: ./smithy-rs/.github/actions/download-all-artifacts + - name: Push smithy-rs changes + shell: bash + working-directory: smithy-rs-release/smithy-rs + run: | + if [[ "${{ inputs.dry_run }}" == "true" ]]; then + echo "Pushing a preview of the release to the smithy-rs-release-preview branch" + git push --force origin HEAD:smithy-rs-release-preview + else + echo "Pushing release commits..." + git push origin + fi + - name: Tag release + uses: actions/github-script@v6 + with: + github-token: ${{ secrets.RELEASE_AUTOMATION_BOT_PAT }} + script: | + const createReleaseScript = require("./smithy-rs/.github/workflows/release-scripts/create-release.js"); + await createReleaseScript({ + github, + isDryRun: ${{ inputs.dry_run }}, + releaseManifestPath: "smithy-rs-release/smithy-rs-release-manifest.json" + }); + - name: Publish to crates.io + shell: bash + working-directory: smithy-rs-release/crates-to-publish + env: + RELEASE_AUTOMATION_BOT_CRATESIO_TOKEN: ${{ secrets.RELEASE_AUTOMATION_BOT_CRATESIO_TOKEN }} + run: | + cargo login -- "${RELEASE_AUTOMATION_BOT_CRATESIO_TOKEN}" + cargo install --path ../smithy-rs/tools/publisher + # Verify the publisher tool installed successfully + publisher --version + + if [[ "${{ inputs.dry_run }}" == "true" ]]; then + if [[ ! -f aws-smithy-types/Cargo.toml ]]; then + echo "Crates to publish not found!" + exit 1 + fi + # The following owner list command fails without a valid crates.io auth token + echo "Checking cargo auth token..." + cargo owner --list aws-smithy-types + else + publisher publish --location . + fi diff --git a/aws/SDK_CHANGELOG.md b/aws/SDK_CHANGELOG.md deleted file mode 100644 index 36f9720c8..000000000 --- a/aws/SDK_CHANGELOG.md +++ /dev/null @@ -1,1107 +0,0 @@ - -v0.14.0 (June 22nd, 2022) -========================= -**New this release:** -- 🐛 ([aws-sdk-rust#547](https://github.com/awslabs/aws-sdk-rust/issues/547), [smithy-rs#1458](https://github.com/awslabs/smithy-rs/issues/1458)) Fix bug in profile file credential provider where a missing `default` profile lead to an unintended error. -- ([smithy-rs#1421](https://github.com/awslabs/smithy-rs/issues/1421)) Add `Debug` implementation to several types in `aws-config` -- 🐛 ([aws-sdk-rust#558](https://github.com/awslabs/aws-sdk-rust/issues/558), [smithy-rs#1478](https://github.com/awslabs/smithy-rs/issues/1478)) Fix bug in retry policy where user configured timeouts were not retried. With this fix, setting - [`with_call_attempt_timeout`](https://docs.rs/aws-smithy-types/0.43.0/aws_smithy_types/timeout/struct.Api.html#method.with_call_attempt_timeout) - will lead to a retry when retries are enabled. - - -v0.13.0 (June 9th, 2022) -======================== -**New this release:** -- 🎉 ([smithy-rs#1390](https://github.com/awslabs/smithy-rs/issues/1390)) Add method `ByteStream::into_async_read`. This makes it easy to convert `ByteStream`s into a struct implementing `tokio:io::AsyncRead`. Available on **crate feature** `rt-tokio` only. -- 🎉 ([smithy-rs#1356](https://github.com/awslabs/smithy-rs/issues/1356), @jszwedko) Add support for `credential_process` in AWS configs for fetching credentials from an external process. -- ([smithy-rs#1404](https://github.com/awslabs/smithy-rs/issues/1404), @petrosagg) Switch to [RustCrypto](https://github.com/RustCrypto)'s implementation of MD5. - -**Contributors** -Thank you for your contributions! ❤ -- @jszwedko ([smithy-rs#1356](https://github.com/awslabs/smithy-rs/issues/1356)) -- @petrosagg ([smithy-rs#1404](https://github.com/awslabs/smithy-rs/issues/1404)) - -v0.12.0 (May 13th, 2022) -======================== -**New this release:** -- ([smithy-rs#1352](https://github.com/awslabs/smithy-rs/issues/1352)) Log a debug event when a retry is going to be peformed - - -0.11.0 (April 28th, 2022) -========================= -**Breaking Changes:** -- ⚠ ([smithy-rs#1318](https://github.com/awslabs/smithy-rs/issues/1318)) Bump [MSRV](https://github.com/awslabs/aws-sdk-rust#supported-rust-versions-msrv) from 1.56.1 to 1.58.1 per our "two versions behind" policy. - -**New this release:** -- 🐛 ([smithy-rs#1344](https://github.com/awslabs/smithy-rs/issues/1344), @ryansb) Suppress irrelevant `$HOME` expansion warning when running in a Lambda Extension - -**Contributors** -Thank you for your contributions! ❤ -- @ryansb ([smithy-rs#1344](https://github.com/awslabs/smithy-rs/issues/1344)) - -0.10.1 (April 14th, 2022) -========================= - -**Breaking Changes:** -- ⚠ ([aws-sdk-rust#490](https://github.com/awslabs/aws-sdk-rust/issues/490)) Update all SDK and runtime crates to [edition 2021](https://blog.rust-lang.org/2021/10/21/Rust-1.56.0.html) - -**New this release:** -- ([smithy-rs#1262](https://github.com/awslabs/smithy-rs/issues/1262), @liubin) Fix link to Developer Guide in crate's README.md -- 🐛 ([aws-sdk-rust#1271](https://github.com/awslabs/aws-sdk-rust/issues/1271), @elrob) Treat blank environment variable credentials (`AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY`) as missing instead of attempting to use them to sign requests. -- ([aws-sdk-rust#479](https://github.com/awslabs/aws-sdk-rust/issues/479), [smithy-rs#1296](https://github.com/awslabs/smithy-rs/issues/1296)) Add support for configuring the session length in [AssumeRoleProvider](https://docs.rs/aws-config/latest/aws_config/sts/struct.AssumeRoleProvider.html) -- ([smithy-rs#1296](https://github.com/awslabs/smithy-rs/issues/1296)) Add caching to [AssumeRoleProvider](https://docs.rs/aws-config/latest/aws_config/sts/struct.AssumeRoleProvider.html) -- ([smithy-rs#1300](https://github.com/awslabs/smithy-rs/issues/1300), @benesch) Add endpoint resolver to SdkConfig. This enables overriding the endpoint resolver for all services build from a single SdkConfig. - -**Contributors** -Thank you for your contributions! ❤ -- @benesch ([smithy-rs#1300](https://github.com/awslabs/smithy-rs/issues/1300)) -- @elrob ([aws-sdk-rust#1271](https://github.com/awslabs/aws-sdk-rust/issues/1271)) -- @liubin ([smithy-rs#1262](https://github.com/awslabs/smithy-rs/issues/1262)) - -0.9.0 (March 17, 2022) -====================== -**Breaking Changes:** -- ⚠ ([aws-sdk-rust#406](https://github.com/awslabs/aws-sdk-rust/issues/406)) `aws_types::config::Config` has been renamed to `aws_types::sdk_config::SdkConfig`. This is to better differentiate it - from service-specific configs like `aws_sdk_s3::Config`. If you were creating shared configs with - `aws_config::load_from_env()`, then you don't have to do anything. If you were directly referring to a shared config, - update your `use` statements and `struct` names. - - _Before:_ - ```rust - use aws_types::config::Config; - - fn main() { - let config = Config::builder() - // config builder methods... - .build() - .await; - } - ``` - - _After:_ - ```rust - // We re-export this type from the root module so it's easier to reference - use aws_types::SdkConfig; - - fn main() { - let config = SdkConfig::builder() - // config builder methods... - .build() - .await; - } - ``` -- ⚠ ([smithy-rs#724](https://github.com/awslabs/smithy-rs/issues/724)) Timeout configuration has been refactored a bit. If you were setting timeouts through environment variables or an AWS - profile, then you shouldn't need to change anything. Take note, however, that we don't currently support HTTP connect, - read, write, or TLS negotiation timeouts. If you try to set any of those timeouts in your profile or environment, we'll - log a warning explaining that those timeouts don't currently do anything. - - If you were using timeouts programmatically, - you'll need to update your code. In previous versions, timeout configuration was stored in a single `TimeoutConfig` - struct. In this new version, timeouts have been broken up into several different config structs that are then collected - in a `timeout::Config` struct. As an example, to get the API per-attempt timeout in previous versions you would access - it with `.api_call_attempt_timeout()` and in this new version you would access it with - `.api.call_attempt_timeout()`. We also made some unimplemented timeouts inaccessible in order to - avoid giving users the impression that setting them had an effect. We plan to re-introduce them once they're made - functional in a future update. - -**New this release:** -- 🎉 ([aws-sdk-rust#475](https://github.com/awslabs/aws-sdk-rust/issues/475), [aws-sdk-rust#473](https://github.com/awslabs/aws-sdk-rust/issues/473)) Enable presigning for S3 operations UploadPart and DeleteObject - - -0.8.0 (Februrary 24, 2022) -========================== -**Breaking Changes:** -- ⚠ ([smithy-rs#1216](https://github.com/awslabs/smithy-rs/issues/1216)) `aws-sigv4` no longer skips the `content-length` and `content-type` headers when signing with `SignatureLocation::QueryParams` - -**New this release:** -- 🎉 ([smithy-rs#1220](https://github.com/awslabs/smithy-rs/issues/1220), [aws-sdk-rust#462](https://github.com/awslabs/aws-sdk-rust/issues/462)) Made it possible to change settings, such as load timeout, on the credential cache used by the `DefaultCredentialsChain`. -- 🐛 ([smithy-rs#1197](https://github.com/awslabs/smithy-rs/issues/1197)) Fixed a bug that caused clients to eventually stop retrying. The cross-request retry allowance wasn't being reimbursed upon receiving a successful response, so once this allowance reached zero, no further retries would ever be attempted. -- 🐛 ([smithy-rs#1217](https://github.com/awslabs/smithy-rs/issues/1217), [aws-sdk-rust#467](https://github.com/awslabs/aws-sdk-rust/issues/467)) `ClientBuilder` helpers `rustls()` and `native_tls()` now return `DynConnector` so that they once again work when constructing clients with custom middleware in the SDK. -- 🐛 ([smithy-rs#1216](https://github.com/awslabs/smithy-rs/issues/1216), [aws-sdk-rust#466](https://github.com/awslabs/aws-sdk-rust/issues/466)) Fixed a bug in S3 that prevented the `content-length` and `content-type` inputs from being included in a presigned request signature. With this fix, customers can generate presigned URLs that enforce `content-length` and `content-type` for requests to S3. - - -0.7.0 (February 18th, 2022) -=========================== -**Breaking Changes:** -- ⚠ ([smithy-rs#1144](https://github.com/awslabs/smithy-rs/issues/1144)) The `aws_config::http_provider` module has been renamed to `aws_config::http_credential_provider` to better reflect its purpose. -- ⚠ ([smithy-rs#1144](https://github.com/awslabs/smithy-rs/issues/1144)) Some APIs required that timeout configuration be specified with an `aws_smithy_client::timeout::Settings` struct while - others required an `aws_smithy_types::timeout::TimeoutConfig` struct. Both were equivalent. Now `aws_smithy_types::timeout::TimeoutConfig` - is used everywhere and `aws_smithy_client::timeout::Settings` has been removed. Here's how to migrate code your code that - depended on `timeout::Settings`: - - The old way: - ```rust - let timeout = timeout::Settings::new() - .with_connect_timeout(Duration::from_secs(1)) - .with_read_timeout(Duration::from_secs(2)); - ``` - - The new way: - ```rust - // This example is passing values, so they're wrapped in `Option::Some`. You can disable a timeout by passing `None`. - let timeout = TimeoutConfig::new() - .with_connect_timeout(Some(Duration::from_secs(1))) - .with_read_timeout(Some(Duration::from_secs(2))); - ``` -- ⚠ ([smithy-rs#1144](https://github.com/awslabs/smithy-rs/issues/1144)) `MakeConnectorFn`, `HttpConnector`, and `HttpSettings` have been moved from `aws_config::provider_config` to - `aws_smithy_client::http_connector`. This is in preparation for a later update that will change how connectors are - created and configured. - - If you were using these structs/enums, you can migrate your old code by importing them from their new location. -- ⚠ ([smithy-rs#1144](https://github.com/awslabs/smithy-rs/issues/1144)) Along with moving `HttpConnector` to `aws_smithy_client`, the `HttpConnector::make_connector` method has been renamed to - `HttpConnector::connector`. - - If you were using this method, you can migrate your old code by calling `connector` instead of `make_connector`. -- ⚠ ([smithy-rs#1085](https://github.com/awslabs/smithy-rs/issues/1085)) Moved the following re-exports into a `types` module for all services: - - `aws_sdk_::AggregatedBytes` -> `aws_sdk_::types::AggregatedBytes` - - `aws_sdk_::Blob` -> `aws_sdk_::types::Blob` - - `aws_sdk_::ByteStream` -> `aws_sdk_::types::ByteStream` - - `aws_sdk_::DateTime` -> `aws_sdk_::types::DateTime` - - `aws_sdk_::SdkError` -> `aws_sdk_::types::SdkError` -- ⚠ ([smithy-rs#1085](https://github.com/awslabs/smithy-rs/issues/1085)) `AggregatedBytes` and `ByteStream` are now only re-exported if the service has streaming operations, - and `Blob`/`DateTime` are only re-exported if the service uses them. -- ⚠ ([smithy-rs#1130](https://github.com/awslabs/smithy-rs/issues/1130)) MSRV increased from `1.54` to `1.56.1` per our 2-behind MSRV policy. -- ⚠ ([smithy-rs#1132](https://github.com/awslabs/smithy-rs/issues/1132)) Fluent clients for all services no longer have generics, and now use `DynConnector` and `DynMiddleware` to allow - for connector/middleware customization. This should only break references to the client that specified generic types for it. - - If you customized the AWS client's connector or middleware with something like the following: - ```rust - let client = aws_sdk_s3::Client::with_config( - aws_sdk_s3::client::Builder::new() - .connector(my_custom_connector) // Connector customization - .middleware(my_custom_middleware) // Middleware customization - .default_async_sleep() - .build(), - config - ); - ``` - Then you will need to wrap the custom connector or middleware in - [`DynConnector`](https://docs.rs/aws-smithy-client/0.36.0/aws_smithy_client/erase/struct.DynConnector.html) - and - [`DynMiddleware`](https://docs.rs/aws-smithy-client/0.36.0/aws_smithy_client/erase/struct.DynMiddleware.html) - respectively: - ```rust - let client = aws_sdk_s3::Client::with_config( - aws_sdk_s3::client::Builder::new() - .connector(DynConnector::new(my_custom_connector)) // Now with `DynConnector` - .middleware(DynMiddleware::new(my_custom_middleware)) // Now with `DynMiddleware` - .default_async_sleep() - .build(), - config - ); - ``` - - If you had functions that took a generic connector, such as the following: - ```rust - fn some_function(conn: C) -> Result<()> - where - C: aws_smithy_client::bounds::SmithyConnector + Send + 'static, - E: Into - { - // ... - } - ``` - - Then the generics and trait bounds will no longer be necessary: - ```rust - fn some_function(conn: DynConnector) -> Result<()> { - // ... - } - ``` - - Similarly, functions that took a generic middleware can replace the generic with `DynMiddleware` and - remove their trait bounds. - -**New this release:** -- 🐛 ([aws-sdk-rust#443](https://github.com/awslabs/aws-sdk-rust/issues/443)) The `ProfileFileRegionProvider` will now respect regions set in chained profiles -- ([smithy-rs#1144](https://github.com/awslabs/smithy-rs/issues/1144)) Several modules defined in the `aws_config` crate that used to be declared within another module's file have been moved to their own files. The moved modules are `sts`, `connector`, and `default_providers`. They still have the exact same import paths. -- 🐛 ([smithy-rs#1129](https://github.com/awslabs/smithy-rs/issues/1129)) Fix some docs links not working because they were escaped when they shouldn't have been -- ([smithy-rs#1085](https://github.com/awslabs/smithy-rs/issues/1085)) The `Client` and `Config` re-exports now have their documentation inlined in the service docs -- 🐛 ([smithy-rs#1180](https://github.com/awslabs/smithy-rs/issues/1180)) Fixed example showing how to use hardcoded credentials in `aws-types` - - -0.6.0 (January 26, 2022) -======================== -**New this release:** -- ([aws-sdk-rust#423](https://github.com/awslabs/aws-sdk-rust/issues/423)) Added `impl Into for PresignedRequest` and a conversion method for turning `PresignedRequest`s into `http::Request`s. -- ([smithy-rs#1087](https://github.com/awslabs/smithy-rs/issues/1087)) Convert several `info` spans to `debug` in aws-config -- ([smithy-rs#1118](https://github.com/awslabs/smithy-rs/issues/1118)) SDK examples now come from [`awsdocs/aws-doc-sdk-examples`](https://github.com/awsdocs/aws-doc-sdk-examples) rather than from `smithy-rs` - - -0.5.2 (January 20th, 2022) -========================== - -**New this release:** -- 🐛 ([smithy-rs#1100](https://github.com/awslabs/smithy-rs/issues/1100)) _Internal:_ Update sync script to run gradle clean. This fixes an issue where codegen was not triggered when only properties changed. - - -v0.5.1 (January 19th, 2022) -=========================== - -**New this release:** -- 🐛 ([smithy-rs#1089](https://github.com/awslabs/smithy-rs/issues/1089)) Fix dev-dependency cycle between aws-sdk-sso and aws-config - - -0.5.0 (January 19, 2022) -======================== -**New this release:** -- 🎉 ([aws-sdk-rust#348](https://github.com/awslabs/aws-sdk-rust/issues/348)) The docs for fluent builders now have easy links to their corresponding Input, Output, and Error structs -- 🎉 ([smithy-rs#1051](https://github.com/awslabs/smithy-rs/issues/1051), [aws-sdk-rust#4](https://github.com/awslabs/aws-sdk-rust/issues/4)) Add support for SSO credentials -- 🐛🎉 ([smithy-rs#1065](https://github.com/awslabs/smithy-rs/issues/1065), [aws-sdk-rust#398](https://github.com/awslabs/aws-sdk-rust/issues/398), @nmoutschen) Silence profile credential warnings in Lambda environment -- 🐛 ([aws-sdk-rust#405](https://github.com/awslabs/aws-sdk-rust/issues/405), [smithy-rs#1083](https://github.com/awslabs/smithy-rs/issues/1083)) Fixed paginator bug impacting EC2 describe VPCs (and others) - -**Contributors** -Thank you for your contributions! ❤ -- @nmoutschen ([aws-sdk-rust#398](https://github.com/awslabs/aws-sdk-rust/issues/398), [smithy-rs#1065](https://github.com/awslabs/smithy-rs/issues/1065)) - - -v0.4.1 (January 10, 2022) -========================= -**New this release:** -- 🐛 (smithy-rs#1050, @nmoutschen) Fix typos for X-Ray trace ID environment variable in aws_http::recursion_detection -- 🐛 (smithy-rs#1054, aws-sdk-rust#391) Fix critical paginator bug where an empty outputToken lead to a never ending stream. - -**Contributors** -Thank you for your contributions! ❤ -- @nmoutschen (smithy-rs#1050) - - -0.4.0 (January 6th, 2022) -========================= -**Breaking Changes:** -- ⚠ (smithy-rs#990) Codegen will no longer produce builders and clients with methods that take `impl Into` except for strings and boxed types. -- ⚠ (smithy-rs#961) The `meta`, `environment`, and `dns` Cargo feature flags were removed from `aws-config`. - The code behind the `dns` flag is now enabled when `rt-tokio` is enabled. The code behind - the `meta` and `environment` flags is always enabled now. -- ⚠ (smithy-rs#1003) `aws_http::AwsErrorRetryPolicy` was moved to `aws_http::retry::AwsErrorRetryPolicy`. -- ⚠ (smithy-rs#1017, smithy-rs#930) Simplify features in aws-config. All features have been removed from `aws-config` with the exception of: `rt-tokio`, `rustls` and `native-tls`. All other features are now included by default. If you depended on those features specifically, remove them from your features listing. - -**New this release:** -- 🎉 (aws-sdk-rust#47, smithy-rs#1006) Add support for paginators! Paginated APIs now include `.into_paginator()` and (when supported) `.into_paginator().items()` to enable paginating responses automatically. The paginator API should be considered in preview and is subject to change pending customer feedback. -- (smithy-rs#712) We removed an example 'telephone-game' that was problematic for our CI. - The APIs that that example demonstrated are also demonstrated by our Polly - and TranscribeStreaming examples so please check those out if you miss it. -- 🐛 (aws-sdk-rust#357) Generated docs should no longer contain links that don't go anywhere -- (aws-sdk-rust#254, @jacco) Made fluent operation structs cloneable -- (smithy-rs#973) Debug implementation of Credentials will print `expiry` in a human readable way. -- 🐛 (smithy-rs#999, smithy-rs#143, aws-sdk-rust#344) Add Route53 customization to trim `/hostedzone/` prefix prior to serialization. This fixes a bug where round-tripping a hosted zone id resulted in an error. -- 🐛 (smithy-rs#998, aws-sdk-rust#359) Fix bug where ECS credential provider could not perform retries. -- (smithy-rs#1003) Add recursion detection middleware to the default stack -- (smithy-rs#1002, aws-sdk-rust#352) aws_types::Config is now `Clone` -- (smithy-rs#670, @jacco) Example for Config builder region function added -- (smithy-rs#1021, @kiiadi) Add function to `aws_config::profile::ProfileSet` that allows listing of loaded profiles by name. -- 🐛 (smithy-rs#1046, aws-sdk-rust#384) Fix IMDS credentials provider bug where the instance profile name was incorrectly cached. - -**Contributors** -Thank you for your contributions! ❤ -- @jacco (aws-sdk-rust#254, smithy-rs#670) -- @kiiadi (smithy-rs#1021) - - -v0.3.0 (December 15th, 2021) -============================ -**Breaking Changes:** -- ⚠ (smithy-rs#930) If you directly depend on AWS or Smithy runtime crates _(e.g., AWS crates not named `aws-config` or prefixed with `aws-sdk-`)_, - the formerly default features from those crates must now be explicitly set in your `Cargo.toml`. - - **Upgrade guide** - - | before | after | - | ------------------------------- | ------------------------------------------------------------------------------------------------ | - | `aws-smithy-async = "VERSION"` | `aws-smithy-async = { version = "VERSION", features = ["rt-tokio"] }` | - | `aws-smithy-client = "VERSION"` | `aws-smithy-client = { version = "VERSION", features = ["client-hyper", "rustls", "rt-tokio"] }` | - | `aws-smithy-http = "VERSION"` | `aws-smithy-http = { version = "VERSION", features = ["rt-tokio"] }` | -- ⚠ (smithy-rs#940) `aws_hyper::Client` which was just a re-export of `aws_smithy_types::Client` with generics set has been removed. If you used - `aws_hyper::Client` or `aws_hyper::Client::https()` you can update your code to use `aws_smithy_client::Builder::https()`. -- ⚠ (smithy-rs#947) The features `aws-hyper/rustls` and `aws-hyper/native-tls` have been removed. If you were using these, use the identical features on `aws-smithy-client`. -- ⚠ (smithy-rs#959, smithy-rs#934) `aws-hyper::AwsMiddleware` is now generated into generated service clients directly. If you used `aws_hyper::Middleware`, use ::middleware::DefaultMiddleware` instead. - -**New this release:** -- 🐛 (aws-sdk-rust#330) A bug that occurred when signing certain query strings has been fixed -- 🐛 (smithy-rs#949, @a-xp) Fix incorrect argument order in the builder for `LazyCachingCredentialsProvider` -- 🐛 (aws-sdk-rust#304) `aws-config` will now work as intended for users that want to use `native-tls` instead of `rustls`. Previously, it was - difficult to ensure that `rustls` was not in use. Also, there is now an example of how to use `native-tls` and a test - that ensures `rustls` is not in the dependency tree -- 🐛 (aws-sdk-rust#317, smithy-rs#907) Removed inaccurate log message when a client was used without a sleep implementation, and - improved context and call to action in logged messages around missing sleep implementations. -- (smithy-rs#923) Use provided `sleep_impl` for retries instead of using Tokio directly. -- (smithy-rs#920) Fix typos in module documentation for generated crates -- 🐛 (aws-sdk-rust#301, smithy-rs#892) Avoid serializing repetitive `xmlns` attributes when serializing XML. This reduces the length of serialized requests and should improve compatibility with localstack. -- 🐛 (smithy-rs#953, aws-sdk-rust#331) Fixed a bug where certain characters caused a panic during URI encoding. - -**Contributors** -Thank you for your contributions! ❤ -- @a-xp (smithy-rs#949) - - -v0.2.0 (December 2nd, 2021) -=========================== - -- This release was a version bump to fix a version number conflict in crates.io - -v0.1.0 (December 2nd, 2021) -=========================== - -**New this release** -- Add docs.rs metadata section to all crates to document all features -- [Added a new example showing how to set all currently supported timeouts](./examples/setting_timeouts/src/main.rs) -- Add a new check so that the SDK doesn't emit an irrelevant `$HOME` dir warning when running in a Lambda (aws-sdk-rust#307) -- :bug: Don't capture empty session tokens from the `AWS_SESSION_TOKEN` environment variable (aws-sdk-rust#316, smithy-rs#906) - -v0.0.26-alpha (November 23rd, 2021) -=================================== - -**Breaking Changes** -- `RetryConfigBuilder::merge_with` has been renamed to `RetryConfigBuilder::take_unset_from` -- `Credentials::from_keys` is now behind a feature flag named `hardcoded-credentials` in `aws-types`. - It is __NOT__ secure to hardcode credentials into your application, and the credentials - providers that come with the AWS SDK should be preferred. (smithy-rs#875, smithy-rs#317) -- (aws-smithy-client): Extraneous `pub use SdkSuccess` removed from `aws_smithy_client::hyper_ext`. (smithy-rs#855) -- The `add_metadata` function was removed from `AwsUserAgent` in `aws-http`. - Use `with_feature_metadata`, `with_config_metadata`, or `with_framework_metadata` now instead. (smithy-rs#865) -- Several breaking changes around `aws_smithy_types::Instant` were introduced by smithy-rs#849: - - `aws_smithy_types::Instant` from was renamed to `DateTime` to avoid confusion with the standard library's monotonically nondecreasing `Instant` type. - - `DateParseError` in `aws_smithy_types` has been renamed to `DateTimeParseError` to match the type that's being parsed. - - The `chrono-conversions` feature and associated functions have been moved to the `aws-smithy-types-convert` crate. - - Calls to `Instant::from_chrono` should be changed to: - ```rust - use aws_smithy_types::DateTime; - use aws_smithy_types_convert::date_time::DateTimeExt; - - // For chrono::DateTime - let date_time = DateTime::from_chrono_utc(chrono_date_time); - // For chrono::DateTime - let date_time = DateTime::from_chrono_offset(chrono_date_time); - ``` - - Calls to `instant.to_chrono()` should be changed to: - ```rust - use aws_smithy_types_convert::date_time::DateTimeExt; - - date_time.to_chrono_utc(); - ``` - - `Instant::from_system_time` and `Instant::to_system_time` have been changed to `From` trait implementations. - - Calls to `from_system_time` should be changed to: - ```rust - DateTime::from(system_time); - // or - let date_time: DateTime = system_time.into(); - ``` - - Calls to `to_system_time` should be changed to: - ```rust - SystemTime::from(date_time); - // or - let system_time: SystemTime = date_time.into(); - ``` - - Several functions in `Instant`/`DateTime` were renamed: - - `Instant::from_f64` -> `DateTime::from_secs_f64` - - `Instant::from_fractional_seconds` -> `DateTime::from_fractional_secs` - - `Instant::from_epoch_seconds` -> `DateTime::from_secs` - - `Instant::from_epoch_millis` -> `DateTime::from_millis` - - `Instant::epoch_fractional_seconds` -> `DateTime::as_secs_f64` - - `Instant::has_nanos` -> `DateTime::has_subsec_nanos` - - `Instant::epoch_seconds` -> `DateTime::secs` - - `Instant::epoch_subsecond_nanos` -> `DateTime::subsec_nanos` - - `Instant::to_epoch_millis` -> `DateTime::to_millis` - - The `DateTime::fmt` method is now fallible and fails when a `DateTime`'s value is outside what can be represented by the desired date format. - -**New this week** - -- :warning: MSRV increased from 1.53.0 to 1.54.0 per our 3-behind MSRV policy. -- Conversions from `aws_smithy_types::DateTime` to `OffsetDateTime` from the `time` crate are now available from the `aws-smithy-types-convert` crate. (smithy-rs#849) -- Fixed links to Usage Examples (smithy-rs#862, @floric) -- Added missing features to user agent formatting, and made it possible to configure an app name for the user agent via service config. (smithy-rs#865) -- :bug: Relaxed profile name validation to allow `@` and other characters (smithy-rs#861, aws-sdk-rust#270) -- :bug: Fixed signing problem with S3 Control (smithy-rs#858, aws-sd-rust#291) -- :tada: Timeouts for requests are now configurable. You can set a timeout for each individual request attempt or for all attempts made for a request. (smithy-rs#831) - - `SdkError` now includes a variant `TimeoutError` for when a request times out (smithy-rs#885) -- Improve docs on `aws-smithy-client` (smithy-rs#855) -- Fix http-body dependency version (smithy-rs#883, aws-sdk-rust#305) - -**Contributions** - -Thank you for your contributions! :heart: - -- @floric (smithy-rs#862) - -v0.0.25-alpha (November 11th, 2021) -=================================== - -No changes since last release except for version bumping since older versions -of the AWS SDK were failing to compile with the `0.27.0-alpha.2` version chosen -for some of the supporting crates. - -v0.0.24-alpha (November 9th, 2021) -================================== -**Breaking Changes** -- Members named `builder` on model structs were renamed to `builder_value` so that their accessors don't conflict with the existing `builder()` methods (smithy-rs#842) - -**New this week** -- Fix epoch seconds date-time parsing bug in `aws-smithy-types` (smithy-rs#834) -- Omit trailing zeros from fraction when formatting HTTP dates in `aws-smithy-types` (smithy-rs#834) -- Moved examples into repository root (aws-sdk-rust#181, smithy-rs#843) -- Model structs now have accessor methods for their members. We recommend updating code to use accessors instead of public fields. A future release will deprecate the public fields before they are made private. (smithy-rs#842) -- :bug: Fix bug that caused signing to fail for requests where the body length was <=9. (smithy-rs#845) - -v0.0.23-alpha (November 3rd, 2021) -================================== -**New this week** -- :tada: Add support for AWS Glacier (smithy-rs#801) -- :tada: Add support for AWS Panorama -- :bug: Fix `native-tls` feature in `aws-config` (aws-sdk-rust#265, smithy-rs#803) -- Add example to aws-sig-auth for generating an IAM Token for RDS (smithy-rs#811, aws-sdk-rust#147) -- :bug: `hyper::Error(IncompleteMessage)` will now be retried (smithy-rs#815) -- :bug: S3 request metadata signing now correctly trims headers fixing [problems like this](https://github.com/awslabs/aws-sdk-rust/issues/248) (smithy-rs#761) -- All unions (eg. `dynamodb::model::AttributeValue`) now include an additional `Unknown` variant. These support cases where a new union variant has been added on the server but the client has not been updated. -- :bug: Fix generated docs on unions like `dynamodb::AttributeValue`. (smithy-rs#826) - -**Breaking Changes** -- `.make_operation(&config)` is now an `async` function for all operations. Code should be updated to call `.await`. This will only impact users using the low-level API. (smithy-rs#797) - -v0.0.22-alpha (October 20th, 2021) -================================== - -**Breaking Changes** - -- `CredentialsError` variants became non-exhaustive. This makes them impossible to construct directly outside of the `aws_types` crate. In order to construct credentials errors, new methods have been added for each variant. Instead of `CredentialsError::Unhandled(...)`, you should instead use `CredentialsError::unhandled`. Matching methods exist for all variants. (#781) -- The default credentials chain now returns `CredentialsError::CredentialsNotLoaded` instead of `ProviderError` when no credentials providers are configured. -- :warning: All Smithy runtime crates have been renamed to have an `aws-` prefix. This may require code changes: - - _Cargo.toml_ changes: - - `smithy-async` -> `aws-smithy-async` - - `smithy-client` -> `aws-smithy-client` - - `smithy-eventstream` -> `aws-smithy-eventstream` - - `smithy-http` -> `aws-smithy-http` - - `smithy-http-tower` -> `aws-smithy-http-tower` - - `smithy-json` -> `aws-smithy-json` - - `smithy-protocol-test` -> `aws-smithy-protocol-test` - - `smithy-query` -> `aws-smithy-query` - - `smithy-types` -> `aws-smithy-types` - - `smithy-xml` -> `aws-smithy-xml` - - Rust `use` statement changes: - - `smithy_async` -> `aws_smithy_async` - - `smithy_client` -> `aws_smithy_client` - - `smithy_eventstream` -> `aws_smithy_eventstream` - - `smithy_http` -> `aws_smithy_http` - - `smithy_http_tower` -> `aws_smithy_http_tower` - - `smithy_json` -> `aws_smithy_json` - - `smithy_protocol_test` -> `aws_smithy_protocol_test` - - `smithy_query` -> `aws_smithy_query` - - `smithy_types` -> `aws_smithy_types` - - `smithy_xml` -> `aws_smithy_xml` - -**New this week** - -- Moved the contents of `aws-auth` into the `aws-http` runtime crate (smithy-rs#783) -- Fix instances where docs were missing in generated services and add `#[warn_missing_docs]` (smithy-rs#779) -- Add tracing output for resolved AWS endpoint (smithy-rs#784) -- Update AWS service models (smithy-rs#790) -- Add support for the following Glacier customizations: - - Set the ApiVersion header (smithy-rs#138, #787) - -v0.0.21-alpha (October 15th, 2021) -================================== - -**New this week** - -- Prepare crate manifests for publishing to crates.io (smithy-rs#755) -- Add support for IAM Roles for tasks credential provider (smithy-rs#765, aws-sdk-rust#123) -- All service crates now have generated README files (smithy-rs#766) -- Update AWS service models (smithy-rs#772) -- :tada: Add support for Amazon Managed Grafana (smithy-rs#772) - -v0.0.20-alpha (October 7, 2021) -=============================== - -**Breaking changes** - -- :warning: MSRV increased from 1.52.1 to 1.53.0 per our 3-behind MSRV policy. -- `SmithyConnector` and `DynConnector` now return `ConnectorError` instead of `Box`. If you have written a custom connector, it will need to be updated to return the new error type. (#744) -- The `DispatchError` variant of `SdkError` now contains `ConnectorError` instead of `Box` (#744). - -**New This Week** - -- :tada: Make retry behavior configurable - - With env vars `AWS_MAX_ATTEMPTS` and `AWS_RETRY_MODE` - - With `~/.aws/config` settings `max_attempts` and `retry_mode` - - By calling the `with_retry_config` method on a `Config` and passing in a `RetryConfig` - - Only the `Standard` retry mode is currently implemented. `Adaptive` retry mode will be implemented at a later - date. - - For more info, see the AWS Reference pages on configuring these settings: - - [Setting global max attempts](https://docs.aws.amazon.com/sdkref/latest/guide/setting-global-max_attempts.html) - - [Setting global retry mode](https://docs.aws.amazon.com/sdkref/latest/guide/setting-global-retry_mode.html) -- :tada: Add presigned request support and examples for S3 GetObject and PutObject (smithy-rs#731, aws-sdk-rust#139) -- :tada: Add presigned request support and example for Polly SynthesizeSpeech (smithy-rs#735, aws-sdk-rust#139) -- Add connect & HTTP read timeouts to IMDS, defaulting to 1 second -- IO and timeout errors from Hyper can now be retried (#744) -- :bug: Fix error when receiving `Cont` event from S3 SelectObjectContent (smithy-rs#736) -- :bug: Fix bug in event stream receiver that could cause the last events in the response stream to be lost when using S3 SelectObjectContent (smithy-rs#736) -- Updated EC2 code examples to include readme; refactored operations from main into separate functions. -- Updated Transcribe code example to take an audio file as a command-line option and added readme. -- Refactored API Gateway code example by moving operation out of main and into a separate function; added readme. -- Updated Auto Scaling code example to move operation from main to separate function; added readme. -- Updated AWS Config code examples to include a readme; added command-line options; added DeleteConfigurationRecorder, DeleteDeliveryChannel, ListConfigurationRecorders, ListDeliveryChannels, ListResources, ShowResourceHistory, and EnableConfig code examples. -- :tada: Add support for 6 new AWS services: - - Wisdom - - VoiceId - - Account - - KafkaConnect - - OpenSearch - - CloudControl - -v0.0.19-alpha (September 24th, 2021) -==================================== - -**New This Week** - -- :tada: IMDS support in the default credential provider chain (aws-sdk-rust#97) -- :tada: Add `sts::AssumeRoleProvider` to `aws-config`. This enables customers to invoke STS directly, - instead of using it via `~/.aws/config`. (smithy-rs#703, aws-sdk-rust#3) -- Add IMDS client to `aws-config` (smithy-rs#701) -- Add IMDS credential provider to `aws-config` (smithy-rs#709) -- Add IMDS region provider to `aws-config` (smithy-rs#715, aws-sdk-rust#97) -- Update event stream `Receiver`s to be `Send` (aws-sdk-rust#224) -- Add query param signing to the `aws-sigv4` crate (smithy-rs#707) -- :bug: Update event stream `Receiver`s to be `Send` (smithy-rs#702, #aws-sdk-rust#224) -- :bug: Fix panic when signing non-ASCII header values (smithy-rs#708, aws-sdk-rust#226) -- Add an example that uses Polly, Transcribe, and S3 called [telephone-game](sdk/examples/telephone-game/src/main.rs) - -**Contributions** - -Thank you for your contributions! :heart: - -- @jonhoo (smithy-rs#703) - -v0.0.18-alpha (September 14th, 2021) -======================= - -- :tada: Add support for `OpenSearch` service & bring in other model updates (#todo) -- Cleanup docs in `aws-config` - -**New This Week** -- :bug: Fixes issue where `Content-Length` header could be duplicated leading to signing failure (aws-sdk-rust#220, smithy-rs#697) - -- Updated AutoScaling code examples to use asynchronous config; added readme file; tested on 0.0.17 bits - -v0.0.17-alpha (September 2nd, 2021) -=================================== - -This release adds support for three commonly requested features: -- More powerful credential chain -- Support for constructing multiple clients from the same configuration -- Support for Transcribe streaming and S3 Select - -In addition, this overhauls client configuration which lead to a number of breaking changes. Detailed changes are inline. - -Current Credential Provider Support: -- [x] Environment variables -- [x] Web Identity Token Credentials -- [ ] Profile file support (partial) - - [ ] Credentials - - [ ] SSO - - [ ] ECS Credential source - - [ ] IMDS credential source - - [x] Assume role from source profile - - [x] Static credentials source profile - - [x] WebTokenIdentity provider - - [x] Region -- [ ] IMDS -- [ ] ECS - -Upgrade Guide -------------- - -### If you use `::Client::from_env` - -`from_env` loaded region & credentials from environment variables _only_. Default sources have been removed from the generated -SDK clients and moved to the `aws-config` package. Note that the `aws-config` package default chain adds support for -profile file and web identity token profiles. - -1. Add a dependency on `aws-config`: - ```toml - [dependencies] - aws-config = { git = "https://github.com/awslabs/aws-sdk-rust", tag = "v0.0.17-alpha" } - ``` -2. Update your client creation code: - ```rust - // `shared_config` can be used to construct multiple different service clients! - let shared_config = aws_config::load_from_env().await; - // before: ::Client::from_env(); - let client = ::Client::new(&shared_config) - ``` - -### If you used `::Config::builder()` - -`Config::build()` has been modified to _not_ fallback to a default provider. Instead, use `aws-config` to load and modify -the default chain. Note that when you switch to `aws-config`, support for profile files and web identity tokens will be added. - -1. Add a dependency on `aws-config`: - ```toml - [dependencies] - aws-config = { git = "https://github.com/awslabs/aws-sdk-rust", tag = "v0.0.17-alpha" } - ``` - -2. Update your client creation code: - - ```rust - fn before() { - let region = aws_types::region::ChainProvider::first_try(<1 provider>).or_default_provider(); - let config = ::Config::builder().region(region).build(); - let client = ::Client::from_conf(&config); - } - - async fn after() { - use aws_config::meta::region::RegionProviderChain; - let region_provider = RegionProviderChain::first_try(<1 provider>).or_default_provider(); - // `shared_config` can be used to construct multiple different service clients! - let shared_config = aws_config::from_env().region(region_provider).load().await; - let client = ::Client::new(&shared_config) - } - ``` - -### If you used `aws-auth-providers` -All credential providers that were in `aws-auth-providers` have been moved to `aws-config`. Unless you have a specific use case -for a specific credential provider, you should use the default provider chain: - -```rust - let shared_config = aws_config::load_from_env().await; - let client = ::Client::new(&shared_config); -``` - -### If you maintain your own credential provider - -`AsyncProvideCredentials` has been renamed to `ProvideCredentials`. The trait has been moved from `aws-auth` to `aws-types`. -The original `ProvideCredentials` trait has been removed. The return type has been changed to by a custom future. - -For synchronous use cases: -```rust -use aws_types::credentials::{ProvideCredentials, future}; - -#[derive(Debug)] -struct CustomCreds; -impl ProvideCredentials for CustomCreds { - fn provide_credentials<'a>(&'a self) -> future::ProvideCredentials<'a> - where - Self: 'a, - { - // if your credentials are synchronous, use `::ready` - // if your credentials are loaded asynchronously, use `::new` - future::ProvideCredentials::ready(todo!()) // your credentials go here - } -} -``` - -For asynchronous use cases: -```rust -use aws_types::credentials::{ProvideCredentials, future, Result}; - -#[derive(Debug)] -struct CustomAsyncCreds; -impl CustomAsyncCreds { - async fn load_credentials(&self) -> Result { - Ok(Credentials::from_keys("my creds...", "secret", None)) - } -} - -impl ProvideCredentials for CustomCreds { - fn provide_credentials<'a>(&'a self) -> future::ProvideCredentials<'a> - where - Self: 'a, - { - future::ProvideCredentials::new(self.load_credentials()) - } -} -``` - -Changes -------- - -**Breaking Changes** - -- Credential providers from `aws-auth-providers` have been moved to `aws-config` (smithy-rs#678) -- `AsyncProvideCredentials` has been renamed to `ProvideCredentials`. The original non-async provide credentials has been - removed. See the migration guide above. -- `::from_env()` has been removed (#675). A drop-in replacement is available: - 1. Add a dependency on `aws-config`: - ```toml - [dependencies] - aws-config = { git = "https://github.com/awslabs/aws-sdk-rust", tag = "v0.0.17-alpha" } - ``` - 2. Update your client creation code: - ```rust - let client = >::Client::new(&aws_config::load_from_env().await) - ``` - -- `ProvideRegion` has been moved to `aws_config::meta::region::ProvideRegion`. (smithy-rs#675) -- `aws_types::region::ChainProvider` has been moved to `aws_config::meta::region::RegionProviderChain` (smithy-rs#675). -- `ProvideRegion` is now asynchronous. Code that called `provider.region()` must be changed to `provider.region().await`. -- `::Config::builder()` will **not** load a default region. To preserve previous behavior: - 1. Add a dependency on `aws-config`: - ```toml - [dependencies] - aws-config = { git = "https://github.com/awslabs/aws-sdk-rust", tag = "v0.0.17-alpha" } - ``` - 2. ```rust - let shared_config = aws_config::load_from_env().await; - let config = ::config::Builder::from(&shared_config)..build(); - ``` - -**New this week** - -- :tada: Add profile file provider for region (smithy-rs#594, smithy-rs#682) -- :tada: Add support for shared configuration between multiple services (smithy-rs#673) -- :tada: Add support for Transcribe `StartStreamTranscription` and S3 `SelectObjectContent` operations (smithy-rs#667) -- :tada: Add support for new MemoryDB service (smithy-rs#677) -- Improve documentation on collection-aware builders (smithy-rs#664) -- Update AWS SDK models (smithy-rs#677) -- :bug: Fix sigv4 signing when request ALPN negotiates to HTTP/2. (smithy-rs#674) -- :bug: Fix integer size on S3 `Size` (smithy-rs#679, #209) -- :bug: Fix MediaLive response parsing issue (smithy-rs#683, #212) - - -v0.0.16-alpha (August 19th, 2021) -================================= - -**New This Week** - -- :tada: Add Chime Identity, Chime Messaging, and Snow Device Management support (smithy-rs#657) -- :tada: Add profile file credential provider implementation. This implementation currently does not support credential sources for assume role providers other than environment variables. (smithy-rs#640) -- :tada: Add support for WebIdentityToken providers via profile & environment variables. (smithy-rs#654) -- :bug: Fix name collision that occurred when a model had both a union and a structure named `Result` (smithy-rs#643) -- :bug: Fix STS Assume Role with WebIdentity & Assume role with SAML to support clients with no credentials provided (smithy-rs#652) -- Update AWS SDK models (smithy-rs#657) -- Add initial implementation of a default provider chain. (smithy-rs#650) - -v0.0.15-alpha (August 11th, 2021) -================================= - -This release primarily contains internal changes to runtime components & updates to AWS models. - -**Breaking changes** - -- (smithy-rs#635) The `config()`, `config_mut()`, `request()`, and `request_mut()` methods on `operation::Request` have been renamed to `properties()`, `properties_mut()`, `http()`, and `http_mut()` respectively. -- (smithy-rs#635) The `Response` type on Tower middleware has been changed from `http::Response` to `operation::Response`. The HTTP response is still available from the `operation::Response` using its `http()` and `http_mut()` methods. -- (smithy-rs#635) The `ParseHttpResponse` trait's `parse_unloaded()` method now takes an `operation::Response` rather than an `http::Response`. -- (smithy-rs#626) `ParseHttpResponse` no longer has a generic argument for the body type, but instead, always uses `SdkBody`. This may cause compilation failures for you if you are using Smithy generated types to parse JSON or XML without using a client to request data from a service. The fix should be as simple as removing `` in the example below: - - Before: - ```rust - let output = >::parse_loaded(&parser, &response).unwrap(); - ``` - - After: - ```rust - let output = ::parse_loaded(&parser, &response).unwrap(); - ``` - -**New This Week** - -- The closure passed to `async_provide_credentials_fn` can now borrow values (smithy-rs#637) -- Bring in the latest AWS models (smithy-rs#630) - -v0.0.14-alpha (July 28th, 2021) -=============================== - -IoT Data Plane is now available! If you discover it isn't functioning as expected, please let us know! - -This week also sees the addition of a robust async caching credentials provider. Take a look at the [STS example](https://github.com/awslabs/smithy-rs/blob/7fa4af4a9367aeca6d55e26fc4d4ba93093b90c4/aws/sdk/examples/sts/src/bin/credentials-provider.rs) to see how to use it. - -To upgrade to the new release, update `tag` to `v0.0.14-alpha`: -``` -[dependencies] -# e.g. S3: -aws-sdk-s3 = { git = "https://github.com/awslabs/aws-sdk-rust", tag = "v0.0.14-alpha" } -``` - -**New This Week** - -- :tada: Add IoT Data Plane (smithy-rs#624) -- :tada: Add LazyCachingCredentialsProvider to aws-auth for use with expiring credentials, such as STS AssumeRole. Update STS example to use this new provider (smithy-rs#578, smithy-rs#595) -- :bug: Correctly encode HTTP Checksums using base64 instead of hex. Fixes #164. (smithy-rs#615) -- Overhaul serialization/deserialization of numeric/boolean types. This resolves issues around serialization of NaN/Infinity and should also reduce the number of allocations required during serialization. (smithy-rs#618) -- Update SQS example to clarify usage of FIFO vs. standard queues (#162, @trevorrobertsjr) - -**Contributions** - -Thank you for your contributions! :heart: - -- @trevorrobertsjr (#622) - - -v0.0.13-alpha (July 28th, 2021) -=============================== - -:tada: This week's release includes most of the remaining AWS services (269 in total!). - -**Breaking changes** -- `test-util` has been made an optional dependency and has moved from - aws-hyper to smithy-http. If you were relying on `aws_hyper::TestConnection`, add `smithy-client` as a dependency - and enable the optional `test-util` feature. This prunes some unnecessary dependencies on `roxmltree` and `serde_json` - for most users. (smithy-rs#608) - -**New This Week** -- :tada: Release all but four remaining AWS services! Glacier, IoT Data Plane, Timestream DB and Transcribe Streaming will be available in a future release. If you discover that a service isn't functioning as expected please let us know! (smithy-rs#607) -- :bug: Bugfix: Fix parsing bug where parsing XML incorrectly stripped whitespace (smithy-rs#590, #153) -- We now run some tests on Windows (smithy-rs#594) -- :bug: Bugfix: Constrain RFC-3339 timestamp formatting to microsecond precision (smithy-rs#596, #152) - - -v0.0.12-alpha (July 19th, 2021) -=============================== - -This week we've added Autoscaling and fixed an S3 bug. - -To update to the new release, change your tag to v0.0.12-alpha. - -**New this Week** -- :tada: Add support for Autoscaling (#576, #582) -- `AsyncProvideCredentials` now introduces an additional lifetime parameter, simplifying bridging it with `#[async_trait]` interfaces -- Fix S3 bug when content type was set explicitly (aws-sdk-rust#131, #566, @eagletmt) - -**Contributions** -Thank you for your contributions! :heart: -- @eagletmt (#566) - - -v0.0.11-alpha (July 6th, 2021) -============================== - -This week, we've added AWS Config, EBS, Cognito, and Snowball. Projects that are implementing the `ProvideCredentials` trait will need to update their imports and should consider using the new `async_provide_credentials_fn` for async credential use-cases. - -To update to the new release, change your tag to `v0.0.11-alpha`. - -**New this Week** -- :warning: **Breaking Change:** `ProvideCredentials` and `CredentialError` were both moved into `aws_auth::provider` when they were previously in `aws_auth` (#572) -- :tada: Add support for AWS Config (#570) -- :tada: Add support for EBS (#567) -- :tada: Add support for Cognito (#573) -- :tada: Add support for Snowball (#579, @landonxjames) -- Make it possible to asynchronously provide credentials with `async_provide_credentials_fn` (#572, #577) -- Improve RDS, QLDB, Polly, and KMS examples (#561, #560, #558, #556, #550) -- Update AWS SDK models (#575) -- :bug: Bugfix: Fill in message from error response even when it doesn't match the modeled case format (#565) - -**Contributions** - -Thank you for your contributions! :heart: - -- landonxjames (#579) - - -v0.0.10-alpha (June 29th, 2021) -=============================== - -This week, we've added EKS, ECR and Cloudwatch. The JSON deserialization implementation has been replaced, please be -on the lookout for potential issues and compile time improvements. - -To update to the new release, change your tag to `v0.0.10-alpha`. - -**New this Week** -- :tada: Add support for ECR (smithy-rs#557) -- :tada: Add support for Cloudwatch (smithy-rs#554) -- :tada: Add support for EKS (smithy-rs#553) -- :warning: **Breaking Change:** httpLabel no longer causes fields to be non-optional. You may need to adapt code that uses models. (#537) -- :warning: **Breaking Change:** `Exception` is **not** renamed to `Error`. Code may need to be updated to replace `Error` with `Exception` when naming error shapes. -- :warning: **Breaking Change:** Models are now in strict pascal case including acronyms (e.g. `dynamodb::model::{SSESpecification => SseSpecification}`) -- Add more SES examples, and improve examples for Batch. -- Improved error handling ergonomics: Errors now provide `is_()` methods to simplify error handling -- :bug: Bugfix: Fix bug in `create_multipart_upload`: #127 (smithy-rs#531, @eagletmt) - -**Contributors** - -Thank you for your contributions! :heart: - -- @eagletmt (#531) - - -v0.0.9-alpha (June 22th, 2021) -============================== - -This week, we've added CloudWatch Logs support and fixed several bugs in the generated S3 clients. -There are breaking changes on builders and unions this week. - -To upgrade to the new release, update `tag` to `v0.0.9-alpha`: - -```toml -[dependencies] -# e.g. Cloudwatch Logs: -aws-sdk-cloudwatchlogs = { git = "https://github.com/awslabs/aws-sdk-rust", tag = "v0.0.9-alpha" } -``` - -**New this Week** -- :tada: Add support for CloudWatch Logs (smithy-rs#526) -- :warning: **Breaking Change:** The `set_*` functions on generated Builders now always take an `Option` (smithy-rs#506) -- :warning: **Breaking Change:** The `as_*` functions on unions now return `Result` rather than `Option` to clearly indicate what the actual value is (smithy-rs#527) -- Add more S3 examples, and improve SNS, SQS, and SageMaker examples. Improve example doc comments (smithy-rs#490, smithy-rs#508, smithy-rs#509, smithy-rs#510, smithy-rs#511, smithy-rs#512, smithy-rs#513, smithy-rs#524) -- Combine individual example packages into per-service example packages with multiple binaries (smithy-rs#481, smithy-rs#490) -- :bug: Bugfix: Show response body in trace logs for calls that don't return a stream (smithy-rs#514) -- :bug: Bugfix: Correctly parse S3's GetBucketLocation response (smithy-rs#516) -- :bug: Bugfix: Fix S3 ListObjectsV2 for prefixes containing tilde characters (smithy-rs#519) -- :bug: Bugfix: Fix S3 PutBucketLifecycle operation by adding support for the `@httpChecksumRequired` Smithy trait (smithy-rs#523) -- :bug: Bugfix: Correctly parse `x-amz-expiration` header on S3 GetObject responses (smithy-rs#525, @eagletmt) - -**Contributions** - -Thank you for your contributions! :heart: - -- @eagletmt (smithy-rs#525) -- @zekisherif (smithy-rs#515) - - -v0.0.8-alpha (June 15th, 2021) -============================== - -This week, we've added CloudFormation, SageMaker, EC2, and SES. More details below. - -To upgrade to the new release, update `tag` to `v0.0.8-alpha`: - -```toml -[dependencies] -# e.g. EC2: -aws-sdk-ec2 = { git = "https://github.com/awslabs/aws-sdk-rust", tag = "v0.0.8-alpha" } -``` - -**New this Week** -- :tada: Add support for CloudFormation (smithy-rs#500, @alistaim) -- :tada: Add support for SageMaker (smithy-rs#473, @alistaim) -- :tada: Add support for EC2 (smithy-rs#495) -- :tada: Add support for SES (smithy-rs#499) -- Add support for the EC2 Query protocol (smithy-rs#475) -- Refactor smithy/hyper connectors to enable client-specified middleware (smithy-rs#496, @jonhoo) -- :bug: Bugfix: RFC-3339 timestamp formatting is no longer truncating zeros off of the number of seconds (smithy-rs#479, smithy-rs#489) - -Contributors: -- @Doug-AWS -- @jdisanti -- @rcoh -- @alistaim -- @jonhoo - -Thanks!! - - -v0.0.7-alpha (June 8th, 2021) -============================= - -This week we’ve added MediaLive, MediaPackage, SNS, Batch, STS, RDS, RDSData, Route53, and IAM. More details below. - -To upgrade to the new release, update `tag` to `v0.0.7-alpha`: -```toml -[dependencies] -# e.g. SNS: -aws-sdk-sns = { git = "https://github.com/awslabs/aws-sdk-rust", tag = "v0.0.7-alpha" } -``` - -**New this Week** -- **Breaking change**: Some string enums have changed case:`DynamoDB::{SSEStatus => SseStatus. SSEType => SseType}` -- :tada: Add support for MediaLive and MediaPackage (#449, @alastaim) -- :tada: Add support for SNS (smithy-rs#450) -- :tada: Add support for Batch (smithy-rs#452) -- :tada: Add support for STS. **Note:** This does not include support for an STS-based credential provider although an example is provided. (smithy-rs#453) -- :tada: Add support for RDS (smithy-rs#455) and RDS-Data (smithy-rs#470). (@LMJW) -- :tada: Add support for Route53 (smithy-rs#457, @alistaim) -- Support AWS Endpoints & Regions. With this update, regions like `iam-fips` and `cn-north-1` will now resolve to the correct endpoint. Please report any issues with endpoint resolution. (smithy-rs#468) -- :bug: Primitive numerics and booleans are now filtered from serialization when they are 0 and not marked as required. This resolves issues where maxResults needed to be set even though it is optional & fixes errors during deserialization. (smithy-rs#451) -- :bug: S3 Head Object returned the wrong error when the object did not exist (smithy-rs#460, fixes smithy-rs#456) - - -Contributors: -- @rcoh -- @jdisanti -- @alistaim -- @LMJW - -Thanks! - - -v0.0.6-alpha (June 1st, 2021) -============================= - -**New this week:** - -- :tada: Add support for SQS. SQS is our first service to use the awsQuery protocol. Please report any issues you may encounter. -- :tada: Add support for ECS. -- **Breaking Change**: Refactored `smithy_types::Error` to be more flexible. Internal fields of `Error` are now private and can now be accessed accessor functions. (smithy-rs#426) -- **Breaking change**: Smithy Enums do not implement `serde::Serialize` -- `ByteStream::from_path` now accepts `implications AsRef` (@LMJW) -- Add support for S3 extended request id (smithy-rs#429) -- Add support for the awsQuery protocol. smithy-rs can now add support for all services except EC2. -- **Bugfix**: Timestamps that fell precisely on minute boundaries were not properly formatted (smithy-rs#435) -- Improve documentation for `ByteStream` & add `pub use ByteStream` to generated crates (smithy-rs#443) -- Add support for `EndpointPrefix` needed for [`s3::WriteGetObjectResponse`](https://awslabs.github.io/aws-sdk-rust/aws_sdk_s3/operation/struct.WriteGetObjectResponse.html) (smithy-rs#420) - -Contributors: -- @jdisanti -- @rcoh -- @LMJW - -Thanks! - - -v0.0.5-alpha (May 25th, 2021) -============================= - -You can install the new release by updating your dependencies to `tag = "v0.0.5-alpha"`, e.g. -```toml -[dependencies] -aws-sdk-s3 = { git = "https://github.com/awslabs/aws-sdk-rust", tag = "v0.0.5-alpha" } -``` - -## New This Week -- :tada: Add S3 support. S3 is the first protocol to use our new XML serializers which increases the likelihood of undiscovered issues. In addition, virtual addressing, dualstack and transfer acceleration are not currently supported. Please try it out and let us know if you run into any problems! (smithy-rs#398) :tada: -- :tada: Add support for SSM. SSM was prioritized based on your votes—Please keep voting for the services and feature most important to you! (smithy-rs#393) :tada: -- Add request/response tracing. These can be enabled via tracing subscriber by setting: `RUST_LOG='smithy_http_tower::dispatch=trace,smithy_http::middleware=trace'` (smithy-rs#397) -- Bugfix: Generated service docs were missing at the module level (smithy-rs#404) -- `ByteStream` can now be created from `Path` and `File` via `ByteStream::from_path` (smithy-rs#412) -- Example code now uses `write_all_buf` (#408, @lmjw) -- The `Authorization` and `x-amz-security-token` headers are now marked as sensitive and will be omitted from logs even when full request/response tracing is enabled - -And more: See the corresponding [smithy-rs release](https://github.com/awslabs/smithy-rs/releases/tag/v0.10). - -Contributors: -- @rcoh -- @jdisanti -- @LMJW - -Thanks! - - -v0.0.4-alpha (May 18th, 2021) -============================= - -You can install the new release by updating your dependencies to `tag = "v0.0.4-alpha"`, e.g. -```toml -[dependencies] -aws-sdk-lambda = { git = "https://github.com/awslabs/aws-sdk-rust", tag = "v0.0.4-alpha" } -``` - -**New this week**: - -- :tada: Add support for AWS Lambda (smithy-rs#361, @richardhboyd) :tada: -- Generate docs automatically and host on GitHub Pages: https://awslabs.github.io/aws-sdk-rust/ (#81) -- Add support for streaming request bodies. This is technically a **breaking change** but no currently generated AWS services expose this type. (smithy-rs#359) -- Types represented by the Smithy `Set` type now generate `Vec` in all cases. This is also technically breaking but not currently exposed. (smithy-rs#270) -- Bugfix: The `.message()`field of errors will now look for both `message` and `Message` in the model (smithy-rs#374) -- Add support for the `AWS_REGION` environment variable. (smithy-rs#362) -- The request type generated by the fluent builders, e.g. `dynamodb.list_tables()` is now `Debug` (smithy-rs#377, @declanvk) - -And more: See the corresponding [smithy-rs release](https://github.com/awslabs/smithy-rs/releases/tag/v0.9). - -Contributors: -- @richardhboyd -- @declanvk -- @jdisanti2019 -- @rcoh - -Thanks! - - -v0.0.3-alpha (May 6th, 2021) -============================ - -**New this week:** - -- Fix stack overflow in `SdkBody` Debug implementation -- Upgrade to Smithy 1.7. This adds support for several new API Gateway endpoints -- Add support for streaming response bodies. This is currently only used in Polly -- Added code examples for Kinesis - -More details in smithy-rs: https://github.com/awslabs/smithy-rs/releases/tag/v0.8 diff --git a/aws/SDK_CHANGELOG.next.json b/aws/SDK_CHANGELOG.next.json new file mode 100644 index 000000000..1aaa166b6 --- /dev/null +++ b/aws/SDK_CHANGELOG.next.json @@ -0,0 +1,49 @@ +# This file will be used by automation when cutting a release of the SDK +# to include code generator change log entries into the release notes. +# This is an auto-generated file. Do not edit. + +{ + "smithy-rs": [], + "aws-sdk-rust": [ + { + "message": "Add method `ByteStream::into_async_read`. This makes it easy to convert `ByteStream`s into a struct implementing `tokio:io::AsyncRead`. Available on **crate feature** `rt-tokio` only.", + "meta": { + "bug": false, + "breaking": false, + "tada": true + }, + "author": "Velfi", + "references": [ + "smithy-rs#1390" + ], + "since-commit": "1d81b4f717c1f4416cd3e9680704624ce07c57c4" + }, + { + "message": "Switch to [RustCrypto](https://github.com/RustCrypto)'s implementation of MD5.", + "meta": { + "bug": false, + "breaking": false, + "tada": false + }, + "author": "petrosagg", + "references": [ + "smithy-rs#1404" + ], + "since-commit": "1d81b4f717c1f4416cd3e9680704624ce07c57c4" + }, + { + "message": "Add support for `credential_process` in AWS configs for fetching credentials from an external process.", + "meta": { + "bug": false, + "breaking": false, + "tada": true + }, + "author": "jszwedko", + "references": [ + "smithy-rs#1356" + ], + "since-commit": "1d81b4f717c1f4416cd3e9680704624ce07c57c4" + } + ], + "aws-sdk-model": [] +} diff --git a/aws/sdk/README.md b/aws/sdk/README.md index 45cdad7c6..160bea4da 100644 --- a/aws/sdk/README.md +++ b/aws/sdk/README.md @@ -22,17 +22,24 @@ Controlling service generation You can use gradle properties to opt/out of generating specific services: ```bash -# generate only s3,ec2,sts +# Generate only s3,ec2,sts ./gradlew -Paws.services=+s3,+ec2,+sts :aws:sdk:assemble -# generate a complete SDK for release -./gradlew -Paws.fullsdk=true :aws:sdk:assemble +# Generate all AWS services using models from the aws-sdk-rust repo +./gradlew \ + -Paws.sdk.models.path=../aws-sdk-rust/aws-models \ + :aws:sdk:assemble + +# Generate only S3 from using the model from the aws-sdk-rust repo +./gradlew \ + -Paws.sdk.models.path=../aws-sdk-rust/aws-models \ + -Paws.services=+s3 \ + :aws:sdk:assemble ``` The generation logic is as follows: -1. If `aws.services` is specified, generate an SDK based on the inclusion/exclusion list. -2. Otherwise, if `aws.fullsdk` is specified generate an SDK based on `aws.services.fullsdk`. -3. Otherwise, generate an SDK based on `aws.services.smoketest` +1. If `aws.sdk.models.path` is specified, take models from that path. Otherwise take them from the local `aws-models` directory. +2. Reference the `aws.services` property to determine which models to include/exclude, based on the service module names. Debugging with IntelliJ ----------------------- diff --git a/aws/sdk/aws-models/README.md b/aws/sdk/aws-models/README.md new file mode 100644 index 000000000..ab03c69f0 --- /dev/null +++ b/aws/sdk/aws-models/README.md @@ -0,0 +1,16 @@ +This directory contains a snapshot of a small subset of AWS service models to test the code generator against. +These were carefully selected to exercise every Smithy AWS protocol: + + - `@awsJson1_0`: dynamodb + - `@awsJson1_1`: config + - `@awsQuery`: sts + - `@ec2Query`: ec2 + - `@restJson1`: polly + - `@restXml`: s3 + - Allow-listed Event Stream: transcribestreaming + +All other services in this directory not listed above have integration tests that need to run in CI. + +When generating the full SDK for releases, the models in [awslabs/aws-sdk-rust]'s `aws-models` directory are used. + +[awslabs/aws-sdk-rust]: https://github.com/awslabs/aws-sdk-rust diff --git a/aws/sdk/build.gradle.kts b/aws/sdk/build.gradle.kts index 394b036b0..46306169e 100644 --- a/aws/sdk/build.gradle.kts +++ b/aws/sdk/build.gradle.kts @@ -51,7 +51,7 @@ dependencies { // Class and functions for service and protocol membership for SDK generation -val awsServices: AwsServices by lazy { discoverServices(loadServiceMembership()) } +val awsServices: AwsServices by lazy { discoverServices(properties.get("aws.sdk.models.path"), loadServiceMembership()) } val eventStreamAllowList: Set by lazy { eventStreamAllowList() } val crateVersioner by lazy { aws.sdk.CrateVersioner.defaultFor(rootProject, properties) } @@ -63,14 +63,8 @@ fun loadServiceMembership(): Membership { val membershipOverride = properties.get("aws.services")?.let { parseMembership(it) } println(membershipOverride) val fullSdk = - parseMembership(properties.get("aws.services.fullsdk") ?: throw Exception("full sdk list missing")) - val tier1 = - parseMembership(properties.get("aws.services.smoketest") ?: throw Exception("smoketest list missing")) - return membershipOverride ?: if ((properties.get("aws.fullsdk") ?: "") == "true") { - fullSdk - } else { - tier1 - } + parseMembership(properties.get("aws.services") ?: throw Exception("aws.services list missing")) + return membershipOverride ?: fullSdk } fun eventStreamAllowList(): Set { diff --git a/aws/sdk/gradle.properties b/aws/sdk/gradle.properties index 21d1c656e..1cd163ac6 100644 --- a/aws/sdk/gradle.properties +++ b/aws/sdk/gradle.properties @@ -3,38 +3,8 @@ # SPDX-License-Identifier: Apache-2.0 # - # timestream requires endpoint discovery: https://github.com/awslabs/aws-sdk-rust/issues/114 -aws.services.fullsdk=-timestreamwrite,-timestreamquery - -# Generate an entire sdk vs. aws.services.smoketest -aws.fullsdk=false - -# Below is a base set of services that are generated unless other options are specified. -# These are carefully selected to exercise every Smithy protocol. -# - @awsJson1_0: dynamodb -# - @awsJson1_1: config -# - @awsQuery: sts -# - @ec2Query: ec2 -# - @restJson1: polly -# - @restXml: s3 -# - Allow-listed Event Stream: transcribestreaming -# All other services in this list have integration tests that need to run in CI. -aws.services.smoketest=\ - +config,\ - +dynamodb,\ - +ec2,\ - +glacier,\ - +iam,\ - +kms,\ - +polly,\ - +qldbsession,\ - +s3,\ - +s3control,\ - +sts,\ - +sso,\ - +transcribestreaming,\ - +route53 +aws.services=-timestreamwrite,-timestreamquery # List of services to generate Event Stream operations for: aws.services.eventstream.allowlist=\ diff --git a/buildSrc/src/main/kotlin/aws/sdk/ServiceLoader.kt b/buildSrc/src/main/kotlin/aws/sdk/ServiceLoader.kt index 5ffe4280a..67c2fdef0 100644 --- a/buildSrc/src/main/kotlin/aws/sdk/ServiceLoader.kt +++ b/buildSrc/src/main/kotlin/aws/sdk/ServiceLoader.kt @@ -56,8 +56,9 @@ class AwsServices(private val project: Project, services: List) { * Since this function parses all models, it is relatively expensive to call. The result should be cached in a property * during build. */ -fun Project.discoverServices(serviceMembership: Membership): AwsServices { - val models = project.file("aws-models") +fun Project.discoverServices(awsModelsPath: String?, serviceMembership: Membership): AwsServices { + val models = awsModelsPath?.let { File(it) } ?: project.file("aws-models") + logger.info("Using model path: $models") val baseServices = fileTree(models) .sortedBy { file -> file.name } .mapNotNull { file -> @@ -96,17 +97,21 @@ fun Project.discoverServices(serviceMembership: Membership): AwsServices { } } val baseModules = baseServices.map { it.module }.toSet() - - // validate the full exclusion list hits - serviceMembership.exclusions.forEach { disabledService -> - check(baseModules.contains(disabledService)) { - "Service $disabledService was explicitly disabled but no service was generated with that name. Generated:\n ${ - baseModules.joinToString( - "\n " - ) - }" + logger.info("Discovered base service modules to generate: $baseModules") + + // validate the full exclusion list hits if the models directory is set + if (awsModelsPath != null) { + serviceMembership.exclusions.forEach { disabledService -> + check(baseModules.contains(disabledService)) { + "Service $disabledService was explicitly disabled but no service was generated with that name. Generated:\n ${ + baseModules.joinToString( + "\n " + ) + }" + } } } + // validate inclusion list hits serviceMembership.inclusions.forEach { service -> check(baseModules.contains(service)) { "Service $service was in explicit inclusion list but not generated!" } @@ -115,6 +120,9 @@ fun Project.discoverServices(serviceMembership: Membership): AwsServices { this, baseServices.filter { serviceMembership.isMember(it.module) + }.also { services -> + val moduleNames = services.map { it.module } + logger.info("Final service module list: $moduleNames") } ) } diff --git a/ci.mk b/ci.mk index b3b3e7da0..236c360b1 100644 --- a/ci.mk +++ b/ci.mk @@ -88,8 +88,8 @@ generate-aws-sdk: generate-codegen-diff: $(CI_ACTION) $@ $(ARGS) -.PHONY: generate-smithy-rs-runtime-bundle -generate-smithy-rs-runtime-bundle: +.PHONY: generate-smithy-rs-release +generate-smithy-rs-release: $(CI_ACTION) $@ $(ARGS) .PHONY: sanity-test diff --git a/tools/Dockerfile b/tools/Dockerfile index f95982a3f..532a0032b 100644 --- a/tools/Dockerfile +++ b/tools/Dockerfile @@ -103,6 +103,7 @@ RUN set -eux; \ fi; \ cargo install --path tools/publisher; \ cargo +${rust_nightly_version} install --path tools/api-linter; \ + cargo install --path tools/changelogger; \ cargo install --path tools/crate-hasher; \ cargo install --path tools/sdk-lints; \ cargo install --path tools/sdk-sync; \ diff --git a/tools/api-linter/Cargo.lock b/tools/api-linter/Cargo.lock index 4e2a2fe65..b8f463338 100644 --- a/tools/api-linter/Cargo.lock +++ b/tools/api-linter/Cargo.lock @@ -438,6 +438,7 @@ dependencies = [ "anyhow", "async-trait", "serde", + "serde_json", "toml", "tracing", ] diff --git a/tools/changelogger/Cargo.lock b/tools/changelogger/Cargo.lock new file mode 100644 index 000000000..8a74e82d4 --- /dev/null +++ b/tools/changelogger/Cargo.lock @@ -0,0 +1,522 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 3 + +[[package]] +name = "ansi_term" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d52a9bb7ec0cf484c551830a7ce27bd20d67eac647e1befb56b0be4ee39a55d2" +dependencies = [ + "winapi", +] + +[[package]] +name = "anyhow" +version = "1.0.57" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08f9b8508dccb7687a1d6c4ce66b2b0ecef467c94667de27d8d7fe1f8d2a9cdc" + +[[package]] +name = "async-trait" +version = "0.1.56" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96cf8829f67d2eab0b2dfa42c5d0ef737e0724e4a82b01b3e292456202b19716" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "atty" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8" +dependencies = [ + "hermit-abi", + "libc", + "winapi", +] + +[[package]] +name = "autocfg" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" + +[[package]] +name = "bitflags" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" + +[[package]] +name = "cfg-if" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" + +[[package]] +name = "changelogger" +version = "0.1.0" +dependencies = [ + "anyhow", + "clap", + "ordinal", + "pretty_assertions", + "serde", + "serde_json", + "smithy-rs-tool-common", + "tempfile", + "time", + "toml", +] + +[[package]] +name = "clap" +version = "3.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a836566fa5f52f7ddf909a8a2f9029b9f78ca584cd95cf7e87f8073110f4c5c9" +dependencies = [ + "atty", + "bitflags", + "clap_derive", + "clap_lex", + "indexmap", + "lazy_static", + "strsim", + "termcolor", + "textwrap", +] + +[[package]] +name = "clap_derive" +version = "3.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "986fd75d1dfd2c34eb8c9275ae38ad87ea9478c9b79e87f1801f7d866dfb1e37" +dependencies = [ + "heck", + "proc-macro-error", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "clap_lex" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5538cd660450ebeb4234cfecf8f2284b844ffc4c50531e66d584ad5b91293613" +dependencies = [ + "os_str_bytes", +] + +[[package]] +name = "ctor" +version = "0.1.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f877be4f7c9f246b183111634f75baa039715e3f46ce860677d3b19a69fb229c" +dependencies = [ + "quote", + "syn", +] + +[[package]] +name = "diff" +version = "0.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e25ea47919b1560c4e3b7fe0aaab9becf5b84a10325ddf7db0f0ba5e1026499" + +[[package]] +name = "fastrand" +version = "1.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3fcf0cee53519c866c09b5de1f6c56ff9d647101f81c1964fa632e148896cdf" +dependencies = [ + "instant", +] + +[[package]] +name = "hashbrown" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab5ef0d4909ef3724cc8cce6ccc8572c5c817592e9285f5464f8e86f8bd3726e" + +[[package]] +name = "heck" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2540771e65fc8cb83cd6e8a237f70c319bd5c29f78ed1084ba5d50eeac86f7f9" + +[[package]] +name = "hermit-abi" +version = "0.1.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62b467343b94ba476dcb2500d242dadbb39557df889310ac77c5d99100aaac33" +dependencies = [ + "libc", +] + +[[package]] +name = "indexmap" +version = "1.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6012d540c5baa3589337a98ce73408de9b5a25ec9fc2c6fd6be8f0d39e0ca5a" +dependencies = [ + "autocfg", + "hashbrown", +] + +[[package]] +name = "instant" +version = "0.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a5bbe824c507c5da5956355e86a746d82e0e1464f65d862cc5e71da70e94b2c" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "itoa" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "112c678d4050afce233f4f2852bb2eb519230b3cf12f33585275537d7e41578d" + +[[package]] +name = "lazy_static" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" + +[[package]] +name = "libc" +version = "0.2.126" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "349d5a591cd28b49e1d1037471617a32ddcda5731b99419008085f72d5a53836" + +[[package]] +name = "num-integer" +version = "0.1.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "225d3389fb3509a24c93f5c29eb6bde2586b98d9f016636dff58d7c6f7569cd9" +dependencies = [ + "autocfg", + "num-traits", +] + +[[package]] +name = "num-traits" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "578ede34cf02f8924ab9447f50c28075b4d3e5b269972345e7e0372b38c6cdcd" +dependencies = [ + "autocfg", +] + +[[package]] +name = "num_threads" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2819ce041d2ee131036f4fc9d6ae7ae125a3a40e97ba64d04fe799ad9dabbb44" +dependencies = [ + "libc", +] + +[[package]] +name = "once_cell" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7709cef83f0c1f58f666e746a08b21e0085f7440fa6a29cc194d68aac97a4225" + +[[package]] +name = "ordinal" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c80c1530f46e9d8985706d7deb80b83172b250538902f607dea6cd6028851083" +dependencies = [ + "num-integer", +] + +[[package]] +name = "os_str_bytes" +version = "6.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "21326818e99cfe6ce1e524c2a805c189a99b5ae555a35d19f9a284b427d86afa" + +[[package]] +name = "output_vt100" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "628223faebab4e3e40667ee0b2336d34a5b960ff60ea743ddfdbcf7770bcfb66" +dependencies = [ + "winapi", +] + +[[package]] +name = "pin-project-lite" +version = "0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e0a7ae3ac2f1173085d398531c705756c94a4c56843785df85a60c1a0afac116" + +[[package]] +name = "pretty_assertions" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c89f989ac94207d048d92db058e4f6ec7342b0971fc58d1271ca148b799b3563" +dependencies = [ + "ansi_term", + "ctor", + "diff", + "output_vt100", +] + +[[package]] +name = "proc-macro-error" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da25490ff9892aab3fcf7c36f08cfb902dd3e71ca0f9f9517bea02a73a5ce38c" +dependencies = [ + "proc-macro-error-attr", + "proc-macro2", + "quote", + "syn", + "version_check", +] + +[[package]] +name = "proc-macro-error-attr" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1be40180e52ecc98ad80b184934baf3d0d29f979574e439af5a55274b35f869" +dependencies = [ + "proc-macro2", + "quote", + "version_check", +] + +[[package]] +name = "proc-macro2" +version = "1.0.39" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c54b25569025b7fc9651de43004ae593a75ad88543b17178aa5e1b9c4f15f56f" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1feb54ed693b93a84e14094943b84b7c4eae204c512b7ccb95ab0c66d278ad1" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "redox_syscall" +version = "0.2.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62f25bc4c7e55e0b0b7a1d43fb893f4fa1361d0abe38b9ce4f323c2adfe6ef42" +dependencies = [ + "bitflags", +] + +[[package]] +name = "remove_dir_all" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3acd125665422973a33ac9d3dd2df85edad0f4ae9b00dafb1a05e43a9f5ef8e7" +dependencies = [ + "winapi", +] + +[[package]] +name = "ryu" +version = "1.0.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3f6f92acf49d1b98f7a81226834412ada05458b7364277387724a237f062695" + +[[package]] +name = "serde" +version = "1.0.137" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61ea8d54c77f8315140a05f4c7237403bf38b72704d031543aa1d16abbf517d1" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.137" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f26faba0c3959972377d3b2d306ee9f71faee9714294e41bb777f83f88578be" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_json" +version = "1.0.81" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b7ce2b32a1aed03c558dc61a5cd328f15aff2dbc17daad8fb8af04d2100e15c" +dependencies = [ + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "smithy-rs-tool-common" +version = "0.1.0" +dependencies = [ + "anyhow", + "async-trait", + "serde", + "serde_json", + "toml", + "tracing", +] + +[[package]] +name = "strsim" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" + +[[package]] +name = "syn" +version = "1.0.96" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0748dd251e24453cb8717f0354206b91557e4ec8703673a4b30208f2abaf1ebf" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "tempfile" +version = "3.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5cdb1ef4eaeeaddc8fbd371e5017057064af0911902ef36b39801f67cc6d79e4" +dependencies = [ + "cfg-if", + "fastrand", + "libc", + "redox_syscall", + "remove_dir_all", + "winapi", +] + +[[package]] +name = "termcolor" +version = "1.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bab24d30b911b2376f3a13cc2cd443142f0c81dda04c118693e35b3835757755" +dependencies = [ + "winapi-util", +] + +[[package]] +name = "textwrap" +version = "0.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1141d4d61095b28419e22cb0bbf02755f5e54e0526f97f1e3d1d160e60885fb" + +[[package]] +name = "time" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2702e08a7a860f005826c6815dcac101b19b5eb330c27fe4a5928fec1d20ddd" +dependencies = [ + "libc", + "num_threads", +] + +[[package]] +name = "toml" +version = "0.5.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8d82e1a7758622a465f8cee077614c73484dac5b836c02ff6a40d5d1010324d7" +dependencies = [ + "indexmap", + "serde", +] + +[[package]] +name = "tracing" +version = "0.1.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a400e31aa60b9d44a52a8ee0343b5b18566b03a8321e0d321f695cf56e940160" +dependencies = [ + "cfg-if", + "pin-project-lite", + "tracing-attributes", + "tracing-core", +] + +[[package]] +name = "tracing-attributes" +version = "0.1.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc6b8ad3567499f98a1db7a752b07a7c8c7c7c34c332ec00effb2b0027974b7c" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tracing-core" +version = "0.1.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7709595b8878a4965ce5e87ebf880a7d39c9afc6837721b21a5a816a8117d921" +dependencies = [ + "once_cell", +] + +[[package]] +name = "unicode-ident" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5bd2fe26506023ed7b5e1e315add59d6f584c621d037f9368fea9cfb988f368c" + +[[package]] +name = "version_check" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" + +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-util" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70ec6ce85bb158151cae5e5c87f95a8e97d2c0c4b001223f33a334e3ce5de178" +dependencies = [ + "winapi", +] + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" diff --git a/tools/changelogger/Cargo.toml b/tools/changelogger/Cargo.toml new file mode 100644 index 000000000..62cf3551c --- /dev/null +++ b/tools/changelogger/Cargo.toml @@ -0,0 +1,29 @@ +[package] +name = "changelogger" +version = "0.1.0" +authors = ["AWS Rust SDK Team "] +description = "A CLI tool render and update changelogs from TOML changelog files" +edition = "2021" +license = "Apache-2.0" +publish = false + +# Having this here prevents this tool from being included in a higher-level workspace +[workspace] + +[profile.release] +# prefer fast compile time over runtime performance +opt-level = 0 + +[dependencies] +anyhow = "1.0.57" +clap = { version = "~3.2.1", features = ["derive"] } +ordinal = "0.3.2" +serde = { version = "1", features = ["derive"]} +serde_json = "1" +smithy-rs-tool-common = { path = "../smithy-rs-tool-common" } +time = { version = "0.3.9", features = ["local-offset"]} +toml = "0.5.8" + +[dev-dependencies] +pretty_assertions = "1.2.1" +tempfile = "3.3.0" diff --git a/tools/sdk-lints/smithy-rs-maintainers.txt b/tools/changelogger/smithy-rs-maintainers.txt similarity index 100% rename from tools/sdk-lints/smithy-rs-maintainers.txt rename to tools/changelogger/smithy-rs-maintainers.txt diff --git a/tools/changelogger/src/entry.rs b/tools/changelogger/src/entry.rs new file mode 100644 index 000000000..e97e00019 --- /dev/null +++ b/tools/changelogger/src/entry.rs @@ -0,0 +1,112 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +use anyhow::{bail, Result}; +use clap::clap_derive::ArgEnum; +use smithy_rs_tool_common::changelog::{Changelog, HandAuthoredEntry, SdkModelEntry}; +use smithy_rs_tool_common::git::Git; +use smithy_rs_tool_common::versions_manifest::VersionsManifest; +use std::path::Path; + +#[derive(ArgEnum, Copy, Clone, Debug, Eq, PartialEq)] +pub enum ChangeSet { + SmithyRs, + AwsSdk, +} + +pub struct ChangelogEntries { + pub aws_sdk_rust: Vec, + pub smithy_rs: Vec, +} + +impl ChangelogEntries { + pub fn filter( + self, + smithy_rs: &dyn Git, + change_set: ChangeSet, + previous_release_versions_manifest: Option<&Path>, + ) -> Result> { + match change_set { + ChangeSet::AwsSdk => { + if let Some(manifest_path) = previous_release_versions_manifest { + let manifest = VersionsManifest::from_file(manifest_path)?; + let revisions = + smithy_rs.rev_list("HEAD", &manifest.smithy_rs_revision, None)?; + Ok(self + .aws_sdk_rust + .into_iter() + .filter(|entry| match entry { + ChangelogEntry::AwsSdkModel(_) => true, + ChangelogEntry::HandAuthored(entry) => { + if let Some(since_commit) = &entry.since_commit { + revisions.iter().any(|rev| rev.as_ref() == since_commit) + } else { + true + } + } + }) + .collect()) + } else { + if self.aws_sdk_rust.iter().any(|entry| { + entry.hand_authored().map(|e| e.since_commit.is_some()) == Some(true) + }) { + bail!("SDK changelog entries have `since_commit` information, but no previous release versions manifest was given"); + } + Ok(self.aws_sdk_rust) + } + } + ChangeSet::SmithyRs => Ok(self.smithy_rs), + } + } +} + +impl From for ChangelogEntries { + fn from(mut changelog: Changelog) -> Self { + changelog.aws_sdk_rust.sort_by_key(|entry| !entry.meta.tada); + changelog.sdk_models.sort_by(|a, b| a.module.cmp(&b.module)); + changelog.smithy_rs.sort_by_key(|entry| !entry.meta.tada); + + ChangelogEntries { + smithy_rs: changelog + .smithy_rs + .into_iter() + .map(ChangelogEntry::HandAuthored) + .collect(), + aws_sdk_rust: changelog + .aws_sdk_rust + .into_iter() + .map(ChangelogEntry::HandAuthored) + .chain( + changelog + .sdk_models + .into_iter() + .map(ChangelogEntry::AwsSdkModel), + ) + .collect(), + } + } +} + +#[derive(Clone, Debug)] +pub enum ChangelogEntry { + HandAuthored(HandAuthoredEntry), + AwsSdkModel(SdkModelEntry), +} + +impl ChangelogEntry { + pub fn hand_authored(&self) -> Option<&HandAuthoredEntry> { + match self { + ChangelogEntry::HandAuthored(hand_authored) => Some(hand_authored), + _ => None, + } + } + + pub fn aws_sdk_model(&self) -> Option<&SdkModelEntry> { + match self { + ChangelogEntry::AwsSdkModel(sdk_model) => Some(sdk_model), + _ => None, + } + } +} diff --git a/tools/changelogger/src/lib.rs b/tools/changelogger/src/lib.rs new file mode 100644 index 000000000..8eb12613f --- /dev/null +++ b/tools/changelogger/src/lib.rs @@ -0,0 +1,8 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +pub mod entry; +pub mod render; +pub mod split; diff --git a/tools/changelogger/src/main.rs b/tools/changelogger/src/main.rs new file mode 100644 index 000000000..5e3655ad6 --- /dev/null +++ b/tools/changelogger/src/main.rs @@ -0,0 +1,152 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +use anyhow::Result; +use clap::Parser; +use render::subcommand_render; +use split::subcommand_split; + +mod entry; +mod render; +mod split; + +#[derive(Parser, Debug, Eq, PartialEq)] +#[clap(name = "changelogger", author, version, about)] +pub enum Args { + /// Split SDK changelog entries into a separate file + Split(split::SplitArgs), + /// Render a TOML/JSON changelog into GitHub-flavored Markdown + Render(render::RenderArgs), +} + +fn main() -> Result<()> { + match Args::parse() { + Args::Split(split) => subcommand_split(&split), + Args::Render(render) => subcommand_render(&render), + } +} + +#[cfg(test)] +mod tests { + use super::Args; + use crate::entry::ChangeSet; + use crate::render::RenderArgs; + use crate::split::SplitArgs; + use clap::Parser; + use std::path::PathBuf; + + #[test] + fn args_parsing() { + assert_eq!( + Args::Split(SplitArgs { + source: PathBuf::from("fromplace"), + destination: PathBuf::from("someplace"), + since_commit: None, + smithy_rs_location: None, + }), + Args::try_parse_from([ + "./changelogger", + "split", + "--source", + "fromplace", + "--destination", + "someplace" + ]) + .unwrap() + ); + + assert_eq!( + Args::Render(RenderArgs { + change_set: ChangeSet::SmithyRs, + independent_versioning: false, + source: vec![PathBuf::from("fromplace")], + source_to_truncate: PathBuf::from("fromplace"), + changelog_output: PathBuf::from("some-changelog"), + release_manifest_output: Some(PathBuf::from("some-manifest")), + previous_release_versions_manifest: None, + date_override: None, + smithy_rs_location: None, + }), + Args::try_parse_from([ + "./changelogger", + "render", + "--change-set", + "smithy-rs", + "--source", + "fromplace", + "--source-to-truncate", + "fromplace", + "--changelog-output", + "some-changelog", + "--release-manifest-output", + "some-manifest" + ]) + .unwrap() + ); + + assert_eq!( + Args::Render(RenderArgs { + change_set: ChangeSet::AwsSdk, + independent_versioning: true, + source: vec![ + PathBuf::from("fromplace"), + PathBuf::from("fromanotherplace") + ], + source_to_truncate: PathBuf::from("fromplace"), + changelog_output: PathBuf::from("some-changelog"), + release_manifest_output: None, + previous_release_versions_manifest: None, + date_override: None, + smithy_rs_location: None, + }), + Args::try_parse_from([ + "./changelogger", + "render", + "--change-set", + "aws-sdk", + "--independent-versioning", + "--source", + "fromplace", + "--source", + "fromanotherplace", + "--source-to-truncate", + "fromplace", + "--changelog-output", + "some-changelog", + ]) + .unwrap() + ); + + assert_eq!( + Args::Render(RenderArgs { + change_set: ChangeSet::AwsSdk, + independent_versioning: true, + source: vec![PathBuf::from("fromplace")], + source_to_truncate: PathBuf::from("fromplace"), + changelog_output: PathBuf::from("some-changelog"), + release_manifest_output: None, + previous_release_versions_manifest: Some(PathBuf::from("path/to/versions.toml")), + date_override: None, + smithy_rs_location: None, + }), + Args::try_parse_from([ + "./changelogger", + "render", + "--change-set", + "aws-sdk", + "--independent-versioning", + "--source", + "fromplace", + "--source-to-truncate", + "fromplace", + "--changelog-output", + "some-changelog", + "--previous-release-versions-manifest", + "path/to/versions.toml" + ]) + .unwrap() + ); + } +} diff --git a/tools/changelogger/src/render.rs b/tools/changelogger/src/render.rs new file mode 100644 index 000000000..20910adde --- /dev/null +++ b/tools/changelogger/src/render.rs @@ -0,0 +1,576 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +use crate::entry::{ChangeSet, ChangelogEntries, ChangelogEntry}; +use anyhow::{Context, Result}; +use clap::Parser; +use ordinal::Ordinal; +use serde::Serialize; +use smithy_rs_tool_common::changelog::{ + Changelog, HandAuthoredEntry, Reference, SdkModelChangeKind, SdkModelEntry, +}; +use smithy_rs_tool_common::git::{find_git_repository_root, Git, GitCLI}; +use std::env; +use std::fmt::Write; +use std::fs; +use std::path::PathBuf; +use time::OffsetDateTime; + +pub const EXAMPLE_ENTRY: &str = r#" +# Example changelog entries +# [[aws-sdk-rust]] +# message = "Fix typos in module documentation for generated crates" +# references = ["smithy-rs#920"] +# meta = { "breaking" = false, "tada" = false, "bug" = false } +# author = "rcoh" +# +# [[smithy-rs]] +# message = "Fix typos in module documentation for generated crates" +# references = ["smithy-rs#920"] +# meta = { "breaking" = false, "tada" = false, "bug" = false } +# author = "rcoh" +"#; + +pub const USE_UPDATE_CHANGELOGS: &str = + ""; + +fn maintainers() -> Vec<&'static str> { + include_str!("../smithy-rs-maintainers.txt") + .lines() + .collect() +} + +#[derive(Parser, Debug, Eq, PartialEq)] +pub struct RenderArgs { + /// Which set of changes to render + #[clap(long, action)] + pub change_set: ChangeSet, + /// Whether or not independent crate versions are being used (defaults to false) + #[clap(long, action)] + pub independent_versioning: bool, + /// Source changelog entries to render + #[clap(long, action, required(true))] + pub source: Vec, + /// Which source to overwrite with an empty changelog template + #[clap(long, action)] + pub source_to_truncate: PathBuf, + #[clap(long, action)] + pub changelog_output: PathBuf, + /// Optional path to output a release manifest file to + #[clap(long, action)] + pub release_manifest_output: Option, + /// Optional path to the SDK's versions.toml file for the previous release. + /// This is used to filter out changelog entries that have `since_commit` information. + #[clap(long, action)] + pub previous_release_versions_manifest: Option, + // Location of the smithy-rs repository. If not specified, the current + // working directory will be used to attempt to find it. + #[clap(long, action)] + pub smithy_rs_location: Option, + + // For testing only + #[clap(skip)] + pub date_override: Option, +} + +pub fn subcommand_render(args: &RenderArgs) -> Result<()> { + let now = args.date_override.unwrap_or_else(OffsetDateTime::now_utc); + + let current_dir = env::current_dir()?; + let repo_root: PathBuf = find_git_repository_root( + "smithy-rs", + args.smithy_rs_location + .as_deref() + .unwrap_or_else(|| current_dir.as_path()), + ) + .context("failed to find smithy-rs repo root")?; + let smithy_rs = GitCLI::new(&repo_root)?; + + if args.independent_versioning { + let smithy_rs_metadata = + date_based_release_metadata(now, "smithy-rs-release-manifest.json"); + let sdk_metadata = date_based_release_metadata(now, "aws-sdk-rust-release-manifest.json"); + update_changelogs(args, &smithy_rs, &smithy_rs_metadata, &sdk_metadata) + } else { + let auto = auto_changelog_meta(&smithy_rs)?; + let smithy_rs_metadata = version_based_release_metadata( + now, + &auto.smithy_version, + "smithy-rs-release-manifest.json", + ); + let sdk_metadata = version_based_release_metadata( + now, + &auto.sdk_version, + "aws-sdk-rust-release-manifest.json", + ); + update_changelogs(args, &smithy_rs, &smithy_rs_metadata, &sdk_metadata) + } +} + +struct ChangelogMeta { + smithy_version: String, + sdk_version: String, +} + +struct ReleaseMetadata { + title: String, + tag: String, + manifest_name: String, +} + +#[derive(Serialize)] +struct ReleaseManifest { + #[serde(rename = "tagName")] + tag_name: String, + name: String, + body: String, + prerelease: bool, +} + +fn date_based_release_metadata( + now: OffsetDateTime, + manifest_name: impl Into, +) -> ReleaseMetadata { + ReleaseMetadata { + title: date_title(&now), + tag: format!( + "release-{year}-{month:02}-{day:02}", + year = now.date().year(), + month = u8::from(now.date().month()), + day = now.date().day() + ), + manifest_name: manifest_name.into(), + } +} + +fn version_based_release_metadata( + now: OffsetDateTime, + version: &str, + manifest_name: impl Into, +) -> ReleaseMetadata { + ReleaseMetadata { + title: format!( + "v{version} ({date})", + version = version, + date = date_title(&now) + ), + tag: format!("v{version}", version = version), + manifest_name: manifest_name.into(), + } +} + +fn date_title(now: &OffsetDateTime) -> String { + format!( + "{month} {day}, {year}", + month = now.date().month(), + day = Ordinal(now.date().day()), + year = now.date().year() + ) +} + +/// Discover the new version for the changelog from gradle.properties and the date. +fn auto_changelog_meta(smithy_rs: &dyn Git) -> Result { + let gradle_props = fs::read_to_string(smithy_rs.path().join("gradle.properties")) + .context("failed to load gradle.properties")?; + let load_gradle_prop = |key: &str| { + let prop = gradle_props + .lines() + .flat_map(|line| line.trim().strip_prefix(key)) + .flat_map(|prop| prop.strip_prefix('=')) + .next(); + prop.map(|prop| prop.to_string()) + .ok_or_else(|| anyhow::Error::msg(format!("missing expected gradle property: {key}"))) + }; + let smithy_version = load_gradle_prop("smithy.rs.runtime.crate.version")?; + let sdk_version = load_gradle_prop("aws.sdk.version")?; + Ok(ChangelogMeta { + smithy_version, + sdk_version, + }) +} + +fn render_model_entry(entry: &SdkModelEntry, out: &mut String) { + write!( + out, + "- `{module}` ({version}): {message}", + module = entry.module, + version = entry.version, + message = entry.message + ) + .unwrap(); +} + +fn to_md_link(reference: &Reference) -> String { + format!( + "[{repo}#{number}](https://github.com/awslabs/{repo}/issues/{number})", + repo = reference.repo, + number = reference.number + ) +} + +/// Write a changelog entry to [out] +/// +/// Example output: +/// `- Add a feature (smithy-rs#123, @contributor)` +fn render_entry(entry: &HandAuthoredEntry, mut out: &mut String) { + let mut meta = String::new(); + if entry.meta.bug { + meta.push('🐛'); + } + if entry.meta.breaking { + meta.push('⚠'); + } + if entry.meta.tada { + meta.push('🎉'); + } + if !meta.is_empty() { + meta.push(' '); + } + let mut references = entry.references.iter().map(to_md_link).collect::>(); + if !maintainers().contains(&entry.author.to_ascii_lowercase().as_str()) { + references.push(format!("@{}", entry.author.to_ascii_lowercase())); + }; + if !references.is_empty() { + write!(meta, "({}) ", references.join(", ")).unwrap(); + } + write!( + &mut out, + "- {meta}{message}", + meta = meta, + message = indented_message(&entry.message), + ) + .unwrap(); +} + +fn indented_message(message: &str) -> String { + let mut out = String::new(); + for (idx, line) in message.lines().enumerate() { + if idx > 0 { + out.push('\n'); + if !line.is_empty() { + out.push_str(" "); + } + } + out.push_str(line); + } + out +} + +fn load_changelogs(args: &RenderArgs) -> Result { + let mut combined = Changelog::new(); + for source in &args.source { + let changelog = Changelog::load_from_file(source) + .map_err(|errs| anyhow::Error::msg(format!("failed to load {source:?}: {errs:#?}")))?; + combined.merge(changelog); + } + Ok(combined) +} + +fn update_changelogs( + args: &RenderArgs, + smithy_rs: &dyn Git, + smithy_rs_metadata: &ReleaseMetadata, + aws_sdk_rust_metadata: &ReleaseMetadata, +) -> Result<()> { + let changelog = load_changelogs(args)?; + let release_metadata = match args.change_set { + ChangeSet::AwsSdk => aws_sdk_rust_metadata, + ChangeSet::SmithyRs => smithy_rs_metadata, + }; + let entries = ChangelogEntries::from(changelog); + let entries = entries.filter( + smithy_rs, + args.change_set, + args.previous_release_versions_manifest.as_deref(), + )?; + let (release_header, release_notes) = render(&entries, &release_metadata.title); + if let Some(output_path) = &args.release_manifest_output { + let release_manifest = ReleaseManifest { + tag_name: release_metadata.tag.clone(), + name: release_metadata.title.clone(), + body: release_notes.clone(), + // All releases are pre-releases for now + prerelease: true, + }; + std::fs::write( + output_path.join(&release_metadata.manifest_name), + serde_json::to_string_pretty(&release_manifest)?, + ) + .context("failed to write release manifest")?; + } + + let mut update = USE_UPDATE_CHANGELOGS.to_string(); + update.push('\n'); + update.push_str(&release_header); + update.push_str(&release_notes); + + let current = std::fs::read_to_string(&args.changelog_output) + .context("failed to read rendered destination changelog")? + .replace(USE_UPDATE_CHANGELOGS, ""); + update.push_str(¤t); + std::fs::write(&args.changelog_output, update).context("failed to write rendered changelog")?; + + std::fs::write(&args.source_to_truncate, EXAMPLE_ENTRY.trim()) + .context("failed to truncate source")?; + eprintln!("Changelogs updated!"); + Ok(()) +} + +fn render_handauthored<'a>(entries: impl Iterator, out: &mut String) { + let (breaking, non_breaking) = entries.partition::, _>(|entry| entry.meta.breaking); + + if !breaking.is_empty() { + out.push_str("**Breaking Changes:**\n"); + for change in breaking { + render_entry(change, out); + out.push('\n'); + } + out.push('\n') + } + + if !non_breaking.is_empty() { + out.push_str("**New this release:**\n"); + for change in non_breaking { + render_entry(change, out); + out.push('\n'); + } + out.push('\n'); + } +} + +fn render_sdk_model_entries<'a>( + entries: impl Iterator, + out: &mut String, +) { + let (features, docs) = + entries.partition::, _>(|entry| matches!(entry.kind, SdkModelChangeKind::Feature)); + if !features.is_empty() { + out.push_str("**Service Features:**\n"); + for entry in features { + render_model_entry(entry, out); + out.push('\n'); + } + out.push('\n'); + } + if !docs.is_empty() { + out.push_str("**Service Documentation:**\n"); + for entry in docs { + render_model_entry(entry, out); + out.push('\n'); + } + out.push('\n'); + } +} + +/// Convert a list of changelog entries into markdown. +/// Returns (header, body) +fn render(entries: &[ChangelogEntry], release_header: &str) -> (String, String) { + let mut header = String::new(); + header.push_str(release_header); + header.push('\n'); + for _ in 0..release_header.len() { + header.push('='); + } + header.push('\n'); + + let mut out = String::new(); + render_handauthored( + entries.iter().filter_map(ChangelogEntry::hand_authored), + &mut out, + ); + render_sdk_model_entries( + entries.iter().filter_map(ChangelogEntry::aws_sdk_model), + &mut out, + ); + + let mut external_contribs = entries + .iter() + .filter_map(|entry| entry.hand_authored().map(|e| e.author.to_ascii_lowercase())) + .filter(|author| !maintainers().contains(&author.as_str())) + .collect::>(); + external_contribs.sort(); + external_contribs.dedup(); + if !external_contribs.is_empty() { + out.push_str("**Contributors**\nThank you for your contributions! ❤\n"); + for contributor_handle in external_contribs { + // retrieve all contributions this author made + let mut contribution_references = entries + .iter() + .filter(|entry| { + entry + .hand_authored() + .map(|e| e.author.eq_ignore_ascii_case(contributor_handle.as_str())) + .unwrap_or(false) + }) + .flat_map(|entry| { + entry + .hand_authored() + .unwrap() + .references + .iter() + .map(to_md_link) + }) + .collect::>(); + contribution_references.sort(); + contribution_references.dedup(); + let contribution_references = contribution_references.as_slice().join(", "); + out.push_str("- @"); + out.push_str(&contributor_handle); + if !contribution_references.is_empty() { + out.push_str(&format!(" ({})", contribution_references)); + } + out.push('\n'); + } + } + + (header, out) +} + +#[cfg(test)] +mod test { + use super::{ + date_based_release_metadata, render, version_based_release_metadata, Changelog, + ChangelogEntries, ChangelogEntry, + }; + use time::OffsetDateTime; + + fn render_full(entries: &[ChangelogEntry], release_header: &str) -> String { + let (header, body) = render(entries, release_header); + return format!("{}{}", header, body); + } + + #[test] + fn end_to_end_changelog() { + let changelog_toml = r#" +[[smithy-rs]] +author = "rcoh" +message = "I made a major change to update the code generator" +meta = { breaking = true, tada = false, bug = false } +references = ["smithy-rs#445"] + +[[smithy-rs]] +author = "external-contrib" +message = "I made a change to update the code generator" +meta = { breaking = false, tada = true, bug = false } +references = ["smithy-rs#446"] + +[[smithy-rs]] +author = "another-contrib" +message = "I made a minor change" +meta = { breaking = false, tada = false, bug = false } + +[[aws-sdk-rust]] +author = "rcoh" +message = "I made a major change to update the AWS SDK" +meta = { breaking = true, tada = false, bug = false } +references = ["smithy-rs#445"] + +[[aws-sdk-rust]] +author = "external-contrib" +message = "I made a change to update the code generator" +meta = { breaking = false, tada = true, bug = false } +references = ["smithy-rs#446"] + +[[smithy-rs]] +author = "external-contrib" +message = """ +I made a change to update the code generator + +**Update guide:** +blah blah +""" +meta = { breaking = false, tada = true, bug = false } +references = ["smithy-rs#446"] + +[[aws-sdk-model]] +module = "aws-sdk-s3" +version = "0.14.0" +kind = "Feature" +message = "Some new API to do X" + +[[aws-sdk-model]] +module = "aws-sdk-ec2" +version = "0.12.0" +kind = "Documentation" +message = "Updated some docs" + +[[aws-sdk-model]] +module = "aws-sdk-ec2" +version = "0.12.0" +kind = "Feature" +message = "Some API change" + "#; + let changelog: Changelog = toml::from_str(changelog_toml).expect("valid changelog"); + let ChangelogEntries { + aws_sdk_rust, + smithy_rs, + } = changelog.into(); + + let smithy_rs_rendered = render_full(&smithy_rs, "v0.3.0 (January 4th, 2022)"); + let smithy_rs_expected = r#" +v0.3.0 (January 4th, 2022) +========================== +**Breaking Changes:** +- ⚠ ([smithy-rs#445](https://github.com/awslabs/smithy-rs/issues/445)) I made a major change to update the code generator + +**New this release:** +- 🎉 ([smithy-rs#446](https://github.com/awslabs/smithy-rs/issues/446), @external-contrib) I made a change to update the code generator +- 🎉 ([smithy-rs#446](https://github.com/awslabs/smithy-rs/issues/446), @external-contrib) I made a change to update the code generator + + **Update guide:** + blah blah +- (@another-contrib) I made a minor change + +**Contributors** +Thank you for your contributions! ❤ +- @another-contrib +- @external-contrib ([smithy-rs#446](https://github.com/awslabs/smithy-rs/issues/446)) +"# + .trim_start(); + pretty_assertions::assert_str_eq!(smithy_rs_expected, smithy_rs_rendered); + + let aws_sdk_rust_rendered = render_full(&aws_sdk_rust, "v0.1.0 (January 4th, 2022)"); + let aws_sdk_expected = r#" +v0.1.0 (January 4th, 2022) +========================== +**Breaking Changes:** +- ⚠ ([smithy-rs#445](https://github.com/awslabs/smithy-rs/issues/445)) I made a major change to update the AWS SDK + +**New this release:** +- 🎉 ([smithy-rs#446](https://github.com/awslabs/smithy-rs/issues/446), @external-contrib) I made a change to update the code generator + +**Service Features:** +- `aws-sdk-ec2` (0.12.0): Some API change +- `aws-sdk-s3` (0.14.0): Some new API to do X + +**Service Documentation:** +- `aws-sdk-ec2` (0.12.0): Updated some docs + +**Contributors** +Thank you for your contributions! ❤ +- @external-contrib ([smithy-rs#446](https://github.com/awslabs/smithy-rs/issues/446)) +"# + .trim_start(); + pretty_assertions::assert_str_eq!(aws_sdk_expected, aws_sdk_rust_rendered); + } + + #[test] + fn test_date_based_release_metadata() { + let now = OffsetDateTime::from_unix_timestamp(100_000_000).unwrap(); + let result = date_based_release_metadata(now, "some-manifest.json"); + assert_eq!("March 3rd, 1973", result.title); + assert_eq!("release-1973-03-03", result.tag); + assert_eq!("some-manifest.json", result.manifest_name); + } + + #[test] + fn test_version_based_release_metadata() { + let now = OffsetDateTime::from_unix_timestamp(100_000_000).unwrap(); + let result = version_based_release_metadata(now, "0.11.0", "some-other-manifest.json"); + assert_eq!("v0.11.0 (March 3rd, 1973)", result.title); + assert_eq!("v0.11.0", result.tag); + assert_eq!("some-other-manifest.json", result.manifest_name); + } +} diff --git a/tools/changelogger/src/split.rs b/tools/changelogger/src/split.rs new file mode 100644 index 000000000..687fd556b --- /dev/null +++ b/tools/changelogger/src/split.rs @@ -0,0 +1,93 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +use anyhow::{Context, Result}; +use clap::Parser; +use smithy_rs_tool_common::changelog::Changelog; +use smithy_rs_tool_common::git::{find_git_repository_root, Git, GitCLI}; +use std::path::{Path, PathBuf}; +use std::{env, fs}; + +const INTERMEDIATE_SOURCE_HEADER: &str = + "# This is an intermediate file that will be replaced after automation is complete.\n\ + # It will be used to generate a changelog entry for smithy-rs.\n\ + # Do not commit the contents of this file!\n"; + +const DEST_HEADER: &str = + "# This file will be used by automation when cutting a release of the SDK\n\ + # to include code generator change log entries into the release notes.\n\ + # This is an auto-generated file. Do not edit.\n"; + +#[derive(Parser, Debug, Eq, PartialEq)] +pub struct SplitArgs { + /// The source path to split changelog entries from + #[clap(long, action)] + pub source: PathBuf, + /// The destination to place changelog entries + #[clap(long, action)] + pub destination: PathBuf, + + // Git revision to use in `since_commit` fields; for testing only + #[clap(skip)] + pub since_commit: Option, + // Location of the smithy-rs repository; for testing only + #[clap(skip)] + pub smithy_rs_location: Option, +} + +pub fn subcommand_split(args: &SplitArgs) -> Result<()> { + let changelog = Changelog::load_from_file(&args.source).map_err(|errs| { + anyhow::Error::msg(format!( + "cannot split changelogs with changelog errors: {:#?}", + errs + )) + })?; + + let (source_log, dest_log) = ( + smithy_rs_entries(changelog.clone()), + sdk_entries(args, changelog).context("failed to filter SDK entries")?, + ); + write_entries(&args.source, INTERMEDIATE_SOURCE_HEADER, &source_log) + .context("failed to write source")?; + write_entries(&args.destination, DEST_HEADER, &dest_log) + .context("failed to write destination")?; + Ok(()) +} + +fn sdk_entries(args: &SplitArgs, mut changelog: Changelog) -> Result { + changelog.smithy_rs.clear(); + + let current_dir = env::current_dir()?; + let repo_root = find_git_repository_root( + "smithy-rs", + args.smithy_rs_location + .as_deref() + .unwrap_or_else(|| current_dir.as_path()), + ) + .context("failed to find smithy-rs root")?; + let last_commit = GitCLI::new(&repo_root)? + .get_head_revision() + .context("failed to get current revision of smithy-rs")?; + let last_commit = args + .since_commit + .as_deref() + .unwrap_or_else(|| last_commit.as_ref()); + for entry in changelog.aws_sdk_rust.iter_mut() { + entry.since_commit = Some(last_commit.to_string()); + } + Ok(changelog) +} + +fn smithy_rs_entries(mut changelog: Changelog) -> Changelog { + changelog.aws_sdk_rust.clear(); + changelog.sdk_models.clear(); + changelog +} + +fn write_entries(into_path: &Path, header: &str, changelog: &Changelog) -> Result<()> { + let json_changelog = changelog.to_json_string()?; + fs::write(into_path, format!("{header}\n{json_changelog}"))?; + Ok(()) +} diff --git a/tools/changelogger/tests/e2e_test.rs b/tools/changelogger/tests/e2e_test.rs new file mode 100644 index 000000000..d0eefefb0 --- /dev/null +++ b/tools/changelogger/tests/e2e_test.rs @@ -0,0 +1,349 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +use changelogger::entry::ChangeSet; +use changelogger::render::{subcommand_render, RenderArgs, EXAMPLE_ENTRY, USE_UPDATE_CHANGELOGS}; +use changelogger::split::{subcommand_split, SplitArgs}; +use smithy_rs_tool_common::git::{CommitHash, Git, GitCLI}; +use smithy_rs_tool_common::shell::handle_failure; +use std::fs; +use std::path::Path; +use std::process::Command; +use tempfile::TempDir; +use time::OffsetDateTime; + +const SOURCE_TOML: &'static str = r#" + [[aws-sdk-rust]] + message = "Some change" + references = ["aws-sdk-rust#123", "smithy-rs#456"] + meta = { "breaking" = false, "tada" = false, "bug" = true } + since-commit = "REPLACE_SINCE_COMMIT_1" + author = "test-dev" + + [[aws-sdk-rust]] + message = "Some other change" + references = ["aws-sdk-rust#234", "smithy-rs#567"] + meta = { "breaking" = false, "tada" = false, "bug" = true } + since-commit = "REPLACE_SINCE_COMMIT_2" + author = "test-dev" + + [[smithy-rs]] + message = "Another change" + references = ["smithy-rs#1234"] + meta = { "breaking" = false, "tada" = false, "bug" = false } + author = "another-dev" + "#; + +const SDK_MODEL_SOURCE_TOML: &'static str = r#" + [[aws-sdk-model]] + module = "aws-sdk-ec2" + version = "0.12.0" + kind = "Feature" + message = "Some API change" +"#; + +fn create_fake_repo_root( + path: &Path, + smithy_rs_version: &str, + aws_sdk_version: &str, +) -> (CommitHash, CommitHash) { + handle_failure( + "git-init", + &Command::new("git") + .arg("init") + .arg(".") + .current_dir(path) + .output() + .unwrap(), + ) + .unwrap(); + fs::write( + path.join("gradle.properties"), + format!( + r#" + smithy.rs.runtime.crate.version={} + aws.sdk.version={} + "#, + smithy_rs_version, aws_sdk_version + ), + ) + .unwrap(); + + // Simulate some release commits + let git = GitCLI::new(path).unwrap(); + let mut release_commits = Vec::new(); + for i in 0..2 { + fs::write(path.join("random-file"), format!("{}", i * 10)).unwrap(); + git.stage(path).unwrap(); + git.commit( + "test-dev", + "test-dev@example.com", + &format!("prepare release {}", i), + ) + .unwrap(); + fs::write(path.join("random-file"), format!("{}", i * 10 + 1)).unwrap(); + git.stage(path).unwrap(); + git.commit( + "test-dev", + "test-dev@example.com", + &format!("finish release {}", i), + ) + .unwrap(); + release_commits.push(git.get_head_revision().unwrap()); + } + (release_commits.remove(0), release_commits.remove(0)) +} + +#[test] +fn split_aws_sdk_test() { + let tmp_dir = TempDir::new().unwrap(); + let source_path = tmp_dir.path().join("source.toml"); + let dest_path = tmp_dir.path().join("dest.toml"); + + create_fake_repo_root(tmp_dir.path(), "0.42.0", "0.12.0"); + + fs::write(&source_path, SOURCE_TOML).unwrap(); + fs::write(&dest_path, "overwrite-me").unwrap(); + + subcommand_split(&SplitArgs { + source: source_path.clone(), + destination: dest_path.clone(), + since_commit: Some("test-commit-hash".into()), + smithy_rs_location: Some(tmp_dir.path().into()), + }) + .unwrap(); + + let source = fs::read_to_string(&source_path).unwrap(); + let dest = fs::read_to_string(&dest_path).unwrap(); + + pretty_assertions::assert_str_eq!( + r#"# This is an intermediate file that will be replaced after automation is complete. +# It will be used to generate a changelog entry for smithy-rs. +# Do not commit the contents of this file! + +{ + "smithy-rs": [ + { + "message": "Another change", + "meta": { + "bug": false, + "breaking": false, + "tada": false + }, + "author": "another-dev", + "references": [ + "smithy-rs#1234" + ], + "since-commit": null + } + ], + "aws-sdk-rust": [], + "aws-sdk-model": [] +}"#, + source + ); + pretty_assertions::assert_str_eq!( + r#"# This file will be used by automation when cutting a release of the SDK +# to include code generator change log entries into the release notes. +# This is an auto-generated file. Do not edit. + +{ + "smithy-rs": [], + "aws-sdk-rust": [ + { + "message": "Some change", + "meta": { + "bug": true, + "breaking": false, + "tada": false + }, + "author": "test-dev", + "references": [ + "aws-sdk-rust#123", + "smithy-rs#456" + ], + "since-commit": "test-commit-hash" + }, + { + "message": "Some other change", + "meta": { + "bug": true, + "breaking": false, + "tada": false + }, + "author": "test-dev", + "references": [ + "aws-sdk-rust#234", + "smithy-rs#567" + ], + "since-commit": "test-commit-hash" + } + ], + "aws-sdk-model": [] +}"#, + dest + ); +} + +#[test] +fn render_smithy_rs_test() { + let tmp_dir = TempDir::new().unwrap(); + let source_path = tmp_dir.path().join("source.toml"); + let dest_path = tmp_dir.path().join("dest.md"); + let release_manifest_path = tmp_dir.path().join("smithy-rs-release-manifest.json"); + + create_fake_repo_root(tmp_dir.path(), "0.42.0", "0.12.0"); + + fs::write(&source_path, SOURCE_TOML).unwrap(); + fs::write( + &dest_path, + format!( + "{}\nv0.41.0 (Some date in the past)\n=========\n\nOld entry contents\n", + USE_UPDATE_CHANGELOGS + ), + ) + .unwrap(); + fs::write(&release_manifest_path, "overwrite-me").unwrap(); + + subcommand_render(&RenderArgs { + change_set: ChangeSet::SmithyRs, + independent_versioning: false, + source: vec![source_path.clone()], + source_to_truncate: source_path.clone(), + changelog_output: dest_path.clone(), + release_manifest_output: Some(tmp_dir.path().into()), + date_override: Some(OffsetDateTime::UNIX_EPOCH), + previous_release_versions_manifest: None, + smithy_rs_location: Some(tmp_dir.path().into()), + }) + .unwrap(); + + let source = fs::read_to_string(&source_path).unwrap(); + let dest = fs::read_to_string(&dest_path).unwrap(); + let release_manifest = fs::read_to_string(&release_manifest_path).unwrap(); + + pretty_assertions::assert_str_eq!(EXAMPLE_ENTRY.trim(), source); + pretty_assertions::assert_str_eq!( + r#" +v0.42.0 (January 1st, 1970) +=========================== +**New this release:** +- ([smithy-rs#1234](https://github.com/awslabs/smithy-rs/issues/1234), @another-dev) Another change + +**Contributors** +Thank you for your contributions! ❤ +- @another-dev ([smithy-rs#1234](https://github.com/awslabs/smithy-rs/issues/1234)) + +v0.41.0 (Some date in the past) +========= + +Old entry contents +"#, + dest + ); + pretty_assertions::assert_str_eq!( + r#"{ + "tagName": "v0.42.0", + "name": "v0.42.0 (January 1st, 1970)", + "body": "**New this release:**\n- ([smithy-rs#1234](https://github.com/awslabs/smithy-rs/issues/1234), @another-dev) Another change\n\n**Contributors**\nThank you for your contributions! ❤\n- @another-dev ([smithy-rs#1234](https://github.com/awslabs/smithy-rs/issues/1234))\n", + "prerelease": true +}"#, + release_manifest + ); +} + +#[test] +fn render_aws_sdk_test() { + let tmp_dir = TempDir::new().unwrap(); + let source1_path = tmp_dir.path().join("source1.toml"); + let source2_path = tmp_dir.path().join("source2.toml"); + let dest_path = tmp_dir.path().join("dest.md"); + let release_manifest_path = tmp_dir.path().join("aws-sdk-rust-release-manifest.json"); + let versions_manifest_path = tmp_dir.path().join("versions.toml"); + + let (release_1_commit, release_2_commit) = + create_fake_repo_root(tmp_dir.path(), "0.42.0", "0.12.0"); + + fs::write( + &source1_path, + SOURCE_TOML + .replace("REPLACE_SINCE_COMMIT_1", release_1_commit.as_ref()) + .replace("REPLACE_SINCE_COMMIT_2", release_2_commit.as_ref()), + ) + .unwrap(); + fs::write(&source2_path, SDK_MODEL_SOURCE_TOML).unwrap(); + fs::write( + &dest_path, + format!( + "{}\nv0.41.0 (Some date in the past)\n=========\n\nOld entry contents\n", + USE_UPDATE_CHANGELOGS + ), + ) + .unwrap(); + fs::write(&release_manifest_path, "overwrite-me").unwrap(); + fs::write( + &versions_manifest_path, + format!( + "smithy_rs_revision = '{release_1_commit}' + aws_doc_sdk_examples_revision = 'not-relevant' + [crates]", + ), + ) + .unwrap(); + + subcommand_render(&RenderArgs { + change_set: ChangeSet::AwsSdk, + independent_versioning: false, + source: vec![source1_path.clone(), source2_path.clone()], + source_to_truncate: source1_path.clone(), + changelog_output: dest_path.clone(), + release_manifest_output: Some(tmp_dir.path().into()), + date_override: Some(OffsetDateTime::UNIX_EPOCH), + previous_release_versions_manifest: Some(versions_manifest_path), + smithy_rs_location: Some(tmp_dir.path().into()), + }) + .unwrap(); + + let source1 = fs::read_to_string(&source1_path).unwrap(); + let source2 = fs::read_to_string(&source2_path).unwrap(); + let dest = fs::read_to_string(&dest_path).unwrap(); + let release_manifest = fs::read_to_string(&release_manifest_path).unwrap(); + + pretty_assertions::assert_str_eq!(EXAMPLE_ENTRY.trim(), source1); + pretty_assertions::assert_str_eq!(SDK_MODEL_SOURCE_TOML, source2); + + // It should only have one of the SDK changelog entries since + // the other should be filtered out by the `since_commit` attribute + pretty_assertions::assert_str_eq!( + r#" +v0.12.0 (January 1st, 1970) +=========================== +**New this release:** +- 🐛 ([aws-sdk-rust#234](https://github.com/awslabs/aws-sdk-rust/issues/234), [smithy-rs#567](https://github.com/awslabs/smithy-rs/issues/567), @test-dev) Some other change + +**Service Features:** +- `aws-sdk-ec2` (0.12.0): Some API change + +**Contributors** +Thank you for your contributions! ❤ +- @test-dev ([aws-sdk-rust#234](https://github.com/awslabs/aws-sdk-rust/issues/234), [smithy-rs#567](https://github.com/awslabs/smithy-rs/issues/567)) + +v0.41.0 (Some date in the past) +========= + +Old entry contents +"#, + dest + ); + pretty_assertions::assert_str_eq!( + r#"{ + "tagName": "v0.12.0", + "name": "v0.12.0 (January 1st, 1970)", + "body": "**New this release:**\n- 🐛 ([aws-sdk-rust#234](https://github.com/awslabs/aws-sdk-rust/issues/234), [smithy-rs#567](https://github.com/awslabs/smithy-rs/issues/567), @test-dev) Some other change\n\n**Service Features:**\n- `aws-sdk-ec2` (0.12.0): Some API change\n\n**Contributors**\nThank you for your contributions! ❤\n- @test-dev ([aws-sdk-rust#234](https://github.com/awslabs/aws-sdk-rust/issues/234), [smithy-rs#567](https://github.com/awslabs/smithy-rs/issues/567))\n", + "prerelease": true +}"#, + release_manifest + ); +} diff --git a/tools/ci-build/ci-create-workspace b/tools/ci-build/ci-create-workspace index f7c764bd9..b7a849eac 100755 --- a/tools/ci-build/ci-create-workspace +++ b/tools/ci-build/ci-create-workspace @@ -29,6 +29,7 @@ cp -r "${SCRIPT_PATH}/scripts" "${ACTION_PATH}/workspace/" # Copy inputs into workspace for input_name in \ "aws-doc-sdk-examples" \ + "aws-sdk-rust" \ "aws-sdk" \ "aws-sdk-smoketest" \ "smithy-rs"; diff --git a/tools/ci-build/scripts/generate-aws-sdk b/tools/ci-build/scripts/generate-aws-sdk index c1a371c3b..f6d69dce8 100755 --- a/tools/ci-build/scripts/generate-aws-sdk +++ b/tools/ci-build/scripts/generate-aws-sdk @@ -8,11 +8,17 @@ C_YELLOW='\033[1;33m' C_RESET='\033[0m' set -eux -# This script takes an optional argument to specify the path to the previous release's versions.toml file -PREVIOUS_RELEASE_VERSIONS_ARG= -if [[ $# -ge 1 ]]; then - echo "Given previous release manifest path: $1" - PREVIOUS_RELEASE_VERSIONS_ARG="-Paws.sdk.previous.release.versions.manifest=$1" +PREVIOUS_RELEASE_VERSIONS_ARG="$(pwd)/aws-sdk-rust/versions.toml" +if [[ ! -f "${PREVIOUS_RELEASE_VERSIONS_ARG}" ]]; then + echo "Failed to find versions.toml file at ${PREVIOUS_RELEASE_VERSIONS_ARG}" + exit 1 +fi + +echo -e "${C_YELLOW}Taking models from 'awslabs/aws-sdk-rust'...${C_RESET}" +AWS_SDK_MODELS_PATH="$(pwd)/aws-sdk-rust/aws-models" +if [[ ! -d "${AWS_SDK_MODELS_PATH}" ]]; then + echo "Failed to find AWS models in ${AWS_SDK_MODELS_PATH}" + exit 1 fi echo -e "${C_YELLOW}Taking examples from 'awsdocs/aws-doc-sdk-examples'...${C_RESET}" @@ -24,11 +30,9 @@ rm smithy-rs/aws/sdk/examples/Cargo.toml echo -e "${C_YELLOW}Generating services...${C_RESET}" cd smithy-rs -# Intentionally not quoting PREVIOUS_RELEASE_VERSIONS_ARG so that if it has no value, it doesn't confuse gradle -# shellcheck disable=SC2086 ./gradlew \ - -Paws.fullsdk=true \ + -Paws.sdk.models.path="${AWS_SDK_MODELS_PATH}" \ -Paws.sdk.examples.revision="${examples_revision}" \ - ${PREVIOUS_RELEASE_VERSIONS_ARG} \ + -Paws.sdk.previous.release.versions.manifest="${PREVIOUS_RELEASE_VERSIONS_ARG}" \ aws:sdk:assemble mv aws/sdk/build/aws-sdk ../artifacts/ diff --git a/tools/ci-build/scripts/generate-aws-sdk-smoketest b/tools/ci-build/scripts/generate-aws-sdk-smoketest index cc00ecd0b..3f2e09609 100755 --- a/tools/ci-build/scripts/generate-aws-sdk-smoketest +++ b/tools/ci-build/scripts/generate-aws-sdk-smoketest @@ -7,6 +7,8 @@ set -eux cd smithy-rs -./gradlew aws:sdk:assemble +# TODO(https://github.com/awslabs/smithy-rs/issues/1493): Remove the `aws.services` property once the +# non-smoketest models have been removed from smithy-rs +./gradlew aws:sdk:assemble '-Paws.services=+config,+dynamodb,+ec2,+glacier,+iam,+kms,+polly,+qldbsession,+s3,+s3control,+sts,+sso,+transcribestreaming,+route53' python3 aws/sdk/test-services.py > aws/sdk/build/aws-sdk/services-with-tests mv aws/sdk/build/aws-sdk ../artifacts/aws-sdk-smoketest diff --git a/tools/ci-build/scripts/generate-smithy-rs-release b/tools/ci-build/scripts/generate-smithy-rs-release new file mode 100755 index 000000000..56908515a --- /dev/null +++ b/tools/ci-build/scripts/generate-smithy-rs-release @@ -0,0 +1,46 @@ +#!/bin/bash +# +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +# + +set -eux + +SMITHY_RS_DIR="$(pwd)/smithy-rs" +ARTIFACTS_DIR="$(pwd)/artifacts/smithy-rs-release" +mkdir -p "${ARTIFACTS_DIR}" + +pushd "${SMITHY_RS_DIR}" +# Split AWS SDK changelog entries into a separate file +changelogger split \ + --source CHANGELOG.next.toml \ + --destination aws/SDK_CHANGELOG.next.json +# Render the remaining smithy-rs changelog entries +changelogger render \ + --change-set smithy-rs \ + --source CHANGELOG.next.toml \ + --source-to-truncate CHANGELOG.next.toml \ + --changelog-output CHANGELOG.md \ + --release-manifest-output "${ARTIFACTS_DIR}" +# Commit changelog changes if there are any +git add . +git diff --staged --quiet || \ + git -c "user.name=AWS SDK Rust Bot" \ + -c "user.email=aws-sdk-rust-primary@amazon.com" \ + commit \ + -am "Update changelog" +# Generate the crates to publish +./gradlew rust-runtime:assemble +popd + +# Move crates into `crates-to-publish` part of the `smithy-rs-release` artifact +mv "${SMITHY_RS_DIR}/rust-runtime/build/smithy-rs/rust-runtime" \ + "${ARTIFACTS_DIR}/crates-to-publish" + +# Clone smithy-rs repo changes (changelog updates) into +# the `smithy-rs-release` artifact for push to GitHub +pushd "${ARTIFACTS_DIR}" +git clone "${SMITHY_RS_DIR}" +# Copy over the original remotes so that it's possibel to push to `origin` +cp "${SMITHY_RS_DIR}/.git/config" smithy-rs/.git/config +popd diff --git a/tools/ci-build/scripts/generate-smithy-rs-runtime-bundle b/tools/ci-build/scripts/generate-smithy-rs-runtime-bundle deleted file mode 100755 index fe1349c64..000000000 --- a/tools/ci-build/scripts/generate-smithy-rs-runtime-bundle +++ /dev/null @@ -1,11 +0,0 @@ -#!/bin/bash -# -# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. -# SPDX-License-Identifier: Apache-2.0 -# - -set -eux -cd smithy-rs - -./gradlew rust-runtime:assemble -mv rust-runtime/build/smithy-rs ../artifacts/smithy-rs-runtime diff --git a/tools/ci-build/scripts/sanity-test b/tools/ci-build/scripts/sanity-test index 58691d881..c46c1931f 100755 --- a/tools/ci-build/scripts/sanity-test +++ b/tools/ci-build/scripts/sanity-test @@ -7,6 +7,7 @@ set -eux cargo --version +changelogger --version crate-hasher --version diff2html --version git --version diff --git a/tools/ci-cdk/canary-runner/Cargo.lock b/tools/ci-cdk/canary-runner/Cargo.lock index 008e3d262..05ceee1c3 100644 --- a/tools/ci-cdk/canary-runner/Cargo.lock +++ b/tools/ci-cdk/canary-runner/Cargo.lock @@ -1435,6 +1435,7 @@ dependencies = [ "anyhow", "async-trait", "serde", + "serde_json", "tokio", "toml", "tracing", diff --git a/tools/ci-cdk/canary-runner/src/run.rs b/tools/ci-cdk/canary-runner/src/run.rs index 53f922b0a..cca51833a 100644 --- a/tools/ci-cdk/canary-runner/src/run.rs +++ b/tools/ci-cdk/canary-runner/src/run.rs @@ -23,9 +23,8 @@ use cloudwatch::model::StandardUnit; use s3::ByteStream; use semver::Version; use serde::Deserialize; -use smithy_rs_tool_common::git; +use smithy_rs_tool_common::git::{find_git_repository_root, Git, GitCLI}; use smithy_rs_tool_common::macros::here; -use smithy_rs_tool_common::shell::ShellOperation; use std::path::PathBuf; use std::time::{Duration, SystemTime}; use std::{env, path::Path}; @@ -190,13 +189,13 @@ pub async fn run(opt: RunOpt) -> Result<()> { } async fn run_canary(options: &Options, config: &aws_config::Config) -> Result { - let repo_root = git_root().await?; - env::set_current_dir(repo_root.join("tools/ci-cdk/canary-lambda")) + let smithy_rs_root = find_git_repository_root("smithy-rs", ".").context(here!())?; + let smithy_rs = GitCLI::new(&smithy_rs_root).context(here!())?; + env::set_current_dir(smithy_rs_root.join("tools/ci-cdk/canary-lambda")) .context("failed to change working directory")?; if let Some(sdk_version) = &options.sdk_version { - use_correct_revision(sdk_version) - .await + use_correct_revision(&smithy_rs, sdk_version) .context(here!("failed to select correct revision of smithy-rs"))?; } @@ -246,7 +245,7 @@ async fn run_canary(options: &Options, config: &aws_config::Config) -> Result Result<()> { +fn use_correct_revision(smithy_rs: &dyn Git, sdk_version: &str) -> Result<()> { let sdk_version = Version::parse(sdk_version).expect("valid version"); if let Some((version, commit_hash)) = PINNED_SMITHY_RS_VERSIONS .iter() @@ -256,13 +255,9 @@ async fn use_correct_revision(sdk_version: &str) -> Result<()> { "SDK version {} requires smithy-rs@{} to successfully compile the canary", version, commit_hash ); - let smithy_rs_root = git::find_git_repository_root("smithy-rs", ".").context(here!())?; // Reset to the revision rather than checkout since the very act of running the // canary-runner can make the working tree dirty by modifying the Cargo.lock file - git::Reset::new(smithy_rs_root, &["--hard", *commit_hash]) - .spawn() - .await - .context(here!())?; + smithy_rs.hard_reset(commit_hash).context(here!())?; } Ok(()) } @@ -417,13 +412,3 @@ async fn delete_lambda_fn(lambda_client: lambda::Client, bundle_name: &str) -> R .context(here!("failed to delete Lambda"))?; Ok(()) } - -async fn git_root() -> Result { - let output = Command::new("git") - .arg("rev-parse") - .arg("--show-toplevel") - .output() - .await - .context("couldn't find repository root")?; - Ok(PathBuf::from(String::from_utf8(output.stdout)?.trim())) -} diff --git a/tools/codegen-diff-revisions.py b/tools/codegen-diff-revisions.py index 0f5a90a1a..03dedfc4f 100755 --- a/tools/codegen-diff-revisions.py +++ b/tools/codegen-diff-revisions.py @@ -92,7 +92,9 @@ def generate_and_commit_generated_code(revision_sha): run("./gradlew codegen:clean codegen-server:clean aws:sdk-codegen:clean") # Generate code - run("./gradlew --rerun-tasks :aws:sdk:assemble") + # TODO(https://github.com/awslabs/smithy-rs/issues/1493): Remove the `aws.services` property once the + # non-smoketest models have been removed from smithy-rs + run("./gradlew --rerun-tasks :aws:sdk:assemble '-Paws.services=+config,+dynamodb,+ec2,+glacier,+iam,+kms,+polly,+qldbsession,+s3,+s3control,+sts,+sso,+transcribestreaming,+route53'") run("./gradlew --rerun-tasks :codegen-server-test:assemble") run("./gradlew --rerun-tasks :codegen-server-test:python:assemble") diff --git a/tools/publisher/Cargo.lock b/tools/publisher/Cargo.lock index b32b7f710..1c1d8bc87 100644 --- a/tools/publisher/Cargo.lock +++ b/tools/publisher/Cargo.lock @@ -1241,6 +1241,7 @@ dependencies = [ "anyhow", "async-trait", "serde", + "serde_json", "tokio", "toml", "tracing", diff --git a/tools/publisher/src/main.rs b/tools/publisher/src/main.rs index 08896fc21..2ed3f846c 100644 --- a/tools/publisher/src/main.rs +++ b/tools/publisher/src/main.rs @@ -18,7 +18,6 @@ use subcommand::publish::PublishArgs; mod cargo; mod fs; mod package; -mod repo; mod retry; mod sort; mod subcommand; diff --git a/tools/publisher/src/repo.rs b/tools/publisher/src/repo.rs deleted file mode 100644 index d888a273b..000000000 --- a/tools/publisher/src/repo.rs +++ /dev/null @@ -1,42 +0,0 @@ -/* - * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. - * SPDX-License-Identifier: Apache-2.0 - */ - -//! Local filesystem git repository discovery. This enables the tool to -//! orient itself despite being run anywhere from within the git repo. - -use crate::{SDK_REPO_CRATE_PATH, SDK_REPO_NAME}; -use anyhow::Result; -use smithy_rs_tool_common::git; -use smithy_rs_tool_common::shell::ShellOperation; -use std::path::{Path, PathBuf}; - -/// Git repository containing crates to be published. -#[derive(Debug)] -pub struct Repository { - pub root: PathBuf, -} - -impl Repository { - pub fn new(repo_name: &str, path: impl Into) -> Result { - let root = git::find_git_repository_root(repo_name, path.into())?; - Ok(Repository { root }) - } - - /// Returns the current tag of this repository - pub async fn current_tag(&self) -> Result { - git::GetCurrentTag::new(&self.root).spawn().await - } -} - -/// Given a `location`, this function looks for the `aws-sdk-rust` git repository. If found, -/// it resolves the `sdk/` directory. Otherwise, it returns the original `location`. -pub fn resolve_publish_location(location: &Path) -> PathBuf { - match Repository::new(SDK_REPO_NAME, location) { - // If the given path was the `aws-sdk-rust` repo root, then resolve the `sdk/` directory to publish from - Ok(sdk_repo) => sdk_repo.root.join(SDK_REPO_CRATE_PATH), - // Otherwise, publish from the given path (likely the smithy-rs runtime bundle) - Err(_) => location.into(), - } -} diff --git a/tools/publisher/src/subcommand/generate_version_manifest.rs b/tools/publisher/src/subcommand/generate_version_manifest.rs index fc6915ba4..54bebadc3 100644 --- a/tools/publisher/src/subcommand/generate_version_manifest.rs +++ b/tools/publisher/src/subcommand/generate_version_manifest.rs @@ -9,9 +9,9 @@ use anyhow::{bail, Context, Result}; use clap::Parser; use semver::Version; use serde::Deserialize; -use smithy_rs_tool_common::git::GetLastCommit; +use smithy_rs_tool_common::git::{find_git_repository_root, Git, GitCLI}; use smithy_rs_tool_common::package::PackageCategory; -use smithy_rs_tool_common::shell::{self, ShellOperation}; +use smithy_rs_tool_common::shell; use smithy_rs_tool_common::versions_manifest::{CrateVersion, Release, VersionsManifest}; use std::collections::BTreeMap; use std::fs::File; @@ -48,9 +48,9 @@ pub async fn subcommand_generate_version_manifest( ) -> Result<()> { verify_crate_hasher_available()?; - let smithy_rs_revision = GetLastCommit::new(std::env::current_dir()?) - .spawn() - .await + let repo_root = find_git_repository_root("smithy-rs", &std::env::current_dir()?)?; + let smithy_rs_revision = GitCLI::new(&repo_root)? + .get_head_revision() .context("get smithy-rs revision")?; info!("Resolved smithy-rs revision to {}", smithy_rs_revision); @@ -94,7 +94,7 @@ pub async fn subcommand_generate_version_manifest( } info!("Discovered and hashed {} crates", crates.len()); let mut versions_manifest = VersionsManifest { - smithy_rs_revision, + smithy_rs_revision: smithy_rs_revision.to_string(), aws_doc_sdk_examples_revision: examples_revision.to_string(), crates, release: None, diff --git a/tools/publisher/src/subcommand/hydrate_readme.rs b/tools/publisher/src/subcommand/hydrate_readme.rs index 29d10d0c7..88c210b74 100644 --- a/tools/publisher/src/subcommand/hydrate_readme.rs +++ b/tools/publisher/src/subcommand/hydrate_readme.rs @@ -3,13 +3,13 @@ * SPDX-License-Identifier: Apache-2.0 */ -use crate::repo::Repository; use crate::SMITHYRS_REPO_NAME; use anyhow::{Context, Result}; use clap::Parser; use handlebars::Handlebars; use semver::Version; use serde_json::json; +use smithy_rs_tool_common::git; use std::fs; use std::path::PathBuf; @@ -34,8 +34,8 @@ pub async fn subcommand_hydrate_readme( }: &HydrateReadmeArgs, ) -> Result<()> { let cwd = std::env::current_dir()?; - let repository = Repository::new(SMITHYRS_REPO_NAME, &cwd)?; - let template_path = repository.root.join("aws/SDK_README.md.hb"); + let repo_root = git::find_git_repository_root(SMITHYRS_REPO_NAME, &cwd)?; + let template_path = repo_root.join("aws/SDK_README.md.hb"); let template_contents = fs::read(&template_path) .with_context(|| format!("Failed to read README template file at {:?}", template_path))?; let template_string = diff --git a/tools/publisher/src/subcommand/publish.rs b/tools/publisher/src/subcommand/publish.rs index ff30bb265..144b1653f 100644 --- a/tools/publisher/src/subcommand/publish.rs +++ b/tools/publisher/src/subcommand/publish.rs @@ -7,17 +7,16 @@ use crate::fs::Fs; use crate::package::{ discover_and_validate_package_batches, Package, PackageBatch, PackageHandle, PackageStats, }; -use crate::repo::{resolve_publish_location, Repository}; use crate::retry::{run_with_retry, BoxError, ErrorClass}; -use crate::CRATE_OWNERS; use crate::{cargo, SDK_REPO_NAME}; +use crate::{CRATE_OWNERS, SDK_REPO_CRATE_PATH}; use anyhow::{bail, Context, Result}; use clap::Parser; use crates_io_api::{AsyncClient, Error}; use dialoguer::Confirm; use lazy_static::lazy_static; +use smithy_rs_tool_common::git; use smithy_rs_tool_common::shell::ShellOperation; -use smithy_rs_tool_common::versions_manifest::VersionsManifest; use std::path::{Path, PathBuf}; use std::sync::Arc; use std::time::Duration; @@ -58,9 +57,6 @@ pub async fn subcommand_publish( let (batches, stats) = discover_and_validate_package_batches(Fs::Real, &location).await?; info!("Finished crate discovery."); - // Sanity check the repository tag if publishing from `aws-sdk-rust` - confirm_correct_tag(&location).await?; - // Don't proceed unless the user confirms the plan confirm_plan(&batches, stats, *skip_confirmation)?; @@ -103,6 +99,17 @@ pub async fn subcommand_publish( Ok(()) } +/// Given a `location`, this function looks for the `aws-sdk-rust` git repository. If found, +/// it resolves the `sdk/` directory. Otherwise, it returns the original `location`. +pub fn resolve_publish_location(location: &Path) -> PathBuf { + match git::find_git_repository_root(SDK_REPO_NAME, location) { + // If the given path was the `aws-sdk-rust` repo root, then resolve the `sdk/` directory to publish from + Ok(sdk_repo) => sdk_repo.join(SDK_REPO_CRATE_PATH), + // Otherwise, publish from the given path (likely the smithy-rs runtime bundle) + Err(_) => location.into(), + } +} + async fn publish(handle: &PackageHandle, crate_path: &Path) -> Result<()> { info!("Publishing `{}`...", handle); run_with_retry( @@ -121,27 +128,6 @@ async fn publish(handle: &PackageHandle, crate_path: &Path) -> Result<()> { Ok(()) } -async fn confirm_correct_tag(location: &Path) -> Result<()> { - let repository = Repository::new(SDK_REPO_NAME, location)?; - let versions_manifest = VersionsManifest::from_file(location.join("../versions.toml"))?; - if versions_manifest.release.is_none() { - // The release metadata is required for yanking in the event of a bad release, so don't - // allow publish if it's missing. - bail!("Generated `versions.toml` doesn't have release metadata. Refusing to publish!"); - } - let expected_tag = versions_manifest.release.unwrap().tag; - let current_tag = repository.current_tag().await?; - if expected_tag != current_tag { - bail!( - "Current tag `{}` in the local `aws-sdk-rust` repository didn't match expected \ - release tag `{}` from the `versions.toml` file", - current_tag, - expected_tag - ); - } - Ok(()) -} - async fn is_published(handle: &PackageHandle) -> Result { run_with_retry( &format!("Checking if `{}` is already published", handle.name), diff --git a/tools/sdk-lints/Cargo.lock b/tools/sdk-lints/Cargo.lock index 9e16f1691..553a33f2d 100644 --- a/tools/sdk-lints/Cargo.lock +++ b/tools/sdk-lints/Cargo.lock @@ -2,21 +2,23 @@ # It is not intended for manual editing. version = 3 -[[package]] -name = "ansi_term" -version = "0.12.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d52a9bb7ec0cf484c551830a7ce27bd20d67eac647e1befb56b0be4ee39a55d2" -dependencies = [ - "winapi", -] - [[package]] name = "anyhow" version = "1.0.57" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "08f9b8508dccb7687a1d6c4ce66b2b0ecef467c94667de27d8d7fe1f8d2a9cdc" +[[package]] +name = "async-trait" +version = "0.1.56" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96cf8829f67d2eab0b2dfa42c5d0ef737e0724e4a82b01b3e292456202b19716" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "atty" version = "0.2.14" @@ -51,6 +53,12 @@ dependencies = [ "toml", ] +[[package]] +name = "cfg-if" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" + [[package]] name = "clap" version = "3.1.18" @@ -90,22 +98,6 @@ dependencies = [ "os_str_bytes", ] -[[package]] -name = "ctor" -version = "0.1.22" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f877be4f7c9f246b183111634f75baa039715e3f46ce860677d3b19a69fb229c" -dependencies = [ - "quote", - "syn", -] - -[[package]] -name = "diff" -version = "0.1.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0e25ea47919b1560c4e3b7fe0aaab9becf5b84a10325ddf7db0f0ba5e1026499" - [[package]] name = "hashbrown" version = "0.11.2" @@ -156,41 +148,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "349d5a591cd28b49e1d1037471617a32ddcda5731b99419008085f72d5a53836" [[package]] -name = "num-integer" -version = "0.1.45" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "225d3389fb3509a24c93f5c29eb6bde2586b98d9f016636dff58d7c6f7569cd9" -dependencies = [ - "autocfg", - "num-traits", -] - -[[package]] -name = "num-traits" -version = "0.2.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "578ede34cf02f8924ab9447f50c28075b4d3e5b269972345e7e0372b38c6cdcd" -dependencies = [ - "autocfg", -] - -[[package]] -name = "num_threads" -version = "0.1.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2819ce041d2ee131036f4fc9d6ae7ae125a3a40e97ba64d04fe799ad9dabbb44" -dependencies = [ - "libc", -] - -[[package]] -name = "ordinal" -version = "0.3.2" +name = "once_cell" +version = "1.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c80c1530f46e9d8985706d7deb80b83172b250538902f607dea6cd6028851083" -dependencies = [ - "num-integer", -] +checksum = "7709cef83f0c1f58f666e746a08b21e0085f7440fa6a29cc194d68aac97a4225" [[package]] name = "os_str_bytes" @@ -199,25 +160,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "21326818e99cfe6ce1e524c2a805c189a99b5ae555a35d19f9a284b427d86afa" [[package]] -name = "output_vt100" -version = "0.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "628223faebab4e3e40667ee0b2336d34a5b960ff60ea743ddfdbcf7770bcfb66" -dependencies = [ - "winapi", -] - -[[package]] -name = "pretty_assertions" -version = "1.2.1" +name = "pin-project-lite" +version = "0.2.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c89f989ac94207d048d92db058e4f6ec7342b0971fc58d1271ca148b799b3563" -dependencies = [ - "ansi_term", - "ctor", - "diff", - "output_vt100", -] +checksum = "e0a7ae3ac2f1173085d398531c705756c94a4c56843785df85a60c1a0afac116" [[package]] name = "proc-macro-error" @@ -275,11 +221,8 @@ dependencies = [ "cargo_toml", "clap", "lazy_static", - "ordinal", - "pretty_assertions", "serde", - "serde_json", - "time", + "smithy-rs-tool-common", "toml", ] @@ -314,6 +257,18 @@ dependencies = [ "serde", ] +[[package]] +name = "smithy-rs-tool-common" +version = "0.1.0" +dependencies = [ + "anyhow", + "async-trait", + "serde", + "serde_json", + "toml", + "tracing", +] + [[package]] name = "strsim" version = "0.10.0" @@ -347,22 +302,45 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b1141d4d61095b28419e22cb0bbf02755f5e54e0526f97f1e3d1d160e60885fb" [[package]] -name = "time" -version = "0.3.9" +name = "toml" +version = "0.5.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c2702e08a7a860f005826c6815dcac101b19b5eb330c27fe4a5928fec1d20ddd" +checksum = "8d82e1a7758622a465f8cee077614c73484dac5b836c02ff6a40d5d1010324d7" dependencies = [ - "libc", - "num_threads", + "indexmap", + "serde", ] [[package]] -name = "toml" -version = "0.5.9" +name = "tracing" +version = "0.1.35" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8d82e1a7758622a465f8cee077614c73484dac5b836c02ff6a40d5d1010324d7" +checksum = "a400e31aa60b9d44a52a8ee0343b5b18566b03a8321e0d321f695cf56e940160" dependencies = [ - "serde", + "cfg-if", + "pin-project-lite", + "tracing-attributes", + "tracing-core", +] + +[[package]] +name = "tracing-attributes" +version = "0.1.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc6b8ad3567499f98a1db7a752b07a7c8c7c7c34c332ec00effb2b0027974b7c" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tracing-core" +version = "0.1.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7709595b8878a4965ce5e87ebf880a7d39c9afc6837721b21a5a816a8117d921" +dependencies = [ + "once_cell", ] [[package]] diff --git a/tools/sdk-lints/Cargo.toml b/tools/sdk-lints/Cargo.toml index 496a762ce..394c12e68 100644 --- a/tools/sdk-lints/Cargo.toml +++ b/tools/sdk-lints/Cargo.toml @@ -16,10 +16,5 @@ cargo_toml = "0.10.1" clap = { version = "~3.1.18", features = ["derive"]} toml = "0.5.8" serde = { version = "1", features = ["derive"]} -serde_json = "1" lazy_static = "1.4.0" -time = { version = "0.3.9", features = ["local-offset"]} -ordinal = "0.3.2" - -[dev-dependencies] -pretty_assertions = "1.2.1" +smithy-rs-tool-common = { path = "../smithy-rs-tool-common" } diff --git a/tools/sdk-lints/src/changelog.rs b/tools/sdk-lints/src/changelog.rs index 67e996c2e..f88e8bcb4 100644 --- a/tools/sdk-lints/src/changelog.rs +++ b/tools/sdk-lints/src/changelog.rs @@ -5,468 +5,12 @@ use crate::lint::LintError; use crate::{repo_root, Check, Lint}; -use anyhow::{bail, Context, Result}; -use serde::{de, Deserialize, Deserializer, Serialize}; -use std::fmt::Write; +use anyhow::Result; +use smithy_rs_tool_common::changelog::Changelog; use std::path::{Path, PathBuf}; -use std::process::Command; -use std::str::FromStr; - -const EXAMPLE_ENTRY: &str = r#" -# Example changelog entries -# [[aws-sdk-rust]] -# message = "Fix typos in module documentation for generated crates" -# references = ["smithy-rs#920"] -# meta = { "breaking" = false, "tada" = false, "bug" = false } -# author = "rcoh" -# -# [[smithy-rs]] -# message = "Fix typos in module documentation for generated crates" -# references = ["smithy-rs#920"] -# meta = { "breaking" = false, "tada" = false, "bug" = false } -# author = "rcoh" -"#; - -const USE_UPDATE_CHANGELOGS: &str = - ""; - -fn maintainers() -> Vec<&'static str> { - include_str!("../smithy-rs-maintainers.txt") - .lines() - .collect() -} - -enum ChangelogEntry { - HandAuthored(HandAuthoredEntry), - AwsSdkModel(SdkModelEntry), -} - -impl ChangelogEntry { - fn hand_authored(&self) -> Option<&HandAuthoredEntry> { - match self { - ChangelogEntry::HandAuthored(hand_authored) => Some(hand_authored), - _ => None, - } - } - - fn aws_sdk_model(&self) -> Option<&SdkModelEntry> { - match self { - ChangelogEntry::AwsSdkModel(sdk_model) => Some(sdk_model), - _ => None, - } - } -} - -#[derive(Deserialize)] -struct HandAuthoredEntry { - message: String, - meta: Meta, - author: String, - #[serde(default)] - references: Vec, -} - -impl HandAuthoredEntry { - /// Validate a changelog entry to ensure it follows standards - fn validate(&self) -> Result<()> { - if self.author.is_empty() { - bail!("Author must be set (was empty)"); - } - if !self.author.chars().all(|c| c.is_alphanumeric() || c == '-') { - bail!("Author must be valid GitHub username: [a-zA-Z0-9\\-]") - } - if self.references.is_empty() { - bail!("Changelog entry must refer to at least one pull request or issue"); - } - - Ok(()) - } - - /// Write a changelog entry to [out] - /// - /// Example output: - /// `- Add a feature (smithy-rs#123, @contributor)` - fn render(&self, mut out: &mut String) { - let mut meta = String::new(); - if self.meta.bug { - meta.push('🐛'); - } - if self.meta.breaking { - meta.push('⚠'); - } - if self.meta.tada { - meta.push('🎉'); - } - if !meta.is_empty() { - meta.push(' '); - } - let mut references = self - .references - .iter() - .map(Reference::to_md_link) - .collect::>(); - if !maintainers().contains(&self.author.to_ascii_lowercase().as_str()) { - references.push(format!("@{}", self.author.to_ascii_lowercase())); - }; - if !references.is_empty() { - write!(meta, "({}) ", references.join(", ")).unwrap(); - } - write!( - &mut out, - "- {meta}{message}", - meta = meta, - message = indented_message(&self.message), - ) - .unwrap(); - } -} - -#[derive(Deserialize)] -enum SdkModelChangeKind { - Documentation, - Feature, -} - -#[derive(Deserialize)] -struct SdkModelEntry { - /// SDK module name (e.g., "aws-sdk-s3" for S3) - module: String, - /// SDK module version number (e.g., "0.14.0") - version: String, - /// What changed - kind: SdkModelChangeKind, - /// More details about the change - message: String, -} - -impl SdkModelEntry { - fn render(&self, out: &mut String) { - write!( - out, - "- `{module}` ({version}): {message}", - module = self.module, - version = self.version, - message = self.message - ) - .unwrap(); - } -} - -struct Reference { - repo: String, - number: usize, -} - -impl Reference { - fn to_md_link(&self) -> String { - format!( - "[{repo}#{number}](https://github.com/awslabs/{repo}/issues/{number})", - repo = self.repo, - number = self.number - ) - } -} - -impl<'de> Deserialize<'de> for Reference { - fn deserialize(deserializer: D) -> Result - where - D: Deserializer<'de>, - { - let s = String::deserialize(deserializer)?; - FromStr::from_str(&s).map_err(de::Error::custom) - } -} - -impl FromStr for Reference { - type Err = anyhow::Error; - - fn from_str(reference: &str) -> std::result::Result { - match reference.split_once('#') { - None => bail!( - "Reference must of the form `repo#number` but found {}", - reference - ), - Some((repo, number)) => { - let number = number.parse::()?; - if !matches!(repo, "smithy-rs" | "aws-sdk-rust") { - bail!("unexpected repo: {}", repo); - } - Ok(Reference { - number, - repo: repo.to_string(), - }) - } - } - } -} - -fn indented_message(message: &str) -> String { - let mut out = String::new(); - for (idx, line) in message.lines().enumerate() { - if idx > 0 { - out.push('\n'); - if !line.is_empty() { - out.push_str(" "); - } - } - out.push_str(line); - } - out -} - -#[derive(Deserialize)] -struct Meta { - bug: bool, - breaking: bool, - tada: bool, -} - -#[derive(Deserialize)] -pub(crate) struct Changelog { - #[serde(rename = "smithy-rs")] - #[serde(default)] - smithy_rs: Vec, - #[serde(rename = "aws-sdk-rust")] - #[serde(default)] - aws_sdk_rust: Vec, - #[serde(rename = "aws-sdk-model")] - #[serde(default)] - sdk_models: Vec, -} - -impl Changelog { - fn into_entries(mut self) -> ChangelogEntries { - self.aws_sdk_rust.sort_by_key(|entry| !entry.meta.tada); - self.sdk_models.sort_by(|a, b| a.module.cmp(&b.module)); - self.smithy_rs.sort_by_key(|entry| !entry.meta.tada); - - ChangelogEntries { - smithy_rs: self - .smithy_rs - .into_iter() - .map(ChangelogEntry::HandAuthored) - .collect(), - aws_sdk_rust: self - .aws_sdk_rust - .into_iter() - .map(ChangelogEntry::HandAuthored) - .chain(self.sdk_models.into_iter().map(ChangelogEntry::AwsSdkModel)) - .collect(), - } - } -} - -struct ChangelogEntries { - smithy_rs: Vec, - aws_sdk_rust: Vec, -} - -/// Ensure that there are no uncommited changes to the changelog -fn no_uncommited_changes(path: &Path) -> Result<()> { - let unstaged = !Command::new("git") - .arg("diff") - .arg("--exit-code") - .arg(path) - .status()? - .success(); - let staged = !Command::new("git") - .arg("diff") - .arg("--exit-code") - .arg("--staged") - .arg(path) - .status()? - .success(); - if unstaged || staged { - bail!("Uncommitted changes to {}", path.display()) - } - Ok(()) -} - -pub struct ReleaseMetadata { - pub title: String, - pub tag: String, - pub manifest_name: String, -} - -#[derive(Serialize)] -struct ReleaseManifest { - #[serde(rename = "tagName")] - tag_name: String, - name: String, - body: String, - prerelease: bool, -} - -pub(crate) fn update_changelogs( - changelog_next: impl AsRef, - smithy_rs_path: impl AsRef, - aws_sdk_rust_path: impl AsRef, - smithy_rs_metadata: &ReleaseMetadata, - aws_sdk_rust_metadata: &ReleaseMetadata, - release_manifest_output_path: Option<&Path>, -) -> Result<()> { - no_uncommited_changes(changelog_next.as_ref()).context( - "CHANGELOG.next.toml had unstaged changes. Refusing to perform changelog update.", - )?; - let changelog = check_changelog_next(changelog_next.as_ref()).map_err(|errs| { - anyhow::Error::msg(format!( - "cannot update changelogs with changelog errors: {:#?}", - errs - )) - })?; - let ChangelogEntries { - smithy_rs, - aws_sdk_rust, - } = changelog.into_entries(); - for (entries, path, release_metadata) in [ - (smithy_rs, smithy_rs_path.as_ref(), smithy_rs_metadata), - ( - aws_sdk_rust, - aws_sdk_rust_path.as_ref(), - aws_sdk_rust_metadata, - ), - ] { - no_uncommited_changes(path) - .with_context(|| format!("{} had unstaged changes", path.display()))?; - let (release_header, release_notes) = render(&entries, &release_metadata.title); - if let Some(output_path) = release_manifest_output_path { - let release_manifest = ReleaseManifest { - tag_name: release_metadata.tag.clone(), - name: release_metadata.title.clone(), - body: release_notes.clone(), - // All releases are pre-releases for now - prerelease: true, - }; - std::fs::write( - output_path.join(&release_metadata.manifest_name), - serde_json::to_string_pretty(&release_manifest)?, - )?; - } - - let mut update = USE_UPDATE_CHANGELOGS.to_string(); - update.push('\n'); - update.push_str(&release_header); - update.push_str(&release_notes); - let current = std::fs::read_to_string(path)?.replace(USE_UPDATE_CHANGELOGS, ""); - update.push_str(¤t); - std::fs::write(path, update)?; - } - std::fs::write(changelog_next.as_ref(), EXAMPLE_ENTRY.trim())?; - eprintln!("Changelogs updated!"); - Ok(()) -} - -fn render_handauthored<'a>(entries: impl Iterator, out: &mut String) { - let (breaking, non_breaking) = entries.partition::, _>(|entry| entry.meta.breaking); - - if !breaking.is_empty() { - out.push_str("**Breaking Changes:**\n"); - for change in breaking { - change.render(out); - out.push('\n'); - } - out.push('\n') - } - - if !non_breaking.is_empty() { - out.push_str("**New this release:**\n"); - for change in non_breaking { - change.render(out); - out.push('\n'); - } - out.push('\n'); - } -} - -fn render_sdk_model_entries<'a>( - entries: impl Iterator, - out: &mut String, -) { - let (features, docs) = - entries.partition::, _>(|entry| matches!(entry.kind, SdkModelChangeKind::Feature)); - if !features.is_empty() { - out.push_str("**Service Features:**\n"); - for entry in features { - entry.render(out); - out.push('\n'); - } - out.push('\n'); - } - if !docs.is_empty() { - out.push_str("**Service Documentation:**\n"); - for entry in docs { - entry.render(out); - out.push('\n'); - } - out.push('\n'); - } -} - -/// Convert a list of changelog entries into markdown. -/// Returns (header, body) -fn render(entries: &[ChangelogEntry], release_header: &str) -> (String, String) { - let mut header = String::new(); - header.push_str(release_header); - header.push('\n'); - for _ in 0..release_header.len() { - header.push('='); - } - header.push('\n'); - - let mut out = String::new(); - render_handauthored( - entries.iter().filter_map(ChangelogEntry::hand_authored), - &mut out, - ); - render_sdk_model_entries( - entries.iter().filter_map(ChangelogEntry::aws_sdk_model), - &mut out, - ); - - let mut external_contribs = entries - .iter() - .filter_map(|entry| entry.hand_authored().map(|e| e.author.to_ascii_lowercase())) - .filter(|author| !maintainers().contains(&author.as_str())) - .collect::>(); - external_contribs.sort(); - external_contribs.dedup(); - if !external_contribs.is_empty() { - out.push_str("**Contributors**\nThank you for your contributions! ❤\n"); - for contributor_handle in external_contribs { - // retrieve all contributions this author made - let mut contribution_references = entries - .iter() - .filter(|entry| { - entry - .hand_authored() - .map(|e| e.author.eq_ignore_ascii_case(contributor_handle.as_str())) - .unwrap_or(false) - }) - .flat_map(|entry| { - entry - .hand_authored() - .unwrap() - .references - .iter() - .map(|it| it.to_md_link()) - }) - .collect::>(); - contribution_references.sort(); - contribution_references.dedup(); - let contribution_references = contribution_references.as_slice().join(", "); - out.push_str("- @"); - out.push_str(&contributor_handle); - if !contribution_references.is_empty() { - out.push_str(&format!(" ({})", contribution_references)); - } - out.push('\n'); - } - } - - (header, out) -} pub(crate) struct ChangelogNext; + impl Lint for ChangelogNext { fn name(&self) -> &str { "Changelog.next" @@ -487,148 +31,12 @@ impl Check for ChangelogNext { } /// Validate that `CHANGELOG.next.toml` follows best practices -fn check_changelog_next(path: impl AsRef) -> std::result::Result> { - let contents = std::fs::read_to_string(path) - .context("failed to read CHANGELOG.next") - .map_err(|e| vec![LintError::via_display(e)])?; - let parsed: Changelog = toml::from_str(&contents) - .context("Invalid changelog format") - .map_err(|e| vec![LintError::via_display(e)])?; - let mut errors = vec![]; - for entry in parsed.aws_sdk_rust.iter().chain(parsed.smithy_rs.iter()) { - if let Err(e) = entry.validate() { - errors.push(LintError::via_display(e)) - } - } - if errors.is_empty() { - Ok(parsed) - } else { - Err(errors) - } -} - -#[cfg(test)] -mod test { - use super::ChangelogEntry; - use crate::changelog::{render, Changelog, ChangelogEntries}; - - fn render_full(entries: &[ChangelogEntry], release_header: &str) -> String { - let (header, body) = render(entries, release_header); - return format!("{}{}", header, body); - } - - #[test] - fn end_to_end_changelog() { - let changelog_toml = r#" -[[smithy-rs]] -author = "rcoh" -message = "I made a major change to update the code generator" -meta = { breaking = true, tada = false, bug = false } -references = ["smithy-rs#445"] - -[[smithy-rs]] -author = "external-contrib" -message = "I made a change to update the code generator" -meta = { breaking = false, tada = true, bug = false } -references = ["smithy-rs#446"] - -[[smithy-rs]] -author = "another-contrib" -message = "I made a minor change" -meta = { breaking = false, tada = false, bug = false } - -[[aws-sdk-rust]] -author = "rcoh" -message = "I made a major change to update the AWS SDK" -meta = { breaking = true, tada = false, bug = false } -references = ["smithy-rs#445"] - -[[aws-sdk-rust]] -author = "external-contrib" -message = "I made a change to update the code generator" -meta = { breaking = false, tada = true, bug = false } -references = ["smithy-rs#446"] - -[[smithy-rs]] -author = "external-contrib" -message = """ -I made a change to update the code generator - -**Update guide:** -blah blah -""" -meta = { breaking = false, tada = true, bug = false } -references = ["smithy-rs#446"] - -[[aws-sdk-model]] -module = "aws-sdk-s3" -version = "0.14.0" -kind = "Feature" -message = "Some new API to do X" - -[[aws-sdk-model]] -module = "aws-sdk-ec2" -version = "0.12.0" -kind = "Documentation" -message = "Updated some docs" - -[[aws-sdk-model]] -module = "aws-sdk-ec2" -version = "0.12.0" -kind = "Feature" -message = "Some API change" - "#; - let changelog: Changelog = toml::from_str(changelog_toml).expect("valid changelog"); - let ChangelogEntries { - aws_sdk_rust, - smithy_rs, - } = changelog.into_entries(); - - let smithy_rs_rendered = render_full(&smithy_rs, "v0.3.0 (January 4th, 2022)"); - let smithy_rs_expected = r#" -v0.3.0 (January 4th, 2022) -========================== -**Breaking Changes:** -- ⚠ ([smithy-rs#445](https://github.com/awslabs/smithy-rs/issues/445)) I made a major change to update the code generator - -**New this release:** -- 🎉 ([smithy-rs#446](https://github.com/awslabs/smithy-rs/issues/446), @external-contrib) I made a change to update the code generator -- 🎉 ([smithy-rs#446](https://github.com/awslabs/smithy-rs/issues/446), @external-contrib) I made a change to update the code generator - - **Update guide:** - blah blah -- (@another-contrib) I made a minor change - -**Contributors** -Thank you for your contributions! ❤ -- @another-contrib -- @external-contrib ([smithy-rs#446](https://github.com/awslabs/smithy-rs/issues/446)) -"# - .trim_start(); - pretty_assertions::assert_str_eq!(smithy_rs_expected, smithy_rs_rendered); - - let aws_sdk_rust_rendered = render_full(&aws_sdk_rust, "v0.1.0 (January 4th, 2022)"); - let aws_sdk_expected = r#" -v0.1.0 (January 4th, 2022) -========================== -**Breaking Changes:** -- ⚠ ([smithy-rs#445](https://github.com/awslabs/smithy-rs/issues/445)) I made a major change to update the AWS SDK - -**New this release:** -- 🎉 ([smithy-rs#446](https://github.com/awslabs/smithy-rs/issues/446), @external-contrib) I made a change to update the code generator - -**Service Features:** -- `aws-sdk-ec2` (0.12.0): Some API change -- `aws-sdk-s3` (0.14.0): Some new API to do X - -**Service Documentation:** -- `aws-sdk-ec2` (0.12.0): Updated some docs - -**Contributors** -Thank you for your contributions! ❤ -- @external-contrib ([smithy-rs#446](https://github.com/awslabs/smithy-rs/issues/446)) -"# - .trim_start(); - pretty_assertions::assert_str_eq!(aws_sdk_expected, aws_sdk_rust_rendered); - } +fn check_changelog_next(path: impl AsRef) -> std::result::Result<(), Vec> { + let parsed = Changelog::load_from_file(path).map_err(|e| vec![LintError::via_display(e)])?; + parsed.validate().map_err(|errs| { + errs.into_iter() + .map(LintError::via_display) + .collect::>() + })?; + Ok(()) } diff --git a/tools/sdk-lints/src/main.rs b/tools/sdk-lints/src/main.rs index b7481f5e7..40fc730b8 100644 --- a/tools/sdk-lints/src/main.rs +++ b/tools/sdk-lints/src/main.rs @@ -10,15 +10,12 @@ use crate::lint_cargo_toml::{CrateAuthor, CrateLicense, DocsRs}; use crate::readmes::{ReadmesExist, ReadmesHaveFooters}; use crate::todos::TodosHaveContext; use anyhow::{bail, Context, Result}; -use changelog::ReleaseMetadata; use clap::Parser; use lazy_static::lazy_static; -use ordinal::Ordinal; use std::env::set_current_dir; use std::path::{Path, PathBuf}; use std::process::Command; use std::{fs, io}; -use time::OffsetDateTime; mod anchor; mod changelog; @@ -65,14 +62,6 @@ enum Args { #[clap(long)] dry_run: Option, }, - UpdateChangelog { - /// Whether or not independent crate versions are being used (defaults to false) - #[clap(long)] - independent_versioning: bool, - /// Optional path to output a release manifest file to - #[clap(long)] - release_manifest_output_path: Option, - }, } fn load_vcs_files() -> Result> { @@ -168,119 +157,10 @@ fn main() -> Result<()> { ok(DocsRs.fix_all(dry_run)?)?; } } - Args::UpdateChangelog { - independent_versioning, - release_manifest_output_path, - } => { - let now = OffsetDateTime::now_local()?; - let changelog_next_path = repo_root().join("CHANGELOG.next.toml"); - let changelog_path = repo_root().join("CHANGELOG.md"); - let aws_changelog_path = repo_root().join("aws/SDK_CHANGELOG.md"); - if independent_versioning { - let smithy_rs_metadata = - date_based_release_metadata(now, "smithy-rs-release-manifest.json"); - let sdk_metadata = - date_based_release_metadata(now, "aws-sdk-rust-release-manifest.json"); - changelog::update_changelogs( - changelog_next_path, - changelog_path, - aws_changelog_path, - &smithy_rs_metadata, - &sdk_metadata, - release_manifest_output_path.as_deref(), - )? - } else { - let auto = auto_changelog_meta()?; - let smithy_rs_metadata = version_based_release_metadata( - now, - &auto.smithy_version, - "smithy-rs-release-manifest.json", - ); - let sdk_metadata = version_based_release_metadata( - now, - &auto.sdk_version, - "aws-sdk-rust-release-manifest.json", - ); - changelog::update_changelogs( - changelog_next_path, - changelog_path, - aws_changelog_path, - &smithy_rs_metadata, - &sdk_metadata, - release_manifest_output_path.as_deref(), - )? - } - } } Ok(()) } -struct ChangelogMeta { - smithy_version: String, - sdk_version: String, -} - -fn date_based_release_metadata( - now: OffsetDateTime, - manifest_name: impl Into, -) -> ReleaseMetadata { - ReleaseMetadata { - title: date_title(&now), - tag: format!( - "release-{year}-{month:02}-{day:02}", - year = now.date().year(), - month = u8::from(now.date().month()), - day = now.date().day() - ), - manifest_name: manifest_name.into(), - } -} - -fn version_based_release_metadata( - now: OffsetDateTime, - version: &str, - manifest_name: impl Into, -) -> ReleaseMetadata { - ReleaseMetadata { - title: format!( - "v{version} ({date})", - version = version, - date = date_title(&now) - ), - tag: format!("v{version}", version = version), - manifest_name: manifest_name.into(), - } -} - -fn date_title(now: &OffsetDateTime) -> String { - format!( - "{month} {day}, {year}", - month = now.date().month(), - day = Ordinal(now.date().day()), - year = now.date().year() - ) -} - -/// Discover the new version for the changelog from gradle.properties and the date. -fn auto_changelog_meta() -> Result { - let gradle_props = fs::read_to_string(repo_root().join("gradle.properties"))?; - let load_gradle_prop = |key: &str| { - let prop = gradle_props - .lines() - .flat_map(|line| line.strip_prefix(key)) - .flat_map(|prop| prop.strip_prefix('=')) - .next(); - prop.map(|prop| prop.to_string()) - .ok_or_else(|| anyhow::Error::msg(format!("missing expected gradle property: {key}"))) - }; - let smithy_version = load_gradle_prop("smithy.rs.runtime.crate.version")?; - let sdk_version = load_gradle_prop("aws.sdk.version")?; - Ok(ChangelogMeta { - smithy_version, - sdk_version, - }) -} - fn ls(path: impl AsRef) -> Result> { Ok(fs::read_dir(path.as_ref()) .with_context(|| format!("failed to ls: {:?}", path.as_ref()))? @@ -310,27 +190,3 @@ fn all_runtime_crates() -> Result> { fn all_cargo_tomls() -> Result> { Ok(all_runtime_crates()?.map(|pkg| pkg.join("Cargo.toml"))) } - -#[cfg(test)] -mod tests { - use crate::{date_based_release_metadata, version_based_release_metadata}; - use time::OffsetDateTime; - - #[test] - fn test_date_based_release_metadata() { - let now = OffsetDateTime::from_unix_timestamp(100_000_000).unwrap(); - let result = date_based_release_metadata(now, "some-manifest.json"); - assert_eq!("March 3rd, 1973", result.title); - assert_eq!("release-1973-03-03", result.tag); - assert_eq!("some-manifest.json", result.manifest_name); - } - - #[test] - fn test_version_based_release_metadata() { - let now = OffsetDateTime::from_unix_timestamp(100_000_000).unwrap(); - let result = version_based_release_metadata(now, "0.11.0", "some-other-manifest.json"); - assert_eq!("v0.11.0 (March 3rd, 1973)", result.title); - assert_eq!("v0.11.0", result.tag); - assert_eq!("some-other-manifest.json", result.manifest_name); - } -} diff --git a/tools/sdk-sync/Cargo.lock b/tools/sdk-sync/Cargo.lock index c2d813cf9..d9afca38e 100644 --- a/tools/sdk-sync/Cargo.lock +++ b/tools/sdk-sync/Cargo.lock @@ -290,6 +290,12 @@ dependencies = [ "either", ] +[[package]] +name = "itoa" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "112c678d4050afce233f4f2852bb2eb519230b3cf12f33585275537d7e41578d" + [[package]] name = "lazy_static" version = "1.4.0" @@ -592,6 +598,12 @@ dependencies = [ "winapi", ] +[[package]] +name = "ryu" +version = "1.0.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3f6f92acf49d1b98f7a81226834412ada05458b7364277387724a237f062695" + [[package]] name = "scopeguard" version = "1.1.0" @@ -641,6 +653,17 @@ dependencies = [ "syn", ] +[[package]] +name = "serde_json" +version = "1.0.81" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b7ce2b32a1aed03c558dc61a5cd328f15aff2dbc17daad8fb8af04d2100e15c" +dependencies = [ + "itoa", + "ryu", + "serde", +] + [[package]] name = "sharded-slab" version = "0.1.4" @@ -663,6 +686,7 @@ dependencies = [ "anyhow", "async-trait", "serde", + "serde_json", "toml", "tracing", ] diff --git a/tools/sdk-sync/fake-cli/git-squash-merge b/tools/sdk-sync/fake-cli/git-squash-merge deleted file mode 100755 index 7be2a54b6..000000000 --- a/tools/sdk-sync/fake-cli/git-squash-merge +++ /dev/null @@ -1,19 +0,0 @@ -#!/usr/bin/env python3 -import os -import sys - -if sys.argv[1] == "merge": - expected = [os.path.realpath("/tmp"), ["merge", "--squash", "test-branch-name"]] - actual = [os.getcwd(), sys.argv[1:]] - if expected != actual: - print(f"ERROR\nExpect: {expected}\nActual: {actual}") - sys.exit(1) -else: - expected = [ - os.path.realpath("/tmp"), - ["-c", "user.name=test-author", "-c", "user.email=test-author-email", "commit", "-m", "test message"] - ] - actual = [os.getcwd(), sys.argv[1:]] - if expected != actual: - print(f"ERROR\nExpect: {expected}\nActual: {actual}") - sys.exit(1) diff --git a/tools/sdk-sync/src/git.rs b/tools/sdk-sync/src/git.rs deleted file mode 100644 index d114ea5ca..000000000 --- a/tools/sdk-sync/src/git.rs +++ /dev/null @@ -1,613 +0,0 @@ -/* - * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. - * SPDX-License-Identifier: Apache-2.0 - */ - -use anyhow::{bail, Result}; -use smithy_rs_tool_common::shell::{handle_failure, output_text}; -use std::borrow::Cow; -use std::fmt; -use std::path::{Path, PathBuf}; -use std::process::Command; -use tracing::debug; - -#[derive(Clone, Debug, Eq, PartialEq)] -pub struct CommitHash(String); - -impl> From for CommitHash { - fn from(hash: T) -> Self { - CommitHash(hash.into()) - } -} - -impl AsRef for CommitHash { - fn as_ref(&self) -> &str { - &self.0 - } -} - -impl fmt::Display for CommitHash { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "{}", self.0) - } -} - -#[derive(Clone, Debug, Eq, PartialEq)] -pub struct Commit { - pub hash: CommitHash, - pub author_name: String, - pub author_email: String, - pub message_subject: String, - pub message_body: String, -} - -impl Commit { - pub fn message(&self) -> Cow<'_, str> { - if self.message_body.is_empty() { - Cow::Borrowed(&self.message_subject) - } else { - Cow::Owned(format!("{}\n\n{}", self.message_subject, self.message_body)) - } - } -} - -/// Easily mockable interface with Git for testing -#[cfg_attr(test, mockall::automock)] -pub trait Git: Send + Sync { - /// Returns the repository path - fn path(&self) -> &Path; - - /// Clones the repository to the given path - fn clone_to(&self, path: &Path) -> Result<()>; - - /// Returns commit hash of HEAD (i.e., `git rev-parse HEAD`) - fn get_head_revision(&self) -> Result; - - /// Stages the given path (i.e., `git add ${path}`) - fn stage(&self, path: &Path) -> Result<()>; - - /// Commits the staged files on behalf of the given author using a bot commiter. - fn commit_on_behalf( - &self, - bot_name: &str, - bot_email: &str, - author_name: &str, - author_email: &str, - message: &str, - ) -> Result<()>; - - /// Commits staged files. - fn commit(&self, name: &str, email: &str, message: &str) -> Result<()>; - - /// Returns a list of commit hashes in reverse chronological order starting with - /// `start_inclusive_revision` and ending before `end_exclusive_revision`. Both of - /// these arguments can be any kind of Git revision (e.g., HEAD, HEAD~2, commit hash, etc). - fn rev_list<'a>( - &self, - start_inclusive_revision: &str, - end_exclusive_revision: &str, - path: Option<&'a Path>, - ) -> Result>; - - /// Returns information about a given revision. - fn show(&self, revision: &str) -> Result; - - /// Hard resets to the given revision. - fn hard_reset(&self, revision: &str) -> Result<()>; - - /// Returns the name of the current branch. - fn current_branch_name(&self) -> Result; - - /// Creates a branch at the given revision. - fn create_branch(&self, branch_name: &str, revision: &str) -> Result<()>; - - /// Deletes a branch. - fn delete_branch(&self, branch_name: &str) -> Result<()>; - - /// Squash merges a branch into the current branch. - fn squash_merge( - &self, - author_name: &str, - author_email: &str, - branch_name: &str, - commit_message: &str, - ) -> Result<()>; - - /// Returns list of untracked files. - fn untracked_files(&self) -> Result>; - - /// Returns list of changed files. - fn changed_files(&self) -> Result>; -} - -enum CommitInfo { - CommitHash, - AuthorName, - AuthorEmail, - MessageSubject, - MessageBody, -} - -pub struct GitCLI { - repo_path: PathBuf, - binary_name: String, -} - -impl GitCLI { - pub fn new(repo_path: &Path) -> Result { - if !repo_path.join(".git").is_dir() { - bail!("{:?} is not a git repository", repo_path); - } - Ok(Self { - repo_path: repo_path.into(), - binary_name: "git".into(), - }) - } - - #[cfg(test)] - pub fn with_binary(repo_path: &Path, name: &str) -> Self { - Self { - repo_path: repo_path.into(), - binary_name: name.into(), - } - } - - fn extract_commit_info(&self, revision: &str, info: CommitInfo) -> Result { - let mut command = Command::new(&self.binary_name); - command.arg("show"); - command.arg("-s"); - command.arg(revision); - command.arg(format!( - "--format={}", - match info { - CommitInfo::CommitHash => "%H", - CommitInfo::AuthorName => "%an", - CommitInfo::AuthorEmail => "%ae", - CommitInfo::MessageSubject => "%s", - CommitInfo::MessageBody => "%b", - } - )); - command.current_dir(&self.repo_path); - - let output = log_command(command).output()?; - handle_failure("extract_commit_info", &output)?; - let (stdout, _) = output_text(&output); - Ok(stdout.trim().into()) - } -} - -impl Git for GitCLI { - fn path(&self) -> &Path { - &self.repo_path - } - - fn clone_to(&self, path: &Path) -> Result<()> { - let mut command = Command::new(&self.binary_name); - command.arg("clone"); - command.arg(&self.repo_path); - command.current_dir(path); - - let output = log_command(command).output()?; - handle_failure("clone_to", &output)?; - Ok(()) - } - - fn get_head_revision(&self) -> Result { - let mut command = Command::new(&self.binary_name); - command.arg("rev-parse"); - command.arg("HEAD"); - command.current_dir(&self.repo_path); - - let output = log_command(command).output()?; - handle_failure("get_head_revision", &output)?; - let (stdout, _) = output_text(&output); - Ok(CommitHash(stdout.trim().into())) - } - - fn stage(&self, path: &Path) -> Result<()> { - let mut command = Command::new(&self.binary_name); - command.arg("add"); - command.arg(path); - command.current_dir(&self.repo_path); - - let output = log_command(command).output()?; - handle_failure("stage", &output)?; - Ok(()) - } - - fn commit_on_behalf( - &self, - bot_name: &str, - bot_email: &str, - author_name: &str, - author_email: &str, - message: &str, - ) -> Result<()> { - let mut command = Command::new(&self.binary_name); - command.arg("-c"); - command.arg(format!("user.name={}", bot_name)); - command.arg("-c"); - command.arg(format!("user.email={}", bot_email)); - command.arg("commit"); - command.arg("-m"); - command.arg(message); - command.arg("--author"); - command.arg(format!("{} <{}>", author_name, author_email)); - command.current_dir(&self.repo_path); - - let output = log_command(command).output()?; - handle_failure("commit_on_behalf", &output)?; - Ok(()) - } - - fn commit(&self, name: &str, email: &str, message: &str) -> Result<()> { - let mut command = Command::new(&self.binary_name); - command.arg("-c"); - command.arg(format!("user.name={}", name)); - command.arg("-c"); - command.arg(format!("user.email={}", email)); - command.arg("commit"); - command.arg("-m"); - command.arg(message); - command.current_dir(&self.repo_path); - - let output = log_command(command).output()?; - handle_failure("commit", &output)?; - Ok(()) - } - - fn rev_list( - &self, - start_inclusive_revision: &str, - end_exclusive_revision: &str, - path: Option<&Path>, - ) -> Result> { - let mut command = Command::new(&self.binary_name); - command.arg("rev-list"); - command.arg(format!( - "{}..{}", - end_exclusive_revision, start_inclusive_revision - )); - if let Some(path) = path { - command.arg("--"); - command.arg(path); - } - command.current_dir(&self.repo_path); - - let output = log_command(command).output()?; - handle_failure("rev_list", &output)?; - let (stdout, _) = output_text(&output); - Ok(stdout - .split_ascii_whitespace() - .into_iter() - .map(CommitHash::from) - .collect()) - } - - fn show(&self, revision: &str) -> Result { - Ok(Commit { - hash: CommitHash::from(self.extract_commit_info(revision, CommitInfo::CommitHash)?), - author_name: self.extract_commit_info(revision, CommitInfo::AuthorName)?, - author_email: self.extract_commit_info(revision, CommitInfo::AuthorEmail)?, - message_subject: self.extract_commit_info(revision, CommitInfo::MessageSubject)?, - message_body: self.extract_commit_info(revision, CommitInfo::MessageBody)?, - }) - } - - fn hard_reset(&self, revision: &str) -> Result<()> { - let mut command = Command::new(&self.binary_name); - command.arg("reset"); - command.arg("--hard"); - command.arg(revision); - command.current_dir(&self.repo_path); - - let output = log_command(command).output()?; - handle_failure("rev_list", &output)?; - Ok(()) - } - - fn current_branch_name(&self) -> Result { - let mut command = Command::new(&self.binary_name); - command.arg("rev-parse"); - command.arg("--abbrev-ref"); - command.arg("HEAD"); - command.current_dir(&self.repo_path); - - let output = log_command(command).output()?; - handle_failure("current_branch_name", &output)?; - let (stdout, _) = output_text(&output); - Ok(stdout.trim().into()) - } - - fn create_branch(&self, branch_name: &str, revision: &str) -> Result<()> { - let mut command = Command::new(&self.binary_name); - command.arg("branch"); - command.arg(branch_name); - command.arg(revision); - command.current_dir(&self.repo_path); - - let output = log_command(command).output()?; - handle_failure("create_branch", &output)?; - Ok(()) - } - - fn delete_branch(&self, branch_name: &str) -> Result<()> { - let mut command = Command::new(&self.binary_name); - command.arg("branch"); - command.arg("-D"); - command.arg(branch_name); - command.current_dir(&self.repo_path); - - let output = log_command(command).output()?; - handle_failure("delete_branch", &output)?; - Ok(()) - } - - fn squash_merge( - &self, - author_name: &str, - author_email: &str, - branch_name: &str, - commit_message: &str, - ) -> Result<()> { - let mut command = Command::new(&self.binary_name); - command.arg("merge"); - command.arg("--squash"); - command.arg(branch_name); - command.current_dir(&self.repo_path); - - let output = log_command(command).output()?; - handle_failure("squash_merge", &output)?; - - // `git merge --squash` only stages changes, so a commit is necessary after - self.commit(author_name, author_email, commit_message) - } - - fn untracked_files(&self) -> Result> { - let mut command = Command::new(&self.binary_name); - command.arg("ls-files"); - command.arg("--exclude-standard"); - command.arg("--others"); - command.current_dir(&self.repo_path); - - let output = log_command(command).output()?; - handle_failure("untracked_files", &output)?; - let (stdout, _) = output_text(&output); - Ok(split_file_names(&stdout)) - } - - fn changed_files(&self) -> Result> { - let mut command = Command::new(&self.binary_name); - command.arg("diff"); - command.arg("--name-only"); - command.current_dir(&self.repo_path); - - let output = log_command(command).output()?; - handle_failure("changed_files", &output)?; - let (stdout, _) = output_text(&output); - Ok(split_file_names(&stdout)) - } -} - -fn is_newline(c: char) -> bool { - c == '\r' || c == '\n' -} - -fn split_file_names(value: &str) -> Vec { - value - .split(is_newline) - .filter(|s| !s.is_empty()) - .map(PathBuf::from) - .collect::>() -} - -fn log_command(command: Command) -> Command { - let mut message = String::new(); - if let Some(cwd) = command.get_current_dir() { - message.push_str(&format!("[in {:?}]: ", cwd)); - } - message.push_str(command.get_program().to_str().expect("valid str")); - for arg in command.get_args() { - message.push(' '); - message.push_str(arg.to_str().expect("valid str")); - } - debug!("{}", message); - command -} - -#[cfg(test)] -mod tests { - use super::*; - use std::env; - - fn bin_path(script: &'static str) -> PathBuf { - env::current_dir() - .expect("current_dir") - .join("fake-cli") - .join(script) - .canonicalize() - .expect("canonicalize") - } - fn cli(script: &'static str) -> GitCLI { - GitCLI::with_binary(&PathBuf::from("/tmp"), &bin_path(script).to_string_lossy()) - } - - #[test] - fn clone_to() { - cli("git-clone") - .clone_to(&PathBuf::from("/tmp")) - .expect("successful invocation"); - } - - #[test] - fn extract_commit_info() { - let result = cli("git-extract-commit-info") - .extract_commit_info("test_revision", CommitInfo::CommitHash) - .expect("successful invocation"); - assert_eq!("success", result); - } - - #[test] - fn changed_files() { - assert_eq!( - vec![ - PathBuf::from("some/file"), - PathBuf::from("some-other-file"), - PathBuf::from("some/file with spaces.txt"), - ], - cli("git-changed-files") - .changed_files() - .expect("successful invocation") - ); - assert_eq!( - Vec::::new(), - cli("git-changed-files-empty") - .changed_files() - .expect("successful invocation") - ); - } - - #[test] - fn untracked_files() { - assert_eq!( - vec![ - PathBuf::from("some-untracked-file"), - PathBuf::from("another-untracked-file"), - PathBuf::from("some/file with spaces.txt"), - ], - cli("git-untracked-files") - .untracked_files() - .expect("successful invocation") - ); - assert_eq!( - Vec::::new(), - cli("git-untracked-files-empty") - .untracked_files() - .expect("successful invocation") - ); - } - - #[test] - fn get_head_revision() { - assert_eq!( - "some-commit-hash", - cli("git-get-head-revision") - .get_head_revision() - .expect("successful invocation") - .as_ref() - ); - } - - #[test] - fn stage() { - cli("git-stage") - .stage(&PathBuf::from("test-path")) - .expect("successful invocation"); - } - - #[test] - fn commit_on_behalf() { - cli("git-commit-on-behalf") - .commit_on_behalf( - "Bot Name", - "bot@example.com", - "Some Author", - "author@example.com", - "Test message", - ) - .expect("successful invocation"); - } - - #[test] - fn commit() { - cli("git-commit") - .commit("Some Author", "author@example.com", "Test message") - .expect("successful invocation"); - } - - #[test] - fn rev_list() { - assert_eq!( - vec![ - CommitHash::from("second-commit"), - CommitHash::from("initial-commit") - ], - cli("git-rev-list") - .rev_list("start_inclusive", "end_exclusive", None) - .expect("successful invocation") - ); - assert_eq!( - vec![ - CommitHash::from("third-commit"), - CommitHash::from("second-commit"), - CommitHash::from("initial-commit") - ], - cli("git-rev-list-path") - .rev_list( - "start_inclusive", - "end_exclusive", - Some(&PathBuf::from("some-path")) - ) - .expect("successful invocation") - ); - } - - #[test] - fn show() { - assert_eq!( - Commit { - hash: "some-commit-hash".into(), - author_name: "Some Author".into(), - author_email: "author@example.com".into(), - message_subject: "Some message subject".into(), - message_body: "Message body\n with multiple lines".into() - }, - cli("git-show") - .show("test_revision") - .expect("successful invocation") - ); - } - - #[test] - fn hard_reset() { - cli("git-reset-hard") - .hard_reset("some-revision") - .expect("successful invocation"); - } - - #[test] - fn current_branch_name() { - assert_eq!( - "some-branch-name", - cli("git-current-branch-name") - .current_branch_name() - .expect("successful invocation") - ); - } - - #[test] - fn create_branch() { - cli("git-create-branch") - .create_branch("test-branch-name", "test-revision") - .expect("successful invocation"); - } - - #[test] - fn delete_branch() { - cli("git-delete-branch") - .delete_branch("test-branch-name") - .expect("successful invocation"); - } - - #[test] - fn squash_merge() { - cli("git-squash-merge") - .squash_merge( - "test-author", - "test-author-email", - "test-branch-name", - "test message", - ) - .expect("successful invocation"); - } -} diff --git a/tools/sdk-sync/src/lib.rs b/tools/sdk-sync/src/lib.rs index 0f45750a5..a150d8166 100644 --- a/tools/sdk-sync/src/lib.rs +++ b/tools/sdk-sync/src/lib.rs @@ -8,7 +8,6 @@ use tracing_subscriber::prelude::*; use tracing_subscriber::EnvFilter; pub mod fs; -pub mod git; pub mod sync; pub mod versions; diff --git a/tools/sdk-sync/src/main.rs b/tools/sdk-sync/src/main.rs index 37b673454..b3df557f2 100644 --- a/tools/sdk-sync/src/main.rs +++ b/tools/sdk-sync/src/main.rs @@ -58,6 +58,7 @@ impl Args { max_gradle_metaspace_megabytes: self .max_gradle_metaspace_megabytes .unwrap_or(defaults.max_gradle_metaspace_megabytes), + aws_models_path: Some(self.aws_sdk_rust.join("aws-models")), } } } diff --git a/tools/sdk-sync/src/sync.rs b/tools/sdk-sync/src/sync.rs index afa1af63c..b9b2da46d 100644 --- a/tools/sdk-sync/src/sync.rs +++ b/tools/sdk-sync/src/sync.rs @@ -5,9 +5,9 @@ use self::gen::{CodeGenSettings, DefaultSdkGenerator, SdkGenerator}; use crate::fs::{DefaultFs, Fs}; -use crate::git::{Commit, Git, GitCLI}; use crate::versions::{DefaultVersions, Versions, VersionsManifest}; use anyhow::{Context, Result}; +use smithy_rs_tool_common::git::{Commit, Git, GitCLI}; use smithy_rs_tool_common::macros::here; use std::collections::BTreeSet; use std::path::{Path, PathBuf}; @@ -186,27 +186,27 @@ impl Sync { #[instrument(skip(self))] fn stash_model_changes(&self) -> Result { info!("Stashing model changes..."); - let original_revision = self.smithy_rs.get_head_revision().context(here!())?; + let original_revision = self.aws_sdk_rust.get_head_revision().context(here!())?; info!( - "smithy-rs revision with model changes: {}", + "aws-sdk-rust revision with model changes: {}", original_revision ); // Create a branch to hold the model changes without switching to it - self.smithy_rs + self.aws_sdk_rust .create_branch(MODEL_STASH_BRANCH_NAME, "HEAD") .context(here!())?; // Get the name of the current branch - let branch_name = self.smithy_rs.current_branch_name().context(here!())?; + let branch_name = self.aws_sdk_rust.current_branch_name().context(here!())?; // Reset the start branch to what's in origin - self.smithy_rs + self.aws_sdk_rust .hard_reset(&format!("origin/{}", branch_name)) .context(here!())?; - let head = self.smithy_rs.get_head_revision().context(here!())?; - info!("smithy-rs revision without model changes: {}", head); + let head = self.aws_sdk_rust.get_head_revision().context(here!())?; + info!("aws-sdk-rust revision without model changes: {}", head); let has_model_changes = head != original_revision; - info!("smithy-rs has model changes: {}", has_model_changes); + info!("aws-sdk-rust has model changes: {}", has_model_changes); Ok(has_model_changes) } @@ -218,18 +218,12 @@ impl Sync { // Restore the model changes. Note: endpoints.json/default config/model changes // may each be in their own commits coming into this, but we want them squashed into // one commit for smithy-rs. - self.smithy_rs - .squash_merge( - BOT_NAME, - BOT_EMAIL, - MODEL_STASH_BRANCH_NAME, - "Update SDK models", - ) + self.aws_sdk_rust + .squash_merge(BOT_NAME, BOT_EMAIL, MODEL_STASH_BRANCH_NAME) .context(here!())?; - self.smithy_rs + self.aws_sdk_rust .delete_branch(MODEL_STASH_BRANCH_NAME) .context(here!())?; - let model_change_commit = self.smithy_rs.show("HEAD").context(here!())?; // Generate with the original examples let sdk_gen = DefaultSdkGenerator::new( @@ -252,7 +246,7 @@ impl Sync { .stage(&PathBuf::from(".")) .context(here!())?; self.aws_sdk_rust - .commit(BOT_NAME, BOT_EMAIL, &model_change_commit.message()) + .commit(BOT_NAME, BOT_EMAIL, "Update SDK models") .context(here!())?; } @@ -459,21 +453,37 @@ impl Sync { Ok(()) } - /// Returns true if the aws-sdk-rust repo has changes (excluding changes to versions.toml). - /// The versions.toml shouldn't be considered since there's a high probability it is only + /// Returns true if the aws-sdk-rust repo has changes (excluding changes to `versions.toml` + /// and `aws-models/`). + /// + /// The `versions.toml` shouldn't be considered since there's a high probability it is only /// a `smithy_rs_revision` change. It can also safely be ignored since any changes to version /// numbers will show up in individual crate manifests. + /// + /// The `aws-models/` can be ignored since they will be updated, but are not code generated. + /// This function only cares about code generated files. #[instrument(skip(self))] fn sdk_has_changes(&self) -> Result { let untracked_files = self.aws_sdk_rust.untracked_files()?; let changed_files = self.aws_sdk_rust.changed_files()?; - let has_changes = !untracked_files.is_empty() - || !changed_files.is_empty() - && (changed_files.len() != 1 || changed_files[0].to_str() != Some("versions.toml")); - debug!("aws-sdk-rust untracked files: {:?}", untracked_files); - debug!("aws-sdk-rust changed files: {:?}", changed_files); + let all_relevant_changed_files: Vec<_> = untracked_files + .into_iter() + .chain(changed_files.into_iter()) + .filter(|path| { + if let Some(path) = path.to_str() { + path != "versions.toml" && !path.starts_with("aws-models/") + } else { + true + } + }) + .collect(); + let has_changes = !all_relevant_changed_files.is_empty(); + debug!( + "aws-sdk-rust relevant changed files: {:?}", + all_relevant_changed_files + ); info!( - "aws-sdk-rust has changes (not considering versions.toml): {}", + "aws-sdk-rust has changes (not considering versions.toml and aws-models): {}", has_changes ); Ok(has_changes) @@ -523,8 +533,41 @@ impl Sync { mod tests { use super::*; use crate::fs::MockFs; - use crate::git::{CommitHash, MockGit}; use crate::versions::MockVersions; + use smithy_rs_tool_common::git::CommitHash; + + mockall::mock! { + Git {} + impl smithy_rs_tool_common::git::Git for Git { + fn path(&self) -> &Path; + fn clone_to(&self, path: &Path) -> Result<()>; + fn get_head_revision(&self) -> Result; + fn stage(&self, path: &Path) -> Result<()>; + fn commit_on_behalf( + &self, + bot_name: &str, + bot_email: &str, + author_name: &str, + author_email: &str, + message: &str, + ) -> Result<()>; + fn commit(&self, name: &str, email: &str, message: &str) -> Result<()>; + fn rev_list<'a>( + &self, + start_inclusive_revision: &str, + end_exclusive_revision: &str, + path: Option<&'a Path>, + ) -> Result>; + fn show(&self, revision: &str) -> Result; + fn hard_reset(&self, revision: &str) -> Result<()>; + fn current_branch_name(&self) -> Result; + fn create_branch(&self, branch_name: &str, revision: &str) -> Result<()>; + fn delete_branch(&self, branch_name: &str) -> Result<()>; + fn squash_merge(&self, author_name: &str, author_email: &str, branch_name: &str) -> Result<()>; + fn untracked_files(&self) -> Result>; + fn changed_files(&self) -> Result>; + } + } // Wish this was in std... fn trim_indent(value: &str) -> String { diff --git a/tools/sdk-sync/src/sync/gen.rs b/tools/sdk-sync/src/sync/gen.rs index d7f7b89fe..a8597134d 100644 --- a/tools/sdk-sync/src/sync/gen.rs +++ b/tools/sdk-sync/src/sync/gen.rs @@ -4,20 +4,21 @@ */ use crate::fs::Fs; -use crate::git::{CommitHash, Git, GitCLI}; use anyhow::{Context, Result}; +use smithy_rs_tool_common::git::{CommitHash, Git, GitCLI}; use smithy_rs_tool_common::here; use smithy_rs_tool_common::shell::handle_failure; use std::path::{Path, PathBuf}; use std::process::Command; use std::sync::Arc; -use tracing::{info, instrument}; +use tracing::{error, info, instrument}; #[derive(Clone, Debug)] pub struct CodeGenSettings { pub smithy_parallelism: usize, pub max_gradle_heap_megabytes: usize, pub max_gradle_metaspace_megabytes: usize, + pub aws_models_path: Option, } impl Default for CodeGenSettings { @@ -26,6 +27,7 @@ impl Default for CodeGenSettings { smithy_parallelism: 1, max_gradle_heap_megabytes: 512, max_gradle_metaspace_megabytes: 512, + aws_models_path: None, } } } @@ -128,8 +130,6 @@ impl DefaultSdkGenerator { } fn do_aws_sdk_assemble(&self) -> Result<()> { - info!("Generating the SDK..."); - let mut command = Command::new("./gradlew"); command.arg("--no-daemon"); // Don't let Gradle continue running after the build command.arg("--no-parallel"); // Disable Gradle parallelism @@ -158,13 +158,23 @@ impl DefaultSdkGenerator { .join(" ") )); + // TODO(https://github.com/awslabs/smithy-rs/issues/1493): Remove this argument once a release goes out with it removed from `build.gradle.kts` + command.arg("-Paws.fullsdk=true"); + // Disable Smithy's codegen parallelism in favor of sdk-sync parallelism command.arg(format!( "-Djava.util.concurrent.ForkJoinPool.common.parallelism={}", self.settings.smithy_parallelism )); - command.arg("-Paws.fullsdk=true"); + if let Some(models_path) = &self.settings.aws_models_path { + command.arg(format!( + "-Paws.sdk.models.path={}", + models_path + .to_str() + .expect("aws models path is a valid str") + )); + } command.arg(format!( "-Paws.sdk.previous.release.versions.manifest={}", self.previous_versions_manifest @@ -178,6 +188,8 @@ impl DefaultSdkGenerator { command.arg("aws:sdk:assemble"); command.current_dir(self.smithy_rs.path()); + info!("Generating the SDK with: {:#?}", command); + let output = command.output()?; handle_failure("aws_sdk_assemble", &output)?; Ok(()) @@ -187,7 +199,8 @@ impl DefaultSdkGenerator { #[instrument(skip(self))] fn aws_sdk_assemble(&self) -> Result<()> { let result = self.do_aws_sdk_assemble(); - if result.is_err() { + if let Err(err) = &result { + error!("Codegen failed: {}", err); // On failure, do a dump of running processes to give more insight into if there is a process leak going on match Command::new("ps").arg("-ef").output() { Ok(output) => info!( diff --git a/tools/sdk-sync/src/versions.rs b/tools/sdk-sync/src/versions.rs index 93ad1e191..a05be2754 100644 --- a/tools/sdk-sync/src/versions.rs +++ b/tools/sdk-sync/src/versions.rs @@ -4,8 +4,8 @@ */ use crate::fs::{DefaultFs, Fs}; -use crate::git::CommitHash; use anyhow::Result; +use smithy_rs_tool_common::git::CommitHash; use std::path::Path; use std::str::FromStr; diff --git a/tools/sdk-sync/tests/create-test-workspace b/tools/sdk-sync/tests/create-test-workspace index da5c7d5b5..23c3dcc9f 100755 --- a/tools/sdk-sync/tests/create-test-workspace +++ b/tools/sdk-sync/tests/create-test-workspace @@ -18,8 +18,6 @@ if [[ $# -eq 1 && "$1" == "--with-model-changes" ]]; then INCLUDE_MODEL_CHANGES=1 fi -ENDPOINTS_JSON_PATH="aws/sdk-codegen/src/main/resources/software/amazon/smithy/rustsdk/endpoints.json" - mkdir aws-doc-sdk-examples mkdir aws-sdk-rust mkdir smithy-rs @@ -40,13 +38,9 @@ popd # Set up a fake smithy-rs repo pushd smithy-rs git init . -mkdir -p aws/sdk/aws-models mkdir -p aws/sdk/examples mkdir -p aws/sdk/build/aws-sdk/examples/s3 mkdir -p aws/sdk/build/aws-sdk/sdk/s3 -mkdir -p $(dirname "${ENDPOINTS_JSON_PATH}") -echo "Ancient S3 model" > aws/sdk/aws-models/s3.json -echo "Old endpoints.json" > "${ENDPOINTS_JSON_PATH}" echo "Some S3 client code" > aws/sdk/build/aws-sdk/sdk/s3/fake_content cat "${SCRIPT_PATH}/fake-sdk-assemble" > gradlew chmod +x gradlew @@ -60,25 +54,14 @@ git add . git -c user.name="Server Dev" -c user.email="serverdev@example.com" commit -m "Make some server changes" popd -# Make an origin for smithy-rs -mv smithy-rs smithy-rs-origin -git clone smithy-rs-origin smithy-rs - -# (Optionally) Add model changes -if [[ "${INCLUDE_MODEL_CHANGES}" == "1" ]]; then - pushd smithy-rs - echo "Updated S3 model" > aws/sdk/aws-models/s3.json - git -c user.name="Automated Process" -c user.email="bot@example.com" commit -am "Update the S3 model" - echo "Updated endpoints.json" > "${ENDPOINTS_JSON_PATH}" - git -c user.name="Automated Process" -c user.email="bot@example.com" commit -am "Update endpoints.json" - popd -fi - # Set up a fake aws-sdk-rust repo pushd aws-sdk-rust git init . +mkdir -p aws-models mkdir -p examples/s3 mkdir -p sdk/s3 +echo "Ancient S3 model" > aws-models/s3.json +echo "Old endpoints.json" > aws-models/endpoints.json echo "Some S3 client code" > sdk/s3/fake_content echo "Ancient S3 model" > sdk/s3/s3.json echo "Some S3 example" > examples/s3/fake_content @@ -87,6 +70,7 @@ echo "Some handwritten file" > some_handwritten echo ".git"; \ echo ".handwritten"; \ echo "some_handwritten"; \ + echo "/aws-models/"; \ ) > .handwritten ( \ echo '# special test comment: this came from the previous release'; \ @@ -101,3 +85,17 @@ echo "Some handwritten file" > some_handwritten git add . git -c user.name="Test Dev" -c user.email="testdev@example.com" commit -m "Initial commit" popd + +# Make an origin for aws-sdk-rust +mv aws-sdk-rust aws-sdk-rust-origin +git clone aws-sdk-rust-origin aws-sdk-rust + +# (Optionally) Add model changes +if [[ "${INCLUDE_MODEL_CHANGES}" == "1" ]]; then + pushd aws-sdk-rust + echo "Updated S3 model" > aws-models/s3.json + git -c user.name="Automated Process" -c user.email="bot@example.com" commit -am "Update the S3 model" + echo "Updated endpoints.json" > aws-models/endpoints.json + git -c user.name="Automated Process" -c user.email="bot@example.com" commit -am "Update endpoints.json" + popd +fi diff --git a/tools/sdk-sync/tests/e2e_test.rs b/tools/sdk-sync/tests/e2e_test.rs index 016add58a..b0264208f 100644 --- a/tools/sdk-sync/tests/e2e_test.rs +++ b/tools/sdk-sync/tests/e2e_test.rs @@ -5,18 +5,16 @@ use once_cell::sync::Lazy; use regex::Regex; -use sdk_sync::git::{Git, GitCLI}; use sdk_sync::init_tracing; +use sdk_sync::sync::gen::CodeGenSettings; use sdk_sync::sync::{Sync, BOT_EMAIL, BOT_NAME}; +use smithy_rs_tool_common::git::{Git, GitCLI}; use smithy_rs_tool_common::shell::handle_failure; use std::fs; use std::path::Path; use std::process::Command; use tempfile::TempDir; -const ENDPOINTS_JSON_PATH: &str = - "aws/sdk-codegen/src/main/resources/software/amazon/smithy/rustsdk/endpoints.json"; - static INIT_TRACING: Lazy = Lazy::new(|| { init_tracing(); true @@ -67,10 +65,12 @@ fn test_without_model_changes() { // Assert pre-conditions assert_file_exists(aws_sdk_rust.path().join(".git")); assert_file_exists(aws_sdk_rust.path().join(".handwritten")); - assert_file_exists(aws_sdk_rust.path().join("some_handwritten")); - assert_file_exists(aws_sdk_rust.path().join("versions.toml")); + assert_file_exists(aws_sdk_rust.path().join("aws-models/endpoints.json")); + assert_file_exists(aws_sdk_rust.path().join("aws-models/s3.json")); assert_file_exists(aws_sdk_rust.path().join("sdk/s3/fake_content")); assert_file_exists(aws_sdk_rust.path().join("sdk/s3/s3.json")); + assert_file_exists(aws_sdk_rust.path().join("some_handwritten")); + assert_file_exists(aws_sdk_rust.path().join("versions.toml")); assert_file_exists(aws_sdk_rust.path().join("examples/s3/fake_content")); assert_no_file(aws_sdk_rust.path().join("examples/Cargo.toml")); @@ -89,10 +89,13 @@ fn test_without_model_changes() { // Run the sync let sync = Sync::new( - &tmp_dir.as_ref().join("aws-doc-sdk-examples"), - &tmp_dir.as_ref().join("aws-sdk-rust"), - &tmp_dir.as_ref().join("smithy-rs"), - Default::default(), + &tmp_dir.path().join("aws-doc-sdk-examples"), + &tmp_dir.path().join("aws-sdk-rust"), + &tmp_dir.path().join("smithy-rs"), + CodeGenSettings { + aws_models_path: Some(tmp_dir.path().join("aws-sdk-rust").join("aws-models")), + ..Default::default() + }, ) .expect("create sync success"); sync.sync().expect("sync success"); @@ -133,6 +136,8 @@ fn test_without_model_changes() { // Verify SDK files assert_file_exists(aws_sdk_rust.path().join(".git")); assert_file_exists(aws_sdk_rust.path().join(".handwritten")); + assert_file_exists(aws_sdk_rust.path().join("aws-models/endpoints.json")); + assert_file_exists(aws_sdk_rust.path().join("aws-models/s3.json")); assert_file_exists(aws_sdk_rust.path().join("some_handwritten")); assert_file_exists(aws_sdk_rust.path().join("versions.toml")); assert_file_exists(aws_sdk_rust.path().join("sdk/s3/fake_content")); @@ -153,7 +158,7 @@ fn test_without_model_changes() { "Some modified S3 example\n", ); - // Verify smithy-rs had no changes since we don't have model updates + // Verify smithy-rs had no changes assert_eq!( smithy_rs_start_revision, smithy_rs.get_head_revision().unwrap() @@ -186,10 +191,12 @@ fn test_with_model_changes() { // Assert pre-conditions assert_file_exists(aws_sdk_rust.path().join(".git")); assert_file_exists(aws_sdk_rust.path().join(".handwritten")); - assert_file_exists(aws_sdk_rust.path().join("some_handwritten")); - assert_file_exists(aws_sdk_rust.path().join("versions.toml")); + assert_file_exists(aws_sdk_rust.path().join("aws-models/endpoints.json")); + assert_file_exists(aws_sdk_rust.path().join("aws-models/s3.json")); assert_file_exists(aws_sdk_rust.path().join("sdk/s3/fake_content")); assert_file_exists(aws_sdk_rust.path().join("sdk/s3/s3.json")); + assert_file_exists(aws_sdk_rust.path().join("some_handwritten")); + assert_file_exists(aws_sdk_rust.path().join("versions.toml")); assert_file_exists(aws_sdk_rust.path().join("examples/s3/fake_content")); assert_no_file(aws_sdk_rust.path().join("examples/Cargo.toml")); @@ -208,10 +215,13 @@ fn test_with_model_changes() { // Run the sync let sync = Sync::new( - &tmp_dir.as_ref().join("aws-doc-sdk-examples"), - &tmp_dir.as_ref().join("aws-sdk-rust"), - &tmp_dir.as_ref().join("smithy-rs"), - Default::default(), + &tmp_dir.path().join("aws-doc-sdk-examples"), + &tmp_dir.path().join("aws-sdk-rust"), + &tmp_dir.path().join("smithy-rs"), + CodeGenSettings { + aws_models_path: Some(tmp_dir.path().join("aws-sdk-rust").join("aws-models")), + ..Default::default() + }, ) .expect("create sync success"); sync.sync().expect("sync success"); @@ -225,7 +235,7 @@ fn test_with_model_changes() { .map(|commit_hash| aws_sdk_rust.show(commit_hash.as_ref()).unwrap()) .collect(); - assert_eq!(3, sdk_commits.len()); + assert_eq!(3, sdk_commits.len(), "commits: {:#?}", sdk_commits); assert_eq!(BOT_NAME, sdk_commits[0].author_name); assert_eq!(BOT_EMAIL, sdk_commits[0].author_email); @@ -257,6 +267,8 @@ fn test_with_model_changes() { // Verify SDK files assert_file_exists(aws_sdk_rust.path().join(".git")); assert_file_exists(aws_sdk_rust.path().join(".handwritten")); + assert_file_exists(aws_sdk_rust.path().join("aws-models/endpoints.json")); + assert_file_exists(aws_sdk_rust.path().join("aws-models/s3.json")); assert_file_exists(aws_sdk_rust.path().join("some_handwritten")); assert_file_exists(aws_sdk_rust.path().join("versions.toml")); assert_file_exists(aws_sdk_rust.path().join("sdk/s3/fake_content")); @@ -268,6 +280,10 @@ fn test_with_model_changes() { aws_sdk_rust.path().join("sdk/s3/s3.json"), "Updated S3 model\n", ); + assert_file_contents( + aws_sdk_rust.path().join("sdk/verify-endpoints.json"), + "Updated endpoints.json\n", + ); assert_file_contents( aws_sdk_rust.path().join("sdk/s3/fake_content"), "Some updated S3 client code\n", @@ -277,13 +293,9 @@ fn test_with_model_changes() { "Some modified S3 example\n", ); - // Verify smithy-rs has the model updates - assert_ne!( + // Verify smithy-rs had no changes + assert_eq!( smithy_rs_start_revision, smithy_rs.get_head_revision().unwrap() ); - assert_file_contents( - smithy_rs.path().join(ENDPOINTS_JSON_PATH), - "Updated endpoints.json\n", - ); } diff --git a/tools/sdk-sync/tests/fake-sdk-assemble b/tools/sdk-sync/tests/fake-sdk-assemble index 6e19fe8b5..0964d3ad6 100755 --- a/tools/sdk-sync/tests/fake-sdk-assemble +++ b/tools/sdk-sync/tests/fake-sdk-assemble @@ -34,7 +34,14 @@ def get_previous_release_versions(): return get_property("aws.sdk.previous.release.versions.manifest") -if get_property("aws.fullsdk") != "true": +def get_models_path(): + return get_property("aws.sdk.models.path") + + +# Verify the models path was set correctly +models_path = get_models_path() +if models_path is None or not os.path.isfile(f"{models_path}/s3.json"): + print(f"Missing or wrong aws.sdk.models.path: {models_path}") sys.exit(1) # Verify the versions manifest path was set correctly @@ -60,7 +67,13 @@ with open("aws/sdk/build/aws-sdk/versions.toml", "w") as versions: # Emulate generating code from the models by just copying the model into the build artifacts. # The model doesn't get copied like this in reality, but this is an easy way to fake it. -subprocess.run(shlex.split("cp aws/sdk/aws-models/s3.json aws/sdk/build/aws-sdk/sdk/s3/"), check=True) +subprocess.run(shlex.split(f"cp {models_path}/s3.json aws/sdk/build/aws-sdk/sdk/s3/"), check=True) + +# Emulate using the updated endpoints.json by copying it into sdk/verify-endpoints.json. +subprocess.run( + shlex.split(f"cp {models_path}/endpoints.json aws/sdk/build/aws-sdk/sdk/verify-endpoints.json"), + check=True +) # Emulate copying the examples into the build output subprocess.run(shlex.split("cp -r aws/sdk/examples aws/sdk/build/aws-sdk/"), check=True) diff --git a/tools/sdk-versioner/Cargo.lock b/tools/sdk-versioner/Cargo.lock index 5a153d93d..d7959e62b 100644 --- a/tools/sdk-versioner/Cargo.lock +++ b/tools/sdk-versioner/Cargo.lock @@ -161,6 +161,12 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "itoa" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "112c678d4050afce233f4f2852bb2eb519230b3cf12f33585275537d7e41578d" + [[package]] name = "lazy_static" version = "1.4.0" @@ -272,6 +278,12 @@ dependencies = [ "winapi", ] +[[package]] +name = "ryu" +version = "1.0.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3f6f92acf49d1b98f7a81226834412ada05458b7364277387724a237f062695" + [[package]] name = "sdk-versioner" version = "0.1.0" @@ -304,6 +316,17 @@ dependencies = [ "syn", ] +[[package]] +name = "serde_json" +version = "1.0.81" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b7ce2b32a1aed03c558dc61a5cd328f15aff2dbc17daad8fb8af04d2100e15c" +dependencies = [ + "itoa", + "ryu", + "serde", +] + [[package]] name = "smithy-rs-tool-common" version = "0.1.0" @@ -311,6 +334,7 @@ dependencies = [ "anyhow", "async-trait", "serde", + "serde_json", "toml", "tracing", ] diff --git a/tools/smithy-rs-tool-common/Cargo.toml b/tools/smithy-rs-tool-common/Cargo.toml index bb5f6eacd..e22f18d24 100644 --- a/tools/smithy-rs-tool-common/Cargo.toml +++ b/tools/smithy-rs-tool-common/Cargo.toml @@ -19,6 +19,7 @@ opt-level = 0 anyhow = "1" async-trait = "0.1" serde = { version = "1", features = ["derive"] } +serde_json = "1" tokio = { version = "1", features = ["rt", "macros"], optional = true } toml = { version = "0.5.8", features = ["preserve_order"] } tracing = "0.1" diff --git a/tools/sdk-sync/fake-cli/git-changed-files b/tools/smithy-rs-tool-common/fake-cli/git-changed-files similarity index 100% rename from tools/sdk-sync/fake-cli/git-changed-files rename to tools/smithy-rs-tool-common/fake-cli/git-changed-files diff --git a/tools/sdk-sync/fake-cli/git-changed-files-empty b/tools/smithy-rs-tool-common/fake-cli/git-changed-files-empty similarity index 100% rename from tools/sdk-sync/fake-cli/git-changed-files-empty rename to tools/smithy-rs-tool-common/fake-cli/git-changed-files-empty diff --git a/tools/sdk-sync/fake-cli/git-clone b/tools/smithy-rs-tool-common/fake-cli/git-clone similarity index 100% rename from tools/sdk-sync/fake-cli/git-clone rename to tools/smithy-rs-tool-common/fake-cli/git-clone diff --git a/tools/sdk-sync/fake-cli/git-commit b/tools/smithy-rs-tool-common/fake-cli/git-commit similarity index 100% rename from tools/sdk-sync/fake-cli/git-commit rename to tools/smithy-rs-tool-common/fake-cli/git-commit diff --git a/tools/sdk-sync/fake-cli/git-commit-on-behalf b/tools/smithy-rs-tool-common/fake-cli/git-commit-on-behalf similarity index 100% rename from tools/sdk-sync/fake-cli/git-commit-on-behalf rename to tools/smithy-rs-tool-common/fake-cli/git-commit-on-behalf diff --git a/tools/sdk-sync/fake-cli/git-create-branch b/tools/smithy-rs-tool-common/fake-cli/git-create-branch similarity index 100% rename from tools/sdk-sync/fake-cli/git-create-branch rename to tools/smithy-rs-tool-common/fake-cli/git-create-branch diff --git a/tools/sdk-sync/fake-cli/git-current-branch-name b/tools/smithy-rs-tool-common/fake-cli/git-current-branch-name similarity index 100% rename from tools/sdk-sync/fake-cli/git-current-branch-name rename to tools/smithy-rs-tool-common/fake-cli/git-current-branch-name diff --git a/tools/sdk-sync/fake-cli/git-delete-branch b/tools/smithy-rs-tool-common/fake-cli/git-delete-branch similarity index 100% rename from tools/sdk-sync/fake-cli/git-delete-branch rename to tools/smithy-rs-tool-common/fake-cli/git-delete-branch diff --git a/tools/sdk-sync/fake-cli/git-extract-commit-info b/tools/smithy-rs-tool-common/fake-cli/git-extract-commit-info similarity index 100% rename from tools/sdk-sync/fake-cli/git-extract-commit-info rename to tools/smithy-rs-tool-common/fake-cli/git-extract-commit-info diff --git a/tools/sdk-sync/fake-cli/git-get-head-revision b/tools/smithy-rs-tool-common/fake-cli/git-get-head-revision similarity index 100% rename from tools/sdk-sync/fake-cli/git-get-head-revision rename to tools/smithy-rs-tool-common/fake-cli/git-get-head-revision diff --git a/tools/sdk-sync/fake-cli/git-reset-hard b/tools/smithy-rs-tool-common/fake-cli/git-reset-hard similarity index 100% rename from tools/sdk-sync/fake-cli/git-reset-hard rename to tools/smithy-rs-tool-common/fake-cli/git-reset-hard diff --git a/tools/sdk-sync/fake-cli/git-rev-list b/tools/smithy-rs-tool-common/fake-cli/git-rev-list similarity index 100% rename from tools/sdk-sync/fake-cli/git-rev-list rename to tools/smithy-rs-tool-common/fake-cli/git-rev-list diff --git a/tools/sdk-sync/fake-cli/git-rev-list-path b/tools/smithy-rs-tool-common/fake-cli/git-rev-list-path similarity index 100% rename from tools/sdk-sync/fake-cli/git-rev-list-path rename to tools/smithy-rs-tool-common/fake-cli/git-rev-list-path diff --git a/tools/sdk-sync/fake-cli/git-show b/tools/smithy-rs-tool-common/fake-cli/git-show similarity index 100% rename from tools/sdk-sync/fake-cli/git-show rename to tools/smithy-rs-tool-common/fake-cli/git-show diff --git a/tools/smithy-rs-tool-common/fake-cli/git-squash-merge b/tools/smithy-rs-tool-common/fake-cli/git-squash-merge new file mode 100755 index 000000000..027bb62f1 --- /dev/null +++ b/tools/smithy-rs-tool-common/fake-cli/git-squash-merge @@ -0,0 +1,13 @@ +#!/usr/bin/env python3 +import os +import sys + +expected = [os.path.realpath("/tmp"), [ + "-c", "user.name=some-dev", + "-c", "user.email=some-email@example.com", + "merge", "--squash", "test-branch-name" +]] +actual = [os.getcwd(), sys.argv[1:]] +if expected != actual: + print(f"ERROR\nExpect: {expected}\nActual: {actual}") + sys.exit(1) diff --git a/tools/sdk-sync/fake-cli/git-stage b/tools/smithy-rs-tool-common/fake-cli/git-stage similarity index 100% rename from tools/sdk-sync/fake-cli/git-stage rename to tools/smithy-rs-tool-common/fake-cli/git-stage diff --git a/tools/sdk-sync/fake-cli/git-untracked-files b/tools/smithy-rs-tool-common/fake-cli/git-untracked-files similarity index 100% rename from tools/sdk-sync/fake-cli/git-untracked-files rename to tools/smithy-rs-tool-common/fake-cli/git-untracked-files diff --git a/tools/sdk-sync/fake-cli/git-untracked-files-empty b/tools/smithy-rs-tool-common/fake-cli/git-untracked-files-empty similarity index 100% rename from tools/sdk-sync/fake-cli/git-untracked-files-empty rename to tools/smithy-rs-tool-common/fake-cli/git-untracked-files-empty diff --git a/tools/smithy-rs-tool-common/fake_git/git_describe_tags b/tools/smithy-rs-tool-common/fake_git/git_describe_tags deleted file mode 100755 index b088d2c5e..000000000 --- a/tools/smithy-rs-tool-common/fake_git/git_describe_tags +++ /dev/null @@ -1,6 +0,0 @@ -#!/bin/bash -if [[ "$1" != "describe" || "$2" != "--tags" ]]; then - echo "wrong arguments" >&2 - exit 1 -fi -echo "some-tag" diff --git a/tools/smithy-rs-tool-common/fake_git/git_fails b/tools/smithy-rs-tool-common/fake_git/git_fails deleted file mode 100755 index 9166af04a..000000000 --- a/tools/smithy-rs-tool-common/fake_git/git_fails +++ /dev/null @@ -1,4 +0,0 @@ -#!/bin/bash -echo "some stdout failure message" ->&2 echo "some stderr failure message" -exit 1 diff --git a/tools/smithy-rs-tool-common/fake_git/git_reset b/tools/smithy-rs-tool-common/fake_git/git_reset deleted file mode 100755 index b3910fe8d..000000000 --- a/tools/smithy-rs-tool-common/fake_git/git_reset +++ /dev/null @@ -1,5 +0,0 @@ -#!/bin/bash -if [[ "$1" != "reset" || "$2" != "--hard" || "$3" != "some-commit-hash" ]]; then - echo "wrong arguments" >&2 - exit 1 -fi diff --git a/tools/smithy-rs-tool-common/fake_git/git_revparse_head b/tools/smithy-rs-tool-common/fake_git/git_revparse_head deleted file mode 100755 index eea4c7d05..000000000 --- a/tools/smithy-rs-tool-common/fake_git/git_revparse_head +++ /dev/null @@ -1,6 +0,0 @@ -#!/bin/bash -if [[ "$1" != "rev-parse" || "$2" != "HEAD" ]]; then - echo "wrong arguments" >&2 - exit 1 -fi -echo "commithash" diff --git a/tools/smithy-rs-tool-common/fake_git/git_revparse_show_toplevel b/tools/smithy-rs-tool-common/fake_git/git_revparse_show_toplevel deleted file mode 100755 index 758001ad0..000000000 --- a/tools/smithy-rs-tool-common/fake_git/git_revparse_show_toplevel +++ /dev/null @@ -1,6 +0,0 @@ -#!/bin/bash -if [[ "$1" != "rev-parse" || "$2" != "--show-toplevel" ]]; then - echo "wrong arguments" >&2 - exit 1 -fi -echo "/git/repo/root/path" diff --git a/tools/smithy-rs-tool-common/src/changelog.rs b/tools/smithy-rs-tool-common/src/changelog.rs new file mode 100644 index 000000000..c2763e2b0 --- /dev/null +++ b/tools/smithy-rs-tool-common/src/changelog.rs @@ -0,0 +1,243 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +//! This module holds deserializable structs for the hand-authored changelog TOML files used in smithy-rs. + +use anyhow::{bail, Context, Result}; +use serde::{de, Deserialize, Deserializer, Serialize}; +use std::fmt; +use std::path::Path; +use std::str::FromStr; + +#[derive(Clone, Debug, Deserialize, Serialize)] +pub struct Meta { + pub bug: bool, + pub breaking: bool, + pub tada: bool, +} + +#[derive(Clone, Debug)] +pub struct Reference { + pub repo: String, + pub number: usize, +} + +impl<'de> Deserialize<'de> for Reference { + fn deserialize(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + let s = String::deserialize(deserializer)?; + FromStr::from_str(&s).map_err(de::Error::custom) + } +} + +impl Serialize for Reference { + fn serialize(&self, serializer: S) -> Result + where + S: serde::Serializer, + { + serializer.serialize_str(&format!("{}#{}", self.repo, self.number)) + } +} + +impl fmt::Display for Reference { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{}#{}", self.repo, self.number) + } +} + +impl FromStr for Reference { + type Err = anyhow::Error; + + fn from_str(reference: &str) -> std::result::Result { + match reference.split_once('#') { + None => bail!( + "Reference must of the form `repo#number` but found {}", + reference + ), + Some((repo, number)) => { + let number = number.parse::()?; + if !matches!(repo, "smithy-rs" | "aws-sdk-rust") { + bail!("unexpected repo: {}", repo); + } + Ok(Reference { + number, + repo: repo.to_string(), + }) + } + } + } +} + +#[derive(Clone, Debug, Deserialize, Serialize)] +pub struct HandAuthoredEntry { + pub message: String, + pub meta: Meta, + pub author: String, + #[serde(default)] + pub references: Vec, + /// Optional commit hash to indicate "since when" these changes were made + #[serde(rename = "since-commit")] + pub since_commit: Option, +} + +impl HandAuthoredEntry { + /// Validate a changelog entry to ensure it follows standards + pub fn validate(&self) -> Result<()> { + if self.author.is_empty() { + bail!("Author must be set (was empty)"); + } + if !self.author.chars().all(|c| c.is_alphanumeric() || c == '-') { + bail!("Author must be valid GitHub username: [a-zA-Z0-9\\-]") + } + if self.references.is_empty() { + bail!("Changelog entry must refer to at least one pull request or issue"); + } + + Ok(()) + } +} + +#[derive(Clone, Debug, Deserialize, Serialize)] +pub enum SdkModelChangeKind { + Documentation, + Feature, +} + +#[derive(Clone, Debug, Deserialize, Serialize)] +pub struct SdkModelEntry { + /// SDK module name (e.g., "aws-sdk-s3" for S3) + pub module: String, + /// SDK module version number (e.g., "0.14.0") + pub version: String, + /// What changed + pub kind: SdkModelChangeKind, + /// More details about the change + pub message: String, +} + +#[derive(Clone, Default, Debug, Deserialize, Serialize)] +pub struct Changelog { + #[serde(rename = "smithy-rs")] + #[serde(default)] + pub smithy_rs: Vec, + #[serde(rename = "aws-sdk-rust")] + #[serde(default)] + pub aws_sdk_rust: Vec, + #[serde(rename = "aws-sdk-model")] + #[serde(default)] + pub sdk_models: Vec, +} + +impl Changelog { + pub fn new() -> Changelog { + Default::default() + } + + pub fn merge(&mut self, other: Changelog) { + self.smithy_rs.extend(other.smithy_rs.into_iter()); + self.aws_sdk_rust.extend(other.aws_sdk_rust.into_iter()); + self.sdk_models.extend(other.sdk_models.into_iter()); + } + + fn parse_str(value: &str) -> Result { + match toml::from_str(value).context("Invalid TOML changelog format") { + Ok(parsed) => Ok(parsed), + Err(toml_err) => { + // Remove comments from the top + let value = value + .split('\n') + .into_iter() + .filter(|line| !line.trim().starts_with('#')) + .collect::>() + .join("\n"); + match serde_json::from_str(&value).context("Invalid JSON changelog format") { + Ok(parsed) => Ok(parsed), + Err(json_err) => bail!( + "Invalid JSON or TOML changelog format:\n{:?}\n{:?}", + toml_err, + json_err + ), + } + } + } + } + + pub fn load_from_file(path: impl AsRef) -> Result { + let contents = std::fs::read_to_string(path.as_ref()) + .with_context(|| format!("failed to read {:?}", path.as_ref()))?; + Self::parse_str(&contents) + } + + pub fn to_json_string(&self) -> Result { + serde_json::to_string_pretty(self).context("failed to serialize changelog JSON") + } + + pub fn validate(&self) -> Result<(), Vec> { + let mut errors = vec![]; + for entry in self.aws_sdk_rust.iter().chain(self.smithy_rs.iter()) { + if let Err(e) = entry.validate() { + errors.push(format!("{}", e)); + } + } + if errors.is_empty() { + Ok(()) + } else { + Err(errors) + } + } +} + +#[cfg(test)] +mod tests { + use super::Changelog; + + #[test] + fn parse_json() { + let json = r#" + # Example changelog entries + # [[aws-sdk-rust]] + # message = "Fix typos in module documentation for generated crates" + # references = ["smithy-rs#920"] + # meta = { "breaking" = false, "tada" = false, "bug" = false } + # author = "rcoh" + # + # [[smithy-rs]] + # message = "Fix typos in module documentation for generated crates" + # references = ["smithy-rs#920"] + # meta = { "breaking" = false, "tada" = false, "bug" = false } + # author = "rcoh" + { + "smithy-rs": [], + "aws-sdk-rust": [ + { + "message": "Some change", + "meta": { "bug": true, "breaking": false, "tada": false }, + "author": "test-dev", + "references": [ + "aws-sdk-rust#123", + "smithy-rs#456" + ] + } + ], + "aws-sdk-model": [ + { + "module": "aws-sdk-ec2", + "version": "0.12.0", + "kind": "Feature", + "message": "Some API change" + } + ] + } + "#; + let changelog = Changelog::parse_str(json).unwrap(); + assert!(changelog.smithy_rs.is_empty()); + assert_eq!(1, changelog.aws_sdk_rust.len()); + assert_eq!("Some change", changelog.aws_sdk_rust[0].message); + assert_eq!(1, changelog.sdk_models.len()); + assert_eq!("Some API change", changelog.sdk_models[0].message); + } +} diff --git a/tools/smithy-rs-tool-common/src/git.rs b/tools/smithy-rs-tool-common/src/git.rs index 1733ee677..22b670e18 100644 --- a/tools/smithy-rs-tool-common/src/git.rs +++ b/tools/smithy-rs-tool-common/src/git.rs @@ -3,27 +3,28 @@ * SPDX-License-Identifier: Apache-2.0 */ -use crate::shell::ShellOperation; -use anyhow::Result; +use crate::shell::{handle_failure, output_text}; +use anyhow::{bail, Context, Result}; +use std::borrow::Cow; use std::ffi::OsStr; +use std::fmt; use std::path::{Path, PathBuf}; +use std::process::Command; +use tracing::debug; use tracing::warn; -mod get_current_tag; -pub use get_current_tag::GetCurrentTag; - -mod get_last_commit; -pub use get_last_commit::GetLastCommit; - -mod get_repo_root; -pub use get_repo_root::GetRepoRoot; - -mod reset; -pub use reset::Reset; - /// Attempts to find git repository root from the given location. pub fn find_git_repository_root(repo_name: &str, location: impl AsRef) -> Result { - let path = GetRepoRoot::new(location.as_ref()).run()?; + let output = Command::new("git") + .arg("rev-parse") + .arg("--show-toplevel") + .current_dir(location.as_ref()) + .output() + .context("failed to run git")?; + handle_failure("determine git repo root", &output)?; + + let (stdout, _) = output_text(&output); + let path = PathBuf::from(stdout.trim()); if path.file_name() != Some(OsStr::new(repo_name)) { warn!( "repository root {:?} doesn't have expected name '{}'", @@ -32,3 +33,587 @@ pub fn find_git_repository_root(repo_name: &str, location: impl AsRef) -> } Ok(path) } + +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct CommitHash(String); + +impl> From for CommitHash { + fn from(hash: T) -> Self { + CommitHash(hash.into()) + } +} + +impl AsRef for CommitHash { + fn as_ref(&self) -> &str { + &self.0 + } +} + +impl fmt::Display for CommitHash { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{}", self.0) + } +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct Commit { + pub hash: CommitHash, + pub author_name: String, + pub author_email: String, + pub message_subject: String, + pub message_body: String, +} + +impl Commit { + pub fn message(&self) -> Cow<'_, str> { + if self.message_body.is_empty() { + Cow::Borrowed(&self.message_subject) + } else { + Cow::Owned(format!("{}\n\n{}", self.message_subject, self.message_body)) + } + } +} + +/// Easily mockable interface with Git for testing +pub trait Git: Send + Sync { + /// Returns the repository path + fn path(&self) -> &Path; + + /// Clones the repository to the given path + fn clone_to(&self, path: &Path) -> Result<()>; + + /// Returns commit hash of HEAD (i.e., `git rev-parse HEAD`) + fn get_head_revision(&self) -> Result; + + /// Stages the given path (i.e., `git add ${path}`) + fn stage(&self, path: &Path) -> Result<()>; + + /// Commits the staged files on behalf of the given author using a bot commiter. + fn commit_on_behalf( + &self, + bot_name: &str, + bot_email: &str, + author_name: &str, + author_email: &str, + message: &str, + ) -> Result<()>; + + /// Commits staged files. + fn commit(&self, name: &str, email: &str, message: &str) -> Result<()>; + + /// Returns a list of commit hashes in reverse chronological order starting with + /// `start_inclusive_revision` and ending before `end_exclusive_revision`. Both of + /// these arguments can be any kind of Git revision (e.g., HEAD, HEAD~2, commit hash, etc). + fn rev_list<'a>( + &self, + start_inclusive_revision: &str, + end_exclusive_revision: &str, + path: Option<&'a Path>, + ) -> Result>; + + /// Returns information about a given revision. + fn show(&self, revision: &str) -> Result; + + /// Hard resets to the given revision. + fn hard_reset(&self, revision: &str) -> Result<()>; + + /// Returns the name of the current branch. + fn current_branch_name(&self) -> Result; + + /// Creates a branch at the given revision. + fn create_branch(&self, branch_name: &str, revision: &str) -> Result<()>; + + /// Deletes a branch. + fn delete_branch(&self, branch_name: &str) -> Result<()>; + + /// Squash merges a branch into the current branch and leaves the changes staged. + fn squash_merge(&self, author_name: &str, author_email: &str, branch_name: &str) -> Result<()>; + + /// Returns list of untracked files. + fn untracked_files(&self) -> Result>; + + /// Returns list of changed files. + fn changed_files(&self) -> Result>; +} + +enum CommitInfo { + CommitHash, + AuthorName, + AuthorEmail, + MessageSubject, + MessageBody, +} + +pub struct GitCLI { + repo_path: PathBuf, + binary_name: String, +} + +impl GitCLI { + pub fn new(repo_path: &Path) -> Result { + if !repo_path.join(".git").is_dir() { + bail!("{:?} is not a git repository", repo_path); + } + Ok(Self { + repo_path: repo_path.into(), + binary_name: "git".into(), + }) + } + + #[cfg(test)] + pub fn with_binary(repo_path: &Path, name: &str) -> Self { + Self { + repo_path: repo_path.into(), + binary_name: name.into(), + } + } + + fn extract_commit_info(&self, revision: &str, info: CommitInfo) -> Result { + let mut command = Command::new(&self.binary_name); + command.arg("show"); + command.arg("-s"); + command.arg(revision); + command.arg(format!( + "--format={}", + match info { + CommitInfo::CommitHash => "%H", + CommitInfo::AuthorName => "%an", + CommitInfo::AuthorEmail => "%ae", + CommitInfo::MessageSubject => "%s", + CommitInfo::MessageBody => "%b", + } + )); + command.current_dir(&self.repo_path); + + let output = log_command(command).output()?; + handle_failure("extract_commit_info", &output)?; + let (stdout, _) = output_text(&output); + Ok(stdout.trim().into()) + } +} + +impl Git for GitCLI { + fn path(&self) -> &Path { + &self.repo_path + } + + fn clone_to(&self, path: &Path) -> Result<()> { + let mut command = Command::new(&self.binary_name); + command.arg("clone"); + command.arg(&self.repo_path); + command.current_dir(path); + + let output = log_command(command).output()?; + handle_failure("clone_to", &output)?; + Ok(()) + } + + fn get_head_revision(&self) -> Result { + let mut command = Command::new(&self.binary_name); + command.arg("rev-parse"); + command.arg("HEAD"); + command.current_dir(&self.repo_path); + + let output = log_command(command).output()?; + handle_failure("get_head_revision", &output)?; + let (stdout, _) = output_text(&output); + Ok(CommitHash(stdout.trim().into())) + } + + fn stage(&self, path: &Path) -> Result<()> { + let mut command = Command::new(&self.binary_name); + command.arg("add"); + command.arg(path); + command.current_dir(&self.repo_path); + + let output = log_command(command).output()?; + handle_failure("stage", &output)?; + Ok(()) + } + + fn commit_on_behalf( + &self, + bot_name: &str, + bot_email: &str, + author_name: &str, + author_email: &str, + message: &str, + ) -> Result<()> { + let mut command = Command::new(&self.binary_name); + command.arg("-c"); + command.arg(format!("user.name={}", bot_name)); + command.arg("-c"); + command.arg(format!("user.email={}", bot_email)); + command.arg("commit"); + command.arg("-m"); + command.arg(message); + command.arg("--author"); + command.arg(format!("{} <{}>", author_name, author_email)); + command.current_dir(&self.repo_path); + + let output = log_command(command).output()?; + handle_failure("commit_on_behalf", &output)?; + Ok(()) + } + + fn commit(&self, name: &str, email: &str, message: &str) -> Result<()> { + let mut command = Command::new(&self.binary_name); + command.arg("-c"); + command.arg(format!("user.name={}", name)); + command.arg("-c"); + command.arg(format!("user.email={}", email)); + command.arg("commit"); + command.arg("-m"); + command.arg(message); + command.current_dir(&self.repo_path); + + let output = log_command(command).output()?; + handle_failure("commit", &output)?; + Ok(()) + } + + fn rev_list( + &self, + start_inclusive_revision: &str, + end_exclusive_revision: &str, + path: Option<&Path>, + ) -> Result> { + let mut command = Command::new(&self.binary_name); + command.arg("rev-list"); + command.arg(format!( + "{}..{}", + end_exclusive_revision, start_inclusive_revision + )); + if let Some(path) = path { + command.arg("--"); + command.arg(path); + } + command.current_dir(&self.repo_path); + + let output = log_command(command).output()?; + handle_failure("rev_list", &output)?; + let (stdout, _) = output_text(&output); + Ok(stdout + .split_ascii_whitespace() + .into_iter() + .map(CommitHash::from) + .collect()) + } + + fn show(&self, revision: &str) -> Result { + Ok(Commit { + hash: CommitHash::from(self.extract_commit_info(revision, CommitInfo::CommitHash)?), + author_name: self.extract_commit_info(revision, CommitInfo::AuthorName)?, + author_email: self.extract_commit_info(revision, CommitInfo::AuthorEmail)?, + message_subject: self.extract_commit_info(revision, CommitInfo::MessageSubject)?, + message_body: self.extract_commit_info(revision, CommitInfo::MessageBody)?, + }) + } + + fn hard_reset(&self, revision: &str) -> Result<()> { + let mut command = Command::new(&self.binary_name); + command.arg("reset"); + command.arg("--hard"); + command.arg(revision); + command.current_dir(&self.repo_path); + + let output = log_command(command).output()?; + handle_failure("hard_reset", &output)?; + Ok(()) + } + + fn current_branch_name(&self) -> Result { + let mut command = Command::new(&self.binary_name); + command.arg("rev-parse"); + command.arg("--abbrev-ref"); + command.arg("HEAD"); + command.current_dir(&self.repo_path); + + let output = log_command(command).output()?; + handle_failure("current_branch_name", &output)?; + let (stdout, _) = output_text(&output); + Ok(stdout.trim().into()) + } + + fn create_branch(&self, branch_name: &str, revision: &str) -> Result<()> { + let mut command = Command::new(&self.binary_name); + command.arg("branch"); + command.arg(branch_name); + command.arg(revision); + command.current_dir(&self.repo_path); + + let output = log_command(command).output()?; + handle_failure("create_branch", &output)?; + Ok(()) + } + + fn delete_branch(&self, branch_name: &str) -> Result<()> { + let mut command = Command::new(&self.binary_name); + command.arg("branch"); + command.arg("-D"); + command.arg(branch_name); + command.current_dir(&self.repo_path); + + let output = log_command(command).output()?; + handle_failure("delete_branch", &output)?; + Ok(()) + } + + fn squash_merge(&self, author_name: &str, author_email: &str, branch_name: &str) -> Result<()> { + let mut command = Command::new(&self.binary_name); + command.arg("-c"); + command.arg(format!("user.name={}", author_name)); + command.arg("-c"); + command.arg(format!("user.email={}", author_email)); + command.arg("merge"); + command.arg("--squash"); + command.arg(branch_name); + command.current_dir(&self.repo_path); + + let output = log_command(command).output()?; + handle_failure("squash_merge", &output) + } + + fn untracked_files(&self) -> Result> { + let mut command = Command::new(&self.binary_name); + command.arg("ls-files"); + command.arg("--exclude-standard"); + command.arg("--others"); + command.current_dir(&self.repo_path); + + let output = log_command(command).output()?; + handle_failure("untracked_files", &output)?; + let (stdout, _) = output_text(&output); + Ok(split_file_names(&stdout)) + } + + fn changed_files(&self) -> Result> { + let mut command = Command::new(&self.binary_name); + command.arg("diff"); + command.arg("--name-only"); + command.current_dir(&self.repo_path); + + let output = log_command(command).output()?; + handle_failure("changed_files", &output)?; + let (stdout, _) = output_text(&output); + Ok(split_file_names(&stdout)) + } +} + +fn is_newline(c: char) -> bool { + c == '\r' || c == '\n' +} + +fn split_file_names(value: &str) -> Vec { + value + .split(is_newline) + .filter(|s| !s.is_empty()) + .map(PathBuf::from) + .collect::>() +} + +fn log_command(command: Command) -> Command { + let mut message = String::new(); + if let Some(cwd) = command.get_current_dir() { + message.push_str(&format!("[in {:?}]: ", cwd)); + } + message.push_str(command.get_program().to_str().expect("valid str")); + for arg in command.get_args() { + message.push(' '); + message.push_str(arg.to_str().expect("valid str")); + } + debug!("{}", message); + command +} + +#[cfg(test)] +mod tests { + use super::*; + use std::env; + + fn bin_path(script: &'static str) -> PathBuf { + env::current_dir() + .expect("current_dir") + .join("fake-cli") + .join(script) + .canonicalize() + .expect("canonicalize") + } + fn cli(script: &'static str) -> GitCLI { + GitCLI::with_binary(&PathBuf::from("/tmp"), &bin_path(script).to_string_lossy()) + } + + #[test] + fn clone_to() { + cli("git-clone") + .clone_to(&PathBuf::from("/tmp")) + .expect("successful invocation"); + } + + #[test] + fn extract_commit_info() { + let result = cli("git-extract-commit-info") + .extract_commit_info("test_revision", CommitInfo::CommitHash) + .expect("successful invocation"); + assert_eq!("success", result); + } + + #[test] + fn changed_files() { + assert_eq!( + vec![ + PathBuf::from("some/file"), + PathBuf::from("some-other-file"), + PathBuf::from("some/file with spaces.txt"), + ], + cli("git-changed-files") + .changed_files() + .expect("successful invocation") + ); + assert_eq!( + Vec::::new(), + cli("git-changed-files-empty") + .changed_files() + .expect("successful invocation") + ); + } + + #[test] + fn untracked_files() { + assert_eq!( + vec![ + PathBuf::from("some-untracked-file"), + PathBuf::from("another-untracked-file"), + PathBuf::from("some/file with spaces.txt"), + ], + cli("git-untracked-files") + .untracked_files() + .expect("successful invocation") + ); + assert_eq!( + Vec::::new(), + cli("git-untracked-files-empty") + .untracked_files() + .expect("successful invocation") + ); + } + + #[test] + fn get_head_revision() { + assert_eq!( + "some-commit-hash", + cli("git-get-head-revision") + .get_head_revision() + .expect("successful invocation") + .as_ref() + ); + } + + #[test] + fn stage() { + cli("git-stage") + .stage(&PathBuf::from("test-path")) + .expect("successful invocation"); + } + + #[test] + fn commit_on_behalf() { + cli("git-commit-on-behalf") + .commit_on_behalf( + "Bot Name", + "bot@example.com", + "Some Author", + "author@example.com", + "Test message", + ) + .expect("successful invocation"); + } + + #[test] + fn commit() { + cli("git-commit") + .commit("Some Author", "author@example.com", "Test message") + .expect("successful invocation"); + } + + #[test] + fn rev_list() { + assert_eq!( + vec![ + CommitHash::from("second-commit"), + CommitHash::from("initial-commit") + ], + cli("git-rev-list") + .rev_list("start_inclusive", "end_exclusive", None) + .expect("successful invocation") + ); + assert_eq!( + vec![ + CommitHash::from("third-commit"), + CommitHash::from("second-commit"), + CommitHash::from("initial-commit") + ], + cli("git-rev-list-path") + .rev_list( + "start_inclusive", + "end_exclusive", + Some(&PathBuf::from("some-path")) + ) + .expect("successful invocation") + ); + } + + #[test] + fn show() { + assert_eq!( + Commit { + hash: "some-commit-hash".into(), + author_name: "Some Author".into(), + author_email: "author@example.com".into(), + message_subject: "Some message subject".into(), + message_body: "Message body\n with multiple lines".into() + }, + cli("git-show") + .show("test_revision") + .expect("successful invocation") + ); + } + + #[test] + fn hard_reset() { + cli("git-reset-hard") + .hard_reset("some-revision") + .expect("successful invocation"); + } + + #[test] + fn current_branch_name() { + assert_eq!( + "some-branch-name", + cli("git-current-branch-name") + .current_branch_name() + .expect("successful invocation") + ); + } + + #[test] + fn create_branch() { + cli("git-create-branch") + .create_branch("test-branch-name", "test-revision") + .expect("successful invocation"); + } + + #[test] + fn delete_branch() { + cli("git-delete-branch") + .delete_branch("test-branch-name") + .expect("successful invocation"); + } + + #[test] + fn squash_merge() { + cli("git-squash-merge") + .squash_merge("some-dev", "some-email@example.com", "test-branch-name") + .expect("successful invocation"); + } +} diff --git a/tools/smithy-rs-tool-common/src/git/get_repo_root.rs b/tools/smithy-rs-tool-common/src/git/get_repo_root.rs deleted file mode 100644 index f2c46f7b4..000000000 --- a/tools/smithy-rs-tool-common/src/git/get_repo_root.rs +++ /dev/null @@ -1,73 +0,0 @@ -/* - * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. - * SPDX-License-Identifier: Apache-2.0 - */ - -use crate::shell::{handle_failure, output_text, ShellOperation}; -use anyhow::Result; -use std::path::PathBuf; -use std::process::Command; - -pub struct GetRepoRoot { - program: &'static str, - start_path: PathBuf, -} - -impl GetRepoRoot { - pub fn new(start_path: impl Into) -> GetRepoRoot { - GetRepoRoot { - program: "git", - start_path: start_path.into(), - } - } -} - -impl ShellOperation for GetRepoRoot { - type Output = PathBuf; - - fn run(&self) -> Result { - let mut command = Command::new(self.program); - command.arg("rev-parse"); - command.arg("--show-toplevel"); - command.current_dir(&self.start_path); - - let output = command.output()?; - handle_failure("determine git repo root", &output)?; - let (stdout, _) = output_text(&output); - Ok(stdout.trim().into()) - } -} - -#[cfg(all(test, not(target_os = "windows")))] -mod tests { - use super::*; - - #[test] - fn get_repo_root_success() { - let last_commit = GetRepoRoot { - program: "./git_revparse_show_toplevel", - start_path: "./fake_git".into(), - } - .run() - .unwrap(); - assert_eq!(PathBuf::from("/git/repo/root/path"), last_commit); - } - - #[test] - fn get_repo_root_failure() { - let result = GetRepoRoot { - program: "./git_fails", - start_path: "./fake_git".into(), - } - .run(); - - assert!(result.is_err(), "expected error, got {:?}", result); - assert_eq!( - "Failed to determine git repo root:\n\ - Status: 1\n\ - Stdout: some stdout failure message\n\n\ - Stderr: some stderr failure message\n\n", - format!("{}", result.err().unwrap()) - ); - } -} diff --git a/tools/smithy-rs-tool-common/src/lib.rs b/tools/smithy-rs-tool-common/src/lib.rs index 1155f4080..b58a3517f 100644 --- a/tools/smithy-rs-tool-common/src/lib.rs +++ b/tools/smithy-rs-tool-common/src/lib.rs @@ -3,6 +3,7 @@ * SPDX-License-Identifier: Apache-2.0 */ +pub mod changelog; pub mod ci; pub mod git; #[macro_use] -- GitLab