Unverified Commit 5e20575e authored by John DiSanti's avatar John DiSanti Committed by GitHub
Browse files

Make SDK runtime patch tool that supports SDK runtime crates (#3369) 

A tool to patch in new versions of smithy-rs runtime crates was
introduced in #3327 so that semver breaks could better be detected for
upcoming releases. However, this tool didn't support patching in SDK
runtime crates, which is necessary for testing #3355.

This PR moves the tool into the runtime-versioner tool, and adjusts it
to support both smithy-rs and SDK runtime crates.

----

_By submitting this pull request, I confirm that you can use, modify,
copy, and redistribute this contribution, under the terms of your
choice._
parent 4c256ef0

tools/ci-build/runtime-release-dryrun/Cargo.lock

deleted100644 → 0
+0 −353
Original line number Diff line number Diff line 
# This file is automatically @generated by Cargo.

# It is not intended for manual editing.

version = 3



[[package]]

name = "anyhow"

version = "1.0.75"

source = "registry+https://github.com/rust-lang/crates.io-index"

checksum = "a4668cab20f66d8d020e1fbc0ebe47217433c1b6c8f2040faf858554e394ace6"



[[package]]

name = "atty"

version = "0.2.14"

source = "registry+https://github.com/rust-lang/crates.io-index"

checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8"

dependencies = [

 "hermit-abi",

 "libc",

 "winapi",

]



[[package]]

name = "bitflags"

version = "1.3.2"

source = "registry+https://github.com/rust-lang/crates.io-index"

checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a"



[[package]]

name = "cfg-if"

version = "1.0.0"

source = "registry+https://github.com/rust-lang/crates.io-index"

checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"



[[package]]

name = "clap"

version = "4.0.8"

source = "registry+https://github.com/rust-lang/crates.io-index"

checksum = "5840cd9093aabeabf7fd932754c435b7674520fc3ddc935c397837050f0f1e4b"

dependencies = [

 "atty",

 "bitflags",

 "clap_derive",

 "clap_lex",

 "once_cell",

 "strsim",

 "termcolor",

]



[[package]]

name = "clap_derive"

version = "4.0.8"

source = "registry+https://github.com/rust-lang/crates.io-index"

checksum = "92289ffc6fb4a85d85c246ddb874c05a87a2e540fb6ad52f7ca07c8c1e1840b1"

dependencies = [

 "heck",

 "proc-macro-error",

 "proc-macro2",

 "quote",

 "syn",

]



[[package]]

name = "clap_lex"

version = "0.3.3"

source = "registry+https://github.com/rust-lang/crates.io-index"

checksum = "033f6b7a4acb1f358c742aaca805c939ee73b4c6209ae4318ec7aca81c42e646"

dependencies = [

 "os_str_bytes",

]



[[package]]

name = "console"

version = "0.15.7"

source = "registry+https://github.com/rust-lang/crates.io-index"

checksum = "c926e00cc70edefdc64d3a5ff31cc65bb97a3460097762bd23afb4d8145fccf8"

dependencies = [

 "encode_unicode",

 "lazy_static",

 "libc",

 "unicode-width",

 "windows-sys",

]



[[package]]

name = "encode_unicode"

version = "0.3.6"

source = "registry+https://github.com/rust-lang/crates.io-index"

checksum = "a357d28ed41a50f9c765dbfe56cbc04a64e53e5fc58ba79fbc34c10ef3df831f"



[[package]]

name = "heck"

version = "0.4.1"

source = "registry+https://github.com/rust-lang/crates.io-index"

checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8"



[[package]]

name = "hermit-abi"

version = "0.1.19"

source = "registry+https://github.com/rust-lang/crates.io-index"

checksum = "62b467343b94ba476dcb2500d242dadbb39557df889310ac77c5d99100aaac33"

dependencies = [

 "libc",

]



[[package]]

name = "indicatif"

version = "0.17.7"

source = "registry+https://github.com/rust-lang/crates.io-index"

checksum = "fb28741c9db9a713d93deb3bb9515c20788cef5815265bee4980e87bde7e0f25"

dependencies = [

 "console",

 "instant",

 "number_prefix",

 "portable-atomic",

 "unicode-width",

]



[[package]]

name = "instant"

version = "0.1.12"

source = "registry+https://github.com/rust-lang/crates.io-index"

checksum = "7a5bbe824c507c5da5956355e86a746d82e0e1464f65d862cc5e71da70e94b2c"

dependencies = [

 "cfg-if",

]



[[package]]

name = "lazy_static"

version = "1.4.0"

source = "registry+https://github.com/rust-lang/crates.io-index"

checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646"



[[package]]

name = "libc"

version = "0.2.151"

source = "registry+https://github.com/rust-lang/crates.io-index"

checksum = "302d7ab3130588088d277783b1e2d2e10c9e9e4a16dd9050e6ec93fb3e7048f4"



[[package]]

name = "number_prefix"

version = "0.4.0"

source = "registry+https://github.com/rust-lang/crates.io-index"

checksum = "830b246a0e5f20af87141b25c173cd1b609bd7779a4617d6ec582abaf90870f3"



[[package]]

name = "once_cell"

version = "1.19.0"

source = "registry+https://github.com/rust-lang/crates.io-index"

checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92"



[[package]]

name = "os_str_bytes"

version = "6.6.1"

source = "registry+https://github.com/rust-lang/crates.io-index"

checksum = "e2355d85b9a3786f481747ced0e0ff2ba35213a1f9bd406ed906554d7af805a1"



[[package]]

name = "portable-atomic"

version = "1.6.0"

source = "registry+https://github.com/rust-lang/crates.io-index"

checksum = "7170ef9988bc169ba16dd36a7fa041e5c4cbeb6a35b76d4c03daded371eae7c0"



[[package]]

name = "proc-macro-error"

version = "1.0.4"

source = "registry+https://github.com/rust-lang/crates.io-index"

checksum = "da25490ff9892aab3fcf7c36f08cfb902dd3e71ca0f9f9517bea02a73a5ce38c"

dependencies = [

 "proc-macro-error-attr",

 "proc-macro2",

 "quote",

 "syn",

 "version_check",

]



[[package]]

name = "proc-macro-error-attr"

version = "1.0.4"

source = "registry+https://github.com/rust-lang/crates.io-index"

checksum = "a1be40180e52ecc98ad80b184934baf3d0d29f979574e439af5a55274b35f869"

dependencies = [

 "proc-macro2",

 "quote",

 "version_check",

]



[[package]]

name = "proc-macro2"

version = "1.0.70"

source = "registry+https://github.com/rust-lang/crates.io-index"

checksum = "39278fbbf5fb4f646ce651690877f89d1c5811a3d4acb27700c1cb3cdb78fd3b"

dependencies = [

 "unicode-ident",

]



[[package]]

name = "quote"

version = "1.0.33"

source = "registry+https://github.com/rust-lang/crates.io-index"

checksum = "5267fca4496028628a95160fc423a33e8b2e6af8a5302579e322e4b520293cae"

dependencies = [

 "proc-macro2",

]



[[package]]

name = "runtime-release-dryrun"

version = "0.1.0"

dependencies = [

 "anyhow",

 "clap",

 "indicatif",

]



[[package]]

name = "strsim"

version = "0.10.0"

source = "registry+https://github.com/rust-lang/crates.io-index"

checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623"



[[package]]

name = "syn"

version = "1.0.109"

source = "registry+https://github.com/rust-lang/crates.io-index"

checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237"

dependencies = [

 "proc-macro2",

 "quote",

 "unicode-ident",

]



[[package]]

name = "termcolor"

version = "1.4.0"

source = "registry+https://github.com/rust-lang/crates.io-index"

checksum = "ff1bc3d3f05aff0403e8ac0d92ced918ec05b666a43f83297ccef5bea8a3d449"

dependencies = [

 "winapi-util",

]



[[package]]

name = "unicode-ident"

version = "1.0.12"

source = "registry+https://github.com/rust-lang/crates.io-index"

checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b"



[[package]]

name = "unicode-width"

version = "0.1.11"

source = "registry+https://github.com/rust-lang/crates.io-index"

checksum = "e51733f11c9c4f72aa0c160008246859e340b00807569a0da0e7a1079b27ba85"



[[package]]

name = "version_check"

version = "0.9.4"

source = "registry+https://github.com/rust-lang/crates.io-index"

checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f"



[[package]]

name = "winapi"

version = "0.3.9"

source = "registry+https://github.com/rust-lang/crates.io-index"

checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419"

dependencies = [

 "winapi-i686-pc-windows-gnu",

 "winapi-x86_64-pc-windows-gnu",

]



[[package]]

name = "winapi-i686-pc-windows-gnu"

version = "0.4.0"

source = "registry+https://github.com/rust-lang/crates.io-index"

checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6"



[[package]]

name = "winapi-util"

version = "0.1.6"

source = "registry+https://github.com/rust-lang/crates.io-index"

checksum = "f29e6f9198ba0d26b4c9f07dbe6f9ed633e1f3d5b8b414090084349e46a52596"

dependencies = [

 "winapi",

]



[[package]]

name = "winapi-x86_64-pc-windows-gnu"

version = "0.4.0"

source = "registry+https://github.com/rust-lang/crates.io-index"

checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"



[[package]]

name = "windows-sys"

version = "0.45.0"

source = "registry+https://github.com/rust-lang/crates.io-index"

checksum = "75283be5efb2831d37ea142365f009c02ec203cd29a3ebecbc093d52315b66d0"

dependencies = [

 "windows-targets",

]



[[package]]

name = "windows-targets"

version = "0.42.2"

source = "registry+https://github.com/rust-lang/crates.io-index"

checksum = "8e5180c00cd44c9b1c88adb3693291f1cd93605ded80c250a75d472756b4d071"

dependencies = [

 "windows_aarch64_gnullvm",

 "windows_aarch64_msvc",

 "windows_i686_gnu",

 "windows_i686_msvc",

 "windows_x86_64_gnu",

 "windows_x86_64_gnullvm",

 "windows_x86_64_msvc",

]



[[package]]

name = "windows_aarch64_gnullvm"

version = "0.42.2"

source = "registry+https://github.com/rust-lang/crates.io-index"

checksum = "597a5118570b68bc08d8d59125332c54f1ba9d9adeedeef5b99b02ba2b0698f8"



[[package]]

name = "windows_aarch64_msvc"

version = "0.42.2"

source = "registry+https://github.com/rust-lang/crates.io-index"

checksum = "e08e8864a60f06ef0d0ff4ba04124db8b0fb3be5776a5cd47641e942e58c4d43"



[[package]]

name = "windows_i686_gnu"

version = "0.42.2"

source = "registry+https://github.com/rust-lang/crates.io-index"

checksum = "c61d927d8da41da96a81f029489353e68739737d3beca43145c8afec9a31a84f"



[[package]]

name = "windows_i686_msvc"

version = "0.42.2"

source = "registry+https://github.com/rust-lang/crates.io-index"

checksum = "44d840b6ec649f480a41c8d80f9c65108b92d89345dd94027bfe06ac444d1060"



[[package]]

name = "windows_x86_64_gnu"

version = "0.42.2"

source = "registry+https://github.com/rust-lang/crates.io-index"

checksum = "8de912b8b8feb55c064867cf047dda097f92d51efad5b491dfb98f6bbb70cb36"



[[package]]

name = "windows_x86_64_gnullvm"

version = "0.42.2"

source = "registry+https://github.com/rust-lang/crates.io-index"

checksum = "26d41b46a36d453748aedef1486d5c7a85db22e56aff34643984ea85514e94a3"



[[package]]

name = "windows_x86_64_msvc"

version = "0.42.2"

source = "registry+https://github.com/rust-lang/crates.io-index"

checksum = "9aec5da331524158c6d1a4ac0ab1541149c0b9505fde06423b02f5ef0106b9f0"

tools/ci-build/runtime-release-dryrun/Cargo.toml

deleted100644 → 0
+0 −11
Original line number Diff line number Diff line 
[package]

name = "runtime-release-dryrun"

version = "0.1.0"

edition = "2021"



# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html



[dependencies]

clap = { version = "4", features = ["derive"] }

anyhow = "1.0.75"

indicatif = "0.17.7"

tools/ci-build/runtime-release-dryrun/src/main.rs

deleted100644 → 0
+0 −182
Original line number Diff line number Diff line 
/*

 * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.

 * SPDX-License-Identifier: Apache-2.0

 */



// this tool is simple and works at what it does

// potential improvements:

// - support the release of rust-runtime crates

// - support patching the users system-wide ~/.cargo/config.toml



use std::path::{Path, PathBuf};

use std::process::Command;

use std::time::Duration;



use anyhow::{bail, Context, Result};

use clap::Parser;

use indicatif::{ProgressBar, ProgressStyle};



#[derive(Parser, Debug)]

struct DryRunSdk {

    /// Path to the aws-sdk-rust repo

    #[clap(long)]

    pub sdk_path: PathBuf,



    /// Tag of the aws-sdk-rust repo to dry-run against

    #[clap(long)]

    pub rust_sdk_tag: String,



    /// Path to the artifact produced by the release-dry run

    #[clap(long)]

    pub smithy_rs_release: PathBuf,

}



#[derive(Parser, Debug)]

#[clap(

    name = "runtime-release-dryrun",

    about = "CLI tool to prepare the aws-sdk-rust to test the result of releasing a new set of runtime crates.",

    version

)]

#[allow(clippy::enum_variant_names)] // Want the "use" prefix in the CLI subcommand names for clarity

enum Args {

    /// Dry run a smithy-rs release against a rust SDK release

    ///

    /// You will need:

    /// 1. An `aws-sdk-rust` repo with a clean working tree

    /// 2. A directory containing the artifacts from a smithy-rs release dry run. This is an artifact

    ///    named `artifacts-generate-smithy-rs-release` from the GH action (e.g. https://github.com/smithy-lang/smithy-rs/actions/runs/7200898068)

    /// 3. An `aws-sdk-rust` release tag you want to test against

    ///

    /// After running the tool, you might want to do something like `cargo test` in `s3`. Make sure

    /// to run `cargo update` to pull in the new dependencies. Use `cargo tree` to confirm you're

    /// actually consuming the new versions.

    #[clap(verbatim_doc_comment)]

    DryRunSdk(DryRunSdk),

}



fn main() -> Result<()> {

    let args = Args::parse();

    match args {

        Args::DryRunSdk(args) => dry_run_sdk(args).map_err(|err| {

            // workaround an indicatif (bug?) where one character is stripped from output on the error message

            eprintln!(" ");

            err

        })?,

    }

    Ok(())

}



fn step<T>(message: &'static str, step: impl FnOnce() -> Result<T>) -> Result<T> {

    let spinner = ProgressBar::new_spinner()

        .with_message(message)

        .with_style(ProgressStyle::with_template("{spinner} {msg} {elapsed}").unwrap());

    spinner.enable_steady_tick(Duration::from_millis(100));

    let result = step();

    let check = match &result {

        Ok(_) => "✅",

        Err(_) => "❌",

    };

    spinner.set_style(ProgressStyle::with_template("{msg} {elapsed}").unwrap());

    spinner.finish_with_message(format!("{check} {message}"));

    result

}



fn run(command: &mut Command) -> anyhow::Result<()> {

    let status = command.output()?;

    if !status.status.success() {

        bail!(

            "command `{:?}` failed:\n{}{}",

            command,

            String::from_utf8_lossy(&status.stdout),

            String::from_utf8_lossy(&status.stderr)

        );

    }

    Ok(())

}



fn dry_run_sdk(args: DryRunSdk) -> Result<()> {

    step("Checking out SDK tag", || {

        run(Command::new("git")

            .arg("checkout")

            .arg(&args.rust_sdk_tag)

            .current_dir(&args.sdk_path))

        .context("failed to checkout aws-sdk-rust revision")?;

        Ok(())

    })?;



    // By default the SDK dependencies also include a path component. This prevents

    // patching from working

    step("Applying version-only dependencies", || {

        run(Command::new("sdk-versioner")

            .args([

                "use-version-dependencies",

                "--versions-toml",

                "versions.toml",

                "sdk",

            ])

            .current_dir(&args.sdk_path))?;



        run(Command::new("git")

            .args(["checkout", "-B", "smithy-release-dryrun"])

            .current_dir(&args.sdk_path))?;

        run(Command::new("git")

            .args([

                "commit",

                "-am",

                "removing path dependencies to allow patching",

            ])

            .current_dir(&args.sdk_path))?;

        Ok(())

    })?;



    let patches = step("computing patches", || {

        let path = Path::new(&args.smithy_rs_release).join("crates-to-publish");

        let crates_to_patch = std::fs::read_dir(&path)

            .context(format!("could list crates in directory {:?}", path))?

            .map(|dir| dir.unwrap().file_name())

            .map(|osstr| osstr.into_string().expect("invalid utf-8 directory"))

            .collect::<Vec<_>>();



        let patch_sections = crates_to_patch

            .iter()

            .map(|crte| {

                let path = Path::new(&args.smithy_rs_release)

                    .join("crates-to-publish")

                    .join(crte);

                assert!(

                    path.exists(),

                    "tried to reference a crate that did not exist!"

                );

                format!(

                    "{crte} = {{ path = '{}' }}",

                    path.canonicalize().unwrap().to_str().unwrap()

                )

            })

            .collect::<Vec<_>>()

            .join("\n");

        Ok(format!("[patch.crates-io]\n{patch_sections}"))

    })?;



    // Note: in the future we could also automatically apply this to the system wide ~/.cargo/config.toml

    step("apply patches to workspace Cargo.toml", || {

        let workspace_cargo_toml = Path::new(&args.sdk_path).join("Cargo.toml");

        if !workspace_cargo_toml.exists() {

            bail!(

                "Could not find the workspace Cargo.toml to patch {:?}",

                workspace_cargo_toml

            );

        }

        let current_contents = std::fs::read_to_string(&workspace_cargo_toml)

            .context("could not read workspace cargo.toml")?;

        std::fs::write(

            workspace_cargo_toml,

            format!("{current_contents}\n{patches}"),

        )?;

        run(Command::new("git")

            .args(["commit", "-am", "patching workspace Cargo.toml"])

            .current_dir(&args.sdk_path))?;

        Ok(())

    })?;

    println!("{:?} has been updated to build against patches. Use `cargo update` to recompute the dependencies.", &args.sdk_path);

    Ok(())

}

tools/ci-build/runtime-versioner/Cargo.lock

+60 −0
Original line number Diff line number Diff line
@@ -209,6 +209,19 @@ version = "1.0.0" 
source = "registry+https://github.com/rust-lang/crates.io-index"

checksum = "acbf1af155f9b9ef647e42cdc158db4b64a1b61f743629225fde6f3e0be2a7c7"



[[package]]

name = "console"

version = "0.15.8"

source = "registry+https://github.com/rust-lang/crates.io-index"

checksum = "0e1f83fc076bd6dd27517eacdf25fef6c4dfe5f1d7448bafaaf3a26f13b5e4eb"

dependencies = [

 "encode_unicode",

 "lazy_static",

 "libc",

 "unicode-width",

 "windows-sys 0.52.0",

]



[[package]]

name = "core-foundation"

version = "0.9.4"
@@ -245,6 +258,12 @@ dependencies = [ 
 "toml 0.8.8",

]



[[package]]

name = "encode_unicode"

version = "0.3.6"

source = "registry+https://github.com/rust-lang/crates.io-index"

checksum = "a357d28ed41a50f9c765dbfe56cbc04a64e53e5fc58ba79fbc34c10ef3df831f"



[[package]]

name = "encoding_rs"

version = "0.8.33"
@@ -522,6 +541,28 @@ dependencies = [ 
 "hashbrown 0.14.3",

]



[[package]]

name = "indicatif"

version = "0.17.7"

source = "registry+https://github.com/rust-lang/crates.io-index"

checksum = "fb28741c9db9a713d93deb3bb9515c20788cef5815265bee4980e87bde7e0f25"

dependencies = [

 "console",

 "instant",

 "number_prefix",

 "portable-atomic",

 "unicode-width",

]



[[package]]

name = "instant"

version = "0.1.12"

source = "registry+https://github.com/rust-lang/crates.io-index"

checksum = "7a5bbe824c507c5da5956355e86a746d82e0e1464f65d862cc5e71da70e94b2c"

dependencies = [

 "cfg-if",

]



[[package]]

name = "ipnet"

version = "2.9.0"
@@ -646,6 +687,12 @@ dependencies = [ 
 "libc",

]



[[package]]

name = "number_prefix"

version = "0.4.0"

source = "registry+https://github.com/rust-lang/crates.io-index"

checksum = "830b246a0e5f20af87141b25c173cd1b609bd7779a4617d6ec582abaf90870f3"



[[package]]

name = "object"

version = "0.32.1"
@@ -735,6 +782,12 @@ version = "0.3.27" 
source = "registry+https://github.com/rust-lang/crates.io-index"

checksum = "26072860ba924cbfa98ea39c8c19b4dd6a4a25423dbdf219c1eca91aa0cf6964"



[[package]]

name = "portable-atomic"

version = "1.6.0"

source = "registry+https://github.com/rust-lang/crates.io-index"

checksum = "7170ef9988bc169ba16dd36a7fa041e5c4cbeb6a35b76d4c03daded371eae7c0"



[[package]]

name = "proc-macro2"

version = "1.0.76"
@@ -852,6 +905,7 @@ dependencies = [ 
 "camino",

 "clap",

 "crates-index",

 "indicatif",

 "reqwest",

 "smithy-rs-tool-common",

 "tempfile",
@@ -1341,6 +1395,12 @@ dependencies = [ 
 "tinyvec",

]



[[package]]

name = "unicode-width"

version = "0.1.11"

source = "registry+https://github.com/rust-lang/crates.io-index"

checksum = "e51733f11c9c4f72aa0c160008246859e340b00807569a0da0e7a1079b27ba85"



[[package]]

name = "url"

version = "2.5.0"

tools/ci-build/runtime-versioner/Cargo.toml

+1 −0
Original line number Diff line number Diff line
@@ -18,6 +18,7 @@ anyhow = "1.0.75" 
camino = "1.1.6"

clap = { version = "4.4.11", features = ["derive"] }

crates-index = "2.3.0"

indicatif = "0.17.7"

reqwest = { version = "0.11.22", features = ["blocking"] }

smithy-rs-tool-common = { version = "0.1", path = "../smithy-rs-tool-common" }

tempfile = "3.9.0"