Unverified Commit da214d90 authored by John DiSanti's avatar John DiSanti Committed by GitHub
Browse files

Add tool for bulk updating references to the AWS Rust SDK (#1080)

parent fd6788ff

tools/sdk-versioner/Cargo.lock

0 → 100644
+366 −0
Original line number Diff line number Diff line 
# This file is automatically @generated by Cargo.

# It is not intended for manual editing.

version = 3



[[package]]

name = "ansi_term"

version = "0.12.1"

source = "registry+https://github.com/rust-lang/crates.io-index"

checksum = "d52a9bb7ec0cf484c551830a7ce27bd20d67eac647e1befb56b0be4ee39a55d2"

dependencies = [

 "winapi",

]



[[package]]

name = "anyhow"

version = "1.0.52"

source = "registry+https://github.com/rust-lang/crates.io-index"

checksum = "84450d0b4a8bd1ba4144ce8ce718fbc5d071358b1e5384bace6536b3d1f2d5b3"



[[package]]

name = "atty"

version = "0.2.14"

source = "registry+https://github.com/rust-lang/crates.io-index"

checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8"

dependencies = [

 "hermit-abi",

 "libc",

 "winapi",

]



[[package]]

name = "autocfg"

version = "1.0.1"

source = "registry+https://github.com/rust-lang/crates.io-index"

checksum = "cdb031dd78e28731d87d56cc8ffef4a8f36ca26c38fe2de700543e627f8a464a"



[[package]]

name = "bitflags"

version = "1.3.2"

source = "registry+https://github.com/rust-lang/crates.io-index"

checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a"



[[package]]

name = "cfg-if"

version = "1.0.0"

source = "registry+https://github.com/rust-lang/crates.io-index"

checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"



[[package]]

name = "clap"

version = "2.34.0"

source = "registry+https://github.com/rust-lang/crates.io-index"

checksum = "a0610544180c38b88101fecf2dd634b174a62eef6946f84dfc6a7127512b381c"

dependencies = [

 "ansi_term",

 "atty",

 "bitflags",

 "strsim",

 "textwrap",

 "unicode-width",

 "vec_map",

]



[[package]]

name = "ctor"

version = "0.1.21"

source = "registry+https://github.com/rust-lang/crates.io-index"

checksum = "ccc0a48a9b826acdf4028595adc9db92caea352f7af011a3034acd172a52a0aa"

dependencies = [

 "quote",

 "syn",

]



[[package]]

name = "diff"

version = "0.1.12"

source = "registry+https://github.com/rust-lang/crates.io-index"

checksum = "0e25ea47919b1560c4e3b7fe0aaab9becf5b84a10325ddf7db0f0ba5e1026499"



[[package]]

name = "fastrand"

version = "1.6.0"

source = "registry+https://github.com/rust-lang/crates.io-index"

checksum = "779d043b6a0b90cc4c0ed7ee380a6504394cee7efd7db050e3774eee387324b2"

dependencies = [

 "instant",

]



[[package]]

name = "hashbrown"

version = "0.11.2"

source = "registry+https://github.com/rust-lang/crates.io-index"

checksum = "ab5ef0d4909ef3724cc8cce6ccc8572c5c817592e9285f5464f8e86f8bd3726e"



[[package]]

name = "heck"

version = "0.3.3"

source = "registry+https://github.com/rust-lang/crates.io-index"

checksum = "6d621efb26863f0e9924c6ac577e8275e5e6b77455db64ffa6c65c904e9e132c"

dependencies = [

 "unicode-segmentation",

]



[[package]]

name = "hermit-abi"

version = "0.1.19"

source = "registry+https://github.com/rust-lang/crates.io-index"

checksum = "62b467343b94ba476dcb2500d242dadbb39557df889310ac77c5d99100aaac33"

dependencies = [

 "libc",

]



[[package]]

name = "indexmap"

version = "1.8.0"

source = "registry+https://github.com/rust-lang/crates.io-index"

checksum = "282a6247722caba404c065016bbfa522806e51714c34f5dfc3e4a3a46fcb4223"

dependencies = [

 "autocfg",

 "hashbrown",

]



[[package]]

name = "instant"

version = "0.1.12"

source = "registry+https://github.com/rust-lang/crates.io-index"

checksum = "7a5bbe824c507c5da5956355e86a746d82e0e1464f65d862cc5e71da70e94b2c"

dependencies = [

 "cfg-if",

]



[[package]]

name = "lazy_static"

version = "1.4.0"

source = "registry+https://github.com/rust-lang/crates.io-index"

checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646"



[[package]]

name = "libc"

version = "0.2.112"

source = "registry+https://github.com/rust-lang/crates.io-index"

checksum = "1b03d17f364a3a042d5e5d46b053bbbf82c92c9430c592dd4c064dc6ee997125"



[[package]]

name = "output_vt100"

version = "0.1.2"

source = "registry+https://github.com/rust-lang/crates.io-index"

checksum = "53cdc5b785b7a58c5aad8216b3dfa114df64b0b06ae6e1501cef91df2fbdf8f9"

dependencies = [

 "winapi",

]



[[package]]

name = "pretty_assertions"

version = "1.0.0"

source = "registry+https://github.com/rust-lang/crates.io-index"

checksum = "ec0cfe1b2403f172ba0f234e500906ee0a3e493fb81092dac23ebefe129301cc"

dependencies = [

 "ansi_term",

 "ctor",

 "diff",

 "output_vt100",

]



[[package]]

name = "proc-macro-error"

version = "1.0.4"

source = "registry+https://github.com/rust-lang/crates.io-index"

checksum = "da25490ff9892aab3fcf7c36f08cfb902dd3e71ca0f9f9517bea02a73a5ce38c"

dependencies = [

 "proc-macro-error-attr",

 "proc-macro2",

 "quote",

 "syn",

 "version_check",

]



[[package]]

name = "proc-macro-error-attr"

version = "1.0.4"

source = "registry+https://github.com/rust-lang/crates.io-index"

checksum = "a1be40180e52ecc98ad80b184934baf3d0d29f979574e439af5a55274b35f869"

dependencies = [

 "proc-macro2",

 "quote",

 "version_check",

]



[[package]]

name = "proc-macro2"

version = "1.0.36"

source = "registry+https://github.com/rust-lang/crates.io-index"

checksum = "c7342d5883fbccae1cc37a2353b09c87c9b0f3afd73f5fb9bba687a1f733b029"

dependencies = [

 "unicode-xid",

]



[[package]]

name = "quote"

version = "1.0.14"

source = "registry+https://github.com/rust-lang/crates.io-index"

checksum = "47aa80447ce4daf1717500037052af176af5d38cc3e571d9ec1c7353fc10c87d"

dependencies = [

 "proc-macro2",

]



[[package]]

name = "redox_syscall"

version = "0.2.10"

source = "registry+https://github.com/rust-lang/crates.io-index"

checksum = "8383f39639269cde97d255a32bdb68c047337295414940c68bdd30c2e13203ff"

dependencies = [

 "bitflags",

]



[[package]]

name = "remove_dir_all"

version = "0.5.3"

source = "registry+https://github.com/rust-lang/crates.io-index"

checksum = "3acd125665422973a33ac9d3dd2df85edad0f4ae9b00dafb1a05e43a9f5ef8e7"

dependencies = [

 "winapi",

]



[[package]]

name = "sdk-versioner"

version = "0.1.0"

dependencies = [

 "anyhow",

 "pretty_assertions",

 "structopt",

 "tempfile",

 "toml",

]



[[package]]

name = "serde"

version = "1.0.133"

source = "registry+https://github.com/rust-lang/crates.io-index"

checksum = "97565067517b60e2d1ea8b268e59ce036de907ac523ad83a0475da04e818989a"



[[package]]

name = "strsim"

version = "0.8.0"

source = "registry+https://github.com/rust-lang/crates.io-index"

checksum = "8ea5119cdb4c55b55d432abb513a0429384878c15dde60cc77b1c99de1a95a6a"



[[package]]

name = "structopt"

version = "0.3.25"

source = "registry+https://github.com/rust-lang/crates.io-index"

checksum = "40b9788f4202aa75c240ecc9c15c65185e6a39ccdeb0fd5d008b98825464c87c"

dependencies = [

 "clap",

 "lazy_static",

 "structopt-derive",

]



[[package]]

name = "structopt-derive"

version = "0.4.18"

source = "registry+https://github.com/rust-lang/crates.io-index"

checksum = "dcb5ae327f9cc13b68763b5749770cb9e048a99bd9dfdfa58d0cf05d5f64afe0"

dependencies = [

 "heck",

 "proc-macro-error",

 "proc-macro2",

 "quote",

 "syn",

]



[[package]]

name = "syn"

version = "1.0.85"

source = "registry+https://github.com/rust-lang/crates.io-index"

checksum = "a684ac3dcd8913827e18cd09a68384ee66c1de24157e3c556c9ab16d85695fb7"

dependencies = [

 "proc-macro2",

 "quote",

 "unicode-xid",

]



[[package]]

name = "tempfile"

version = "3.3.0"

source = "registry+https://github.com/rust-lang/crates.io-index"

checksum = "5cdb1ef4eaeeaddc8fbd371e5017057064af0911902ef36b39801f67cc6d79e4"

dependencies = [

 "cfg-if",

 "fastrand",

 "libc",

 "redox_syscall",

 "remove_dir_all",

 "winapi",

]



[[package]]

name = "textwrap"

version = "0.11.0"

source = "registry+https://github.com/rust-lang/crates.io-index"

checksum = "d326610f408c7a4eb6f51c37c330e496b08506c9457c9d34287ecc38809fb060"

dependencies = [

 "unicode-width",

]



[[package]]

name = "toml"

version = "0.5.8"

source = "registry+https://github.com/rust-lang/crates.io-index"

checksum = "a31142970826733df8241ef35dc040ef98c679ab14d7c3e54d827099b3acecaa"

dependencies = [

 "indexmap",

 "serde",

]



[[package]]

name = "unicode-segmentation"

version = "1.8.0"

source = "registry+https://github.com/rust-lang/crates.io-index"

checksum = "8895849a949e7845e06bd6dc1aa51731a103c42707010a5b591c0038fb73385b"



[[package]]

name = "unicode-width"

version = "0.1.9"

source = "registry+https://github.com/rust-lang/crates.io-index"

checksum = "3ed742d4ea2bd1176e236172c8429aaf54486e7ac098db29ffe6529e0ce50973"



[[package]]

name = "unicode-xid"

version = "0.2.2"

source = "registry+https://github.com/rust-lang/crates.io-index"

checksum = "8ccb82d61f80a663efe1f787a51b16b5a51e3314d6ac365b08639f52387b33f3"



[[package]]

name = "vec_map"

version = "0.8.2"

source = "registry+https://github.com/rust-lang/crates.io-index"

checksum = "f1bddf1187be692e79c5ffeab891132dfb0f236ed36a43c7ed39f1165ee20191"



[[package]]

name = "version_check"

version = "0.9.4"

source = "registry+https://github.com/rust-lang/crates.io-index"

checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f"



[[package]]

name = "winapi"

version = "0.3.9"

source = "registry+https://github.com/rust-lang/crates.io-index"

checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419"

dependencies = [

 "winapi-i686-pc-windows-gnu",

 "winapi-x86_64-pc-windows-gnu",

]



[[package]]

name = "winapi-i686-pc-windows-gnu"

version = "0.4.0"

source = "registry+https://github.com/rust-lang/crates.io-index"

checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6"



[[package]]

name = "winapi-x86_64-pc-windows-gnu"

version = "0.4.0"

source = "registry+https://github.com/rust-lang/crates.io-index"

checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"

tools/sdk-versioner/Cargo.toml

0 → 100644
+18 −0
Original line number Diff line number Diff line 
[package]

name = "sdk-versioner"

version = "0.1.0"

authors = ["AWS Rust SDK Team <aws-sdk-rust@amazon.com>"]

edition = "2018"

license = "Apache-2.0"

publish = false



[workspace]



[dependencies]

anyhow = "1.0"

structopt = "0.3"

toml = { version = "0.5.8", features = ["preserve_order"] }



[dev-dependencies]

pretty_assertions = "1"

tempfile = "3"

tools/sdk-versioner/README.md

0 → 100644
+23 −0
Original line number Diff line number Diff line 
sdk-versioner

============



This is a CLI tool that will recursively update all references to the AWS Rust SDK

in Cargo.toml files in a given directory. That is, it finds every Cargo.toml file nested

within that directory, and modifies dependency lines that point to the AWS Rust SDK

or its supporting crates. These dependencies can be updated to be based on file-system

path, crates.io version, or both.



Example updating SDK examples to use SDK version 0.5.0 with Smithy version 0.35.0:

```bash

$ sdk-versioner \

  --sdk-version 0.5.0 \

  --smithy-version 0.35.0 \

  path/to/aws-doc-sdk-examples/rust_dev_preview

```



Example updating SDK examples to refer to local generated code:

```bash

$ sdk-versioner \

  --sdk-path path/to/smithy-rs/aws/sdk/build/aws-sdk/sdk \

  path/to/aws-doc-sdk-examples/rust_dev_preview

```

tools/sdk-versioner/src/main.rs

0 → 100644
+294 −0
Original line number Diff line number Diff line 
/*

 * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.

 * SPDX-License-Identifier: Apache-2.0.

 */



use anyhow::bail;

use std::ffi::OsStr;

use std::fs;

use std::path::{Path, PathBuf};

use std::time::Instant;

use structopt::StructOpt;

use toml::value::{Table, Value};



const AWS_CONFIG: &str = "aws-config";

const AWS_RUNTIME_CRATES: &[&str] = &[

    "aws-endpoint",

    "aws-http",

    "aws-hyper",

    "aws-sig-auth",

    "aws-sigv4",

    "aws-types",

];

const SDK_PREFIX: &str = "aws-sdk-";

const SMITHY_PREFIX: &str = "aws-smithy-";



#[derive(StructOpt, Debug)]

#[structopt(

    name = "sdk-versioner",

    about = "CLI tool to recursively update SDK/Smithy crate references in Cargo.toml files"

)]

struct Opt {

    /// Path(s) to recursively update Cargo.toml files in

    #[structopt()]

    crate_paths: Vec<PathBuf>,



    /// SDK version to point to

    #[structopt(long)]

    sdk_version: Option<String>,

    /// Smithy version to point to

    #[structopt(long)]

    smithy_version: Option<String>,



    /// Path to generated SDK to point to

    #[structopt(long)]

    sdk_path: Option<PathBuf>,

}



impl Opt {

    fn validate(self) -> anyhow::Result<Self> {

        if self.crate_paths.is_empty() {

            bail!("Must provide at least one crate path to recursively update");

        }

        if self.sdk_version.is_none() && self.sdk_path.is_none() {

            bail!("Must provide either an SDK version or an SDK path to update to");

        }

        if self.sdk_version.is_some() != self.smithy_version.is_some() {

            bail!("Must provide a Smithy version when providing an SDK version to update to");

        }

        Ok(self)

    }

}



fn main() -> anyhow::Result<()> {

    let opt = Opt::from_args().validate()?;



    let start_time = Instant::now();

    let mut manifest_paths = Vec::new();

    for crate_path in &opt.crate_paths {

        discover_manifests(&mut manifest_paths, crate_path)?;

    }



    for manifest_path in manifest_paths {

        update_manifest(&manifest_path, &opt)?;

    }



    println!("Finished in {:?}", start_time.elapsed());

    Ok(())

}



fn update_manifest(manifest_path: &Path, opt: &Opt) -> anyhow::Result<()> {

    println!("Updating {:?}...", manifest_path);



    let mut metadata: Value = toml::from_slice(&fs::read(manifest_path)?)?;

    let mut changed = false;

    for set in ["dependencies", "dev-dependencies", "build-dependencies"] {

        if let Some(dependencies) = metadata.get_mut(set) {

            if !dependencies.is_table() {

                bail!(

                    "Unexpected non-table value named `{}` in {:?}",

                    set,

                    manifest_path

                );

            }

            changed = update_dependencies(dependencies.as_table_mut().unwrap(), opt)? || changed;

        }

    }



    if changed {

        fs::write(manifest_path, &toml::to_vec(&metadata)?)?;

    }



    Ok(())

}



fn update_dependencies(dependencies: &mut Table, opt: &Opt) -> anyhow::Result<bool> {

    let mut changed = false;

    for (key, value) in dependencies.iter_mut() {

        if is_sdk_or_runtime_crate(key) {

            if !value.is_table() {

                *value = Value::Table(Table::new());

            }

            update_dependency_value(key, value.as_table_mut().unwrap(), opt);

            changed = true;

        }

    }

    Ok(changed)

}



fn is_sdk_crate(name: &str) -> bool {

    name.starts_with(SDK_PREFIX) || name == AWS_CONFIG

}



fn is_sdk_or_runtime_crate(name: &str) -> bool {

    is_sdk_crate(name)

        || name.starts_with(SMITHY_PREFIX)

        || AWS_RUNTIME_CRATES.iter().any(|&k| k == name)

}



fn update_dependency_value(crate_name: &str, value: &mut Table, opt: &Opt) {

    let is_sdk_crate = is_sdk_crate(crate_name);



    // Remove keys that will be replaced

    value.remove("version");

    value.remove("path");



    // Set the `path` if one was given

    if let Some(path) = &opt.sdk_path {

        let crate_path = if is_sdk_crate && crate_name != AWS_CONFIG {

            path.join(&crate_name[SDK_PREFIX.len()..])

        } else {

            path.join(crate_name)

        };

        value.insert(

            "path".to_string(),

            Value::String(

                crate_path

                    .as_os_str()

                    .to_str()

                    .expect("valid utf-8 path")

                    .to_string(),

            ),

        );

    }



    // Set the `version` if one was given

    if opt.sdk_version.is_some() {

        value.insert(

            "version".to_string(),

            Value::String(

                if is_sdk_crate {

                    &opt.sdk_version

                } else {

                    &opt.smithy_version

                }

                .clone()

                .unwrap(),

            ),

        );

    }

}



/// Recursively discovers Cargo.toml files in the given `path` and adds them to `manifests`.

fn discover_manifests(manifests: &mut Vec<PathBuf>, path: impl AsRef<Path>) -> anyhow::Result<()> {

    let path = path.as_ref();



    for entry in fs::read_dir(path)? {

        let entry = entry?;

        if entry.path().is_dir() {

            discover_manifests(manifests, entry.path())?;

        } else if entry.path().is_file()

            && entry.path().file_name() == Some(OsStr::new("Cargo.toml"))

        {

            manifests.push(entry.path());

        }

    }



    Ok(())

}



#[cfg(test)]

mod tests {

    use crate::{update_manifest, Opt};

    use pretty_assertions::assert_eq;

    use toml::Value;



    const TEST_MANIFEST: &[u8] = br#"

        [package]

        name = "test"

        version = "0.1.0"



        [dependencies]

        aws-config = "0.4.1"

        aws-sdk-s3 = "0.4.1"

        aws-smithy-types = "0.34.1"

        aws-smithy-http = { version = "0.34.1", features = ["test-util"] }

        something-else = "0.1"

    "#;



    #[track_caller]

    fn test_with_opt(opt: Opt, expected: &[u8]) {

        let manifest_file = tempfile::NamedTempFile::new().unwrap();

        let manifest_path = manifest_file.into_temp_path();

        std::fs::write(&manifest_path, TEST_MANIFEST).unwrap();



        update_manifest(&manifest_path, &opt).expect("success");



        let actual = toml::from_slice(&std::fs::read(&manifest_path).expect("read tmp file"))

            .expect("valid toml");

        let expected: Value = toml::from_slice(expected).unwrap();

        assert_eq!(expected, actual);

    }



    #[test]

    fn update_dependencies_with_versions() {

        test_with_opt(

            Opt {

                crate_paths: Vec::new(),

                sdk_path: None,

                sdk_version: Some("0.5.0".to_string()),

                smithy_version: Some("0.35.0".to_string()),

            },

            br#"

            [package]

            name = "test"

            version = "0.1.0"



            [dependencies]

            aws-config = { version = "0.5.0" }

            aws-sdk-s3 = { version = "0.5.0" }

            aws-smithy-types = { version = "0.35.0" }

            aws-smithy-http = { version = "0.35.0", features = ["test-util"] }

            something-else = "0.1"

            "#,

        );

    }



    #[test]

    fn update_dependencies_with_paths() {

        test_with_opt(

            Opt {

                crate_paths: Vec::new(),

                sdk_path: Some("/foo/asdf/".into()),

                sdk_version: None,

                smithy_version: None,

            },

            br#"

            [package]

            name = "test"

            version = "0.1.0"



            [dependencies]

            aws-config = { path = "/foo/asdf/aws-config" }

            aws-sdk-s3 = { path = "/foo/asdf/s3" }

            aws-smithy-types = { path = "/foo/asdf/aws-smithy-types" }

            aws-smithy-http = { path = "/foo/asdf/aws-smithy-http", features = ["test-util"] }

            something-else = "0.1"

            "#,

        );

    }



    #[test]

    fn update_dependencies_with_versions_and_paths() {

        test_with_opt(

            Opt {

                crate_paths: Vec::new(),

                sdk_path: Some("/foo/asdf/".into()),

                sdk_version: Some("0.5.0".to_string()),

                smithy_version: Some("0.35.0".to_string()),

            },

        br#"

            [package]

            name = "test"

            version = "0.1.0"



            [dependencies]

            aws-config = { version = "0.5.0", path = "/foo/asdf/aws-config" }

            aws-sdk-s3 = { version = "0.5.0", path = "/foo/asdf/s3" }

            aws-smithy-types = { version = "0.35.0", path = "/foo/asdf/aws-smithy-types" }

            aws-smithy-http = { version = "0.35.0", path = "/foo/asdf/aws-smithy-http", features = ["test-util"] }

            something-else = "0.1"

            "#

        );

    }

}