Unverified Commit 9ac9fbc4 authored by ysaito1001's avatar ysaito1001 Committed by GitHub
Browse files

Fix `canary-wasm` not being able to locate generated crates (#3472) 

## Motivation and Context
Fixes the issue of `canary-wasm` not being able to locate generated
crates in our release pipeline

## Description
The `canary-wasm` crate has
[Cargo.toml](https://github.com/smithy-lang/smithy-rs/blob/main/tools/ci-cdk/canary-wasm/Cargo.toml)
checked-in, whose SDK dependencies assume path locations with respect to
build artifacts within the `smithy-rs` repository. However, those
locations can be different in our release pipeline (the same reason why
there's no manifest file checked-in for
[canary-lambda](https://github.com/smithy-lang/smithy-rs/tree/main/tools/ci-cdk/canary-lambda)
and it needs to be generated on the fly by `canary-runner`'s
`build-bundle` subcommand).

To fix this, `canary-runner` now generates the manifest for
`canary-wasm`.

## Testing
- [x] Added unit tests for `canary-runner`
- [x] Verified in our release pipeline that canary built and passed

----

_By submitting this pull request, I confirm that you can use, modify,
copy, and redistribute this contribution, under the terms of your
choice._
parent f68b5eea

tools/ci-cdk/.gitignore

+0 −3
Original line number Diff line number Diff line
@@ -8,6 +8,3 @@ build 
.cdk.staging

cdk.out

cdk-outputs.json
 


# Generated Rust file

canary-wasm/src/bindings.rs

tools/ci-cdk/canary-runner/Cargo.lock

+101 −1
Original line number Diff line number Diff line
@@ -736,6 +736,26 @@ dependencies = [ 
 "libc",

]



[[package]]

name = "crates-index"

version = "2.5.1"

source = "registry+https://github.com/rust-lang/crates.io-index"

checksum = "1d9efa03a974d583ad530bbfe00e3d0021de7f26217120437b128dc4c331aa4f"

dependencies = [

 "hex",

 "home",

 "http",

 "memchr",

 "rustc-hash",

 "semver",

 "serde",

 "serde_derive",

 "serde_json",

 "smol_str",

 "thiserror",

 "toml 0.8.10",

]



[[package]]

name = "crc32c"

version = "0.6.4"
@@ -1142,6 +1162,9 @@ name = "hex" 
version = "0.4.3"

source = "registry+https://github.com/rust-lang/crates.io-index"

checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70"

dependencies = [

 "serde",

]



[[package]]

name = "hmac"
@@ -1152,6 +1175,15 @@ dependencies = [ 
 "digest",

]



[[package]]

name = "home"

version = "0.5.9"

source = "registry+https://github.com/rust-lang/crates.io-index"

checksum = "e3d1354bf6b7235cb4a0576c2619fd4ed18183f689b12b006a0ee7329eeff9a5"

dependencies = [

 "windows-sys 0.52.0",

]



[[package]]

name = "http"

version = "0.2.11"
@@ -2057,6 +2089,12 @@ version = "0.1.23" 
source = "registry+https://github.com/rust-lang/crates.io-index"

checksum = "d626bb9dae77e28219937af045c257c28bfd3f69333c512553507f5f9798cb76"



[[package]]

name = "rustc-hash"

version = "1.1.0"

source = "registry+https://github.com/rust-lang/crates.io-index"

checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2"



[[package]]

name = "rustc_version"

version = "0.4.0"
@@ -2266,6 +2304,15 @@ dependencies = [ 
 "serde",

]



[[package]]

name = "serde_spanned"

version = "0.6.5"

source = "registry+https://github.com/rust-lang/crates.io-index"

checksum = "eb3622f419d1296904700073ea6cc23ad690adbd66f13ea683df73298736f0c1"

dependencies = [

 "serde",

]



[[package]]

name = "serde_urlencoded"

version = "0.7.1"
@@ -2361,6 +2408,7 @@ version = "0.1.0" 
dependencies = [

 "anyhow",

 "async-trait",

 "crates-index",

 "lazy_static",

 "regex",

 "reqwest",
@@ -2369,10 +2417,19 @@ dependencies = [ 
 "serde_json",

 "thiserror",

 "tokio",

 "toml",

 "toml 0.5.11",

 "tracing",

]



[[package]]

name = "smol_str"

version = "0.2.1"

source = "registry+https://github.com/rust-lang/crates.io-index"

checksum = "e6845563ada680337a52d43bb0b29f396f2d911616f6573012645b9e3d048a49"

dependencies = [

 "serde",

]



[[package]]

name = "socket2"

version = "0.4.10"
@@ -2655,6 +2712,40 @@ dependencies = [ 
 "serde",

]



[[package]]

name = "toml"

version = "0.8.10"

source = "registry+https://github.com/rust-lang/crates.io-index"

checksum = "9a9aad4a3066010876e8dcf5a8a06e70a558751117a145c6ce2b82c2e2054290"

dependencies = [

 "serde",

 "serde_spanned",

 "toml_datetime",

 "toml_edit",

]



[[package]]

name = "toml_datetime"

version = "0.6.5"

source = "registry+https://github.com/rust-lang/crates.io-index"

checksum = "3550f4e9685620ac18a50ed434eb3aec30db8ba93b0287467bca5826ea25baf1"

dependencies = [

 "serde",

]



[[package]]

name = "toml_edit"

version = "0.22.6"

source = "registry+https://github.com/rust-lang/crates.io-index"

checksum = "2c1b5fd4128cc8d3e0cb74d4ed9a9cc7c7284becd4df68f5f940e1ad123606f6"

dependencies = [

 "indexmap 2.1.0",

 "serde",

 "serde_spanned",

 "toml_datetime",

 "winnow",

]



[[package]]

name = "tower-service"

version = "0.3.2"
@@ -3112,6 +3203,15 @@ version = "0.52.0" 
source = "registry+https://github.com/rust-lang/crates.io-index"

checksum = "dff9641d1cd4be8d1a070daf9e3773c5f67e78b4d9d42263020c057706765c04"



[[package]]

name = "winnow"

version = "0.6.5"

source = "registry+https://github.com/rust-lang/crates.io-index"

checksum = "dffa400e67ed5a4dd237983829e66475f0a4a26938c4b04c21baede6262215b8"

dependencies = [

 "memchr",

]



[[package]]

name = "winreg"

version = "0.50.0"

tools/ci-cdk/canary-runner/src/build_bundle.rs

+255 −72
Original line number Diff line number Diff line
@@ -19,6 +19,90 @@ use smithy_rs_tool_common::release_tag::ReleaseTag; 
use smithy_rs_tool_common::shell::handle_failure;

use smithy_rs_tool_common::versions_manifest::VersionsManifest;



struct RequiredDependency {

    name: &'static str,

    features: Option<Vec<&'static str>>,

    disable_default_feature: bool,

}



impl RequiredDependency {

    fn new(name: &'static str) -> Self {

        Self {

            name,

            features: None,

            disable_default_feature: false,

        }

    }

    fn with_features(mut self, features: impl IntoIterator<Item = &'static str>) -> Self {

        self.features = Some(features.into_iter().collect());

        self

    }

    fn with_default_feature_disabled(mut self) -> Self {

        self.disable_default_feature = true;

        self

    }

    fn to_string(&self, crate_source: &CrateSource) -> Result<String> {

        match crate_source {

            CrateSource::Path(path) => {

                let path_name = match self.name.strip_prefix("aws-sdk-") {

                    Some(path) => path,

                    None => self.name,

                };

                let crate_path = path.join(path_name);

                let mut result = format!(

                    r#"{name} = {{ path = "{path}""#,

                    name = self.name,

                    path = crate_path.to_string_lossy(),

                );

                if let Some(features) = &self.features {

                    self.write_features(features, &mut result);

                }

                if self.disable_default_feature {

                    result.push_str(", default-features = false");

                }

                result.push_str(" }");

                Ok(result)

            }

            CrateSource::VersionsManifest {

                versions,

                release_tag,

            } => match versions.crates.get(self.name) {

                Some(version) => {

                    let mut result = format!(

                        r#"{name} = {{ version = "{version}""#,

                        name = self.name,

                        version = version.version,

                    );

                    if let Some(features) = &self.features {

                        self.write_features(features, &mut result);

                    }

                    if self.disable_default_feature {

                        result.push_str(", default-features = false");

                    }

                    result.push_str(" }");

                    Ok(result)

                }

                None => {

                    bail!(

                        "Couldn't find `{name}` in versions.toml for `{release_tag}`",

                        name = self.name,

                    )

                }

            },

        }

    }

    fn write_features(&self, features: &[&'static str], output: &mut String) {

        output.push_str(&format!(

            r#", features = [{features}]"#,

            features = features

                .iter()

                .map(|feature| format!(r#""{feature}""#))

                .collect::<Vec<_>>()

                .join(","),

        ));

    }

}



const BASE_MANIFEST: &str = r#"

# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.

# SPDX-License-Identifier: Apache-2.0
@@ -61,13 +145,51 @@ wasmtime-wasi = "17.0.1" 
wasmtime-wasi-http = "17.0.1"

"#;



const REQUIRED_SDK_CRATES: &[&str] = &[

    "aws-config",

    "aws-sdk-s3",

    "aws-sdk-ec2",

    "aws-sdk-transcribestreaming",

    "aws-smithy-wasm",

lazy_static! {

    static ref REQUIRED_SDK_CRATES: Vec<RequiredDependency> = vec![

        RequiredDependency::new("aws-config").with_features(["behavior-version-latest"]),

        RequiredDependency::new("aws-sdk-s3"),

        RequiredDependency::new("aws-sdk-ec2"),

        RequiredDependency::new("aws-sdk-transcribestreaming"),

        RequiredDependency::new("aws-smithy-wasm"),

    ];

}



const WASM_BASE_MANIFEST: &str = r#"

# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.

# SPDX-License-Identifier: Apache-2.0

#

# IMPORTANT: Don't edit this file directly! Run `canary-runner build-bundle` to modify this file instead.

[package]

name = "aws-sdk-rust-lambda-canary-wasm"

version = "0.1.0"

edition = "2021"

license = "Apache-2.0"



[lib]

crate-type = ["cdylib"]



# metadata used by cargo-component to identify which wit world to embed in the binary

[package.metadata.component]

package = "aws:component"



[dependencies]

tokio = { version = "1.36.0", features = ["macros", "rt", "time"] }

wit-bindgen = { version = "0.16.0", features = ["macros", "realloc"] }

"#;



lazy_static! {

    static ref WASM_REQUIRED_SDK_CRATES: Vec<RequiredDependency> = vec![

        RequiredDependency::new("aws-config")

            .with_features(["behavior-version-latest"])

            .with_default_feature_disabled(),

        RequiredDependency::new("aws-sdk-s3").with_default_feature_disabled(),

        RequiredDependency::new("aws-smithy-async")

            .with_features(["rt-tokio"])

            .with_default_feature_disabled(),

        RequiredDependency::new("aws-smithy-wasm"),

    ];

}



// The elements in this `Vec` should be sorted in an ascending order by the release date.

lazy_static! {
@@ -112,6 +234,7 @@ pub struct BuildBundleArgs { 
    pub manifest_only: bool,

}



#[derive(Clone)]

enum CrateSource {

    Path(PathBuf),

    VersionsManifest {
@@ -136,7 +259,7 @@ fn enabled_feature(crate_source: &CrateSource) -> String { 


fn generate_crate_manifest(crate_source: CrateSource) -> Result<String> {

    let mut output = BASE_MANIFEST.to_string();

    write_dependencies(REQUIRED_SDK_CRATES, &mut output, &crate_source)?;

    write_dependencies(&REQUIRED_SDK_CRATES, &mut output, &crate_source)?;

    write!(output, "\n[features]\n").unwrap();

    writeln!(output, "latest = []").unwrap();

    for release_tag in NOTABLE_SDK_RELEASE_TAGS.iter() {
@@ -157,55 +280,12 @@ fn generate_crate_manifest(crate_source: CrateSource) -> Result<String> { 
}



fn write_dependencies(

    required_crates: &[&str],

    required_crates: &[RequiredDependency],

    output: &mut String,

    crate_source: &CrateSource,

) -> Result<()> {

    for &required_crate in required_crates {

        match &crate_source {

            CrateSource::Path(path) => {

                let path_name = match required_crate.strip_prefix("aws-sdk-") {

                    Some(path) => path,

                    None => required_crate,

                };

                let crate_path = path.join(path_name);

                write!(

                    output,

                    r#"{required_crate} = {{ path = "{path}""#,

                    path = crate_path.to_string_lossy()

                )

                .unwrap();

                if required_crate == "aws-config" {

                    write!(output, r#", features = ["behavior-version-latest"]"#).unwrap();

                }

                writeln!(output, " }}").unwrap();

            }

            CrateSource::VersionsManifest {

                versions,

                release_tag,

            } => match versions.crates.get(required_crate) {

                Some(version) => {

                    if required_crate == "aws-config" {

                        writeln!(

                            output,

                            r#"{required_crate} = {{ version = "{version}", features = ["behavior-version-latest"] }}"#,

                            version = version.version

                        )

                        .unwrap();

                    } else {

                        writeln!(

                            output,

                            r#"{required_crate} = "{version}""#,

                            version = version.version

                        )

                        .unwrap();

                    }

                }

                None => {

                    bail!("Couldn't find `{required_crate}` in versions.toml for `{release_tag}`")

                }

            },

        }

    for required_crate in required_crates {

        writeln!(output, "{}", required_crate.to_string(crate_source)?).unwrap();

    }

    Ok(())

}
@@ -260,8 +340,21 @@ pub async fn build_bundle(opt: BuildBundleArgs) -> Result<Option<PathBuf>> { 


    // Generate the canary Lambda's Cargo.toml

    let manifest_path = canary_path.join("Cargo.toml");

    let crate_manifest_content = generate_crate_manifest(crate_source)?;

    fs::write(&manifest_path, crate_manifest_content).context("failed to write Cargo.toml")?;

    let crate_manifest_content = generate_crate_manifest(crate_source.clone())?;

    fs::write(&manifest_path, crate_manifest_content)

        .context(format!("failed to write Cargo.toml in {manifest_path:?}"))?;



    // Generate the canary-wasm's Cargo.toml

    let wasm_manifest_path = canary_path.join("../canary-wasm/Cargo.toml");

    let mut wasm_crate_manifest_content = WASM_BASE_MANIFEST.to_string();

    write_dependencies(

        &WASM_REQUIRED_SDK_CRATES,

        &mut wasm_crate_manifest_content,

        &crate_source,

    )?;

    fs::write(&wasm_manifest_path, wasm_crate_manifest_content).context(format!(

        "failed to write Cargo.toml in {wasm_manifest_path:?}"

    ))?;



    let wasm_manifest_path = std::env::current_dir()

        .expect("Current dir")
@@ -356,6 +449,18 @@ mod tests { 


    use super::*;



    fn crate_version(name: &str, version: &str) -> (String, CrateVersion) {

        (

            name.to_string(),

            CrateVersion {

                category: PackageCategory::AwsRuntime, // doesn't matter for this test

                version: version.into(),

                source_hash: "some-hash".into(),

                model_hash: None,

            },

        )

    }



    #[test]

    fn test_arg_parsing() {

        assert!(Args::try_parse_from(["./canary-runner", "build-bundle"]).is_err());
@@ -511,18 +616,6 @@ default = ["latest"] 


    #[test]

    fn test_generate_crate_manifest_with_release_tag() {

        fn crate_version(name: &str, version: &str) -> (String, CrateVersion) {

            (

                name.to_string(),

                CrateVersion {

                    category: PackageCategory::AwsRuntime, // doesn't matter for this test

                    version: version.into(),

                    source_hash: "some-hash".into(),

                    model_hash: None,

                },

            )

        }



        pretty_assertions::assert_str_eq!(

            r#"

# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
@@ -565,10 +658,10 @@ wasmtime = { version = "17.0.1", features = ["component-model"] } 
wasmtime-wasi = "17.0.1"

wasmtime-wasi-http = "17.0.1"

aws-config = { version = "0.46.0", features = ["behavior-version-latest"] }

aws-sdk-s3 = "0.20.0"

aws-sdk-ec2 = "0.19.0"

aws-sdk-transcribestreaming = "0.16.0"

aws-smithy-wasm = "0.1.0"

aws-sdk-s3 = { version = "0.20.0" }

aws-sdk-ec2 = { version = "0.19.0" }

aws-sdk-transcribestreaming = { version = "0.16.0" }

aws-smithy-wasm = { version = "0.1.0" }



[features]

latest = []
@@ -597,6 +690,96 @@ default = ["latest"] 
        );

    }



    #[test]

    fn test_generate_canary_wasm_crate_manifest_with_paths() {

        let mut output = WASM_BASE_MANIFEST.to_string();

        let crate_source = CrateSource::Path("some/sdk/path".into());

        write_dependencies(&WASM_REQUIRED_SDK_CRATES, &mut output, &crate_source).expect("success");



        pretty_assertions::assert_str_eq!(

            r#"

# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.

# SPDX-License-Identifier: Apache-2.0

#

# IMPORTANT: Don't edit this file directly! Run `canary-runner build-bundle` to modify this file instead.

[package]

name = "aws-sdk-rust-lambda-canary-wasm"

version = "0.1.0"

edition = "2021"

license = "Apache-2.0"



[lib]

crate-type = ["cdylib"]



# metadata used by cargo-component to identify which wit world to embed in the binary

[package.metadata.component]

package = "aws:component"



[dependencies]

tokio = { version = "1.36.0", features = ["macros", "rt", "time"] }

wit-bindgen = { version = "0.16.0", features = ["macros", "realloc"] }

aws-config = { path = "some/sdk/path/aws-config", features = ["behavior-version-latest"], default-features = false }

aws-sdk-s3 = { path = "some/sdk/path/s3", default-features = false }

aws-smithy-async = { path = "some/sdk/path/aws-smithy-async", features = ["rt-tokio"], default-features = false }

aws-smithy-wasm = { path = "some/sdk/path/aws-smithy-wasm" }

"#,

            output,

        );

    }



    #[test]

    fn test_generate_canary_wasm_crate_manifest_with_release_tag() {

        let mut output = WASM_BASE_MANIFEST.to_string();

        let crate_source = CrateSource::VersionsManifest {

            versions: VersionsManifest {

                smithy_rs_revision: "some-revision-smithy-rs".into(),

                aws_doc_sdk_examples_revision: "some-revision-docs".into(),

                manual_interventions: Default::default(),

                crates: [

                    crate_version("aws-config", "0.46.0"),

                    crate_version("aws-sdk-s3", "0.20.0"),

                    crate_version("aws-smithy-async", "0.46.0"),

                    crate_version("aws-smithy-wasm", "0.1.0"),

                ]

                .into_iter()

                .collect(),

                release: None,

            },

            release_tag: ReleaseTag::from_str("release-9999-12-31").unwrap(),

        };

        write_dependencies(&WASM_REQUIRED_SDK_CRATES, &mut output, &crate_source).expect("success");



        pretty_assertions::assert_str_eq!(

            r#"

# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.

# SPDX-License-Identifier: Apache-2.0

#

# IMPORTANT: Don't edit this file directly! Run `canary-runner build-bundle` to modify this file instead.

[package]

name = "aws-sdk-rust-lambda-canary-wasm"

version = "0.1.0"

edition = "2021"

license = "Apache-2.0"



[lib]

crate-type = ["cdylib"]



# metadata used by cargo-component to identify which wit world to embed in the binary

[package.metadata.component]

package = "aws:component"



[dependencies]

tokio = { version = "1.36.0", features = ["macros", "rt", "time"] }

wit-bindgen = { version = "0.16.0", features = ["macros", "realloc"] }

aws-config = { version = "0.46.0", features = ["behavior-version-latest"], default-features = false }

aws-sdk-s3 = { version = "0.20.0", default-features = false }

aws-smithy-async = { version = "0.46.0", features = ["rt-tokio"], default-features = false }

aws-smithy-wasm = { version = "0.1.0" }

"#,

            output

        );

    }



    #[test]

    fn test_name_hashed_bundle() {

        fn check(expected: &str, actual: &str) {

tools/ci-cdk/canary-wasm/.gitignore

0 → 100644
+6 −0
Original line number Diff line number Diff line 
# Generated Rust file

/src/bindings.rs



# The Cargo.toml should be generated every time it's needed

/Cargo.toml

/Cargo.lock

tools/ci-cdk/canary-wasm/Cargo.lock

deleted100644 → 0
+0 −1286

File deleted.

Preview size limit exceeded, changes collapsed.