From 69efe3a72867b0f269ee840f8042f5f477d7476a Mon Sep 17 00:00:00 2001 From: John DiSanti Date: Mon, 8 Apr 2024 17:41:21 -0700 Subject: [PATCH] Don't generate a SDK for runtime-versioner patching (#3540) This PR removes the requirement of generating a SDK in order to patch smithy-rs runtime crates into an already released SDK version for testing. This is possible now that all the runtime crates are independently versioned. The tool also takes multiple paths as input for the runtime crate locations. This makes it possible to pass in the `rust-runtime` and `aws/rust-runtime` directories from smithy-rs (done by default by one of the subcommands). Some of the publisher tool's package resolution code is moved into smithy-rs-tool-common so it can be better shared across the tools, and as part of this, the version in the package handle was made optional. ---- _By submitting this pull request, I confirm that you can use, modify, copy, and redistribute this contribution, under the terms of your choice._ --- tools/ci-build/changelogger/Cargo.lock | 149 ++++++++- tools/ci-build/publisher/Cargo.lock | 13 +- tools/ci-build/publisher/src/cargo/publish.rs | 25 +- tools/ci-build/publisher/src/package.rs | 259 ++------------- tools/ci-build/publisher/src/publish.rs | 3 +- tools/ci-build/publisher/src/sort.rs | 8 +- .../src/subcommand/claim_crate_names.rs | 11 +- .../publisher/src/subcommand/fix_manifests.rs | 6 +- .../subcommand/generate_version_manifest.rs | 5 +- .../publisher/src/subcommand/publish.rs | 18 +- tools/ci-build/runtime-versioner/Cargo.lock | 1 + .../runtime-versioner/src/command/patch.rs | 183 ++++++----- tools/ci-build/runtime-versioner/src/main.rs | 20 +- tools/ci-build/sdk-lints/Cargo.lock | 13 +- tools/ci-build/sdk-versioner/Cargo.lock | 11 + .../ci-build/smithy-rs-tool-common/Cargo.toml | 1 + .../ci-build/smithy-rs-tool-common/src/lib.rs | 3 + .../smithy-rs-tool-common/src/package.rs | 297 ++++++++++++++++++ tools/ci-cdk/canary-runner/Cargo.lock | 11 + 19 files changed, 668 insertions(+), 369 deletions(-) diff --git a/tools/ci-build/changelogger/Cargo.lock b/tools/ci-build/changelogger/Cargo.lock index c7de5cafa..593c64701 100644 --- a/tools/ci-build/changelogger/Cargo.lock +++ b/tools/ci-build/changelogger/Cargo.lock @@ -49,7 +49,7 @@ version = "0.2.14" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8" dependencies = [ - "hermit-abi", + "hermit-abi 0.1.19", "libc", "winapi", ] @@ -105,6 +105,16 @@ version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a2bd12c1caf447e69cd4528f47f94d203fd2582878ecb9e9465484c4148a8223" +[[package]] +name = "cargo_toml" +version = "0.19.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a98356df42a2eb1bd8f1793ae4ee4de48e384dd974ce5eac8eee802edb7492be" +dependencies = [ + "serde", + "toml 0.8.12", +] + [[package]] name = "cc" version = "1.0.83" @@ -134,7 +144,7 @@ dependencies = [ "smithy-rs-tool-common", "tempfile", "time", - "toml", + "toml 0.5.11", ] [[package]] @@ -192,6 +202,26 @@ version = "0.8.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "06ea2b9bc92be3c2baa9334a323ebca2d6f074ff852cd1d7b11064035cd3868f" +[[package]] +name = "crates-index" +version = "2.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d9efa03a974d583ad530bbfe00e3d0021de7f26217120437b128dc4c331aa4f" +dependencies = [ + "hex", + "home", + "http", + "memchr", + "rustc-hash", + "semver", + "serde", + "serde_derive", + "serde_json", + "smol_str", + "thiserror", + "toml 0.8.12", +] + [[package]] name = "deranged" version = "0.3.10" @@ -283,6 +313,12 @@ version = "0.3.29" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "eb1d22c66e66d9d72e1758f0bd7d4fd0bee04cad842ee34587d68c07e45d088c" +[[package]] +name = "futures-io" +version = "0.3.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a44623e20b9681a318efdd71c299b6b222ed6f231972bfe2f224ebad6311f0c1" + [[package]] name = "futures-sink" version = "0.3.29" @@ -302,9 +338,12 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a19526d624e703a3179b3d322efec918b6246ea0fa51d41124525f00f1cc8104" dependencies = [ "futures-core", + "futures-io", "futures-task", + "memchr", "pin-project-lite", "pin-utils", + "slab", ] [[package]] @@ -359,6 +398,30 @@ dependencies = [ "libc", ] +[[package]] +name = "hermit-abi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d231dfb89cfffdbc30e7fc41579ed6066ad03abda9e567ccafae602b97ec5024" + +[[package]] +name = "hex" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" +dependencies = [ + "serde", +] + +[[package]] +name = "home" +version = "0.5.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3d1354bf6b7235cb4a0576c2619fd4ed18183f689b12b006a0ee7329eeff9a5" +dependencies = [ + "windows-sys 0.52.0", +] + [[package]] name = "http" version = "0.2.11" @@ -574,6 +637,16 @@ dependencies = [ "autocfg", ] +[[package]] +name = "num_cpus" +version = "1.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4161fcb6d602d4d2081af7c3a45852d875a03dd337a6bfdd6e06407b61342a43" +dependencies = [ + "hermit-abi 0.3.9", + "libc", +] + [[package]] name = "num_threads" version = "0.1.6" @@ -821,6 +894,12 @@ version = "0.1.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d626bb9dae77e28219937af045c257c28bfd3f69333c512553507f5f9798cb76" +[[package]] +name = "rustc-hash" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" + [[package]] name = "rustix" version = "0.38.26" @@ -909,6 +988,15 @@ dependencies = [ "serde", ] +[[package]] +name = "serde_spanned" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eb3622f419d1296904700073ea6cc23ad690adbd66f13ea683df73298736f0c1" +dependencies = [ + "serde", +] + [[package]] name = "serde_urlencoded" version = "0.7.1" @@ -936,6 +1024,8 @@ version = "0.1.0" dependencies = [ "anyhow", "async-trait", + "cargo_toml", + "crates-index", "lazy_static", "regex", "reqwest", @@ -943,10 +1033,19 @@ dependencies = [ "serde", "serde_json", "thiserror", - "toml", + "toml 0.5.11", "tracing", ] +[[package]] +name = "smol_str" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6845563ada680337a52d43bb0b29f396f2d911616f6573012645b9e3d048a49" +dependencies = [ + "serde", +] + [[package]] name = "socket2" version = "0.4.10" @@ -1109,6 +1208,7 @@ dependencies = [ "bytes", "libc", "mio", + "num_cpus", "pin-project-lite", "socket2 0.5.5", "windows-sys 0.48.0", @@ -1148,6 +1248,40 @@ dependencies = [ "serde", ] +[[package]] +name = "toml" +version = "0.8.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e9dd1545e8208b4a5af1aa9bbd0b4cf7e9ea08fabc5d0a5c67fcaafa17433aa3" +dependencies = [ + "serde", + "serde_spanned", + "toml_datetime", + "toml_edit", +] + +[[package]] +name = "toml_datetime" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3550f4e9685620ac18a50ed434eb3aec30db8ba93b0287467bca5826ea25baf1" +dependencies = [ + "serde", +] + +[[package]] +name = "toml_edit" +version = "0.22.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e40bb779c5187258fd7aad0eb68cb8706a0a81fa712fbea808ab43c4b8374c4" +dependencies = [ + "indexmap 2.1.0", + "serde", + "serde_spanned", + "toml_datetime", + "winnow", +] + [[package]] name = "tower-service" version = "0.3.2" @@ -1489,6 +1623,15 @@ version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dff9641d1cd4be8d1a070daf9e3773c5f67e78b4d9d42263020c057706765c04" +[[package]] +name = "winnow" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dffa400e67ed5a4dd237983829e66475f0a4a26938c4b04c21baede6262215b8" +dependencies = [ + "memchr", +] + [[package]] name = "winreg" version = "0.50.0" diff --git a/tools/ci-build/publisher/Cargo.lock b/tools/ci-build/publisher/Cargo.lock index 6aacb8502..9b0a0f2c7 100644 --- a/tools/ci-build/publisher/Cargo.lock +++ b/tools/ci-build/publisher/Cargo.lock @@ -135,6 +135,16 @@ dependencies = [ "toml 0.8.12", ] +[[package]] +name = "cargo_toml" +version = "0.19.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a98356df42a2eb1bd8f1793ae4ee4de48e384dd974ce5eac8eee802edb7492be" +dependencies = [ + "serde", + "toml 0.8.12", +] + [[package]] name = "cc" version = "1.0.90" @@ -956,7 +966,7 @@ dependencies = [ "anyhow", "async-recursion", "async-trait", - "cargo_toml", + "cargo_toml 0.16.3", "clap", "dialoguer", "fs-err", @@ -1278,6 +1288,7 @@ version = "0.1.0" dependencies = [ "anyhow", "async-trait", + "cargo_toml 0.19.2", "crates-index", "lazy_static", "regex", diff --git a/tools/ci-build/publisher/src/cargo/publish.rs b/tools/ci-build/publisher/src/cargo/publish.rs index 71e79e381..3f0f2a50d 100644 --- a/tools/ci-build/publisher/src/cargo/publish.rs +++ b/tools/ci-build/publisher/src/cargo/publish.rs @@ -3,9 +3,11 @@ * SPDX-License-Identifier: Apache-2.0 */ -use crate::package::PackageHandle; use anyhow::Result; -use smithy_rs_tool_common::shell::{capture_error, output_text, ShellOperation}; +use smithy_rs_tool_common::{ + package::PackageHandle, + shell::{capture_error, output_text, ShellOperation}, +}; use std::path::PathBuf; use std::process::Command; use tracing::info; @@ -18,6 +20,10 @@ pub struct Publish { impl Publish { pub fn new(package_handle: PackageHandle, package_path: impl Into) -> Publish { + assert!( + package_handle.version.is_some(), + "crate version number required; given {package_handle}" + ); Publish { program: "cargo", package_handle, @@ -42,12 +48,12 @@ impl ShellOperation for Publish { let (stdout, stderr) = output_text(&output); let already_uploaded_msg = format!( "error: crate version `{}` is already uploaded", - self.package_handle.version + self.package_handle.expect_version() ); if stdout.contains(&already_uploaded_msg) || stderr.contains(&already_uploaded_msg) { info!( - "{}-{} has already been published to crates.io.", - self.package_handle.name, self.package_handle.version + "{} has already been published to crates.io.", + self.package_handle ); } else { return Err(capture_error("cargo publish", &output)); @@ -69,7 +75,7 @@ mod tests { program: "./fake_cargo/cargo_success", package_handle: PackageHandle::new( "aws-sdk-dynamodb", - Version::parse("0.0.22-alpha").unwrap(), + Version::parse("0.0.22-alpha").ok(), ), package_path: env::current_dir().unwrap(), } @@ -82,10 +88,7 @@ mod tests { async fn publish_fails() { let result = Publish { program: "./fake_cargo/cargo_fails", - package_handle: PackageHandle::new( - "something", - Version::parse("0.0.22-alpha").unwrap(), - ), + package_handle: PackageHandle::new("something", Version::parse("0.0.22-alpha").ok()), package_path: env::current_dir().unwrap(), } .spawn() @@ -106,7 +109,7 @@ mod tests { program: "./fake_cargo/cargo_publish_already_published", package_handle: PackageHandle::new( "aws-sdk-dynamodb", - Version::parse("0.0.22-alpha").unwrap(), + Version::parse("0.0.22-alpha").ok(), ), package_path: env::current_dir().unwrap(), } diff --git a/tools/ci-build/publisher/src/package.rs b/tools/ci-build/publisher/src/package.rs index 4fb572a68..7f2f823df 100644 --- a/tools/ci-build/publisher/src/package.rs +++ b/tools/ci-build/publisher/src/package.rs @@ -7,101 +7,15 @@ use crate::fs::Fs; use crate::sort::dependency_order; -use crate::RUST_SDK_CI_OWNER; -use anyhow::{Context, Result}; -use cargo_toml::{Dependency, DepsSet, Manifest}; +use anyhow::Result; use semver::Version; -use smithy_rs_tool_common::package::PackageCategory; -use std::collections::{BTreeMap, BTreeSet, HashSet}; +use smithy_rs_tool_common::package::{Package, PackageCategory, PackageHandle, Publish}; use std::error::Error as StdError; -use std::fmt; -use std::path::{Path, PathBuf}; +use std::path::PathBuf; +use std::{collections::BTreeMap, path::Path}; use tokio::fs; use tracing::warn; -/// Information required to identify a package (crate). -#[derive(Clone, Debug, Eq, PartialEq, Ord, PartialOrd)] -pub struct PackageHandle { - pub name: String, - pub version: Version, -} - -impl PackageHandle { - pub fn new(name: impl Into, version: Version) -> Self { - Self { - name: name.into(), - version, - } - } -} - -impl fmt::Display for PackageHandle { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "{}-{}", self.name, self.version) - } -} - -#[derive(Debug, Eq, PartialEq, PartialOrd, Ord)] -pub enum Publish { - Allowed, - NotAllowed, -} - -/// Represents a crate (called Package since crate is a reserved word). -#[derive(Debug, Eq, PartialEq, PartialOrd, Ord)] -pub struct Package { - /// Package name and version information - pub handle: PackageHandle, - /// Package category (Generated, SmithyRuntime, AwsRuntime, etc.) - pub category: PackageCategory, - /// Location to the crate on the current file system - pub crate_path: PathBuf, - /// Location to the crate manifest on the current file system - pub manifest_path: PathBuf, - /// Dependencies used by this package - pub local_dependencies: BTreeSet, - /// Whether or not the package should be published - pub publish: Publish, -} - -impl Package { - pub fn new( - handle: PackageHandle, - manifest_path: impl Into, - local_dependencies: BTreeSet, - publish: Publish, - ) -> Self { - let manifest_path = manifest_path.into(); - let category = PackageCategory::from_package_name(&handle.name); - Self { - handle, - category, - crate_path: manifest_path.parent().unwrap().into(), - manifest_path, - local_dependencies, - publish, - } - } - - /// Returns `true` if this package depends on `other` - pub fn locally_depends_on(&self, other: &PackageHandle) -> bool { - self.local_dependencies.contains(other) - } - - /// Returns the expected owners of the crate. - pub fn expected_owners(&self) -> HashSet { - expected_package_owners(&self.category, &self.handle.name) - } -} - -/// Returns the expected owners of the crate. -pub fn expected_package_owners( - _category: &PackageCategory, - _package_name: &str, -) -> HashSet { - [RUST_SDK_CI_OWNER.to_string()].into_iter().collect() -} - /// Batch of packages. pub type PackageBatch = Vec; @@ -145,7 +59,7 @@ pub async fn discover_and_validate_package_batches( fs: Fs, path: impl AsRef, ) -> Result<(Vec, PackageStats)> { - let packages = discover_packages(fs, path.as_ref().into()) + let packages = discover_packages(fs, path.as_ref()) .await? .into_iter() .filter(|package| package.publish == Publish::Allowed) @@ -175,9 +89,9 @@ pub enum Error { /// Discovers all Cargo.toml files under the given path recursively #[async_recursion::async_recursion] -pub async fn discover_manifests(path: PathBuf) -> Result> { +pub async fn discover_manifests(path: &Path) -> Result> { let mut manifests = Vec::new(); - let mut read_dir = fs::read_dir(&path).await?; + let mut read_dir = fs::read_dir(path).await?; while let Some(entry) = read_dir.next_entry().await? { let package_path = entry.path(); if package_path.is_dir() { @@ -185,91 +99,35 @@ pub async fn discover_manifests(path: PathBuf) -> Result> { if manifest_path.exists() { manifests.push(manifest_path); } - manifests.extend(discover_manifests(package_path).await?.into_iter()); + manifests.extend(discover_manifests(&package_path).await?.into_iter()); } } Ok(manifests) } /// Discovers and parses all Cargo.toml files that are packages (as opposed to being exclusively workspaces) -pub async fn discover_packages(fs: Fs, path: PathBuf) -> Result> { +pub async fn discover_packages(fs: Fs, path: &Path) -> Result> { let manifest_paths = discover_manifests(path).await?; read_packages(fs, manifest_paths).await } -/// Parses a semver version number and adds additional error context when parsing fails. -pub fn parse_version(manifest_path: &Path, version: &str) -> Result { - Version::parse(version) - .map_err(|err| Error::InvalidCrateVersion(manifest_path.into(), version.into(), err.into())) -} - -fn read_dependencies(path: &Path, dependencies: &DepsSet) -> Result> { - let mut result = Vec::new(); - for (name, metadata) in dependencies { - match metadata { - Dependency::Simple(_) => {} - Dependency::Detailed(detailed) => { - if detailed.path.is_some() { - let version = detailed - .version - .as_ref() - .map(|version| parse_version(path, version)) - .ok_or_else(|| Error::MissingVersion(path.into(), name.into()))??; - result.push(PackageHandle::new(name, version)); - } - } - Dependency::Inherited(_) => panic!("workspace deps are unsupported"), - } - } - Ok(result) -} - -/// Returns `Ok(None)` when the Cargo.toml is a workspace rather than a package -fn read_package(path: &Path, manifest_bytes: &[u8]) -> Result> { - let mut manifest = Manifest::from_slice(manifest_bytes) - .with_context(|| format!("failed to load package manifest for {:?}", path))?; - manifest.complete_from_path(path)?; - if let Some(package) = manifest.package { - let name = package.name; - let version = parse_version(path, &package.version.unwrap())?; - let handle = PackageHandle { name, version }; - let publish = match package.publish.unwrap() { - cargo_toml::Publish::Flag(true) => Publish::Allowed, - _ => Publish::NotAllowed, - }; - - let mut local_dependencies = BTreeSet::new(); - local_dependencies.extend(read_dependencies(path, &manifest.dependencies)?); - local_dependencies.extend(read_dependencies(path, &manifest.dev_dependencies)?); - local_dependencies.extend(read_dependencies(path, &manifest.build_dependencies)?); - Ok(Some(Package::new( - handle, - path, - local_dependencies, - publish, - ))) - } else { - Ok(None) - } -} - /// Validates that all of the publishable crates use consistent version numbers /// across all of their local dependencies. fn validate_packages(packages: &[Package]) -> Result<()> { let mut versions: BTreeMap = BTreeMap::new(); let track_version = &mut |handle: &PackageHandle| -> Result<(), Error> { if let Some(version) = versions.get(&handle.name) { - if *version != handle.version { + if version != handle.expect_version() { Err(Error::MultipleVersions( (&handle.name).into(), versions[&handle.name].clone(), - handle.version.clone(), + handle.expect_version().clone(), )) } else { Ok(()) } } else { - versions.insert(handle.name.clone(), handle.version.clone()); + versions.insert(handle.name.clone(), handle.expect_version().clone()); Ok(()) } }; @@ -287,7 +145,7 @@ pub async fn read_packages(fs: Fs, manifest_paths: Vec) -> Result = fs.read_file(path).await?; - if let Some(package) = read_package(path, &contents)? { + if let Some(package) = Package::try_load_manifest(path, &contents)? { result.push(package); } } @@ -337,79 +195,14 @@ fn batch_packages(packages: Vec) -> Result> { mod tests { use super::*; use semver::Version; - use std::path::PathBuf; - - fn version(version: &str) -> Version { - Version::parse(version).unwrap() - } - - #[test] - fn read_package_success() { - let manifest = br#" - [package] - name = "test" - version = "1.2.0-preview" - - [build-dependencies] - build_something = "1.3" - local_build_something = { version = "0.2.0", path = "../local_build_something" } - - [dev-dependencies] - dev_something = "1.1" - local_dev_something = { version = "0.1.0", path = "../local_dev_something" } - - [dependencies] - something = "1.0" - local_something = { version = "1.1.3", path = "../local_something" } - "#; - let path: PathBuf = "test/Cargo.toml".into(); - - let package = read_package(&path, manifest) - .expect("parse success") - .expect("is a package"); - assert_eq!("test", package.handle.name); - assert_eq!(version("1.2.0-preview"), package.handle.version); - - let mut expected = BTreeSet::new(); - expected.insert(PackageHandle::new( - "local_build_something", - version("0.2.0"), - )); - expected.insert(PackageHandle::new("local_dev_something", version("0.1.0"))); - expected.insert(PackageHandle::new("local_something", version("1.1.3"))); - assert_eq!(expected, package.local_dependencies); - } - - #[test] - fn read_package_version_requirement_invalid() { - let manifest = br#" - [package] - name = "test" - version = "1.2.0-preview" - - [dependencies] - local_something = { version = "1.0", path = "../local_something" } - "#; - let path: PathBuf = "test/Cargo.toml".into(); - - let error = format!( - "{}", - read_package(&path, manifest).expect_err("should fail") - ); - assert!( - error.contains("Invalid crate version"), - "'{}' should contain 'Invalid crate version'", - error - ); - } fn package(name: &str, dependencies: &[&str]) -> Package { Package::new( - PackageHandle::new(name, Version::parse("1.0.0").unwrap()), + PackageHandle::new(name, Version::parse("1.0.0").ok()), format!("{}/Cargo.toml", name), dependencies .iter() - .map(|d| PackageHandle::new(*d, Version::parse("1.0.0").unwrap())) + .map(|d| PackageHandle::new(*d, Version::parse("1.0.0").ok())) .collect(), Publish::Allowed, ) @@ -487,11 +280,11 @@ mod tests { fn pkg_ver(name: &str, version: &str, dependencies: &[(&str, &str)]) -> Package { Package::new( - PackageHandle::new(name, Version::parse(version).unwrap()), + PackageHandle::new(name, Some(Version::parse(version).unwrap())), format!("{}/Cargo.toml", name), dependencies .iter() - .map(|p| PackageHandle::new(p.0, Version::parse(p.1).unwrap())) + .map(|p| PackageHandle::new(p.0, Some(Version::parse(p.1).unwrap()))) .collect(), Publish::Allowed, ) @@ -540,33 +333,21 @@ mod tests { package("aws-smithy-http-server-typescript", &[]), ]; for pkg in server_packages { - assert_eq!( - [String::from("aws-sdk-rust-ci")] - .into_iter() - .collect::>(), - pkg.expected_owners() - ); + assert_eq!(&["aws-sdk-rust-ci"], pkg.expected_owners()); } } #[test] fn test_expected_package_owners_sdk_crate() { let sdk_package = package("aws-types", &[]); - assert_eq!( - [String::from("aws-sdk-rust-ci")] - .into_iter() - .collect::>(), - sdk_package.expected_owners() - ); + assert_eq!(&["aws-sdk-rust-ci"], sdk_package.expected_owners()); } #[test] fn test_expected_package_owners_smithy_runtime_crate() { let smithy_runtime_package = package("aws-smithy-types", &[]); assert_eq!( - [String::from("aws-sdk-rust-ci")] - .into_iter() - .collect::>(), + &["aws-sdk-rust-ci"], smithy_runtime_package.expected_owners() ); } diff --git a/tools/ci-build/publisher/src/publish.rs b/tools/ci-build/publisher/src/publish.rs index 3c8a1fb96..6fc8ad123 100644 --- a/tools/ci-build/publisher/src/publish.rs +++ b/tools/ci-build/publisher/src/publish.rs @@ -4,13 +4,12 @@ */ use crate::cargo; -use crate::package::PackageHandle; use anyhow::Result; -use smithy_rs_tool_common::shell::ShellOperation; use smithy_rs_tool_common::{ index::CratesIndex, retry::{run_with_retry, BoxError, ErrorClass}, }; +use smithy_rs_tool_common::{package::PackageHandle, shell::ShellOperation}; use std::time::Duration; use std::{path::Path, sync::Arc}; use tracing::info; diff --git a/tools/ci-build/publisher/src/sort.rs b/tools/ci-build/publisher/src/sort.rs index 6362e9c43..3b0e2d570 100644 --- a/tools/ci-build/publisher/src/sort.rs +++ b/tools/ci-build/publisher/src/sort.rs @@ -5,8 +5,8 @@ //! Logic for topological sorting packages by dependencies. -use crate::package::{Package, PackageHandle}; use anyhow::{anyhow, bail, Result}; +use smithy_rs_tool_common::package::{Package, PackageHandle}; use std::collections::{BTreeMap, BTreeSet}; /// Determines the dependency order of the given packages. @@ -72,16 +72,16 @@ fn dependency_order_visit( #[cfg(test)] mod tests { use super::*; - use crate::package::Publish; use semver::Version; + use smithy_rs_tool_common::package::Publish; fn package(name: &str, dependencies: &[&str]) -> Package { Package::new( - PackageHandle::new(name, Version::parse("1.0.0").unwrap()), + PackageHandle::new(name, Version::parse("1.0.0").ok()), format!("{}/Cargo.toml", name), dependencies .iter() - .map(|d| PackageHandle::new(*d, Version::parse("1.0.0").unwrap())) + .map(|d| PackageHandle::new(*d, Version::parse("1.0.0").ok())) .collect(), Publish::Allowed, ) diff --git a/tools/ci-build/publisher/src/subcommand/claim_crate_names.rs b/tools/ci-build/publisher/src/subcommand/claim_crate_names.rs index 11303f368..caef6abb2 100644 --- a/tools/ci-build/publisher/src/subcommand/claim_crate_names.rs +++ b/tools/ci-build/publisher/src/subcommand/claim_crate_names.rs @@ -2,17 +2,17 @@ * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. * SPDX-License-Identifier: Apache-2.0 */ +use crate::publish::is_published; use crate::publish::publish; use crate::subcommand::publish::correct_owner; use crate::{cargo, SDK_REPO_NAME}; use crate::{fs::Fs, package::discover_manifests}; -use crate::{package::PackageHandle, publish::is_published}; use anyhow::{Context, Result}; use cargo_toml::Manifest; use clap::Parser; use dialoguer::Confirm; use semver::Version; -use smithy_rs_tool_common::package::PackageCategory; +use smithy_rs_tool_common::package::PackageHandle; use smithy_rs_tool_common::{git, index::CratesIndex}; use std::time::Duration; use std::{collections::HashSet, fs}; @@ -66,20 +66,19 @@ async fn claim_crate_name(name: &str) -> Result<()> { let crate_dir_path = temporary_directory.path(); create_dummy_lib_crate(Fs::Real, name, crate_dir_path.to_path_buf()).await?; - let category = PackageCategory::from_package_name(name); - let package_handle = PackageHandle::new(name, Version::new(0, 0, 1)); + let package_handle = PackageHandle::new(name, Some(Version::new(0, 0, 1))); publish(&package_handle, crate_dir_path).await?; // Keep things slow to avoid getting throttled by crates.io tokio::time::sleep(Duration::from_secs(2)).await; - correct_owner(&package_handle, &category).await?; + correct_owner(&package_handle).await?; info!("Successfully published `{}`", package_handle); Ok(()) } async fn load_publishable_crate_names(path: &Path) -> Result> { - let manifest_paths = discover_manifests(path.into()).await?; + let manifest_paths = discover_manifests(path).await?; let mut result = HashSet::new(); for manifest_path in &manifest_paths { let content = diff --git a/tools/ci-build/publisher/src/subcommand/fix_manifests.rs b/tools/ci-build/publisher/src/subcommand/fix_manifests.rs index 78814cdad..cb42abc30 100644 --- a/tools/ci-build/publisher/src/subcommand/fix_manifests.rs +++ b/tools/ci-build/publisher/src/subcommand/fix_manifests.rs @@ -10,12 +10,12 @@ //! version numbers in addition to the dependency path. use crate::fs::Fs; -use crate::package::{discover_manifests, parse_version}; +use crate::package::discover_manifests; use crate::SDK_REPO_NAME; use anyhow::{bail, Context, Result}; use clap::Parser; use semver::Version; -use smithy_rs_tool_common::ci::running_in_ci; +use smithy_rs_tool_common::{ci::running_in_ci, package::parse_version}; use std::collections::BTreeMap; use std::ffi::OsStr; use std::path::{Path, PathBuf}; @@ -54,7 +54,7 @@ pub async fn subcommand_fix_manifests( true => Mode::Check, false => Mode::Execute, }; - let manifest_paths = discover_manifests(location.into()).await?; + let manifest_paths = discover_manifests(location).await?; let mut manifests = read_manifests(Fs::Real, manifest_paths).await?; let versions = package_versions(&manifests)?; diff --git a/tools/ci-build/publisher/src/subcommand/generate_version_manifest.rs b/tools/ci-build/publisher/src/subcommand/generate_version_manifest.rs index 6b9ac3539..9a5a53273 100644 --- a/tools/ci-build/publisher/src/subcommand/generate_version_manifest.rs +++ b/tools/ci-build/publisher/src/subcommand/generate_version_manifest.rs @@ -70,7 +70,7 @@ pub async fn subcommand_generate_version_manifest( (None, Some(output_location)) => output_location, _ => bail!("Only one of `--location` or `--output-location` should be provided"), }; - let packages = discover_packages(Fs::Real, input_location.into()) + let packages = discover_packages(Fs::Real, input_location) .await .context("read packages")?; @@ -94,11 +94,12 @@ pub async fn subcommand_generate_version_manifest( matches!(package.category, PackageCategory::AwsSdk) == model_hash.is_some(), "all generated SDK crates should have a model hash" ); + let version = package.handle.expect_version().to_string(); crates.insert( package.handle.name, CrateVersion { category: package.category, - version: package.handle.version.to_string(), + version, source_hash: hash_crate(&package.crate_path).context("hash crate")?, model_hash, }, diff --git a/tools/ci-build/publisher/src/subcommand/publish.rs b/tools/ci-build/publisher/src/subcommand/publish.rs index d724557f9..2c568b3f6 100644 --- a/tools/ci-build/publisher/src/subcommand/publish.rs +++ b/tools/ci-build/publisher/src/subcommand/publish.rs @@ -4,16 +4,13 @@ */ use crate::fs::Fs; -use crate::package::{ - discover_and_validate_package_batches, expected_package_owners, Package, PackageBatch, - PackageHandle, PackageStats, -}; +use crate::package::{discover_and_validate_package_batches, PackageBatch, PackageStats}; use crate::publish::publish; use crate::{cargo, SDK_REPO_CRATE_PATH, SDK_REPO_NAME}; use anyhow::{bail, Context, Result}; use clap::Parser; use dialoguer::Confirm; -use smithy_rs_tool_common::package::PackageCategory; +use smithy_rs_tool_common::package::{Package, PackageHandle}; use smithy_rs_tool_common::retry::{run_with_retry, BoxError, ErrorClass}; use smithy_rs_tool_common::shell::ShellOperation; use smithy_rs_tool_common::{git, index::CratesIndex}; @@ -90,7 +87,7 @@ pub async fn subcommand_publish( for batch in &batches { for package in batch { - correct_owner(&package.handle, &package.category).await?; + correct_owner(&package.handle).await?; } } @@ -110,7 +107,7 @@ pub fn resolve_publish_location(location: &Path) -> PathBuf { async fn is_published(index: Arc, handle: &PackageHandle) -> Result { let name = handle.name.clone(); - let version = handle.version.clone(); + let version = handle.expect_version().clone(); tokio::task::spawn_blocking(move || { smithy_rs_tool_common::index::is_published(index.as_ref(), &name, &version) }) @@ -137,7 +134,7 @@ async fn wait_for_eventual_consistency(index: Arc, package: &Packag } /// Corrects the crate ownership. -pub async fn correct_owner(handle: &PackageHandle, category: &PackageCategory) -> Result<()> { +pub async fn correct_owner(handle: &PackageHandle) -> Result<()> { // https://github.com/orgs/awslabs/teams/smithy-rs-server const SMITHY_RS_SERVER_OWNER: &str = "github:awslabs:smithy-rs-server"; // https://github.com/orgs/awslabs/teams/rust-sdk-owners @@ -149,7 +146,7 @@ pub async fn correct_owner(handle: &PackageHandle, category: &PackageCategory) - Duration::from_secs(5), || async { let actual_owners: HashSet = cargo::GetOwners::new(&handle.name).spawn().await?.into_iter().collect(); - let expected_owners = expected_package_owners(category, &handle.name); + let expected_owners = handle.expected_owners().iter().map(|s| s.to_string()).collect::>(); let owners_to_be_added = expected_owners.difference(&actual_owners); let owners_to_be_removed = actual_owners.difference(&expected_owners); @@ -203,7 +200,8 @@ fn confirm_plan( for package in batch { full_plan.push(format!( "Publish version `{}` of `{}`", - package.handle.version, package.handle.name + package.handle.expect_version(), + package.handle.name )); } full_plan.push("-- wait --".into()); diff --git a/tools/ci-build/runtime-versioner/Cargo.lock b/tools/ci-build/runtime-versioner/Cargo.lock index 025690ed2..6b9818a6c 100644 --- a/tools/ci-build/runtime-versioner/Cargo.lock +++ b/tools/ci-build/runtime-versioner/Cargo.lock @@ -1061,6 +1061,7 @@ version = "0.1.0" dependencies = [ "anyhow", "async-trait", + "cargo_toml", "crates-index", "lazy_static", "regex", diff --git a/tools/ci-build/runtime-versioner/src/command/patch.rs b/tools/ci-build/runtime-versioner/src/command/patch.rs index 2984fbc17..bf701ccdb 100644 --- a/tools/ci-build/runtime-versioner/src/command/patch.rs +++ b/tools/ci-build/runtime-versioner/src/command/patch.rs @@ -9,12 +9,12 @@ use crate::{ PatchRuntime, PatchRuntimeWith, }; use anyhow::{bail, Context, Result}; -use camino::Utf8Path; +use camino::Utf8PathBuf; use cargo_toml::Manifest; use indicatif::{ProgressBar, ProgressStyle}; -use smithy_rs_tool_common::command::sync::CommandExt; +use smithy_rs_tool_common::{command::sync::CommandExt, package::Package}; use std::{fs, time::Duration}; -use toml_edit::DocumentMut; +use toml_edit::{DocumentMut, Item}; pub fn patch(args: PatchRuntime) -> Result<()> { let smithy_rs = step("Resolving smithy-rs", || { @@ -29,22 +29,14 @@ pub fn patch(args: PatchRuntime) -> Result<()> { bail!("aws-sdk-rust has a dirty working tree. Aborting."); } - // Use aws:sdk:assemble to generate both the smithy-rs runtime and AWS SDK - // runtime crates with the correct version numbers. - step("Generating an AWS SDK", || { - smithy_rs - .cmd( - "./gradlew", - // limit services down to minimum required to reduce generation time - ["-Paws.services=+sts,+sso,+ssooidc", "aws:sdk:assemble"], - ) - .expect_success_output("assemble SDK") - })?; - patch_with(PatchRuntimeWith { sdk_path: args.sdk_path, - runtime_crate_path: smithy_rs.root.join("aws/sdk/build/aws-sdk/sdk"), + runtime_crate_path: vec![ + smithy_rs.root.join("rust-runtime"), + smithy_rs.root.join("aws/rust-runtime"), + ], previous_release_tag: args.previous_release_tag, + no_checkout_sdk_release: args.no_checkout_sdk_release, })?; Ok(()) @@ -56,22 +48,24 @@ pub fn patch_with(args: PatchRuntimeWith) -> Result<()> { bail!("aws-sdk-rust has a dirty working tree. Aborting."); } - // Make sure the aws-sdk-rust repo is on the correct release tag - let release_tags = step("Resolving aws-sdk-rust release tags", || { - release_tags(&aws_sdk_rust) - })?; - let previous_release_tag = step("Resolving release tag", || { - previous_release_tag( - &aws_sdk_rust, - &release_tags, - args.previous_release_tag.as_deref(), - ) - })?; - step("Checking out release tag", || { - aws_sdk_rust - .git(["checkout", previous_release_tag.as_str()]) - .expect_success_output("check out release tag in aws-sdk-rust") - })?; + if !args.no_checkout_sdk_release { + // Make sure the aws-sdk-rust repo is on the correct release tag + let release_tags = step("Resolving aws-sdk-rust release tags", || { + release_tags(&aws_sdk_rust) + })?; + let previous_release_tag = step("Resolving release tag", || { + previous_release_tag( + &aws_sdk_rust, + &release_tags, + args.previous_release_tag.as_deref(), + ) + })?; + step("Checking out release tag", || { + aws_sdk_rust + .git(["checkout", previous_release_tag.as_str()]) + .expect_success_output("check out release tag in aws-sdk-rust") + })?; + } // Patch the new runtime crates into the old SDK step("Applying version-only dependencies", || { @@ -80,7 +74,7 @@ pub fn patch_with(args: PatchRuntimeWith) -> Result<()> { step("Patching aws-sdk-rust root Cargo.toml", || { let crates_to_patch = remove_unchanged_dependencies(&aws_sdk_rust, &args.runtime_crate_path)?; - patch_workspace_cargo_toml(&aws_sdk_rust, &args.runtime_crate_path, crates_to_patch) + patch_workspace_cargo_toml(&aws_sdk_rust, crates_to_patch) })?; step("Running cargo update", || { aws_sdk_rust @@ -107,19 +101,18 @@ fn apply_version_only_dependencies(aws_sdk_rust: &Repo) -> Result<()> { } /// Determine if a given crate has a new version vs. the release we're comparing -fn crate_version_has_changed( - crate_name: &str, - aws_sdk_rust: &Repo, - runtime_crate_path: &Utf8Path, -) -> Result { +fn crate_version_has_changed(runtime_crate: &Package, aws_sdk_rust: &Repo) -> Result { let sdk_cargo_toml = aws_sdk_rust .root .join("sdk") - .join(crate_name) + .join(&runtime_crate.handle.name) .join("Cargo.toml"); - let to_patch_cargo_toml = runtime_crate_path.join(crate_name).join("Cargo.toml"); + let to_patch_cargo_toml = &runtime_crate.manifest_path; if !sdk_cargo_toml.exists() { - tracing::trace!("`{crate_name}` is a new crate, so there is nothing to patch."); + tracing::trace!( + "`{}` is a new crate, so there is nothing to patch.", + runtime_crate.handle + ); // This is a new runtime crate, so there is nothing to patch. return Ok(false); } @@ -128,25 +121,25 @@ fn crate_version_has_changed( "{:?} did not exist!", to_patch_cargo_toml ); - let sdk_cargo_toml = Manifest::from_path(sdk_cargo_toml).context("could not parse")?; - let to_patch_toml = Manifest::from_path(to_patch_cargo_toml).context("could not parse")?; + let sdk_cargo_toml = Manifest::from_path(&sdk_cargo_toml) + .context("could not parse SDK Cargo.toml") + .context(sdk_cargo_toml)?; + let to_patch_toml = Manifest::from_path(to_patch_cargo_toml) + .context("could not parse Cargo.toml to patch") + .with_context(|| to_patch_cargo_toml.display().to_string())?; Ok(sdk_cargo_toml.package().version() != to_patch_toml.package().version()) } + fn patch_workspace_cargo_toml( aws_sdk_rust: &Repo, - runtime_crate_path: &Utf8Path, - crates_to_patch: impl Iterator, + crates_to_patch: impl Iterator, ) -> Result<()> { let patch_sections = crates_to_patch - .map(|crate_name| { - let path = runtime_crate_path.join(&crate_name); - assert!( - path.exists(), - "tried to reference a crate that did not exist!" - ); + .map(|runtime_crate| { format!( - "{crate_name} = {{ path = '{}' }}", - path.canonicalize_utf8().unwrap() + "{} = {{ path = '{}' }}", + runtime_crate.handle.name, + runtime_crate.crate_path.canonicalize().unwrap().display() ) }) .collect::>() @@ -166,46 +159,54 @@ fn patch_workspace_cargo_toml( /// Removes Path dependencies referring to unchanged crates & returns a list of crates to patch fn remove_unchanged_dependencies( aws_sdk_rust: &Repo, - runtime_crate_path: &Utf8Path, -) -> Result> { - let all_crates = fs::read_dir(runtime_crate_path) - .context(format!( - "could list crates in directory {:?}", - runtime_crate_path - ))? - .map(|dir| dir.unwrap().file_name()) - .map(|osstr| osstr.into_string().expect("invalid utf-8 directory")) - .collect::>(); + runtime_crate_paths: &[Utf8PathBuf], +) -> Result> { + let mut all_crates = Vec::new(); + for runtime_crate_path in runtime_crate_paths { + let read_dir = fs::read_dir(runtime_crate_path).context(format!( + "could list crates in directory {runtime_crate_path:?}" + ))?; + for directory in read_dir { + let path = directory?.path(); + if let Some(runtime_crate) = Package::try_load_path(path)? { + let name = &runtime_crate.handle.name; + if name.starts_with("aws-") && name != "aws-config" { + all_crates.push(runtime_crate); + } + } + } + } let (crates_to_patch, unchanged_crates): (Vec<_>, Vec<_>) = - all_crates.clone().into_iter().partition(|crate_dir| { - crate_version_has_changed(crate_dir, aws_sdk_rust, runtime_crate_path) + all_crates.clone().into_iter().partition(|runtime_crate| { + crate_version_has_changed(runtime_crate, aws_sdk_rust) .expect("failed to determine change-status") }); for patched_crate in &all_crates { - tracing::trace!("removing unchanged path dependencies for {patched_crate}"); - remove_unchanged_path_dependencies(runtime_crate_path, &unchanged_crates, patched_crate)?; + tracing::trace!( + "removing unchanged path dependencies for {}", + patched_crate.handle + ); + remove_unchanged_path_dependencies(&unchanged_crates, patched_crate)?; } - Ok(crates_to_patch - .into_iter() - .filter(|crte| crte.starts_with("aws-"))) + Ok(crates_to_patch.into_iter()) } -/// Remove `path = ...` from the dependency section for unchanged crates +/// Remove `path = ...` from the dependency section for unchanged crates, +/// and add version numbers for those where necessary. /// /// If we leave these path dependencies in, we'll get an error when we try to patch because the /// version numbers are the same. fn remove_unchanged_path_dependencies( - runtime_crate_path: &Utf8Path, - unchanged_crates: &[String], - patched_crate: &String, + unchanged_crates: &[Package], + patched_crate: &Package, ) -> Result<()> { - let path = runtime_crate_path.join(patched_crate).join("Cargo.toml"); - let manifest = Manifest::from_path(&path)?; - let mut mutable_manifest = fs::read_to_string(&path) + let manifest_path = &patched_crate.manifest_path; + let manifest = Manifest::from_path(manifest_path)?; + let mut mutable_manifest = fs::read_to_string(manifest_path) .context("failed to read file") - .context(path.clone())? + .with_context(|| manifest_path.display().to_string())? .parse::() .context("invalid toml in manifest!")?; let mut updates = false; @@ -215,25 +216,37 @@ fn remove_unchanged_path_dependencies( ]; for (deps_set, key) in sections { for (dependency_name, dependency_metadata) in deps_set.iter() { - if unchanged_crates.iter().any(|crate_name| { - crate_name.as_str() + let runtime_crate = unchanged_crates.iter().find(|rt_crate| { + rt_crate.handle.name.as_str() == dependency_metadata .package() .unwrap_or(dependency_name.as_str()) - }) { + }); + if let Some(runtime_crate) = runtime_crate { let it = &mut mutable_manifest[key][dependency_name]; match it.as_table_like_mut() { - Some(table_like) => table_like.remove("path"), - None => { - panic!("crate `{patched_crate}` depends on crate `{dependency_name}` crate by version instead of by path. Please update it to use path dependencies for all runtime crates."); + Some(table_like) => { + table_like.remove("path"); + if !table_like.contains_key("version") { + table_like.insert( + "version", + Item::Value(runtime_crate.handle.expect_version().to_string().into()), + ); + } } + None => panic!( + "crate `{}` depends on crate `{dependency_name}` crate by version instead \ + of by path. Please update it to use path dependencies for all runtime crates.", + patched_crate.handle + ) }; updates = true } } } if updates { - fs::write(&path, mutable_manifest.to_string()).context("failed to write back manifest")? + fs::write(manifest_path, mutable_manifest.to_string()) + .context("failed to write back manifest")? } Ok(()) } diff --git a/tools/ci-build/runtime-versioner/src/main.rs b/tools/ci-build/runtime-versioner/src/main.rs index 853395cb3..e8602cfec 100644 --- a/tools/ci-build/runtime-versioner/src/main.rs +++ b/tools/ci-build/runtime-versioner/src/main.rs @@ -58,6 +58,13 @@ pub struct PatchRuntime { /// Explicitly state the previous release's tag. Discovers it if not provided. #[arg(long)] previous_release_tag: Option, + /// Disable checking out the release tag in the SDK repo. + /// + /// This is useful if you need to test changes in runtime crates against + /// local changes in the SDK. + #[arg(long)] + no_checkout_sdk_release: bool, + /// Version number for stable crates. /// /// Deprecated: this argument is ignored @@ -75,15 +82,24 @@ pub struct PatchRuntimeWith { /// Path to aws-sdk-rust. #[arg(long)] sdk_path: Utf8PathBuf, - /// Path to runtime crates to patch in. + /// Path(s) to runtime crates to patch in. + /// + /// Multiple paths can be passed in, for example, if patching SDK and Smithy + /// runtime crates that are in different directories. /// /// Note: this doesn't need to be a complete set of runtime crates. It will /// only patch the crates included in the provided path. #[arg(long)] - runtime_crate_path: Utf8PathBuf, + runtime_crate_path: Vec, /// Explicitly state the previous release's tag. Discovers it if not provided. #[arg(long)] previous_release_tag: Option, + /// Disable checking out the release tag in the SDK repo. + /// + /// This is useful if you need to test changes in runtime crates against + /// local changes in the SDK. + #[arg(long)] + no_checkout_sdk_release: bool, } #[derive(clap::Parser, Clone)] diff --git a/tools/ci-build/sdk-lints/Cargo.lock b/tools/ci-build/sdk-lints/Cargo.lock index 17071c9c4..73db9414d 100644 --- a/tools/ci-build/sdk-lints/Cargo.lock +++ b/tools/ci-build/sdk-lints/Cargo.lock @@ -115,6 +115,16 @@ dependencies = [ "toml 0.8.8", ] +[[package]] +name = "cargo_toml" +version = "0.19.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a98356df42a2eb1bd8f1793ae4ee4de48e384dd974ce5eac8eee802edb7492be" +dependencies = [ + "serde", + "toml 0.8.8", +] + [[package]] name = "cc" version = "1.0.83" @@ -848,7 +858,7 @@ name = "sdk-lints" version = "0.1.0" dependencies = [ "anyhow", - "cargo_toml", + "cargo_toml 0.18.0", "clap", "lazy_static", "serde", @@ -952,6 +962,7 @@ version = "0.1.0" dependencies = [ "anyhow", "async-trait", + "cargo_toml 0.19.2", "crates-index", "lazy_static", "regex", diff --git a/tools/ci-build/sdk-versioner/Cargo.lock b/tools/ci-build/sdk-versioner/Cargo.lock index 709b9ae16..dd6572db4 100644 --- a/tools/ci-build/sdk-versioner/Cargo.lock +++ b/tools/ci-build/sdk-versioner/Cargo.lock @@ -105,6 +105,16 @@ version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a2bd12c1caf447e69cd4528f47f94d203fd2582878ecb9e9465484c4148a8223" +[[package]] +name = "cargo_toml" +version = "0.19.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a98356df42a2eb1bd8f1793ae4ee4de48e384dd974ce5eac8eee802edb7492be" +dependencies = [ + "serde", + "toml 0.8.12", +] + [[package]] name = "cc" version = "1.0.90" @@ -947,6 +957,7 @@ version = "0.1.0" dependencies = [ "anyhow", "async-trait", + "cargo_toml", "crates-index", "lazy_static", "regex", diff --git a/tools/ci-build/smithy-rs-tool-common/Cargo.toml b/tools/ci-build/smithy-rs-tool-common/Cargo.toml index a9cbfa762..d9fd0093e 100644 --- a/tools/ci-build/smithy-rs-tool-common/Cargo.toml +++ b/tools/ci-build/smithy-rs-tool-common/Cargo.toml @@ -18,6 +18,7 @@ opt-level = 0 [dependencies] anyhow = "1" async-trait = "0.1.74" +cargo_toml = "0.19.2" crates-index = { version = "2.5.1", features = ["sparse"] } lazy_static = "1" regex = "1.6.0" diff --git a/tools/ci-build/smithy-rs-tool-common/src/lib.rs b/tools/ci-build/smithy-rs-tool-common/src/lib.rs index 567617de2..27e174d72 100644 --- a/tools/ci-build/smithy-rs-tool-common/src/lib.rs +++ b/tools/ci-build/smithy-rs-tool-common/src/lib.rs @@ -15,3 +15,6 @@ pub mod release_tag; pub mod retry; pub mod shell; pub mod versions_manifest; + +// https://github.com/aws-sdk-rust-ci +pub const RUST_SDK_CI_OWNER: &str = "aws-sdk-rust-ci"; diff --git a/tools/ci-build/smithy-rs-tool-common/src/package.rs b/tools/ci-build/smithy-rs-tool-common/src/package.rs index 4197099b9..e6f6a8d23 100644 --- a/tools/ci-build/smithy-rs-tool-common/src/package.rs +++ b/tools/ci-build/smithy-rs-tool-common/src/package.rs @@ -3,7 +3,16 @@ * SPDX-License-Identifier: Apache-2.0 */ +use crate::RUST_SDK_CI_OWNER; +use anyhow::{Context, Result}; +use cargo_toml::{Dependency, DepsSet, Manifest}; +use semver::Version; use serde::{Deserialize, Serialize}; +use std::{ + collections::BTreeSet, + fmt, fs, + path::{Path, PathBuf}, +}; pub const SMITHY_PREFIX: &str = "aws-smithy-"; pub const SDK_PREFIX: &str = "aws-sdk-"; @@ -46,3 +55,291 @@ pub enum PackageStability { Stable, Unstable, } + +/// Information required to identify a package (crate). +#[derive(Clone, Debug, Eq, PartialEq, Ord, PartialOrd)] +pub struct PackageHandle { + pub name: String, + pub version: Option, +} + +impl PackageHandle { + pub fn new(name: impl Into, version: Option) -> Self { + Self { + name: name.into(), + version, + } + } + + /// Returns the expected owners of the crate. + pub fn expected_owners(&self) -> &[&str] { + expected_owners() + } + + /// Returns the version number or panics + pub fn expect_version(&self) -> &Version { + if let Some(version) = &self.version { + version + } else { + panic!("Crate version number required for {}", self.name) + } + } +} + +impl fmt::Display for PackageHandle { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match &self.version { + Some(version) => write!(f, "{}-{version}", self.name), + _ => f.write_str(&self.name), + } + } +} + +#[derive(Copy, Clone, Debug, Eq, PartialEq, PartialOrd, Ord)] +pub enum Publish { + Allowed, + NotAllowed, +} + +/// Represents a crate (called Package since crate is a reserved word). +#[derive(Clone, Debug, Eq, PartialEq, PartialOrd, Ord)] +pub struct Package { + /// Package name and version information + pub handle: PackageHandle, + /// Package category (Generated, SmithyRuntime, AwsRuntime, etc.) + pub category: PackageCategory, + /// Location to the crate on the current file system + pub crate_path: PathBuf, + /// Location to the crate manifest on the current file system + pub manifest_path: PathBuf, + /// Dependencies used by this package + pub local_dependencies: BTreeSet, + /// Whether or not the package should be published + pub publish: Publish, +} + +impl Package { + pub fn new( + handle: PackageHandle, + manifest_path: impl Into, + local_dependencies: BTreeSet, + publish: Publish, + ) -> Self { + let manifest_path = manifest_path.into(); + let category = PackageCategory::from_package_name(&handle.name); + Self { + handle, + category, + crate_path: manifest_path.parent().unwrap().into(), + manifest_path, + local_dependencies, + publish, + } + } + + /// Try to load a package from the given path. + pub fn try_load_path(path: impl AsRef) -> Result> { + let path = path.as_ref(); + let manifest_path = path.join("Cargo.toml"); + if path.is_dir() && manifest_path.exists() { + let manifest = fs::read(&manifest_path) + .context("failed to read manifest") + .with_context(|| format!("{manifest_path:?}"))?; + Self::try_load_manifest(manifest_path, &manifest) + } else { + Ok(None) + } + } + + /// Returns `Ok(None)` when the Cargo.toml is a workspace rather than a package + pub fn try_load_manifest( + manifest_path: impl AsRef, + manifest: &[u8], + ) -> Result> { + let manifest_path = manifest_path.as_ref(); + let mut manifest = Manifest::from_slice(manifest) + .with_context(|| format!("failed to load package manifest for {:?}", manifest_path))?; + manifest.complete_from_path(manifest_path)?; + if let Some(package) = manifest.package { + let name = package.name; + let version = parse_version(manifest_path, &package.version.unwrap())?; + let handle = PackageHandle { + name, + version: Some(version), + }; + let publish = match package.publish.unwrap() { + cargo_toml::Publish::Flag(true) => Publish::Allowed, + _ => Publish::NotAllowed, + }; + + let mut local_dependencies = BTreeSet::new(); + local_dependencies.extend(read_dependencies(manifest_path, &manifest.dependencies)?); + local_dependencies.extend(read_dependencies( + manifest_path, + &manifest.dev_dependencies, + )?); + local_dependencies.extend(read_dependencies( + manifest_path, + &manifest.build_dependencies, + )?); + Ok(Some(Package::new( + handle, + manifest_path, + local_dependencies, + publish, + ))) + } else { + Ok(None) + } + } + + /// Returns `true` if this package depends on `other` + pub fn locally_depends_on(&self, other: &PackageHandle) -> bool { + self.local_dependencies.contains(other) + } + + /// Returns the expected owners of the crate. + pub fn expected_owners(&self) -> &[&str] { + expected_owners() + } +} + +fn expected_owners() -> &'static [&'static str] { + &[RUST_SDK_CI_OWNER] +} + +/// Parses a semver version number and adds additional error context when parsing fails. +pub fn parse_version(manifest_path: &Path, version: &str) -> Result { + Version::parse(version) + .with_context(|| format!("Invalid crate version {version} in {manifest_path:?}.")) +} + +fn read_dependencies(path: &Path, dependencies: &DepsSet) -> Result> { + let mut result = Vec::new(); + for (name, metadata) in dependencies { + match metadata { + Dependency::Simple(_) => {} + Dependency::Detailed(detailed) => { + if detailed.path.is_some() { + let version = detailed + .version + .as_ref() + .map(|version| parse_version(path, version)) + .transpose()?; + result.push(PackageHandle::new(name, version)); + } + } + Dependency::Inherited(_) => panic!("workspace deps are unsupported"), + } + } + Ok(result) +} + +#[cfg(test)] +mod tests { + use super::*; + use semver::Version; + + fn version(version: &str) -> Option { + Some(Version::parse(version).unwrap()) + } + + #[test] + fn try_load_manifest_success() { + let manifest = br#" + [package] + name = "test" + version = "1.2.0-preview" + + [build-dependencies] + build_something = "1.3" + local_build_something = { version = "0.2.0", path = "../local_build_something" } + + [dev-dependencies] + dev_something = "1.1" + local_dev_something = { version = "0.1.0", path = "../local_dev_something" } + + [dependencies] + something = "1.0" + local_something = { version = "1.1.3", path = "../local_something" } + "#; + let path: PathBuf = "test/Cargo.toml".into(); + + let package = Package::try_load_manifest(&path, manifest) + .expect("parse success") + .expect("is a package"); + assert_eq!("test", package.handle.name); + assert_eq!(version("1.2.0-preview"), package.handle.version); + + let mut expected = BTreeSet::new(); + expected.insert(PackageHandle::new( + "local_build_something", + version("0.2.0"), + )); + expected.insert(PackageHandle::new("local_dev_something", version("0.1.0"))); + expected.insert(PackageHandle::new("local_something", version("1.1.3"))); + assert_eq!(expected, package.local_dependencies); + } + + #[test] + fn try_load_manifest_version_requirement_invalid() { + let manifest = br#" + [package] + name = "test" + version = "1.2.0-preview" + + [dependencies] + local_something = { version = "1.0", path = "../local_something" } + "#; + let path: PathBuf = "test/Cargo.toml".into(); + + let error = format!( + "{}", + Package::try_load_manifest(&path, manifest).expect_err("should fail") + ); + assert!( + error.contains("Invalid crate version"), + "'{}' should contain 'Invalid crate version'", + error + ); + } + + fn package(name: &str, dependencies: &[&str]) -> Package { + Package::new( + PackageHandle::new(name, version("1.0.0")), + format!("{}/Cargo.toml", name), + dependencies + .iter() + .map(|d| PackageHandle::new(*d, version("1.0.0"))) + .collect(), + Publish::Allowed, + ) + } + + #[test] + fn test_expected_package_owners_server_crate() { + let server_packages = vec![ + package("aws-smithy-http-server", &[]), + package("aws-smithy-http-server-python", &[]), + package("aws-smithy-http-server-typescript", &[]), + ]; + for pkg in server_packages { + assert_eq!(&["aws-sdk-rust-ci"], pkg.expected_owners()); + } + } + + #[test] + fn test_expected_package_owners_sdk_crate() { + let sdk_package = package("aws-types", &[]); + assert_eq!(&["aws-sdk-rust-ci"], sdk_package.expected_owners()); + } + + #[test] + fn test_expected_package_owners_smithy_runtime_crate() { + let smithy_runtime_package = package("aws-smithy-types", &[]); + assert_eq!( + &["aws-sdk-rust-ci"], + smithy_runtime_package.expected_owners() + ); + } +} diff --git a/tools/ci-cdk/canary-runner/Cargo.lock b/tools/ci-cdk/canary-runner/Cargo.lock index 043ed5ae8..c5a705dbd 100644 --- a/tools/ci-cdk/canary-runner/Cargo.lock +++ b/tools/ci-cdk/canary-runner/Cargo.lock @@ -638,6 +638,16 @@ dependencies = [ "zip", ] +[[package]] +name = "cargo_toml" +version = "0.19.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a98356df42a2eb1bd8f1793ae4ee4de48e384dd974ce5eac8eee802edb7492be" +dependencies = [ + "serde", + "toml 0.8.10", +] + [[package]] name = "cc" version = "1.0.83" @@ -2408,6 +2418,7 @@ version = "0.1.0" dependencies = [ "anyhow", "async-trait", + "cargo_toml", "crates-index", "lazy_static", "regex", -- GitLab