diff --git a/tools/ci-build/changelogger/Cargo.lock b/tools/ci-build/changelogger/Cargo.lock index c7de5cafaa90e2a9b71d06ee64e17150d706019d..593c64701d2515e2cd20ee8df610951a2a15a266 100644 --- a/tools/ci-build/changelogger/Cargo.lock +++ b/tools/ci-build/changelogger/Cargo.lock @@ -49,7 +49,7 @@ version = "0.2.14" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8" dependencies = [ - "hermit-abi", + "hermit-abi 0.1.19", "libc", "winapi", ] @@ -105,6 +105,16 @@ version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a2bd12c1caf447e69cd4528f47f94d203fd2582878ecb9e9465484c4148a8223" +[[package]] +name = "cargo_toml" +version = "0.19.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a98356df42a2eb1bd8f1793ae4ee4de48e384dd974ce5eac8eee802edb7492be" +dependencies = [ + "serde", + "toml 0.8.12", +] + [[package]] name = "cc" version = "1.0.83" @@ -134,7 +144,7 @@ dependencies = [ "smithy-rs-tool-common", "tempfile", "time", - "toml", + "toml 0.5.11", ] [[package]] @@ -192,6 +202,26 @@ version = "0.8.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "06ea2b9bc92be3c2baa9334a323ebca2d6f074ff852cd1d7b11064035cd3868f" +[[package]] +name = "crates-index" +version = "2.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d9efa03a974d583ad530bbfe00e3d0021de7f26217120437b128dc4c331aa4f" +dependencies = [ + "hex", + "home", + "http", + "memchr", + "rustc-hash", + "semver", + "serde", + "serde_derive", + "serde_json", + "smol_str", + "thiserror", + "toml 0.8.12", +] + [[package]] name = "deranged" version = "0.3.10" @@ -283,6 +313,12 @@ version = "0.3.29" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "eb1d22c66e66d9d72e1758f0bd7d4fd0bee04cad842ee34587d68c07e45d088c" +[[package]] +name = "futures-io" +version = "0.3.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a44623e20b9681a318efdd71c299b6b222ed6f231972bfe2f224ebad6311f0c1" + [[package]] name = "futures-sink" version = "0.3.29" @@ -302,9 +338,12 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a19526d624e703a3179b3d322efec918b6246ea0fa51d41124525f00f1cc8104" dependencies = [ "futures-core", + "futures-io", "futures-task", + "memchr", "pin-project-lite", "pin-utils", + "slab", ] [[package]] @@ -359,6 +398,30 @@ dependencies = [ "libc", ] +[[package]] +name = "hermit-abi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d231dfb89cfffdbc30e7fc41579ed6066ad03abda9e567ccafae602b97ec5024" + +[[package]] +name = "hex" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" +dependencies = [ + "serde", +] + +[[package]] +name = "home" +version = "0.5.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3d1354bf6b7235cb4a0576c2619fd4ed18183f689b12b006a0ee7329eeff9a5" +dependencies = [ + "windows-sys 0.52.0", +] + [[package]] name = "http" version = "0.2.11" @@ -574,6 +637,16 @@ dependencies = [ "autocfg", ] +[[package]] +name = "num_cpus" +version = "1.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4161fcb6d602d4d2081af7c3a45852d875a03dd337a6bfdd6e06407b61342a43" +dependencies = [ + "hermit-abi 0.3.9", + "libc", +] + [[package]] name = "num_threads" version = "0.1.6" @@ -821,6 +894,12 @@ version = "0.1.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d626bb9dae77e28219937af045c257c28bfd3f69333c512553507f5f9798cb76" +[[package]] +name = "rustc-hash" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" + [[package]] name = "rustix" version = "0.38.26" @@ -909,6 +988,15 @@ dependencies = [ "serde", ] +[[package]] +name = "serde_spanned" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eb3622f419d1296904700073ea6cc23ad690adbd66f13ea683df73298736f0c1" +dependencies = [ + "serde", +] + [[package]] name = "serde_urlencoded" version = "0.7.1" @@ -936,6 +1024,8 @@ version = "0.1.0" dependencies = [ "anyhow", "async-trait", + "cargo_toml", + "crates-index", "lazy_static", "regex", "reqwest", @@ -943,10 +1033,19 @@ dependencies = [ "serde", "serde_json", "thiserror", - "toml", + "toml 0.5.11", "tracing", ] +[[package]] +name = "smol_str" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6845563ada680337a52d43bb0b29f396f2d911616f6573012645b9e3d048a49" +dependencies = [ + "serde", +] + [[package]] name = "socket2" version = "0.4.10" @@ -1109,6 +1208,7 @@ dependencies = [ "bytes", "libc", "mio", + "num_cpus", "pin-project-lite", "socket2 0.5.5", "windows-sys 0.48.0", @@ -1148,6 +1248,40 @@ dependencies = [ "serde", ] +[[package]] +name = "toml" +version = "0.8.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e9dd1545e8208b4a5af1aa9bbd0b4cf7e9ea08fabc5d0a5c67fcaafa17433aa3" +dependencies = [ + "serde", + "serde_spanned", + "toml_datetime", + "toml_edit", +] + +[[package]] +name = "toml_datetime" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3550f4e9685620ac18a50ed434eb3aec30db8ba93b0287467bca5826ea25baf1" +dependencies = [ + "serde", +] + +[[package]] +name = "toml_edit" +version = "0.22.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e40bb779c5187258fd7aad0eb68cb8706a0a81fa712fbea808ab43c4b8374c4" +dependencies = [ + "indexmap 2.1.0", + "serde", + "serde_spanned", + "toml_datetime", + "winnow", +] + [[package]] name = "tower-service" version = "0.3.2" @@ -1489,6 +1623,15 @@ version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dff9641d1cd4be8d1a070daf9e3773c5f67e78b4d9d42263020c057706765c04" +[[package]] +name = "winnow" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dffa400e67ed5a4dd237983829e66475f0a4a26938c4b04c21baede6262215b8" +dependencies = [ + "memchr", +] + [[package]] name = "winreg" version = "0.50.0" diff --git a/tools/ci-build/publisher/Cargo.lock b/tools/ci-build/publisher/Cargo.lock index 6aacb8502adb06ff39f02747ba9a138d4a963315..9b0a0f2c7f06e22df3a09ae236b277d829caa7c1 100644 --- a/tools/ci-build/publisher/Cargo.lock +++ b/tools/ci-build/publisher/Cargo.lock @@ -135,6 +135,16 @@ dependencies = [ "toml 0.8.12", ] +[[package]] +name = "cargo_toml" +version = "0.19.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a98356df42a2eb1bd8f1793ae4ee4de48e384dd974ce5eac8eee802edb7492be" +dependencies = [ + "serde", + "toml 0.8.12", +] + [[package]] name = "cc" version = "1.0.90" @@ -956,7 +966,7 @@ dependencies = [ "anyhow", "async-recursion", "async-trait", - "cargo_toml", + "cargo_toml 0.16.3", "clap", "dialoguer", "fs-err", @@ -1278,6 +1288,7 @@ version = "0.1.0" dependencies = [ "anyhow", "async-trait", + "cargo_toml 0.19.2", "crates-index", "lazy_static", "regex", diff --git a/tools/ci-build/publisher/src/cargo/publish.rs b/tools/ci-build/publisher/src/cargo/publish.rs index 71e79e3814e257cf2e4103bd1a2499b9b719e157..3f0f2a50d0054f34755f9a1cd5b515483d90804c 100644 --- a/tools/ci-build/publisher/src/cargo/publish.rs +++ b/tools/ci-build/publisher/src/cargo/publish.rs @@ -3,9 +3,11 @@ * SPDX-License-Identifier: Apache-2.0 */ -use crate::package::PackageHandle; use anyhow::Result; -use smithy_rs_tool_common::shell::{capture_error, output_text, ShellOperation}; +use smithy_rs_tool_common::{ + package::PackageHandle, + shell::{capture_error, output_text, ShellOperation}, +}; use std::path::PathBuf; use std::process::Command; use tracing::info; @@ -18,6 +20,10 @@ pub struct Publish { impl Publish { pub fn new(package_handle: PackageHandle, package_path: impl Into) -> Publish { + assert!( + package_handle.version.is_some(), + "crate version number required; given {package_handle}" + ); Publish { program: "cargo", package_handle, @@ -42,12 +48,12 @@ impl ShellOperation for Publish { let (stdout, stderr) = output_text(&output); let already_uploaded_msg = format!( "error: crate version `{}` is already uploaded", - self.package_handle.version + self.package_handle.expect_version() ); if stdout.contains(&already_uploaded_msg) || stderr.contains(&already_uploaded_msg) { info!( - "{}-{} has already been published to crates.io.", - self.package_handle.name, self.package_handle.version + "{} has already been published to crates.io.", + self.package_handle ); } else { return Err(capture_error("cargo publish", &output)); @@ -69,7 +75,7 @@ mod tests { program: "./fake_cargo/cargo_success", package_handle: PackageHandle::new( "aws-sdk-dynamodb", - Version::parse("0.0.22-alpha").unwrap(), + Version::parse("0.0.22-alpha").ok(), ), package_path: env::current_dir().unwrap(), } @@ -82,10 +88,7 @@ mod tests { async fn publish_fails() { let result = Publish { program: "./fake_cargo/cargo_fails", - package_handle: PackageHandle::new( - "something", - Version::parse("0.0.22-alpha").unwrap(), - ), + package_handle: PackageHandle::new("something", Version::parse("0.0.22-alpha").ok()), package_path: env::current_dir().unwrap(), } .spawn() @@ -106,7 +109,7 @@ mod tests { program: "./fake_cargo/cargo_publish_already_published", package_handle: PackageHandle::new( "aws-sdk-dynamodb", - Version::parse("0.0.22-alpha").unwrap(), + Version::parse("0.0.22-alpha").ok(), ), package_path: env::current_dir().unwrap(), } diff --git a/tools/ci-build/publisher/src/package.rs b/tools/ci-build/publisher/src/package.rs index 4fb572a687f1db956f575eebdbe1f73b6bda0e71..7f2f823df2f5831a5240866190c535261b4197cc 100644 --- a/tools/ci-build/publisher/src/package.rs +++ b/tools/ci-build/publisher/src/package.rs @@ -7,101 +7,15 @@ use crate::fs::Fs; use crate::sort::dependency_order; -use crate::RUST_SDK_CI_OWNER; -use anyhow::{Context, Result}; -use cargo_toml::{Dependency, DepsSet, Manifest}; +use anyhow::Result; use semver::Version; -use smithy_rs_tool_common::package::PackageCategory; -use std::collections::{BTreeMap, BTreeSet, HashSet}; +use smithy_rs_tool_common::package::{Package, PackageCategory, PackageHandle, Publish}; use std::error::Error as StdError; -use std::fmt; -use std::path::{Path, PathBuf}; +use std::path::PathBuf; +use std::{collections::BTreeMap, path::Path}; use tokio::fs; use tracing::warn; -/// Information required to identify a package (crate). -#[derive(Clone, Debug, Eq, PartialEq, Ord, PartialOrd)] -pub struct PackageHandle { - pub name: String, - pub version: Version, -} - -impl PackageHandle { - pub fn new(name: impl Into, version: Version) -> Self { - Self { - name: name.into(), - version, - } - } -} - -impl fmt::Display for PackageHandle { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "{}-{}", self.name, self.version) - } -} - -#[derive(Debug, Eq, PartialEq, PartialOrd, Ord)] -pub enum Publish { - Allowed, - NotAllowed, -} - -/// Represents a crate (called Package since crate is a reserved word). -#[derive(Debug, Eq, PartialEq, PartialOrd, Ord)] -pub struct Package { - /// Package name and version information - pub handle: PackageHandle, - /// Package category (Generated, SmithyRuntime, AwsRuntime, etc.) - pub category: PackageCategory, - /// Location to the crate on the current file system - pub crate_path: PathBuf, - /// Location to the crate manifest on the current file system - pub manifest_path: PathBuf, - /// Dependencies used by this package - pub local_dependencies: BTreeSet, - /// Whether or not the package should be published - pub publish: Publish, -} - -impl Package { - pub fn new( - handle: PackageHandle, - manifest_path: impl Into, - local_dependencies: BTreeSet, - publish: Publish, - ) -> Self { - let manifest_path = manifest_path.into(); - let category = PackageCategory::from_package_name(&handle.name); - Self { - handle, - category, - crate_path: manifest_path.parent().unwrap().into(), - manifest_path, - local_dependencies, - publish, - } - } - - /// Returns `true` if this package depends on `other` - pub fn locally_depends_on(&self, other: &PackageHandle) -> bool { - self.local_dependencies.contains(other) - } - - /// Returns the expected owners of the crate. - pub fn expected_owners(&self) -> HashSet { - expected_package_owners(&self.category, &self.handle.name) - } -} - -/// Returns the expected owners of the crate. -pub fn expected_package_owners( - _category: &PackageCategory, - _package_name: &str, -) -> HashSet { - [RUST_SDK_CI_OWNER.to_string()].into_iter().collect() -} - /// Batch of packages. pub type PackageBatch = Vec; @@ -145,7 +59,7 @@ pub async fn discover_and_validate_package_batches( fs: Fs, path: impl AsRef, ) -> Result<(Vec, PackageStats)> { - let packages = discover_packages(fs, path.as_ref().into()) + let packages = discover_packages(fs, path.as_ref()) .await? .into_iter() .filter(|package| package.publish == Publish::Allowed) @@ -175,9 +89,9 @@ pub enum Error { /// Discovers all Cargo.toml files under the given path recursively #[async_recursion::async_recursion] -pub async fn discover_manifests(path: PathBuf) -> Result> { +pub async fn discover_manifests(path: &Path) -> Result> { let mut manifests = Vec::new(); - let mut read_dir = fs::read_dir(&path).await?; + let mut read_dir = fs::read_dir(path).await?; while let Some(entry) = read_dir.next_entry().await? { let package_path = entry.path(); if package_path.is_dir() { @@ -185,91 +99,35 @@ pub async fn discover_manifests(path: PathBuf) -> Result> { if manifest_path.exists() { manifests.push(manifest_path); } - manifests.extend(discover_manifests(package_path).await?.into_iter()); + manifests.extend(discover_manifests(&package_path).await?.into_iter()); } } Ok(manifests) } /// Discovers and parses all Cargo.toml files that are packages (as opposed to being exclusively workspaces) -pub async fn discover_packages(fs: Fs, path: PathBuf) -> Result> { +pub async fn discover_packages(fs: Fs, path: &Path) -> Result> { let manifest_paths = discover_manifests(path).await?; read_packages(fs, manifest_paths).await } -/// Parses a semver version number and adds additional error context when parsing fails. -pub fn parse_version(manifest_path: &Path, version: &str) -> Result { - Version::parse(version) - .map_err(|err| Error::InvalidCrateVersion(manifest_path.into(), version.into(), err.into())) -} - -fn read_dependencies(path: &Path, dependencies: &DepsSet) -> Result> { - let mut result = Vec::new(); - for (name, metadata) in dependencies { - match metadata { - Dependency::Simple(_) => {} - Dependency::Detailed(detailed) => { - if detailed.path.is_some() { - let version = detailed - .version - .as_ref() - .map(|version| parse_version(path, version)) - .ok_or_else(|| Error::MissingVersion(path.into(), name.into()))??; - result.push(PackageHandle::new(name, version)); - } - } - Dependency::Inherited(_) => panic!("workspace deps are unsupported"), - } - } - Ok(result) -} - -/// Returns `Ok(None)` when the Cargo.toml is a workspace rather than a package -fn read_package(path: &Path, manifest_bytes: &[u8]) -> Result> { - let mut manifest = Manifest::from_slice(manifest_bytes) - .with_context(|| format!("failed to load package manifest for {:?}", path))?; - manifest.complete_from_path(path)?; - if let Some(package) = manifest.package { - let name = package.name; - let version = parse_version(path, &package.version.unwrap())?; - let handle = PackageHandle { name, version }; - let publish = match package.publish.unwrap() { - cargo_toml::Publish::Flag(true) => Publish::Allowed, - _ => Publish::NotAllowed, - }; - - let mut local_dependencies = BTreeSet::new(); - local_dependencies.extend(read_dependencies(path, &manifest.dependencies)?); - local_dependencies.extend(read_dependencies(path, &manifest.dev_dependencies)?); - local_dependencies.extend(read_dependencies(path, &manifest.build_dependencies)?); - Ok(Some(Package::new( - handle, - path, - local_dependencies, - publish, - ))) - } else { - Ok(None) - } -} - /// Validates that all of the publishable crates use consistent version numbers /// across all of their local dependencies. fn validate_packages(packages: &[Package]) -> Result<()> { let mut versions: BTreeMap = BTreeMap::new(); let track_version = &mut |handle: &PackageHandle| -> Result<(), Error> { if let Some(version) = versions.get(&handle.name) { - if *version != handle.version { + if version != handle.expect_version() { Err(Error::MultipleVersions( (&handle.name).into(), versions[&handle.name].clone(), - handle.version.clone(), + handle.expect_version().clone(), )) } else { Ok(()) } } else { - versions.insert(handle.name.clone(), handle.version.clone()); + versions.insert(handle.name.clone(), handle.expect_version().clone()); Ok(()) } }; @@ -287,7 +145,7 @@ pub async fn read_packages(fs: Fs, manifest_paths: Vec) -> Result = fs.read_file(path).await?; - if let Some(package) = read_package(path, &contents)? { + if let Some(package) = Package::try_load_manifest(path, &contents)? { result.push(package); } } @@ -337,79 +195,14 @@ fn batch_packages(packages: Vec) -> Result> { mod tests { use super::*; use semver::Version; - use std::path::PathBuf; - - fn version(version: &str) -> Version { - Version::parse(version).unwrap() - } - - #[test] - fn read_package_success() { - let manifest = br#" - [package] - name = "test" - version = "1.2.0-preview" - - [build-dependencies] - build_something = "1.3" - local_build_something = { version = "0.2.0", path = "../local_build_something" } - - [dev-dependencies] - dev_something = "1.1" - local_dev_something = { version = "0.1.0", path = "../local_dev_something" } - - [dependencies] - something = "1.0" - local_something = { version = "1.1.3", path = "../local_something" } - "#; - let path: PathBuf = "test/Cargo.toml".into(); - - let package = read_package(&path, manifest) - .expect("parse success") - .expect("is a package"); - assert_eq!("test", package.handle.name); - assert_eq!(version("1.2.0-preview"), package.handle.version); - - let mut expected = BTreeSet::new(); - expected.insert(PackageHandle::new( - "local_build_something", - version("0.2.0"), - )); - expected.insert(PackageHandle::new("local_dev_something", version("0.1.0"))); - expected.insert(PackageHandle::new("local_something", version("1.1.3"))); - assert_eq!(expected, package.local_dependencies); - } - - #[test] - fn read_package_version_requirement_invalid() { - let manifest = br#" - [package] - name = "test" - version = "1.2.0-preview" - - [dependencies] - local_something = { version = "1.0", path = "../local_something" } - "#; - let path: PathBuf = "test/Cargo.toml".into(); - - let error = format!( - "{}", - read_package(&path, manifest).expect_err("should fail") - ); - assert!( - error.contains("Invalid crate version"), - "'{}' should contain 'Invalid crate version'", - error - ); - } fn package(name: &str, dependencies: &[&str]) -> Package { Package::new( - PackageHandle::new(name, Version::parse("1.0.0").unwrap()), + PackageHandle::new(name, Version::parse("1.0.0").ok()), format!("{}/Cargo.toml", name), dependencies .iter() - .map(|d| PackageHandle::new(*d, Version::parse("1.0.0").unwrap())) + .map(|d| PackageHandle::new(*d, Version::parse("1.0.0").ok())) .collect(), Publish::Allowed, ) @@ -487,11 +280,11 @@ mod tests { fn pkg_ver(name: &str, version: &str, dependencies: &[(&str, &str)]) -> Package { Package::new( - PackageHandle::new(name, Version::parse(version).unwrap()), + PackageHandle::new(name, Some(Version::parse(version).unwrap())), format!("{}/Cargo.toml", name), dependencies .iter() - .map(|p| PackageHandle::new(p.0, Version::parse(p.1).unwrap())) + .map(|p| PackageHandle::new(p.0, Some(Version::parse(p.1).unwrap()))) .collect(), Publish::Allowed, ) @@ -540,33 +333,21 @@ mod tests { package("aws-smithy-http-server-typescript", &[]), ]; for pkg in server_packages { - assert_eq!( - [String::from("aws-sdk-rust-ci")] - .into_iter() - .collect::>(), - pkg.expected_owners() - ); + assert_eq!(&["aws-sdk-rust-ci"], pkg.expected_owners()); } } #[test] fn test_expected_package_owners_sdk_crate() { let sdk_package = package("aws-types", &[]); - assert_eq!( - [String::from("aws-sdk-rust-ci")] - .into_iter() - .collect::>(), - sdk_package.expected_owners() - ); + assert_eq!(&["aws-sdk-rust-ci"], sdk_package.expected_owners()); } #[test] fn test_expected_package_owners_smithy_runtime_crate() { let smithy_runtime_package = package("aws-smithy-types", &[]); assert_eq!( - [String::from("aws-sdk-rust-ci")] - .into_iter() - .collect::>(), + &["aws-sdk-rust-ci"], smithy_runtime_package.expected_owners() ); } diff --git a/tools/ci-build/publisher/src/publish.rs b/tools/ci-build/publisher/src/publish.rs index 3c8a1fb96f5d8458346366a063c5047f8ac82071..6fc8ad12397b1c53df2ab4263e443462598c74ff 100644 --- a/tools/ci-build/publisher/src/publish.rs +++ b/tools/ci-build/publisher/src/publish.rs @@ -4,13 +4,12 @@ */ use crate::cargo; -use crate::package::PackageHandle; use anyhow::Result; -use smithy_rs_tool_common::shell::ShellOperation; use smithy_rs_tool_common::{ index::CratesIndex, retry::{run_with_retry, BoxError, ErrorClass}, }; +use smithy_rs_tool_common::{package::PackageHandle, shell::ShellOperation}; use std::time::Duration; use std::{path::Path, sync::Arc}; use tracing::info; diff --git a/tools/ci-build/publisher/src/sort.rs b/tools/ci-build/publisher/src/sort.rs index 6362e9c43daa60be94382420e158c8dd3aa3fbef..3b0e2d5704f61903f971a08848b1383af1665385 100644 --- a/tools/ci-build/publisher/src/sort.rs +++ b/tools/ci-build/publisher/src/sort.rs @@ -5,8 +5,8 @@ //! Logic for topological sorting packages by dependencies. -use crate::package::{Package, PackageHandle}; use anyhow::{anyhow, bail, Result}; +use smithy_rs_tool_common::package::{Package, PackageHandle}; use std::collections::{BTreeMap, BTreeSet}; /// Determines the dependency order of the given packages. @@ -72,16 +72,16 @@ fn dependency_order_visit( #[cfg(test)] mod tests { use super::*; - use crate::package::Publish; use semver::Version; + use smithy_rs_tool_common::package::Publish; fn package(name: &str, dependencies: &[&str]) -> Package { Package::new( - PackageHandle::new(name, Version::parse("1.0.0").unwrap()), + PackageHandle::new(name, Version::parse("1.0.0").ok()), format!("{}/Cargo.toml", name), dependencies .iter() - .map(|d| PackageHandle::new(*d, Version::parse("1.0.0").unwrap())) + .map(|d| PackageHandle::new(*d, Version::parse("1.0.0").ok())) .collect(), Publish::Allowed, ) diff --git a/tools/ci-build/publisher/src/subcommand/claim_crate_names.rs b/tools/ci-build/publisher/src/subcommand/claim_crate_names.rs index 11303f36850614fdb7355cd1a0102a15702b18bd..caef6abb22a6100664a1c5dc118ad971cf0c49b5 100644 --- a/tools/ci-build/publisher/src/subcommand/claim_crate_names.rs +++ b/tools/ci-build/publisher/src/subcommand/claim_crate_names.rs @@ -2,17 +2,17 @@ * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. * SPDX-License-Identifier: Apache-2.0 */ +use crate::publish::is_published; use crate::publish::publish; use crate::subcommand::publish::correct_owner; use crate::{cargo, SDK_REPO_NAME}; use crate::{fs::Fs, package::discover_manifests}; -use crate::{package::PackageHandle, publish::is_published}; use anyhow::{Context, Result}; use cargo_toml::Manifest; use clap::Parser; use dialoguer::Confirm; use semver::Version; -use smithy_rs_tool_common::package::PackageCategory; +use smithy_rs_tool_common::package::PackageHandle; use smithy_rs_tool_common::{git, index::CratesIndex}; use std::time::Duration; use std::{collections::HashSet, fs}; @@ -66,20 +66,19 @@ async fn claim_crate_name(name: &str) -> Result<()> { let crate_dir_path = temporary_directory.path(); create_dummy_lib_crate(Fs::Real, name, crate_dir_path.to_path_buf()).await?; - let category = PackageCategory::from_package_name(name); - let package_handle = PackageHandle::new(name, Version::new(0, 0, 1)); + let package_handle = PackageHandle::new(name, Some(Version::new(0, 0, 1))); publish(&package_handle, crate_dir_path).await?; // Keep things slow to avoid getting throttled by crates.io tokio::time::sleep(Duration::from_secs(2)).await; - correct_owner(&package_handle, &category).await?; + correct_owner(&package_handle).await?; info!("Successfully published `{}`", package_handle); Ok(()) } async fn load_publishable_crate_names(path: &Path) -> Result> { - let manifest_paths = discover_manifests(path.into()).await?; + let manifest_paths = discover_manifests(path).await?; let mut result = HashSet::new(); for manifest_path in &manifest_paths { let content = diff --git a/tools/ci-build/publisher/src/subcommand/fix_manifests.rs b/tools/ci-build/publisher/src/subcommand/fix_manifests.rs index 78814cdad50ad53c856057b5844ba68343132cef..cb42abc3081dceb553e77d0c0e94606880a63f85 100644 --- a/tools/ci-build/publisher/src/subcommand/fix_manifests.rs +++ b/tools/ci-build/publisher/src/subcommand/fix_manifests.rs @@ -10,12 +10,12 @@ //! version numbers in addition to the dependency path. use crate::fs::Fs; -use crate::package::{discover_manifests, parse_version}; +use crate::package::discover_manifests; use crate::SDK_REPO_NAME; use anyhow::{bail, Context, Result}; use clap::Parser; use semver::Version; -use smithy_rs_tool_common::ci::running_in_ci; +use smithy_rs_tool_common::{ci::running_in_ci, package::parse_version}; use std::collections::BTreeMap; use std::ffi::OsStr; use std::path::{Path, PathBuf}; @@ -54,7 +54,7 @@ pub async fn subcommand_fix_manifests( true => Mode::Check, false => Mode::Execute, }; - let manifest_paths = discover_manifests(location.into()).await?; + let manifest_paths = discover_manifests(location).await?; let mut manifests = read_manifests(Fs::Real, manifest_paths).await?; let versions = package_versions(&manifests)?; diff --git a/tools/ci-build/publisher/src/subcommand/generate_version_manifest.rs b/tools/ci-build/publisher/src/subcommand/generate_version_manifest.rs index 6b9ac35397b24b907a27255e3715407507fae675..9a5a53273e9ebd30085a037dc38046f9f390699f 100644 --- a/tools/ci-build/publisher/src/subcommand/generate_version_manifest.rs +++ b/tools/ci-build/publisher/src/subcommand/generate_version_manifest.rs @@ -70,7 +70,7 @@ pub async fn subcommand_generate_version_manifest( (None, Some(output_location)) => output_location, _ => bail!("Only one of `--location` or `--output-location` should be provided"), }; - let packages = discover_packages(Fs::Real, input_location.into()) + let packages = discover_packages(Fs::Real, input_location) .await .context("read packages")?; @@ -94,11 +94,12 @@ pub async fn subcommand_generate_version_manifest( matches!(package.category, PackageCategory::AwsSdk) == model_hash.is_some(), "all generated SDK crates should have a model hash" ); + let version = package.handle.expect_version().to_string(); crates.insert( package.handle.name, CrateVersion { category: package.category, - version: package.handle.version.to_string(), + version, source_hash: hash_crate(&package.crate_path).context("hash crate")?, model_hash, }, diff --git a/tools/ci-build/publisher/src/subcommand/publish.rs b/tools/ci-build/publisher/src/subcommand/publish.rs index d724557f9be82b9874286d114d7622ca26949909..2c568b3f6565b3a5601a3a6919e2e112b6f8b9a9 100644 --- a/tools/ci-build/publisher/src/subcommand/publish.rs +++ b/tools/ci-build/publisher/src/subcommand/publish.rs @@ -4,16 +4,13 @@ */ use crate::fs::Fs; -use crate::package::{ - discover_and_validate_package_batches, expected_package_owners, Package, PackageBatch, - PackageHandle, PackageStats, -}; +use crate::package::{discover_and_validate_package_batches, PackageBatch, PackageStats}; use crate::publish::publish; use crate::{cargo, SDK_REPO_CRATE_PATH, SDK_REPO_NAME}; use anyhow::{bail, Context, Result}; use clap::Parser; use dialoguer::Confirm; -use smithy_rs_tool_common::package::PackageCategory; +use smithy_rs_tool_common::package::{Package, PackageHandle}; use smithy_rs_tool_common::retry::{run_with_retry, BoxError, ErrorClass}; use smithy_rs_tool_common::shell::ShellOperation; use smithy_rs_tool_common::{git, index::CratesIndex}; @@ -90,7 +87,7 @@ pub async fn subcommand_publish( for batch in &batches { for package in batch { - correct_owner(&package.handle, &package.category).await?; + correct_owner(&package.handle).await?; } } @@ -110,7 +107,7 @@ pub fn resolve_publish_location(location: &Path) -> PathBuf { async fn is_published(index: Arc, handle: &PackageHandle) -> Result { let name = handle.name.clone(); - let version = handle.version.clone(); + let version = handle.expect_version().clone(); tokio::task::spawn_blocking(move || { smithy_rs_tool_common::index::is_published(index.as_ref(), &name, &version) }) @@ -137,7 +134,7 @@ async fn wait_for_eventual_consistency(index: Arc, package: &Packag } /// Corrects the crate ownership. -pub async fn correct_owner(handle: &PackageHandle, category: &PackageCategory) -> Result<()> { +pub async fn correct_owner(handle: &PackageHandle) -> Result<()> { // https://github.com/orgs/awslabs/teams/smithy-rs-server const SMITHY_RS_SERVER_OWNER: &str = "github:awslabs:smithy-rs-server"; // https://github.com/orgs/awslabs/teams/rust-sdk-owners @@ -149,7 +146,7 @@ pub async fn correct_owner(handle: &PackageHandle, category: &PackageCategory) - Duration::from_secs(5), || async { let actual_owners: HashSet = cargo::GetOwners::new(&handle.name).spawn().await?.into_iter().collect(); - let expected_owners = expected_package_owners(category, &handle.name); + let expected_owners = handle.expected_owners().iter().map(|s| s.to_string()).collect::>(); let owners_to_be_added = expected_owners.difference(&actual_owners); let owners_to_be_removed = actual_owners.difference(&expected_owners); @@ -203,7 +200,8 @@ fn confirm_plan( for package in batch { full_plan.push(format!( "Publish version `{}` of `{}`", - package.handle.version, package.handle.name + package.handle.expect_version(), + package.handle.name )); } full_plan.push("-- wait --".into()); diff --git a/tools/ci-build/runtime-versioner/Cargo.lock b/tools/ci-build/runtime-versioner/Cargo.lock index 025690ed202511af1e54f51e039502d7efdb083f..6b9818a6c41fb48065a173a49dd5153dac102569 100644 --- a/tools/ci-build/runtime-versioner/Cargo.lock +++ b/tools/ci-build/runtime-versioner/Cargo.lock @@ -1061,6 +1061,7 @@ version = "0.1.0" dependencies = [ "anyhow", "async-trait", + "cargo_toml", "crates-index", "lazy_static", "regex", diff --git a/tools/ci-build/runtime-versioner/src/command/patch.rs b/tools/ci-build/runtime-versioner/src/command/patch.rs index 2984fbc17c26f558fd30084f7c3de37ba60aaf69..bf701ccdb9bbae82d20f9502fba50a1b163ef632 100644 --- a/tools/ci-build/runtime-versioner/src/command/patch.rs +++ b/tools/ci-build/runtime-versioner/src/command/patch.rs @@ -9,12 +9,12 @@ use crate::{ PatchRuntime, PatchRuntimeWith, }; use anyhow::{bail, Context, Result}; -use camino::Utf8Path; +use camino::Utf8PathBuf; use cargo_toml::Manifest; use indicatif::{ProgressBar, ProgressStyle}; -use smithy_rs_tool_common::command::sync::CommandExt; +use smithy_rs_tool_common::{command::sync::CommandExt, package::Package}; use std::{fs, time::Duration}; -use toml_edit::DocumentMut; +use toml_edit::{DocumentMut, Item}; pub fn patch(args: PatchRuntime) -> Result<()> { let smithy_rs = step("Resolving smithy-rs", || { @@ -29,22 +29,14 @@ pub fn patch(args: PatchRuntime) -> Result<()> { bail!("aws-sdk-rust has a dirty working tree. Aborting."); } - // Use aws:sdk:assemble to generate both the smithy-rs runtime and AWS SDK - // runtime crates with the correct version numbers. - step("Generating an AWS SDK", || { - smithy_rs - .cmd( - "./gradlew", - // limit services down to minimum required to reduce generation time - ["-Paws.services=+sts,+sso,+ssooidc", "aws:sdk:assemble"], - ) - .expect_success_output("assemble SDK") - })?; - patch_with(PatchRuntimeWith { sdk_path: args.sdk_path, - runtime_crate_path: smithy_rs.root.join("aws/sdk/build/aws-sdk/sdk"), + runtime_crate_path: vec![ + smithy_rs.root.join("rust-runtime"), + smithy_rs.root.join("aws/rust-runtime"), + ], previous_release_tag: args.previous_release_tag, + no_checkout_sdk_release: args.no_checkout_sdk_release, })?; Ok(()) @@ -56,22 +48,24 @@ pub fn patch_with(args: PatchRuntimeWith) -> Result<()> { bail!("aws-sdk-rust has a dirty working tree. Aborting."); } - // Make sure the aws-sdk-rust repo is on the correct release tag - let release_tags = step("Resolving aws-sdk-rust release tags", || { - release_tags(&aws_sdk_rust) - })?; - let previous_release_tag = step("Resolving release tag", || { - previous_release_tag( - &aws_sdk_rust, - &release_tags, - args.previous_release_tag.as_deref(), - ) - })?; - step("Checking out release tag", || { - aws_sdk_rust - .git(["checkout", previous_release_tag.as_str()]) - .expect_success_output("check out release tag in aws-sdk-rust") - })?; + if !args.no_checkout_sdk_release { + // Make sure the aws-sdk-rust repo is on the correct release tag + let release_tags = step("Resolving aws-sdk-rust release tags", || { + release_tags(&aws_sdk_rust) + })?; + let previous_release_tag = step("Resolving release tag", || { + previous_release_tag( + &aws_sdk_rust, + &release_tags, + args.previous_release_tag.as_deref(), + ) + })?; + step("Checking out release tag", || { + aws_sdk_rust + .git(["checkout", previous_release_tag.as_str()]) + .expect_success_output("check out release tag in aws-sdk-rust") + })?; + } // Patch the new runtime crates into the old SDK step("Applying version-only dependencies", || { @@ -80,7 +74,7 @@ pub fn patch_with(args: PatchRuntimeWith) -> Result<()> { step("Patching aws-sdk-rust root Cargo.toml", || { let crates_to_patch = remove_unchanged_dependencies(&aws_sdk_rust, &args.runtime_crate_path)?; - patch_workspace_cargo_toml(&aws_sdk_rust, &args.runtime_crate_path, crates_to_patch) + patch_workspace_cargo_toml(&aws_sdk_rust, crates_to_patch) })?; step("Running cargo update", || { aws_sdk_rust @@ -107,19 +101,18 @@ fn apply_version_only_dependencies(aws_sdk_rust: &Repo) -> Result<()> { } /// Determine if a given crate has a new version vs. the release we're comparing -fn crate_version_has_changed( - crate_name: &str, - aws_sdk_rust: &Repo, - runtime_crate_path: &Utf8Path, -) -> Result { +fn crate_version_has_changed(runtime_crate: &Package, aws_sdk_rust: &Repo) -> Result { let sdk_cargo_toml = aws_sdk_rust .root .join("sdk") - .join(crate_name) + .join(&runtime_crate.handle.name) .join("Cargo.toml"); - let to_patch_cargo_toml = runtime_crate_path.join(crate_name).join("Cargo.toml"); + let to_patch_cargo_toml = &runtime_crate.manifest_path; if !sdk_cargo_toml.exists() { - tracing::trace!("`{crate_name}` is a new crate, so there is nothing to patch."); + tracing::trace!( + "`{}` is a new crate, so there is nothing to patch.", + runtime_crate.handle + ); // This is a new runtime crate, so there is nothing to patch. return Ok(false); } @@ -128,25 +121,25 @@ fn crate_version_has_changed( "{:?} did not exist!", to_patch_cargo_toml ); - let sdk_cargo_toml = Manifest::from_path(sdk_cargo_toml).context("could not parse")?; - let to_patch_toml = Manifest::from_path(to_patch_cargo_toml).context("could not parse")?; + let sdk_cargo_toml = Manifest::from_path(&sdk_cargo_toml) + .context("could not parse SDK Cargo.toml") + .context(sdk_cargo_toml)?; + let to_patch_toml = Manifest::from_path(to_patch_cargo_toml) + .context("could not parse Cargo.toml to patch") + .with_context(|| to_patch_cargo_toml.display().to_string())?; Ok(sdk_cargo_toml.package().version() != to_patch_toml.package().version()) } + fn patch_workspace_cargo_toml( aws_sdk_rust: &Repo, - runtime_crate_path: &Utf8Path, - crates_to_patch: impl Iterator, + crates_to_patch: impl Iterator, ) -> Result<()> { let patch_sections = crates_to_patch - .map(|crate_name| { - let path = runtime_crate_path.join(&crate_name); - assert!( - path.exists(), - "tried to reference a crate that did not exist!" - ); + .map(|runtime_crate| { format!( - "{crate_name} = {{ path = '{}' }}", - path.canonicalize_utf8().unwrap() + "{} = {{ path = '{}' }}", + runtime_crate.handle.name, + runtime_crate.crate_path.canonicalize().unwrap().display() ) }) .collect::>() @@ -166,46 +159,54 @@ fn patch_workspace_cargo_toml( /// Removes Path dependencies referring to unchanged crates & returns a list of crates to patch fn remove_unchanged_dependencies( aws_sdk_rust: &Repo, - runtime_crate_path: &Utf8Path, -) -> Result> { - let all_crates = fs::read_dir(runtime_crate_path) - .context(format!( - "could list crates in directory {:?}", - runtime_crate_path - ))? - .map(|dir| dir.unwrap().file_name()) - .map(|osstr| osstr.into_string().expect("invalid utf-8 directory")) - .collect::>(); + runtime_crate_paths: &[Utf8PathBuf], +) -> Result> { + let mut all_crates = Vec::new(); + for runtime_crate_path in runtime_crate_paths { + let read_dir = fs::read_dir(runtime_crate_path).context(format!( + "could list crates in directory {runtime_crate_path:?}" + ))?; + for directory in read_dir { + let path = directory?.path(); + if let Some(runtime_crate) = Package::try_load_path(path)? { + let name = &runtime_crate.handle.name; + if name.starts_with("aws-") && name != "aws-config" { + all_crates.push(runtime_crate); + } + } + } + } let (crates_to_patch, unchanged_crates): (Vec<_>, Vec<_>) = - all_crates.clone().into_iter().partition(|crate_dir| { - crate_version_has_changed(crate_dir, aws_sdk_rust, runtime_crate_path) + all_crates.clone().into_iter().partition(|runtime_crate| { + crate_version_has_changed(runtime_crate, aws_sdk_rust) .expect("failed to determine change-status") }); for patched_crate in &all_crates { - tracing::trace!("removing unchanged path dependencies for {patched_crate}"); - remove_unchanged_path_dependencies(runtime_crate_path, &unchanged_crates, patched_crate)?; + tracing::trace!( + "removing unchanged path dependencies for {}", + patched_crate.handle + ); + remove_unchanged_path_dependencies(&unchanged_crates, patched_crate)?; } - Ok(crates_to_patch - .into_iter() - .filter(|crte| crte.starts_with("aws-"))) + Ok(crates_to_patch.into_iter()) } -/// Remove `path = ...` from the dependency section for unchanged crates +/// Remove `path = ...` from the dependency section for unchanged crates, +/// and add version numbers for those where necessary. /// /// If we leave these path dependencies in, we'll get an error when we try to patch because the /// version numbers are the same. fn remove_unchanged_path_dependencies( - runtime_crate_path: &Utf8Path, - unchanged_crates: &[String], - patched_crate: &String, + unchanged_crates: &[Package], + patched_crate: &Package, ) -> Result<()> { - let path = runtime_crate_path.join(patched_crate).join("Cargo.toml"); - let manifest = Manifest::from_path(&path)?; - let mut mutable_manifest = fs::read_to_string(&path) + let manifest_path = &patched_crate.manifest_path; + let manifest = Manifest::from_path(manifest_path)?; + let mut mutable_manifest = fs::read_to_string(manifest_path) .context("failed to read file") - .context(path.clone())? + .with_context(|| manifest_path.display().to_string())? .parse::() .context("invalid toml in manifest!")?; let mut updates = false; @@ -215,25 +216,37 @@ fn remove_unchanged_path_dependencies( ]; for (deps_set, key) in sections { for (dependency_name, dependency_metadata) in deps_set.iter() { - if unchanged_crates.iter().any(|crate_name| { - crate_name.as_str() + let runtime_crate = unchanged_crates.iter().find(|rt_crate| { + rt_crate.handle.name.as_str() == dependency_metadata .package() .unwrap_or(dependency_name.as_str()) - }) { + }); + if let Some(runtime_crate) = runtime_crate { let it = &mut mutable_manifest[key][dependency_name]; match it.as_table_like_mut() { - Some(table_like) => table_like.remove("path"), - None => { - panic!("crate `{patched_crate}` depends on crate `{dependency_name}` crate by version instead of by path. Please update it to use path dependencies for all runtime crates."); + Some(table_like) => { + table_like.remove("path"); + if !table_like.contains_key("version") { + table_like.insert( + "version", + Item::Value(runtime_crate.handle.expect_version().to_string().into()), + ); + } } + None => panic!( + "crate `{}` depends on crate `{dependency_name}` crate by version instead \ + of by path. Please update it to use path dependencies for all runtime crates.", + patched_crate.handle + ) }; updates = true } } } if updates { - fs::write(&path, mutable_manifest.to_string()).context("failed to write back manifest")? + fs::write(manifest_path, mutable_manifest.to_string()) + .context("failed to write back manifest")? } Ok(()) } diff --git a/tools/ci-build/runtime-versioner/src/main.rs b/tools/ci-build/runtime-versioner/src/main.rs index 853395cb31f26389d5f01dc262770f48a69d87d5..e8602cfecef9d6d7d60714eb2237eb615aad8cac 100644 --- a/tools/ci-build/runtime-versioner/src/main.rs +++ b/tools/ci-build/runtime-versioner/src/main.rs @@ -58,6 +58,13 @@ pub struct PatchRuntime { /// Explicitly state the previous release's tag. Discovers it if not provided. #[arg(long)] previous_release_tag: Option, + /// Disable checking out the release tag in the SDK repo. + /// + /// This is useful if you need to test changes in runtime crates against + /// local changes in the SDK. + #[arg(long)] + no_checkout_sdk_release: bool, + /// Version number for stable crates. /// /// Deprecated: this argument is ignored @@ -75,15 +82,24 @@ pub struct PatchRuntimeWith { /// Path to aws-sdk-rust. #[arg(long)] sdk_path: Utf8PathBuf, - /// Path to runtime crates to patch in. + /// Path(s) to runtime crates to patch in. + /// + /// Multiple paths can be passed in, for example, if patching SDK and Smithy + /// runtime crates that are in different directories. /// /// Note: this doesn't need to be a complete set of runtime crates. It will /// only patch the crates included in the provided path. #[arg(long)] - runtime_crate_path: Utf8PathBuf, + runtime_crate_path: Vec, /// Explicitly state the previous release's tag. Discovers it if not provided. #[arg(long)] previous_release_tag: Option, + /// Disable checking out the release tag in the SDK repo. + /// + /// This is useful if you need to test changes in runtime crates against + /// local changes in the SDK. + #[arg(long)] + no_checkout_sdk_release: bool, } #[derive(clap::Parser, Clone)] diff --git a/tools/ci-build/sdk-lints/Cargo.lock b/tools/ci-build/sdk-lints/Cargo.lock index 17071c9c47e3e923c1bc01a4305e533278057878..73db9414d327979ad8c2f54d37ed7b1e8949edbe 100644 --- a/tools/ci-build/sdk-lints/Cargo.lock +++ b/tools/ci-build/sdk-lints/Cargo.lock @@ -115,6 +115,16 @@ dependencies = [ "toml 0.8.8", ] +[[package]] +name = "cargo_toml" +version = "0.19.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a98356df42a2eb1bd8f1793ae4ee4de48e384dd974ce5eac8eee802edb7492be" +dependencies = [ + "serde", + "toml 0.8.8", +] + [[package]] name = "cc" version = "1.0.83" @@ -848,7 +858,7 @@ name = "sdk-lints" version = "0.1.0" dependencies = [ "anyhow", - "cargo_toml", + "cargo_toml 0.18.0", "clap", "lazy_static", "serde", @@ -952,6 +962,7 @@ version = "0.1.0" dependencies = [ "anyhow", "async-trait", + "cargo_toml 0.19.2", "crates-index", "lazy_static", "regex", diff --git a/tools/ci-build/sdk-versioner/Cargo.lock b/tools/ci-build/sdk-versioner/Cargo.lock index 709b9ae162465211370770001acd984757982c74..dd6572db45b729e472dc2a3441f775fe6073dbe7 100644 --- a/tools/ci-build/sdk-versioner/Cargo.lock +++ b/tools/ci-build/sdk-versioner/Cargo.lock @@ -105,6 +105,16 @@ version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a2bd12c1caf447e69cd4528f47f94d203fd2582878ecb9e9465484c4148a8223" +[[package]] +name = "cargo_toml" +version = "0.19.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a98356df42a2eb1bd8f1793ae4ee4de48e384dd974ce5eac8eee802edb7492be" +dependencies = [ + "serde", + "toml 0.8.12", +] + [[package]] name = "cc" version = "1.0.90" @@ -947,6 +957,7 @@ version = "0.1.0" dependencies = [ "anyhow", "async-trait", + "cargo_toml", "crates-index", "lazy_static", "regex", diff --git a/tools/ci-build/smithy-rs-tool-common/Cargo.toml b/tools/ci-build/smithy-rs-tool-common/Cargo.toml index a9cbfa762c19bdbe201fbd24c7a8c5edaaf019af..d9fd0093e72edd29c7fa74f2d1f810626e7d0d1b 100644 --- a/tools/ci-build/smithy-rs-tool-common/Cargo.toml +++ b/tools/ci-build/smithy-rs-tool-common/Cargo.toml @@ -18,6 +18,7 @@ opt-level = 0 [dependencies] anyhow = "1" async-trait = "0.1.74" +cargo_toml = "0.19.2" crates-index = { version = "2.5.1", features = ["sparse"] } lazy_static = "1" regex = "1.6.0" diff --git a/tools/ci-build/smithy-rs-tool-common/src/lib.rs b/tools/ci-build/smithy-rs-tool-common/src/lib.rs index 567617de26453b3e5fc35fd00e91dfc4bb5a2707..27e174d72b076f6d0f61e39df62231002643eba9 100644 --- a/tools/ci-build/smithy-rs-tool-common/src/lib.rs +++ b/tools/ci-build/smithy-rs-tool-common/src/lib.rs @@ -15,3 +15,6 @@ pub mod release_tag; pub mod retry; pub mod shell; pub mod versions_manifest; + +// https://github.com/aws-sdk-rust-ci +pub const RUST_SDK_CI_OWNER: &str = "aws-sdk-rust-ci"; diff --git a/tools/ci-build/smithy-rs-tool-common/src/package.rs b/tools/ci-build/smithy-rs-tool-common/src/package.rs index 4197099b9ba12dda92b7ab11fae21d21a4297daa..e6f6a8d239b1731bf51d8ed2ec133f1fdfedfa90 100644 --- a/tools/ci-build/smithy-rs-tool-common/src/package.rs +++ b/tools/ci-build/smithy-rs-tool-common/src/package.rs @@ -3,7 +3,16 @@ * SPDX-License-Identifier: Apache-2.0 */ +use crate::RUST_SDK_CI_OWNER; +use anyhow::{Context, Result}; +use cargo_toml::{Dependency, DepsSet, Manifest}; +use semver::Version; use serde::{Deserialize, Serialize}; +use std::{ + collections::BTreeSet, + fmt, fs, + path::{Path, PathBuf}, +}; pub const SMITHY_PREFIX: &str = "aws-smithy-"; pub const SDK_PREFIX: &str = "aws-sdk-"; @@ -46,3 +55,291 @@ pub enum PackageStability { Stable, Unstable, } + +/// Information required to identify a package (crate). +#[derive(Clone, Debug, Eq, PartialEq, Ord, PartialOrd)] +pub struct PackageHandle { + pub name: String, + pub version: Option, +} + +impl PackageHandle { + pub fn new(name: impl Into, version: Option) -> Self { + Self { + name: name.into(), + version, + } + } + + /// Returns the expected owners of the crate. + pub fn expected_owners(&self) -> &[&str] { + expected_owners() + } + + /// Returns the version number or panics + pub fn expect_version(&self) -> &Version { + if let Some(version) = &self.version { + version + } else { + panic!("Crate version number required for {}", self.name) + } + } +} + +impl fmt::Display for PackageHandle { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match &self.version { + Some(version) => write!(f, "{}-{version}", self.name), + _ => f.write_str(&self.name), + } + } +} + +#[derive(Copy, Clone, Debug, Eq, PartialEq, PartialOrd, Ord)] +pub enum Publish { + Allowed, + NotAllowed, +} + +/// Represents a crate (called Package since crate is a reserved word). +#[derive(Clone, Debug, Eq, PartialEq, PartialOrd, Ord)] +pub struct Package { + /// Package name and version information + pub handle: PackageHandle, + /// Package category (Generated, SmithyRuntime, AwsRuntime, etc.) + pub category: PackageCategory, + /// Location to the crate on the current file system + pub crate_path: PathBuf, + /// Location to the crate manifest on the current file system + pub manifest_path: PathBuf, + /// Dependencies used by this package + pub local_dependencies: BTreeSet, + /// Whether or not the package should be published + pub publish: Publish, +} + +impl Package { + pub fn new( + handle: PackageHandle, + manifest_path: impl Into, + local_dependencies: BTreeSet, + publish: Publish, + ) -> Self { + let manifest_path = manifest_path.into(); + let category = PackageCategory::from_package_name(&handle.name); + Self { + handle, + category, + crate_path: manifest_path.parent().unwrap().into(), + manifest_path, + local_dependencies, + publish, + } + } + + /// Try to load a package from the given path. + pub fn try_load_path(path: impl AsRef) -> Result> { + let path = path.as_ref(); + let manifest_path = path.join("Cargo.toml"); + if path.is_dir() && manifest_path.exists() { + let manifest = fs::read(&manifest_path) + .context("failed to read manifest") + .with_context(|| format!("{manifest_path:?}"))?; + Self::try_load_manifest(manifest_path, &manifest) + } else { + Ok(None) + } + } + + /// Returns `Ok(None)` when the Cargo.toml is a workspace rather than a package + pub fn try_load_manifest( + manifest_path: impl AsRef, + manifest: &[u8], + ) -> Result> { + let manifest_path = manifest_path.as_ref(); + let mut manifest = Manifest::from_slice(manifest) + .with_context(|| format!("failed to load package manifest for {:?}", manifest_path))?; + manifest.complete_from_path(manifest_path)?; + if let Some(package) = manifest.package { + let name = package.name; + let version = parse_version(manifest_path, &package.version.unwrap())?; + let handle = PackageHandle { + name, + version: Some(version), + }; + let publish = match package.publish.unwrap() { + cargo_toml::Publish::Flag(true) => Publish::Allowed, + _ => Publish::NotAllowed, + }; + + let mut local_dependencies = BTreeSet::new(); + local_dependencies.extend(read_dependencies(manifest_path, &manifest.dependencies)?); + local_dependencies.extend(read_dependencies( + manifest_path, + &manifest.dev_dependencies, + )?); + local_dependencies.extend(read_dependencies( + manifest_path, + &manifest.build_dependencies, + )?); + Ok(Some(Package::new( + handle, + manifest_path, + local_dependencies, + publish, + ))) + } else { + Ok(None) + } + } + + /// Returns `true` if this package depends on `other` + pub fn locally_depends_on(&self, other: &PackageHandle) -> bool { + self.local_dependencies.contains(other) + } + + /// Returns the expected owners of the crate. + pub fn expected_owners(&self) -> &[&str] { + expected_owners() + } +} + +fn expected_owners() -> &'static [&'static str] { + &[RUST_SDK_CI_OWNER] +} + +/// Parses a semver version number and adds additional error context when parsing fails. +pub fn parse_version(manifest_path: &Path, version: &str) -> Result { + Version::parse(version) + .with_context(|| format!("Invalid crate version {version} in {manifest_path:?}.")) +} + +fn read_dependencies(path: &Path, dependencies: &DepsSet) -> Result> { + let mut result = Vec::new(); + for (name, metadata) in dependencies { + match metadata { + Dependency::Simple(_) => {} + Dependency::Detailed(detailed) => { + if detailed.path.is_some() { + let version = detailed + .version + .as_ref() + .map(|version| parse_version(path, version)) + .transpose()?; + result.push(PackageHandle::new(name, version)); + } + } + Dependency::Inherited(_) => panic!("workspace deps are unsupported"), + } + } + Ok(result) +} + +#[cfg(test)] +mod tests { + use super::*; + use semver::Version; + + fn version(version: &str) -> Option { + Some(Version::parse(version).unwrap()) + } + + #[test] + fn try_load_manifest_success() { + let manifest = br#" + [package] + name = "test" + version = "1.2.0-preview" + + [build-dependencies] + build_something = "1.3" + local_build_something = { version = "0.2.0", path = "../local_build_something" } + + [dev-dependencies] + dev_something = "1.1" + local_dev_something = { version = "0.1.0", path = "../local_dev_something" } + + [dependencies] + something = "1.0" + local_something = { version = "1.1.3", path = "../local_something" } + "#; + let path: PathBuf = "test/Cargo.toml".into(); + + let package = Package::try_load_manifest(&path, manifest) + .expect("parse success") + .expect("is a package"); + assert_eq!("test", package.handle.name); + assert_eq!(version("1.2.0-preview"), package.handle.version); + + let mut expected = BTreeSet::new(); + expected.insert(PackageHandle::new( + "local_build_something", + version("0.2.0"), + )); + expected.insert(PackageHandle::new("local_dev_something", version("0.1.0"))); + expected.insert(PackageHandle::new("local_something", version("1.1.3"))); + assert_eq!(expected, package.local_dependencies); + } + + #[test] + fn try_load_manifest_version_requirement_invalid() { + let manifest = br#" + [package] + name = "test" + version = "1.2.0-preview" + + [dependencies] + local_something = { version = "1.0", path = "../local_something" } + "#; + let path: PathBuf = "test/Cargo.toml".into(); + + let error = format!( + "{}", + Package::try_load_manifest(&path, manifest).expect_err("should fail") + ); + assert!( + error.contains("Invalid crate version"), + "'{}' should contain 'Invalid crate version'", + error + ); + } + + fn package(name: &str, dependencies: &[&str]) -> Package { + Package::new( + PackageHandle::new(name, version("1.0.0")), + format!("{}/Cargo.toml", name), + dependencies + .iter() + .map(|d| PackageHandle::new(*d, version("1.0.0"))) + .collect(), + Publish::Allowed, + ) + } + + #[test] + fn test_expected_package_owners_server_crate() { + let server_packages = vec![ + package("aws-smithy-http-server", &[]), + package("aws-smithy-http-server-python", &[]), + package("aws-smithy-http-server-typescript", &[]), + ]; + for pkg in server_packages { + assert_eq!(&["aws-sdk-rust-ci"], pkg.expected_owners()); + } + } + + #[test] + fn test_expected_package_owners_sdk_crate() { + let sdk_package = package("aws-types", &[]); + assert_eq!(&["aws-sdk-rust-ci"], sdk_package.expected_owners()); + } + + #[test] + fn test_expected_package_owners_smithy_runtime_crate() { + let smithy_runtime_package = package("aws-smithy-types", &[]); + assert_eq!( + &["aws-sdk-rust-ci"], + smithy_runtime_package.expected_owners() + ); + } +} diff --git a/tools/ci-cdk/canary-runner/Cargo.lock b/tools/ci-cdk/canary-runner/Cargo.lock index 043ed5ae8476dac460d4315a9aa61c497493ba78..c5a705dbd15a75ea9c3b40eee26c46bd0936c815 100644 --- a/tools/ci-cdk/canary-runner/Cargo.lock +++ b/tools/ci-cdk/canary-runner/Cargo.lock @@ -638,6 +638,16 @@ dependencies = [ "zip", ] +[[package]] +name = "cargo_toml" +version = "0.19.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a98356df42a2eb1bd8f1793ae4ee4de48e384dd974ce5eac8eee802edb7492be" +dependencies = [ + "serde", + "toml 0.8.10", +] + [[package]] name = "cc" version = "1.0.83" @@ -2408,6 +2418,7 @@ version = "0.1.0" dependencies = [ "anyhow", "async-trait", + "cargo_toml", "crates-index", "lazy_static", "regex",