Add retry to the `publish` tool (#1376) (4b3f9e06) · Commits · Public Repositories / Smithy Rust

tools/publisher/Cargo.lock

+67 −55

Original line number	Diff line number	Diff line
		@@ -167,9 +167,9 @@ dependencies = [

		[[package]]
		name = "clap"
		version = "3.1.12"
		version = "3.1.15"
		source = "registry+https://github.com/rust-lang/crates.io-index"
		checksum = "7c167e37342afc5f33fd87bbc870cedd020d2a6dffa05d45ccd9241fbdd146db"
		checksum = "85a35a599b11c089a7f49105658d089b8f2cf0882993c17daf6de15285c2c35d"
		dependencies = [
		"atty",
		"bitflags",
		@@ -197,9 +197,9 @@ dependencies = [

		[[package]]
		name = "clap_lex"
		version = "0.1.1"
		version = "0.2.0"
		source = "registry+https://github.com/rust-lang/crates.io-index"
		checksum = "189ddd3b5d32a70b35e7686054371742a937b0d99128e76dde6340210e966669"
		checksum = "a37c35f1112dad5e6e0b1adaff798507497a18fceeb30cceb3bae7d1427b9213"
		dependencies = [
		"os_str_bytes",
		]
		@@ -522,9 +522,9 @@ checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70"

		[[package]]
		name = "http"
		version = "0.2.6"
		version = "0.2.7"
		source = "registry+https://github.com/rust-lang/crates.io-index"
		checksum = "31f4c6746584866f0feabcc69893c5b51beef3831656a968ed7ae254cdc4fd03"
		checksum = "ff8670570af52249509a86f5e3e18a08c60b177071826898fde8997cf5f6bfbb"
		dependencies = [
		"bytes",
		"fnv",
		@@ -650,9 +650,9 @@ checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646"

		[[package]]
		name = "libc"
		version = "0.2.124"
		version = "0.2.125"
		source = "registry+https://github.com/rust-lang/crates.io-index"
		checksum = "21a41fed9d98f27ab1c6d161da622a4fa35e8a54a8adc24bbf3ddd0ef70b0e50"
		checksum = "5916d2ae698f6de9bfb891ad7a8d65c09d232dc58cc4ac433c7da3b2fd84bc2b"

		[[package]]
		name = "lock_api"
		@@ -666,9 +666,9 @@ dependencies = [

		[[package]]
		name = "log"
		version = "0.4.16"
		version = "0.4.17"
		source = "registry+https://github.com/rust-lang/crates.io-index"
		checksum = "6389c490849ff5bc16be905ae24bc913a9c8892e19b2341dbc175e14c341c2b8"
		checksum = "abb12e687cfb44aa40f41fc3978ef76448f9b6038cad6aef4259d3c095a2382e"
		dependencies = [
		"cfg-if",
		]
		@@ -696,9 +696,9 @@ checksum = "a3e378b66a060d48947b590737b30a1be76706c8dd7b8ba0f2fe3989c68a853f"

		[[package]]
		name = "memchr"
		version = "2.4.1"
		version = "2.5.0"
		source = "registry+https://github.com/rust-lang/crates.io-index"
		checksum = "308cc39be01b73d0d18f82a0e7b2a3df85245f84af96fdddc5d202d27e47b86a"
		checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d"

		[[package]]
		name = "mime"
		@@ -758,9 +758,9 @@ dependencies = [

		[[package]]
		name = "num-integer"
		version = "0.1.44"
		version = "0.1.45"
		source = "registry+https://github.com/rust-lang/crates.io-index"
		checksum = "d2cc698a63b549a70bc047073d2949cce27cd1c7b0a4a862d08a8031bc2801db"
		checksum = "225d3389fb3509a24c93f5c29eb6bde2586b98d9f016636dff58d7c6f7569cd9"
		dependencies = [
		"autocfg",
		"num-traits",
		@@ -768,9 +768,9 @@ dependencies = [

		[[package]]
		name = "num-traits"
		version = "0.2.14"
		version = "0.2.15"
		source = "registry+https://github.com/rust-lang/crates.io-index"
		checksum = "9a64b1ec5cda2586e284722486d802acf1f7dbdc623e2bfc57e65ca1cd099290"
		checksum = "578ede34cf02f8924ab9447f50c28075b4d3e5b269972345e7e0372b38c6cdcd"
		dependencies = [
		"autocfg",
		]
		@@ -805,18 +805,30 @@ checksum = "624a8340c38c1b80fd549087862da4ba43e08858af025b236e509b6649fc13d5"

		[[package]]
		name = "openssl"
		version = "0.10.38"
		version = "0.10.40"
		source = "registry+https://github.com/rust-lang/crates.io-index"
		checksum = "0c7ae222234c30df141154f159066c5093ff73b63204dcda7121eb082fc56a95"
		checksum = "fb81a6430ac911acb25fe5ac8f1d2af1b4ea8a4fdfda0f1ee4292af2e2d8eb0e"
		dependencies = [
		"bitflags",
		"cfg-if",
		"foreign-types",
		"libc",
		"once_cell",
		"openssl-macros",
		"openssl-sys",
		]

		[[package]]
		name = "openssl-macros"
		version = "0.1.0"
		source = "registry+https://github.com/rust-lang/crates.io-index"
		checksum = "b501e44f11665960c7e7fcf062c7d96a14ade4aa98116c004b2e37b5be7d736c"
		dependencies = [
		"proc-macro2",
		"quote",
		"syn",
		]

		[[package]]
		name = "openssl-probe"
		version = "0.1.5"
		@@ -825,9 +837,9 @@ checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf"

		[[package]]
		name = "openssl-sys"
		version = "0.9.72"
		version = "0.9.73"
		source = "registry+https://github.com/rust-lang/crates.io-index"
		checksum = "7e46109c383602735fa0a2e48dd2b7c892b048e1bf69e5c3b1d804b7d9c203cb"
		checksum = "9d5fd19fb3e0a8191c1e34935718976a3e70c112ab9a24af6d7cadccd9d90bc0"
		dependencies = [
		"autocfg",
		"cc",
		@@ -854,9 +866,9 @@ dependencies = [

		[[package]]
		name = "parking_lot_core"
		version = "0.9.2"
		version = "0.9.3"
		source = "registry+https://github.com/rust-lang/crates.io-index"
		checksum = "995f667a6c822200b0433ac218e05582f0e2efa1b922a3fd2fbaadc5f87bab37"
		checksum = "09a279cbf25cb0757810394fbc1e359949b59e348145c643a939a525692e6929"
		dependencies = [
		"cfg-if",
		"libc",
		@@ -967,7 +979,7 @@ dependencies = [

		[[package]]
		name = "publisher"
		version = "0.2.0"
		version = "0.3.0"
		dependencies = [
		"anyhow",
		"async-recursion",
		@@ -1134,24 +1146,24 @@ dependencies = [

		[[package]]
		name = "semver"
		version = "1.0.7"
		version = "1.0.9"
		source = "registry+https://github.com/rust-lang/crates.io-index"
		checksum = "d65bd28f48be7196d222d95b9243287f48d27aca604e08497513019ff0502cc4"
		checksum = "8cb243bdfdb5936c8dc3c45762a19d12ab4550cdc753bc247637d4ec35a040fd"

		[[package]]
		name = "serde"
		version = "1.0.136"
		version = "1.0.137"
		source = "registry+https://github.com/rust-lang/crates.io-index"
		checksum = "ce31e24b01e1e524df96f1c2fdd054405f8d7376249a5110886fb4b658484789"
		checksum = "61ea8d54c77f8315140a05f4c7237403bf38b72704d031543aa1d16abbf517d1"
		dependencies = [
		"serde_derive",
		]

		[[package]]
		name = "serde_derive"
		version = "1.0.136"
		version = "1.0.137"
		source = "registry+https://github.com/rust-lang/crates.io-index"
		checksum = "08597e7152fcd306f41838ed3e37be9eaeed2b61c42e2117266a554fab4662f9"
		checksum = "1f26faba0c3959972377d3b2d306ee9f71faee9714294e41bb777f83f88578be"
		dependencies = [
		"proc-macro2",
		"quote",
		@@ -1160,9 +1172,9 @@ dependencies = [

		[[package]]
		name = "serde_json"
		version = "1.0.79"
		version = "1.0.81"
		source = "registry+https://github.com/rust-lang/crates.io-index"
		checksum = "8e8d9fa5c3b304765ce1fd9c4c8a3de2c8db365a5b91be52f186efc675681d95"
		checksum = "9b7ce2b32a1aed03c558dc61a5cd328f15aff2dbc17daad8fb8af04d2100e15c"
		dependencies = [
		"itoa",
		"ryu",
		@@ -1275,9 +1287,9 @@ checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623"

		[[package]]
		name = "syn"
		version = "1.0.91"
		version = "1.0.92"
		source = "registry+https://github.com/rust-lang/crates.io-index"
		checksum = "b683b2b825c8eef438b77c36a06dc262294da3d5a5813fac20da149241dcd44d"
		checksum = "7ff7c592601f11445996a06f8ad0c27f094a58857c2f89e97974ab9235b92c52"
		dependencies = [
		"proc-macro2",
		"quote",
		@@ -1325,18 +1337,18 @@ checksum = "b1141d4d61095b28419e22cb0bbf02755f5e54e0526f97f1e3d1d160e60885fb"

		[[package]]
		name = "thiserror"
		version = "1.0.30"
		version = "1.0.31"
		source = "registry+https://github.com/rust-lang/crates.io-index"
		checksum = "854babe52e4df1653706b98fcfc05843010039b406875930a70e4d9644e5c417"
		checksum = "bd829fe32373d27f76265620b5309d0340cb8550f523c1dda251d6298069069a"
		dependencies = [
		"thiserror-impl",
		]

		[[package]]
		name = "thiserror-impl"
		version = "1.0.30"
		version = "1.0.31"
		source = "registry+https://github.com/rust-lang/crates.io-index"
		checksum = "aa32fd3f627f367fe16f893e2597ae3c05020f8bba2666a4e6ea73d377e5714b"
		checksum = "0396bc89e626244658bef819e22d0cc459e795a5ebe878e6ec336d1674a8d79a"
		dependencies = [
		"proc-macro2",
		"quote",
		@@ -1369,9 +1381,9 @@ checksum = "cda74da7e1a664f795bb1f8a87ec406fb89a02522cf6e50620d016add6dbbf5c"

		[[package]]
		name = "tokio"
		version = "1.18.0"
		version = "1.18.1"
		source = "registry+https://github.com/rust-lang/crates.io-index"
		checksum = "0f48b6d60512a392e34dbf7fd456249fd2de3c83669ab642e021903f4015185b"
		checksum = "dce653fb475565de9f6fb0614b28bca8df2c430c0cf84bcd9c843f15de5414cc"
		dependencies = [
		"bytes",
		"libc",
		@@ -1541,9 +1553,9 @@ checksum = "3ed742d4ea2bd1176e236172c8429aaf54486e7ac098db29ffe6529e0ce50973"

		[[package]]
		name = "unicode-xid"
		version = "0.2.2"
		version = "0.2.3"
		source = "registry+https://github.com/rust-lang/crates.io-index"
		checksum = "8ccb82d61f80a663efe1f787a51b16b5a51e3314d6ac365b08639f52387b33f3"
		checksum = "957e51f3646910546462e67d5f7599b9e4fb8acdd304b087a6494730f9eebf04"

		[[package]]
		name = "url"
		@@ -1700,9 +1712,9 @@ checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"

		[[package]]
		name = "windows-sys"
		version = "0.34.0"
		version = "0.36.1"
		source = "registry+https://github.com/rust-lang/crates.io-index"
		checksum = "5acdd78cb4ba54c0045ac14f62d8f94a03d10047904ae2a40afa1e99d8f70825"
		checksum = "ea04155a16a59f9eab786fe12a4a450e75cdb175f9e0d80da1e17db09f55b8d2"
		dependencies = [
		"windows_aarch64_msvc",
		"windows_i686_gnu",
		@@ -1713,33 +1725,33 @@ dependencies = [

		[[package]]
		name = "windows_aarch64_msvc"
		version = "0.34.0"
		version = "0.36.1"
		source = "registry+https://github.com/rust-lang/crates.io-index"
		checksum = "17cffbe740121affb56fad0fc0e421804adf0ae00891205213b5cecd30db881d"
		checksum = "9bb8c3fd39ade2d67e9874ac4f3db21f0d710bee00fe7cab16949ec184eeaa47"

		[[package]]
		name = "windows_i686_gnu"
		version = "0.34.0"
		version = "0.36.1"
		source = "registry+https://github.com/rust-lang/crates.io-index"
		checksum = "2564fde759adb79129d9b4f54be42b32c89970c18ebf93124ca8870a498688ed"
		checksum = "180e6ccf01daf4c426b846dfc66db1fc518f074baa793aa7d9b9aaeffad6a3b6"

		[[package]]
		name = "windows_i686_msvc"
		version = "0.34.0"
		version = "0.36.1"
		source = "registry+https://github.com/rust-lang/crates.io-index"
		checksum = "9cd9d32ba70453522332c14d38814bceeb747d80b3958676007acadd7e166956"
		checksum = "e2e7917148b2812d1eeafaeb22a97e4813dfa60a3f8f78ebe204bcc88f12f024"

		[[package]]
		name = "windows_x86_64_gnu"
		version = "0.34.0"
		version = "0.36.1"
		source = "registry+https://github.com/rust-lang/crates.io-index"
		checksum = "cfce6deae227ee8d356d19effc141a509cc503dfd1f850622ec4b0f84428e1f4"
		checksum = "4dcd171b8776c41b97521e5da127a2d86ad280114807d0b2ab1e462bc764d9e1"

		[[package]]
		name = "windows_x86_64_msvc"
		version = "0.34.0"
		version = "0.36.1"
		source = "registry+https://github.com/rust-lang/crates.io-index"
		checksum = "d19538ccc21819d01deaf88d6a17eae6596a12e9aafdbb97916fb49896d89de9"
		checksum = "c811ca4a8c853ef420abd8592ba53ddbbac90410fab6903b3e79972a631f7680"

		[[package]]
		name = "winreg"
		@@ -1752,6 +1764,6 @@ dependencies = [

		[[package]]
		name = "zeroize"
		version = "1.5.4"
		version = "1.5.5"
		source = "registry+https://github.com/rust-lang/crates.io-index"
		checksum = "7eb5728b8afd3f280a869ce1d4c554ffaed35f45c231fc41bfbd0381bef50317"
		checksum = "94693807d016b2f2d2e14420eb3bfcca689311ff775dcf113d74ea624b7cdf07"

tools/publisher/Cargo.toml

+1 −1

Original line number	Diff line number	Diff line
		[package]
		name = "publisher"
		version = "0.2.0"
		version = "0.3.0"
		authors = ["AWS Rust SDK Team <aws-sdk-rust@amazon.com>"]
		description = "Tool used to publish the AWS SDK to crates.io"
		edition = "2021"

tools/publisher/src/main.rs

+1 −0

Original line number	Diff line number	Diff line
		@@ -17,6 +17,7 @@ mod cargo;
		mod fs;
		mod package;
		mod repo;
		mod retry;
		mod sort;
		mod subcommand;

tools/publisher/src/retry.rs

0 → 100644

+182 −0

Original line number	Diff line number	Diff line
		/*
		* Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
		* SPDX-License-Identifier: Apache-2.0
		*/

		use std::error::Error;
		use std::error::Error as StdError;
		use std::future::Future;
		use std::time::Duration;
		use tracing::{error, info};

		pub type BoxError = Box<dyn StdError + Send + Sync + 'static>;

		pub enum ErrorClass {
		Retry,
		NoRetry,
		}

		#[derive(thiserror::Error, Debug)]
		pub enum RetryError {
		#[error("failed with unretryable error")]
		FailedUnretryable(#[source] Box<dyn Error + Send + Sync + 'static>),
		#[error("failed {0} times and won't be retried again")]
		FailedMaxAttempts(usize),
		}

		pub async fn run_with_retry<F, Ft, C, O, E>(
		what: &str,
		max_attempts: usize,
		backoff: Duration,
		create_future: F,
		classify_error: C,
		) -> Result<O, RetryError>
		where
		F: Fn() -> Ft,
		Ft: Future<Output = Result<O, E>> + Send,
		C: Fn(&E) -> ErrorClass,
		E: Into<BoxError>,
		{
		assert!(max_attempts > 0);

		let mut attempt = 1;
		loop {
		let future = create_future();
		match future.await {
		Ok(output) => return Ok(output),
		Err(err) => {
		match classify_error(&err) {
		ErrorClass::NoRetry => {
		return Err(RetryError::FailedUnretryable(err.into()));
		}
		ErrorClass::Retry => {
		info!(
		"{} failed on attempt {} with retryable error: {}. Will retry after {:?}",
		what, attempt, err.into(), backoff
		);
		}
		}
		}
		}

		// If we made it this far, we're retrying or failing at max retries
		if attempt == max_attempts {
		return Err(RetryError::FailedMaxAttempts(max_attempts));
		}
		attempt += 1;
		tokio::time::sleep(backoff).await;
		}
		}

		#[cfg(test)]
		mod tests {
		use super::*;
		use std::sync::atomic::{AtomicU8, Ordering};
		use std::sync::Arc;

		#[derive(thiserror::Error, Debug)]
		#[error("FakeError")]
		struct FakeError;

		#[derive(thiserror::Error, Debug)]
		#[error("UnretryableError")]
		struct UnretryableError;

		fn assert_send<T: Send>(thing: T) -> T {
		thing
		}

		#[tokio::test]
		async fn fail_max_attempts() {
		let attempt = Arc::new(AtomicU8::new(1));
		let result = {
		let attempt = attempt.clone();
		assert_send(run_with_retry(
		"test",
		3,
		Duration::from_millis(0),
		move \|\| {
		let attempt = attempt.clone();
		Box::pin(async move {
		attempt.fetch_add(1, Ordering::Relaxed);
		Result::<(), _>::Err(FakeError)
		})
		},
		\|_err\| ErrorClass::Retry,
		))
		.await
		};

		assert!(matches!(result, Err(RetryError::FailedMaxAttempts(3))));
		// `attempt` holds the number of the next attempt, so 4 instead of 3 in this case
		assert_eq!(4, attempt.load(Ordering::Relaxed));
		}

		#[tokio::test]
		async fn fail_then_succeed() {
		let attempt = Arc::new(AtomicU8::new(1));
		let result = {
		let attempt = attempt.clone();
		run_with_retry(
		"test",
		3,
		Duration::from_millis(0),
		move \|\| {
		let attempt = attempt.clone();
		Box::pin(async move {
		if attempt.fetch_add(1, Ordering::Relaxed) == 1 {
		Err(FakeError)
		} else {
		Ok(2)
		}
		})
		},
		\|_err\| ErrorClass::Retry,
		)
		.await
		};

		assert!(matches!(result, Ok(2)));
		// `attempt` holds the number of the next attempt, so 3 instead of 2 in this case
		assert_eq!(3, attempt.load(Ordering::Relaxed));
		}

		#[tokio::test]
		async fn unretryable_error() {
		let attempt = Arc::new(AtomicU8::new(1));
		let result = {
		let attempt = attempt.clone();
		run_with_retry(
		"test",
		3,
		Duration::from_millis(0),
		move \|\| {
		let attempt = attempt.clone();
		Box::pin(async move {
		if attempt.fetch_add(1, Ordering::Relaxed) == 1 {
		Err(UnretryableError)
		} else {
		Ok(2)
		}
		})
		},
		\|err\| {
		if matches!(err, UnretryableError) {
		ErrorClass::NoRetry
		} else {
		ErrorClass::Retry
		}
		},
		)
		.await
		};

		match result {
		Err(RetryError::FailedUnretryable(err)) => {
		assert!(err.downcast_ref::<UnretryableError>().is_some());
		}
		_ => panic!("should be an unretryable error"),
		}
		assert_eq!(2, attempt.load(Ordering::Relaxed));
		}
		}

tools/publisher/src/subcommand/publish.rs

+63 −23

Original line number	Diff line number	Diff line
		@@ -8,9 +8,10 @@ use crate::package::{
		discover_and_validate_package_batches, Package, PackageBatch, PackageHandle, PackageStats,
		};
		use crate::repo::{resolve_publish_location, Repository};
		use crate::retry::{run_with_retry, BoxError, ErrorClass};
		use crate::CRATE_OWNER;
		use crate::{cargo, SDK_REPO_NAME};
		use anyhow::{bail, Result};
		use anyhow::{bail, Context, Result};
		use crates_io_api::{AsyncClient, Error};
		use dialoguer::Confirm;
		use lazy_static::lazy_static;
		@@ -59,10 +60,8 @@ pub async fn subcommand_publish(location: &Path) -> Result<()> {
		tasks.push(tokio::spawn(async move {
		// Only publish if it hasn't been published yet.
		if !is_published(&package.handle).await? {
		info!("Publishing `{}`...", package.handle);
		cargo::Publish::new(package.handle.clone(), &package.crate_path)
		.spawn()
		.await?;
		publish(&package.handle, &package.crate_path).await?;

		// Sometimes it takes a little bit of time for the new package version
		// to become available after publish. If we proceed too quickly, then
		// the next package publish can fail if it depends on this package.
		@@ -86,6 +85,24 @@ pub async fn subcommand_publish(location: &Path) -> Result<()> {
		Ok(())
		}

		async fn publish(handle: &PackageHandle, crate_path: &Path) -> Result<()> {
		info!("Publishing `{}`...", handle);
		run_with_retry(
		&format!("Publishing `{}`", handle),
		3,
		Duration::from_secs(5),
		\|\| async {
		cargo::Publish::new(handle.clone(), &crate_path)
		.spawn()
		.await?;
		Result::<_, BoxError>::Ok(())
		},
		\|_err\| ErrorClass::Retry,
		)
		.await?;
		Ok(())
		}

		async fn confirm_correct_tag(batches: &[Vec<Package>], location: &Path) -> Result<()> {
		let aws_config_version = batches
		.iter()
		@@ -108,16 +125,29 @@ async fn confirm_correct_tag(batches: &[Vec<Package>], location: &Path) -> Resul
		}

		async fn is_published(handle: &PackageHandle) -> Result<bool> {
		let expected_version = handle.version.to_string();
		run_with_retry(
		&format!("Checking if `{}` is already published", handle.name),
		3,
		Duration::from_secs(5),
		\|\| async {
		let expected_version = (&handle.version).to_string();
		let crate_info = match CRATES_IO_CLIENT.get_crate(&handle.name).await {
		Ok(info) => info,
		Err(Error::NotFound(_)) => return Ok(false),
		Err(other) => return Err(other.into()),
		Err(other) => return Err(other),
		};
		Ok(crate_info
		.versions
		.iter()
		.any(\|crate_version\| crate_version.num == expected_version))
		},
		\|err\| match err {
		Error::Http(_) => ErrorClass::Retry,
		_ => ErrorClass::NoRetry,
		},
		)
		.await
		.context("is_published")
		}

		/// Waits for the given package to show up on crates.io
		@@ -141,6 +171,11 @@ async fn wait_for_eventual_consistency(package: &Package) -> Result<()> {

		/// Corrects the crate ownership.
		async fn correct_owner(package: &Package) -> Result<()> {
		run_with_retry(
		&format!("Correcting ownership of `{}`", package.handle.name),
		3,
		Duration::from_secs(5),
		\|\| async {
		let owners = cargo::GetOwners::new(&package.handle.name).spawn().await?;
		if !owners.iter().any(\|owner\| owner == CRATE_OWNER) {
		cargo::AddOwner::new(&package.handle.name, CRATE_OWNER)
		@@ -148,7 +183,12 @@ async fn correct_owner(package: &Package) -> Result<()> {
		.await?;
		info!("Corrected crate ownership of `{}`", package.handle);
		}
		Ok(())
		Result::<_, BoxError>::Ok(())
		},
		\|_err\| ErrorClass::Retry,
		)
		.await
		.context("correct_owner")
		}

		fn confirm_plan(batches: &[PackageBatch], stats: PackageStats) -> Result<()> {

Admin message