Unverified Commit ce1ffa41 authored by John DiSanti's avatar John DiSanti Committed by GitHub
Browse files

Slow down crate publish (#1691) 

Crate publish used to be slow due to the verification step that `cargo`
would run, which was basically a full compile of the crate prior to
uploading it to crates.io. Now that the `publisher` tool passes
`--no-verify` to `cargo`, it publishes fast enough to get throttled.

This change completely removes the publishing parallelism, adds a fixed
time between publishing each crate, and increases the publish retry
backoff time.
parent 600c26da

tools/publisher/Cargo.lock

+0 −1
Original line number Diff line number Diff line
@@ -1007,7 +1007,6 @@ dependencies = [ 
 "dialoguer",

 "handlebars",

 "lazy_static",

 "num_cpus",

 "pretty_assertions",

 "regex",

 "semver",

tools/publisher/Cargo.toml

+0 −1
Original line number Diff line number Diff line
@@ -23,7 +23,6 @@ crates_io_api = "0.7.3" 
dialoguer = "0.8"

handlebars = "4.2"

lazy_static = "1"

num_cpus = "1.13"

regex = "1.5.4"

semver = "1.0"

serde = { version = "1", features = ["derive"] }

tools/publisher/src/subcommand/publish.rs

+24 −33
Original line number Diff line number Diff line
@@ -19,9 +19,7 @@ use smithy_rs_tool_common::git; 
use smithy_rs_tool_common::shell::ShellOperation;

use std::collections::HashSet;

use std::path::{Path, PathBuf};

use std::sync::Arc;

use std::time::Duration;

use tokio::sync::Semaphore;

use tracing::info;



lazy_static! {
@@ -61,18 +59,9 @@ pub async fn subcommand_publish( 
    // Don't proceed unless the user confirms the plan

    confirm_plan(&batches, stats, *skip_confirmation)?;



    // Use a semaphore to only allow a few concurrent publishes

    let max_concurrency = num_cpus::get_physical();

    let semaphore = Arc::new(Semaphore::new(max_concurrency));

    info!(

        "Will publish {} crates in parallel where possible.",

        max_concurrency

    );

    for batch in batches {

        let mut tasks = Vec::new();

        let mut any_published = false;

        for package in batch {

            let permit = semaphore.clone().acquire_owned().await.unwrap();

            tasks.push(tokio::spawn(async move {

            // Only publish if it hasn't been published yet.

            if !is_published(&package.handle).await? {

                publish(&package.handle, &package.crate_path).await?;
@@ -82,19 +71,21 @@ pub async fn subcommand_publish( 
                // the next package publish can fail if it depends on this package.

                wait_for_eventual_consistency(&package).await?;

                info!("Successfully published `{}`", package.handle);

                any_published = true;



                // Keep things slow to avoid getting throttled by crates.io

                tokio::time::sleep(Duration::from_secs(1)).await;

            } else {

                info!("`{}` was already published", package.handle);

            }

            correct_owner(&package).await?;

                drop(permit);

                Ok::<_, anyhow::Error>(())

            }));

        }

        for task in tasks {

            task.await??;

        }

        info!("sleeping 30 seconds after completion of the batch");

        if any_published {

            info!("Sleeping 30 seconds after completion of the batch");

            tokio::time::sleep(Duration::from_secs(30)).await;

        } else {

            info!("No packages in the batch needed publishing. Proceeding with the next batch immediately.")

        }

    }



    Ok(())
@@ -115,8 +106,8 @@ async fn publish(handle: &PackageHandle, crate_path: &Path) -> Result<()> { 
    info!("Publishing `{}`...", handle);

    run_with_retry(

        &format!("Publishing `{}`", handle),

        3,

        Duration::from_secs(5),

        5,

        Duration::from_secs(30),

        || async {

            cargo::Publish::new(handle.clone(), &crate_path)

                .spawn()