Unverified Commit 4604aa5b authored by John DiSanti's avatar John DiSanti Committed by GitHub
Browse files

Replace the crates.io API with the sparse index (#3447) 

The sparse index is preferred to the crates.io API for the checks we
need, according to the [documentation](https://crates.io/data-access).
When the tools were first implemented, the sparse index didn't exist, so
the API was used.

----

_By submitting this pull request, I confirm that you can use, modify,
copy, and redistribute this contribution, under the terms of your
choice._
parent 49ec3046

tools/ci-build/publisher/Cargo.lock

+42 −72
Original line number Diff line number Diff line
@@ -150,16 +150,6 @@ version = "1.0.0" 
source = "registry+https://github.com/rust-lang/crates.io-index"

checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"



[[package]]

name = "chrono"

version = "0.4.31"

source = "registry+https://github.com/rust-lang/crates.io-index"

checksum = "7f2c685bad3eb3d45a01354cedb7d5faa66194d1d58ba6e267a8de788f79db38"

dependencies = [

 "num-traits",

 "serde",

]



[[package]]

name = "clap"

version = "3.1.18"
@@ -240,20 +230,23 @@ dependencies = [ 
]



[[package]]

name = "crates_io_api"

version = "0.7.3"

name = "crates-index"

version = "2.5.1"

source = "registry+https://github.com/rust-lang/crates.io-index"

checksum = "f0ab4553806e9495b7e78a66454c16c5021cd3816d7b8e7e8fe28aa5474346bb"

checksum = "1d9efa03a974d583ad530bbfe00e3d0021de7f26217120437b128dc4c331aa4f"

dependencies = [

 "chrono",

 "futures",

 "log",

 "reqwest",

 "hex",

 "home",

 "http",

 "memchr",

 "rustc-hash",

 "semver",

 "serde",

 "serde_derive",

 "serde_json",

 "tokio",

 "url",

 "smol_str",

 "thiserror",

 "toml 0.8.8",

]



[[package]]
@@ -370,21 +363,6 @@ dependencies = [ 
 "autocfg",

]



[[package]]

name = "futures"

version = "0.3.29"

source = "registry+https://github.com/rust-lang/crates.io-index"

checksum = "da0290714b38af9b4a7b094b8a37086d1b4e61f2df9122c3cad2577669145335"

dependencies = [

 "futures-channel",

 "futures-core",

 "futures-executor",

 "futures-io",

 "futures-sink",

 "futures-task",

 "futures-util",

]



[[package]]

name = "futures-channel"

version = "0.3.29"
@@ -392,7 +370,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index" 
checksum = "ff4dd66668b557604244583e3e1e1eada8c5c2e96a6d0d6653ede395b78bbacb"

dependencies = [

 "futures-core",

 "futures-sink",

]



[[package]]
@@ -401,33 +378,11 @@ version = "0.3.29" 
source = "registry+https://github.com/rust-lang/crates.io-index"

checksum = "eb1d22c66e66d9d72e1758f0bd7d4fd0bee04cad842ee34587d68c07e45d088c"



[[package]]

name = "futures-executor"

version = "0.3.29"

source = "registry+https://github.com/rust-lang/crates.io-index"

checksum = "0f4fb8693db0cf099eadcca0efe2a5a22e4550f98ed16aba6c48700da29597bc"

dependencies = [

 "futures-core",

 "futures-task",

 "futures-util",

]



[[package]]

name = "futures-io"

version = "0.3.29"

source = "registry+https://github.com/rust-lang/crates.io-index"

checksum = "8bf34a163b5c4c52d0478a4d757da8fb65cabef42ba90515efee0f6f9fa45aaa"



[[package]]

name = "futures-macro"

version = "0.3.29"

version = "0.3.30"

source = "registry+https://github.com/rust-lang/crates.io-index"

checksum = "53b153fd91e4b0147f4aced87be237c98248656bb01050b96bf3ee89220a8ddb"

dependencies = [

 "proc-macro2",

 "quote",

 "syn 2.0.48",

]

checksum = "a44623e20b9681a318efdd71c299b6b222ed6f231972bfe2f224ebad6311f0c1"



[[package]]

name = "futures-sink"
@@ -447,11 +402,8 @@ version = "0.3.29" 
source = "registry+https://github.com/rust-lang/crates.io-index"

checksum = "a19526d624e703a3179b3d322efec918b6246ea0fa51d41124525f00f1cc8104"

dependencies = [

 "futures-channel",

 "futures-core",

 "futures-io",

 "futures-macro",

 "futures-sink",

 "futures-task",

 "memchr",

 "pin-project-lite",
@@ -546,6 +498,18 @@ name = "hex" 
version = "0.4.3"

source = "registry+https://github.com/rust-lang/crates.io-index"

checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70"

dependencies = [

 "serde",

]



[[package]]

name = "home"

version = "0.5.9"

source = "registry+https://github.com/rust-lang/crates.io-index"

checksum = "e3d1354bf6b7235cb4a0576c2619fd4ed18183f689b12b006a0ee7329eeff9a5"

dependencies = [

 "windows-sys 0.52.0",

]



[[package]]

name = "http"
@@ -772,15 +736,6 @@ dependencies = [ 
 "winapi",

]



[[package]]

name = "num-traits"

version = "0.2.17"

source = "registry+https://github.com/rust-lang/crates.io-index"

checksum = "39e3200413f237f41ab11ad6d161bc7239c84dcb631773ccd7de3dfe4b5c267c"

dependencies = [

 "autocfg",

]



[[package]]

name = "num_cpus"

version = "1.16.0"
@@ -1006,7 +961,6 @@ dependencies = [ 
 "async-trait",

 "cargo_toml",

 "clap",

 "crates_io_api",

 "dialoguer",

 "fs-err",

 "handlebars",
@@ -1132,6 +1086,12 @@ version = "0.1.23" 
source = "registry+https://github.com/rust-lang/crates.io-index"

checksum = "d626bb9dae77e28219937af045c257c28bfd3f69333c512553507f5f9798cb76"



[[package]]

name = "rustc-hash"

version = "1.1.0"

source = "registry+https://github.com/rust-lang/crates.io-index"

checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2"



[[package]]

name = "rustix"

version = "0.38.26"
@@ -1310,6 +1270,7 @@ version = "0.1.0" 
dependencies = [

 "anyhow",

 "async-trait",

 "crates-index",

 "lazy_static",

 "regex",

 "reqwest",
@@ -1322,6 +1283,15 @@ dependencies = [ 
 "tracing",

]



[[package]]

name = "smol_str"

version = "0.2.1"

source = "registry+https://github.com/rust-lang/crates.io-index"

checksum = "e6845563ada680337a52d43bb0b29f396f2d911616f6573012645b9e3d048a49"

dependencies = [

 "serde",

]



[[package]]

name = "socket2"

version = "0.4.10"

tools/ci-build/publisher/Cargo.toml

+0 −1
Original line number Diff line number Diff line
@@ -19,7 +19,6 @@ async-recursion = "0.3.2" 
async-trait = "0.1.74"

cargo_toml = "0.16.3"

clap = { version = "~3.1.18", features = ["derive"] }

crates_io_api = "0.7.3"

dialoguer = "0.8"

fs-err = "2"

handlebars = "4.2"

tools/ci-build/publisher/src/publish.rs

+12 −22
Original line number Diff line number Diff line
@@ -5,35 +5,25 @@ 


use crate::cargo;

use crate::package::PackageHandle;

use crates_io_api::{AsyncClient, Error};

use once_cell::sync::Lazy;

use smithy_rs_tool_common::retry::{run_with_retry, BoxError, ErrorClass};

use anyhow::Result;

use smithy_rs_tool_common::shell::ShellOperation;

use std::path::Path;

use smithy_rs_tool_common::{

    index::CratesIndex,

    retry::{run_with_retry, BoxError, ErrorClass},

};

use std::time::Duration;

use std::{path::Path, sync::Arc};

use tracing::info;



pub static CRATES_IO_CLIENT: Lazy<AsyncClient> = Lazy::new(|| {

    AsyncClient::new(

        "AWS_RUST_SDK_PUBLISHER (aws-sdk-rust@amazon.com)",

        Duration::from_secs(1),

    )

    .expect("valid client")

});



/// Return `true` if there is at least one version published on crates.io associated with

/// the specified crate name.

#[tracing::instrument]

pub async fn has_been_published_on_crates_io(crate_name: &str) -> anyhow::Result<bool> {

    match CRATES_IO_CLIENT.get_crate(crate_name).await {

        Ok(_) => Ok(true),

        Err(Error::NotFound(_)) => Ok(false),

        Err(e) => Err(e.into()),

    }

pub async fn is_published(index: Arc<CratesIndex>, crate_name: &str) -> Result<bool> {

    let crate_name = crate_name.to_string();

    let versions =

        tokio::task::spawn_blocking(move || index.published_versions(&crate_name)).await??;

    Ok(!versions.is_empty())

}



#[tracing::instrument]

pub async fn publish(handle: &PackageHandle, crate_path: &Path) -> anyhow::Result<()> {

pub async fn publish(handle: &PackageHandle, crate_path: &Path) -> Result<()> {

    info!("Publishing `{}`...", handle);

    run_with_retry(

        &format!("Publishing `{}`", handle),

tools/ci-build/publisher/src/subcommand/claim_crate_names.rs

+12 −6
Original line number Diff line number Diff line
@@ -2,21 +2,24 @@ 
 * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.

 * SPDX-License-Identifier: Apache-2.0

 */

use crate::package::PackageHandle;

use crate::publish::{has_been_published_on_crates_io, publish};

use crate::publish::publish;

use crate::subcommand::publish::correct_owner;

use crate::{cargo, SDK_REPO_NAME};

use crate::{fs::Fs, package::discover_manifests};

use crate::{package::PackageHandle, publish::is_published};

use anyhow::{Context, Result};

use cargo_toml::Manifest;

use clap::Parser;

use dialoguer::Confirm;

use semver::Version;

use smithy_rs_tool_common::git;

use smithy_rs_tool_common::package::PackageCategory;

use std::path::{Path, PathBuf};

use smithy_rs_tool_common::{git, index::CratesIndex};

use std::time::Duration;

use std::{collections::HashSet, fs};

use std::{

    path::{Path, PathBuf},

    sync::Arc,

};

use tracing::info;



#[derive(Parser, Debug)]
@@ -33,11 +36,12 @@ pub async fn subcommand_claim_crate_names(args: &ClaimCrateNamesArgs) -> Result< 


    let smithy_rs_repository_root =

        git::find_git_repository_root(SDK_REPO_NAME, std::env::current_dir()?)?;

    let index = Arc::new(CratesIndex::real()?);

    let packages = discover_publishable_crate_names(&smithy_rs_repository_root).await?;

    let unpublished_package_names = {

        let mut s = HashSet::new();

        for package_name in packages {

            if !has_been_published_on_crates_io(&package_name).await? {

            if !is_published(index.clone(), &package_name).await? {

                s.insert(package_name);

            }

        }
@@ -65,10 +69,12 @@ async fn claim_crate_name(name: &str) -> Result<()> { 
    let category = PackageCategory::from_package_name(name);

    let package_handle = PackageHandle::new(name, Version::new(0, 0, 1));

    publish(&package_handle, crate_dir_path).await?;



    // Keep things slow to avoid getting throttled by crates.io

    tokio::time::sleep(Duration::from_secs(2)).await;

    info!("Successfully published `{}`", package_handle);

    correct_owner(&package_handle, &category).await?;



    info!("Successfully published `{}`", package_handle);

    Ok(())

}

tools/ci-build/publisher/src/subcommand/publish.rs

+14 −49
Original line number Diff line number Diff line
@@ -8,19 +8,18 @@ use crate::package::{ 
    discover_and_validate_package_batches, expected_package_owners, Package, PackageBatch,

    PackageHandle, PackageStats,

};

use crate::publish::{publish, CRATES_IO_CLIENT};

use crate::publish::publish;

use crate::{cargo, SDK_REPO_CRATE_PATH, SDK_REPO_NAME};

use anyhow::{bail, Context, Result};

use clap::Parser;

use crates_io_api::Error;

use dialoguer::Confirm;

use smithy_rs_tool_common::git;

use smithy_rs_tool_common::package::PackageCategory;

use smithy_rs_tool_common::retry::{run_with_retry, BoxError, ErrorClass};

use smithy_rs_tool_common::shell::ShellOperation;

use std::collections::HashSet;

use smithy_rs_tool_common::{git, index::CratesIndex};

use std::path::{Path, PathBuf};

use std::time::Duration;

use std::{collections::HashSet, sync::Arc};

use tracing::info;



const DEFAULT_DELAY_MILLIS: usize = 1000;
@@ -60,11 +59,12 @@ pub async fn subcommand_publish( 
    // Don't proceed unless the user confirms the plan

    confirm_plan(&batches, stats, *skip_confirmation)?;



    let index = Arc::new(CratesIndex::real()?);

    for batch in &batches {

        let mut any_published = false;

        for package in batch {

            // Only publish if it hasn't been published yet.

            if !is_published(&package.handle).await? {

            if !is_published(index.clone(), &package.handle).await? {

                publish(&package.handle, &package.crate_path).await?;



                // Keep things slow to avoid getting throttled by crates.io
@@ -73,7 +73,7 @@ pub async fn subcommand_publish( 
                // Sometimes it takes a little bit of time for the new package version

                // to become available after publish. If we proceed too quickly, then

                // the next package publish can fail if it depends on this package.

                wait_for_eventual_consistency(package).await?;

                wait_for_eventual_consistency(index.clone(), package).await?;

                info!("Successfully published `{}`", &package.handle);

                any_published = true;

            } else {
@@ -108,43 +108,24 @@ pub fn resolve_publish_location(location: &Path) -> PathBuf { 
    }

}



async fn is_published(handle: &PackageHandle) -> Result<bool> {

    run_with_retry(

        &format!("Checking if `{}` is already published", handle.name),

        3,

        Duration::from_secs(5),

        || async {

            let expected_version = handle.version.to_string();

            let crate_info = match CRATES_IO_CLIENT.get_crate(&handle.name).await {

                Ok(info) => info,

                Err(Error::NotFound(_)) => return Ok(false),

                Err(other) => return Err(other),

            };

            Ok(crate_info

                .versions

                .iter()

                .any(|crate_version| crate_version.num == expected_version))

        },

        |err| match err {

            Error::Http(_) => ErrorClass::Retry,

            _ => ErrorClass::NoRetry,

        },

    )

    .await

    .context("is_published")

async fn is_published(index: Arc<CratesIndex>, handle: &PackageHandle) -> Result<bool> {

    let crate_name = handle.name.clone();

    let versions =

        tokio::task::spawn_blocking(move || index.published_versions(&crate_name)).await??;

    Ok(!versions.is_empty())

}



/// Waits for the given package to show up on crates.io

async fn wait_for_eventual_consistency(package: &Package) -> Result<()> {

async fn wait_for_eventual_consistency(index: Arc<CratesIndex>, package: &Package) -> Result<()> {

    let max_wait_time = 10usize;

    for _ in 0..max_wait_time {

        if !is_published(&package.handle).await? {

        if !is_published(index.clone(), &package.handle).await? {

            tokio::time::sleep(Duration::from_secs(1)).await;

        } else {

            return Ok(());

        }

    }

    if !is_published(&package.handle).await? {

    if !is_published(index.clone(), &package.handle).await? {

        return Err(anyhow::Error::msg(format!(

            "package wasn't found on crates.io {} seconds after publish",

            max_wait_time
@@ -248,19 +229,3 @@ fn confirm_plan( 
        bail!("aborted")

    }

}
 


#[cfg(test)]

mod test {

    use super::*;

    use crate::package::PackageHandle;



    #[ignore]

    #[tokio::test]

    async fn crate_published_works() {

        let handle = PackageHandle::new("aws-smithy-http", "0.27.0-alpha.1".parse().unwrap());

        assert!(is_published(&handle).await.expect("failed"));

        // we will never publish this version

        let handle = PackageHandle::new("aws-smithy-http", "0.21.0-alpha.1".parse().unwrap());

        assert!(!is_published(&handle).await.expect("failed"));

    }

}