Unverified Commit f1a47826 authored by John DiSanti's avatar John DiSanti Committed by GitHub
Browse files

Reintroduce `sdk-sync` parallelism by disabling Smithy parallelism (#1443) 

* Revert "remove parallelism from the sync tool (#1436)"

This reverts commit f0568e1f.

* Reintroduce `sdk-sync` parallelism by disabling Smithy parallelism

* Make the Smithy parallelism configurable
parent a49076a1

tools/sdk-sync/Cargo.lock

+96 −0
Original line number Diff line number Diff line
@@ -102,6 +102,51 @@ version = "0.8.3" 
source = "registry+https://github.com/rust-lang/crates.io-index"

checksum = "5827cebf4670468b8772dd191856768aedcb1b0278a04f989f7766351917b9dc"



[[package]]

name = "crossbeam-channel"

version = "0.5.4"

source = "registry+https://github.com/rust-lang/crates.io-index"

checksum = "5aaa7bd5fb665c6864b5f963dd9097905c54125909c7aa94c9e18507cdbe6c53"

dependencies = [

 "cfg-if",

 "crossbeam-utils",

]



[[package]]

name = "crossbeam-deque"

version = "0.8.1"

source = "registry+https://github.com/rust-lang/crates.io-index"

checksum = "6455c0ca19f0d2fbf751b908d5c55c1f5cbc65e03c4225427254b46890bdde1e"

dependencies = [

 "cfg-if",

 "crossbeam-epoch",

 "crossbeam-utils",

]



[[package]]

name = "crossbeam-epoch"

version = "0.9.8"

source = "registry+https://github.com/rust-lang/crates.io-index"

checksum = "1145cf131a2c6ba0615079ab6a638f7e1973ac9c2634fcbeaaad6114246efe8c"

dependencies = [

 "autocfg",

 "cfg-if",

 "crossbeam-utils",

 "lazy_static",

 "memoffset",

 "scopeguard",

]



[[package]]

name = "crossbeam-utils"

version = "0.8.8"

source = "registry+https://github.com/rust-lang/crates.io-index"

checksum = "0bf124c720b7686e3c2663cf54062ab0f68a88af2fb6a030e87e30bf721fcb38"

dependencies = [

 "cfg-if",

 "lazy_static",

]



[[package]]

name = "ctor"

version = "0.1.21"
@@ -260,6 +305,15 @@ version = "2.4.1" 
source = "registry+https://github.com/rust-lang/crates.io-index"

checksum = "308cc39be01b73d0d18f82a0e7b2a3df85245f84af96fdddc5d202d27e47b86a"



[[package]]

name = "memoffset"

version = "0.6.5"

source = "registry+https://github.com/rust-lang/crates.io-index"

checksum = "5aa361d4faea93603064a027415f07bd8e1d5c88c9fbf68bf56a285428fd79ce"

dependencies = [

 "autocfg",

]



[[package]]

name = "mockall"

version = "0.11.0"
@@ -311,6 +365,16 @@ dependencies = [ 
 "autocfg",

]



[[package]]

name = "num_cpus"

version = "1.13.1"

source = "registry+https://github.com/rust-lang/crates.io-index"

checksum = "19e64526ebdee182341572e50e9ad03965aa510cd94427a4549448f285e957a1"

dependencies = [

 "hermit-abi",

 "libc",

]



[[package]]

name = "once_cell"

version = "1.10.0"
@@ -425,6 +489,30 @@ dependencies = [ 
 "proc-macro2",

]



[[package]]

name = "rayon"

version = "1.5.2"

source = "registry+https://github.com/rust-lang/crates.io-index"

checksum = "fd249e82c21598a9a426a4e00dd7adc1d640b22445ec8545feef801d1a74c221"

dependencies = [

 "autocfg",

 "crossbeam-deque",

 "either",

 "rayon-core",

]



[[package]]

name = "rayon-core"

version = "1.9.2"

source = "registry+https://github.com/rust-lang/crates.io-index"

checksum = "9f51245e1e62e1f1629cbfec37b5793bbabcaeb90f30e94d2ba03564687353e4"

dependencies = [

 "crossbeam-channel",

 "crossbeam-deque",

 "crossbeam-utils",

 "num_cpus",

]



[[package]]

name = "redox_syscall"

version = "0.2.13"
@@ -469,6 +557,12 @@ dependencies = [ 
 "winapi",

]



[[package]]

name = "scopeguard"

version = "1.1.0"

source = "registry+https://github.com/rust-lang/crates.io-index"

checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd"



[[package]]

name = "sdk-sync"

version = "0.1.0"
@@ -477,8 +571,10 @@ dependencies = [ 
 "clap",

 "gitignore",

 "mockall",

 "num_cpus",

 "once_cell",

 "pretty_assertions",

 "rayon",

 "regex",

 "serde",

 "smithy-rs-tool-common",

tools/sdk-sync/Cargo.toml

+2 −0
Original line number Diff line number Diff line
@@ -18,6 +18,8 @@ opt-level = 0 
anyhow = "1.0.56"

clap = { version = "3.1.8", features = ["derive"] }

gitignore = "1.0.7"

num_cpus = "1.13.1"

rayon = "1.5.2"

serde = { version = "1.0.136", features = ["derive"] }

smithy-rs-tool-common = { version = "0.1", path = "../smithy-rs-tool-common" }

sysinfo = { version = "0.23.11", default-features = false }

tools/sdk-sync/src/main.rs

+35 −0
Original line number Diff line number Diff line
@@ -9,6 +9,10 @@ use sdk_sync::init_tracing; 
use sdk_sync::sync::Sync;

use smithy_rs_tool_common::macros::here;

use std::path::PathBuf;

use sysinfo::{System, SystemExt};

use tracing::info;



const CODEGEN_MIN_RAM_REQUIRED_GB: usize = 2;



/// A CLI tool to replay commits from smithy-rs, generate code, and commit that code to aws-rust-sdk.

#[derive(Parser, Debug)]
@@ -23,6 +27,15 @@ struct Args { 
    /// Path to the aws-doc-sdk-examples repository.

    #[clap(long, parse(from_os_str))]

    aws_doc_sdk_examples: PathBuf,



    /// Number of threads that `sdk-sync` will use. Defaults to the physical number of CPUs,

    /// or the available RAM divided by the RAM required for codegen. Whichever is smaller.

    #[clap(long)]

    sync_threads: Option<usize>,

    /// The Java parallelism (corresponding to the `java.util.concurrent.ForkJoinPool.common.parallelism`

    /// system property) to use for Smithy codegen. Defaults to 1.

    #[clap(long)]

    smithy_parallelism: Option<usize>,

}



/// This tool syncs codegen changes from smithy-rs, examples changes from aws-doc-sdk-examples,
@@ -46,10 +59,32 @@ fn main() -> Result<()> { 
    init_tracing();

    let args = Args::parse();



    let sys = System::new_all();

    let available_ram_gb = (sys.available_memory() / 1024 / 1024) as usize;

    let num_cpus = num_cpus::get_physical();

    info!("Available RAM (GB): {available_ram_gb}");

    info!("Num physical CPUs: {num_cpus}");



    let smithy_parallelism = args.smithy_parallelism.unwrap_or(1);

    let sync_threads = if let Some(sync_threads) = args.sync_threads {

        sync_threads

    } else {

        (available_ram_gb / CODEGEN_MIN_RAM_REQUIRED_GB)

            .max(1) // Must use at least 1 thread

            .min(num_cpus) // Don't exceed the number of physical CPUs

    };

    info!("Sync thread pool size: {sync_threads}");



    rayon::ThreadPoolBuilder::new()

        .num_threads(sync_threads)

        .build_global()

        .unwrap();



    let sync = Sync::new(

        &args.aws_doc_sdk_examples.canonicalize().context(here!())?,

        &args.aws_sdk_rust.canonicalize().context(here!())?,

        &args.smithy_rs.canonicalize().context(here!())?,

        smithy_parallelism,

    )?;



    sync.sync().map_err(|e| e.context("The sync failed"))

tools/sdk-sync/src/sync.rs

+11 −1
Original line number Diff line number Diff line
@@ -28,6 +28,7 @@ pub struct Sync { 
    fs: Arc<dyn Fs>,

    versions: Arc<dyn Versions>,

    previous_versions_manifest: Arc<PathBuf>,

    smithy_parallelism: usize,

    // Keep a reference to the temp directory so that it doesn't get cleaned up until the sync is complete

    _temp_dir: Arc<tempfile::TempDir>,

}
@@ -37,6 +38,7 @@ impl Sync { 
        aws_doc_sdk_examples_path: &Path,

        aws_sdk_rust_path: &Path,

        smithy_rs_path: &Path,

        smithy_parallelism: usize,

    ) -> Result<Self> {

        let _temp_dir = Arc::new(tempfile::tempdir().context(here!("create temp dir"))?);

        let aws_sdk_rust = Arc::new(GitCLI::new(aws_sdk_rust_path)?);
@@ -56,6 +58,7 @@ impl Sync { 
            fs,

            versions: Arc::new(DefaultVersions::new()),

            previous_versions_manifest,

            smithy_parallelism,

            _temp_dir,

        })

    }
@@ -75,6 +78,7 @@ impl Sync { 
            fs: Arc::new(fs),

            versions: Arc::new(versions),

            previous_versions_manifest: Arc::new(PathBuf::from("doesnt-matter-for-tests")),

            smithy_parallelism: 1,

            _temp_dir: Arc::new(tempfile::tempdir().unwrap()),

        }

    }
@@ -158,6 +162,7 @@ impl Sync { 
            self.fs.clone(),

            None,

            self.smithy_rs.path(),

            self.smithy_parallelism,

        )

        .context(here!())?;

        let generated_sdk = sdk_gen.generate_sdk().context(here!())?;
@@ -180,6 +185,8 @@ impl Sync { 
    /// Run through all commits made to `smithy-rs` since last sync and "replay" them onto `aws-sdk-rust`.

    #[instrument(skip(self, versions))]

    fn sync_smithy_rs(&self, versions: &VersionsManifest) -> Result<()> {

        use rayon::prelude::*;



        info!(

            "Checking for smithy-rs commits in range HEAD..{}",

            versions.smithy_rs_revision
@@ -204,9 +211,10 @@ impl Sync { 
            let examples_revision = versions.aws_doc_sdk_examples_revision.clone();

            let examples_path = self.aws_sdk_rust.path().join("examples");

            let fs = self.fs.clone();

            let smithy_parallelism = self.smithy_parallelism;



            commits

                .iter()

                .par_iter()

                .enumerate()

                .map(move |(commit_num, commit_hash)| {

                    let span = info_span!(
@@ -227,6 +235,7 @@ impl Sync { 
                        fs.clone(),

                        Some(commit.hash.clone()),

                        smithy_rs.path(),

                        smithy_parallelism,

                    )

                    .context(here!())?;

                    let sdk_path = sdk_gen.generate_sdk().context(here!())?;
@@ -278,6 +287,7 @@ impl Sync { 
            self.fs.clone(),

            None,

            self.smithy_rs.path(),

            self.smithy_parallelism,

        )

        .context(here!())?;

        let generated_sdk = sdk_gen.generate_sdk().context(here!())?;

tools/sdk-sync/src/sync/gen.rs

+9 −0
Original line number Diff line number Diff line
@@ -48,6 +48,7 @@ pub struct DefaultSdkGenerator { 
    examples_path: PathBuf,

    fs: Arc<dyn Fs>,

    smithy_rs: Box<dyn Git>,

    smithy_parallelism: usize,

    temp_dir: Arc<tempfile::TempDir>,

}
@@ -60,6 +61,7 @@ impl DefaultSdkGenerator { 
        fs: Arc<dyn Fs>,

        reset_to_commit: Option<CommitHash>,

        original_smithy_rs_path: &Path,

        smithy_parallelism: usize,

    ) -> Result<Self> {

        let temp_dir = tempfile::tempdir().context(here!("create temp dir"))?;

        GitCLI::new(original_smithy_rs_path)
@@ -80,6 +82,7 @@ impl DefaultSdkGenerator { 
            examples_path: examples_path.into(),

            fs,

            smithy_rs: Box::new(smithy_rs) as Box<dyn Git>,

            smithy_parallelism,

            temp_dir: Arc::new(temp_dir),

        })

    }
@@ -114,6 +117,12 @@ impl DefaultSdkGenerator { 
        let mut command = Command::new("./gradlew");

        command.arg("--no-daemon"); // Don't let Gradle continue running after the build

        command.arg("--info"); // Increase logging verbosity for failure debugging



        // Disable Smithy's codegen parallelism in favor of sdk-sync parallelism

        command.arg(format!(

            "-Djava.util.concurrent.ForkJoinPool.common.parallelism={}",

            self.smithy_parallelism

        ));

        command.arg("-Paws.fullsdk=true");

        command.arg(format!(

            "-Paws.sdk.previous.release.versions.manifest={}",