Unverified Commit f0568e1f authored by Russell Cohen's avatar Russell Cohen Committed by GitHub
Browse files

remove parallelism from the sync tool (#1436) 

smithy-build actually utilizes `ForkJoinPool.common()` when executing build projections. Because each service is its own projection, the build internally is _already_ running in parallel. Because of this, running the sync in parallel produces uneccessary memory pressure.

This change:
- removes parallelism
- runs gradle `--no-daemon --info` for better log output
parent b72c807f

tools/sdk-sync/Cargo.lock

+0 −96
Original line number Diff line number Diff line
@@ -102,51 +102,6 @@ version = "0.8.3" 
source = "registry+https://github.com/rust-lang/crates.io-index"

checksum = "5827cebf4670468b8772dd191856768aedcb1b0278a04f989f7766351917b9dc"



[[package]]

name = "crossbeam-channel"

version = "0.5.4"

source = "registry+https://github.com/rust-lang/crates.io-index"

checksum = "5aaa7bd5fb665c6864b5f963dd9097905c54125909c7aa94c9e18507cdbe6c53"

dependencies = [

 "cfg-if",

 "crossbeam-utils",

]



[[package]]

name = "crossbeam-deque"

version = "0.8.1"

source = "registry+https://github.com/rust-lang/crates.io-index"

checksum = "6455c0ca19f0d2fbf751b908d5c55c1f5cbc65e03c4225427254b46890bdde1e"

dependencies = [

 "cfg-if",

 "crossbeam-epoch",

 "crossbeam-utils",

]



[[package]]

name = "crossbeam-epoch"

version = "0.9.8"

source = "registry+https://github.com/rust-lang/crates.io-index"

checksum = "1145cf131a2c6ba0615079ab6a638f7e1973ac9c2634fcbeaaad6114246efe8c"

dependencies = [

 "autocfg",

 "cfg-if",

 "crossbeam-utils",

 "lazy_static",

 "memoffset",

 "scopeguard",

]



[[package]]

name = "crossbeam-utils"

version = "0.8.8"

source = "registry+https://github.com/rust-lang/crates.io-index"

checksum = "0bf124c720b7686e3c2663cf54062ab0f68a88af2fb6a030e87e30bf721fcb38"

dependencies = [

 "cfg-if",

 "lazy_static",

]



[[package]]

name = "ctor"

version = "0.1.21"
@@ -305,15 +260,6 @@ version = "2.4.1" 
source = "registry+https://github.com/rust-lang/crates.io-index"

checksum = "308cc39be01b73d0d18f82a0e7b2a3df85245f84af96fdddc5d202d27e47b86a"



[[package]]

name = "memoffset"

version = "0.6.5"

source = "registry+https://github.com/rust-lang/crates.io-index"

checksum = "5aa361d4faea93603064a027415f07bd8e1d5c88c9fbf68bf56a285428fd79ce"

dependencies = [

 "autocfg",

]



[[package]]

name = "mockall"

version = "0.11.0"
@@ -365,16 +311,6 @@ dependencies = [ 
 "autocfg",

]



[[package]]

name = "num_cpus"

version = "1.13.1"

source = "registry+https://github.com/rust-lang/crates.io-index"

checksum = "19e64526ebdee182341572e50e9ad03965aa510cd94427a4549448f285e957a1"

dependencies = [

 "hermit-abi",

 "libc",

]



[[package]]

name = "once_cell"

version = "1.10.0"
@@ -489,30 +425,6 @@ dependencies = [ 
 "proc-macro2",

]



[[package]]

name = "rayon"

version = "1.5.2"

source = "registry+https://github.com/rust-lang/crates.io-index"

checksum = "fd249e82c21598a9a426a4e00dd7adc1d640b22445ec8545feef801d1a74c221"

dependencies = [

 "autocfg",

 "crossbeam-deque",

 "either",

 "rayon-core",

]



[[package]]

name = "rayon-core"

version = "1.9.2"

source = "registry+https://github.com/rust-lang/crates.io-index"

checksum = "9f51245e1e62e1f1629cbfec37b5793bbabcaeb90f30e94d2ba03564687353e4"

dependencies = [

 "crossbeam-channel",

 "crossbeam-deque",

 "crossbeam-utils",

 "num_cpus",

]



[[package]]

name = "redox_syscall"

version = "0.2.13"
@@ -557,12 +469,6 @@ dependencies = [ 
 "winapi",

]



[[package]]

name = "scopeguard"

version = "1.1.0"

source = "registry+https://github.com/rust-lang/crates.io-index"

checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd"



[[package]]

name = "sdk-sync"

version = "0.1.0"
@@ -571,10 +477,8 @@ dependencies = [ 
 "clap",

 "gitignore",

 "mockall",

 "num_cpus",

 "once_cell",

 "pretty_assertions",

 "rayon",

 "regex",

 "serde",

 "smithy-rs-tool-common",

tools/sdk-sync/Cargo.toml

+0 −2
Original line number Diff line number Diff line
@@ -18,8 +18,6 @@ opt-level = 0 
anyhow = "1.0.56"

clap = { version = "3.1.8", features = ["derive"] }

gitignore = "1.0.7"

num_cpus = "1.13.1"

rayon = "1.5.2"

serde = { version = "1.0.136", features = ["derive"] }

smithy-rs-tool-common = { version = "0.1", path = "../smithy-rs-tool-common" }

sysinfo = { version = "0.23.11", default-features = false }

tools/sdk-sync/src/main.rs

+0 −18
Original line number Diff line number Diff line
@@ -9,10 +9,6 @@ use sdk_sync::init_tracing; 
use sdk_sync::sync::Sync;

use smithy_rs_tool_common::macros::here;

use std::path::PathBuf;

use sysinfo::{System, SystemExt};

use tracing::info;



const CODEGEN_MIN_RAM_REQUIRED_GB: usize = 2;



/// A CLI tool to replay commits from smithy-rs, generate code, and commit that code to aws-rust-sdk.

#[derive(Parser, Debug)]
@@ -50,20 +46,6 @@ fn main() -> Result<()> { 
    init_tracing();

    let args = Args::parse();



    let sys = System::new_all();

    let available_ram_gb = (sys.available_memory() / 1024 / 1024) as usize;

    let num_cpus = num_cpus::get_physical();

    let threads = (available_ram_gb / CODEGEN_MIN_RAM_REQUIRED_GB)

        .max(1) // Must use at least 1 thread

        .min(num_cpus); // Don't exceed the number of physical CPUs

    info!("Available RAM (GB): {available_ram_gb}");

    info!("Num physical CPUs: {num_cpus}");

    info!("Thread pool size: {threads}");

    rayon::ThreadPoolBuilder::new()

        .num_threads(threads)

        .build_global()

        .unwrap();



    let sync = Sync::new(

        &args.aws_doc_sdk_examples.canonicalize().context(here!())?,

        &args.aws_sdk_rust.canonicalize().context(here!())?,

tools/sdk-sync/src/sync.rs

+1 −3
Original line number Diff line number Diff line
@@ -180,8 +180,6 @@ impl Sync { 
    /// Run through all commits made to `smithy-rs` since last sync and "replay" them onto `aws-sdk-rust`.

    #[instrument(skip(self, versions))]

    fn sync_smithy_rs(&self, versions: &VersionsManifest) -> Result<()> {

        use rayon::prelude::*;



        info!(

            "Checking for smithy-rs commits in range HEAD..{}",

            versions.smithy_rs_revision
@@ -208,7 +206,7 @@ impl Sync { 
            let fs = self.fs.clone();



            commits

                .par_iter()

                .iter()

                .enumerate()

                .map(move |(commit_num, commit_hash)| {

                    let span = info_span!(

tools/sdk-sync/src/sync/gen.rs

+2 −0
Original line number Diff line number Diff line
@@ -112,6 +112,8 @@ impl DefaultSdkGenerator { 
    fn aws_sdk_assemble(&self) -> Result<()> {

        info!("Generating the SDK...");

        let mut command = Command::new("./gradlew");

        command.arg("--no-daemon"); // Don't let Gradle continue running after the build

        command.arg("--info"); // Increase logging verbosity for failure debugging

        command.arg("-Paws.fullsdk=true");

        command.arg(format!(

            "-Paws.sdk.previous.release.versions.manifest={}",