Unverified Commit 40c0b04d authored by John DiSanti's avatar John DiSanti Committed by GitHub
Browse files

Use the same smithy-rs revision for all commits in `sdk-sync` (#1694)

parent 2fa4cdcb

codegen/src/main/kotlin/software/amazon/smithy/rust/codegen/smithy/Version.kt

+4 −5
Original line number Diff line number Diff line
@@ -28,9 +28,8 @@ data class Version(val fullVersion: String, val crateVersion: String) { 
        fun crateVersion(): String =

            fromDefaultResource().crateVersion



        private fun fromDefaultResource(): Version =

            parse(

                object {}.javaClass.getResource(VERSION_FILENAME)?.readText()

        private fun fromDefaultResource(): Version = parse(

            Version::class.java.getResource(VERSION_FILENAME)?.readText()

                ?: throw CodegenException("$VERSION_FILENAME does not exist"),

        )

    }

tools/sdk-sync/src/sync.rs

+40 −32
Original line number Diff line number Diff line
@@ -7,7 +7,7 @@ use self::gen::{CodeGenSettings, DefaultSdkGenerator, SdkGenerator}; 
use crate::fs::{DefaultFs, Fs};

use crate::versions::{DefaultVersions, Versions, VersionsManifest};

use anyhow::{bail, Context, Result};

use smithy_rs_tool_common::git::{Commit, Git, GitCLI};

use smithy_rs_tool_common::git::{Commit, CommitHash, Git, GitCLI};

use smithy_rs_tool_common::macros::here;

use std::collections::BTreeSet;

use std::path::{Path, PathBuf};
@@ -97,6 +97,7 @@ pub struct Sync { 
    aws_doc_sdk_examples: Arc<dyn Git>,

    aws_sdk_rust: Arc<dyn Git>,

    smithy_rs: Arc<dyn Git>,

    smithy_rs_head: CommitHash,

    fs: Arc<dyn Fs>,

    versions: Arc<dyn Versions>,

    previous_versions_manifest: Arc<PathBuf>,
@@ -124,10 +125,14 @@ impl Sync { 
        )

        .context("failed to copy versions.toml to temp dir")?;



        let smithy_rs = Arc::new(GitCLI::new(smithy_rs_path)?);

        let smithy_rs_head = smithy_rs.get_head_revision().context(here!())?;



        Ok(Self {

            aws_doc_sdk_examples: Arc::new(GitCLI::new(aws_doc_sdk_examples_path)?),

            aws_sdk_rust,

            smithy_rs: Arc::new(GitCLI::new(smithy_rs_path)?),

            smithy_rs,

            smithy_rs_head,

            fs,

            versions: Arc::new(DefaultVersions::new()),

            previous_versions_manifest,
@@ -149,6 +154,7 @@ impl Sync { 
            aws_doc_sdk_examples: Arc::new(aws_doc_sdk_examples),

            aws_sdk_rust: Arc::new(aws_sdk_rust),

            smithy_rs: Arc::new(smithy_rs),

            smithy_rs_head: "test-commit-hash".into(),

            fs: Arc::new(fs),

            versions: Arc::new(versions),

            previous_versions_manifest: Arc::new(PathBuf::from("doesnt-matter-for-tests")),
@@ -226,15 +232,15 @@ impl Sync { 
            .context(here!())?;



        // Generate with the original examples

        let sdk_gen = DefaultSdkGenerator::new(

            &self.previous_versions_manifest,

            &versions.aws_doc_sdk_examples_revision,

            &self.aws_sdk_rust.path().join("examples"),

            self.fs.clone(),

            None,

            self.smithy_rs.path(),

            &self.codegen_settings,

        )

        let sdk_gen = DefaultSdkGenerator::builder()

            .previous_versions_manifest(self.previous_versions_manifest.as_ref())

            .aws_doc_sdk_examples_revision(versions.aws_doc_sdk_examples_revision.clone())

            .examples_path(&self.aws_sdk_rust.path().join("examples"))

            .fs(self.fs.clone())

            .original_smithy_rs_path(self.smithy_rs.path())

            .smithy_rs_revision_override(self.smithy_rs_head.clone())

            .settings(self.codegen_settings.clone())

            .build()

            .context(here!())?;

        let generated_sdk = sdk_gen.generate_sdk().context(here!())?;

        self.copy_sdk(generated_sdk.path())
@@ -282,6 +288,7 @@ impl Sync { 
        let code_gen_paths = {

            let previous_versions_manifest = self.previous_versions_manifest.clone();

            let smithy_rs = self.smithy_rs.clone();

            let smithy_rs_head = self.smithy_rs_head.clone();

            let examples_revision = versions.aws_doc_sdk_examples_revision.clone();

            let examples_path = self.aws_sdk_rust.path().join("examples");

            let fs = self.fs.clone();
@@ -303,15 +310,16 @@ impl Sync { 
                        format!("couldn't find commit {} in smithy-rs", commit_hash)

                    })?;



                    let sdk_gen = DefaultSdkGenerator::new(

                        &previous_versions_manifest,

                        &examples_revision,

                        &examples_path,

                        fs.clone(),

                        Some(commit.hash.clone()),

                        smithy_rs.path(),

                        &codegen_settings,

                    )

                    let sdk_gen = DefaultSdkGenerator::builder()

                        .previous_versions_manifest(previous_versions_manifest.as_ref())

                        .aws_doc_sdk_examples_revision(examples_revision.clone())

                        .examples_path(examples_path.clone())

                        .fs(fs.clone())

                        .reset_to_commit(Some(commit.hash.clone()))

                        .original_smithy_rs_path(smithy_rs.path())

                        .smithy_rs_revision_override(smithy_rs_head.clone())

                        .settings(codegen_settings.clone())

                        .build()

                        .context(here!())?;

                    let sdk_path = sdk_gen.generate_sdk().context(here!())?;

                    progress.commits_completed.fetch_add(1, Ordering::Relaxed);
@@ -356,15 +364,15 @@ impl Sync { 
        }

        let examples_head = example_revisions.get(0).unwrap();



        let sdk_gen = DefaultSdkGenerator::new(

            &self.previous_versions_manifest,

            examples_head,

            &self.aws_doc_sdk_examples.path().join("rust_dev_preview"),

            self.fs.clone(),

            None,

            self.smithy_rs.path(),

            &self.codegen_settings,

        )

        let sdk_gen = DefaultSdkGenerator::builder()

            .previous_versions_manifest(self.previous_versions_manifest.as_ref())

            .aws_doc_sdk_examples_revision(examples_head.clone())

            .examples_path(self.aws_doc_sdk_examples.path().join("rust_dev_preview"))

            .fs(self.fs.clone())

            .original_smithy_rs_path(self.smithy_rs.path())

            .smithy_rs_revision_override(self.smithy_rs_head.clone())

            .settings(self.codegen_settings.clone())

            .build()

            .context(here!())?;

        let generated_sdk = sdk_gen.generate_sdk().context(here!())?;

        self.copy_sdk(generated_sdk.path())

tools/sdk-sync/src/sync/gen.rs

+86 −29
Original line number Diff line number Diff line
@@ -61,52 +61,103 @@ pub trait SdkGenerator { 
    fn generate_sdk(&self) -> Result<GeneratedSdk>;

}



/// SDK generator that creates a temporary directory and clones the given `smithy-rs` into it

/// so that generation can safely be done in parallel for other commits.

pub struct DefaultSdkGenerator {

    previous_versions_manifest: PathBuf,

    aws_doc_sdk_examples_revision: CommitHash,

    examples_path: PathBuf,

    fs: Arc<dyn Fs>,

    smithy_rs: Box<dyn Git>,

    settings: CodeGenSettings,

    temp_dir: Arc<tempfile::TempDir>,

#[derive(Default)]

pub struct Builder {

    previous_versions_manifest: Option<PathBuf>,

    aws_doc_sdk_examples_revision: Option<CommitHash>,

    examples_path: Option<PathBuf>,

    fs: Option<Arc<dyn Fs>>,

    reset_to_commit: Option<CommitHash>,

    original_smithy_rs_path: Option<PathBuf>,

    smithy_rs_revision_override: Option<CommitHash>,

    settings: Option<CodeGenSettings>,

}



impl DefaultSdkGenerator {

    #[instrument(skip(fs))]

    pub fn new(

        previous_versions_manifest: &Path,

        aws_doc_sdk_examples_revision: &CommitHash,

        examples_path: &Path,

        fs: Arc<dyn Fs>,

        reset_to_commit: Option<CommitHash>,

        original_smithy_rs_path: &Path,

        settings: &CodeGenSettings,

    ) -> Result<Self> {

impl Builder {

    pub fn previous_versions_manifest(mut self, path: impl Into<PathBuf>) -> Self {

        self.previous_versions_manifest = Some(path.into());

        self

    }



    pub fn aws_doc_sdk_examples_revision(mut self, revision: impl Into<CommitHash>) -> Self {

        self.aws_doc_sdk_examples_revision = Some(revision.into());

        self

    }



    pub fn examples_path(mut self, path: impl Into<PathBuf>) -> Self {

        self.examples_path = Some(path.into());

        self

    }



    pub fn fs(mut self, fs: Arc<dyn Fs>) -> Self {

        self.fs = Some(fs);

        self

    }



    pub fn reset_to_commit(mut self, maybe_commit: Option<CommitHash>) -> Self {

        self.reset_to_commit = maybe_commit;

        self

    }



    pub fn original_smithy_rs_path(mut self, path: impl Into<PathBuf>) -> Self {

        self.original_smithy_rs_path = Some(path.into());

        self

    }



    pub fn smithy_rs_revision_override(mut self, commit: impl Into<CommitHash>) -> Self {

        self.smithy_rs_revision_override = Some(commit.into());

        self

    }



    pub fn settings(mut self, settings: CodeGenSettings) -> Self {

        self.settings = Some(settings);

        self

    }



    pub fn build(self) -> Result<DefaultSdkGenerator> {

        let temp_dir = tempfile::tempdir().context(here!("create temp dir"))?;

        GitCLI::new(original_smithy_rs_path)

        GitCLI::new(&self.original_smithy_rs_path.expect("required"))

            .context(here!())?

            .clone_to(temp_dir.path())

            .context(here!())?;



        let smithy_rs = GitCLI::new(&temp_dir.path().join("smithy-rs")).context(here!())?;

        if let Some(smithy_rs_commit) = reset_to_commit {

        if let Some(smithy_rs_commit) = self.reset_to_commit {

            smithy_rs

                .hard_reset(smithy_rs_commit.as_ref())

                .with_context(|| format!("failed to reset to {} in smithy-rs", smithy_rs_commit))?;

        }



        Ok(Self {

            previous_versions_manifest: previous_versions_manifest.into(),

            aws_doc_sdk_examples_revision: aws_doc_sdk_examples_revision.clone(),

            examples_path: examples_path.into(),

            fs,

        Ok(DefaultSdkGenerator {

            previous_versions_manifest: self.previous_versions_manifest.expect("required"),

            aws_doc_sdk_examples_revision: self.aws_doc_sdk_examples_revision.expect("required"),

            examples_path: self.examples_path.expect("required"),

            fs: self.fs.expect("required"),

            smithy_rs: Box::new(smithy_rs) as Box<dyn Git>,

            settings: settings.clone(),

            smithy_rs_revision_override: self.smithy_rs_revision_override.expect("required"),

            settings: self.settings.expect("required"),

            temp_dir: Arc::new(temp_dir),

        })

    }

}



/// SDK generator that creates a temporary directory and clones the given `smithy-rs` into it

/// so that generation can safely be done in parallel for other commits.

pub struct DefaultSdkGenerator {

    previous_versions_manifest: PathBuf,

    aws_doc_sdk_examples_revision: CommitHash,

    examples_path: PathBuf,

    fs: Arc<dyn Fs>,

    smithy_rs: Box<dyn Git>,

    smithy_rs_revision_override: CommitHash,

    settings: CodeGenSettings,

    temp_dir: Arc<tempfile::TempDir>,

}



impl DefaultSdkGenerator {

    pub fn builder() -> Builder {

        Builder::default()

    }



    /// Copies examples into smithy-rs.

    #[instrument(skip(self))]
@@ -133,6 +184,12 @@ impl DefaultSdkGenerator { 


    fn do_aws_sdk_assemble(&self, attempt: u32) -> Result<()> {

        let mut command = Command::new("./gradlew");

        // Use the latest smithy-rs commit hash for all the codegen versions so that commits

        // that aren't actually making real changes aren't synced to the SDK

        command.env(

            "SMITHY_RS_VERSION_COMMIT_HASH_OVERRIDE",

            self.smithy_rs_revision_override.as_ref(),

        );

        command.arg("--no-daemon"); // Don't let Gradle continue running after the build

        command.arg("--no-parallel"); // Disable Gradle parallelism

        command.arg("--max-workers=1"); // Cap the Gradle workers at 1

tools/sdk-sync/tests/fake-sdk-assemble

+4 −0
Original line number Diff line number Diff line
@@ -64,6 +64,10 @@ else: 


examples_revision = get_examples_revision()

smithy_rs_revision = get_cmd_output("git rev-parse HEAD")

smithy_rs_version_commit_hash_override = os.getenv("SMITHY_RS_VERSION_COMMIT_HASH_OVERRIDE")

if not smithy_rs_version_commit_hash_override:

    print("Missing SMITHY_RS_VERSION_COMMIT_HASH_OVERRIDE env var")

    sys.exit(1)



# Emulate generating the versions.toml

with open("aws/sdk/build/aws-sdk/versions.toml", "w") as versions: