Unverified Commit 30421e13 authored by Russell Cohen's avatar Russell Cohen Committed by GitHub
Browse files

Support unchanging versions in the versioner (#3414) 

## Motivation and Context
The nested path dependencies in our generated runtime crates cause
issues when simulating a release. This strips those out in order to
support testing a release where some versions _don't_ change.

## Testing
https://github.com/smithy-lang/smithy-rs/actions/runs/7917462892


----

_By submitting this pull request, I confirm that you can use, modify,
copy, and redistribute this contribution, under the terms of your
choice._
parent 1ea9d055

tools/ci-build/runtime-versioner/Cargo.lock

+34 −2
Original line number Diff line number Diff line
@@ -148,6 +148,16 @@ version = "1.1.6" 
source = "registry+https://github.com/rust-lang/crates.io-index"

checksum = "c59e92b5a388f549b863a7bea62612c09f24c8393560709a54558a9abdfb3b9c"



[[package]]

name = "cargo_toml"

version = "0.19.1"

source = "registry+https://github.com/rust-lang/crates.io-index"

checksum = "3dc9f7a067415ab5058020f04c60ec7b557084dbec0e021217bbabc7a8d38d14"

dependencies = [

 "serde",

 "toml 0.8.8",

]



[[package]]

name = "cc"

version = "1.0.83"
@@ -903,6 +913,7 @@ version = "0.1.0" 
dependencies = [

 "anyhow",

 "camino",

 "cargo_toml",

 "clap",

 "crates-index",

 "indicatif",
@@ -911,6 +922,7 @@ dependencies = [ 
 "tempfile",

 "test-common",

 "toml 0.5.11",

 "toml_edit 0.22.5",

 "tracing",

 "tracing-subscriber",

]
@@ -1276,7 +1288,7 @@ dependencies = [ 
 "serde",

 "serde_spanned",

 "toml_datetime",

 "toml_edit",

 "toml_edit 0.21.0",

]



[[package]]
@@ -1298,7 +1310,18 @@ dependencies = [ 
 "serde",

 "serde_spanned",

 "toml_datetime",

 "winnow",

 "winnow 0.5.28",

]



[[package]]

name = "toml_edit"

version = "0.22.5"

source = "registry+https://github.com/rust-lang/crates.io-index"

checksum = "99e68c159e8f5ba8a28c4eb7b0c0c190d77bb479047ca713270048145a9ad28a"

dependencies = [

 "indexmap 2.1.0",

 "toml_datetime",

 "winnow 0.6.1",

]



[[package]]
@@ -1684,6 +1707,15 @@ dependencies = [ 
 "memchr",

]



[[package]]

name = "winnow"

version = "0.6.1"

source = "registry+https://github.com/rust-lang/crates.io-index"

checksum = "d90f4e0f530c4c69f62b80d839e9ef3855edc9cba471a160c4d692deed62b401"

dependencies = [

 "memchr",

]



[[package]]

name = "winreg"

version = "0.50.0"

tools/ci-build/runtime-versioner/Cargo.toml

+3 −0
Original line number Diff line number Diff line
@@ -25,6 +25,9 @@ tempfile = "3.9.0" 
toml = { version = "0.5.8", features = ["preserve_order"] }

tracing = "0.1.40"

tracing-subscriber = { version = "0.3.18", features = ["env-filter"] }

# why both? cargo_toml can't write out to a file because of a toml_rs longstanding issue/bug

cargo_toml = "0.19.0"

toml_edit = "0.22"



[dev-dependencies]

test-common = { path = "./test-common" }

tools/ci-build/runtime-versioner/src/command/patch.rs

+115 −16
Original line number Diff line number Diff line
@@ -10,9 +10,11 @@ use crate::{ 
};

use anyhow::{bail, Context, Result};

use camino::Utf8Path;

use cargo_toml::Manifest;

use indicatif::{ProgressBar, ProgressStyle};

use smithy_rs_tool_common::command::sync::CommandExt;

use std::{fs, time::Duration};

use toml_edit::Document;



pub fn patch(args: PatchRuntime) -> Result<()> {

    let smithy_rs = step("Resolving smithy-rs", || {
@@ -22,6 +24,11 @@ pub fn patch(args: PatchRuntime) -> Result<()> { 
        bail!("smithy-rs has a dirty working tree. Aborting.");

    }



    let aws_sdk_rust = step("Resolving aws-sdk-rust", || Repo::new(Some(&args.sdk_path)))?;

    if is_dirty(&aws_sdk_rust)? {

        bail!("aws-sdk-rust has a dirty working tree. Aborting.");

    }



    step(

        "Patching smithy-rs/gradle.properties with given crate version numbers",

        || patch_gradle_properties(&smithy_rs, &args),
@@ -76,7 +83,9 @@ pub fn patch_with(args: PatchRuntimeWith) -> Result<()> { 
        apply_version_only_dependencies(&aws_sdk_rust)

    })?;

    step("Patching aws-sdk-rust root Cargo.toml", || {

        patch_workspace_cargo_toml(&aws_sdk_rust, &args.runtime_crate_path)

        let crates_to_patch =

            remove_unchanged_dependencies(&aws_sdk_rust, &args.runtime_crate_path)?;

        patch_workspace_cargo_toml(&aws_sdk_rust, &args.runtime_crate_path, crates_to_patch)

    })?;

    step("Running cargo update", || {

        aws_sdk_rust
@@ -127,27 +136,46 @@ fn apply_version_only_dependencies(aws_sdk_rust: &Repo) -> Result<()> { 
    Ok(())

}



fn patch_workspace_cargo_toml(aws_sdk_rust: &Repo, runtime_crate_path: &Utf8Path) -> Result<()> {

    let crates_to_patch = fs::read_dir(runtime_crate_path)

        .context(format!(

            "could list crates in directory {:?}",

            runtime_crate_path

        ))?

        .map(|dir| dir.unwrap().file_name())

        .map(|osstr| osstr.into_string().expect("invalid utf-8 directory"))

        .filter(|name| name.starts_with("aws-"))

        .collect::<Vec<_>>();



/// Determine if a given crate has a new version vs. the release we're comparing

fn crate_version_has_changed(

    crate_name: &str,

    aws_sdk_rust: &Repo,

    runtime_crate_path: &Utf8Path,

) -> Result<bool> {

    let sdk_cargo_toml = aws_sdk_rust

        .root

        .join("sdk")

        .join(crate_name)

        .join("Cargo.toml");

    let to_patch_cargo_toml = runtime_crate_path.join(crate_name).join("Cargo.toml");

    assert!(

        sdk_cargo_toml.exists(),

        "{:?} did not exist!",

        sdk_cargo_toml

    );

    assert!(

        to_patch_cargo_toml.exists(),

        "{:?} did not exist!",

        to_patch_cargo_toml

    );

    let sdk_cargo_toml = Manifest::from_path(sdk_cargo_toml).context("could not parse")?;

    let to_patch_toml = Manifest::from_path(to_patch_cargo_toml).context("could not parse")?;

    Ok(sdk_cargo_toml.package().version() != to_patch_toml.package().version())

}

fn patch_workspace_cargo_toml(

    aws_sdk_rust: &Repo,

    runtime_crate_path: &Utf8Path,

    crates_to_patch: impl Iterator<Item = String>,

) -> Result<()> {

    let patch_sections = crates_to_patch

        .iter()

        .map(|crte| {

            let path = runtime_crate_path.join(crte);

        .map(|crate_name| {

            let path = runtime_crate_path.join(&crate_name);

            assert!(

                path.exists(),

                "tried to reference a crate that did not exist!"

            );

            format!(

                "{crte} = {{ path = '{}' }}",

                "{crate_name} = {{ path = '{}' }}",

                path.canonicalize_utf8().unwrap()

            )

        })
@@ -164,6 +192,77 @@ fn patch_workspace_cargo_toml(aws_sdk_rust: &Repo, runtime_crate_path: &Utf8Path 
    Ok(())

}



/// Removes Path dependencies referring to unchanged crates & returns a list of crates to patch

fn remove_unchanged_dependencies(

    aws_sdk_rust: &Repo,

    runtime_crate_path: &Utf8Path,

) -> Result<impl Iterator<Item = String>> {

    let all_crates = fs::read_dir(runtime_crate_path)

        .context(format!(

            "could list crates in directory {:?}",

            runtime_crate_path

        ))?

        .map(|dir| dir.unwrap().file_name())

        .map(|osstr| osstr.into_string().expect("invalid utf-8 directory"))

        .collect::<Vec<_>>();



    let (crates_to_patch, unchanged_crates): (Vec<_>, Vec<_>) =

        all_crates.clone().into_iter().partition(|crate_dir| {

            crate_version_has_changed(crate_dir, aws_sdk_rust, runtime_crate_path)

                .expect("failed to determine change-status")

        });



    for patched_crate in &all_crates {

        remove_unchanged_path_dependencies(runtime_crate_path, &unchanged_crates, patched_crate)?;

    }

    Ok(crates_to_patch

        .into_iter()

        .filter(|crte| crte.starts_with("aws-")))

}



/// Remove `path = ...` from the dependency section for unchanged crates

///

/// If we leave these path dependencies in, we'll get an error when we try to patch because the

/// version numbers are the same.

fn remove_unchanged_path_dependencies(

    runtime_crate_path: &Utf8Path,

    unchanged_crates: &[String],

    patched_crate: &String,

) -> Result<()> {

    let path = runtime_crate_path.join(patched_crate).join("Cargo.toml");

    let manifest = Manifest::from_path(&path)?;

    let mut mutable_manifest = fs::read_to_string(&path)

        .context("failed to read file")

        .context(path.clone())?

        .parse::<Document>()

        .context("invalid toml in manifest!")?;

    let mut updates = false;

    let sections = [

        (manifest.dependencies, "dependencies"),

        (manifest.dev_dependencies, "dev-dependencies"),

    ];

    for (deps_set, key) in sections {

        for (dependency_name, dependency_metadata) in deps_set.iter() {

            if unchanged_crates.iter().any(|crate_name| {

                crate_name.as_str()

                    == dependency_metadata

                        .package()

                        .unwrap_or(dependency_name.as_str())

            }) {

                mutable_manifest[key][dependency_name]

                    .as_table_mut()

                    .unwrap()

                    .remove("path");

                updates = true

            }

        }

    }

    if updates {

        fs::write(&path, mutable_manifest.to_string()).context("failed to write back manifest")?

    }

    Ok(())

}



fn is_dirty(repo: &Repo) -> Result<bool> {

    let result = repo

        .git(["status", "--porcelain"])