Unverified Commit 76ae89ae authored by ysaito1001's avatar ysaito1001 Committed by GitHub
Browse files

Fix running semver-checks on `aws-config` (#3272) 

## Motivation and Context
Fixes https://github.com/smithy-lang/smithy-rs/issues/3265

## Description
To check semver, `semver checks` needs to compile two versions of a
crate, i.e. baseline and current. Our CI failed to compile `aws-config`
for a baseline version.

The following diagram shows what was missing to cause compilation
failure:
```
┌───────────┐
│ smithy-rs │
└─────┬─────┘
      │
      │
      │     ┌────────┐
      └─────┤ target │
            └───┬────┘
                │        ┌───────────────┐
                └────────┤ semver_checks │
                         └───────┬───────┘
                                 │
                                 │
                                 │       ┌──────────┐
                                 └───────┤ git-base │
                                         └────┬─────┘
                                              │
                                              │        ┌──────────────┐
                                              ├────────┤ <rev number> │
                                              │        └──────┬───────┘
                                              │               │
                                              │               │
                                              │               │       ┌───────┐
                                              │               ├───────┤  aws  │
                                              │               │       └───┬───┘
                                              │               │           │
                                              │               │           │       ┌──────────────┐
                                              │               │           ├───────┤ rust-runtime │
                                              │               │           │       └──────┬───────┘
                                              │               │           │              │
                                              │               │           │              │       ┌────────────┐
                                              │               │           │              └───────┤ aws-config │ ◄***************
                                              │               │           │                      └────────────┘                *
                                              │               │           │                            *                       *
                                              │               │           │                            *                       *
                                              │               │           │          *****depends*on****                       *
                                              │               │           │          ▼                                         *
                                              │               │           │                                                    *
                                              │               │           │       ┌─────┐                                      *
                                              │               │           └───────┤ sdk │ (with no "build" directory)          *
                                              │               │                   └─────┘                                      *
                                              │               │                                                                *
                                              │               │                                                                *
                                              │               │      ┌────────────────────┐                                depends on
                                              │               └──────┤  tmp-codegen-diff  │                                    *
                                              │                      └─────────┬──────────┘                                    *
                                              │                                │                                               *
                                              │                                │       ┌─────────┐                             *
                                              │                                └───────┤ aws-sdk │                             *
                                              │                                        └────┬────┘                             *
                                              │                                             │     ┌─────┐                      *
                                              │                                             └─────┤ sdk │                      *
                                              │                                                   └─────┘                      *
                                              │                                                                                *
                                              │                                                                                *
                                              │           ┌───────────────────────────────────────┐                            *
                                              └───────────┤ local-aws_config-0_0_0_smithy_rs_head │*****************************
                                                          └───────────────────────────────────────┘
```
`local-aws_config-0_0_0_smithy_rs_head` under the `git-base` directory
is a special crate created by `semver-checks` for a baseline version of
`aws-config` and its `Cargo.toml` depends on
`target/semver_checks/git-base/<rev
number>/aws/rust-runtime/aws-config`. However, that `aws-config` in turn
depends upon crates in `target/semver_checks/git-base/<rev
number>/aws/sdk/build/` (as shown
[here](https://github.com/smithy-lang/smithy-rs/blob/main/aws/rust-runtime/aws-config/Cargo.toml#L23-L33)),
which does not exist in a baseline branch.

When `semver-checks.py` [creates a baseline
branch](https://github.com/smithy-lang/smithy-rs/blob/3d0cb5c3b179d5ed1d14531882657b481c4469f0/tools/ci-scripts/codegen-diff/semver-checks.py#L31)
to prepare for running `cargo semver-checks`, it [moves aws/sdk/build to
tmp-codegen-diff](https://github.com/smithy-lang/smithy-rs/blob/3d0cb5c3b179d5ed1d14531882657b481c4469f0/tools/ci-scripts/codegen-diff/diff_lib.py#L59),
leaving nothing behind in `aws/sdk/build/`. The fix, therefore, is to
`cp -r aws/sdk/build/aws-sdk` instead of `mv aws/sdk/build/aws-sdk` when
preparing a baseline branch.

The issue is specific to `aws-config`, probably because that's the only
runtime crate that depends on those in `aws/sdk/build`.

## Testing
Verified a clean run for `cargo semver-checks` in [an investigation
branch](https://github.com/smithy-lang/smithy-rs/actions/runs/7035082815/job/19144676499#step:4:1101).

----

_By submitting this pull request, I confirm that you can use, modify,
copy, and redistribute this contribution, under the terms of your
choice._
parent edab8cfb

tools/ci-scripts/codegen-diff/diff_lib.py

+11 −4
Original line number Diff line number Diff line
@@ -26,7 +26,7 @@ def running_in_docker_build(): 
    return os.environ.get("SMITHY_RS_DOCKER_BUILD_IMAGE") == "1"





def checkout_commit_and_generate(revision_sha, branch_name, targets=None):

def checkout_commit_and_generate(revision_sha, branch_name, targets=None, preserve_aws_sdk_build=False):

    if running_in_docker_build():

        eprint(f"Fetching base revision {revision_sha} from GitHub...")

        run(f"git fetch --no-tags --progress --no-recurse-submodules --depth=1 origin {revision_sha}")
@@ -34,10 +34,10 @@ def checkout_commit_and_generate(revision_sha, branch_name, targets=None): 
    # Generate code for HEAD

    eprint(f"Creating temporary branch {branch_name} with generated code for {revision_sha}")

    run(f"git checkout {revision_sha} -B {branch_name}")

    generate_and_commit_generated_code(revision_sha, targets)

    generate_and_commit_generated_code(revision_sha, targets, preserve_aws_sdk_build)





def generate_and_commit_generated_code(revision_sha, targets=None):

def generate_and_commit_generated_code(revision_sha, targets=None, preserve_aws_sdk_build=False):

    targets = targets or [

        target_codegen_client,

        target_codegen_server,
@@ -56,6 +56,10 @@ def generate_and_commit_generated_code(revision_sha, targets=None): 
    get_cmd_output(f"rm -rf {OUTPUT_PATH}")

    get_cmd_output(f"mkdir {OUTPUT_PATH}")

    if target_aws_sdk in targets:

        # Compiling aws-config for semver checks baseline requires build artifacts to exist under aws/sdk/build

        if preserve_aws_sdk_build:

            get_cmd_output(f"cp -r aws/sdk/build/aws-sdk {OUTPUT_PATH}/")

        else:

            get_cmd_output(f"mv aws/sdk/build/aws-sdk {OUTPUT_PATH}/")

    for target in [target_codegen_client, target_codegen_server]:

        if target in targets:
@@ -89,6 +93,9 @@ def generate_and_commit_generated_code(revision_sha, targets=None): 
        f"xargs rm -f", shell=True)



    get_cmd_output(f"git add -f {OUTPUT_PATH}")

    if preserve_aws_sdk_build:

        get_cmd_output(f"git add -f aws/sdk/build")



    get_cmd_output(f"git -c 'user.name=GitHub Action (generated code preview)' "

                   f"-c 'user.name={COMMIT_AUTHOR_NAME}' "

                   f"-c 'user.email={COMMIT_AUTHOR_EMAIL}' "

tools/ci-scripts/codegen-diff/semver-checks.py

+1 −3
Original line number Diff line number Diff line
@@ -28,7 +28,7 @@ def main(skip_generation=False): 


    if not skip_generation:

        checkout_commit_and_generate(head_commit_sha, CURRENT_BRANCH, targets=['aws:sdk'])

        checkout_commit_and_generate(base_commit_sha, BASE_BRANCH, targets=['aws:sdk'])

        checkout_commit_and_generate(base_commit_sha, BASE_BRANCH, targets=['aws:sdk'], preserve_aws_sdk_build=True)

    get_cmd_output(f'git checkout {CURRENT_BRANCH}')

    sdk_directory = os.path.join(OUTPUT_PATH, 'aws-sdk', 'sdk')

    os.chdir(sdk_directory)
@@ -36,8 +36,6 @@ def main(skip_generation=False): 
    failures = []

    deny_list = [

        # add crate names here to exclude them from the semver checks

        # TODO(https://github.com/smithy-lang/smithy-rs/issues/3265)

        'aws-config'

    ]

    for path in list(os.listdir())[:10]:

        eprint(f'checking {path}...', end='')