Unverified Commit 6e63635e authored by John DiSanti's avatar John DiSanti Committed by GitHub
Browse files

Enable running the canary with a path to a generated SDK (#1303)

parent 6bbc776d

tools/ci-cdk/canary-lambda/README.md

+15 −3
Original line number Diff line number Diff line
@@ -13,8 +13,20 @@ the previous version of the SDK continues to work after the deployment 
of the new version.





Building locally for Lambda

---------------------------

Building locally for Lambda from Amazon Linux 2

-----------------------------------------------



1. Build a code bundle:



```

$ ./build-bundle --sdk-version <version>

```



This will place a zip file in `smithy-rs/target/release` that can be uploaded and tested against Lambda.





Building locally for Lambda from non Amazon Linux 2 system

----------------------------------------------------------



1. Make sure the `musl-gcc` wrapper is installed.

2. Add the musl target for Rust:
@@ -26,7 +38,7 @@ $ rustup target add x86_64-unknown-linux-musl 
3. Build a code bundle:



```

$ ./build-bundle.sh

$ ./build-bundle --sdk-version <version> --musl

```



This will place a zip file in `smithy-rs/target/x86_64-unknown-linux-musl/release` that can be

tools/ci-cdk/canary-lambda/additional-ci

+2 −2
Original line number Diff line number Diff line
@@ -5,12 +5,12 @@ 
#

# Run by CI to check the canary-lambda

set -e

cd "$(dirname $0)"

cd "$(dirname "$0")"



SDK_PATH="$(git rev-parse --show-toplevel)"/aws/sdk/build/aws-sdk/sdk

if [[ "${GITHUB_ACTIONS}" == "true" ]]; then

   SDK_PATH="$(git rev-parse --show-toplevel)"/aws-sdk/sdk

fi



./write-cargo-toml.py --path "${SDK_PATH}"

./build-bundle --sdk-path "${SDK_PATH}" --manifest-only

cargo clippy

tools/ci-cdk/canary-lambda/write-cargo-toml.py→tools/ci-cdk/canary-lambda/build-bundle

+175 −0
Original line number Diff line number Diff line
@@ -7,13 +7,18 @@ 
# Generates a Cargo.toml with the given AWS SDK version for this canary



import argparse

import hashlib

import os

import shlex

import subprocess

import sys

import zipfile



BASE_MANIFEST = """

# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.

# SPDX-License-Identifier: Apache-2.0.

#

# IMPORTANT: Don't edit this file directly! Run `write-cargo-toml.py` to modify this file instead.

# IMPORTANT: Don't edit this file directly! Run `build-bundle` to modify this file instead.

[package]

name = "aws-sdk-rust-lambda-canary"

version = "0.1.0"
@@ -50,31 +55,56 @@ notable_versions = [ 
    "0.4.1"

]





def main():

    args = Args()

    script_path = os.path.dirname(os.path.realpath(__file__))

    repository_root = get_cmd_output("git rev-parse --show-toplevel", cwd=script_path)[1]



    with open("Cargo.toml", "w") as file:

        print(BASE_MANIFEST, file=file)

        print(format_dependency("aws-config",

              args.path, args.sdk_version), file=file)

        print(format_dependency("aws-sdk-s3",

              args.path, args.sdk_version), file=file)

        print(format_dependency("aws-sdk-ec2",

                                args.path, args.sdk_version), file=file)

        print(format_dependency("aws-sdk-transcribestreaming",

              args.path, args.sdk_version), file=file)

        print(format_dependency("aws-config", args.sdk_path, args.sdk_version), file=file)

        print(format_dependency("aws-sdk-s3", args.sdk_path, args.sdk_version), file=file)

        print(format_dependency("aws-sdk-ec2", args.sdk_path, args.sdk_version), file=file)

        print(format_dependency("aws-sdk-transcribestreaming", args.sdk_path, args.sdk_version), file=file)

        print("[features]", file=file)

        for version in notable_versions:

            print(f'"v{version}" = []', file=file)

        enabled = ', '.join(enabled_versions(args.sdk_version))

        print(f'default = [{enabled}]', file=file)



    if args.manifest_only:

        sys.exit(0)



    target_arg = "--target=x86_64-unknown-linux-musl" if args.musl else ""

    run(f"cargo build --release {target_arg}", cwd=script_path)



    target_path = f"{repository_root}/tools/target/x86_64-unknown-linux-musl/release" \

        if args.musl else f"{repository_root}/tools/target/release"

    bin_hash = sha1_file(f"{target_path}/bootstrap")

    bundle_path = f"{target_path}/canary-lambda-{bin_hash}.zip"



    with zipfile.ZipFile(bundle_path, 'w') as zip:

        zip.write(f"{target_path}/bootstrap", arcname="bootstrap", compress_type=zipfile.ZIP_DEFLATED, compresslevel=5)



    print(bundle_path)





def sha1_file(path):

    hasher = hashlib.sha1()

    with open(path, 'rb') as file:

        contents = file.read()

        hasher.update(contents)

    return hasher.hexdigest()





def enabled_versions(sdk_version):

    if sdk_version is None:

        return [f'"v{version}"' for version in notable_versions]

    else:

        return [f'"v{version}"' for version in notable_versions if version.split('.') <= sdk_version.split('.')]





def format_dependency(crate, path, version):

    if path is None:

        return f'{crate} = "{version}"'
@@ -88,19 +118,56 @@ def format_dependency(crate, path, version): 
            return f'{crate} = {{ path = "{path}/{crate_path}", version = "{version}" }}'





def run(command, cwd=None):

    subprocess.run(shlex.split(command), stdout=sys.stderr, stderr=sys.stderr, cwd=cwd, check=True)





# Returns (status, output) from a shell command

def get_cmd_output(command, cwd=None, check=True):

    result = subprocess.run(

        shlex.split(command),

        capture_output=True,

        check=check,

        cwd=cwd

    )

    return (result.returncode, result.stdout.decode("utf-8").strip())





class Args:

    def __init__(self):

        parser = argparse.ArgumentParser()

        parser.add_argument("--path", dest="path", type=str,

                            help="Path to the generated AWS Rust SDK")

        parser.add_argument("--sdk-version", dest="sdk_version",

                            type=str, help="AWS Rust SDK version")

        parser.add_argument(

            "--sdk-path",

            dest="sdk_path",

            type=str,

            help="Path to the generated AWS Rust SDK"

        )

        parser.add_argument(

            "--sdk-version",

            dest="sdk_version",

            type=str,

            help="AWS Rust SDK version"

        )

        parser.add_argument(

            "--musl",

            dest="musl",

            action="store_true",

            help="Whether target MUSL instead of GLIBC"

        )

        parser.add_argument(

            "--manifest-only",

            dest="manifest_only",

            action="store_true",

            help="Only generate the Cargo.toml but don't build"

        )



        args = parser.parse_args()

        self.path = args.path

        self.sdk_path = args.sdk_path

        self.sdk_version = args.sdk_version

        if self.path == None and self.sdk_version == None:

            print("Either of path or sdk-version are required")

        self.musl = args.musl

        self.manifest_only = args.manifest_only

        if self.sdk_path is None and self.sdk_version is None:

            print("Either of sdk-path or sdk-version are required")

            sys.exit(1)

tools/ci-cdk/canary-lambda/build-bundle.sh

deleted100755 → 0
+0 −30
Original line number Diff line number Diff line 
#!/bin/bash

#

# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.

# SPDX-License-Identifier: Apache-2.0.

#



set -ex



if [[ $# -ne 1 ]]; then

    echo "Usage: $0 <sdk version>"

    exit 1

fi

SDK_VERSION=$1



cd `dirname $0`

./write-cargo-toml.py --sdk-version ${SDK_VERSION}

cargo build --release --target=x86_64-unknown-linux-musl



TARGET_PATH="$(git rev-parse --show-toplevel)/tools/target/x86_64-unknown-linux-musl/release"

pushd "${TARGET_PATH}" &>/dev/null



BIN_NAME="bootstrap"

BIN_SHA1_HASH="$(sha1sum ${BIN_NAME} | awk '{ print $1; }')"

BUNDLE_NAME="canary-lambda-${BIN_SHA1_HASH}.zip"



zip --quiet "${BUNDLE_NAME}" "${BIN_NAME}"

popd &>/dev/null



# Output: the path to the bundle

echo "${TARGET_PATH}/${BUNDLE_NAME}"

tools/ci-cdk/canary-runner/Cargo.lock

+53 −59
Original line number Diff line number Diff line
@@ -406,13 +406,13 @@ dependencies = [ 
 "aws-sdk-lambda",

 "aws-sdk-s3",

 "base64",

 "clap",

 "crates_io_api",

 "lazy_static",

 "semver",

 "serde",

 "serde_json",

 "smithy-rs-tool-common",

 "structopt",

 "tokio",

 "tracing",

 "tracing-subscriber",
@@ -443,17 +443,32 @@ dependencies = [ 


[[package]]

name = "clap"

version = "2.34.0"

version = "3.1.8"

source = "registry+https://github.com/rust-lang/crates.io-index"

checksum = "a0610544180c38b88101fecf2dd634b174a62eef6946f84dfc6a7127512b381c"

checksum = "71c47df61d9e16dc010b55dba1952a57d8c215dbb533fd13cdd13369aac73b1c"

dependencies = [

 "ansi_term",

 "atty",

 "bitflags",

 "clap_derive",

 "indexmap",

 "lazy_static",

 "os_str_bytes",

 "strsim",

 "termcolor",

 "textwrap",

 "unicode-width",

 "vec_map",

]



[[package]]

name = "clap_derive"

version = "3.1.7"

source = "registry+https://github.com/rust-lang/crates.io-index"

checksum = "a3aab4734e083b809aaf5794e14e756d1c798d2c69c7f7de7a09a2f5214993c1"

dependencies = [

 "heck",

 "proc-macro-error",

 "proc-macro2",

 "quote",

 "syn",

]



[[package]]
@@ -678,12 +693,9 @@ checksum = "ab5ef0d4909ef3724cc8cce6ccc8572c5c817592e9285f5464f8e86f8bd3726e" 


[[package]]

name = "heck"

version = "0.3.3"

version = "0.4.0"

source = "registry+https://github.com/rust-lang/crates.io-index"

checksum = "6d621efb26863f0e9924c6ac577e8275e5e6b77455db64ffa6c65c904e9e132c"

dependencies = [

 "unicode-segmentation",

]

checksum = "2540771e65fc8cb83cd6e8a237f70c319bd5c29f78ed1084ba5d50eeac86f7f9"



[[package]]

name = "hermit-abi"
@@ -1025,6 +1037,15 @@ dependencies = [ 
 "vcpkg",

]



[[package]]

name = "os_str_bytes"

version = "6.0.0"

source = "registry+https://github.com/rust-lang/crates.io-index"

checksum = "8e22443d1643a904602595ba1cd8f7d896afe56d26712531c5ff73a15b2fbf64"

dependencies = [

 "memchr",

]



[[package]]

name = "parking_lot"

version = "0.11.2"
@@ -1427,33 +1448,9 @@ checksum = "6e63cff320ae2c57904679ba7cb63280a3dc4613885beafb148ee7bf9aa9042d" 


[[package]]

name = "strsim"

version = "0.8.0"

source = "registry+https://github.com/rust-lang/crates.io-index"

checksum = "8ea5119cdb4c55b55d432abb513a0429384878c15dde60cc77b1c99de1a95a6a"



[[package]]

name = "structopt"

version = "0.3.26"

version = "0.10.0"

source = "registry+https://github.com/rust-lang/crates.io-index"

checksum = "0c6b5c64445ba8094a6ab0c3cd2ad323e07171012d9c98b0b15651daf1787a10"

dependencies = [

 "clap",

 "lazy_static",

 "structopt-derive",

]



[[package]]

name = "structopt-derive"

version = "0.4.18"

source = "registry+https://github.com/rust-lang/crates.io-index"

checksum = "dcb5ae327f9cc13b68763b5749770cb9e048a99bd9dfdfa58d0cf05d5f64afe0"

dependencies = [

 "heck",

 "proc-macro-error",

 "proc-macro2",

 "quote",

 "syn",

]

checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623"



[[package]]

name = "syn"
@@ -1481,14 +1478,20 @@ dependencies = [ 
]



[[package]]

name = "textwrap"

version = "0.11.0"

name = "termcolor"

version = "1.1.3"

source = "registry+https://github.com/rust-lang/crates.io-index"

checksum = "d326610f408c7a4eb6f51c37c330e496b08506c9457c9d34287ecc38809fb060"

checksum = "bab24d30b911b2376f3a13cc2cd443142f0c81dda04c118693e35b3835757755"

dependencies = [

 "unicode-width",

 "winapi-util",

]



[[package]]

name = "textwrap"

version = "0.15.0"

source = "registry+https://github.com/rust-lang/crates.io-index"

checksum = "b1141d4d61095b28419e22cb0bbf02755f5e54e0526f97f1e3d1d160e60885fb"



[[package]]

name = "thiserror"

version = "1.0.30"
@@ -1718,18 +1721,6 @@ dependencies = [ 
 "tinyvec",

]



[[package]]

name = "unicode-segmentation"

version = "1.8.0"

source = "registry+https://github.com/rust-lang/crates.io-index"

checksum = "8895849a949e7845e06bd6dc1aa51731a103c42707010a5b591c0038fb73385b"



[[package]]

name = "unicode-width"

version = "0.1.9"

source = "registry+https://github.com/rust-lang/crates.io-index"

checksum = "3ed742d4ea2bd1176e236172c8429aaf54486e7ac098db29ffe6529e0ce50973"



[[package]]

name = "unicode-xid"

version = "0.2.2"
@@ -1766,12 +1757,6 @@ version = "0.2.15" 
source = "registry+https://github.com/rust-lang/crates.io-index"

checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426"



[[package]]

name = "vec_map"

version = "0.8.2"

source = "registry+https://github.com/rust-lang/crates.io-index"

checksum = "f1bddf1187be692e79c5ffeab891132dfb0f236ed36a43c7ed39f1165ee20191"



[[package]]

name = "version_check"

version = "0.9.4"
@@ -1890,6 +1875,15 @@ version = "0.4.0" 
source = "registry+https://github.com/rust-lang/crates.io-index"

checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6"



[[package]]

name = "winapi-util"

version = "0.1.5"

source = "registry+https://github.com/rust-lang/crates.io-index"

checksum = "70ec6ce85bb158151cae5e5c87f95a8e97d2c0c4b001223f33a334e3ce5de178"

dependencies = [

 "winapi",

]



[[package]]

name = "winapi-x86_64-pc-windows-gnu"

version = "0.4.0"