Unverified Commit d7593493 authored by Landon James's avatar Landon James Committed by GitHub
Browse files

Adding new gradle tasks for generating Cargo.lock (#3689) 

## Motivation and Context
<!--- Why is this change required? What problem does it solve? -->
<!--- If it fixes an open issue, please link to the issue here -->
We have had numerous recent issues where updated dependencies have
broken our build/release process. This seeks to fix that in a brute
force way by manually generating Cargo.lock files for our main
workspaces.

## Description
<!--- Describe your changes in detail -->
Introduce a new gradle task `generateAllLockfiles` (and some child tasks
that it invokes) that generates lock files for
* `aws-config` crate
* `aws/rust-runtime` workspace
* `rust-runtime` workspace
* the superset workspace generated by the `assemble` task
* NOTE: This Cargo.lock is actually generated from the workspace in the
`aws-sdk-rust` repo, saved in this repo, and copied into place at build
time. Generating this one requires a local copy of the `aws-sdk-rust`
repo whose absolute path is indicated by the `-Paws-sdk-rust-path`
parameter.

## Testing
<!--- Please describe in detail how you tested your changes -->
<!--- Include details of your testing environment, and the tests you ran
to -->
<!--- see how your change affects other areas of the code, etc. -->

I ran our E2E tests against 3 commits to confirm that the Cargo.lock
files operate as we expect:
* `9d84980ee2e37deadb7e45450d65c5e6bc69e07c` The head commit from this
PR which has our Cargo.lock files
	* ✅ Passes
* `d9d9fcec2de0e898bcc804ee0bb31393d7098e72` with lockfiles and with the
`httparse` and `crc323c` crates unpinned
	* ✅ Passes
* `df75beb160890b2a9c382c8a2f7da8d7d0ddd3c8` with no lockfiles and has
the `httparse` and `crc32c` crates unpinned
	* ❌ Fails (as expected)

This series of tests confirms that that Cargo.lock files will protect us
against bad dependency updates going forward.

## Checklist
<!--- If a checkbox below is not applicable, then please DELETE it
rather than leaving it unchecked -->

----

_By submitting this pull request, I confirm that you can use, modify,
copy, and redistribute this contribution, under the terms of your
choice._
parent 3177a6b1

aws/rust-runtime/.gitignore

+0 −1
Original line number Diff line number Diff line 
target/
 
Cargo.lock

aws/rust-runtime/Cargo.lock

0 → 100644
+2557 −0

File added.

Preview size limit exceeded, changes collapsed.

aws/rust-runtime/aws-config/Cargo.lock

0 → 100644
+2056 −0

File added.

Preview size limit exceeded, changes collapsed.

aws/sdk/Cargo.lock

0 → 100644
+13050 −0

File added.

Preview size limit exceeded, changes collapsed.

aws/sdk/build.gradle.kts

+65 −4
Original line number Diff line number Diff line
@@ -37,10 +37,15 @@ val properties = PropertyRetriever(rootProject, project) 
val crateHasherToolPath = rootProject.projectDir.resolve("tools/ci-build/crate-hasher")

val publisherToolPath = rootProject.projectDir.resolve("tools/ci-build/publisher")

val sdkVersionerToolPath = rootProject.projectDir.resolve("tools/ci-build/sdk-versioner")

val awsConfigPath = rootProject.projectDir.resolve("aws/rust-runtime/aws-config")

val rustRuntimePath = rootProject.projectDir.resolve("rust-runtime")

val awsRustRuntimePath = rootProject.projectDir.resolve("aws/rust-runtime")

val checkedInCargoLock = rootProject.projectDir.resolve("aws/sdk/Cargo.lock")

val outputDir = layout.buildDirectory.dir("aws-sdk").get()

val sdkOutputDir = outputDir.dir("sdk")

val examplesOutputDir = outputDir.dir("examples")





dependencies {

    implementation(project(":aws:sdk-codegen"))

    implementation("software.amazon.smithy:smithy-protocol-test-traits:$smithyVersion")
@@ -87,9 +92,12 @@ fun generateSmithyBuild(services: AwsServices): String { 
        }

        val moduleName = "aws-sdk-${service.module}"

        val eventStreamAllowListMembers = eventStreamAllowList.joinToString(", ") { "\"$it\"" }

        val defaultConfigPath = services.defaultConfigPath.let { software.amazon.smithy.utils.StringUtils.escapeJavaString(it, "") }

        val partitionsConfigPath = services.partitionsConfigPath.let { software.amazon.smithy.utils.StringUtils.escapeJavaString(it, "") }

        val integrationTestPath = project.projectDir.resolve("integration-tests").let { software.amazon.smithy.utils.StringUtils.escapeJavaString(it, "") }

        val defaultConfigPath =

            services.defaultConfigPath.let { software.amazon.smithy.utils.StringUtils.escapeJavaString(it, "") }

        val partitionsConfigPath =

            services.partitionsConfigPath.let { software.amazon.smithy.utils.StringUtils.escapeJavaString(it, "") }

        val integrationTestPath = project.projectDir.resolve("integration-tests")

            .let { software.amazon.smithy.utils.StringUtils.escapeJavaString(it, "") }

        """

            "${service.module}": {

                "imports": [${files.joinToString()}],
@@ -293,7 +301,10 @@ tasks.register("relocateAwsRuntime") { 
        // Patch the Cargo.toml files

        CrateSet.AWS_SDK_RUNTIME.forEach { module ->

            patchFile(sdkOutputDir.file("${module.name}/Cargo.toml").asFile) { line ->

                rewriteRuntimeCrateVersion(properties.get(module.versionPropertyName)!!, line.let(::rewritePathDependency))

                rewriteRuntimeCrateVersion(

                    properties.get(module.versionPropertyName)!!,

                    line.let(::rewritePathDependency),

                )

            }

        }

    }
@@ -439,9 +450,17 @@ tasks["assemble"].apply { 
        "hydrateReadme",

        "relocateChangelog",

    )

    finalizedBy("copyCheckedInCargoLock")

    outputs.upToDateWhen { false }

}



tasks.register<Copy>("copyCheckedInCargoLock") {

    description = "Copy the checked in Cargo.lock file back to the build directory"

    this.outputs.upToDateWhen { false }

    from(checkedInCargoLock)

    into(outputDir)

}



project.registerCargoCommandsTasks(outputDir.asFile, defaultRustDocFlags)

project.registerGenerateCargoConfigTomlTask(outputDir.asFile)
@@ -452,6 +471,48 @@ tasks.register("sdkTest") { 
    finalizedBy(Cargo.CLIPPY.toString, Cargo.TEST.toString, Cargo.DOCS.toString)

}



//Tasks for generating individual Cargo.lock files

fun Project.registerLockfileGeneration(

    dir: File,

    name: String,

): TaskProvider<Exec> {

    return tasks.register<Exec>("generate${name}Lockfile") {

        workingDir(dir)

        environment("RUSTFLAGS", "--cfg aws_sdk_unstable")

        commandLine("cargo", "generate-lockfile")

    }

}



val generateAwsConfigLockfile = registerLockfileGeneration(awsConfigPath, "AwsConfig")

val generateAwsRuntimeLockfile = registerLockfileGeneration(awsRustRuntimePath, "AwsRustRuntime")

val generateSmithytRuntimeLockfile = registerLockfileGeneration(rustRuntimePath, "RustRuntime")



//Generates a lockfile from the aws-sdk-rust repo and copies it into the smithy-rs repo

val generateAwsSdkRustLockfile = tasks.register<Exec>("generateAwsSdkRustLockfile") {

    val sdkRustPath: String =

        properties.get("aws-sdk-rust-path") ?: throw Exception("A -Paws-sdk-rust-path argument must be specified")

    workingDir(sdkRustPath)

    environment("RUSTFLAGS", "--cfg aws_sdk_unstable")

    commandLine("cargo", "generate-lockfile")

    copy {

        from("${sdkRustPath}/Cargo.lock")

        into(rootProject.projectDir.resolve("aws/sdk"))

    }

}



//Parent task to generate all the Cargo.lock files

tasks.register("generateAllLockfiles") {

    description =

        "Create Cargo.lock files for aws-config, aws/rust-runtime, rust-runtime, and the workspace created by" +

            "the assemble task."

    finalizedBy(

        generateAwsSdkRustLockfile,

        generateAwsConfigLockfile,

        generateAwsRuntimeLockfile,

        generateSmithytRuntimeLockfile,

    )

}



tasks.register<Delete>("deleteSdk") {

    delete(

        fileTree(outputDir) {