Unverified Commit d7593493 authored by Landon James's avatar Landon James Committed by GitHub
Browse files

Adding new gradle tasks for generating Cargo.lock (#3689)

## Motivation and Context
<!--- Why is this change required? What problem does it solve? -->
<!--- If it fixes an open issue, please link to the issue here -->
We have had numerous recent issues where updated dependencies have
broken our build/release process. This seeks to fix that in a brute
force way by manually generating Cargo.lock files for our main
workspaces.

## Description
<!--- Describe your changes in detail -->
Introduce a new gradle task `generateAllLockfiles` (and some child tasks
that it invokes) that generates lock files for
* `aws-config` crate
* `aws/rust-runtime` workspace
* `rust-runtime` workspace
* the superset workspace generated by the `assemble` task
* NOTE: This Cargo.lock is actually generated from the workspace in the
`aws-sdk-rust` repo, saved in this repo, and copied into place at build
time. Generating this one requires a local copy of the `aws-sdk-rust`
repo whose absolute path is indicated by the `-Paws-sdk-rust-path`
parameter.

## Testing
<!--- Please describe in detail how you tested your changes -->
<!--- Include details of your testing environment, and the tests you ran
to -->
<!--- see how your change affects other areas of the code, etc. -->

I ran our E2E tests against 3 commits to confirm that the Cargo.lock
files operate as we expect:
* `9d84980ee2e37deadb7e45450d65c5e6bc69e07c` The head commit from this
PR which has our Cargo.lock files
	*  Passes
* `d9d9fcec2de0e898bcc804ee0bb31393d7098e72` with lockfiles and with the
`httparse` and `crc323c` crates unpinned
	*  Passes
* `df75beb160890b2a9c382c8a2f7da8d7d0ddd3c8` with no lockfiles and has
the `httparse` and `crc32c` crates unpinned
	*  Fails (as expected)

This series of tests confirms that that Cargo.lock files will protect us
against bad dependency updates going forward.

## Checklist
<!--- If a checkbox below is not applicable, then please DELETE it
rather than leaving it unchecked -->

----

_By submitting this pull request, I confirm that you can use, modify,
copy, and redistribute this contribution, under the terms of your
choice._
parent 3177a6b1
Loading
Loading
Loading
Loading
+0 −1
Original line number Diff line number Diff line
target/
Cargo.lock
+2557 −0

File added.

Preview size limit exceeded, changes collapsed.

+2056 −0

File added.

Preview size limit exceeded, changes collapsed.

aws/sdk/Cargo.lock

0 → 100644
+13050 −0

File added.

Preview size limit exceeded, changes collapsed.

+65 −4
Original line number Diff line number Diff line
@@ -37,10 +37,15 @@ val properties = PropertyRetriever(rootProject, project)
val crateHasherToolPath = rootProject.projectDir.resolve("tools/ci-build/crate-hasher")
val publisherToolPath = rootProject.projectDir.resolve("tools/ci-build/publisher")
val sdkVersionerToolPath = rootProject.projectDir.resolve("tools/ci-build/sdk-versioner")
val awsConfigPath = rootProject.projectDir.resolve("aws/rust-runtime/aws-config")
val rustRuntimePath = rootProject.projectDir.resolve("rust-runtime")
val awsRustRuntimePath = rootProject.projectDir.resolve("aws/rust-runtime")
val checkedInCargoLock = rootProject.projectDir.resolve("aws/sdk/Cargo.lock")
val outputDir = layout.buildDirectory.dir("aws-sdk").get()
val sdkOutputDir = outputDir.dir("sdk")
val examplesOutputDir = outputDir.dir("examples")


dependencies {
    implementation(project(":aws:sdk-codegen"))
    implementation("software.amazon.smithy:smithy-protocol-test-traits:$smithyVersion")
@@ -87,9 +92,12 @@ fun generateSmithyBuild(services: AwsServices): String {
        }
        val moduleName = "aws-sdk-${service.module}"
        val eventStreamAllowListMembers = eventStreamAllowList.joinToString(", ") { "\"$it\"" }
        val defaultConfigPath = services.defaultConfigPath.let { software.amazon.smithy.utils.StringUtils.escapeJavaString(it, "") }
        val partitionsConfigPath = services.partitionsConfigPath.let { software.amazon.smithy.utils.StringUtils.escapeJavaString(it, "") }
        val integrationTestPath = project.projectDir.resolve("integration-tests").let { software.amazon.smithy.utils.StringUtils.escapeJavaString(it, "") }
        val defaultConfigPath =
            services.defaultConfigPath.let { software.amazon.smithy.utils.StringUtils.escapeJavaString(it, "") }
        val partitionsConfigPath =
            services.partitionsConfigPath.let { software.amazon.smithy.utils.StringUtils.escapeJavaString(it, "") }
        val integrationTestPath = project.projectDir.resolve("integration-tests")
            .let { software.amazon.smithy.utils.StringUtils.escapeJavaString(it, "") }
        """
            "${service.module}": {
                "imports": [${files.joinToString()}],
@@ -293,7 +301,10 @@ tasks.register("relocateAwsRuntime") {
        // Patch the Cargo.toml files
        CrateSet.AWS_SDK_RUNTIME.forEach { module ->
            patchFile(sdkOutputDir.file("${module.name}/Cargo.toml").asFile) { line ->
                rewriteRuntimeCrateVersion(properties.get(module.versionPropertyName)!!, line.let(::rewritePathDependency))
                rewriteRuntimeCrateVersion(
                    properties.get(module.versionPropertyName)!!,
                    line.let(::rewritePathDependency),
                )
            }
        }
    }
@@ -439,9 +450,17 @@ tasks["assemble"].apply {
        "hydrateReadme",
        "relocateChangelog",
    )
    finalizedBy("copyCheckedInCargoLock")
    outputs.upToDateWhen { false }
}

tasks.register<Copy>("copyCheckedInCargoLock") {
    description = "Copy the checked in Cargo.lock file back to the build directory"
    this.outputs.upToDateWhen { false }
    from(checkedInCargoLock)
    into(outputDir)
}

project.registerCargoCommandsTasks(outputDir.asFile, defaultRustDocFlags)
project.registerGenerateCargoConfigTomlTask(outputDir.asFile)

@@ -452,6 +471,48 @@ tasks.register("sdkTest") {
    finalizedBy(Cargo.CLIPPY.toString, Cargo.TEST.toString, Cargo.DOCS.toString)
}

//Tasks for generating individual Cargo.lock files
fun Project.registerLockfileGeneration(
    dir: File,
    name: String,
): TaskProvider<Exec> {
    return tasks.register<Exec>("generate${name}Lockfile") {
        workingDir(dir)
        environment("RUSTFLAGS", "--cfg aws_sdk_unstable")
        commandLine("cargo", "generate-lockfile")
    }
}

val generateAwsConfigLockfile = registerLockfileGeneration(awsConfigPath, "AwsConfig")
val generateAwsRuntimeLockfile = registerLockfileGeneration(awsRustRuntimePath, "AwsRustRuntime")
val generateSmithytRuntimeLockfile = registerLockfileGeneration(rustRuntimePath, "RustRuntime")

//Generates a lockfile from the aws-sdk-rust repo and copies it into the smithy-rs repo
val generateAwsSdkRustLockfile = tasks.register<Exec>("generateAwsSdkRustLockfile") {
    val sdkRustPath: String =
        properties.get("aws-sdk-rust-path") ?: throw Exception("A -Paws-sdk-rust-path argument must be specified")
    workingDir(sdkRustPath)
    environment("RUSTFLAGS", "--cfg aws_sdk_unstable")
    commandLine("cargo", "generate-lockfile")
    copy {
        from("${sdkRustPath}/Cargo.lock")
        into(rootProject.projectDir.resolve("aws/sdk"))
    }
}

//Parent task to generate all the Cargo.lock files
tasks.register("generateAllLockfiles") {
    description =
        "Create Cargo.lock files for aws-config, aws/rust-runtime, rust-runtime, and the workspace created by" +
            "the assemble task."
    finalizedBy(
        generateAwsSdkRustLockfile,
        generateAwsConfigLockfile,
        generateAwsRuntimeLockfile,
        generateSmithytRuntimeLockfile,
    )
}

tasks.register<Delete>("deleteSdk") {
    delete(
        fileTree(outputDir) {
Loading