From 17818ca7e5221ff1a82356a9b6259f852dc8cb16 Mon Sep 17 00:00:00 2001 From: John DiSanti Date: Thu, 16 Dec 2021 13:27:21 -0800 Subject: [PATCH] Produce two publishable bundles in CI (#986) * Move the publisher tool from `aws-sdk-rust` * Add `rust-runtime:assemble` target to generate a publishable bundle * Run `fix-manifests` on assemble output * Produce publish-ready Smithy runtime bundle during CI * Allow publish from any arbitrary directory * Add safe-guard to prevent accidental publish from local dev * Fix unit test target * Incorporate feedback * Add `buildSrc` tests and publisher tool to CI --- .github/workflows/ci-sdk.yaml | 2 +- .github/workflows/ci.yaml | 66 +- aws/sdk/build.gradle.kts | 182 +- aws/sdk/examples/iam/Cargo.toml | 2 +- .../examples/iam/src/bin/iam-hello-world.rs | 2 + aws/sdk/examples/kms/Cargo.toml | 2 +- aws/sdk/examples/kms/src/bin/create-key.rs | 2 +- aws/sdk/examples/kms/src/bin/decrypt.rs | 2 +- aws/sdk/examples/kms/src/bin/encrypt.rs | 2 +- .../generate-data-key-without-plaintext.rs | 4 +- .../examples/kms/src/bin/generate-data-key.rs | 4 +- .../examples/kms/src/bin/generate-random.rs | 2 +- .../examples/kms/src/bin/reencrypt-data.rs | 2 +- aws/sdk/examples/polly/Cargo.toml | 2 +- .../examples/polly/src/bin/describe-voices.rs | 2 +- .../examples/polly/src/bin/list-lexicons.rs | 2 +- .../polly/src/bin/polly-helloworld.rs | 4 +- aws/sdk/examples/polly/src/bin/put-lexicon.rs | 2 +- .../src/bin/synthesize-speech-presigned.rs | 8 +- .../polly/src/bin/synthesize-speech.rs | 4 +- buildSrc/build.gradle.kts | 22 + buildSrc/src/main/kotlin/CrateSet.kt | 36 + buildSrc/src/main/kotlin/ManifestPatcher.kt | 31 + buildSrc/src/main/kotlin/Properties.kt | 28 + .../kotlin/{ => aws/sdk}/DocsLandingPage.kt | 6 +- .../kotlin/{ => aws/sdk}/ServiceLoader.kt | 52 +- .../src/test/kotlin/ManifestPatcherTest.kt | 18 + rust-runtime/build.gradle.kts | 38 + tools/publisher/Cargo.lock | 1474 +++++++++++++++++ tools/publisher/Cargo.toml | 28 + tools/publisher/README.md | 1 + tools/publisher/fake_cargo/cargo_fails | 4 + tools/publisher/fake_cargo/cargo_owner_list | 4 + .../cargo_publish_already_published | 3 + .../publisher/fake_cargo/cargo_search_success | 4 + tools/publisher/fake_cargo/cargo_success | 2 + .../publisher/fake_cargo/cargo_yank_not_found | 10 + tools/publisher/src/cargo.rs | 74 + tools/publisher/src/cargo/add_owner.rs | 84 + tools/publisher/src/cargo/get_owners.rs | 99 ++ tools/publisher/src/cargo/publish.rs | 127 ++ tools/publisher/src/cargo/yank.rs | 125 ++ tools/publisher/src/fs.rs | 52 + tools/publisher/src/main.rs | 113 ++ tools/publisher/src/package.rs | 584 +++++++ tools/publisher/src/repo.rs | 51 + tools/publisher/src/sort.rs | 125 ++ .../publisher/src/subcommand/fix_manifests.rs | 260 +++ tools/publisher/src/subcommand/mod.rs | 8 + tools/publisher/src/subcommand/publish.rs | 183 ++ .../publisher/src/subcommand/yank_category.rs | 104 ++ 51 files changed, 3887 insertions(+), 161 deletions(-) create mode 100644 buildSrc/src/main/kotlin/CrateSet.kt create mode 100644 buildSrc/src/main/kotlin/ManifestPatcher.kt create mode 100644 buildSrc/src/main/kotlin/Properties.kt rename buildSrc/src/main/kotlin/{ => aws/sdk}/DocsLandingPage.kt (92%) rename buildSrc/src/main/kotlin/{ => aws/sdk}/ServiceLoader.kt (73%) create mode 100644 buildSrc/src/test/kotlin/ManifestPatcherTest.kt create mode 100644 tools/publisher/Cargo.lock create mode 100644 tools/publisher/Cargo.toml create mode 100644 tools/publisher/README.md create mode 100755 tools/publisher/fake_cargo/cargo_fails create mode 100755 tools/publisher/fake_cargo/cargo_owner_list create mode 100755 tools/publisher/fake_cargo/cargo_publish_already_published create mode 100755 tools/publisher/fake_cargo/cargo_search_success create mode 100755 tools/publisher/fake_cargo/cargo_success create mode 100755 tools/publisher/fake_cargo/cargo_yank_not_found create mode 100644 tools/publisher/src/cargo.rs create mode 100644 tools/publisher/src/cargo/add_owner.rs create mode 100644 tools/publisher/src/cargo/get_owners.rs create mode 100644 tools/publisher/src/cargo/publish.rs create mode 100644 tools/publisher/src/cargo/yank.rs create mode 100644 tools/publisher/src/fs.rs create mode 100644 tools/publisher/src/main.rs create mode 100644 tools/publisher/src/package.rs create mode 100644 tools/publisher/src/repo.rs create mode 100644 tools/publisher/src/sort.rs create mode 100644 tools/publisher/src/subcommand/fix_manifests.rs create mode 100644 tools/publisher/src/subcommand/mod.rs create mode 100644 tools/publisher/src/subcommand/publish.rs create mode 100644 tools/publisher/src/subcommand/yank_category.rs diff --git a/.github/workflows/ci-sdk.yaml b/.github/workflows/ci-sdk.yaml index 4a806778b..9fc628077 100644 --- a/.github/workflows/ci-sdk.yaml +++ b/.github/workflows/ci-sdk.yaml @@ -62,7 +62,7 @@ jobs: matrix: test: - name: Unit Tests - run: cargo test $(cat service-with-tests) + run: cargo test --all-features - name: Docs run: cargo doc --no-deps --document-private-items - name: Clippy diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 561d6dfea..7d0b1b08a 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -41,6 +41,8 @@ jobs: test: - name: Kotlin Style run: ./gradlew ktlint + - name: BuildSrc Tests + run: ./gradlew -p buildSrc test - name: Client Unit Tests run: ./gradlew :codegen:test - name: SDK Unit Tests @@ -80,13 +82,22 @@ jobs: - name: ${{ matrix.test.name }} run: ${{ matrix.test.run }} - runtime-tests: - name: Rust Runtime Tests + rust-tests: + name: Rust Tests + runs-on: ${{ matrix.os }} strategy: matrix: os: [ubuntu-latest, windows-latest] - runtime: [., ./aws] - runs-on: ${{ matrix.os }} + runtime: [rust-runtime, aws/rust-runtime, tools/publisher] + exclude: + # Don't need to test the publisher tool on Windows + - os: windows-latest + runtime: tools/publisher + env: + # Disable incremental compilation to reduce disk space use + CARGO_INCREMENTAL: 0 + RUSTDOCFLAGS: -D warnings + RUSTFLAGS: -D warnings steps: - uses: actions/checkout@v2 # Pinned to the commit hash of v1.3.0 @@ -105,14 +116,51 @@ jobs: if: ${{ matrix.os == 'ubuntu-latest' }} - name: clippy check run: cargo clippy -- -D warnings - working-directory: ${{ matrix.runtime }}/rust-runtime/ + working-directory: ${{ matrix.runtime }} # don't bother running Clippy twice, it will have the same results on Windows if: ${{ matrix.os == 'ubuntu-latest' }} - name: run tests run: cargo test --all-features - working-directory: ${{ matrix.runtime }}/rust-runtime/ + working-directory: ${{ matrix.runtime }} - name: generate docs run: cargo doc --no-deps --document-private-items --all-features - working-directory: ${{ matrix.runtime }}/rust-runtime/ - env: - RUSTDOCFLAGS: -D warnings + working-directory: ${{ matrix.runtime }} + + # Psuedo-job that depends on the rust-tests job so that we don't have to enter + # the myriad of test matrix combinations into GitHub's protected branch rules + require-rust-tests: + needs: rust-tests + runs-on: ubuntu-latest + name: Rust Tests Matrix Success + steps: + - name: Run + run: echo "We should only get this far if the rust-tests matrix succeeded." + + runtime-bundle: + name: Produce smithy-rs runtime bundle + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - uses: actions/cache@v2 + name: Gradle Cache + with: + path: | + ~/.gradle/caches + ~/.gradle/wrapper + key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle*') }} + restore-keys: | + ${{ runner.os }}-gradle- + # Pinned to the commit hash of v1.3.0 + - name: Set up JDK + uses: actions/setup-java@v1 + with: + java-version: ${{ env.java_version }} + - name: Produce bundle + run: | + ./gradlew rust-runtime:assemble + tar cfvz smithy-rs-runtime.tar.gz -C rust-runtime/build smithy-rs + - uses: actions/upload-artifact@v2 + name: Upload bundle + with: + name: smithy-rs-runtime-${{ github.sha }} + path: smithy-rs-runtime.tar.gz diff --git a/aws/sdk/build.gradle.kts b/aws/sdk/build.gradle.kts index 400bec008..13f7f44e7 100644 --- a/aws/sdk/build.gradle.kts +++ b/aws/sdk/build.gradle.kts @@ -3,7 +3,11 @@ * SPDX-License-Identifier: Apache-2.0. */ -import java.util.Properties +import aws.sdk.AwsServices +import aws.sdk.Membership +import aws.sdk.discoverServices +import aws.sdk.docsLandingPage +import aws.sdk.parseMembership extra["displayName"] = "Smithy :: Rust :: AWS-SDK" extra["moduleName"] = "software.amazon.smithy.rust.awssdk" @@ -15,34 +19,12 @@ plugins { } val smithyVersion: String by project +val properties = PropertyRetriever(rootProject, project) val outputDir = buildDir.resolve("aws-sdk") val sdkOutputDir = outputDir.resolve("sdk") val examplesOutputDir = outputDir.resolve("examples") -val runtimeModules = listOf( - "aws-smithy-async", - "aws-smithy-client", - "aws-smithy-eventstream", - "aws-smithy-http", - "aws-smithy-http-tower", - "aws-smithy-json", - "aws-smithy-protocol-test", - "aws-smithy-query", - "aws-smithy-types", - "aws-smithy-types-convert", - "aws-smithy-xml" -) -val awsModules = listOf( - "aws-config", - "aws-endpoint", - "aws-http", - "aws-hyper", - "aws-sig-auth", - "aws-sigv4", - "aws-types" -) - buildscript { val smithyVersion: String by project dependencies { @@ -59,37 +41,19 @@ dependencies { implementation("software.amazon.smithy:smithy-aws-cloudformation-traits:$smithyVersion") } -// get a project property by name if it exists (including from local.properties) -fun getProperty(name: String): String? { - if (project.hasProperty(name)) { - return project.properties[name].toString() - } - - val localProperties = Properties() - val propertiesFile: File = rootProject.file("local.properties") - if (propertiesFile.exists()) { - propertiesFile.inputStream().use { localProperties.load(it) } - - if (localProperties.containsKey(name)) { - return localProperties[name].toString() - } - } - return null -} - // Class and functions for service and protocol membership for SDK generation -val awsServices: List by lazy { discoverServices(loadServiceMembership()) } +val awsServices: AwsServices by lazy { discoverServices(loadServiceMembership()) } val eventStreamAllowList: Set by lazy { eventStreamAllowList() } fun loadServiceMembership(): Membership { - val membershipOverride = getProperty("aws.services")?.let { parseMembership(it) } + val membershipOverride = properties.get("aws.services")?.let { parseMembership(it) } println(membershipOverride) val fullSdk = - parseMembership(getProperty("aws.services.fullsdk") ?: throw kotlin.Exception("full sdk list missing")) + parseMembership(properties.get("aws.services.fullsdk") ?: throw kotlin.Exception("full sdk list missing")) val tier1 = - parseMembership(getProperty("aws.services.smoketest") ?: throw kotlin.Exception("smoketest list missing")) - return membershipOverride ?: if ((getProperty("aws.fullsdk") ?: "") == "true") { + parseMembership(properties.get("aws.services.smoketest") ?: throw kotlin.Exception("smoketest list missing")) + return membershipOverride ?: if ((properties.get("aws.fullsdk") ?: "") == "true") { fullSdk } else { tier1 @@ -97,12 +61,12 @@ fun loadServiceMembership(): Membership { } fun eventStreamAllowList(): Set { - val list = getProperty("aws.services.eventstream.allowlist") ?: "" + val list = properties.get("aws.services.eventstream.allowlist") ?: "" return list.split(",").map { it.trim() }.toSet() } -fun generateSmithyBuild(services: List): String { - val serviceProjections = services.map { service -> +fun generateSmithyBuild(services: AwsServices): String { + val serviceProjections = services.services.map { service -> val files = service.files().map { extraFile -> software.amazon.smithy.utils.StringUtils.escapeJavaString( extraFile.absolutePath, @@ -127,7 +91,7 @@ fun generateSmithyBuild(services: List): String { }, "service": "${service.service}", "module": "aws-sdk-${service.module}", - "moduleVersion": "${getProperty("aws.sdk.version")}", + "moduleVersion": "${properties.get("aws.sdk.version")}", "moduleAuthors": ["AWS Rust SDK Team ", "Russell Cohen "], "moduleDescription": "${service.moduleDescription}", ${service.examplesUri(project)?.let { """"examples": "$it",""" } ?: ""} @@ -150,7 +114,7 @@ fun generateSmithyBuild(services: List): String { task("generateSmithyBuild") { description = "generate smithy-build.json" - inputs.property("servicelist", awsServices.sortedBy { it.module }.toString()) + inputs.property("servicelist", awsServices.services.toString()) inputs.property("eventStreamAllowList", eventStreamAllowList) inputs.dir(projectDir.resolve("aws-models")) outputs.file(projectDir.resolve("smithy-build.json")) @@ -161,7 +125,7 @@ task("generateSmithyBuild") { } task("generateDocs") { - inputs.property("servicelist", awsServices.sortedBy { it.module }.toString()) + inputs.property("servicelist", awsServices.services.toString()) outputs.file(outputDir.resolve("docs.md")) doLast { project.docsLandingPage(awsServices, outputDir) @@ -171,7 +135,7 @@ task("generateDocs") { task("relocateServices") { description = "relocate AWS services to their final destination" doLast { - awsServices.forEach { + awsServices.services.forEach { logger.info("Relocating ${it.module}...") copy { from("$buildDir/smithyprojections/sdk/${it.module}/rust-codegen") @@ -198,7 +162,9 @@ task("relocateExamples") { doLast { copy { from(projectDir) - include("examples/**") + awsServices.examples.forEach { example -> + include("$example/**") + } into(outputDir) exclude("**/target") filter { line -> line.replace("build/aws-sdk/sdk/", "sdk/") } @@ -219,36 +185,12 @@ fun rewritePathDependency(line: String): String { .replace("../../rust-runtime/", "") } -fun rewriteCrateVersion(line: String, version: String): String = line.replace( - """^\s*version\s+=\s+"0.0.0-smithy-rs-head"$""".toRegex(), - "version = \"$version\"" -) - -/** - * AWS runtime crate versions are all `0.0.0-smithy-rs-head`. When copying over to the AWS SDK, - * these should be changed to the AWS SDK version. - */ -fun rewriteAwsSdkCrateVersion(line: String): String = rewriteCrateVersion(line, getProperty("aws.sdk.version")!!) - -/** - * Smithy runtime crate versions in smithy-rs are all `0.0.0-smithy-rs-head`. When copying over to the AWS SDK, - * these should be changed to the smithy-rs version. - */ -fun rewriteSmithyRsCrateVersion(line: String): String = - rewriteCrateVersion(line, getProperty("smithy.rs.runtime.crate.version")!!) - -/** Patches a file with the result of the given `operation` being run on each line */ -fun patchFile(path: File, operation: (String) -> String) { - val patchedContents = path.readLines().joinToString("\n", transform = operation) - path.writeText(patchedContents) -} - tasks.register("copyAllRuntimes") { from("$rootDir/aws/rust-runtime") { - awsModules.forEach { include("$it/**") } + CrateSet.AWS_SDK_RUNTIME.forEach { include("$it/**") } } from("$rootDir/rust-runtime") { - runtimeModules.forEach { include("$it/**") } + CrateSet.AWS_SDK_SMITHY_RUNTIME.forEach { include("$it/**") } } exclude("**/target") exclude("**/Cargo.lock") @@ -260,10 +202,9 @@ tasks.register("relocateAwsRuntime") { dependsOn("copyAllRuntimes") doLast { // Patch the Cargo.toml files - awsModules.forEach { moduleName -> + CrateSet.AWS_SDK_RUNTIME.forEach { moduleName -> patchFile(sdkOutputDir.resolve("$moduleName/Cargo.toml")) { line -> - line.let(::rewritePathDependency) - .let(::rewriteAwsSdkCrateVersion) + rewriteAwsSdkCrateVersion(properties, line.let(::rewritePathDependency)) } } } @@ -272,57 +213,51 @@ tasks.register("relocateRuntime") { dependsOn("copyAllRuntimes") doLast { // Patch the Cargo.toml files - runtimeModules.forEach { moduleName -> + CrateSet.AWS_SDK_SMITHY_RUNTIME.forEach { moduleName -> patchFile(sdkOutputDir.resolve("$moduleName/Cargo.toml")) { line -> - line.let(::rewriteSmithyRsCrateVersion) + rewriteSmithyRsCrateVersion(properties, line) } } } } -fun generateCargoWorkspace(services: List): String { - val generatedModules = services.map { it.module }.toSet() - val examples = projectDir.resolve("examples") - .listFiles { file -> !file.name.startsWith(".") }.orEmpty().toList() - .filter { file -> - val cargoToml = File(file, "Cargo.toml") - if (cargoToml.exists()) { - val usedModules = cargoToml.readLines() - .map { line -> line.substringBefore('=').trim() } - .filter { line -> line.startsWith("aws-sdk-") } - .map { line -> line.substringAfter("aws-sdk-") } - .toSet() - generatedModules.containsAll(usedModules) - } else { - false - } - } - .map { "examples/${it.name}" } - - val modules = ( - services.map(AwsService::module).map { "sdk/$it" } + - runtimeModules.map { "sdk/$it" } + - awsModules.map { "sdk/$it" } + - examples.toList() - ).sorted() +fun generateCargoWorkspace(services: AwsServices): String { return """ |[workspace] - |members = [${"\n"}${modules.joinToString(",\n") { "| \"$it\"" }} + |members = [${"\n"}${services.allModules.joinToString(",\n") { "| \"$it\"" }} |] """.trimMargin() } + task("generateCargoWorkspace") { description = "generate Cargo.toml workspace file" doFirst { outputDir.mkdirs() outputDir.resolve("Cargo.toml").writeText(generateCargoWorkspace(awsServices)) } - inputs.property("servicelist", awsServices.sortedBy { it.module }.toString()) + inputs.property("servicelist", awsServices.moduleNames.toString()) inputs.dir(projectDir.resolve("examples")) outputs.file(outputDir.resolve("Cargo.toml")) outputs.upToDateWhen { false } } +tasks.register("fixManifests") { + description = "Run the publisher tool's `fix-manifests` sub-command on the generated services" + + val publisherPath = rootProject.projectDir.resolve("tools/publisher") + inputs.dir(publisherPath) + outputs.dir(outputDir) + + workingDir(publisherPath) + commandLine("cargo", "run", "--", "fix-manifests", "--location", outputDir.absolutePath) + + dependsOn("assemble") + dependsOn("relocateServices") + dependsOn("relocateRuntime") + dependsOn("relocateAwsRuntime") + dependsOn("relocateExamples") +} + task("finalizeSdk") { dependsOn("assemble") outputs.upToDateWhen { false } @@ -331,17 +266,22 @@ task("finalizeSdk") { "relocateRuntime", "relocateAwsRuntime", "relocateExamples", - "generateDocs" + "generateDocs", + "fixManifests" ) } -tasks["smithyBuildJar"].inputs.file(projectDir.resolve("smithy-build.json")) -tasks["smithyBuildJar"].inputs.dir(projectDir.resolve("aws-models")) -tasks["smithyBuildJar"].dependsOn("generateSmithyBuild") -tasks["smithyBuildJar"].dependsOn("generateCargoWorkspace") -tasks["smithyBuildJar"].outputs.upToDateWhen { false } -tasks["assemble"].dependsOn("smithyBuildJar") -tasks["assemble"].finalizedBy("finalizeSdk") +tasks["smithyBuildJar"].apply { + inputs.file(projectDir.resolve("smithy-build.json")) + inputs.dir(projectDir.resolve("aws-models")) + dependsOn("generateSmithyBuild") + dependsOn("generateCargoWorkspace") + outputs.upToDateWhen { false } +} +tasks["assemble"].apply { + dependsOn("smithyBuildJar") + finalizedBy("finalizeSdk") +} tasks.register("cargoCheck") { workingDir(outputDir) diff --git a/aws/sdk/examples/iam/Cargo.toml b/aws/sdk/examples/iam/Cargo.toml index 0d176b2e5..eafd69b9a 100644 --- a/aws/sdk/examples/iam/Cargo.toml +++ b/aws/sdk/examples/iam/Cargo.toml @@ -8,6 +8,6 @@ edition = "2018" [dependencies] aws-config = { path = "../../build/aws-sdk/sdk/aws-config" } -iam = { package = "aws-sdk-iam", path = "../../build/aws-sdk/sdk/iam" } +aws-sdk-iam = { path = "../../build/aws-sdk/sdk/iam" } tokio = { version = "1", features = ["full"] } tracing-subscriber = "0.2.18" diff --git a/aws/sdk/examples/iam/src/bin/iam-hello-world.rs b/aws/sdk/examples/iam/src/bin/iam-hello-world.rs index 66b034615..2e9a23bac 100644 --- a/aws/sdk/examples/iam/src/bin/iam-hello-world.rs +++ b/aws/sdk/examples/iam/src/bin/iam-hello-world.rs @@ -3,6 +3,8 @@ * SPDX-License-Identifier: Apache-2.0. */ +use aws_sdk_iam as iam; + #[tokio::main] async fn main() -> Result<(), iam::Error> { tracing_subscriber::fmt::init(); diff --git a/aws/sdk/examples/kms/Cargo.toml b/aws/sdk/examples/kms/Cargo.toml index 6a38648c9..74b4c4078 100644 --- a/aws/sdk/examples/kms/Cargo.toml +++ b/aws/sdk/examples/kms/Cargo.toml @@ -7,7 +7,7 @@ description = "Example usage of the KMS service" [dependencies] aws-config = { path = "../../build/aws-sdk/sdk/aws-config" } -kms = { package = "aws-sdk-kms", path = "../../build/aws-sdk/sdk/kms" } +aws-sdk-kms = { path = "../../build/aws-sdk/sdk/kms" } aws-types = { path = "../../build/aws-sdk/sdk/aws-types" } tokio = { version = "1", features = ["full"]} structopt = { version = "0.3", default-features = false } diff --git a/aws/sdk/examples/kms/src/bin/create-key.rs b/aws/sdk/examples/kms/src/bin/create-key.rs index 60c2fb740..06d9f86f6 100644 --- a/aws/sdk/examples/kms/src/bin/create-key.rs +++ b/aws/sdk/examples/kms/src/bin/create-key.rs @@ -4,7 +4,7 @@ */ use aws_config::meta::region::RegionProviderChain; -use kms::{Client, Error, Region, PKG_VERSION}; +use aws_sdk_kms::{Client, Error, Region, PKG_VERSION}; use structopt::StructOpt; #[derive(Debug, StructOpt)] diff --git a/aws/sdk/examples/kms/src/bin/decrypt.rs b/aws/sdk/examples/kms/src/bin/decrypt.rs index 8e4c80c0c..f7b0547a5 100644 --- a/aws/sdk/examples/kms/src/bin/decrypt.rs +++ b/aws/sdk/examples/kms/src/bin/decrypt.rs @@ -4,7 +4,7 @@ */ use aws_config::meta::region::RegionProviderChain; -use kms::{Blob, Client, Error, Region, PKG_VERSION}; +use aws_sdk_kms::{Blob, Client, Error, Region, PKG_VERSION}; use std::fs; use structopt::StructOpt; diff --git a/aws/sdk/examples/kms/src/bin/encrypt.rs b/aws/sdk/examples/kms/src/bin/encrypt.rs index 69cb065ae..25bfcfc43 100644 --- a/aws/sdk/examples/kms/src/bin/encrypt.rs +++ b/aws/sdk/examples/kms/src/bin/encrypt.rs @@ -4,7 +4,7 @@ */ use aws_config::meta::region::RegionProviderChain; -use kms::{Blob, Client, Error, Region, PKG_VERSION}; +use aws_sdk_kms::{Blob, Client, Error, Region, PKG_VERSION}; use std::fs::File; use std::io::Write; use structopt::StructOpt; diff --git a/aws/sdk/examples/kms/src/bin/generate-data-key-without-plaintext.rs b/aws/sdk/examples/kms/src/bin/generate-data-key-without-plaintext.rs index 9f3f5f421..95a6974de 100644 --- a/aws/sdk/examples/kms/src/bin/generate-data-key-without-plaintext.rs +++ b/aws/sdk/examples/kms/src/bin/generate-data-key-without-plaintext.rs @@ -3,8 +3,8 @@ * SPDX-License-Identifier: Apache-2.0. */ use aws_config::meta::region::RegionProviderChain; -use kms::model::DataKeySpec; -use kms::{Client, Error, Region, PKG_VERSION}; +use aws_sdk_kms::model::DataKeySpec; +use aws_sdk_kms::{Client, Error, Region, PKG_VERSION}; use structopt::StructOpt; #[derive(Debug, StructOpt)] diff --git a/aws/sdk/examples/kms/src/bin/generate-data-key.rs b/aws/sdk/examples/kms/src/bin/generate-data-key.rs index 4817b40cd..8fd38896f 100644 --- a/aws/sdk/examples/kms/src/bin/generate-data-key.rs +++ b/aws/sdk/examples/kms/src/bin/generate-data-key.rs @@ -4,8 +4,8 @@ */ use aws_config::meta::region::RegionProviderChain; -use kms::model::DataKeySpec; -use kms::{Client, Error, Region, PKG_VERSION}; +use aws_sdk_kms::model::DataKeySpec; +use aws_sdk_kms::{Client, Error, Region, PKG_VERSION}; use structopt::StructOpt; #[derive(Debug, StructOpt)] diff --git a/aws/sdk/examples/kms/src/bin/generate-random.rs b/aws/sdk/examples/kms/src/bin/generate-random.rs index 77374b1d8..c4f64a3cc 100644 --- a/aws/sdk/examples/kms/src/bin/generate-random.rs +++ b/aws/sdk/examples/kms/src/bin/generate-random.rs @@ -4,7 +4,7 @@ */ use aws_config::meta::region::RegionProviderChain; -use kms::{Client, Error, Region, PKG_VERSION}; +use aws_sdk_kms::{Client, Error, Region, PKG_VERSION}; use std::process; use structopt::StructOpt; diff --git a/aws/sdk/examples/kms/src/bin/reencrypt-data.rs b/aws/sdk/examples/kms/src/bin/reencrypt-data.rs index 2c27ecc50..3a7eaee86 100644 --- a/aws/sdk/examples/kms/src/bin/reencrypt-data.rs +++ b/aws/sdk/examples/kms/src/bin/reencrypt-data.rs @@ -4,7 +4,7 @@ */ use aws_config::meta::region::RegionProviderChain; -use kms::{Blob, Client, Error, Region, PKG_VERSION}; +use aws_sdk_kms::{Blob, Client, Error, Region, PKG_VERSION}; use std::fs; use std::fs::File; use std::io::Write; diff --git a/aws/sdk/examples/polly/Cargo.toml b/aws/sdk/examples/polly/Cargo.toml index f5187365d..32aae8fb2 100644 --- a/aws/sdk/examples/polly/Cargo.toml +++ b/aws/sdk/examples/polly/Cargo.toml @@ -8,7 +8,7 @@ edition = "2018" [dependencies] aws-config = { path = "../../build/aws-sdk/sdk/aws-config" } -polly = { package = "aws-sdk-polly", path = "../../build/aws-sdk/sdk/polly" } +aws-sdk-polly = { path = "../../build/aws-sdk/sdk/polly" } aws-types = { path = "../../build/aws-sdk/sdk/aws-types" } tokio = { version = "1", features = ["full"] } structopt = { version = "0.3", default-features = false } diff --git a/aws/sdk/examples/polly/src/bin/describe-voices.rs b/aws/sdk/examples/polly/src/bin/describe-voices.rs index d3b40f000..10f038e78 100644 --- a/aws/sdk/examples/polly/src/bin/describe-voices.rs +++ b/aws/sdk/examples/polly/src/bin/describe-voices.rs @@ -4,7 +4,7 @@ */ use aws_config::meta::region::RegionProviderChain; -use polly::{Client, Error, Region, PKG_VERSION}; +use aws_sdk_polly::{Client, Error, Region, PKG_VERSION}; use structopt::StructOpt; #[derive(Debug, StructOpt)] diff --git a/aws/sdk/examples/polly/src/bin/list-lexicons.rs b/aws/sdk/examples/polly/src/bin/list-lexicons.rs index 895e80e5b..d86599359 100644 --- a/aws/sdk/examples/polly/src/bin/list-lexicons.rs +++ b/aws/sdk/examples/polly/src/bin/list-lexicons.rs @@ -4,7 +4,7 @@ */ use aws_config::meta::region::RegionProviderChain; -use polly::{Client, Error, Region, PKG_VERSION}; +use aws_sdk_polly::{Client, Error, Region, PKG_VERSION}; use structopt::StructOpt; #[derive(Debug, StructOpt)] diff --git a/aws/sdk/examples/polly/src/bin/polly-helloworld.rs b/aws/sdk/examples/polly/src/bin/polly-helloworld.rs index 4e2f3ef3f..3a018cb0e 100644 --- a/aws/sdk/examples/polly/src/bin/polly-helloworld.rs +++ b/aws/sdk/examples/polly/src/bin/polly-helloworld.rs @@ -4,8 +4,8 @@ */ use aws_config::meta::region::RegionProviderChain; -use polly::model::{Engine, Voice}; -use polly::{Client, Error, Region, PKG_VERSION}; +use aws_sdk_polly::model::{Engine, Voice}; +use aws_sdk_polly::{Client, Error, Region, PKG_VERSION}; use structopt::StructOpt; #[derive(Debug, StructOpt)] diff --git a/aws/sdk/examples/polly/src/bin/put-lexicon.rs b/aws/sdk/examples/polly/src/bin/put-lexicon.rs index 4f5bd90cf..531dc4947 100644 --- a/aws/sdk/examples/polly/src/bin/put-lexicon.rs +++ b/aws/sdk/examples/polly/src/bin/put-lexicon.rs @@ -4,7 +4,7 @@ */ use aws_config::meta::region::RegionProviderChain; -use polly::{Client, Error, Region, PKG_VERSION}; +use aws_sdk_polly::{Client, Error, Region, PKG_VERSION}; use structopt::StructOpt; diff --git a/aws/sdk/examples/polly/src/bin/synthesize-speech-presigned.rs b/aws/sdk/examples/polly/src/bin/synthesize-speech-presigned.rs index 11149511c..6aab27ca6 100644 --- a/aws/sdk/examples/polly/src/bin/synthesize-speech-presigned.rs +++ b/aws/sdk/examples/polly/src/bin/synthesize-speech-presigned.rs @@ -3,10 +3,10 @@ * SPDX-License-Identifier: Apache-2.0. */ use aws_config::meta::region::RegionProviderChain; -use polly::input::SynthesizeSpeechInput; -use polly::model::{OutputFormat, VoiceId}; -use polly::presigning::config::PresigningConfig; -use polly::{Client, Config, Region, PKG_VERSION}; +use aws_sdk_polly::input::SynthesizeSpeechInput; +use aws_sdk_polly::model::{OutputFormat, VoiceId}; +use aws_sdk_polly::presigning::config::PresigningConfig; +use aws_sdk_polly::{Client, Config, Region, PKG_VERSION}; use std::error::Error; use std::fs; use std::time::Duration; diff --git a/aws/sdk/examples/polly/src/bin/synthesize-speech.rs b/aws/sdk/examples/polly/src/bin/synthesize-speech.rs index d82dec5dc..55d999f19 100644 --- a/aws/sdk/examples/polly/src/bin/synthesize-speech.rs +++ b/aws/sdk/examples/polly/src/bin/synthesize-speech.rs @@ -3,8 +3,8 @@ * SPDX-License-Identifier: Apache-2.0. */ use aws_config::meta::region::RegionProviderChain; -use polly::model::{OutputFormat, VoiceId}; -use polly::{Client, Error, Region, PKG_VERSION}; +use aws_sdk_polly::model::{OutputFormat, VoiceId}; +use aws_sdk_polly::{Client, Error, Region, PKG_VERSION}; use std::fs; use structopt::StructOpt; use tokio::io::AsyncWriteExt; diff --git a/buildSrc/build.gradle.kts b/buildSrc/build.gradle.kts index 96519ba51..f0cc5b048 100644 --- a/buildSrc/build.gradle.kts +++ b/buildSrc/build.gradle.kts @@ -7,6 +7,7 @@ import java.util.Properties plugins { `kotlin-dsl` + jacoco } repositories { maven("https://plugins.gradle.org/m2") @@ -33,4 +34,25 @@ dependencies { implementation("software.amazon.smithy:smithy-aws-iam-traits:$smithyVersion") implementation("software.amazon.smithy:smithy-aws-cloudformation-traits:$smithyVersion") implementation(gradleApi()) + testImplementation("org.junit.jupiter:junit-jupiter:5.6.1") } + +tasks.test { + useJUnitPlatform() + testLogging { + events("passed", "skipped", "failed") + showStandardStreams = true + } +} + +// Configure jacoco (code coverage) to generate an HTML report +tasks.jacocoTestReport { + reports { + xml.isEnabled = false + csv.isEnabled = false + html.destination = file("$buildDir/reports/jacoco") + } +} + +// Always run the jacoco test report after testing. +tasks["test"].finalizedBy(tasks["jacocoTestReport"]) diff --git a/buildSrc/src/main/kotlin/CrateSet.kt b/buildSrc/src/main/kotlin/CrateSet.kt new file mode 100644 index 000000000..3ad3b5710 --- /dev/null +++ b/buildSrc/src/main/kotlin/CrateSet.kt @@ -0,0 +1,36 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0. + */ + +object CrateSet { + val AWS_SDK_RUNTIME = listOf( + "aws-config", + "aws-endpoint", + "aws-http", + "aws-hyper", + "aws-sig-auth", + "aws-sigv4", + "aws-types" + ) + + private val SMITHY_RUNTIME_COMMON = listOf( + "aws-smithy-async", + "aws-smithy-client", + "aws-smithy-eventstream", + "aws-smithy-http", + "aws-smithy-http-tower", + "aws-smithy-json", + "aws-smithy-protocol-test", + "aws-smithy-query", + "aws-smithy-types", + "aws-smithy-types-convert", + "aws-smithy-xml" + ) + + val AWS_SDK_SMITHY_RUNTIME = SMITHY_RUNTIME_COMMON + + val SERVER_SMITHY_RUNTIME = SMITHY_RUNTIME_COMMON + listOf("aws-smithy-http-server") + + val ENTIRE_SMITHY_RUNTIME = (AWS_SDK_SMITHY_RUNTIME + SERVER_SMITHY_RUNTIME).toSortedSet() +} diff --git a/buildSrc/src/main/kotlin/ManifestPatcher.kt b/buildSrc/src/main/kotlin/ManifestPatcher.kt new file mode 100644 index 000000000..f387d201c --- /dev/null +++ b/buildSrc/src/main/kotlin/ManifestPatcher.kt @@ -0,0 +1,31 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0. + */ + +import java.io.File + +fun rewriteCrateVersion(line: String, version: String): String = line.replace( + """^\s*version\s*=\s*"0.0.0-smithy-rs-head"$""".toRegex(), + "version = \"$version\"" +) + +/** + * AWS runtime crate versions are all `0.0.0-smithy-rs-head`. When copying over to the AWS SDK, + * these should be changed to the AWS SDK version. + */ +fun rewriteAwsSdkCrateVersion(properties: PropertyRetriever, line: String): String = + rewriteCrateVersion(line, properties.get("aws.sdk.version")!!) + +/** + * Smithy runtime crate versions in smithy-rs are all `0.0.0-smithy-rs-head`. When copying over to the AWS SDK, + * these should be changed to the smithy-rs version. + */ +fun rewriteSmithyRsCrateVersion(properties: PropertyRetriever, line: String): String = + rewriteCrateVersion(line, properties.get("smithy.rs.runtime.crate.version")!!) + +/** Patches a file with the result of the given `operation` being run on each line */ +fun patchFile(path: File, operation: (String) -> String) { + val patchedContents = path.readLines().joinToString("\n", transform = operation) + path.writeText(patchedContents) +} diff --git a/buildSrc/src/main/kotlin/Properties.kt b/buildSrc/src/main/kotlin/Properties.kt new file mode 100644 index 000000000..b73c1431b --- /dev/null +++ b/buildSrc/src/main/kotlin/Properties.kt @@ -0,0 +1,28 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0. + */ + +import org.gradle.api.Project +import java.io.File +import java.util.Properties + +class PropertyRetriever(private val rootProject: Project, private val project: Project) { + /** Get a project property by name if it exists (including from local.properties) */ + fun get(name: String): String? { + if (project.hasProperty(name)) { + return project.properties[name].toString() + } + + val localProperties = Properties() + val propertiesFile: File = rootProject.file("local.properties") + if (propertiesFile.exists()) { + propertiesFile.inputStream().use { localProperties.load(it) } + + if (localProperties.containsKey(name)) { + return localProperties[name].toString() + } + } + return null + } +} diff --git a/buildSrc/src/main/kotlin/DocsLandingPage.kt b/buildSrc/src/main/kotlin/aws/sdk/DocsLandingPage.kt similarity index 92% rename from buildSrc/src/main/kotlin/DocsLandingPage.kt rename to buildSrc/src/main/kotlin/aws/sdk/DocsLandingPage.kt index 841bb950b..888755d9f 100644 --- a/buildSrc/src/main/kotlin/DocsLandingPage.kt +++ b/buildSrc/src/main/kotlin/aws/sdk/DocsLandingPage.kt @@ -3,6 +3,8 @@ * SPDX-License-Identifier: Apache-2.0. */ +package aws.sdk + import org.gradle.api.Project import software.amazon.smithy.utils.CodeWriter import java.io.File @@ -13,7 +15,7 @@ import java.io.File * The generated docs will include links to crates.io, docs.rs and GitHub examples for all generated services. The generated docs will * be written to `docs.md` in the provided [outputDir]. */ -fun Project.docsLandingPage(awsServices: List, outputDir: File) { +fun Project.docsLandingPage(awsServices: AwsServices, outputDir: File) { val project = this val writer = CodeWriter() with(writer) { @@ -29,7 +31,7 @@ fun Project.docsLandingPage(awsServices: List, outputDir: File) { /* generate a basic markdown table */ writer.write("| Service | Package |") writer.write("| ------- | ------- |") - awsServices.sortedBy { it.humanName }.forEach { + awsServices.services.sortedBy { it.humanName }.forEach { val items = listOfNotNull(cratesIo(it), docsRs(it), examplesLink(it, project)).joinToString(" ") writer.write( "| ${it.humanName} | $items |" diff --git a/buildSrc/src/main/kotlin/ServiceLoader.kt b/buildSrc/src/main/kotlin/aws/sdk/ServiceLoader.kt similarity index 73% rename from buildSrc/src/main/kotlin/ServiceLoader.kt rename to buildSrc/src/main/kotlin/aws/sdk/ServiceLoader.kt index 5ee836978..3a84ea52b 100644 --- a/buildSrc/src/main/kotlin/ServiceLoader.kt +++ b/buildSrc/src/main/kotlin/aws/sdk/ServiceLoader.kt @@ -3,6 +3,8 @@ * SPDX-License-Identifier: Apache-2.0. */ +package aws.sdk + import org.gradle.api.Project import software.amazon.smithy.aws.traits.ServiceTrait import software.amazon.smithy.model.Model @@ -11,13 +13,50 @@ import software.amazon.smithy.model.traits.TitleTrait import java.io.File import kotlin.streams.toList +class AwsServices(private val project: Project, services: List) { + val services: List + val moduleNames: Set by lazy { services.map { it.module }.toSortedSet() } + + val allModules: Set by lazy { + ( + services.map(AwsService::module).map { "sdk/$it" } + + CrateSet.AWS_SDK_SMITHY_RUNTIME.map { "sdk/$it" } + + CrateSet.AWS_SDK_RUNTIME.map { "sdk/$it" } + + examples + ).toSortedSet() + } + + val examples: List by lazy { + project.projectDir.resolve("examples") + .listFiles { file -> !file.name.startsWith(".") }.orEmpty().toList() + .filter { file -> + val cargoToml = File(file, "Cargo.toml") + if (cargoToml.exists()) { + val usedModules = cargoToml.readLines() + .map { line -> line.substringBefore('=').trim() } + .filter { line -> line.startsWith("aws-sdk-") } + .map { line -> line.substringAfter("aws-sdk-") } + .toSet() + moduleNames.containsAll(usedModules) + } else { + false + } + } + .map { "examples/${it.name}" } + } + + init { + this.services = services.sortedBy { it.module } + } +} + /** * Discovers services from the `aws-models` directory within the project. * * Since this function parses all models, it is relatively expensive to call. The result should be cached in a property * during build. */ -fun Project.discoverServices(serviceMembership: Membership): List { +fun Project.discoverServices(serviceMembership: Membership): AwsServices { val models = project.file("aws-models") val baseServices = fileTree(models) .sortedBy { file -> file.name } @@ -72,9 +111,12 @@ fun Project.discoverServices(serviceMembership: Membership): List { serviceMembership.inclusions.forEach { service -> check(baseModules.contains(service)) { "Service $service was in explicit inclusion list but not generated!" } } - return baseServices.filter { - serviceMembership.isMember(it.module) - } + return AwsServices( + this, + baseServices.filter { + serviceMembership.isMember(it.module) + } + ) } data class Membership(val inclusions: Set = emptySet(), val exclusions: Set = emptySet()) @@ -102,7 +144,7 @@ data class AwsService( fun AwsService.crate(): String = "aws-sdk-$module" -fun Membership.isMember(member: String): Boolean = when { +private fun Membership.isMember(member: String): Boolean = when { exclusions.contains(member) -> false inclusions.contains(member) -> true inclusions.isEmpty() -> true diff --git a/buildSrc/src/test/kotlin/ManifestPatcherTest.kt b/buildSrc/src/test/kotlin/ManifestPatcherTest.kt new file mode 100644 index 000000000..85431272c --- /dev/null +++ b/buildSrc/src/test/kotlin/ManifestPatcherTest.kt @@ -0,0 +1,18 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0. + */ + +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Test + +class ManifestPatcherTest { + @Test + fun `it should rewrite crate versions`() { + val version = "0.10.0" + assertEquals("", rewriteCrateVersion("", version)) + assertEquals("version = \"0.10.0\"", rewriteCrateVersion("version=\"0.0.0-smithy-rs-head\"", version)) + assertEquals("version = \"0.10.0\"", rewriteCrateVersion(" version = \"0.0.0-smithy-rs-head\"", version)) + assertEquals("version = \"1.5.0\"", rewriteCrateVersion("version = \"1.5.0\"", version)) + } +} diff --git a/rust-runtime/build.gradle.kts b/rust-runtime/build.gradle.kts index 0623435a0..94dd0a6ab 100644 --- a/rust-runtime/build.gradle.kts +++ b/rust-runtime/build.gradle.kts @@ -18,3 +18,41 @@ tasks.jar { include("inlineable/Cargo.toml") } } + +val properties = PropertyRetriever(rootProject, project) +val outputDir = buildDir.resolve("smithy-rs") +val runtimeOutputDir = outputDir.resolve("rust-runtime") + +tasks["assemble"].apply { + dependsOn("copyRuntimeCrates") + dependsOn("fixRuntimeCrateVersions") + dependsOn("fixManifests") +} + +tasks.register("copyRuntimeCrates") { + from("$rootDir/rust-runtime") { + CrateSet.ENTIRE_SMITHY_RUNTIME.forEach { include("$it/**") } + } + exclude("**/target") + exclude("**/Cargo.lock") + exclude("**/node_modules") + into(runtimeOutputDir) +} + +task("fixRuntimeCrateVersions") { + dependsOn("copyRuntimeCrates") + doLast { + CrateSet.ENTIRE_SMITHY_RUNTIME.forEach { moduleName -> + patchFile(runtimeOutputDir.resolve("$moduleName/Cargo.toml")) { line -> + rewriteSmithyRsCrateVersion(properties, line) + } + } + } +} + +tasks.register("fixManifests") { + description = "Run the publisher tool's `fix-manifests` sub-command on the runtime crates" + workingDir(rootProject.projectDir.resolve("tools/publisher")) + commandLine("cargo", "run", "--", "fix-manifests", "--location", runtimeOutputDir.absolutePath) + dependsOn("fixRuntimeCrateVersions") +} diff --git a/tools/publisher/Cargo.lock b/tools/publisher/Cargo.lock new file mode 100644 index 000000000..c7904f4ec --- /dev/null +++ b/tools/publisher/Cargo.lock @@ -0,0 +1,1474 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 3 + +[[package]] +name = "aho-corasick" +version = "0.7.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e37cfd5e7657ada45f742d6e99ca5788580b5c529dc78faf11ece6dc702656f" +dependencies = [ + "memchr", +] + +[[package]] +name = "ansi_term" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee49baf6cb617b853aa8d93bf420db2383fab46d314482ca2803b40d5fde979b" +dependencies = [ + "winapi", +] + +[[package]] +name = "ansi_term" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d52a9bb7ec0cf484c551830a7ce27bd20d67eac647e1befb56b0be4ee39a55d2" +dependencies = [ + "winapi", +] + +[[package]] +name = "anyhow" +version = "1.0.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61604a8f862e1d5c3229fdd78f8b02c68dcf73a4c4b05fd636d12240aaa242c1" + +[[package]] +name = "async-recursion" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7d78656ba01f1b93024b7c3a0467f1608e4be67d725749fdcd7d2c7678fd7a2" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "async-trait" +version = "0.1.51" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44318e776df68115a881de9a8fd1b9e53368d7a4a5ce4cc48517da3393233a5e" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "atty" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8" +dependencies = [ + "hermit-abi", + "libc", + "winapi", +] + +[[package]] +name = "autocfg" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cdb031dd78e28731d87d56cc8ffef4a8f36ca26c38fe2de700543e627f8a464a" + +[[package]] +name = "base64" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "904dfeac50f3cdaba28fc6f57fdcddb75f49ed61346676a78c4ffe55877802fd" + +[[package]] +name = "bitflags" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" + +[[package]] +name = "bumpalo" +version = "3.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f1e260c3a9040a7c19a12468758f4c16f31a81a1fe087482be9570ec864bb6c" + +[[package]] +name = "bytes" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4872d67bab6358e59559027aa3b9157c53d9358c51423c17554809a8858e0f8" + +[[package]] +name = "cargo_toml" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6d613611c914a7db07f28526941ce1e956d2f977b0c5e2014fbfa42230d420f" +dependencies = [ + "serde", + "serde_derive", + "toml", +] + +[[package]] +name = "cc" +version = "1.0.71" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "79c2681d6594606957bbb8631c4b90a7fcaaa72cdb714743a437b156d6a7eedd" + +[[package]] +name = "cfg-if" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" + +[[package]] +name = "chrono" +version = "0.4.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "670ad68c9088c2a963aaa298cb369688cf3f9465ce5e2d4ca10e6e0098a1ce73" +dependencies = [ + "libc", + "num-integer", + "num-traits", + "serde", + "winapi", +] + +[[package]] +name = "clap" +version = "2.33.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37e58ac78573c40708d45522f0d80fa2f01cc4f9b4e2bf749807255454312002" +dependencies = [ + "ansi_term 0.11.0", + "atty", + "bitflags", + "strsim", + "textwrap", + "unicode-width", + "vec_map", +] + +[[package]] +name = "console" +version = "0.14.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3993e6445baa160675931ec041a5e03ca84b9c6e32a056150d3aa2bdda0a1f45" +dependencies = [ + "encode_unicode", + "lazy_static", + "libc", + "regex", + "terminal_size", + "unicode-width", + "winapi", +] + +[[package]] +name = "core-foundation" +version = "0.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6888e10551bb93e424d8df1d07f1a8b4fceb0001a3a4b048bfc47554946f47b3" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "core-foundation-sys" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5827cebf4670468b8772dd191856768aedcb1b0278a04f989f7766351917b9dc" + +[[package]] +name = "crates_io_api" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0ab4553806e9495b7e78a66454c16c5021cd3816d7b8e7e8fe28aa5474346bb" +dependencies = [ + "chrono", + "futures", + "log", + "reqwest", + "serde", + "serde_derive", + "serde_json", + "tokio", + "url", +] + +[[package]] +name = "dialoguer" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c9dd058f8b65922819fabb4a41e7d1964e56344042c26efbccd465202c23fa0c" +dependencies = [ + "console", + "lazy_static", + "tempfile", + "zeroize", +] + +[[package]] +name = "encode_unicode" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a357d28ed41a50f9c765dbfe56cbc04a64e53e5fc58ba79fbc34c10ef3df831f" + +[[package]] +name = "encoding_rs" +version = "0.8.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a74ea89a0a1b98f6332de42c95baff457ada66d1cb4030f9ff151b2041a1c746" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + +[[package]] +name = "foreign-types" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1" +dependencies = [ + "foreign-types-shared", +] + +[[package]] +name = "foreign-types-shared" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" + +[[package]] +name = "form_urlencoded" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5fc25a87fa4fd2094bffb06925852034d90a17f0d1e05197d4956d3555752191" +dependencies = [ + "matches", + "percent-encoding", +] + +[[package]] +name = "futures" +version = "0.3.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a12aa0eb539080d55c3f2d45a67c3b58b6b0773c1a3ca2dfec66d58c97fd66ca" +dependencies = [ + "futures-channel", + "futures-core", + "futures-executor", + "futures-io", + "futures-sink", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-channel" +version = "0.3.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5da6ba8c3bb3c165d3c7319fc1cc8304facf1fb8db99c5de877183c08a273888" +dependencies = [ + "futures-core", + "futures-sink", +] + +[[package]] +name = "futures-core" +version = "0.3.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "88d1c26957f23603395cd326b0ffe64124b818f4449552f960d815cfba83a53d" + +[[package]] +name = "futures-executor" +version = "0.3.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "45025be030969d763025784f7f355043dc6bc74093e4ecc5000ca4dc50d8745c" +dependencies = [ + "futures-core", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-io" +version = "0.3.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "522de2a0fe3e380f1bc577ba0474108faf3f6b18321dbf60b3b9c39a75073377" + +[[package]] +name = "futures-macro" +version = "0.3.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "18e4a4b95cea4b4ccbcf1c5675ca7c4ee4e9e75eb79944d07defde18068f79bb" +dependencies = [ + "autocfg", + "proc-macro-hack", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "futures-sink" +version = "0.3.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "36ea153c13024fe480590b3e3d4cad89a0cfacecc24577b68f86c6ced9c2bc11" + +[[package]] +name = "futures-task" +version = "0.3.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d3d00f4eddb73e498a54394f228cd55853bdf059259e8e7bc6e69d408892e99" + +[[package]] +name = "futures-util" +version = "0.3.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "36568465210a3a6ee45e1f165136d68671471a501e632e9a98d96872222b5481" +dependencies = [ + "autocfg", + "futures-channel", + "futures-core", + "futures-io", + "futures-macro", + "futures-sink", + "futures-task", + "memchr", + "pin-project-lite", + "pin-utils", + "proc-macro-hack", + "proc-macro-nested", + "slab", +] + +[[package]] +name = "getrandom" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7fcd999463524c52659517fe2cea98493cfe485d10565e7b0fb07dbba7ad2753" +dependencies = [ + "cfg-if", + "libc", + "wasi", +] + +[[package]] +name = "h2" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7fd819562fcebdac5afc5c113c3ec36f902840b70fd4fc458799c8ce4607ae55" +dependencies = [ + "bytes", + "fnv", + "futures-core", + "futures-sink", + "futures-util", + "http", + "indexmap", + "slab", + "tokio", + "tokio-util", + "tracing", +] + +[[package]] +name = "hashbrown" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab5ef0d4909ef3724cc8cce6ccc8572c5c817592e9285f5464f8e86f8bd3726e" + +[[package]] +name = "hermit-abi" +version = "0.1.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62b467343b94ba476dcb2500d242dadbb39557df889310ac77c5d99100aaac33" +dependencies = [ + "libc", +] + +[[package]] +name = "http" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1323096b05d41827dadeaee54c9981958c0f94e670bc94ed80037d1a7b8b186b" +dependencies = [ + "bytes", + "fnv", + "itoa", +] + +[[package]] +name = "http-body" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ff4f84919677303da5f147645dbea6b1881f368d03ac84e1dc09031ebd7b2c6" +dependencies = [ + "bytes", + "http", + "pin-project-lite", +] + +[[package]] +name = "httparse" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "acd94fdbe1d4ff688b67b04eee2e17bd50995534a61539e45adfefb45e5e5503" + +[[package]] +name = "httpdate" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6456b8a6c8f33fee7d958fcd1b60d55b11940a79e63ae87013e6d22e26034440" + +[[package]] +name = "hyper" +version = "0.14.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b91bb1f221b6ea1f1e4371216b70f40748774c2fb5971b450c07773fb92d26b" +dependencies = [ + "bytes", + "futures-channel", + "futures-core", + "futures-util", + "h2", + "http", + "http-body", + "httparse", + "httpdate", + "itoa", + "pin-project-lite", + "socket2", + "tokio", + "tower-service", + "tracing", + "want", +] + +[[package]] +name = "hyper-tls" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6183ddfa99b85da61a140bea0efc93fdf56ceaa041b37d553518030827f9905" +dependencies = [ + "bytes", + "hyper", + "native-tls", + "tokio", + "tokio-native-tls", +] + +[[package]] +name = "idna" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "418a0a6fab821475f634efe3ccc45c013f742efe03d853e8d3355d5cb850ecf8" +dependencies = [ + "matches", + "unicode-bidi", + "unicode-normalization", +] + +[[package]] +name = "indexmap" +version = "1.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc633605454125dec4b66843673f01c7df2b89479b32e0ed634e43a91cff62a5" +dependencies = [ + "autocfg", + "hashbrown", +] + +[[package]] +name = "instant" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "716d3d89f35ac6a34fd0eed635395f4c3b76fa889338a4632e5231a8684216bd" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "ipnet" +version = "2.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68f2d64f2edebec4ce84ad108148e67e1064789bee435edc5b60ad398714a3a9" + +[[package]] +name = "itoa" +version = "0.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b71991ff56294aa922b450139ee08b3bfc70982c6b2c7562771375cf73542dd4" + +[[package]] +name = "js-sys" +version = "0.3.55" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7cc9ffccd38c451a86bf13657df244e9c3f37493cce8e5e21e940963777acc84" +dependencies = [ + "wasm-bindgen", +] + +[[package]] +name = "lazy_static" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" + +[[package]] +name = "libc" +version = "0.2.103" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dd8f7255a17a627354f321ef0055d63b898c6fb27eff628af4d1b66b7331edf6" + +[[package]] +name = "lock_api" +version = "0.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712a4d093c9976e24e7dbca41db895dabcbac38eb5f4045393d17a95bdfb1109" +dependencies = [ + "scopeguard", +] + +[[package]] +name = "log" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51b9bbe6c47d51fc3e1a9b945965946b4c44142ab8792c50835a980d362c2710" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "matchers" +version = "0.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f099785f7595cc4b4553a174ce30dd7589ef93391ff414dbb67f62392b9e0ce1" +dependencies = [ + "regex-automata", +] + +[[package]] +name = "matches" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a3e378b66a060d48947b590737b30a1be76706c8dd7b8ba0f2fe3989c68a853f" + +[[package]] +name = "memchr" +version = "2.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "308cc39be01b73d0d18f82a0e7b2a3df85245f84af96fdddc5d202d27e47b86a" + +[[package]] +name = "mime" +version = "0.3.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a60c7ce501c71e03a9c9c0d35b861413ae925bd979cc7a4e30d060069aaac8d" + +[[package]] +name = "mio" +version = "0.7.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8c2bdb6314ec10835cd3293dd268473a835c02b7b352e788be788b3c6ca6bb16" +dependencies = [ + "libc", + "log", + "miow", + "ntapi", + "winapi", +] + +[[package]] +name = "miow" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9f1c5b025cda876f66ef43a113f91ebc9f4ccef34843000e0adf6ebbab84e21" +dependencies = [ + "winapi", +] + +[[package]] +name = "native-tls" +version = "0.2.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "48ba9f7719b5a0f42f338907614285fb5fd70e53858141f69898a1fb7203b24d" +dependencies = [ + "lazy_static", + "libc", + "log", + "openssl", + "openssl-probe", + "openssl-sys", + "schannel", + "security-framework", + "security-framework-sys", + "tempfile", +] + +[[package]] +name = "ntapi" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f6bb902e437b6d86e03cce10a7e2af662292c5dfef23b65899ea3ac9354ad44" +dependencies = [ + "winapi", +] + +[[package]] +name = "num-integer" +version = "0.1.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2cc698a63b549a70bc047073d2949cce27cd1c7b0a4a862d08a8031bc2801db" +dependencies = [ + "autocfg", + "num-traits", +] + +[[package]] +name = "num-traits" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a64b1ec5cda2586e284722486d802acf1f7dbdc623e2bfc57e65ca1cd099290" +dependencies = [ + "autocfg", +] + +[[package]] +name = "num_cpus" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05499f3756671c15885fee9034446956fff3f243d6077b91e5767df161f766b3" +dependencies = [ + "hermit-abi", + "libc", +] + +[[package]] +name = "once_cell" +version = "1.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "692fcb63b64b1758029e0a96ee63e049ce8c5948587f2f7208df04625e5f6b56" + +[[package]] +name = "openssl" +version = "0.10.38" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c7ae222234c30df141154f159066c5093ff73b63204dcda7121eb082fc56a95" +dependencies = [ + "bitflags", + "cfg-if", + "foreign-types", + "libc", + "once_cell", + "openssl-sys", +] + +[[package]] +name = "openssl-probe" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "28988d872ab76095a6e6ac88d99b54fd267702734fd7ffe610ca27f533ddb95a" + +[[package]] +name = "openssl-sys" +version = "0.9.70" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6517987b3f8226b5da3661dad65ff7f300cc59fb5ea8333ca191fc65fde3edf" +dependencies = [ + "autocfg", + "cc", + "libc", + "pkg-config", + "vcpkg", +] + +[[package]] +name = "parking_lot" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d17b78036a60663b797adeaee46f5c9dfebb86948d1255007a1d6be0271ff99" +dependencies = [ + "instant", + "lock_api", + "parking_lot_core", +] + +[[package]] +name = "parking_lot_core" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d76e8e1493bcac0d2766c42737f34458f1c8c50c0d23bcb24ea953affb273216" +dependencies = [ + "cfg-if", + "instant", + "libc", + "redox_syscall", + "smallvec", + "winapi", +] + +[[package]] +name = "percent-encoding" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d4fd5641d01c8f18a23da7b6fe29298ff4b55afcccdf78973b24cf3175fee32e" + +[[package]] +name = "pin-project-lite" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8d31d11c69a6b52a174b42bdc0c30e5e11670f90788b2c471c31c1d17d449443" + +[[package]] +name = "pin-utils" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" + +[[package]] +name = "pkg-config" +version = "0.3.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "12295df4f294471248581bc09bef3c38a5e46f1e36d6a37353621a0c6c357e1f" + +[[package]] +name = "ppv-lite86" +version = "0.2.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac74c624d6b2d21f425f752262f42188365d7b8ff1aff74c82e45136510a4857" + +[[package]] +name = "proc-macro-hack" +version = "0.5.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dbf0c48bc1d91375ae5c3cd81e3722dff1abcf81a30960240640d223f59fe0e5" + +[[package]] +name = "proc-macro-nested" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc881b2c22681370c6a780e47af9840ef841837bc98118431d4e1868bd0c1086" + +[[package]] +name = "proc-macro2" +version = "1.0.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9f5105d4fdaab20335ca9565e106a5d9b82b6219b5ba735731124ac6711d23d" +dependencies = [ + "unicode-xid", +] + +[[package]] +name = "publisher" +version = "0.1.0" +dependencies = [ + "anyhow", + "async-recursion", + "async-trait", + "cargo_toml", + "clap", + "crates_io_api", + "dialoguer", + "lazy_static", + "num_cpus", + "regex", + "semver", + "thiserror", + "tokio", + "toml", + "tracing", + "tracing-subscriber", +] + +[[package]] +name = "quote" +version = "1.0.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38bc8cc6a5f2e3655e0899c1b848643b2562f853f114bfec7be120678e3ace05" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "rand" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e7573632e6454cf6b99d7aac4ccca54be06da05aca2ef7423d22d27d4d4bcd8" +dependencies = [ + "libc", + "rand_chacha", + "rand_core", + "rand_hc", +] + +[[package]] +name = "rand_chacha" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" +dependencies = [ + "ppv-lite86", + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d34f1408f55294453790c48b2f1ebbb1c5b4b7563eb1f418bcfcfdbb06ebb4e7" +dependencies = [ + "getrandom", +] + +[[package]] +name = "rand_hc" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d51e9f596de227fda2ea6c84607f5558e196eeaf43c986b724ba4fb8fdf497e7" +dependencies = [ + "rand_core", +] + +[[package]] +name = "redox_syscall" +version = "0.2.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8383f39639269cde97d255a32bdb68c047337295414940c68bdd30c2e13203ff" +dependencies = [ + "bitflags", +] + +[[package]] +name = "regex" +version = "1.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d07a8629359eb56f1e2fb1652bb04212c072a87ba68546a04065d525673ac461" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c230d73fb8d8c1b9c0b3135c5142a8acee3a0558fb8db5cf1cb65f8d7862132" +dependencies = [ + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.6.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f497285884f3fcff424ffc933e56d7cbca511def0c9831a7f9b5f6153e3cc89b" + +[[package]] +name = "remove_dir_all" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3acd125665422973a33ac9d3dd2df85edad0f4ae9b00dafb1a05e43a9f5ef8e7" +dependencies = [ + "winapi", +] + +[[package]] +name = "reqwest" +version = "0.11.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "66d2927ca2f685faf0fc620ac4834690d29e7abb153add10f5812eef20b5e280" +dependencies = [ + "base64", + "bytes", + "encoding_rs", + "futures-core", + "futures-util", + "http", + "http-body", + "hyper", + "hyper-tls", + "ipnet", + "js-sys", + "lazy_static", + "log", + "mime", + "native-tls", + "percent-encoding", + "pin-project-lite", + "serde", + "serde_json", + "serde_urlencoded", + "tokio", + "tokio-native-tls", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", + "winreg", +] + +[[package]] +name = "ryu" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "71d301d4193d031abdd79ff7e3dd721168a9572ef3fe51a1517aba235bd8f86e" + +[[package]] +name = "schannel" +version = "0.1.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f05ba609c234e60bee0d547fe94a4c7e9da733d1c962cf6e59efa4cd9c8bc75" +dependencies = [ + "lazy_static", + "winapi", +] + +[[package]] +name = "scopeguard" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd" + +[[package]] +name = "security-framework" +version = "2.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "525bc1abfda2e1998d152c45cf13e696f76d0a4972310b22fac1658b05df7c87" +dependencies = [ + "bitflags", + "core-foundation", + "core-foundation-sys", + "libc", + "security-framework-sys", +] + +[[package]] +name = "security-framework-sys" +version = "2.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9dd14d83160b528b7bfd66439110573efcfbe281b17fc2ca9f39f550d619c7e" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "semver" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "568a8e6258aa33c13358f81fd834adb854c6f7c9468520910a9b1e8fac068012" + +[[package]] +name = "serde" +version = "1.0.130" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f12d06de37cf59146fbdecab66aa99f9fe4f78722e3607577a5375d66bd0c913" + +[[package]] +name = "serde_derive" +version = "1.0.130" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7bc1a1ab1961464eae040d96713baa5a724a8152c1222492465b54322ec508b" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_json" +version = "1.0.68" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0f690853975602e1bfe1ccbf50504d67174e3bcf340f23b5ea9992e0587a52d8" +dependencies = [ + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "serde_urlencoded" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "edfa57a7f8d9c1d260a549e7224100f6c43d43f9103e06dd8b4095a9b2b43ce9" +dependencies = [ + "form_urlencoded", + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "sharded-slab" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "740223c51853f3145fe7c90360d2d4232f2b62e3449489c207eccde818979982" +dependencies = [ + "lazy_static", +] + +[[package]] +name = "signal-hook-registry" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e51e73328dc4ac0c7ccbda3a494dfa03df1de2f46018127f60c693f2648455b0" +dependencies = [ + "libc", +] + +[[package]] +name = "slab" +version = "0.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9def91fd1e018fe007022791f865d0ccc9b3a0d5001e01aabb8b40e46000afb5" + +[[package]] +name = "smallvec" +version = "1.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ecab6c735a6bb4139c0caafd0cc3635748bbb3acf4550e8138122099251f309" + +[[package]] +name = "socket2" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5dc90fe6c7be1a323296982db1836d1ea9e47b6839496dde9a541bc496df3516" +dependencies = [ + "libc", + "winapi", +] + +[[package]] +name = "strsim" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ea5119cdb4c55b55d432abb513a0429384878c15dde60cc77b1c99de1a95a6a" + +[[package]] +name = "syn" +version = "1.0.80" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d010a1623fbd906d51d650a9916aaefc05ffa0e4053ff7fe601167f3e715d194" +dependencies = [ + "proc-macro2", + "quote", + "unicode-xid", +] + +[[package]] +name = "tempfile" +version = "3.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dac1c663cfc93810f88aed9b8941d48cabf856a1b111c29a40439018d870eb22" +dependencies = [ + "cfg-if", + "libc", + "rand", + "redox_syscall", + "remove_dir_all", + "winapi", +] + +[[package]] +name = "terminal_size" +version = "0.1.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "633c1a546cee861a1a6d0dc69ebeca693bf4296661ba7852b9d21d159e0506df" +dependencies = [ + "libc", + "winapi", +] + +[[package]] +name = "textwrap" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d326610f408c7a4eb6f51c37c330e496b08506c9457c9d34287ecc38809fb060" +dependencies = [ + "unicode-width", +] + +[[package]] +name = "thiserror" +version = "1.0.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "854babe52e4df1653706b98fcfc05843010039b406875930a70e4d9644e5c417" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa32fd3f627f367fe16f893e2597ae3c05020f8bba2666a4e6ea73d377e5714b" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "thread_local" +version = "1.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8018d24e04c95ac8790716a5987d0fec4f8b27249ffa0f7d33f1369bdfb88cbd" +dependencies = [ + "once_cell", +] + +[[package]] +name = "tinyvec" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f83b2a3d4d9091d0abd7eba4dc2710b1718583bd4d8992e2190720ea38f391f7" +dependencies = [ + "tinyvec_macros", +] + +[[package]] +name = "tinyvec_macros" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cda74da7e1a664f795bb1f8a87ec406fb89a02522cf6e50620d016add6dbbf5c" + +[[package]] +name = "tokio" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2c2416fdedca8443ae44b4527de1ea633af61d8f7169ffa6e72c5b53d24efcc" +dependencies = [ + "autocfg", + "bytes", + "libc", + "memchr", + "mio", + "num_cpus", + "once_cell", + "parking_lot", + "pin-project-lite", + "signal-hook-registry", + "tokio-macros", + "winapi", +] + +[[package]] +name = "tokio-macros" +version = "1.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "154794c8f499c2619acd19e839294703e9e32e7630ef5f46ea80d4ef0fbee5eb" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tokio-native-tls" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f7d995660bd2b7f8c1568414c1126076c13fbb725c40112dc0120b78eb9b717b" +dependencies = [ + "native-tls", + "tokio", +] + +[[package]] +name = "tokio-util" +version = "0.6.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e99e1983e5d376cd8eb4b66604d2e99e79f5bd988c3055891dcd8c9e2604cc0" +dependencies = [ + "bytes", + "futures-core", + "futures-sink", + "log", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "toml" +version = "0.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a31142970826733df8241ef35dc040ef98c679ab14d7c3e54d827099b3acecaa" +dependencies = [ + "indexmap", + "serde", +] + +[[package]] +name = "tower-service" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "360dfd1d6d30e05fda32ace2c8c70e9c0a9da713275777f5a4dbb8a1893930c6" + +[[package]] +name = "tracing" +version = "0.1.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "375a639232caf30edfc78e8d89b2d4c375515393e7af7e16f01cd96917fb2105" +dependencies = [ + "cfg-if", + "pin-project-lite", + "tracing-attributes", + "tracing-core", +] + +[[package]] +name = "tracing-attributes" +version = "0.1.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f4f480b8f81512e825f337ad51e94c1eb5d3bbdf2b363dcd01e2b19a9ffe3f8e" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tracing-core" +version = "0.1.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f4ed65637b8390770814083d20756f87bfa2c21bf2f110babdc5438351746e4" +dependencies = [ + "lazy_static", +] + +[[package]] +name = "tracing-log" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6923477a48e41c1951f1999ef8bb5a3023eb723ceadafe78ffb65dc366761e3" +dependencies = [ + "lazy_static", + "log", + "tracing-core", +] + +[[package]] +name = "tracing-serde" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb65ea441fbb84f9f6748fd496cf7f63ec9af5bca94dd86456978d055e8eb28b" +dependencies = [ + "serde", + "tracing-core", +] + +[[package]] +name = "tracing-subscriber" +version = "0.2.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e0d2eaa99c3c2e41547cfa109e910a68ea03823cccad4a0525dcbc9b01e8c71" +dependencies = [ + "ansi_term 0.12.1", + "chrono", + "lazy_static", + "matchers", + "regex", + "serde", + "serde_json", + "sharded-slab", + "smallvec", + "thread_local", + "tracing", + "tracing-core", + "tracing-log", + "tracing-serde", +] + +[[package]] +name = "try-lock" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59547bce71d9c38b83d9c0e92b6066c4253371f15005def0c30d9657f50c7642" + +[[package]] +name = "unicode-bidi" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a01404663e3db436ed2746d9fefef640d868edae3cceb81c3b8d5732fda678f" + +[[package]] +name = "unicode-normalization" +version = "0.1.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d54590932941a9e9266f0832deed84ebe1bf2e4c9e4a3554d393d18f5e854bf9" +dependencies = [ + "tinyvec", +] + +[[package]] +name = "unicode-width" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ed742d4ea2bd1176e236172c8429aaf54486e7ac098db29ffe6529e0ce50973" + +[[package]] +name = "unicode-xid" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ccb82d61f80a663efe1f787a51b16b5a51e3314d6ac365b08639f52387b33f3" + +[[package]] +name = "url" +version = "2.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a507c383b2d33b5fc35d1861e77e6b383d158b2da5e14fe51b83dfedf6fd578c" +dependencies = [ + "form_urlencoded", + "idna", + "matches", + "percent-encoding", +] + +[[package]] +name = "vcpkg" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" + +[[package]] +name = "vec_map" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1bddf1187be692e79c5ffeab891132dfb0f236ed36a43c7ed39f1165ee20191" + +[[package]] +name = "want" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ce8a968cb1cd110d136ff8b819a556d6fb6d919363c61534f6860c7eb172ba0" +dependencies = [ + "log", + "try-lock", +] + +[[package]] +name = "wasi" +version = "0.10.2+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fd6fbd9a79829dd1ad0cc20627bf1ed606756a7f77edff7b66b7064f9cb327c6" + +[[package]] +name = "wasm-bindgen" +version = "0.2.78" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "632f73e236b219150ea279196e54e610f5dbafa5d61786303d4da54f84e47fce" +dependencies = [ + "cfg-if", + "wasm-bindgen-macro", +] + +[[package]] +name = "wasm-bindgen-backend" +version = "0.2.78" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a317bf8f9fba2476b4b2c85ef4c4af8ff39c3c7f0cdfeed4f82c34a880aa837b" +dependencies = [ + "bumpalo", + "lazy_static", + "log", + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-futures" +version = "0.4.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e8d7523cb1f2a4c96c1317ca690031b714a51cc14e05f712446691f413f5d39" +dependencies = [ + "cfg-if", + "js-sys", + "wasm-bindgen", + "web-sys", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.78" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d56146e7c495528bf6587663bea13a8eb588d39b36b679d83972e1a2dbbdacf9" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.78" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7803e0eea25835f8abdc585cd3021b3deb11543c6fe226dcd30b228857c5c5ab" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-backend", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.78" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0237232789cf037d5480773fe568aac745bfe2afbc11a863e97901780a6b47cc" + +[[package]] +name = "web-sys" +version = "0.3.55" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38eb105f1c59d9eaa6b5cdc92b859d85b926e82cb2e0945cd0c9259faa6fe9fb" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + +[[package]] +name = "winreg" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0120db82e8a1e0b9fb3345a539c478767c0048d842860994d96113d5b667bd69" +dependencies = [ + "winapi", +] + +[[package]] +name = "zeroize" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf68b08513768deaa790264a7fac27a58cbf2705cfcdc9448362229217d7e970" diff --git a/tools/publisher/Cargo.toml b/tools/publisher/Cargo.toml new file mode 100644 index 000000000..0159055d9 --- /dev/null +++ b/tools/publisher/Cargo.toml @@ -0,0 +1,28 @@ +[package] +name = "publisher" +version = "0.1.0" +authors = ["AWS Rust SDK Team "] +description = "Tool used to publish the AWS SDK to crates.io" +edition = "2018" +license = "Apache-2.0" +publish = false + +[workspace] + +[dependencies] +anyhow = "1.0" +async-recursion = "0.3.2" +async-trait = "0.1.51" +cargo_toml = "0.10.1" +clap = "2.33" +crates_io_api = "0.7.3" +lazy_static = "1" +dialoguer = "0.8" +num_cpus = "1.13" +regex = "1.5.4" +semver = "1.0" +thiserror = "1.0" +tokio = { version = "1.12", features = ["full"] } +toml = { version = "0.5.8", features = ["preserve_order"] } +tracing = "0.1.29" +tracing-subscriber = "0.2.25" diff --git a/tools/publisher/README.md b/tools/publisher/README.md new file mode 100644 index 000000000..4d9093f6e --- /dev/null +++ b/tools/publisher/README.md @@ -0,0 +1 @@ +This is a tool that the SDK developer team uses to publish the AWS SDK to crates.io. diff --git a/tools/publisher/fake_cargo/cargo_fails b/tools/publisher/fake_cargo/cargo_fails new file mode 100755 index 000000000..9166af04a --- /dev/null +++ b/tools/publisher/fake_cargo/cargo_fails @@ -0,0 +1,4 @@ +#!/bin/bash +echo "some stdout failure message" +>&2 echo "some stderr failure message" +exit 1 diff --git a/tools/publisher/fake_cargo/cargo_owner_list b/tools/publisher/fake_cargo/cargo_owner_list new file mode 100755 index 000000000..d065bddef --- /dev/null +++ b/tools/publisher/fake_cargo/cargo_owner_list @@ -0,0 +1,4 @@ +#!/bin/bash +>&2 echo " Updating crates.io index" +echo "rcoh (Russell Cohen)" +echo "github:awslabs:rust-sdk-owners (rust-sdk-owners)" diff --git a/tools/publisher/fake_cargo/cargo_publish_already_published b/tools/publisher/fake_cargo/cargo_publish_already_published new file mode 100755 index 000000000..02cf5fc2a --- /dev/null +++ b/tools/publisher/fake_cargo/cargo_publish_already_published @@ -0,0 +1,3 @@ +#!/bin/bash +echo 'error: crate version `0.0.22-alpha` is already uploaded' +exit 1 diff --git a/tools/publisher/fake_cargo/cargo_search_success b/tools/publisher/fake_cargo/cargo_search_success new file mode 100755 index 000000000..6d1395c85 --- /dev/null +++ b/tools/publisher/fake_cargo/cargo_search_success @@ -0,0 +1,4 @@ +#!/bin/bash +# Fake `cargo search aws-sdk-dynamodb` for unit testing +echo 'aws-sdk-dynamodb = "0.0.22-alpha" # AWS SDK for Amazon DynamoDB' +echo 'aws-sdk-dynamodbstreams = "0.0.22-alpha" # AWS SDK for Amazon DynamoDB Streams' diff --git a/tools/publisher/fake_cargo/cargo_success b/tools/publisher/fake_cargo/cargo_success new file mode 100755 index 000000000..06bd98656 --- /dev/null +++ b/tools/publisher/fake_cargo/cargo_success @@ -0,0 +1,2 @@ +#!/bin/bash +exit 0 diff --git a/tools/publisher/fake_cargo/cargo_yank_not_found b/tools/publisher/fake_cargo/cargo_yank_not_found new file mode 100755 index 000000000..3f399b9ae --- /dev/null +++ b/tools/publisher/fake_cargo/cargo_yank_not_found @@ -0,0 +1,10 @@ +#!/bin/bash +{ + echo " Updating crates.io index" + echo " Yank aws-sigv4:0.0.0" + echo "error: failed to yank from the registry at https://crates.io" + echo + echo "Caused by:" + echo ' the remote server responded with an error: crate `aws-sigv4` does not have a version `0.0.0`' +} >&2 +exit 101 diff --git a/tools/publisher/src/cargo.rs b/tools/publisher/src/cargo.rs new file mode 100644 index 000000000..f0e051d8a --- /dev/null +++ b/tools/publisher/src/cargo.rs @@ -0,0 +1,74 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0. + */ + +//! Module for interacting with Cargo. + +mod add_owner; +mod get_owners; +mod publish; +mod yank; + +pub use add_owner::AddOwner; +pub use get_owners::GetOwners; +pub use publish::Publish; +pub use yank::Yank; + +use anyhow::{Context, Result}; +use async_trait::async_trait; +use std::borrow::Cow; +use std::process::{Command, Output}; + +#[async_trait] +pub trait CargoOperation { + type Output; + + /// Runs the command asynchronously. + async fn spawn(&self) -> Result; + + /// Returns a plan string that can be output to the user to describe the command. + fn plan(&self) -> Option>; +} + +/// Confirms that cargo exists on the path. +pub fn confirm_installed_on_path() -> Result<()> { + handle_failure( + "discover cargo version", + &Command::new("cargo") + .arg("version") + .output() + .context("cargo is not installed on the PATH")?, + ) + .context("cargo is not installed on the PATH") +} + +/// Returns (stdout, stderr) +fn output_text(output: &Output) -> (String, String) { + ( + String::from_utf8_lossy(&output.stdout).to_string(), + String::from_utf8_lossy(&output.stderr).to_string(), + ) +} + +fn handle_failure(operation_name: &str, output: &Output) -> Result<(), anyhow::Error> { + if !output.status.success() { + return Err(capture_error(operation_name, output)); + } + Ok(()) +} + +fn capture_error(operation_name: &str, output: &Output) -> anyhow::Error { + let message = format!( + "Failed to {name}:\nStatus: {status}\nStdout: {stdout}\nStderr: {stderr}\n", + name = operation_name, + status = if let Some(code) = output.status.code() { + format!("{}", code) + } else { + "Killed by signal".to_string() + }, + stdout = String::from_utf8_lossy(&output.stdout), + stderr = String::from_utf8_lossy(&output.stderr) + ); + anyhow::Error::msg(message) +} diff --git a/tools/publisher/src/cargo/add_owner.rs b/tools/publisher/src/cargo/add_owner.rs new file mode 100644 index 000000000..a90db64c5 --- /dev/null +++ b/tools/publisher/src/cargo/add_owner.rs @@ -0,0 +1,84 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0. + */ + +use crate::cargo::{handle_failure, CargoOperation}; +use anyhow::Result; +use async_trait::async_trait; +use std::borrow::Cow; +use std::process::Command; + +pub struct AddOwner<'a> { + program: &'static str, + package_name: &'a str, + owner: &'a str, +} + +impl<'a> AddOwner<'a> { + pub fn new(package_name: &'a str, owner: &'a str) -> AddOwner<'a> { + AddOwner { + program: "cargo", + package_name, + owner, + } + } +} + +#[async_trait] +impl<'a> CargoOperation for AddOwner<'a> { + type Output = (); + + async fn spawn(&self) -> Result<()> { + let mut command = Command::new(self.program); + command + .arg("owner") + .arg("--add") + .arg(self.owner) + .arg(self.package_name); + let output = tokio::task::spawn_blocking(move || command.output()).await??; + handle_failure("add owner", &output)?; + Ok(()) + } + + fn plan(&self) -> Option> { + None + } +} + +#[cfg(all(test, not(target_os = "windows")))] +mod tests { + use super::*; + + #[tokio::test] + async fn add_owner_success() { + AddOwner { + program: "./fake_cargo/cargo_success", + package_name: "aws-sdk-s3", + owner: "github:awslabs:rust-sdk-owners", + } + .spawn() + .await + .unwrap(); + } + + #[tokio::test] + async fn get_owners_failed() { + let result = AddOwner { + program: "./fake_cargo/cargo_fails", + package_name: "aws-sdk-s3", + owner: "github:awslabs:rust-sdk-owners", + } + .spawn() + .await; + + assert!(result.is_err(), "expected error, got {:?}", result); + assert_eq!( + "Failed to add owner:\n\ + Status: 1\n\ + Stdout: some stdout failure message\n\n\ + Stderr: some stderr failure message\n\n", + format!("{}", result.err().unwrap()) + ); + } +} diff --git a/tools/publisher/src/cargo/get_owners.rs b/tools/publisher/src/cargo/get_owners.rs new file mode 100644 index 000000000..83b32f188 --- /dev/null +++ b/tools/publisher/src/cargo/get_owners.rs @@ -0,0 +1,99 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0. + */ + +use crate::cargo::{handle_failure, output_text, CargoOperation}; +use anyhow::Result; +use async_trait::async_trait; +use regex::Regex; +use std::borrow::Cow; +use std::process::Command; + +pub struct GetOwners<'a> { + program: &'static str, + package_name: &'a str, +} + +impl<'a> GetOwners<'a> { + pub fn new(package_name: &'a str) -> GetOwners<'a> { + GetOwners { + program: "cargo", + package_name, + } + } +} + +#[async_trait] +impl<'a> CargoOperation for GetOwners<'a> { + type Output = Vec; + + async fn spawn(&self) -> Result> { + let mut command = Command::new(self.program); + command.arg("owner").arg("--list").arg(self.package_name); + let output = tokio::task::spawn_blocking(move || command.output()).await??; + handle_failure("get crate owners", &output)?; + + let mut result = Vec::new(); + let (stdout, _) = output_text(&output); + let line_re = Regex::new(r#"^([\w\d\-_:]+)\s+\([\w\d\s\-_]+\)$"#).unwrap(); + for line in stdout.lines() { + if let Some(captures) = line_re.captures(line) { + let user_id = captures.get(1).unwrap().as_str(); + result.push(user_id.to_string()); + } else { + return Err(anyhow::Error::msg(format!( + "unrecognized line in `cargo owner` output: {}", + line + ))); + } + } + Ok(result) + } + + fn plan(&self) -> Option> { + None + } +} + +#[cfg(all(test, not(target_os = "windows")))] +mod tests { + use super::*; + + #[tokio::test] + async fn get_owners_success() { + let owners = GetOwners { + program: "./fake_cargo/cargo_owner_list", + package_name: "aws-sdk-s3", + } + .spawn() + .await + .unwrap(); + assert_eq!( + vec![ + "rcoh".to_string(), + "github:awslabs:rust-sdk-owners".to_string() + ], + owners + ); + } + + #[tokio::test] + async fn get_owners_failed() { + let result = GetOwners { + program: "./fake_cargo/cargo_fails", + package_name: "aws-sdk-s3", + } + .spawn() + .await; + + assert!(result.is_err(), "expected error, got {:?}", result); + assert_eq!( + "Failed to get crate owners:\n\ + Status: 1\n\ + Stdout: some stdout failure message\n\n\ + Stderr: some stderr failure message\n\n", + format!("{}", result.err().unwrap()) + ); + } +} diff --git a/tools/publisher/src/cargo/publish.rs b/tools/publisher/src/cargo/publish.rs new file mode 100644 index 000000000..5797236a3 --- /dev/null +++ b/tools/publisher/src/cargo/publish.rs @@ -0,0 +1,127 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0. + */ + +use crate::cargo::{capture_error, output_text, CargoOperation}; +use crate::package::PackageHandle; +use anyhow::Result; +use async_trait::async_trait; +use std::borrow::Cow; +use std::path::Path; +use std::process::Command; +use tracing::info; + +pub struct Publish<'a> { + program: &'static str, + package_handle: &'a PackageHandle, + package_path: &'a Path, +} + +impl<'a> Publish<'a> { + pub fn new(package_handle: &'a PackageHandle, package_path: &'a Path) -> Publish<'a> { + Publish { + program: "cargo", + package_handle, + package_path, + } + } +} + +#[async_trait] +impl<'a> CargoOperation for Publish<'a> { + type Output = (); + + async fn spawn(&self) -> Result<()> { + let mut command = Command::new(self.program); + command + .current_dir(self.package_path) + .env("CARGO_INCREMENTAL", "0") // Disable incremental compilation to reduce disk space used + .arg("publish") + .arg("--jobs") + .arg("1"); + let output = tokio::task::spawn_blocking(move || command.output()).await??; + if !output.status.success() { + let (stdout, stderr) = output_text(&output); + let already_uploaded_msg = format!( + "error: crate version `{}` is already uploaded", + self.package_handle.version + ); + if stdout.contains(&already_uploaded_msg) || stderr.contains(&already_uploaded_msg) { + info!( + "{}-{} has already been published to crates.io.", + self.package_handle.name, self.package_handle.version + ); + } else { + return Err(capture_error("cargo publish", &output)); + } + } + Ok(()) + } + + fn plan(&self) -> Option> { + Some(Cow::Owned(format!( + "[in {:?}]: cargo publish --jobs 1", + self.package_path + ))) + } +} + +#[cfg(all(test, not(target_os = "windows")))] +mod tests { + use super::*; + use semver::Version; + use std::env; + + #[tokio::test] + async fn publish_succeeds() { + Publish { + program: "./fake_cargo/cargo_success", + package_handle: &PackageHandle::new( + "aws-sdk-dynamodb", + Version::parse("0.0.22-alpha").unwrap(), + ), + package_path: &env::current_dir().unwrap(), + } + .spawn() + .await + .unwrap(); + } + + #[tokio::test] + async fn publish_fails() { + let result = Publish { + program: "./fake_cargo/cargo_fails", + package_handle: &PackageHandle::new( + "something", + Version::parse("0.0.22-alpha").unwrap(), + ), + package_path: &env::current_dir().unwrap(), + } + .spawn() + .await; + assert!(result.is_err(), "expected error, got {:?}", result); + assert_eq!( + "Failed to cargo publish:\n\ + Status: 1\n\ + Stdout: some stdout failure message\n\n\ + Stderr: some stderr failure message\n\n", + format!("{}", result.err().unwrap()) + ); + } + + #[tokio::test] + async fn publish_fails_already_uploaded() { + Publish { + program: "./fake_cargo/cargo_publish_already_published", + package_handle: &PackageHandle::new( + "aws-sdk-dynamodb", + Version::parse("0.0.22-alpha").unwrap(), + ), + package_path: &env::current_dir().unwrap(), + } + .spawn() + .await + .unwrap(); + } +} diff --git a/tools/publisher/src/cargo/yank.rs b/tools/publisher/src/cargo/yank.rs new file mode 100644 index 000000000..e16b79334 --- /dev/null +++ b/tools/publisher/src/cargo/yank.rs @@ -0,0 +1,125 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0. + */ + +use crate::cargo::{capture_error, output_text, CargoOperation}; +use crate::package::PackageHandle; +use anyhow::Result; +use async_trait::async_trait; +use std::borrow::Cow; +use std::path::Path; +use std::process::Command; +use tracing::info; + +/// Yanks a package version from crates.io +pub struct Yank<'a> { + program: &'static str, + package_handle: &'a PackageHandle, + package_path: &'a Path, +} + +impl<'a> Yank<'a> { + pub fn new(package_handle: &'a PackageHandle, package_path: &'a Path) -> Yank<'a> { + Yank { + program: "cargo", + package_handle, + package_path, + } + } +} + +#[async_trait] +impl<'a> CargoOperation for Yank<'a> { + type Output = (); + + async fn spawn(&self) -> Result<()> { + let mut command = Command::new(self.program); + command + .current_dir(self.package_path) + .arg("yank") + .arg("--vers") + .arg(format!("{}", self.package_handle.version)) + .arg(&self.package_handle.name); + let output = tokio::task::spawn_blocking(move || command.output()).await??; + if !output.status.success() { + let (_, stderr) = output_text(&output); + let no_such_version = format!( + "error: crate `{}` does not have a version `{}`", + self.package_handle.name, self.package_handle.version + ); + if stderr.contains(&no_such_version) { + info!( + "{} never had a version {}.", + self.package_handle.name, self.package_handle.version + ); + } else { + return Err(capture_error("cargo yank", &output)); + } + } + Ok(()) + } + + fn plan(&self) -> Option> { + Some(Cow::Owned(format!( + "[in {:?}] cargo yank --vers {} {}", + self.package_path, self.package_handle.version, self.package_handle.name + ))) + } +} + +#[cfg(all(test, not(target_os = "windows")))] +mod tests { + use super::*; + use semver::Version; + use std::env; + + #[tokio::test] + async fn yank_succeeds() { + Yank { + program: "./fake_cargo/cargo_success", + package_handle: &PackageHandle::new( + "aws-sdk-dynamodb", + Version::parse("0.0.22-alpha").unwrap(), + ), + package_path: &env::current_dir().unwrap(), + } + .spawn() + .await + .unwrap(); + } + + #[tokio::test] + async fn yank_fails() { + let result = Yank { + program: "./fake_cargo/cargo_fails", + package_handle: &PackageHandle::new( + "something", + Version::parse("0.0.22-alpha").unwrap(), + ), + package_path: &env::current_dir().unwrap(), + } + .spawn() + .await; + assert!(result.is_err(), "expected error, got {:?}", result); + assert_eq!( + "Failed to cargo yank:\n\ + Status: 1\n\ + Stdout: some stdout failure message\n\n\ + Stderr: some stderr failure message\n\n", + format!("{}", result.err().unwrap()) + ); + } + + #[tokio::test] + async fn yank_no_such_version() { + Yank { + program: "./fake_cargo/cargo_yank_not_found", + package_handle: &PackageHandle::new("aws-sigv4", Version::parse("0.0.0").unwrap()), + package_path: &env::current_dir().unwrap(), + } + .spawn() + .await + .unwrap(); + } +} diff --git a/tools/publisher/src/fs.rs b/tools/publisher/src/fs.rs new file mode 100644 index 000000000..1bcb53c5d --- /dev/null +++ b/tools/publisher/src/fs.rs @@ -0,0 +1,52 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0. + */ + +use anyhow::{Context, Result}; +use std::path::Path; +use tokio::fs::File; +use tokio::io::{AsyncReadExt, AsyncWriteExt}; + +/// Abstraction of the filesystem to allow for more tests to be added in the future. +#[derive(Clone, Debug)] +pub enum Fs { + Real, +} + +impl Fs { + /// Reads entire file into `Vec` + pub async fn read_file(&self, path: impl AsRef) -> Result> { + match self { + Fs::Real => tokio_read_file(path.as_ref()).await, + } + } + + /// Writes an entire file from a `&[u8]` + pub async fn write_file(&self, path: impl AsRef, contents: &[u8]) -> Result<()> { + match self { + Fs::Real => tokio_write_file(path.as_ref(), contents).await, + } + } +} + +async fn tokio_read_file(path: &Path) -> Result> { + let mut contents = Vec::new(); + let mut file = File::open(path) + .await + .with_context(|| format!("failed to open {:?}", path))?; + file.read_to_end(&mut contents) + .await + .with_context(|| format!("failed to read {:?}", path))?; + Ok(contents) +} + +async fn tokio_write_file(path: &Path, contents: &[u8]) -> Result<()> { + let mut file = File::create(path) + .await + .with_context(|| format!("failed to create {:?}", path))?; + file.write_all(contents) + .await + .with_context(|| format!("failed to write {:?}", path))?; + Ok(()) +} diff --git a/tools/publisher/src/main.rs b/tools/publisher/src/main.rs new file mode 100644 index 000000000..b913885a5 --- /dev/null +++ b/tools/publisher/src/main.rs @@ -0,0 +1,113 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0. + */ + +use crate::subcommand::fix_manifests::{subcommand_fix_manifests, Mode}; +use crate::subcommand::publish::subcommand_publish; +use crate::subcommand::yank_category::subcommand_yank_category; +use anyhow::Result; +use clap::{crate_authors, crate_description, crate_name, crate_version}; + +mod cargo; +mod fs; +mod package; +mod repo; +mod sort; +mod subcommand; + +pub const REPO_NAME: &str = "aws-sdk-rust"; +pub const REPO_CRATE_PATH: &str = "sdk"; +pub const CRATE_OWNER: &str = "github:awslabs:rust-sdk-owners"; + +#[tokio::main] +async fn main() -> Result<()> { + tracing_subscriber::fmt() + .with_env_filter( + std::env::var("RUST_LOG").unwrap_or_else(|_| "error,publisher=info".to_owned()), + ) + .init(); + + let matches = clap_app().get_matches(); + if let Some(matches) = matches.subcommand_matches("publish") { + let continue_from = matches.value_of("continue-from"); + subcommand_publish(matches.value_of("location").unwrap(), continue_from).await?; + } else if let Some(fix_manifests) = matches.subcommand_matches("fix-manifests") { + let mode = match fix_manifests.is_present("check") { + true => Mode::Check, + false => Mode::Execute, + }; + subcommand_fix_manifests(mode, fix_manifests.value_of("location").unwrap()).await?; + } else if let Some(matches) = matches.subcommand_matches("yank-category") { + let category = matches.value_of("category").unwrap(); + let version = matches.value_of("version").unwrap(); + subcommand_yank_category(category, version).await?; + } else { + clap_app().print_long_help().unwrap(); + } + Ok(()) +} + +fn clap_app() -> clap::App<'static, 'static> { + clap::App::new(crate_name!()) + .version(crate_version!()) + .author(crate_authors!()) + .about(crate_description!()) + // In the future, there may be another subcommand for yanking + .subcommand( + clap::SubCommand::with_name("fix-manifests") + .about("fixes path dependencies in manifests to also have version numbers") + .arg( + clap::Arg::with_name("location") + .required(true) + .takes_value(true) + .long("location") + .help("Path containing the manifests to fix. Manifests will be discovered recursively"), + ) + .arg( + clap::Arg::with_name("check") + .required(false) + .takes_value(false) + .long("check"), + ), + ) + .subcommand( + clap::SubCommand::with_name("publish") + .about("publishes crates to crates.io") + .arg( + clap::Arg::with_name("location") + .required(true) + .takes_value(true) + .long("location") + .help("Path containing the crates to publish. Crates will be discovered recursively"), + ) + .arg( + clap::Arg::with_name("continue-from") + .long("continue-from") + .required(false) + .takes_value(true) + .help( + "Crate name to continue publishing from, if, for example, \ + publishing failed half way through previously.", + ), + ), + ) + .subcommand( + clap::SubCommand::with_name("yank-category") + .about("yanks a category of packages with the given version number") + .arg( + clap::Arg::with_name("category") + .long("category") + .required(true) + .takes_value(true) + .help("package category to yank (smithy-runtime, aws-runtime, or aws-sdk)"), + ) + .arg( + clap::Arg::with_name("version") + .long("version") + .required(true) + .takes_value(true) + .help("version number to yank"), + ), + ) +} diff --git a/tools/publisher/src/package.rs b/tools/publisher/src/package.rs new file mode 100644 index 000000000..9b6663082 --- /dev/null +++ b/tools/publisher/src/package.rs @@ -0,0 +1,584 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0. + */ + +//! Packages, package discovery, and package batching logic. + +use crate::fs::Fs; +use crate::sort::dependency_order; +use anyhow::{Context, Result}; +use cargo_toml::{Dependency, DepsSet, Manifest}; +use semver::Version; +use std::collections::{BTreeMap, BTreeSet}; +use std::error::Error as StdError; +use std::fmt; +use std::path::{Path, PathBuf}; +use tokio::fs; +use tracing::warn; + +#[derive(Clone, Copy, Debug, Eq, PartialEq, Ord, PartialOrd)] +pub enum PackageCategory { + SmithyRuntime, + AwsRuntime, + AwsSdk, + Unknown, +} + +/// Information required to identify a package (crate). +#[derive(Clone, Debug, Eq, PartialEq, Ord, PartialOrd)] +pub struct PackageHandle { + pub name: String, + pub version: Version, +} + +impl PackageHandle { + pub fn new(name: impl Into, version: Version) -> Self { + Self { + name: name.into(), + version, + } + } +} + +impl fmt::Display for PackageHandle { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{}-{}", self.name, self.version) + } +} + +/// Represents a crate (called Package since crate is a reserved word). +#[derive(Debug, Eq, PartialEq, PartialOrd, Ord)] +pub struct Package { + /// Package name and version information + pub handle: PackageHandle, + /// Package category (Generated, SmithyRuntime, AwsRuntime, etc.) + pub category: PackageCategory, + /// Location to the crate on the current file system + pub crate_path: PathBuf, + /// Location to the crate manifest on the current file system + pub manifest_path: PathBuf, + /// Dependencies used by this package + pub local_dependencies: BTreeSet, +} + +impl Package { + pub fn new( + handle: PackageHandle, + manifest_path: impl Into, + local_dependencies: BTreeSet, + ) -> Self { + let manifest_path = manifest_path.into(); + let category = if handle.name.starts_with("aws-smithy-") { + PackageCategory::SmithyRuntime + } else if handle.name.starts_with("aws-sdk-") { + PackageCategory::AwsSdk + } else if handle.name.starts_with("aws-") { + PackageCategory::AwsRuntime + } else { + PackageCategory::Unknown + }; + Self { + handle, + category, + crate_path: manifest_path.parent().unwrap().into(), + manifest_path, + local_dependencies, + } + } + + /// Returns `true` if this package depends on `other` + pub fn locally_depends_on(&self, other: &PackageHandle) -> bool { + self.local_dependencies.contains(other) + } +} + +/// Batch of packages. +pub type PackageBatch = Vec; + +/// Stats about the packages. +#[derive(Copy, Clone, Debug, Default, Eq, PartialEq)] +pub struct PackageStats { + /// Number of Smithy runtime crates + pub smithy_runtime_crates: usize, + /// Number of AWS runtime crates + pub aws_runtime_crates: usize, + /// Number of AWS service crates + pub aws_sdk_crates: usize, +} + +impl PackageStats { + pub fn total(&self) -> usize { + self.smithy_runtime_crates + self.aws_runtime_crates + self.aws_sdk_crates + } + + fn calculate(batches: &[PackageBatch]) -> PackageStats { + let mut stats = PackageStats::default(); + for batch in batches { + for package in batch { + match package.category { + PackageCategory::SmithyRuntime => stats.smithy_runtime_crates += 1, + PackageCategory::AwsRuntime => stats.aws_runtime_crates += 1, + PackageCategory::AwsSdk => stats.aws_sdk_crates += 1, + PackageCategory::Unknown => { + warn!("Unrecognized crate: {}", package.handle.name); + } + } + } + } + stats + } +} + +/// Discovers publishable packages in the given directory and returns them as +/// batches that can be published in order. +pub async fn discover_package_batches( + fs: Fs, + path: impl AsRef, +) -> Result<(Vec, PackageStats)> { + let manifest_paths = discover_package_manifests(path.as_ref().into()).await?; + let packages = read_packages(fs, manifest_paths).await?; + validate_packages(&packages)?; + + let batches = batch_packages(packages)?; + let stats = PackageStats::calculate(&batches); + Ok((batches, stats)) +} + +/// Modifies the given `batches` so that publishing will continue from the given +/// `package_name`. The `stats` are modified to reflect how many crates will be published +/// after the filtering. +pub fn continue_batches_from( + package_name: &str, + batches: &mut Vec, + stats: &mut PackageStats, +) -> Result<(), anyhow::Error> { + while !batches.is_empty() { + let found = { + let first_batch = batches.iter().next().unwrap(); + first_batch.iter().any(|p| p.handle.name == package_name) + }; + if !found { + batches.remove(0); + } else { + let first_batch = &mut batches[0]; + while !first_batch.is_empty() && first_batch[0].handle.name != package_name { + first_batch.remove(0); + } + break; + } + } + *stats = PackageStats::calculate(batches); + if batches.is_empty() { + Err(anyhow::Error::msg("no more batches to publish")) + } else { + Ok(()) + } +} + +type BoxError = Box; + +#[derive(Debug, thiserror::Error)] +pub enum Error { + #[error("Invalid manifest {0:?}")] + InvalidManifest(PathBuf), + #[error( + "Invalid crate version {1} in {0:?}: {2}. NOTE: All local dependencies \ + must have complete version numbers rather than version requirements." + )] + InvalidCrateVersion(PathBuf, String, BoxError), + #[error("{0:?} missing version in dependency {1}")] + MissingVersion(PathBuf, String), + #[error("crate {0} has multiple versions: {1} and {2}")] + MultipleVersions(String, Version, Version), +} + +/// Discovers all Cargo.toml files under the given path recursively +#[async_recursion::async_recursion] +pub async fn discover_package_manifests(path: PathBuf) -> Result> { + let mut manifests = Vec::new(); + let mut read_dir = fs::read_dir(&path).await?; + while let Some(entry) = read_dir.next_entry().await? { + let package_path = entry.path(); + if package_path.is_dir() { + let manifest_path = package_path.join("Cargo.toml"); + if manifest_path.exists() { + manifests.push(manifest_path); + } else { + manifests.extend(discover_package_manifests(package_path).await?.into_iter()); + } + } + } + Ok(manifests) +} + +/// Parses a semver version number and adds additional error context when parsing fails. +pub fn parse_version(manifest_path: &Path, version: &str) -> Result { + Version::parse(version) + .map_err(|err| Error::InvalidCrateVersion(manifest_path.into(), version.into(), err.into())) +} + +fn read_dependencies(path: &Path, dependencies: &DepsSet) -> Result> { + let mut result = Vec::new(); + for (name, metadata) in dependencies { + match metadata { + Dependency::Simple(_) => {} + Dependency::Detailed(detailed) => { + if detailed.path.is_some() { + let version = detailed + .version + .as_ref() + .map(|version| parse_version(path, version)) + .ok_or_else(|| Error::MissingVersion(path.into(), name.into()))??; + result.push(PackageHandle::new(name, version)); + } + } + } + } + Ok(result) +} + +fn read_package(path: &Path, manifest_bytes: &[u8]) -> Result { + let manifest = Manifest::from_slice(manifest_bytes) + .with_context(|| format!("failed to load package manifest for {:?}", path))?; + let package = manifest + .package + .ok_or_else(|| Error::InvalidManifest(path.into())) + .context("crate manifest doesn't have a `[package]` section")?; + let name = package.name; + let version = parse_version(path, &package.version)?; + let handle = PackageHandle { name, version }; + + let mut local_dependencies = BTreeSet::new(); + local_dependencies.extend(read_dependencies(path, &manifest.dependencies)?.into_iter()); + local_dependencies.extend(read_dependencies(path, &manifest.dev_dependencies)?.into_iter()); + local_dependencies.extend(read_dependencies(path, &manifest.build_dependencies)?.into_iter()); + Ok(Package::new(handle, path, local_dependencies)) +} + +/// Validates that all of the publishable crates use consistent version numbers +/// across all of their local dependencies. +fn validate_packages(packages: &[Package]) -> Result<()> { + let mut versions: BTreeMap = BTreeMap::new(); + let track_version = &mut |handle: &PackageHandle| -> Result<(), Error> { + if let Some(version) = versions.get(&handle.name) { + if *version != handle.version { + Err(Error::MultipleVersions( + (&handle.name).into(), + versions[&handle.name].clone(), + handle.version.clone(), + )) + } else { + Ok(()) + } + } else { + versions.insert(handle.name.clone(), handle.version.clone()); + Ok(()) + } + }; + for package in packages { + track_version(&package.handle)?; + for dependency in &package.local_dependencies { + track_version(dependency)?; + } + } + + Ok(()) +} + +async fn read_packages(fs: Fs, manifest_paths: Vec) -> Result> { + let mut result = Vec::new(); + for path in &manifest_paths { + let contents: Vec = fs.read_file(path).await?; + result.push(read_package(path, &contents)?); + } + Ok(result) +} + +/// Splits the given packages into a list of batches that can be published in order. +/// All of the packages in a given batch can be safely published in parallel. +fn batch_packages(packages: Vec) -> Result> { + // Sort packages in order of local dependencies + let mut packages = dependency_order(packages)?; + + // Discover batches + let mut batches = Vec::new(); + 'outer: while packages.len() > 1 { + for run in 0..packages.len() { + let next = &packages[run]; + // If the next package depends on any prior package, then we've discovered the end of the batch + for index in 0..run { + let previous = &packages[index]; + if next.locally_depends_on(&previous.handle) { + let remaining = packages.split_off(run); + let batch = packages; + packages = remaining; + batches.push(batch); + continue 'outer; + } + } + } + // If the current run is the length of the package vec, then we have exactly one batch left + break; + } + + // Push the final batch + if !packages.is_empty() { + batches.push(packages); + } + + // Sort packages within batches so that `--continue-from` work consistently + for batch in batches.iter_mut() { + batch.sort(); + } + Ok(batches) +} + +#[cfg(test)] +mod tests { + use super::*; + use semver::Version; + use std::path::PathBuf; + + fn version(version: &str) -> Version { + Version::parse(version).unwrap() + } + + #[test] + fn read_package_success() { + let manifest = br#" + [package] + name = "test" + version = "1.2.0-preview" + + [build-dependencies] + build_something = "1.3" + local_build_something = { version = "0.2.0", path = "../local_build_something" } + + [dev-dependencies] + dev_something = "1.1" + local_dev_something = { version = "0.1.0", path = "../local_dev_something" } + + [dependencies] + something = "1.0" + local_something = { version = "1.1.3", path = "../local_something" } + "#; + let path: PathBuf = "test/Cargo.toml".into(); + + let package = read_package(&path, manifest).expect("parse success"); + assert_eq!("test", package.handle.name); + assert_eq!(version("1.2.0-preview"), package.handle.version); + + let mut expected = BTreeSet::new(); + expected.insert(PackageHandle::new( + "local_build_something", + version("0.2.0"), + )); + expected.insert(PackageHandle::new("local_dev_something", version("0.1.0"))); + expected.insert(PackageHandle::new("local_something", version("1.1.3"))); + assert_eq!(expected, package.local_dependencies); + } + + #[test] + fn read_package_version_requirement_invalid() { + let manifest = br#" + [package] + name = "test" + version = "1.2.0-preview" + + [dependencies] + local_something = { version = "1.0", path = "../local_something" } + "#; + let path: PathBuf = "test/Cargo.toml".into(); + + let error = format!( + "{}", + read_package(&path, manifest).err().expect("should fail") + ); + assert!( + error.contains("Invalid crate version"), + "'{}' should contain 'Invalid crate version'", + error + ); + } + + fn package(name: &str, dependencies: &[&str]) -> Package { + Package::new( + PackageHandle::new(name, Version::parse("1.0.0").unwrap()), + format!("{}/Cargo.toml", name), + dependencies + .iter() + .map(|d| PackageHandle::new(*d, Version::parse("1.0.0").unwrap())) + .collect(), + ) + } + + fn fmt_batches(batches: Vec) -> String { + let mut result = String::new(); + for batch in batches { + result.push_str( + &batch + .iter() + .map(|p| p.handle.name.as_str()) + .collect::>() + .join(","), + ); + result.push(';'); + } + result + } + + #[test] + fn test_batch_packages() { + assert_eq!("", fmt_batches(batch_packages(vec![]).unwrap())); + assert_eq!( + "A;", + fmt_batches(batch_packages(vec![package("A", &[])]).unwrap()) + ); + assert_eq!( + "A,B;", + fmt_batches(batch_packages(vec![package("A", &[]), package("B", &[])]).unwrap()) + ); + assert_eq!( + "A,B;C;", + fmt_batches( + batch_packages(vec![ + package("C", &["A", "B"]), + package("B", &[]), + package("A", &[]), + ]) + .unwrap() + ) + ); + assert_eq!( + "A,B;C,D,F;E;", + fmt_batches( + batch_packages(vec![ + package("A", &[]), + package("B", &[]), + package("C", &["A"]), + package("D", &["A", "B"]), + package("F", &["B"]), + package("E", &["C", "D", "F"]), + ]) + .unwrap() + ) + ); + assert_eq!( + "A,F;B;C;E,G;D,H,I;", + fmt_batches( + batch_packages(vec![ + package("F", &[]), + package("G", &["C"]), + package("I", &["G"]), + package("H", &["G"]), + package("D", &["B", "C"]), + package("E", &["C"]), + package("C", &["B"]), + package("A", &[]), + package("B", &["A"]), + ]) + .unwrap() + ) + ); + } + + fn pkg_ver(name: &str, version: &str, dependencies: &[(&str, &str)]) -> Package { + Package::new( + PackageHandle::new(name, Version::parse(version).unwrap()), + format!("{}/Cargo.toml", name), + dependencies + .iter() + .map(|p| PackageHandle::new(p.0, Version::parse(p.1).unwrap())) + .collect(), + ) + } + + #[test] + fn test_validate_packages() { + validate_packages(&vec![ + pkg_ver("A", "1.0.0", &[]), + pkg_ver("B", "1.1.0", &[]), + pkg_ver("C", "1.2.0", &[("A", "1.0.0"), ("B", "1.1.0")]), + pkg_ver("D", "1.3.0", &[("A", "1.0.0")]), + pkg_ver("F", "1.4.0", &[("B", "1.1.0")]), + pkg_ver( + "E", + "1.5.0", + &[("C", "1.2.0"), ("D", "1.3.0"), ("F", "1.4.0")], + ), + ]) + .expect("success"); + + let error = validate_packages(&vec![ + pkg_ver("A", "1.1.0", &[]), + pkg_ver("B", "1.1.0", &[]), + pkg_ver("C", "1.2.0", &[("A", "1.1.0"), ("B", "1.1.0")]), + pkg_ver("D", "1.3.0", &[("A", "1.0.0")]), + pkg_ver("F", "1.4.0", &[("B", "1.1.0")]), + pkg_ver( + "E", + "1.5.0", + &[("C", "1.2.0"), ("D", "1.3.0"), ("F", "1.4.0")], + ), + ]) + .err() + .expect("fail"); + assert_eq!( + "crate A has multiple versions: 1.1.0 and 1.0.0", + format!("{}", error) + ); + } + + #[test] + fn test_continue_batches_from() { + let mut batches = vec![ + vec![ + pkg_ver("aws-a", "1.0.0", &[]), + pkg_ver("aws-b", "1.1.0", &[]), + ], + vec![ + pkg_ver("aws-smithy-c", "1.0.0", &[]), + pkg_ver("aws-smithy-d", "1.1.0", &[]), + ], + vec![ + pkg_ver("aws-sdk-e", "1.0.0", &[]), + pkg_ver("aws-sdk-f", "1.1.0", &[]), + ], + ]; + let mut stats = PackageStats::default(); + continue_batches_from("aws-smithy-d", &mut batches, &mut stats).unwrap(); + + assert_eq!( + vec![ + vec![pkg_ver("aws-smithy-d", "1.1.0", &[])], + vec![ + pkg_ver("aws-sdk-e", "1.0.0", &[]), + pkg_ver("aws-sdk-f", "1.1.0", &[]) + ], + ], + batches + ); + assert_eq!( + PackageStats { + smithy_runtime_crates: 1, + aws_runtime_crates: 0, + aws_sdk_crates: 2, + }, + stats + ); + } + + #[test] + fn test_continue_batches_from_package_not_found() { + let mut batches = vec![vec![ + pkg_ver("aws-a", "1.0.0", &[]), + pkg_ver("aws-b", "1.1.0", &[]), + ]]; + let mut stats = PackageStats::default(); + assert!(continue_batches_from("does-not-exist", &mut batches, &mut stats).is_err()); + + let mut batches = vec![]; + assert!(continue_batches_from("does-not-exist", &mut batches, &mut stats).is_err()); + } +} diff --git a/tools/publisher/src/repo.rs b/tools/publisher/src/repo.rs new file mode 100644 index 000000000..265ede659 --- /dev/null +++ b/tools/publisher/src/repo.rs @@ -0,0 +1,51 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0. + */ + +//! Local filesystem git repository discovery. This enables the tool to +//! orient itself despite being run anywhere from within the git repo. + +use anyhow::Result; +use std::env; +use std::ffi::OsStr; +use std::path::{Path, PathBuf}; + +/// Git repository containing crates to be published. +#[derive(Debug)] +pub struct Repository { + pub root: PathBuf, + pub crates_root: PathBuf, +} + +#[derive(Debug, thiserror::Error)] +pub enum Error { + #[error("failed to find {0} repository root")] + RepositoryRootNotFound(String), +} + +/// Attempts to find git repository root from current working directory. +pub fn discover_repository(name: &str, crate_path: &str) -> Result { + let mut current_dir = env::current_dir()?.canonicalize()?; + let os_name = OsStr::new(name); + loop { + if is_git_root(¤t_dir) { + if let Some(file_name) = current_dir.file_name() { + if os_name == file_name { + return Ok(Repository { + crates_root: current_dir.join(crate_path), + root: current_dir, + }); + } + } + return Err(Error::RepositoryRootNotFound(name.into()).into()); + } else if !current_dir.pop() { + return Err(Error::RepositoryRootNotFound(name.into()).into()); + } + } +} + +fn is_git_root(path: &Path) -> bool { + let path = path.join(".git"); + path.exists() && path.is_dir() +} diff --git a/tools/publisher/src/sort.rs b/tools/publisher/src/sort.rs new file mode 100644 index 000000000..637fe6527 --- /dev/null +++ b/tools/publisher/src/sort.rs @@ -0,0 +1,125 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0. + */ + +//! Logic for topological sorting packages by dependencies. + +use crate::package::{Package, PackageHandle}; +use anyhow::Result; +use std::collections::{BTreeMap, BTreeSet}; + +/// Determines the dependency order of the given packages. +pub fn dependency_order(packages: Vec) -> Result> { + let mut order = Vec::new(); + let mut packages: BTreeMap = packages + .into_iter() + .map(|p| (p.handle.clone(), p)) + .collect(); + let mut visited = BTreeSet::new(); + + let mut to_visit: Vec<&Package> = packages.iter().map(|e| e.1).collect(); + to_visit.sort_by(|a, b| { + (*a).local_dependencies + .len() + .cmp(&(*b).local_dependencies.len()) + }); + + // Depth-first search topological sort + while let Some(package) = to_visit.iter().find(|e| !visited.contains(&e.handle)) { + dependency_order_visit( + &package.handle, + &packages, + &mut BTreeSet::new(), + &mut visited, + &mut order, + )?; + } + + Ok(order + .into_iter() + .map(&mut |handle| packages.remove(&handle).unwrap()) + .collect()) +} + +#[derive(Debug, thiserror::Error)] +enum Error { + #[error("dependency cycle detected")] + DependencyCycle, +} + +fn dependency_order_visit( + package_handle: &PackageHandle, + packages: &BTreeMap, + stack: &mut BTreeSet, + visited: &mut BTreeSet, + result: &mut Vec, +) -> Result<(), Error> { + visited.insert(package_handle.clone()); + stack.insert(package_handle.clone()); + + let local_dependencies = &packages[package_handle].local_dependencies; + for dependency in local_dependencies { + if visited.contains(dependency) && stack.contains(dependency) { + return Err(Error::DependencyCycle); + } + if package_handle != dependency + && packages.contains_key(dependency) + && !visited.contains(dependency) + { + dependency_order_visit(dependency, packages, stack, visited, result)?; + } + } + result.push(package_handle.clone()); + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + use semver::Version; + + fn package(name: &str, dependencies: &[&str]) -> Package { + Package::new( + PackageHandle::new(name, Version::parse("1.0.0").unwrap()), + format!("{}/Cargo.toml", name), + dependencies + .iter() + .map(|d| PackageHandle::new(*d, Version::parse("1.0.0").unwrap())) + .collect(), + ) + } + + #[test] + pub fn test_dependency_order() { + let packages = vec![ + package("E", &["B", "C", "A"]), + package("B", &[]), + package("F", &["E", "D"]), + package("C", &["A"]), + package("A", &[]), + package("D", &["C"]), + ]; + + let result = dependency_order(packages).unwrap(); + assert_eq!( + "ABCDEF", + result.iter().fold(String::new(), |mut acc, p| { + acc.push_str(&p.handle.name); + acc + }) + ); + } + + #[test] + pub fn test_dependency_cycles() { + let packages = vec![ + package("A", &["C"]), + package("B", &["A"]), + package("C", &["B"]), + ]; + + let error = dependency_order(packages).err().expect("cycle"); + assert_eq!("dependency cycle detected", format!("{}", error)); + } +} diff --git a/tools/publisher/src/subcommand/fix_manifests.rs b/tools/publisher/src/subcommand/fix_manifests.rs new file mode 100644 index 000000000..4faa2f1bd --- /dev/null +++ b/tools/publisher/src/subcommand/fix_manifests.rs @@ -0,0 +1,260 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0. + */ + +//! Subcommand for fixing manifest dependency version numbers. +//! +//! Finds all of the version numbers for every crate in the repo crate path, and then +//! finds all references to the crates in that path and updates them to have the correct +//! version numbers in addition to the dependency path. + +use crate::fs::Fs; +use crate::package::{discover_package_manifests, parse_version}; +use anyhow::{bail, Context, Result}; +use semver::Version; +use std::collections::BTreeMap; +use std::env; +use std::path::{Path, PathBuf}; +use toml::value::Table; +use tracing::info; + +#[derive(Debug, Copy, Clone, Eq, PartialEq)] +pub enum Mode { + Check, + Execute, +} + +pub async fn subcommand_fix_manifests(mode: Mode, location: &str) -> Result<()> { + let manifest_paths = discover_package_manifests(location.into()).await?; + let mut manifests = read_manifests(Fs::Real, manifest_paths).await?; + let versions = package_versions(&manifests)?; + fix_manifests(Fs::Real, &versions, &mut manifests, mode).await?; + Ok(()) +} + +struct Manifest { + path: PathBuf, + metadata: toml::Value, +} + +async fn read_manifests(fs: Fs, manifest_paths: Vec) -> Result> { + let mut result = Vec::new(); + for path in manifest_paths { + let contents = fs.read_file(&path).await?; + let metadata = toml::from_slice(&contents) + .with_context(|| format!("failed to load package manifest for {:?}", &path))?; + result.push(Manifest { path, metadata }); + } + Ok(result) +} + +/// Returns a map of crate name to semver version number +fn package_versions(manifests: &[Manifest]) -> Result> { + let mut versions = BTreeMap::new(); + for manifest in manifests { + let name = manifest.metadata["package"]["name"] + .as_str() + .ok_or_else(|| { + anyhow::Error::msg(format!("{:?} is missing a package name", manifest.path)) + })?; + let version = manifest.metadata["package"]["version"] + .as_str() + .ok_or_else(|| { + anyhow::Error::msg(format!("{:?} is missing a package version", manifest.path)) + })?; + let version = parse_version(&manifest.path, version)?; + versions.insert(name.into(), version); + } + Ok(versions) +} + +fn fix_dep_set( + versions: &BTreeMap, + key: &str, + metadata: &mut toml::Value, +) -> Result { + let mut changed = 0; + if let Some(dependencies) = metadata.as_table_mut().unwrap().get_mut(key) { + if let Some(dependencies) = dependencies.as_table_mut() { + for (dep_name, dep) in dependencies.iter_mut() { + changed += match dep.as_table_mut() { + None => { + if !dep.is_str() { + bail!("unexpected dependency (must be table or string): {:?}", dep) + } + 0 + } + Some(ref mut table) => update_dep(table, dep_name, versions)?, + }; + } + } + } + Ok(changed) +} + +fn update_dep( + table: &mut Table, + dep_name: &str, + versions: &BTreeMap, +) -> Result { + if !table.contains_key("path") { + return Ok(0); + } + let package_version = match versions.get(dep_name) { + Some(version) => version.to_string(), + None => bail!("version not found for crate {}", dep_name), + }; + let previous_version = table.insert( + "version".into(), + toml::Value::String(package_version.to_string()), + ); + match previous_version { + None => Ok(1), + Some(prev_version) if prev_version.as_str() == Some(&package_version) => Ok(0), + Some(mismatched_version) => { + tracing::warn!(expected = ?package_version, actual = ?mismatched_version, "version was set but it did not match"); + Ok(1) + } + } +} + +fn fix_dep_sets(versions: &BTreeMap, metadata: &mut toml::Value) -> Result { + let mut changed = fix_dep_set(versions, "dependencies", metadata)?; + changed += fix_dep_set(versions, "dev-dependencies", metadata)?; + changed += fix_dep_set(versions, "build-dependencies", metadata)?; + Ok(changed) +} + +fn block_local_publish(manifest_path: &Path, metadata: &mut toml::Value) -> Result { + // Safe-guard to prevent accidental publish to crates.io. Add some friction + // to publishing from a local development machine by detecting that the tool + // is not being run from CI, and disallow publish in that case. + if env::var("GITHUB_ACTIONS").unwrap_or_default() != "true" { + if let Some(package) = metadata.as_table_mut().unwrap().get_mut("package") { + info!( + "Detected local build. Disallowing publish for {:?}.", + manifest_path + ); + package + .as_table_mut() + .unwrap() + .insert("publish".into(), toml::Value::Boolean(false)); + return Ok(true); + } + } + Ok(false) +} + +async fn fix_manifests( + fs: Fs, + versions: &BTreeMap, + manifests: &mut Vec, + mode: Mode, +) -> Result<()> { + for manifest in manifests { + let package_changed = block_local_publish(&manifest.path, &mut manifest.metadata)?; + let dependencies_changed = fix_dep_sets(versions, &mut manifest.metadata)?; + if package_changed || dependencies_changed > 0 { + let contents = + "# Code generated by software.amazon.smithy.rust.codegen.smithy-rs. DO NOT EDIT.\n" + .to_string() + + &toml::to_string(&manifest.metadata).with_context(|| { + format!("failed to serialize to toml for {:?}", manifest.path) + })?; + match mode { + Mode::Execute => { + fs.write_file(&manifest.path, contents.as_bytes()).await?; + info!( + "Changed {} dependencies in {:?}.", + dependencies_changed, manifest.path + ); + } + Mode::Check => { + bail!( + "{manifest:?} contained invalid versions", + manifest = manifest.path + ) + } + } + } + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_fix_dep_sets() { + let manifest = br#" + [package] + name = "test" + version = "1.2.0-preview" + + [build-dependencies] + build_something = "1.3" + local_build_something = { path = "../local_build_something", version = "0.4.0-different" } + + [dev-dependencies] + dev_something = "1.1" + local_dev_something = { path = "../local_dev_something" } + + [dependencies] + something = "1.0" + local_something = { path = "../local_something" } + "#; + let metadata = toml::from_slice(manifest).unwrap(); + let mut manifest = Manifest { + path: "test".into(), + metadata, + }; + let versions = vec![ + ("local_build_something", "0.2.0"), + ("local_dev_something", "0.1.0"), + ("local_something", "1.1.3"), + ] + .into_iter() + .map(|e| (e.0.to_string(), Version::parse(e.1).unwrap())) + .collect(); + + fix_dep_sets(&versions, &mut manifest.metadata).expect("success"); + + let actual_deps = &manifest.metadata["dependencies"]; + assert_eq!( + "\ + something = \"1.0\"\n\ + \n\ + [local_something]\n\ + path = \"../local_something\"\n\ + version = \"1.1.3\"\n\ + ", + actual_deps.to_string() + ); + + let actual_dev_deps = &manifest.metadata["dev-dependencies"]; + assert_eq!( + "\ + dev_something = \"1.1\"\n\ + \n\ + [local_dev_something]\n\ + path = \"../local_dev_something\"\n\ + version = \"0.1.0\"\n\ + ", + actual_dev_deps.to_string() + ); + + let actual_build_deps = &manifest.metadata["build-dependencies"]; + assert_eq!( + "\ + build_something = \"1.3\"\n\ + \n\ + [local_build_something]\n\ + path = \"../local_build_something\"\n\ + version = \"0.2.0\"\n\ + ", + actual_build_deps.to_string() + ); + } +} diff --git a/tools/publisher/src/subcommand/mod.rs b/tools/publisher/src/subcommand/mod.rs new file mode 100644 index 000000000..6ecfc0027 --- /dev/null +++ b/tools/publisher/src/subcommand/mod.rs @@ -0,0 +1,8 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0. + */ + +pub mod fix_manifests; +pub mod publish; +pub mod yank_category; diff --git a/tools/publisher/src/subcommand/publish.rs b/tools/publisher/src/subcommand/publish.rs new file mode 100644 index 000000000..6c55ef15e --- /dev/null +++ b/tools/publisher/src/subcommand/publish.rs @@ -0,0 +1,183 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0. + */ + +use crate::cargo::{self, CargoOperation}; +use crate::fs::Fs; +use crate::package::{ + continue_batches_from, discover_package_batches, Package, PackageBatch, PackageHandle, + PackageStats, +}; +use crate::CRATE_OWNER; +use anyhow::Result; +use crates_io_api::{AsyncClient, Error}; +use dialoguer::Confirm; +use lazy_static::lazy_static; +use std::sync::Arc; +use std::time::Duration; +use tokio::sync::Semaphore; +use tracing::info; + +lazy_static! { + static ref CRATES_IO_CLIENT: AsyncClient = AsyncClient::new( + "AWS_RUST_SDK_PUBLISHER (aws-sdk-rust@amazon.com)", + Duration::from_secs(1) + ) + .expect("valid client"); +} + +pub async fn subcommand_publish(location: &str, continue_from: Option<&str>) -> Result<()> { + // Make sure cargo exists + cargo::confirm_installed_on_path()?; + + info!("Discovering crates to publish..."); + let (mut batches, mut stats) = discover_package_batches(Fs::Real, &location).await?; + if let Some(continue_from) = continue_from { + info!( + "Filtering batches so that publishing starts from {}.", + continue_from + ); + continue_batches_from(continue_from, &mut batches, &mut stats)?; + } + info!("Finished crate discovery."); + + // Don't proceed unless the user confirms the plan + confirm_plan(&batches, stats)?; + + // Use a semaphore to only allow a few concurrent publishes + let max_concurrency = num_cpus::get_physical(); + let semaphore = Arc::new(Semaphore::new(max_concurrency)); + info!( + "Will publish {} crates in parallel where possible.", + max_concurrency + ); + for batch in batches { + let mut tasks = Vec::new(); + for package in batch { + let permit = semaphore.clone().acquire_owned().await.unwrap(); + tasks.push(tokio::spawn(async move { + // Only publish if it hasn't been published yet. + if !is_published(&package.handle).await? { + info!("Publishing `{}`...", package.handle); + cargo::Publish::new(&package.handle, &package.crate_path) + .spawn() + .await?; + // Sometimes it takes a little bit of time for the new package version + // to become available after publish. If we proceed too quickly, then + // the next package publish can fail if it depends on this package. + wait_for_eventual_consistency(&package).await?; + info!("Successfully published `{}`", package.handle); + } else { + info!("`{}` was already published", package.handle); + } + correct_owner(&package).await?; + drop(permit); + Ok::<_, anyhow::Error>(()) + })); + } + for task in tasks { + task.await??; + } + info!("sleeping 30 seconds after completion of the batch"); + tokio::time::sleep(Duration::from_secs(30)).await; + } + + Ok(()) +} + +async fn is_published(handle: &PackageHandle) -> Result { + let expected_version = handle.version.to_string(); + let crate_info = match CRATES_IO_CLIENT.get_crate(&handle.name).await { + Ok(info) => info, + Err(Error::NotFound(_)) => return Ok(false), + Err(other) => return Err(other.into()), + }; + Ok(crate_info + .versions + .iter() + .any(|crate_version| crate_version.num == expected_version)) +} + +/// Waits for the given package to show up on crates.io +async fn wait_for_eventual_consistency(package: &Package) -> Result<()> { + let max_wait_time = 10usize; + for _ in 0..max_wait_time { + if !is_published(&package.handle).await? { + tokio::time::sleep(Duration::from_secs(1)).await; + } else { + return Ok(()); + } + } + if !is_published(&package.handle).await? { + return Err(anyhow::Error::msg(format!( + "package wasn't found on crates.io {} seconds after publish", + max_wait_time + ))); + } + Ok(()) +} + +/// Corrects the crate ownership. +async fn correct_owner(package: &Package) -> Result<()> { + let owners = cargo::GetOwners::new(&package.handle.name).spawn().await?; + if !owners.iter().any(|owner| owner == CRATE_OWNER) { + cargo::AddOwner::new(&package.handle.name, CRATE_OWNER) + .spawn() + .await?; + info!("Corrected crate ownership of `{}`", package.handle); + } + Ok(()) +} + +fn confirm_plan(batches: &[PackageBatch], stats: PackageStats) -> Result<()> { + let mut full_plan = Vec::new(); + for batch in batches { + for package in batch { + full_plan.push( + cargo::Publish::new(&package.handle, &package.crate_path) + .plan() + .unwrap(), + ); + } + full_plan.push("wait".into()); + } + + info!("Publish plan:"); + for item in full_plan { + println!(" {}", item); + } + info!( + "Will publish {} crates total ({} Smithy runtime, {} AWS runtime, {} AWS SDK).", + stats.total(), + stats.smithy_runtime_crates, + stats.aws_runtime_crates, + stats.aws_sdk_crates + ); + + if Confirm::new() + .with_prompt("Continuing will publish to crates.io. Do you wish to continue?") + .interact()? + { + Ok(()) + } else { + Err(anyhow::Error::msg("aborted")) + } +} + +#[cfg(test)] +mod test { + + use super::*; + use crate::package::PackageHandle; + + #[ignore] + #[tokio::test] + async fn crate_published_works() { + let handle = PackageHandle::new("aws-smithy-http", "0.27.0-alpha.1".parse().unwrap()); + assert_eq!(is_published(&handle).await.expect("failed"), true); + // we will never publish this version + let handle = PackageHandle::new("aws-smithy-http", "0.21.0-alpha.1".parse().unwrap()); + assert_eq!(is_published(&handle).await.expect("failed"), false); + } +} diff --git a/tools/publisher/src/subcommand/yank_category.rs b/tools/publisher/src/subcommand/yank_category.rs new file mode 100644 index 000000000..66d9bca78 --- /dev/null +++ b/tools/publisher/src/subcommand/yank_category.rs @@ -0,0 +1,104 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0. + */ + +use crate::cargo::{self, CargoOperation}; +use crate::fs::Fs; +use crate::package::{discover_package_batches, Package, PackageCategory, PackageHandle}; +use crate::repo::discover_repository; +use crate::{REPO_CRATE_PATH, REPO_NAME}; +use anyhow::{Context, Result}; +use dialoguer::Confirm; +use semver::Version; +use std::sync::Arc; +use tokio::sync::Semaphore; +use tracing::info; + +const MAX_CONCURRENCY: usize = 5; + +pub async fn subcommand_yank_category(category: &str, version: &str) -> Result<()> { + let category = match category { + "aws-runtime" => PackageCategory::AwsRuntime, + "aws-sdk" => PackageCategory::AwsSdk, + "smithy-runtime" => PackageCategory::SmithyRuntime, + _ => { + return Err(anyhow::Error::msg(format!( + "unrecognized package category: {}", + category + ))); + } + }; + let version = Version::parse(version).context("failed to parse inputted version number")?; + + // Make sure cargo exists + cargo::confirm_installed_on_path()?; + + info!("Discovering crates to yank..."); + let repo = discover_repository(REPO_NAME, REPO_CRATE_PATH)?; + let (batches, _) = discover_package_batches(Fs::Real, &repo.crates_root).await?; + let packages: Vec = batches + .into_iter() + .flatten() + .filter(|p| p.category == category) + .map(|p| { + Package::new( + // Replace the version with the version given on the CLI + PackageHandle::new(p.handle.name, version.clone()), + p.manifest_path, + p.local_dependencies, + ) + }) + .collect(); + info!("Finished crate discovery."); + + // Don't proceed unless the user confirms the plan + confirm_plan(&packages)?; + + // Use a semaphore to only allow a few concurrent yanks + let semaphore = Arc::new(Semaphore::new(MAX_CONCURRENCY)); + info!( + "Will yank {} crates in parallel where possible.", + MAX_CONCURRENCY + ); + + let mut tasks = Vec::new(); + for package in packages { + let permit = semaphore.clone().acquire_owned().await.unwrap(); + tasks.push(tokio::spawn(async move { + info!("Yanking `{}`...", package.handle); + let result = cargo::Yank::new(&package.handle, &package.crate_path) + .spawn() + .await; + drop(permit); + info!("Successfully yanked `{}`", package.handle); + result + })); + } + for task in tasks { + task.await??; + } + + Ok(()) +} + +fn confirm_plan(packages: &[Package]) -> Result<()> { + info!("Yank plan:"); + for package in packages { + println!( + " {}", + cargo::Yank::new(&package.handle, &package.crate_path) + .plan() + .unwrap() + ); + } + + if Confirm::new() + .with_prompt("Continuing will yank crate versions from crates.io. Do you wish to continue?") + .interact()? + { + Ok(()) + } else { + Err(anyhow::Error::msg("aborted")) + } +} -- GitLab