Produce two publishable bundles in CI (#986) (17818ca7) · Commits · Public Repositories / Smithy Rust

.github/workflows/ci-sdk.yaml

+1 −1

Original line number	Diff line number	Diff line
		@@ -62,7 +62,7 @@ jobs:
		matrix:
		test:
		- name: Unit Tests
		run: cargo test $(cat service-with-tests)
		run: cargo test --all-features
		- name: Docs
		run: cargo doc --no-deps --document-private-items
		- name: Clippy

.github/workflows/ci.yaml

+57 −9

Original line number	Diff line number	Diff line
		@@ -41,6 +41,8 @@ jobs:
		test:
		- name: Kotlin Style
		run: ./gradlew ktlint
		- name: BuildSrc Tests
		run: ./gradlew -p buildSrc test
		- name: Client Unit Tests
		run: ./gradlew :codegen:test
		- name: SDK Unit Tests
		@@ -80,13 +82,22 @@ jobs:
		- name: ${{ matrix.test.name }}
		run: ${{ matrix.test.run }}

		runtime-tests:
		name: Rust Runtime Tests
		rust-tests:
		name: Rust Tests
		runs-on: ${{ matrix.os }}
		strategy:
		matrix:
		os: [ubuntu-latest, windows-latest]
		runtime: [., ./aws]
		runs-on: ${{ matrix.os }}
		runtime: [rust-runtime, aws/rust-runtime, tools/publisher]
		exclude:
		# Don't need to test the publisher tool on Windows
		- os: windows-latest
		runtime: tools/publisher
		env:
		# Disable incremental compilation to reduce disk space use
		CARGO_INCREMENTAL: 0
		RUSTDOCFLAGS: -D warnings
		RUSTFLAGS: -D warnings
		steps:
		- uses: actions/checkout@v2
		# Pinned to the commit hash of v1.3.0
		@@ -105,14 +116,51 @@ jobs:
		if: ${{ matrix.os == 'ubuntu-latest' }}
		- name: clippy check
		run: cargo clippy -- -D warnings
		working-directory: ${{ matrix.runtime }}/rust-runtime/
		working-directory: ${{ matrix.runtime }}
		# don't bother running Clippy twice, it will have the same results on Windows
		if: ${{ matrix.os == 'ubuntu-latest' }}
		- name: run tests
		run: cargo test --all-features
		working-directory: ${{ matrix.runtime }}/rust-runtime/
		working-directory: ${{ matrix.runtime }}
		- name: generate docs
		run: cargo doc --no-deps --document-private-items --all-features
		working-directory: ${{ matrix.runtime }}/rust-runtime/
		env:
		RUSTDOCFLAGS: -D warnings
		working-directory: ${{ matrix.runtime }}

		# Psuedo-job that depends on the rust-tests job so that we don't have to enter
		# the myriad of test matrix combinations into GitHub's protected branch rules
		require-rust-tests:
		needs: rust-tests
		runs-on: ubuntu-latest
		name: Rust Tests Matrix Success
		steps:
		- name: Run
		run: echo "We should only get this far if the rust-tests matrix succeeded."

		runtime-bundle:
		name: Produce smithy-rs runtime bundle
		runs-on: ubuntu-latest
		steps:
		- uses: actions/checkout@v2
		- uses: actions/cache@v2
		name: Gradle Cache
		with:
		path: \|
		~/.gradle/caches
		~/.gradle/wrapper
		key: ${{ runner.os }}-gradle-${{ hashFiles('*/.gradle*') }}
		restore-keys: \|
		${{ runner.os }}-gradle-
		# Pinned to the commit hash of v1.3.0
		- name: Set up JDK
		uses: actions/setup-java@v1
		with:
		java-version: ${{ env.java_version }}
		- name: Produce bundle
		run: \|
		./gradlew rust-runtime:assemble
		tar cfvz smithy-rs-runtime.tar.gz -C rust-runtime/build smithy-rs
		- uses: actions/upload-artifact@v2
		name: Upload bundle
		with:
		name: smithy-rs-runtime-${{ github.sha }}
		path: smithy-rs-runtime.tar.gz

aws/sdk/build.gradle.kts

+61 −121

Original line number	Diff line number	Diff line
		@@ -3,7 +3,11 @@
		* SPDX-License-Identifier: Apache-2.0.
		*/

		import java.util.Properties
		import aws.sdk.AwsServices
		import aws.sdk.Membership
		import aws.sdk.discoverServices
		import aws.sdk.docsLandingPage
		import aws.sdk.parseMembership

		extra["displayName"] = "Smithy :: Rust :: AWS-SDK"
		extra["moduleName"] = "software.amazon.smithy.rust.awssdk"
		@@ -15,34 +19,12 @@ plugins {
		}

		val smithyVersion: String by project
		val properties = PropertyRetriever(rootProject, project)

		val outputDir = buildDir.resolve("aws-sdk")
		val sdkOutputDir = outputDir.resolve("sdk")
		val examplesOutputDir = outputDir.resolve("examples")

		val runtimeModules = listOf(
		"aws-smithy-async",
		"aws-smithy-client",
		"aws-smithy-eventstream",
		"aws-smithy-http",
		"aws-smithy-http-tower",
		"aws-smithy-json",
		"aws-smithy-protocol-test",
		"aws-smithy-query",
		"aws-smithy-types",
		"aws-smithy-types-convert",
		"aws-smithy-xml"
		)
		val awsModules = listOf(
		"aws-config",
		"aws-endpoint",
		"aws-http",
		"aws-hyper",
		"aws-sig-auth",
		"aws-sigv4",
		"aws-types"
		)

		buildscript {
		val smithyVersion: String by project
		dependencies {
		@@ -59,37 +41,19 @@ dependencies {
		implementation("software.amazon.smithy:smithy-aws-cloudformation-traits:$smithyVersion")
		}

		// get a project property by name if it exists (including from local.properties)
		fun getProperty(name: String): String? {
		if (project.hasProperty(name)) {
		return project.properties[name].toString()
		}

		val localProperties = Properties()
		val propertiesFile: File = rootProject.file("local.properties")
		if (propertiesFile.exists()) {
		propertiesFile.inputStream().use { localProperties.load(it) }

		if (localProperties.containsKey(name)) {
		return localProperties[name].toString()
		}
		}
		return null
		}

		// Class and functions for service and protocol membership for SDK generation

		val awsServices: List<AwsService> by lazy { discoverServices(loadServiceMembership()) }
		val awsServices: AwsServices by lazy { discoverServices(loadServiceMembership()) }
		val eventStreamAllowList: Set<String> by lazy { eventStreamAllowList() }

		fun loadServiceMembership(): Membership {
		val membershipOverride = getProperty("aws.services")?.let { parseMembership(it) }
		val membershipOverride = properties.get("aws.services")?.let { parseMembership(it) }
		println(membershipOverride)
		val fullSdk =
		parseMembership(getProperty("aws.services.fullsdk") ?: throw kotlin.Exception("full sdk list missing"))
		parseMembership(properties.get("aws.services.fullsdk") ?: throw kotlin.Exception("full sdk list missing"))
		val tier1 =
		parseMembership(getProperty("aws.services.smoketest") ?: throw kotlin.Exception("smoketest list missing"))
		return membershipOverride ?: if ((getProperty("aws.fullsdk") ?: "") == "true") {
		parseMembership(properties.get("aws.services.smoketest") ?: throw kotlin.Exception("smoketest list missing"))
		return membershipOverride ?: if ((properties.get("aws.fullsdk") ?: "") == "true") {
		fullSdk
		} else {
		tier1
		@@ -97,12 +61,12 @@ fun loadServiceMembership(): Membership {
		}

		fun eventStreamAllowList(): Set<String> {
		val list = getProperty("aws.services.eventstream.allowlist") ?: ""
		val list = properties.get("aws.services.eventstream.allowlist") ?: ""
		return list.split(",").map { it.trim() }.toSet()
		}

		fun generateSmithyBuild(services: List<AwsService>): String {
		val serviceProjections = services.map { service ->
		fun generateSmithyBuild(services: AwsServices): String {
		val serviceProjections = services.services.map { service ->
		val files = service.files().map { extraFile ->
		software.amazon.smithy.utils.StringUtils.escapeJavaString(
		extraFile.absolutePath,
		@@ -127,7 +91,7 @@ fun generateSmithyBuild(services: List<AwsService>): String {
		},
		"service": "${service.service}",
		"module": "aws-sdk-${service.module}",
		"moduleVersion": "${getProperty("aws.sdk.version")}",
		"moduleVersion": "${properties.get("aws.sdk.version")}",
		"moduleAuthors": ["AWS Rust SDK Team <aws-sdk-rust@amazon.com>", "Russell Cohen <rcoh@amazon.com>"],
		"moduleDescription": "${service.moduleDescription}",
		${service.examplesUri(project)?.let { """"examples": "$it",""" } ?: ""}
		@@ -150,7 +114,7 @@ fun generateSmithyBuild(services: List<AwsService>): String {

		task("generateSmithyBuild") {
		description = "generate smithy-build.json"
		inputs.property("servicelist", awsServices.sortedBy { it.module }.toString())
		inputs.property("servicelist", awsServices.services.toString())
		inputs.property("eventStreamAllowList", eventStreamAllowList)
		inputs.dir(projectDir.resolve("aws-models"))
		outputs.file(projectDir.resolve("smithy-build.json"))
		@@ -161,7 +125,7 @@ task("generateSmithyBuild") {
		}

		task("generateDocs") {
		inputs.property("servicelist", awsServices.sortedBy { it.module }.toString())
		inputs.property("servicelist", awsServices.services.toString())
		outputs.file(outputDir.resolve("docs.md"))
		doLast {
		project.docsLandingPage(awsServices, outputDir)
		@@ -171,7 +135,7 @@ task("generateDocs") {
		task("relocateServices") {
		description = "relocate AWS services to their final destination"
		doLast {
		awsServices.forEach {
		awsServices.services.forEach {
		logger.info("Relocating ${it.module}...")
		copy {
		from("$buildDir/smithyprojections/sdk/${it.module}/rust-codegen")
		@@ -198,7 +162,9 @@ task("relocateExamples") {
		doLast {
		copy {
		from(projectDir)
		include("examples/**")
		awsServices.examples.forEach { example ->
		include("$example/**")
		}
		into(outputDir)
		exclude("**/target")
		filter { line -> line.replace("build/aws-sdk/sdk/", "sdk/") }
		@@ -219,36 +185,12 @@ fun rewritePathDependency(line: String): String {
		.replace("../../rust-runtime/", "")
		}

		fun rewriteCrateVersion(line: String, version: String): String = line.replace(
		"""^\s*version\s+=\s+"0.0.0-smithy-rs-head"$""".toRegex(),
		"version = \"$version\""
		)

		/**
		* AWS runtime crate versions are all `0.0.0-smithy-rs-head`. When copying over to the AWS SDK,
		* these should be changed to the AWS SDK version.
		*/
		fun rewriteAwsSdkCrateVersion(line: String): String = rewriteCrateVersion(line, getProperty("aws.sdk.version")!!)

		/**
		* Smithy runtime crate versions in smithy-rs are all `0.0.0-smithy-rs-head`. When copying over to the AWS SDK,
		* these should be changed to the smithy-rs version.
		*/
		fun rewriteSmithyRsCrateVersion(line: String): String =
		rewriteCrateVersion(line, getProperty("smithy.rs.runtime.crate.version")!!)

		/** Patches a file with the result of the given `operation` being run on each line */
		fun patchFile(path: File, operation: (String) -> String) {
		val patchedContents = path.readLines().joinToString("\n", transform = operation)
		path.writeText(patchedContents)
		}

		tasks.register<Copy>("copyAllRuntimes") {
		from("$rootDir/aws/rust-runtime") {
		awsModules.forEach { include("$it/**") }
		CrateSet.AWS_SDK_RUNTIME.forEach { include("$it/**") }
		}
		from("$rootDir/rust-runtime") {
		runtimeModules.forEach { include("$it/**") }
		CrateSet.AWS_SDK_SMITHY_RUNTIME.forEach { include("$it/**") }
		}
		exclude("**/target")
		exclude("**/Cargo.lock")
		@@ -260,10 +202,9 @@ tasks.register("relocateAwsRuntime") {
		dependsOn("copyAllRuntimes")
		doLast {
		// Patch the Cargo.toml files
		awsModules.forEach { moduleName ->
		CrateSet.AWS_SDK_RUNTIME.forEach { moduleName ->
		patchFile(sdkOutputDir.resolve("$moduleName/Cargo.toml")) { line ->
		line.let(::rewritePathDependency)
		.let(::rewriteAwsSdkCrateVersion)
		rewriteAwsSdkCrateVersion(properties, line.let(::rewritePathDependency))
		}
		}
		}
		@@ -272,57 +213,51 @@ tasks.register("relocateRuntime") {
		dependsOn("copyAllRuntimes")
		doLast {
		// Patch the Cargo.toml files
		runtimeModules.forEach { moduleName ->
		CrateSet.AWS_SDK_SMITHY_RUNTIME.forEach { moduleName ->
		patchFile(sdkOutputDir.resolve("$moduleName/Cargo.toml")) { line ->
		line.let(::rewriteSmithyRsCrateVersion)
		rewriteSmithyRsCrateVersion(properties, line)
		}
		}
		}
		}

		fun generateCargoWorkspace(services: List<AwsService>): String {
		val generatedModules = services.map { it.module }.toSet()
		val examples = projectDir.resolve("examples")
		.listFiles { file -> !file.name.startsWith(".") }.orEmpty().toList()
		.filter { file ->
		val cargoToml = File(file, "Cargo.toml")
		if (cargoToml.exists()) {
		val usedModules = cargoToml.readLines()
		.map { line -> line.substringBefore('=').trim() }
		.filter { line -> line.startsWith("aws-sdk-") }
		.map { line -> line.substringAfter("aws-sdk-") }
		.toSet()
		generatedModules.containsAll(usedModules)
		} else {
		false
		}
		}
		.map { "examples/${it.name}" }

		val modules = (
		services.map(AwsService::module).map { "sdk/$it" } +
		runtimeModules.map { "sdk/$it" } +
		awsModules.map { "sdk/$it" } +
		examples.toList()
		).sorted()
		fun generateCargoWorkspace(services: AwsServices): String {
		return """
		\|[workspace]
		\|members = [${"\n"}${modules.joinToString(",\n") { "\| \"$it\"" }}
		\|members = [${"\n"}${services.allModules.joinToString(",\n") { "\| \"$it\"" }}
		\|]
		""".trimMargin()
		}

		task("generateCargoWorkspace") {
		description = "generate Cargo.toml workspace file"
		doFirst {
		outputDir.mkdirs()
		outputDir.resolve("Cargo.toml").writeText(generateCargoWorkspace(awsServices))
		}
		inputs.property("servicelist", awsServices.sortedBy { it.module }.toString())
		inputs.property("servicelist", awsServices.moduleNames.toString())
		inputs.dir(projectDir.resolve("examples"))
		outputs.file(outputDir.resolve("Cargo.toml"))
		outputs.upToDateWhen { false }
		}

		tasks.register<Exec>("fixManifests") {
		description = "Run the publisher tool's `fix-manifests` sub-command on the generated services"

		val publisherPath = rootProject.projectDir.resolve("tools/publisher")
		inputs.dir(publisherPath)
		outputs.dir(outputDir)

		workingDir(publisherPath)
		commandLine("cargo", "run", "--", "fix-manifests", "--location", outputDir.absolutePath)

		dependsOn("assemble")
		dependsOn("relocateServices")
		dependsOn("relocateRuntime")
		dependsOn("relocateAwsRuntime")
		dependsOn("relocateExamples")
		}

		task("finalizeSdk") {
		dependsOn("assemble")
		outputs.upToDateWhen { false }
		@@ -331,17 +266,22 @@ task("finalizeSdk") {
		"relocateRuntime",
		"relocateAwsRuntime",
		"relocateExamples",
		"generateDocs"
		"generateDocs",
		"fixManifests"
		)
		}

		tasks["smithyBuildJar"].inputs.file(projectDir.resolve("smithy-build.json"))
		tasks["smithyBuildJar"].inputs.dir(projectDir.resolve("aws-models"))
		tasks["smithyBuildJar"].dependsOn("generateSmithyBuild")
		tasks["smithyBuildJar"].dependsOn("generateCargoWorkspace")
		tasks["smithyBuildJar"].outputs.upToDateWhen { false }
		tasks["assemble"].dependsOn("smithyBuildJar")
		tasks["assemble"].finalizedBy("finalizeSdk")
		tasks["smithyBuildJar"].apply {
		inputs.file(projectDir.resolve("smithy-build.json"))
		inputs.dir(projectDir.resolve("aws-models"))
		dependsOn("generateSmithyBuild")
		dependsOn("generateCargoWorkspace")
		outputs.upToDateWhen { false }
		}
		tasks["assemble"].apply {
		dependsOn("smithyBuildJar")
		finalizedBy("finalizeSdk")
		}

		tasks.register<Exec>("cargoCheck") {
		workingDir(outputDir)

aws/sdk/examples/iam/Cargo.toml

+1 −1

Original line number	Diff line number	Diff line
		@@ -8,6 +8,6 @@ edition = "2018"

		[dependencies]
		aws-config = { path = "../../build/aws-sdk/sdk/aws-config" }
		iam = { package = "aws-sdk-iam", path = "../../build/aws-sdk/sdk/iam" }
		aws-sdk-iam = { path = "../../build/aws-sdk/sdk/iam" }
		tokio = { version = "1", features = ["full"] }
		tracing-subscriber = "0.2.18"

aws/sdk/examples/iam/src/bin/iam-hello-world.rs

+2 −0

Original line number	Diff line number	Diff line
		@@ -3,6 +3,8 @@
		* SPDX-License-Identifier: Apache-2.0.
		*/

		use aws_sdk_iam as iam;

		#[tokio::main]
		async fn main() -> Result<(), iam::Error> {
		tracing_subscriber::fmt::init();