diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 19da5c9e3d2739937bc4205d049f529520b0d888..00a725106a7b069933eb9fe105b9367f8a593b5c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -58,6 +58,8 @@ jobs: - action: check-sdk-codegen-unit-tests - action: check-server-codegen-integration-tests - action: check-server-codegen-unit-tests + - action: check-server-codegen-integration-tests-python + - action: check-server-codegen-unit-tests-python - action: check-server-e2e-test - action: check-style-and-lints steps: diff --git a/ci.mk b/ci.mk index 74c29569f1d63693009b9f603b068aff2a975e15..860facf4b214e068de794894371ce44d27a349c9 100644 --- a/ci.mk +++ b/ci.mk @@ -59,6 +59,14 @@ check-server-codegen-integration-tests: check-server-codegen-unit-tests: $(CI_ACTION) $@ +.PHONY: check-server-codegen-integration-tests-python +check-server-codegen-integration-tests-python: + $(CI_ACTION) $@ + +.PHONY: check-server-codegen-unit-tests-python +check-server-codegen-unit-tests-python: + $(CI_ACTION) $@ + .PHONY: check-server-e2e-test check-server-e2e-test: $(CI_ACTION) $@ diff --git a/codegen-server-test/build.gradle.kts b/codegen-server-test/build.gradle.kts index ce9ca740157c8cfb7bb63e9002a07a6606ba925f..bbb5aba6e33c5513d60191c073f2ef6f2d8a9b14 100644 --- a/codegen-server-test/build.gradle.kts +++ b/codegen-server-test/build.gradle.kts @@ -3,6 +3,7 @@ * SPDX-License-Identifier: Apache-2.0 */ +description = "Generates Rust code from Smithy models and runs the protocol tests" extra["displayName"] = "Smithy :: Rust :: Codegen :: Server :: Test" extra["moduleName"] = "software.amazon.smithy.rust.kotlin.codegen.server.test" diff --git a/codegen-server-test/python/build.gradle.kts b/codegen-server-test/python/build.gradle.kts new file mode 100644 index 0000000000000000000000000000000000000000..9bc81a5f24790166763ca4ad55d668de433e14aa --- /dev/null +++ b/codegen-server-test/python/build.gradle.kts @@ -0,0 +1,107 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +description = "Generates Rust/Python code from Smithy models and runs the protocol tests" +extra["displayName"] = "Smithy :: Rust :: Codegen :: Server :: Python :: Test" +extra["moduleName"] = "software.amazon.smithy.rust.kotlin.codegen.server.python.test" + +tasks["jar"].enabled = false + +plugins { id("software.amazon.smithy") } + +val smithyVersion: String by project +val defaultRustFlags: String by project +val defaultRustDocFlags: String by project +val properties = PropertyRetriever(rootProject, project) + +val pluginName = "rust-server-codegen" +val workingDirUnderBuildDir = "smithyprojections/codegen-server-test-python/" + +configure { + outputDirectory = file("$buildDir/$workingDirUnderBuildDir") +} + +buildscript { + val smithyVersion: String by project + dependencies { + classpath("software.amazon.smithy:smithy-cli:$smithyVersion") + } +} + +dependencies { + implementation(project(":codegen-server:python")) + implementation("software.amazon.smithy:smithy-aws-protocol-tests:$smithyVersion") + implementation("software.amazon.smithy:smithy-protocol-test-traits:$smithyVersion") + implementation("software.amazon.smithy:smithy-aws-traits:$smithyVersion") +} + +val allCodegenTests = listOf( + CodegenTest("com.amazonaws.simple#SimpleService", "simple"), + CodegenTest("aws.protocoltests.restjson#RestJson", "rest_json"), + CodegenTest("aws.protocoltests.restjson.validation#RestJsonValidation", "rest_json_validation"), + CodegenTest("aws.protocoltests.json10#JsonRpc10", "json_rpc10"), + CodegenTest("aws.protocoltests.json#JsonProtocol", "json_rpc11"), + CodegenTest("aws.protocoltests.misc#MiscService", "misc"), + CodegenTest("com.amazonaws.ebs#Ebs", "ebs"), + CodegenTest("com.amazonaws.s3#AmazonS3", "s3"), + CodegenTest("com.aws.example#PokemonService", "pokemon_service_sdk") +) + +task("generateSmithyBuild") { + description = "generate smithy-build.json" + doFirst { + projectDir.resolve("smithy-build.json") + .writeText( + generateSmithyBuild( + rootProject.projectDir.absolutePath, + pluginName, + codegenTests(properties, allCodegenTests) + ) + ) + } +} + +task("generateCargoWorkspace") { + description = "generate Cargo.toml workspace file" + doFirst { + buildDir.resolve("$workingDirUnderBuildDir/Cargo.toml") + .writeText(generateCargoWorkspace(pluginName, codegenTests(properties, allCodegenTests))) + } +} + +tasks["smithyBuildJar"].dependsOn("generateSmithyBuild") +tasks["assemble"].finalizedBy("generateCargoWorkspace") + +tasks.register(Cargo.CHECK.toString) { + workingDir("$buildDir/$workingDirUnderBuildDir") + environment("RUSTFLAGS", defaultRustFlags) + commandLine("cargo", "check") + dependsOn("assemble") +} + +tasks.register(Cargo.TEST.toString) { + workingDir("$buildDir/$workingDirUnderBuildDir") + environment("RUSTFLAGS", defaultRustFlags) + commandLine("cargo", "test") + dependsOn("assemble") +} + +tasks.register(Cargo.DOCS.toString) { + workingDir("$buildDir/$workingDirUnderBuildDir") + environment("RUSTDOCFLAGS", defaultRustDocFlags) + commandLine("cargo", "doc", "--no-deps") + dependsOn("assemble") +} + +tasks.register(Cargo.CLIPPY.toString) { + workingDir("$buildDir/$workingDirUnderBuildDir") + environment("RUSTFLAGS", defaultRustFlags) + commandLine("cargo", "clippy") + dependsOn("assemble") +} + +tasks["test"].finalizedBy(cargoCommands(properties).map { it.toString }) + +tasks["clean"].doFirst { delete("smithy-build.json") } diff --git a/codegen-server-test/python/model b/codegen-server-test/python/model new file mode 120000 index 0000000000000000000000000000000000000000..fe0e728f649c965c2d46391090ac2ba74e188015 --- /dev/null +++ b/codegen-server-test/python/model @@ -0,0 +1 @@ +../model \ No newline at end of file diff --git a/codegen-server/build.gradle.kts b/codegen-server/build.gradle.kts index c1893db763a721da8c981cd5fc470a308f602c55..315bf3c44f4b4b183bae328e9e06ccb707189ede 100644 --- a/codegen-server/build.gradle.kts +++ b/codegen-server/build.gradle.kts @@ -7,8 +7,6 @@ import org.gradle.api.tasks.testing.logging.TestExceptionFormat plugins { kotlin("jvm") - id("org.jetbrains.dokka") - jacoco maven `maven-publish` } @@ -27,22 +25,14 @@ val smithyVersion: String by project val kotestVersion: String by project dependencies { - implementation(kotlin("stdlib-jdk8")) - api("software.amazon.smithy:smithy-codegen-core:$smithyVersion") - api("com.moandjiezana.toml:toml4j:0.7.2") implementation(project(":codegen")) implementation("software.amazon.smithy:smithy-aws-traits:$smithyVersion") implementation("software.amazon.smithy:smithy-protocol-test-traits:$smithyVersion") - implementation("software.amazon.smithy:smithy-waiters:$smithyVersion") - runtimeOnly(project(":rust-runtime")) testImplementation("org.junit.jupiter:junit-jupiter:5.6.1") testImplementation("io.kotest:kotest-assertions-core-jvm:$kotestVersion") } -// unlike the client-runtime, software.amazon.smithy.rust.codegen.smithy-kotlin codegen package is -// not expected to run on Android...we can target 1.8 tasks.compileKotlin { kotlinOptions.jvmTarget = "1.8" } - tasks.compileTestKotlin { kotlinOptions.jvmTarget = "1.8" } // Reusable license copySpec @@ -77,26 +67,6 @@ tasks.test { } } -tasks.dokka { - outputFormat = "html" - outputDirectory = "$buildDir/javadoc" -} - -// Always build documentation -tasks["build"].finalizedBy(tasks["dokka"]) - -// Configure jacoco (code coverage) to generate an HTML report -tasks.jacocoTestReport { - reports { - xml.isEnabled = false - csv.isEnabled = false - html.destination = file("$buildDir/reports/jacoco") - } -} - -// Always run the jacoco test report after testing. -tasks["test"].finalizedBy(tasks["jacocoTestReport"]) - publishing { publications { create("default") { diff --git a/codegen-server/python/build.gradle.kts b/codegen-server/python/build.gradle.kts new file mode 100644 index 0000000000000000000000000000000000000000..c042410caf3242b62849f9367fdc4109e6dc4c67 --- /dev/null +++ b/codegen-server/python/build.gradle.kts @@ -0,0 +1,79 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +import org.gradle.api.tasks.testing.logging.TestExceptionFormat + +plugins { + kotlin("jvm") + maven + `maven-publish` +} + +description = "Generates Rust/Python server-side code from Smithy models" + +extra["displayName"] = "Smithy :: Rust :: Codegen :: Server :: Python" + +extra["moduleName"] = "software.amazon.smithy.rust.codegen.server.python" + +group = "software.amazon.smithy.rust.codegen.server.python.smithy" + +version = "0.1.0" + +val smithyVersion: String by project +val kotestVersion: String by project + +dependencies { + implementation(project(":codegen")) + implementation(project(":codegen-server")) + implementation("software.amazon.smithy:smithy-aws-traits:$smithyVersion") + implementation("software.amazon.smithy:smithy-protocol-test-traits:$smithyVersion") + testImplementation("org.junit.jupiter:junit-jupiter:5.6.1") + testImplementation("io.kotest:kotest-assertions-core-jvm:$kotestVersion") +} + +tasks.compileKotlin { kotlinOptions.jvmTarget = "1.8" } +tasks.compileTestKotlin { kotlinOptions.jvmTarget = "1.8" } + +// Reusable license copySpec +val licenseSpec = copySpec { + from("${project.rootDir}/LICENSE") + from("${project.rootDir}/NOTICE") +} + +// Configure jars to include license related info +tasks.jar { + metaInf.with(licenseSpec) + inputs.property("moduleName", project.name) + manifest { attributes["Automatic-Module-Name"] = project.name } +} + +val sourcesJar by tasks.creating(Jar::class) { + group = "publishing" + description = "Assembles Kotlin sources jar" + classifier = "sources" + from(sourceSets.getByName("main").allSource) +} + +tasks.test { + useJUnitPlatform() + testLogging { + events("passed", "skipped", "failed") + exceptionFormat = TestExceptionFormat.FULL + showCauses = true + showExceptions = true + showStackTraces = true + showStandardStreams = true + } +} + +publishing { + publications { + create("default") { + from(components["java"]) + artifact(sourcesJar) + } + } + repositories { maven { url = uri("$buildDir/repository") } } +} diff --git a/codegen-server/python/src/main/kotlin/software/amazon/smithy/rust/codegen/server/python/smithy/RustCodegenServerPlugin.kt b/codegen-server/python/src/main/kotlin/software/amazon/smithy/rust/codegen/server/python/smithy/RustCodegenServerPlugin.kt new file mode 100644 index 0000000000000000000000000000000000000000..1bd6ac5405b35a8d394a3b037654f8dd3c90f36d --- /dev/null +++ b/codegen-server/python/src/main/kotlin/software/amazon/smithy/rust/codegen/server/python/smithy/RustCodegenServerPlugin.kt @@ -0,0 +1,78 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +package software.amazon.smithy.rust.codegen.server.python.smithy + +import software.amazon.smithy.build.PluginContext +import software.amazon.smithy.build.SmithyBuildPlugin +import software.amazon.smithy.codegen.core.ReservedWordSymbolProvider +import software.amazon.smithy.model.Model +import software.amazon.smithy.model.shapes.ServiceShape +import software.amazon.smithy.rust.codegen.rustlang.RustReservedWordSymbolProvider +import software.amazon.smithy.rust.codegen.server.smithy.ServerCodegenVisitor +import software.amazon.smithy.rust.codegen.smithy.BaseSymbolMetadataProvider +import software.amazon.smithy.rust.codegen.smithy.DefaultConfig +import software.amazon.smithy.rust.codegen.smithy.EventStreamSymbolProvider +import software.amazon.smithy.rust.codegen.smithy.StreamingShapeMetadataProvider +import software.amazon.smithy.rust.codegen.smithy.StreamingShapeSymbolProvider +import software.amazon.smithy.rust.codegen.smithy.SymbolVisitor +import software.amazon.smithy.rust.codegen.smithy.SymbolVisitorConfig +import software.amazon.smithy.rust.codegen.smithy.customize.CombinedCodegenDecorator +import java.util.logging.Level +import java.util.logging.Logger + +/** Rust with Python bindings Codegen Plugin. + * This is the entrypoint for code generation, triggered by the smithy-build plugin. + * `resources/META-INF.services/software.amazon.smithy.build.SmithyBuildPlugin` refers to this class by name which + * enables the smithy-build plugin to invoke `execute` with all of the Smithy plugin context + models. + */ +class RustCodegenServerPlugin : SmithyBuildPlugin { + private val logger = Logger.getLogger(javaClass.name) + + override fun getName(): String = "rust-server-codegen-python" + + override fun execute(context: PluginContext) { + // Suppress extremely noisy logs about reserved words + Logger.getLogger(ReservedWordSymbolProvider::class.java.name).level = Level.OFF + // Discover [RustCodegenDecorators] on the classpath. [RustCodegenDectorator] return different types of + // customization. A customization is a function of: + // - location (e.g. the mutate section of an operation) + // - context (e.g. the of the operation) + // - writer: The active RustWriter at the given location + val codegenDecorator = CombinedCodegenDecorator.fromClasspath(context) + + // ServerCodegenVisitor is the main driver of code generation that traverses the model and generates code + logger.info("Loaded plugin to generate Rust/Python bindings for the server SSDK") + ServerCodegenVisitor(context, codegenDecorator).execute() + } + + companion object { + /** SymbolProvider + * When generating code, smithy types need to be converted into Rust types—that is the core role of the symbol provider + * + * The Symbol provider is composed of a base [SymbolVisitor] which handles the core functionality, then is layered + * with other symbol providers, documented inline, to handle the full scope of Smithy types. + */ + fun baseSymbolProvider( + model: Model, + serviceShape: ServiceShape, + symbolVisitorConfig: SymbolVisitorConfig = DefaultConfig + ) = + SymbolVisitor(model, serviceShape = serviceShape, config = symbolVisitorConfig) + // Generate different types for EventStream shapes (e.g. transcribe streaming) + .let { + EventStreamSymbolProvider(symbolVisitorConfig.runtimeConfig, it, model) + } + // Generate [ByteStream] instead of `Blob` for streaming binary shapes (e.g. S3 GetObject) + .let { StreamingShapeSymbolProvider(it, model) } + // Add Rust attributes (like `#[derive(PartialEq)]`) to generated shapes + .let { BaseSymbolMetadataProvider(it, additionalAttributes = listOf()) } + // Streaming shapes need different derives (e.g. they cannot derive Eq) + .let { StreamingShapeMetadataProvider(it, model) } + // Rename shapes that clash with Rust reserved words & and other SDK specific features e.g. `send()` cannot + // be the name of an operation input + .let { RustReservedWordSymbolProvider(it, model) } + } +} diff --git a/codegen-server/python/src/main/resources/META-INF/services/software.amazon.smithy.build.SmithyBuildPlugin b/codegen-server/python/src/main/resources/META-INF/services/software.amazon.smithy.build.SmithyBuildPlugin new file mode 100644 index 0000000000000000000000000000000000000000..ea7c4546af81e06c65fad71cf02ddce550372d2e --- /dev/null +++ b/codegen-server/python/src/main/resources/META-INF/services/software.amazon.smithy.build.SmithyBuildPlugin @@ -0,0 +1,5 @@ +# +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +# +software.amazon.smithy.rust.codegen.server.python.smithy.RustCodegenServerPlugin diff --git a/codegen-server/src/main/kotlin/software/amazon/smithy/rust/codegen/server/smithy/RustCodegenServerPlugin.kt b/codegen-server/src/main/kotlin/software/amazon/smithy/rust/codegen/server/smithy/RustCodegenServerPlugin.kt index 76dcbfd13ed10e3c91c0fa6929be38b65e0df4d9..d0f840e8d0816b6de0585a552f5255ba226e8e21 100644 --- a/codegen-server/src/main/kotlin/software/amazon/smithy/rust/codegen/server/smithy/RustCodegenServerPlugin.kt +++ b/codegen-server/src/main/kotlin/software/amazon/smithy/rust/codegen/server/smithy/RustCodegenServerPlugin.kt @@ -43,6 +43,7 @@ class RustCodegenServerPlugin : SmithyBuildPlugin { val codegenDecorator = CombinedCodegenDecorator.fromClasspath(context) // ServerCodegenVisitor is the main driver of code generation that traverses the model and generates code + logger.info("Loaded plugin to generate pure Rust bindings for the server SSDK") ServerCodegenVisitor(context, codegenDecorator).execute() } diff --git a/settings.gradle.kts b/settings.gradle.kts index 529a0b3e4bc750e9795d0fed78e29bb3d37fd5e8..d2576c79391fd7640c0949aab576ff636092e019 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -9,7 +9,9 @@ enableFeaturePreview("GRADLE_METADATA") include(":codegen") include(":codegen-test") include(":codegen-server") +include(":codegen-server:python") include(":codegen-server-test") +include(":codegen-server-test:python") include(":rust-runtime") include(":aws:sdk-codegen") include(":aws:sdk-codegen-test") diff --git a/tools/ci-build/scripts/check-server-codegen-integration-tests-python b/tools/ci-build/scripts/check-server-codegen-integration-tests-python new file mode 100755 index 0000000000000000000000000000000000000000..d0ea58026ffb9bef69b785cae06e1f04b2e142b6 --- /dev/null +++ b/tools/ci-build/scripts/check-server-codegen-integration-tests-python @@ -0,0 +1,9 @@ +#!/bin/bash +# +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +# + +set -eux +cd smithy-rs +./gradlew codegen-server-test:python:test diff --git a/tools/ci-build/scripts/check-server-codegen-unit-tests-python b/tools/ci-build/scripts/check-server-codegen-unit-tests-python new file mode 100755 index 0000000000000000000000000000000000000000..0338f4d6866ace1248d22ede20ed9c0ee36f00ee --- /dev/null +++ b/tools/ci-build/scripts/check-server-codegen-unit-tests-python @@ -0,0 +1,9 @@ +#!/bin/bash +# +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +# + +set -eux +cd smithy-rs +./gradlew codegen-server:python:test diff --git a/tools/codegen-diff-revisions.py b/tools/codegen-diff-revisions.py index 0e12b68d3054885f6177cb7bc8b8b1d58db8272e..0f5a90a1ad5ebea7bed3b8a568c1836dd7aec3a4 100755 --- a/tools/codegen-diff-revisions.py +++ b/tools/codegen-diff-revisions.py @@ -94,18 +94,24 @@ def generate_and_commit_generated_code(revision_sha): # Generate code run("./gradlew --rerun-tasks :aws:sdk:assemble") run("./gradlew --rerun-tasks :codegen-server-test:assemble") + run("./gradlew --rerun-tasks :codegen-server-test:python:assemble") # Move generated code into codegen-diff/ directory run(f"rm -rf {OUTPUT_PATH}") run(f"mkdir {OUTPUT_PATH}") run(f"mv aws/sdk/build/aws-sdk {OUTPUT_PATH}") run(f"mv codegen-server-test/build/smithyprojections/codegen-server-test {OUTPUT_PATH}") + run(f"mv codegen-server-test/python/build/smithyprojections/codegen-server-test-python {OUTPUT_PATH}") # Clean up the server-test folder run(f"rm -rf {OUTPUT_PATH}/codegen-server-test/source") + run(f"rm -rf {OUTPUT_PATH}/codegen-server-test-python/source") run(f"find {OUTPUT_PATH}/codegen-server-test | " f"grep -E 'smithy-build-info.json|sources/manifest|model.json' | " f"xargs rm -f", shell=True) + run(f"find {OUTPUT_PATH}/codegen-server-test-python | " + f"grep -E 'smithy-build-info.json|sources/manifest|model.json' | " + f"xargs rm -f", shell=True) run(f"git add -f {OUTPUT_PATH}") run(f"git -c 'user.name=GitHub Action (generated code preview)' " @@ -187,15 +193,22 @@ def make_diffs(base_commit_sha, head_commit_sha): head_commit_sha, "server-test", whitespace=True) server_nows = make_diff("Server Test", f"{OUTPUT_PATH}/codegen-server-test", base_commit_sha, head_commit_sha, "server-test-ignore-whitespace", whitespace=False) + server_ws_python = make_diff("Server Test Python", f"{OUTPUT_PATH}/codegen-server-test-python", base_commit_sha, + head_commit_sha, "server-test-python", whitespace=True) + server_nows_python = make_diff("Server Test Python", f"{OUTPUT_PATH}/codegen-server-test-python", base_commit_sha, + head_commit_sha, "server-test-python-ignore-whitespace", whitespace=False) sdk_links = diff_link('AWS SDK', 'No codegen difference in the AWS SDK', sdk_ws, 'ignoring whitespace', sdk_nows) server_links = diff_link('Server Test', 'No codegen difference in the Server Test', server_ws, 'ignoring whitespace', server_nows) + server_links_python = diff_link('Server Test Python', 'No codegen difference in the Server Test Python', + server_ws_python, 'ignoring whitespace', server_nows_python) # Save escaped newlines so that the GitHub Action script gets the whole message return "A new generated diff is ready to view.\\n"\ f"- {sdk_links}\\n"\ - f"- {server_links}\\n" + f"- {server_links}\\n"\ + f"- {server_links_python}\\n" def write_to_file(path, text):