diff --git a/codegen-server-test/python/build.gradle.kts b/codegen-server-test/python/build.gradle.kts index 9bc81a5f24790166763ca4ad55d668de433e14aa..51d7971a82b3c87d72b6241fdbfc49bf69697d71 100644 --- a/codegen-server-test/python/build.gradle.kts +++ b/codegen-server-test/python/build.gradle.kts @@ -16,7 +16,7 @@ val defaultRustFlags: String by project val defaultRustDocFlags: String by project val properties = PropertyRetriever(rootProject, project) -val pluginName = "rust-server-codegen" +val pluginName = "rust-server-codegen-python" val workingDirUnderBuildDir = "smithyprojections/codegen-server-test-python/" configure { @@ -39,13 +39,6 @@ dependencies { val allCodegenTests = listOf( CodegenTest("com.amazonaws.simple#SimpleService", "simple"), - CodegenTest("aws.protocoltests.restjson#RestJson", "rest_json"), - CodegenTest("aws.protocoltests.restjson.validation#RestJsonValidation", "rest_json_validation"), - CodegenTest("aws.protocoltests.json10#JsonRpc10", "json_rpc10"), - CodegenTest("aws.protocoltests.json#JsonProtocol", "json_rpc11"), - CodegenTest("aws.protocoltests.misc#MiscService", "misc"), - CodegenTest("com.amazonaws.ebs#Ebs", "ebs"), - CodegenTest("com.amazonaws.s3#AmazonS3", "s3"), CodegenTest("com.aws.example#PokemonService", "pokemon_service_sdk") ) diff --git a/codegen-server/python/src/main/kotlin/software/amazon/smithy/rust/codegen/server/python/smithy/RustCodegenServerPlugin.kt b/codegen-server/python/src/main/kotlin/software/amazon/smithy/rust/codegen/server/python/smithy/PythonCodegenServerPlugin.kt similarity index 80% rename from codegen-server/python/src/main/kotlin/software/amazon/smithy/rust/codegen/server/python/smithy/RustCodegenServerPlugin.kt rename to codegen-server/python/src/main/kotlin/software/amazon/smithy/rust/codegen/server/python/smithy/PythonCodegenServerPlugin.kt index 1bd6ac5405b35a8d394a3b037654f8dd3c90f36d..58bbc74482986d77fe3a2c7635dcf19e6343991d 100644 --- a/codegen-server/python/src/main/kotlin/software/amazon/smithy/rust/codegen/server/python/smithy/RustCodegenServerPlugin.kt +++ b/codegen-server/python/src/main/kotlin/software/amazon/smithy/rust/codegen/server/python/smithy/PythonCodegenServerPlugin.kt @@ -11,7 +11,6 @@ import software.amazon.smithy.codegen.core.ReservedWordSymbolProvider import software.amazon.smithy.model.Model import software.amazon.smithy.model.shapes.ServiceShape import software.amazon.smithy.rust.codegen.rustlang.RustReservedWordSymbolProvider -import software.amazon.smithy.rust.codegen.server.smithy.ServerCodegenVisitor import software.amazon.smithy.rust.codegen.smithy.BaseSymbolMetadataProvider import software.amazon.smithy.rust.codegen.smithy.DefaultConfig import software.amazon.smithy.rust.codegen.smithy.EventStreamSymbolProvider @@ -23,12 +22,13 @@ import software.amazon.smithy.rust.codegen.smithy.customize.CombinedCodegenDecor import java.util.logging.Level import java.util.logging.Logger -/** Rust with Python bindings Codegen Plugin. - * This is the entrypoint for code generation, triggered by the smithy-build plugin. - * `resources/META-INF.services/software.amazon.smithy.build.SmithyBuildPlugin` refers to this class by name which - * enables the smithy-build plugin to invoke `execute` with all of the Smithy plugin context + models. +/** + * Rust with Python bindings Codegen Plugin. + * This is the entrypoint for code generation, triggered by the smithy-build plugin. + * `resources/META-INF.services/software.amazon.smithy.build.SmithyBuildPlugin` refers to this class by name which + * enables the smithy-build plugin to invoke `execute` with all of the Smithy plugin context + models. */ -class RustCodegenServerPlugin : SmithyBuildPlugin { +class PythonCodegenServerPlugin : SmithyBuildPlugin { private val logger = Logger.getLogger(javaClass.name) override fun getName(): String = "rust-server-codegen-python" @@ -43,9 +43,9 @@ class RustCodegenServerPlugin : SmithyBuildPlugin { // - writer: The active RustWriter at the given location val codegenDecorator = CombinedCodegenDecorator.fromClasspath(context) - // ServerCodegenVisitor is the main driver of code generation that traverses the model and generates code - logger.info("Loaded plugin to generate Rust/Python bindings for the server SSDK") - ServerCodegenVisitor(context, codegenDecorator).execute() + // PythonServerCodegenVisitor is the main driver of code generation that traverses the model and generates code + logger.info("Loaded plugin to generate Rust/Python bindings for the server SSDK for projection ${context.projectionName}") + PythonServerCodegenVisitor(context, codegenDecorator).execute() } companion object { @@ -61,6 +61,9 @@ class RustCodegenServerPlugin : SmithyBuildPlugin { symbolVisitorConfig: SymbolVisitorConfig = DefaultConfig ) = SymbolVisitor(model, serviceShape = serviceShape, config = symbolVisitorConfig) + // Rename a set of symbols that do not implement `PyClass` and have been wrapped in + // `aws_smithy_http_server_python::types`. + .let { PythonServerSymbolProvider(it) } // Generate different types for EventStream shapes (e.g. transcribe streaming) .let { EventStreamSymbolProvider(symbolVisitorConfig.runtimeConfig, it, model) diff --git a/codegen-server/python/src/main/kotlin/software/amazon/smithy/rust/codegen/server/python/smithy/PythonServerCargoDependency.kt b/codegen-server/python/src/main/kotlin/software/amazon/smithy/rust/codegen/server/python/smithy/PythonServerCargoDependency.kt new file mode 100644 index 0000000000000000000000000000000000000000..65c90262d31ec33746944a5b952a0f738bd9076a --- /dev/null +++ b/codegen-server/python/src/main/kotlin/software/amazon/smithy/rust/codegen/server/python/smithy/PythonServerCargoDependency.kt @@ -0,0 +1,29 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +package software.amazon.smithy.rust.codegen.server.python.smithy + +import software.amazon.smithy.rust.codegen.rustlang.CargoDependency +import software.amazon.smithy.rust.codegen.rustlang.CratesIo +import software.amazon.smithy.rust.codegen.smithy.RuntimeConfig + +/** + * Object used *exclusively* in the runtime of the Python server, for separation concerns. + * Analogous to the companion object in [CargoDependency] and [ServerCargoDependency]; see its documentation for details. + * For a dependency that is used in the client, or in both the client and the server, use [CargoDependency] directly. + */ +object PythonServerCargoDependency { + val PyO3: CargoDependency = CargoDependency("pyo3", CratesIo("0.16"), features = setOf("extension-module")) + val PyO3Asyncio: CargoDependency = CargoDependency("pyo3-asyncio", CratesIo("0.16"), features = setOf("attributes", "tokio-runtime")) + val Tokio: CargoDependency = CargoDependency("tokio", CratesIo("1.0"), features = setOf("full")) + val Tracing: CargoDependency = CargoDependency("tracing", CratesIo("0.1")) + val Tower: CargoDependency = CargoDependency("tower", CratesIo("0.4")) + val TowerHttp: CargoDependency = CargoDependency("tower-http", CratesIo("0.3"), features = setOf("trace")) + val Hyper: CargoDependency = CargoDependency("hyper", CratesIo("0.14"), features = setOf("server", "http1", "http2", "tcp", "stream")) + val NumCpus: CargoDependency = CargoDependency("num_cpus", CratesIo("1.13")) + + fun SmithyHttpServer(runtimeConfig: RuntimeConfig) = runtimeConfig.runtimeCrate("http-server") + fun SmithyHttpServerPython(runtimeConfig: RuntimeConfig) = runtimeConfig.runtimeCrate("http-server-python") +} diff --git a/codegen-server/python/src/main/kotlin/software/amazon/smithy/rust/codegen/server/python/smithy/PythonServerCodegenVisitor.kt b/codegen-server/python/src/main/kotlin/software/amazon/smithy/rust/codegen/server/python/smithy/PythonServerCodegenVisitor.kt new file mode 100644 index 0000000000000000000000000000000000000000..9e8679e5a9fbab18c798b6fb67afe708409b59bd --- /dev/null +++ b/codegen-server/python/src/main/kotlin/software/amazon/smithy/rust/codegen/server/python/smithy/PythonServerCodegenVisitor.kt @@ -0,0 +1,131 @@ + +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +package software.amazon.smithy.rust.codegen.server.python.smithy + +import software.amazon.smithy.build.PluginContext +import software.amazon.smithy.model.shapes.ServiceShape +import software.amazon.smithy.model.shapes.StringShape +import software.amazon.smithy.model.shapes.StructureShape +import software.amazon.smithy.model.traits.EnumTrait +import software.amazon.smithy.rust.codegen.server.python.smithy.generators.PythonServerEnumGenerator +import software.amazon.smithy.rust.codegen.server.python.smithy.generators.PythonServerServiceGenerator +import software.amazon.smithy.rust.codegen.server.python.smithy.generators.PythonServerStructureGenerator +import software.amazon.smithy.rust.codegen.server.smithy.ServerCodegenVisitor +import software.amazon.smithy.rust.codegen.server.smithy.protocols.ServerProtocolLoader +import software.amazon.smithy.rust.codegen.smithy.CodegenContext +import software.amazon.smithy.rust.codegen.smithy.DefaultPublicModules +import software.amazon.smithy.rust.codegen.smithy.RustCrate +import software.amazon.smithy.rust.codegen.smithy.SymbolVisitorConfig +import software.amazon.smithy.rust.codegen.smithy.customize.RustCodegenDecorator +import software.amazon.smithy.rust.codegen.smithy.generators.BuilderGenerator +import software.amazon.smithy.rust.codegen.smithy.generators.CodegenTarget +import software.amazon.smithy.rust.codegen.smithy.generators.implBlock +import software.amazon.smithy.rust.codegen.util.getTrait + +/** + * Entrypoint for Python server-side code generation. This class will walk the in-memory model and + * generate all the needed types by calling the accept() function on the available shapes. + * + * This class inherits from [ServerCodegenVisitor] since it uses most of the functionlities of the super class + * and have to override the symbol provider with [PythonServerSymbolProvider]. + */ +class PythonServerCodegenVisitor(context: PluginContext, codegenDecorator: RustCodegenDecorator) : + ServerCodegenVisitor(context, codegenDecorator) { + + init { + val symbolVisitorConfig = + SymbolVisitorConfig( + runtimeConfig = settings.runtimeConfig, + codegenConfig = settings.codegenConfig, + handleRequired = true + ) + val baseModel = baselineTransform(context.model) + val service = settings.getService(baseModel) + val (protocol, generator) = + ServerProtocolLoader( + codegenDecorator.protocols( + service.id, + ServerProtocolLoader.DefaultProtocols + ) + ) + .protocolFor(context.model, service) + protocolGeneratorFactory = generator + model = generator.transformModel(codegenDecorator.transformModel(service, baseModel)) + val baseProvider = PythonCodegenServerPlugin.baseSymbolProvider(model, service, symbolVisitorConfig) + // Override symbolProvider. + symbolProvider = + codegenDecorator.symbolProvider(generator.symbolProvider(model, baseProvider)) + + // Override `codegenContext` which carries the symbolProvider. + codegenContext = CodegenContext(model, symbolProvider, service, protocol, settings, target = CodegenTarget.SERVER) + + // Override `rustCrate` which carries the symbolProvider. + rustCrate = RustCrate(context.fileManifest, symbolProvider, DefaultPublicModules, settings.codegenConfig) + // Override `protocolGenerator` which carries the symbolProvider. + protocolGenerator = protocolGeneratorFactory.buildProtocolGenerator(codegenContext) + } + + /** + * Structure Shape Visitor + * + * For each structure shape, generate: + * - A Rust structure for the shape ([StructureGenerator]). + * - `pyo3::PyClass` trait implementation. + * - A builder for the shape. + * + * This function _does not_ generate any serializers. + */ + override fun structureShape(shape: StructureShape) { + logger.info("[python-server-codegen] Generating a structure $shape") + rustCrate.useShapeWriter(shape) { writer -> + // Use Python specific structure generator that adds the #[pyclass] attribute + // and #[pymethods] implementation. + PythonServerStructureGenerator(model, symbolProvider, writer, shape).render(CodegenTarget.SERVER) + val builderGenerator = + BuilderGenerator(codegenContext.model, codegenContext.symbolProvider, shape) + builderGenerator.render(writer) + writer.implBlock(shape, symbolProvider) { + builderGenerator.renderConvenienceMethod(this) + } + } + } + + /** + * String Shape Visitor + * + * Although raw strings require no code generation, enums are actually [EnumTrait] applied to string shapes. + */ + override fun stringShape(shape: StringShape) { + logger.info("[rust-server-codegen] Generating an enum $shape") + shape.getTrait()?.also { enum -> + rustCrate.useShapeWriter(shape) { writer -> + PythonServerEnumGenerator(model, symbolProvider, writer, shape, enum, codegenContext.runtimeConfig).render() + } + } + } + + /** + * Generate service-specific code for the model: + * - Serializers + * - Deserializers + * - Trait implementations + * - Protocol tests + * - Operation structures + * - Python operation handlers + */ + override fun serviceShape(shape: ServiceShape) { + logger.info("[python-server-codegen] Generating a service $shape") + PythonServerServiceGenerator( + rustCrate, + protocolGenerator, + protocolGeneratorFactory.support(), + protocolGeneratorFactory.protocol(codegenContext).httpBindingResolver, + codegenContext, + ) + .render() + } +} diff --git a/codegen-server/python/src/main/kotlin/software/amazon/smithy/rust/codegen/server/python/smithy/PythonServerRuntimeType.kt b/codegen-server/python/src/main/kotlin/software/amazon/smithy/rust/codegen/server/python/smithy/PythonServerRuntimeType.kt new file mode 100644 index 0000000000000000000000000000000000000000..0e87af702f1839e35fdac3456304a03b79a0e06f --- /dev/null +++ b/codegen-server/python/src/main/kotlin/software/amazon/smithy/rust/codegen/server/python/smithy/PythonServerRuntimeType.kt @@ -0,0 +1,26 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +package software.amazon.smithy.rust.codegen.server.python.smithy + +import software.amazon.smithy.rust.codegen.smithy.RuntimeConfig +import software.amazon.smithy.rust.codegen.smithy.RuntimeType + +/** + * Object used *exclusively* in the runtime of the Python server, for separation concerns. + * Analogous to the companion object in [RuntimeType] and [ServerRuntimeType]; see its documentation for details. + * For a runtime type that is used in the client, or in both the client and the server, use [RuntimeType] directly. + */ +object PythonServerRuntimeType { + + fun SharedSocket(runtimeConfig: RuntimeConfig) = + RuntimeType("SharedSocket", PythonServerCargoDependency.SmithyHttpServerPython(runtimeConfig), "${runtimeConfig.crateSrcPrefix}_http_server_python") + + fun Blob(runtimeConfig: RuntimeConfig) = + RuntimeType("Blob", PythonServerCargoDependency.SmithyHttpServerPython(runtimeConfig), "${runtimeConfig.crateSrcPrefix}_http_server_python::types") + + fun PyError(runtimeConfig: RuntimeConfig) = + RuntimeType("Error", PythonServerCargoDependency.SmithyHttpServerPython(runtimeConfig), "${runtimeConfig.crateSrcPrefix}_http_server_python") +} diff --git a/codegen-server/python/src/main/kotlin/software/amazon/smithy/rust/codegen/server/python/smithy/PythonServerSymbolProvider.kt b/codegen-server/python/src/main/kotlin/software/amazon/smithy/rust/codegen/server/python/smithy/PythonServerSymbolProvider.kt new file mode 100644 index 0000000000000000000000000000000000000000..4dc949958bf1247e89e6a0b56a4d548100e7fd93 --- /dev/null +++ b/codegen-server/python/src/main/kotlin/software/amazon/smithy/rust/codegen/server/python/smithy/PythonServerSymbolProvider.kt @@ -0,0 +1,44 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +package software.amazon.smithy.rust.codegen.server.python.smithy + +import software.amazon.smithy.codegen.core.Symbol +import software.amazon.smithy.model.shapes.Shape +import software.amazon.smithy.rust.codegen.smithy.RuntimeType +import software.amazon.smithy.rust.codegen.smithy.RustSymbolProvider +import software.amazon.smithy.rust.codegen.smithy.WrappingSymbolProvider +import software.amazon.smithy.rust.codegen.smithy.rustType + +/** + * Input / output / error structures can refer to complex types like the ones implemented inside + * `aws_smithy_types` (a good example is `aws_smithy_types::Blob`). + * `aws_smithy_http_server_python::types` wraps those types that do not implement directly the + * `pyo3::PyClass` trait and cannot be shared safely with Python, providing an idiomatic Python / Rust API. + * + * This symbol provider ensures types not implementing `pyo3::PyClass` are swapped with their wrappers from + * `aws_smithy_http_server_python::types`. + */ +class PythonServerSymbolProvider(private val base: RustSymbolProvider) : + WrappingSymbolProvider(base) { + + private val runtimeConfig = config().runtimeConfig + + /** + * Convert a shape to a Symbol. + * + * Swap the shape's symbol if its associated type does not implement `pyo3::PyClass`. + */ + override fun toSymbol(shape: Shape): Symbol { + return when (base.toSymbol(shape).rustType()) { + RuntimeType.Blob(runtimeConfig).toSymbol().rustType() -> { + PythonServerRuntimeType.Blob(runtimeConfig).toSymbol() + } + else -> { + base.toSymbol(shape) + } + } + } +} diff --git a/codegen-server/python/src/main/kotlin/software/amazon/smithy/rust/codegen/server/python/smithy/customizations/PythonServerCodegenDecorator.kt b/codegen-server/python/src/main/kotlin/software/amazon/smithy/rust/codegen/server/python/smithy/customizations/PythonServerCodegenDecorator.kt new file mode 100644 index 0000000000000000000000000000000000000000..be3c3b9b8a137accf3d4d3da16fbcf8e4aa79998 --- /dev/null +++ b/codegen-server/python/src/main/kotlin/software/amazon/smithy/rust/codegen/server/python/smithy/customizations/PythonServerCodegenDecorator.kt @@ -0,0 +1,88 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +package software.amazon.smithy.rust.codegen.server.python.smithy.customizations + +import software.amazon.smithy.rust.codegen.rustlang.Writable +import software.amazon.smithy.rust.codegen.rustlang.docs +import software.amazon.smithy.rust.codegen.rustlang.rust +import software.amazon.smithy.rust.codegen.rustlang.rustBlock +import software.amazon.smithy.rust.codegen.rustlang.writable +import software.amazon.smithy.rust.codegen.server.python.smithy.PythonServerRuntimeType +import software.amazon.smithy.rust.codegen.server.smithy.customizations.AddInternalServerErrorToAllOperationsDecorator +import software.amazon.smithy.rust.codegen.smithy.CodegenContext +import software.amazon.smithy.rust.codegen.smithy.RuntimeConfig +import software.amazon.smithy.rust.codegen.smithy.customize.CombinedCodegenDecorator +import software.amazon.smithy.rust.codegen.smithy.customize.RustCodegenDecorator +import software.amazon.smithy.rust.codegen.smithy.generators.LibRsCustomization +import software.amazon.smithy.rust.codegen.smithy.generators.LibRsSection +import software.amazon.smithy.rust.codegen.smithy.generators.ManifestCustomizations + +/** + * Configure the [lib] section of `Cargo.toml`. + * + * [lib] + * name = "$CRATE_NAME" + * crate-type = ["cdylib"] + */ +class CdylibManifestDecorator : RustCodegenDecorator { + override val name: String = "CdylibDecorator" + override val order: Byte = 0 + + override fun crateManifestCustomizations( + codegenContext: CodegenContext + ): ManifestCustomizations = + mapOf("lib" to mapOf("name" to codegenContext.settings.moduleName, "crate-type" to listOf("cdylib"))) +} + +/** + * Add `pub use aws_smithy_http_server_python::types::$TYPE` to lib.rs. + */ +class PubUsePythonTypes(private val runtimeConfig: RuntimeConfig) : LibRsCustomization() { + override fun section(section: LibRsSection): Writable { + return when (section) { + is LibRsSection.Body -> writable { + docs("Re-exported Python types from supporting crates.") + rustBlock("pub mod python_types") { + rust("pub use #T;", PythonServerRuntimeType.Blob(runtimeConfig).toSymbol()) + } + } + else -> emptySection + } + } +} + +/** + * Decorator applying the customization from [PubUsePythonTypes] class. + */ +class PubUsePythonTypesDecorator : RustCodegenDecorator { + override val name: String = "PubUsePythonTypesDecorator" + override val order: Byte = 0 + + override fun libRsCustomizations( + codegenContext: CodegenContext, + baseCustomizations: List + ): List { + return baseCustomizations + PubUsePythonTypes(codegenContext.runtimeConfig) + } +} + +val DECORATORS = listOf( + /** + * Add the [InternalServerError] error to all operations. + * This is done because the Python interpreter can raise exceptions during execution + */ + AddInternalServerErrorToAllOperationsDecorator(), + // Add the [lib] section to Cargo.toml to configure the generation of the shared library: + CdylibManifestDecorator(), + // Add `pub use` of `aws_smithy_http_server_python::types`. + PubUsePythonTypesDecorator() +) + +// Combined codegen decorator for Python services. +class PythonServerCodegenDecorator : CombinedCodegenDecorator(DECORATORS) { + override val name: String = "PythonServerCodegenDecorator" + override val order: Byte = -1 +} diff --git a/codegen-server/python/src/main/kotlin/software/amazon/smithy/rust/codegen/server/python/smithy/generators/PythonApplicationGenerator.kt b/codegen-server/python/src/main/kotlin/software/amazon/smithy/rust/codegen/server/python/smithy/generators/PythonApplicationGenerator.kt new file mode 100644 index 0000000000000000000000000000000000000000..641e930abad8da3b97ee821b0074953162bff4ee --- /dev/null +++ b/codegen-server/python/src/main/kotlin/software/amazon/smithy/rust/codegen/server/python/smithy/generators/PythonApplicationGenerator.kt @@ -0,0 +1,151 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +package software.amazon.smithy.rust.codegen.server.python.smithy.generators + +import software.amazon.smithy.model.shapes.OperationShape +import software.amazon.smithy.rust.codegen.rustlang.RustWriter +import software.amazon.smithy.rust.codegen.rustlang.asType +import software.amazon.smithy.rust.codegen.rustlang.rustBlockTemplate +import software.amazon.smithy.rust.codegen.rustlang.rustTemplate +import software.amazon.smithy.rust.codegen.server.python.smithy.PythonServerCargoDependency +import software.amazon.smithy.rust.codegen.server.smithy.ServerCargoDependency +import software.amazon.smithy.rust.codegen.smithy.CodegenContext +import software.amazon.smithy.rust.codegen.util.toSnakeCase + +/** + * Generates a Python compatible application and server that can be configured from Python. + * + * Example: + * from pool import DatabasePool + * from my_library import App, OperationInput, OperationOutput + + * @dataclass + * class Context: + * db = DatabasePool() + * + * app = App() + * app.context(Context()) + * + * @app.operation + * def operation(input: OperationInput, ctx: State) -> OperationOutput: + * description = await ctx.db.get_description(input.name) + * return OperationOutput(description) + * + * app.run() + * + * The application holds a mapping between operation names (lowercase, snakecase), + * the context as defined in Python and some task local with the Python event loop + * for the current process. + * + * The application exposes several methods to Python: + * * `App()`: constructor to create an instance of `App`. + * * `run()`: run the application on a number of workers. + * * `context()`: register the context object that is passed to the Python handlers. + * * One register method per operation that can be used as decorator. For example if + * the model has one operation called `RegisterServer`, it will codegenerate a method + * of `App` called `register_service()` that can be used to decorate the Python implementation + * of this operation. + * + * This class also renders the implementation of the `aws_smity_http_server_python::PyServer` trait, + * that abstracts the processes / event loops / workers lifecycles. + */ +class PythonApplicationGenerator( + codegenContext: CodegenContext, + private val operations: List, +) { + private val symbolProvider = codegenContext.symbolProvider + private val runtimeConfig = codegenContext.runtimeConfig + private val codegenScope = + arrayOf( + "SmithyPython" to PythonServerCargoDependency.SmithyHttpServerPython(runtimeConfig).asType(), + "SmithyServer" to ServerCargoDependency.SmithyHttpServer(runtimeConfig).asType(), + "pyo3" to PythonServerCargoDependency.PyO3.asType(), + "pyo3_asyncio" to PythonServerCargoDependency.PyO3Asyncio.asType(), + "tokio" to PythonServerCargoDependency.Tokio.asType(), + "tracing" to PythonServerCargoDependency.Tracing.asType(), + "tower" to PythonServerCargoDependency.Tower.asType(), + "tower_http" to PythonServerCargoDependency.TowerHttp.asType(), + "num_cpus" to PythonServerCargoDependency.NumCpus.asType(), + "hyper" to PythonServerCargoDependency.Hyper.asType() + ) + + fun render(writer: RustWriter) { + writer.rustTemplate( + """ + ##[#{pyo3}::pyclass(extends = #{SmithyPython}::PyApp)] + ##[derive(Debug, Clone)] + pub struct App { } + """, + *codegenScope + ) + + renderPyMethods(writer) + } + + private fun renderPyMethods(writer: RustWriter) { + writer.rustBlockTemplate( + """ + ##[#{pyo3}::pymethods] + impl App + """, + *codegenScope + ) { + rustBlockTemplate( + """ + /// Override the `router()` function of #{SmithyPython}::PyApp allowing to dynamically + /// codegenerate the routes. + pub fn router(self_: #{pyo3}::PyRef<'_, Self>) -> Option<#{pyo3}::PyObject> + """, + *codegenScope + ) { + rustTemplate( + """ + let router = crate::operation_registry::OperationRegistryBuilder::default(); + let sup = self_.as_ref(); + """, + *codegenScope + ) + for (operation in operations) { + val operationName = symbolProvider.toSymbol(operation).name + val name = operationName.toSnakeCase() + rustTemplate( + """ + let locals = sup.locals.clone(); + let handler = sup.handlers.get("$name").expect("Python handler for `{$name}` not found").clone(); + let router = router.$name(move |input, state| { + #{pyo3_asyncio}::tokio::scope(locals.clone(), crate::operation_handler::$name(input, state, handler)) + }); + """, + *codegenScope + ) + } + rustTemplate( + """ + let router: #{SmithyServer}::Router = router.build().expect("Unable to build operation registry").into(); + use #{pyo3}::IntoPy; + Some(#{SmithyPython}::PyRouter(router).into_py(self_.py())) + """, + *codegenScope + ) + } + operations.map { operation -> + val operationName = symbolProvider.toSymbol(operation).name + val name = operationName.toSnakeCase() + rustTemplate( + """ + /// Method to register `$name` Python implementation inside the handlers map. + /// It can be used as a function decorator in Python. + pub fn $name(self_: #{pyo3}::PyRefMut<'_, Self>, func: #{pyo3}::PyObject) -> #{pyo3}::PyResult<()> { + let mut sup = self_.into_super(); + #{pyo3}::Python::with_gil(|py| sup.register_operation(py, "$name", func)) + } + """, + *codegenScope + ) + } + } + } +} diff --git a/codegen-server/python/src/main/kotlin/software/amazon/smithy/rust/codegen/server/python/smithy/generators/PythonServerAttributeUtils.kt b/codegen-server/python/src/main/kotlin/software/amazon/smithy/rust/codegen/server/python/smithy/generators/PythonServerAttributeUtils.kt new file mode 100644 index 0000000000000000000000000000000000000000..f65eb27d104552440f1482983b47693682d5feec --- /dev/null +++ b/codegen-server/python/src/main/kotlin/software/amazon/smithy/rust/codegen/server/python/smithy/generators/PythonServerAttributeUtils.kt @@ -0,0 +1,43 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +package software.amazon.smithy.rust.codegen.server.python.smithy.generators + +import software.amazon.smithy.model.shapes.Shape +import software.amazon.smithy.model.traits.ErrorTrait +import software.amazon.smithy.rust.codegen.rustlang.RustWriter +import software.amazon.smithy.rust.codegen.rustlang.asType +import software.amazon.smithy.rust.codegen.rustlang.rustTemplate +import software.amazon.smithy.rust.codegen.server.python.smithy.PythonServerCargoDependency +import software.amazon.smithy.rust.codegen.util.hasTrait + +/** + * This module contains utilities to render PyO3 attributes. + * + * TODO(https://github.com/awslabs/smithy-rs/issues/1465): Switch to `Attribute.Custom` and get rid of this class. + */ + +private val codegenScope = arrayOf( + "pyo3" to PythonServerCargoDependency.PyO3.asType(), +) + +// Renders #[pyo3::pyclass] attribute, inheriting from `Exception` if the shape has the `ErrorTrait` attached. +fun RustWriter.renderPyClass(shape: Shape) { + if (shape.hasTrait()) { + rustTemplate("##[#{pyo3}::pyclass(extends = #{pyo3}::exceptions::PyException)]", *codegenScope) + } else { + rustTemplate("##[#{pyo3}::pyclass]", *codegenScope) + } +} + +// Renders #[pyo3::pymethods] attribute. +fun RustWriter.renderPyMethods() { + rustTemplate("##[#{pyo3}::pymethods]", *codegenScope) +} + +// Renders #[pyo3(get, set)] attribute. +fun RustWriter.renderPyGetterSetter() { + rustTemplate("##[#{pyo3}(get, set)]", *codegenScope) +} diff --git a/codegen-server/python/src/main/kotlin/software/amazon/smithy/rust/codegen/server/python/smithy/generators/PythonServerCombinedErrorGenerator.kt b/codegen-server/python/src/main/kotlin/software/amazon/smithy/rust/codegen/server/python/smithy/generators/PythonServerCombinedErrorGenerator.kt new file mode 100644 index 0000000000000000000000000000000000000000..076af0713759773ba13a0e52e225256ebf581a25 --- /dev/null +++ b/codegen-server/python/src/main/kotlin/software/amazon/smithy/rust/codegen/server/python/smithy/generators/PythonServerCombinedErrorGenerator.kt @@ -0,0 +1,47 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +package software.amazon.smithy.rust.codegen.server.python.smithy.generators + +import software.amazon.smithy.model.Model +import software.amazon.smithy.model.shapes.OperationShape +import software.amazon.smithy.rust.codegen.rustlang.RustWriter +import software.amazon.smithy.rust.codegen.rustlang.asType +import software.amazon.smithy.rust.codegen.rustlang.rust +import software.amazon.smithy.rust.codegen.rustlang.rustTemplate +import software.amazon.smithy.rust.codegen.server.python.smithy.PythonServerCargoDependency +import software.amazon.smithy.rust.codegen.server.smithy.generators.ServerCombinedErrorGenerator +import software.amazon.smithy.rust.codegen.smithy.RuntimeType +import software.amazon.smithy.rust.codegen.smithy.RustSymbolProvider +import software.amazon.smithy.rust.codegen.smithy.generators.error.errorSymbol + +/** + * Generates a unified error enum for [operation]. It depends on [ServerCombinedErrorGenerator] + * to generate the errors from the model and adds the Rust implementation `From`. + */ +class PythonServerCombinedErrorGenerator( + model: Model, + private val symbolProvider: RustSymbolProvider, + private val operation: OperationShape +) : ServerCombinedErrorGenerator(model, symbolProvider, operation) { + + override fun render(writer: RustWriter) { + super.render(writer) + writer.rustTemplate( + """ + impl #{From}<#{pyo3}::PyErr> for #{Error} { + fn from(variant: #{pyo3}::PyErr) -> #{Error} { + crate::error::InternalServerError { + message: variant.to_string() + }.into() + } + } + """, + "pyo3" to PythonServerCargoDependency.PyO3.asType(), + "Error" to operation.errorSymbol(symbolProvider), + "From" to RuntimeType.From + ) + } +} diff --git a/codegen-server/python/src/main/kotlin/software/amazon/smithy/rust/codegen/server/python/smithy/generators/PythonServerEnumGenerator.kt b/codegen-server/python/src/main/kotlin/software/amazon/smithy/rust/codegen/server/python/smithy/generators/PythonServerEnumGenerator.kt new file mode 100644 index 0000000000000000000000000000000000000000..4cb1708a7776ebd1e94caeaf26d578d011c7fa8c --- /dev/null +++ b/codegen-server/python/src/main/kotlin/software/amazon/smithy/rust/codegen/server/python/smithy/generators/PythonServerEnumGenerator.kt @@ -0,0 +1,57 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +package software.amazon.smithy.rust.codegen.server.python.smithy.generators + +import software.amazon.smithy.model.Model +import software.amazon.smithy.model.shapes.StringShape +import software.amazon.smithy.model.traits.EnumTrait +import software.amazon.smithy.rust.codegen.rustlang.RustWriter +import software.amazon.smithy.rust.codegen.rustlang.rust +import software.amazon.smithy.rust.codegen.server.smithy.generators.ServerEnumGenerator +import software.amazon.smithy.rust.codegen.smithy.RuntimeConfig +import software.amazon.smithy.rust.codegen.smithy.RustSymbolProvider + +/** + * To share enums defined in Rust with Python, `pyo3` provides the `PyClass` trait. + * This class generates enums definitions, implements the `PyClass` trait and adds + * some utility functions like `__str__()` and `__repr__()`. + */ +class PythonServerEnumGenerator( + model: Model, + symbolProvider: RustSymbolProvider, + private val writer: RustWriter, + private val shape: StringShape, + enumTrait: EnumTrait, + runtimeConfig: RuntimeConfig, +) : ServerEnumGenerator(model, symbolProvider, writer, shape, enumTrait, runtimeConfig) { + + override fun render() { + writer.renderPyClass(shape) + super.render() + renderPyO3Methods() + } + + override fun renderFromForStr() { + writer.renderPyClass(shape) + super.renderFromForStr() + } + + private fun renderPyO3Methods() { + writer.renderPyMethods() + writer.rust( + """ + impl $enumName { + fn __repr__(&self) -> String { + self.as_str().to_owned() + } + fn __str__(&self) -> String { + self.as_str().to_owned() + } + } + """ + ) + } +} diff --git a/codegen-server/python/src/main/kotlin/software/amazon/smithy/rust/codegen/server/python/smithy/generators/PythonServerOperationHandlerGenerator.kt b/codegen-server/python/src/main/kotlin/software/amazon/smithy/rust/codegen/server/python/smithy/generators/PythonServerOperationHandlerGenerator.kt new file mode 100644 index 0000000000000000000000000000000000000000..17dea610b6681de181a01c395e46399a1c4c8945 --- /dev/null +++ b/codegen-server/python/src/main/kotlin/software/amazon/smithy/rust/codegen/server/python/smithy/generators/PythonServerOperationHandlerGenerator.kt @@ -0,0 +1,152 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +package software.amazon.smithy.rust.codegen.server.python.smithy.generators + +import software.amazon.smithy.model.shapes.OperationShape +import software.amazon.smithy.rust.codegen.rustlang.RustWriter +import software.amazon.smithy.rust.codegen.rustlang.Writable +import software.amazon.smithy.rust.codegen.rustlang.asType +import software.amazon.smithy.rust.codegen.rustlang.rustTemplate +import software.amazon.smithy.rust.codegen.rustlang.writable +import software.amazon.smithy.rust.codegen.server.python.smithy.PythonServerCargoDependency +import software.amazon.smithy.rust.codegen.server.smithy.ServerCargoDependency +import software.amazon.smithy.rust.codegen.server.smithy.generators.ServerOperationHandlerGenerator +import software.amazon.smithy.rust.codegen.smithy.CodegenContext +import software.amazon.smithy.rust.codegen.util.toSnakeCase + +/** + * The Rust code responsible to run the Python business logic on the Python interpreter + * is implemented in this class, which inherits from [ServerOperationHandlerGenerator]. + * + * We codegenerate all operations handlers (steps usually left to the developer in a pure + * Rust application), which are built into a `Router` by [PythonApplicationGenerator]. + * + * To call a Python function from Rust, anything dealing with Python runs inside an async + * block that allows to catch stacktraces. The handler function is extracted from `PyHandler` + * and called with the necessary arguments inside a blocking Tokio task. + * At the end the block is awaited and errors are collected and reported. + * + * To call a Python coroutine, the same happens, but scheduled in a `tokio::Future`. + */ +class PythonServerOperationHandlerGenerator( + codegenContext: CodegenContext, + private val operations: List, +) : ServerOperationHandlerGenerator(codegenContext, operations) { + private val symbolProvider = codegenContext.symbolProvider + private val runtimeConfig = codegenContext.runtimeConfig + private val codegenScope = + arrayOf( + "SmithyPython" to PythonServerCargoDependency.SmithyHttpServerPython(runtimeConfig).asType(), + "SmithyServer" to ServerCargoDependency.SmithyHttpServer(runtimeConfig).asType(), + "pyo3" to PythonServerCargoDependency.PyO3.asType(), + "pyo3asyncio" to PythonServerCargoDependency.PyO3Asyncio.asType(), + "tokio" to PythonServerCargoDependency.Tokio.asType(), + "tracing" to PythonServerCargoDependency.Tracing.asType() + ) + + override fun render(writer: RustWriter) { + super.render(writer) + renderPythonOperationHandlerImpl(writer) + } + + private fun renderPythonOperationHandlerImpl(writer: RustWriter) { + for (operation in operations) { + val operationName = symbolProvider.toSymbol(operation).name + val input = "crate::input::${operationName}Input" + val output = "crate::output::${operationName}Output" + val error = "crate::error::${operationName}Error" + val fnName = operationName.toSnakeCase() + + writer.rustTemplate( + """ + /// Python handler for operation `$operationName`. + pub async fn $fnName( + input: $input, + state: #{SmithyServer}::Extension<#{SmithyPython}::PyState>, + handler: std::sync::Arc<#{SmithyPython}::PyHandler>, + ) -> std::result::Result<$output, $error> { + // Async block used to run the handler and catch any Python error. + let result = async { + let handler = handler.clone(); + if handler.is_coroutine { + #{pycoroutine:W} + } else { + #{pyfunction:W} + } + }; + #{pyerror:W} + } + """, + *codegenScope, + "pycoroutine" to renderPyCoroutine(fnName, output), + "pyfunction" to renderPyFunction(fnName, output), + "pyerror" to renderPyError(), + ) + } + } + + private fun renderPyFunction(name: String, output: String): Writable = + writable { + rustTemplate( + """ + #{tracing}::debug!("Executing Python handler function `$name()`"); + #{tokio}::task::spawn_blocking(move || { + #{pyo3}::Python::with_gil(|py| { + let pyhandler: &#{pyo3}::types::PyFunction = handler.extract(py)?; + let output = if handler.args == 1 { + pyhandler.call1((input,))? + } else { + pyhandler.call1((input, &*state.0.context))? + }; + output.extract::<$output>() + }) + }) + .await.map_err(|e| #{pyo3}::exceptions::PyRuntimeError::new_err(e.to_string()))? + """, + *codegenScope + ) + } + + private fun renderPyCoroutine(name: String, output: String): Writable = + writable { + rustTemplate( + """ + #{tracing}::debug!("Executing Python handler coroutine `$name()`"); + let result = #{pyo3}::Python::with_gil(|py| { + let pyhandler: &#{pyo3}::types::PyFunction = handler.extract(py)?; + let coroutine = if handler.args == 1 { + pyhandler.call1((input,))? + } else { + pyhandler.call1((input, &*state.0.context))? + }; + #{pyo3asyncio}::tokio::into_future(coroutine) + })?; + result.await.map(|r| #{pyo3}::Python::with_gil(|py| r.extract::<$output>(py)))? + """, + *codegenScope + ) + } + + private fun renderPyError(): Writable = + writable { + rustTemplate( + """ + // Catch and record a Python traceback. + result.await.map_err(|e| { + #{pyo3}::Python::with_gil(|py| { + let traceback = match e.traceback(py) { + Some(t) => t.format().unwrap_or_else(|e| e.to_string()), + None => "Unknown traceback".to_string() + }; + #{tracing}::error!("{}\n{}", e, traceback); + }); + e.into() + }) + """, + *codegenScope + ) + } +} diff --git a/codegen-server/python/src/main/kotlin/software/amazon/smithy/rust/codegen/server/python/smithy/generators/PythonServerServiceGenerator.kt b/codegen-server/python/src/main/kotlin/software/amazon/smithy/rust/codegen/server/python/smithy/generators/PythonServerServiceGenerator.kt new file mode 100644 index 0000000000000000000000000000000000000000..86835adb2a2cbc50b56995c7e8af2f3e4b19561a --- /dev/null +++ b/codegen-server/python/src/main/kotlin/software/amazon/smithy/rust/codegen/server/python/smithy/generators/PythonServerServiceGenerator.kt @@ -0,0 +1,46 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +package software.amazon.smithy.rust.codegen.server.python.smithy.generators + +import software.amazon.smithy.model.shapes.OperationShape +import software.amazon.smithy.rust.codegen.rustlang.RustModule +import software.amazon.smithy.rust.codegen.rustlang.RustWriter +import software.amazon.smithy.rust.codegen.server.smithy.generators.ServerServiceGenerator +import software.amazon.smithy.rust.codegen.smithy.CodegenContext +import software.amazon.smithy.rust.codegen.smithy.RustCrate +import software.amazon.smithy.rust.codegen.smithy.generators.protocol.ProtocolGenerator +import software.amazon.smithy.rust.codegen.smithy.generators.protocol.ProtocolSupport +import software.amazon.smithy.rust.codegen.smithy.protocols.HttpBindingResolver + +/** + * PythonServerServiceGenerator + * + * Service generator is the main codegeneration entry point for Smithy services. Individual structures and unions are + * generated in codegen visitor, but this class handles all protocol-specific code generation (i.e. operations). + */ +class PythonServerServiceGenerator( + private val rustCrate: RustCrate, + protocolGenerator: ProtocolGenerator, + protocolSupport: ProtocolSupport, + httpBindingResolver: HttpBindingResolver, + private val context: CodegenContext, +) : ServerServiceGenerator(rustCrate, protocolGenerator, protocolSupport, httpBindingResolver, context) { + + override fun renderCombinedErrors(writer: RustWriter, operation: OperationShape) { + PythonServerCombinedErrorGenerator(context.model, context.symbolProvider, operation).render(writer) + } + + override fun renderOperationHandler(writer: RustWriter, operations: List) { + PythonServerOperationHandlerGenerator(context, operations).render(writer) + } + + override fun renderExtras(operations: List) { + rustCrate.withModule(RustModule.public("python_server_application", "Python server and application implementation.")) { writer -> + PythonApplicationGenerator(context, operations) + .render(writer) + } + } +} diff --git a/codegen-server/python/src/main/kotlin/software/amazon/smithy/rust/codegen/server/python/smithy/generators/PythonServerStructureGenerator.kt b/codegen-server/python/src/main/kotlin/software/amazon/smithy/rust/codegen/server/python/smithy/generators/PythonServerStructureGenerator.kt new file mode 100644 index 0000000000000000000000000000000000000000..7c3b59d2e080834e0febcc2d902569b32076653c --- /dev/null +++ b/codegen-server/python/src/main/kotlin/software/amazon/smithy/rust/codegen/server/python/smithy/generators/PythonServerStructureGenerator.kt @@ -0,0 +1,85 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +package software.amazon.smithy.rust.codegen.server.python.smithy.generators + +import software.amazon.smithy.codegen.core.Symbol +import software.amazon.smithy.model.Model +import software.amazon.smithy.model.shapes.MemberShape +import software.amazon.smithy.model.shapes.StructureShape +import software.amazon.smithy.model.traits.ErrorTrait +import software.amazon.smithy.rust.codegen.rustlang.RustWriter +import software.amazon.smithy.rust.codegen.rustlang.Writable +import software.amazon.smithy.rust.codegen.rustlang.render +import software.amazon.smithy.rust.codegen.rustlang.rust +import software.amazon.smithy.rust.codegen.rustlang.rustTemplate +import software.amazon.smithy.rust.codegen.rustlang.writable +import software.amazon.smithy.rust.codegen.smithy.RustSymbolProvider +import software.amazon.smithy.rust.codegen.smithy.generators.StructureGenerator +import software.amazon.smithy.rust.codegen.smithy.rustType +import software.amazon.smithy.rust.codegen.util.hasTrait + +/** + * To share structures defined in Rust with Python, `pyo3` provides the `PyClass` trait. + * This class generates input / output / error structures definitions and implements the + * `PyClass` trait. + */ +open class PythonServerStructureGenerator( + model: Model, + private val symbolProvider: RustSymbolProvider, + private val writer: RustWriter, + private val shape: StructureShape +) : StructureGenerator(model, symbolProvider, writer, shape) { + + override fun renderStructure() { + writer.renderPyClass(shape) + super.renderStructure() + renderPyO3Methods() + } + + override fun renderStructureMember(writer: RustWriter, member: MemberShape, memberName: String, memberSymbol: Symbol) { + writer.renderPyGetterSetter() + super.renderStructureMember(writer, member, memberName, memberSymbol) + } + + private fun renderPyO3Methods() { + if (shape.hasTrait() || accessorMembers.isNotEmpty()) { + writer.renderPyMethods() + writer.rustTemplate( + """ + impl $name { + ##[new] + pub fn new(#{bodysignature:W}) -> Self { + Self { + #{bodymembers:W} + } + } + fn __repr__(&self) -> String { + format!("{self:?}") + } + fn __str__(&self) -> String { + format!("{self:?}") + } + } + """, + "bodysignature" to renderStructSignatureMembers(), + "bodymembers" to renderStructBodyMembers() + ) + } + } + + private fun renderStructSignatureMembers(): Writable = + writable { + forEachMember(members) { _, memberName, memberSymbol -> + val memberType = memberSymbol.rustType() + rust("$memberName: ${memberType.render()},") + } + } + + private fun renderStructBodyMembers(): Writable = + writable { + forEachMember(members) { _, memberName, _ -> rust("$memberName,") } + } +} diff --git a/codegen-server/python/src/main/resources/META-INF/services/software.amazon.smithy.build.SmithyBuildPlugin b/codegen-server/python/src/main/resources/META-INF/services/software.amazon.smithy.build.SmithyBuildPlugin index ea7c4546af81e06c65fad71cf02ddce550372d2e..6dc5b76c780b95e41732c4168794d6574efc5012 100644 --- a/codegen-server/python/src/main/resources/META-INF/services/software.amazon.smithy.build.SmithyBuildPlugin +++ b/codegen-server/python/src/main/resources/META-INF/services/software.amazon.smithy.build.SmithyBuildPlugin @@ -2,4 +2,4 @@ # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. # SPDX-License-Identifier: Apache-2.0 # -software.amazon.smithy.rust.codegen.server.python.smithy.RustCodegenServerPlugin +software.amazon.smithy.rust.codegen.server.python.smithy.PythonCodegenServerPlugin diff --git a/codegen-server/python/src/main/resources/META-INF/services/software.amazon.smithy.rust.codegen.smithy.customize.RustCodegenDecorator b/codegen-server/python/src/main/resources/META-INF/services/software.amazon.smithy.rust.codegen.smithy.customize.RustCodegenDecorator new file mode 100644 index 0000000000000000000000000000000000000000..13a1fd23750a07674e326fc691799e22d87c2dd3 --- /dev/null +++ b/codegen-server/python/src/main/resources/META-INF/services/software.amazon.smithy.rust.codegen.smithy.customize.RustCodegenDecorator @@ -0,0 +1,6 @@ +# +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +# + +software.amazon.smithy.rust.codegen.server.python.smithy.customizations.PythonServerCodegenDecorator diff --git a/codegen-server/src/main/kotlin/software/amazon/smithy/rust/codegen/server/smithy/ServerCodegenVisitor.kt b/codegen-server/src/main/kotlin/software/amazon/smithy/rust/codegen/server/smithy/ServerCodegenVisitor.kt index d9816f9ec19d7d2c66785d229b878c7fada8763d..ca1fb540af30fdd2b6d43925c112a391b5cce2c4 100644 --- a/codegen-server/src/main/kotlin/software/amazon/smithy/rust/codegen/server/smithy/ServerCodegenVisitor.kt +++ b/codegen-server/src/main/kotlin/software/amazon/smithy/rust/codegen/server/smithy/ServerCodegenVisitor.kt @@ -48,19 +48,20 @@ import java.util.logging.Logger * Entrypoint for server-side code generation. This class will walk the in-memory model and * generate all the needed types by calling the accept() function on the available shapes. */ -class ServerCodegenVisitor(context: PluginContext, private val codegenDecorator: RustCodegenDecorator) : +open class ServerCodegenVisitor(context: PluginContext, private val codegenDecorator: RustCodegenDecorator) : ShapeVisitor.Default() { - private val logger = Logger.getLogger(javaClass.name) - private val settings = ServerRustSettings.from(context.model, context.settings) - - private val symbolProvider: RustSymbolProvider - private val rustCrate: RustCrate private val fileManifest = context.fileManifest - private val model: Model - private val codegenContext: CodegenContext - private val protocolGeneratorFactory: ProtocolGeneratorFactory - private val protocolGenerator: ProtocolGenerator + + protected val logger = Logger.getLogger(javaClass.name) + protected val settings = ServerRustSettings.from(context.model, context.settings) + + var model: Model + var protocolGeneratorFactory: ProtocolGeneratorFactory + var protocolGenerator: ProtocolGenerator + var codegenContext: CodegenContext + var symbolProvider: RustSymbolProvider + var rustCrate: RustCrate init { val symbolVisitorConfig = @@ -95,7 +96,7 @@ class ServerCodegenVisitor(context: PluginContext, private val codegenDecorator: * Base model transformation applied to all services. * See below for details. */ - private fun baselineTransform(model: Model) = + protected fun baselineTransform(model: Model) = model // Add errors attached at the service level to the models .let { ModelTransformer.create().copyServiceErrorsToOperations(it, settings.getService(it)) } @@ -175,7 +176,7 @@ class ServerCodegenVisitor(context: PluginContext, private val codegenDecorator: } /** - * String Shape Visitor + * Enum Shape Visitor * * Although raw strings require no code generation, enums are actually [EnumTrait] applied to string shapes. */ diff --git a/codegen-server/src/main/kotlin/software/amazon/smithy/rust/codegen/server/smithy/customizations/AdditionalErrorsDecorator.kt b/codegen-server/src/main/kotlin/software/amazon/smithy/rust/codegen/server/smithy/customizations/AdditionalErrorsDecorator.kt index 0038af876b0297888206a0cd8d9bf9e707ea613f..af82b645b8331551723ab1d3d7d28781974b0319 100644 --- a/codegen-server/src/main/kotlin/software/amazon/smithy/rust/codegen/server/smithy/customizations/AdditionalErrorsDecorator.kt +++ b/codegen-server/src/main/kotlin/software/amazon/smithy/rust/codegen/server/smithy/customizations/AdditionalErrorsDecorator.kt @@ -18,23 +18,23 @@ import software.amazon.smithy.rust.codegen.smithy.customize.RustCodegenDecorator /** * Add at least one error to all operations in the model. * - * When this decorator is applied, even operations that do not have a Smithy error attatched, + * When this decorator is applied, even operations that do not have a Smithy error attached, * will return `Result`. * * To enable this decorator write its class name to a resource file like this: * ``` - * C="software.amazon.smithy.rust.codegen.server.smithy.customizations.AddInternalServerErrorToInfallibleOpsDecorator" + * C="software.amazon.smithy.rust.codegen.server.smithy.customizations.AddInternalServerErrorToInfallibleOperationsDecorator" * F="software.amazon.smithy.rust.codegen.smithy.customize.RustCodegenDecorator" * D="codegen-server/src/main/resources/META-INF/services" * mkdir -p "$D" && echo "$C" > "$D/$F" * ``` */ -class AddInternalServerErrorToInfallibleOpsDecorator : RustCodegenDecorator { - override val name: String = "AddInternalServerErrorToInfallibleOps" +class AddInternalServerErrorToInfallibleOperationsDecorator : RustCodegenDecorator { + override val name: String = "AddInternalServerErrorToInfallibleOperations" override val order: Byte = 0 override fun transformModel(service: ServiceShape, model: Model): Model = - addErrorShapeToModelOps(service, model, { shape -> shape.errors.isEmpty() }) + addErrorShapeToModelOperations(service, model, { shape -> shape.errors.isEmpty() }) } /** @@ -44,26 +44,26 @@ class AddInternalServerErrorToInfallibleOpsDecorator : RustCodegenDecorator { * and there is no native mapping of these actual errors to the API errors, servers can generate * the code with this decorator to add an internal error shape on-the-fly to all the operations. * - * When this decorator is applied, even operations that do not have a Smithy error attatched, + * When this decorator is applied, even operations that do not have a Smithy error attached, * will return `Result`. * * To enable this decorator write its class name to a resource file like this: * ``` - * C="software.amazon.smithy.rust.codegen.server.smithy.customizations.AddInternalServerErrorToAllOpsDecorator" + * C="software.amazon.smithy.rust.codegen.server.smithy.customizations.AddInternalServerErrorToAllOperationsDecorator" * F="software.amazon.smithy.rust.codegen.smithy.customize.RustCodegenDecorator" * D="codegen-server/src/main/resources/META-INF/services" * mkdir -p "$D" && echo "$C" > "$D/$F" * ``` */ -class AddInternalServerErrorToAllOpsDecorator : RustCodegenDecorator { - override val name: String = "AddInternalServerErrorToAllOps" +class AddInternalServerErrorToAllOperationsDecorator : RustCodegenDecorator { + override val name: String = "AddInternalServerErrorToAllOperations" override val order: Byte = 0 override fun transformModel(service: ServiceShape, model: Model): Model = - addErrorShapeToModelOps(service, model, { _ -> true }) + addErrorShapeToModelOperations(service, model, { _ -> true }) } -fun addErrorShapeToModelOps(service: ServiceShape, model: Model, opSelector: (OperationShape) -> Boolean): Model { +fun addErrorShapeToModelOperations(service: ServiceShape, model: Model, opSelector: (OperationShape) -> Boolean): Model { val errorShape = internalServerError(service.id.getNamespace()) val modelShapes = model.toBuilder().addShapes(listOf(errorShape)).build() return ModelTransformer.create().mapShapes(modelShapes) { shape -> diff --git a/codegen-server/src/main/kotlin/software/amazon/smithy/rust/codegen/server/smithy/generators/ServerCombinedErrorGenerator.kt b/codegen-server/src/main/kotlin/software/amazon/smithy/rust/codegen/server/smithy/generators/ServerCombinedErrorGenerator.kt index edb7a891403b24170bd7c8219b67aabe51d3efa1..64683f4448dd397f56f1cace657afd7e0140351e 100644 --- a/codegen-server/src/main/kotlin/software/amazon/smithy/rust/codegen/server/smithy/generators/ServerCombinedErrorGenerator.kt +++ b/codegen-server/src/main/kotlin/software/amazon/smithy/rust/codegen/server/smithy/generators/ServerCombinedErrorGenerator.kt @@ -25,14 +25,14 @@ import software.amazon.smithy.rust.codegen.util.toSnakeCase * Generates a unified error enum for [operation]. [ErrorGenerator] handles generating the individual variants, * but we must still combine those variants into an enum covering all possible errors for a given operation. */ -class ServerCombinedErrorGenerator( +open class ServerCombinedErrorGenerator( private val model: Model, private val symbolProvider: RustSymbolProvider, private val operation: OperationShape ) { private val operationIndex = OperationIndex.of(model) - fun render(writer: RustWriter) { + open fun render(writer: RustWriter) { val errors = operationIndex.getErrors(operation) val operationSymbol = symbolProvider.toSymbol(operation) val symbol = operation.errorSymbol(symbolProvider) @@ -87,7 +87,7 @@ class ServerCombinedErrorGenerator( for (error in errors) { val errorSymbol = symbolProvider.toSymbol(error) - writer.rustBlock("impl From<#T> for #T", errorSymbol, symbol) { + writer.rustBlock("impl #T<#T> for #T", RuntimeType.From, errorSymbol, symbol) { rustBlock("fn from(variant: #T) -> #T", errorSymbol, symbol) { rust("Self::${errorSymbol.name}(variant)") } diff --git a/codegen-server/src/main/kotlin/software/amazon/smithy/rust/codegen/server/smithy/generators/ServerEnumGenerator.kt b/codegen-server/src/main/kotlin/software/amazon/smithy/rust/codegen/server/smithy/generators/ServerEnumGenerator.kt index 830de8acfd1b68c7ff5440afc264e0928d341180..9b1da8934b19f85c47cb8a3af651b74ae31dfaa8 100644 --- a/codegen-server/src/main/kotlin/software/amazon/smithy/rust/codegen/server/smithy/generators/ServerEnumGenerator.kt +++ b/codegen-server/src/main/kotlin/software/amazon/smithy/rust/codegen/server/smithy/generators/ServerEnumGenerator.kt @@ -19,7 +19,7 @@ import software.amazon.smithy.rust.codegen.smithy.generators.CodegenTarget import software.amazon.smithy.rust.codegen.smithy.generators.EnumGenerator import software.amazon.smithy.rust.codegen.util.dq -class ServerEnumGenerator( +open class ServerEnumGenerator( model: Model, symbolProvider: RustSymbolProvider, private val writer: RustWriter, @@ -55,15 +55,12 @@ class ServerEnumGenerator( Self::EnumVariantNotFound(Box::new(e)) } } - impl #{From}<$errorStruct> for #{JsonDeserialize} { fn from(e: $errorStruct) -> Self { Self::custom(format!("unknown variant {}", e)) } } - impl #{StdError} for $errorStruct { } - impl #{Display} for $errorStruct { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { self.0.fmt(f) @@ -83,7 +80,6 @@ class ServerEnumGenerator( """ impl std::str::FromStr for $enumName { type Err = $errorStruct; - fn from_str(s: &str) -> std::result::Result { $enumName::try_from(s) } diff --git a/codegen-server/src/main/kotlin/software/amazon/smithy/rust/codegen/server/smithy/generators/ServerOperationHandlerGenerator.kt b/codegen-server/src/main/kotlin/software/amazon/smithy/rust/codegen/server/smithy/generators/ServerOperationHandlerGenerator.kt index e1c1b5db903006ed6c3019a3b58a974905fc0eea..fb4d8013064a2799b28c76883a1a418d653b04e6 100644 --- a/codegen-server/src/main/kotlin/software/amazon/smithy/rust/codegen/server/smithy/generators/ServerOperationHandlerGenerator.kt +++ b/codegen-server/src/main/kotlin/software/amazon/smithy/rust/codegen/server/smithy/generators/ServerOperationHandlerGenerator.kt @@ -25,7 +25,7 @@ import software.amazon.smithy.rust.codegen.util.toPascalCase /** * ServerOperationHandlerGenerator */ -class ServerOperationHandlerGenerator( +open class ServerOperationHandlerGenerator( codegenContext: CodegenContext, private val operations: List, ) { @@ -48,7 +48,7 @@ class ServerOperationHandlerGenerator( "http" to RuntimeType.http, ) - fun render(writer: RustWriter) { + open fun render(writer: RustWriter) { renderHandlerImplementations(writer, false) renderHandlerImplementations(writer, true) } diff --git a/codegen-server/src/main/kotlin/software/amazon/smithy/rust/codegen/server/smithy/generators/ServerServiceGenerator.kt b/codegen-server/src/main/kotlin/software/amazon/smithy/rust/codegen/server/smithy/generators/ServerServiceGenerator.kt index 73ff3780734a588553739b8e359eba358e131a86..4abc64460e9ccb8a0a777d73cf6b5c031d5fb601 100644 --- a/codegen-server/src/main/kotlin/software/amazon/smithy/rust/codegen/server/smithy/generators/ServerServiceGenerator.kt +++ b/codegen-server/src/main/kotlin/software/amazon/smithy/rust/codegen/server/smithy/generators/ServerServiceGenerator.kt @@ -6,7 +6,9 @@ package software.amazon.smithy.rust.codegen.server.smithy.generators import software.amazon.smithy.model.knowledge.TopDownIndex +import software.amazon.smithy.model.shapes.OperationShape import software.amazon.smithy.rust.codegen.rustlang.RustModule +import software.amazon.smithy.rust.codegen.rustlang.RustWriter import software.amazon.smithy.rust.codegen.server.smithy.generators.protocol.ServerProtocolTestGenerator import software.amazon.smithy.rust.codegen.smithy.CodegenContext import software.amazon.smithy.rust.codegen.smithy.RustCrate @@ -20,7 +22,7 @@ import software.amazon.smithy.rust.codegen.smithy.protocols.HttpBindingResolver * Service generator is the main codegeneration entry point for Smithy services. Individual structures and unions are * generated in codegen visitor, but this class handles all protocol-specific code generation (i.e. operations). */ -class ServerServiceGenerator( +open class ServerServiceGenerator( private val rustCrate: RustCrate, private val protocolGenerator: ProtocolGenerator, private val protocolSupport: ProtocolSupport, @@ -28,13 +30,13 @@ class ServerServiceGenerator( private val context: CodegenContext, ) { private val index = TopDownIndex.of(context.model) + protected val operations = index.getContainedOperations(context.serviceShape).sortedBy { it.id } /** * Render Service Specific code. Code will end up in different files via [useShapeWriter]. See `SymbolVisitor.kt` * which assigns a symbol location to each shape. */ fun render() { - val operations = index.getContainedOperations(context.serviceShape).sortedBy { it.id } for (operation in operations) { rustCrate.useShapeWriter(operation) { operationWriter -> protocolGenerator.serverRenderOperation( @@ -44,21 +46,36 @@ class ServerServiceGenerator( ServerProtocolTestGenerator(context, protocolSupport, operation, operationWriter) .render() } - if (operation.errors.isNotEmpty()) { rustCrate.withModule(RustModule.Error) { writer -> - ServerCombinedErrorGenerator(context.model, context.symbolProvider, operation) - .render(writer) + renderCombinedErrors(writer, operation) } } } rustCrate.withModule(RustModule.public("operation_handler", "Operation handlers definition and implementation.")) { writer -> - ServerOperationHandlerGenerator(context, operations) - .render(writer) + renderOperationHandler(writer, operations) } rustCrate.withModule(RustModule.public("operation_registry", "A registry of your service's operations.")) { writer -> - ServerOperationRegistryGenerator(context, httpBindingResolver, operations) - .render(writer) + renderOperationRegistry(writer, operations) } + renderExtras(operations) + } + + // Render any extra section needed by subclasses of `ServerServiceGenerator`. + open fun renderExtras(operations: List) { } + + // Render combined errors. + open fun renderCombinedErrors(writer: RustWriter, operation: OperationShape) { + ServerCombinedErrorGenerator(context.model, context.symbolProvider, operation).render(writer) + } + + // Render operations handler. + open fun renderOperationHandler(writer: RustWriter, operations: List) { + ServerOperationHandlerGenerator(context, operations).render(writer) + } + + // Render operations registry. + private fun renderOperationRegistry(writer: RustWriter, operations: List) { + ServerOperationRegistryGenerator(context, httpBindingResolver, operations).render(writer) } } diff --git a/codegen-server/src/main/kotlin/software/amazon/smithy/rust/codegen/server/smithy/protocols/ServerHttpBoundProtocolGenerator.kt b/codegen-server/src/main/kotlin/software/amazon/smithy/rust/codegen/server/smithy/protocols/ServerHttpBoundProtocolGenerator.kt index c6f7d9832f549f431b67dcf77b10851e3d7fa658..9ad961d399c5053c16f6de8e134786b07072b269 100644 --- a/codegen-server/src/main/kotlin/software/amazon/smithy/rust/codegen/server/smithy/protocols/ServerHttpBoundProtocolGenerator.kt +++ b/codegen-server/src/main/kotlin/software/amazon/smithy/rust/codegen/server/smithy/protocols/ServerHttpBoundProtocolGenerator.kt @@ -306,7 +306,7 @@ private class ServerHttpBoundProtocolTraitImplGenerator( if (operationShape.errors.isNotEmpty()) { rustTemplate( """ - impl From> for $outputName { + impl #{From}> for $outputName { fn from(res: Result<#{O}, #{E}>) -> Self { match res { Ok(v) => Self::Output(v), @@ -316,31 +316,34 @@ private class ServerHttpBoundProtocolTraitImplGenerator( } """.trimIndent(), "O" to outputSymbol, - "E" to errorSymbol + "E" to errorSymbol, + "From" to RuntimeType.From ) } else { rustTemplate( """ - impl From<#{O}> for $outputName { + impl #{From}<#{O}> for $outputName { fn from(o: #{O}) -> Self { Self(o) } } """.trimIndent(), - "O" to outputSymbol + "O" to outputSymbol, + "From" to RuntimeType.From ) } // Implement conversion function to "unwrap" into the model operation input types. rustTemplate( """ - impl From<$inputName> for #{I} { + impl #{From}<$inputName> for #{I} { fn from(i: $inputName) -> Self { i.0 } } """.trimIndent(), - "I" to inputSymbol + "I" to inputSymbol, + "From" to RuntimeType.From ) } diff --git a/codegen-server/src/test/kotlin/software/amazon/smithy/rust/codegen/server/smithy/customizations/AdditionalErrorsDecoratorTest.kt b/codegen-server/src/test/kotlin/software/amazon/smithy/rust/codegen/server/smithy/customizations/AdditionalErrorsDecoratorTest.kt index e49e29188797de3baa4a41222e3f3ff20a007b63..4607c4460e4e992ae82bb6166eb04c0961cf7e90 100644 --- a/codegen-server/src/test/kotlin/software/amazon/smithy/rust/codegen/server/smithy/customizations/AdditionalErrorsDecoratorTest.kt +++ b/codegen-server/src/test/kotlin/software/amazon/smithy/rust/codegen/server/smithy/customizations/AdditionalErrorsDecoratorTest.kt @@ -27,9 +27,9 @@ class AdditionalErrorsDecoratorTest { output: InputOutput, errors: [AnError] } - + structure InputOutput { } - + @error("client") structure AnError { } """.asSmithyModel() @@ -40,7 +40,7 @@ class AdditionalErrorsDecoratorTest { fun `add InternalServerError to infallible operations only`() { model.lookup("test#Infallible").errors.isEmpty() shouldBe true model.lookup("test#Fallible").errors.size shouldBe 1 - val transformedModel = AddInternalServerErrorToInfallibleOpsDecorator().transformModel(service, model) + val transformedModel = AddInternalServerErrorToInfallibleOperationsDecorator().transformModel(service, model) transformedModel.lookup("test#Infallible").errors.size shouldBe 1 transformedModel.lookup("test#Fallible").errors.size shouldBe 1 } @@ -49,7 +49,7 @@ class AdditionalErrorsDecoratorTest { fun `add InternalServerError to all model operations`() { model.lookup("test#Infallible").errors.isEmpty() shouldBe true model.lookup("test#Fallible").errors.size shouldBe 1 - val transformedModel = AddInternalServerErrorToAllOpsDecorator().transformModel(service, model) + val transformedModel = AddInternalServerErrorToAllOperationsDecorator().transformModel(service, model) transformedModel.lookup("test#Infallible").errors.size shouldBe 1 transformedModel.lookup("test#Fallible").errors.size shouldBe 2 } diff --git a/codegen/src/main/kotlin/software/amazon/smithy/rust/codegen/smithy/generators/EnumGenerator.kt b/codegen/src/main/kotlin/software/amazon/smithy/rust/codegen/smithy/generators/EnumGenerator.kt index 8f2d1abbb6b1509e030dea0fdc9fbeb001479a18..2e761ff3791e673c4e69742032d918a32c970981 100644 --- a/codegen/src/main/kotlin/software/amazon/smithy/rust/codegen/smithy/generators/EnumGenerator.kt +++ b/codegen/src/main/kotlin/software/amazon/smithy/rust/codegen/smithy/generators/EnumGenerator.kt @@ -92,7 +92,7 @@ open class EnumGenerator( const val Values = "values" } - fun render() { + open fun render() { if (enumTrait.hasNames()) { // pub enum Blah { V1, V2, .. } renderEnum() diff --git a/codegen/src/main/kotlin/software/amazon/smithy/rust/codegen/smithy/generators/StructureGenerator.kt b/codegen/src/main/kotlin/software/amazon/smithy/rust/codegen/smithy/generators/StructureGenerator.kt index d4c6f143f71aeab30fa6eee0dffb84005eb616da..cb545391f7f971f2f9a6eeb80f4e21de6c0452e7 100644 --- a/codegen/src/main/kotlin/software/amazon/smithy/rust/codegen/smithy/generators/StructureGenerator.kt +++ b/codegen/src/main/kotlin/software/amazon/smithy/rust/codegen/smithy/generators/StructureGenerator.kt @@ -50,20 +50,20 @@ fun redactIfNecessary(member: MemberShape, model: Model, safeToPrint: String): S } } -class StructureGenerator( +open class StructureGenerator( val model: Model, private val symbolProvider: RustSymbolProvider, private val writer: RustWriter, private val shape: StructureShape ) { private val errorTrait = shape.getTrait() - private val members: List = shape.allMembers.values.toList() - private val accessorMembers: List = when (errorTrait) { + protected val members: List = shape.allMembers.values.toList() + protected val accessorMembers: List = when (errorTrait) { null -> members // Let the ErrorGenerator render the error message accessor if this is an error struct else -> members.filter { "message" != symbolProvider.toMemberName(it) } } - private val name = symbolProvider.toSymbol(shape).name + protected val name = symbolProvider.toSymbol(shape).name fun render(forWhom: CodegenTarget = CodegenTarget.CLIENT) { renderStructure() @@ -153,7 +153,13 @@ class StructureGenerator( } } - private fun renderStructure() { + open fun renderStructureMember(writer: RustWriter, member: MemberShape, memberName: String, memberSymbol: Symbol) { + writer.renderMemberDoc(member, memberSymbol) + memberSymbol.expectRustMetadata().render(writer) + writer.write("$memberName: #T,", symbolProvider.toSymbol(member)) + } + + open fun renderStructure() { val symbol = symbolProvider.toSymbol(shape) val containerMeta = symbol.expectRustMetadata() writer.documentShape(shape, model) @@ -161,10 +167,8 @@ class StructureGenerator( containerMeta.copy(derives = withoutDebug).render(writer) writer.rustBlock("struct $name ${lifetimeDeclaration()}") { - forEachMember(members) { member, memberName, memberSymbol -> - renderMemberDoc(member, memberSymbol) - memberSymbol.expectRustMetadata().render(this) - write("$memberName: #T,", symbolProvider.toSymbol(member)) + writer.forEachMember(members) { member, memberName, memberSymbol -> + renderStructureMember(writer, member, memberName, memberSymbol) } } @@ -172,7 +176,7 @@ class StructureGenerator( renderDebugImpl() } - private fun RustWriter.forEachMember( + protected fun RustWriter.forEachMember( toIterate: List, block: RustWriter.(MemberShape, String, Symbol) -> Unit ) { diff --git a/codegen/src/main/kotlin/software/amazon/smithy/rust/codegen/smithy/generators/http/HttpBindingGenerator.kt b/codegen/src/main/kotlin/software/amazon/smithy/rust/codegen/smithy/generators/http/HttpBindingGenerator.kt index 83f3ccf9ef8348076d23d3f87a2d5d7fe3bc57bc..7ebfb9b130cc2c009b6fc65c60021ddd37eec4f3 100644 --- a/codegen/src/main/kotlin/software/amazon/smithy/rust/codegen/smithy/generators/http/HttpBindingGenerator.kt +++ b/codegen/src/main/kotlin/software/amazon/smithy/rust/codegen/smithy/generators/http/HttpBindingGenerator.kt @@ -297,7 +297,7 @@ class HttpBindingGenerator( } is BlobShape -> rust( "Ok(#T::new(body))", - RuntimeType.Blob(runtimeConfig) + symbolProvider.toSymbol(targetShape) ) // `httpPayload` can be applied to set/map/list shapes. // However, none of the AWS protocols support it. diff --git a/rust-runtime/aws-smithy-http-server-python/Cargo.toml b/rust-runtime/aws-smithy-http-server-python/Cargo.toml index e0d8e1d252276ffd6ad1b0042660a0a5266bfa74..96e074d3f7cce404a6d84f993270b9bbc0708680 100644 --- a/rust-runtime/aws-smithy-http-server-python/Cargo.toml +++ b/rust-runtime/aws-smithy-http-server-python/Cargo.toml @@ -14,10 +14,20 @@ Python server runtime for Smithy Rust Server Framework. publish = false [dependencies] -pyo3 = { version = "0.16" } -pyo3-asyncio = { version = "0.16", features = ["attributes", "tokio-runtime"] } +aws-smithy-http-server = { path = "../aws-smithy-http-server" } +aws-smithy-types = { path = "../aws-smithy-types" } +bytes = "1.1" +delegate = "0.6" +http = "0.2" +hyper = { version = "0.14", features = ["server", "http1", "http2", "tcp", "stream"] } +num_cpus = "1.13" +paste = "1.0" +pyo3 = { version = "0.16.5" } +pyo3-asyncio = { version = "0.16.0", features = ["attributes", "tokio-runtime"] } socket2 = { version = "0.4", features = ["all"] } +thiserror = "1.0.31" tokio = { version = "1", features = ["full"] } +tower = "0.4" tracing = "0.1.34" tracing-subscriber = { version = "0.3", features = ["env-filter"] } diff --git a/rust-runtime/aws-smithy-http-server-python/src/error.rs b/rust-runtime/aws-smithy-http-server-python/src/error.rs new file mode 100644 index 0000000000000000000000000000000000000000..0b1bc1549426c7bd0570c20894a47ef3891a52dd --- /dev/null +++ b/rust-runtime/aws-smithy-http-server-python/src/error.rs @@ -0,0 +1,22 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +//! Python error definition. + +use thiserror::Error; + +/// Python error that implements foreign errors. +#[derive(Error, Debug)] +pub enum Error { + /// Custom error. + #[error("{0}")] + Custom(String), + /// Errors coming from `pyo3::PyErr`. + #[error("PyO3 error: {0}")] + PyO3(#[from] pyo3::PyErr), + /// Error coming from `tokio::task::JoinError`. + #[error("Tokio task join error: {0}")] + TaskJoin(#[from] tokio::task::JoinError), +} diff --git a/rust-runtime/aws-smithy-http-server-python/src/lib.rs b/rust-runtime/aws-smithy-http-server-python/src/lib.rs index 090642833f06cbd69bd9ab5b59b90abbeca4f710..60a4f0f7c7c5b2d1db6b3fadce6c9c38dfab1b9a 100644 --- a/rust-runtime/aws-smithy-http-server-python/src/lib.rs +++ b/rust-runtime/aws-smithy-http-server-python/src/lib.rs @@ -11,10 +11,33 @@ //! //! [PyO3]: https://pyo3.rs/ +mod error; mod logging; +mod server; mod socket; +mod state; +pub mod types; +#[doc(inline)] +pub use error::Error; #[doc(inline)] pub use logging::{setup, LogLevel}; #[doc(inline)] +pub use server::{PyApp, PyRouter}; +#[doc(inline)] pub use socket::SharedSocket; +#[doc(inline)] +pub use state::{PyHandler, PyHandlers, PyState}; + +#[cfg(test)] +mod tests { + use std::sync::Once; + + static INIT: Once = Once::new(); + + pub(crate) fn initialize() { + INIT.call_once(|| { + pyo3::prepare_freethreaded_python(); + }); + } +} diff --git a/rust-runtime/aws-smithy-http-server-python/src/logging.rs b/rust-runtime/aws-smithy-http-server-python/src/logging.rs index 063ce412df4620b3262c98286c592e3a3cf12975..595bf4cb981f97965c4bc8f0fe593bb14fcdd3f1 100644 --- a/rust-runtime/aws-smithy-http-server-python/src/logging.rs +++ b/rust-runtime/aws-smithy-http-server-python/src/logging.rs @@ -88,7 +88,7 @@ impl From for Level { } } -/// Modifies the Python `logging` module to deliver its log messages using `tracing::Subscriber` events. +/// Modifies the Python `logging` module to deliver its log messages using [tracing::Subscriber] events. /// /// To achieve this goal, the following changes are made to the module: /// - A new builtin function `logging.python_tracing` transcodes `logging.LogRecord`s to `tracing::Event`s. This function @@ -131,9 +131,10 @@ def basicConfig(*pargs, **kwargs): Ok(()) } -/// Consumes a Python `logging.LogRecord` and emits a Rust `tracing::Event` instead. +/// Consumes a Python `logging.LogRecord` and emits a Rust [tracing::Event] instead. #[cfg(not(test))] #[pyfunction] +#[pyo3(text_signature = "(record)")] fn python_tracing(record: &PyAny) -> PyResult<()> { let level = record.getattr("levelno")?; let message = record.getattr("getMessage")?.call0()?; @@ -154,6 +155,7 @@ fn python_tracing(record: &PyAny) -> PyResult<()> { #[cfg(test)] #[pyfunction] +#[pyo3(text_signature = "(record)")] fn python_tracing(record: &PyAny) -> PyResult<()> { let message = record.getattr("getMessage")?.call0()?; pretty_assertions::assert_eq!(message.to_string(), "a message"); @@ -163,19 +165,10 @@ fn python_tracing(record: &PyAny) -> PyResult<()> { #[cfg(test)] mod tests { use super::*; - use std::sync::Once; - - static INIT: Once = Once::new(); - - fn initialize() { - INIT.call_once(|| { - pyo3::prepare_freethreaded_python(); - }); - } #[test] fn tracing_handler_is_injected_in_python() { - initialize(); + crate::tests::initialize(); Python::with_gil(|py| { setup_python_logging(py, LogLevel::Info).unwrap(); let logging = py.import("logging").unwrap(); diff --git a/rust-runtime/aws-smithy-http-server-python/src/server.rs b/rust-runtime/aws-smithy-http-server-python/src/server.rs new file mode 100644 index 0000000000000000000000000000000000000000..83fa804f086c02df8e65e4d540a3acc0fd024ee6 --- /dev/null +++ b/rust-runtime/aws-smithy-http-server-python/src/server.rs @@ -0,0 +1,213 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ +// Code generated by software.amazon.smithy.rust.codegen.smithy-rs. DO NOT EDIT. + +use std::sync::Arc; + +use aws_smithy_http_server::Router; +use pyo3::prelude::*; + +use crate::{PyHandler, PyHandlers, PyState, SharedSocket}; + +/// Python compatible wrapper for the [aws_smithy_http_server::Router] type. +#[pyclass(text_signature = "(router)")] +#[derive(Debug, Clone)] +pub struct PyRouter(pub Router); + +/// Python application definition, holding the handlers map, the optional Python context object +/// and the asyncio task locals with the running event loop. +#[pyclass(subclass, text_signature = "()")] +#[derive(Debug, Clone)] +pub struct PyApp { + pub handlers: PyHandlers, + pub context: Option>, + pub locals: pyo3_asyncio::TaskLocals, +} + +#[pymethods] +impl PyApp { + /// Create a new instance of [PyApp]. + #[new] + pub fn new(py: Python) -> PyResult { + let asyncio = py.import("asyncio")?; + let event_loop = asyncio.call_method0("get_event_loop")?; + let locals = pyo3_asyncio::TaskLocals::new(event_loop); + Ok(Self { + handlers: PyHandlers::new(), + context: None, + locals, + }) + } + + /// Start a single worker with its own Tokio and Python async runtime and provided shared socket. + /// + /// Python asynchronous loop needs to be started and handled during the lifetime of the process. + /// First of all we install [uvloop] as the main Python event loop. Thanks to libuv, uvloop + /// performs ~20% better than Python standard event loop in most benchmarks, while being 100% + /// compatible. + /// We retrieve the Python context object, if setup by the user calling [PyApp::context] method, + /// generate the [PyState] structure and build the [aws_smithy_http_server::Router], filling + /// it with the functions generated by `PythonServerOperationHandlerGenerator.kt`. + /// At last we get a cloned reference to the underlying [socket2::Socket]. + /// + /// Now that all the setup is done, we can start the two runtimes and run the [hyper] server. + /// We spawn a thread with a new [tokio::runtime], setup the middlewares and finally block the + /// thread on `hyper::serve`. + /// The main process continues and at the end it is blocked on Python `loop.run_forever()`. + /// + /// [uvloop]: https://github.com/MagicStack/uvloop + #[pyo3(text_signature = "($self, socket, worker_number)")] + fn start_hyper_thread( + &mut self, + py: Python, + socket: &PyCell, + worker_number: isize, + ) -> PyResult<()> { + // Setup the Python asyncio loop to use `uvloop`. + let asyncio = py.import("asyncio")?; + let uvloop = py.import("uvloop")?; + uvloop.call_method0("install")?; + tracing::debug!("Setting up uvloop for current process"); + let event_loop = asyncio.call_method0("new_event_loop")?; + asyncio.call_method1("set_event_loop", (event_loop,))?; + // Create the `PyState` object from the Python context object. + let context = self.context.clone().unwrap_or_else(|| Arc::new(py.None())); + let state = PyState::new(context); + // Build the router. + let router: PyRouter = self.router(py).expect("`start_hyper_thread()` is meant to be called only by subclasses implementing the `router()` method").extract(py)?; + // Clone the socket. + let borrow = socket.try_borrow_mut()?; + let held_socket: &SharedSocket = &*borrow; + let raw_socket = held_socket.get_socket()?; + // Store Python event loop locals. + self.locals = pyo3_asyncio::TaskLocals::new(event_loop); + + // Spawn a new background [std::thread] to run the application. + tracing::debug!("Start the Tokio runtime in a background task"); + std::thread::spawn(move || { + // The thread needs a new [tokio] runtime. + let rt = tokio::runtime::Builder::new_current_thread() + .enable_all() + .thread_name(format!("smithy-rs[{}]", worker_number)) + .build() + .expect("Unable to start a new tokio runtime for this process"); + // Register operations into a Router, add middleware and start the `hyper` server, + // all inside a [tokio] blocking function. + tracing::debug!("Add middlewares to Rust Python router"); + let app = router.0.layer( + tower::ServiceBuilder::new() + .layer(aws_smithy_http_server::AddExtensionLayer::new(state)), + ); + let server = hyper::Server::from_tcp( + raw_socket + .try_into() + .expect("Unable to convert `socket2::Socket` into `std::net::TcpListener`"), + ) + .expect("Unable to create hyper server from shared socket") + .serve(app.into_make_service()); + + tracing::debug!("Starting hyper server from shared socket"); + rt.block_on(async move { + // Run forever-ish... + if let Err(err) = server.await { + tracing::error!("server error: {}", err); + } + }); + }); + // Block on the event loop forever. + tracing::debug!("Run and block on the Python event loop"); + let event_loop = (*event_loop).call_method0("run_forever"); + tracing::info!("Rust Python server started successfully"); + if event_loop.is_err() { + tracing::warn!("Ctrl-c handler, quitting"); + } + Ok(()) + } + + /// Register a new operation in the handlers map. + /// + /// The operation registered in the map are used inside the code-generated `router()` method + /// and passed to the [aws_smithy_http_server::Router] as part of the operation handlers call. + #[pyo3(text_signature = "($self, name, func)")] + pub fn register_operation(&mut self, py: Python, name: &str, func: PyObject) -> PyResult<()> { + let inspect = py.import("inspect")?; + // Check if the function is a coroutine. + // NOTE: that `asyncio.iscoroutine()` doesn't work here. + let is_coroutine = inspect + .call_method1("iscoroutinefunction", (&func,))? + .extract::()?; + // Find number of expected methods (a Python implementation could not accept the context). + let func_args = inspect + .call_method1("getargs", (func.getattr(py, "__code__")?,))? + .getattr("args")? + .extract::>()?; + let func = PyHandler { + func, + is_coroutine, + args: func_args.len(), + }; + tracing::info!( + "Registering {} function `{}` for operation {} with {} arguments", + if func.is_coroutine { "async" } else { "sync" }, + name, + func.func, + func.args + ); + // Insert the handler in the handlers map. + self.handlers + .insert(String::from(name), std::sync::Arc::new(func)); + Ok(()) + } + + /// Register a new context object inside the Rust state. + #[pyo3(text_signature = "($self, context)")] + pub fn context(&mut self, _py: Python, context: PyObject) { + self.context = Some(Arc::new(context)); + } + + /// This method is here because it is meant to be overriden by the code-generated + /// `App` structure (see PythonServerApplicationGenerator.kt) with the code needed + /// to build the [aws_smithy_http_server::Router] and register the operations on it. + #[pyo3(text_signature = "($self)")] + pub fn router(&self, _py: Python) -> Option { + None + } + + /// Main entrypoint: start the server on multiple workers. + /// + /// The multiprocessing server is achieved using the ability of a Python interpreter + /// to clone and start itself as a new process. + /// The shared sockets is created and Using the [multiprocessing::Process] module, multiple + /// workers with the method `self.start_single_python_worker()` as target are started. + /// + /// [multiprocessing::Process]: https://docs.python.org/3/library/multiprocessing.html + #[pyo3(text_signature = "($self, address, port, backlog, workers)")] + fn run( + &mut self, + py: Python, + address: Option, + port: Option, + backlog: Option, + workers: Option, + ) -> PyResult<()> { + let mp = py.import("multiprocessing")?; + mp.call_method0("allow_connection_pickling")?; + let address = address.unwrap_or_else(|| String::from("127.0.0.1")); + let port = port.unwrap_or(8080); + let socket = SharedSocket::new(address, port, backlog)?; + for idx in 0..workers.unwrap_or_else(num_cpus::get) { + let sock = socket.try_clone()?; + let process = mp.getattr("Process")?; + let handle = process.call1(( + py.None(), + self.clone().into_py(py).getattr(py, "start_hyper_worker")?, + format!("smithy-rs[{}]", idx), + (sock.into_py(py), idx), + ))?; + handle.call_method0("start")?; + } + Ok(()) + } +} diff --git a/rust-runtime/aws-smithy-http-server-python/src/socket.rs b/rust-runtime/aws-smithy-http-server-python/src/socket.rs index aa305209b89adf368c6676edbbca40b32d576e24..1d24b2f5a3a4f63999b16cede654c9e3b54e40cd 100644 --- a/rust-runtime/aws-smithy-http-server-python/src/socket.rs +++ b/rust-runtime/aws-smithy-http-server-python/src/socket.rs @@ -74,12 +74,20 @@ impl SharedSocket { /// Clone the inner socket allowing it to be shared between multiple /// Python processes. + #[pyo3(text_signature = "($self, socket, worker_number)")] pub fn try_clone(&self) -> PyResult { let copied = self.inner.try_clone()?; Ok(SharedSocket { inner: copied }) } } +impl SharedSocket { + /// Get a cloned inner socket. + pub fn get_socket(&self) -> Result { + self.inner.try_clone() + } +} + #[cfg(test)] mod tests { use super::*; diff --git a/rust-runtime/aws-smithy-http-server-python/src/state.rs b/rust-runtime/aws-smithy-http-server-python/src/state.rs new file mode 100644 index 0000000000000000000000000000000000000000..fe1fcb8d63764b2aa9ea9834788a166a076b6285 --- /dev/null +++ b/rust-runtime/aws-smithy-http-server-python/src/state.rs @@ -0,0 +1,47 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +//! [PyState] and Python handlers.. +use std::{collections::HashMap, ops::Deref, sync::Arc}; + +use pyo3::prelude::*; + +/// The Python business logic implementation needs to carry some information +/// to be executed properly like the size of its arguments and if it is +/// a coroutine. +#[derive(Debug, Clone)] +pub struct PyHandler { + pub func: PyObject, + pub args: usize, + pub is_coroutine: bool, +} + +impl Deref for PyHandler { + type Target = PyObject; + + fn deref(&self) -> &Self::Target { + &self.func + } +} + +/// Mapping holding the Python business logic handlers. +pub type PyHandlers = HashMap>; + +/// [PyState] structure holding the Python context. +/// +/// The possibility of passing the State or not is decided in Python if the method +/// `context()` is called on the `App` to register a context object. +#[pyclass] +#[derive(Debug, Clone)] +pub struct PyState { + pub context: Arc, +} + +impl PyState { + /// Create a new [PyState] structure. + pub fn new(context: Arc) -> Self { + Self { context } + } +} diff --git a/rust-runtime/aws-smithy-http-server-python/src/types.rs b/rust-runtime/aws-smithy-http-server-python/src/types.rs new file mode 100644 index 0000000000000000000000000000000000000000..42bc82c218c6d10c43591434851e69a462715a4b --- /dev/null +++ b/rust-runtime/aws-smithy-http-server-python/src/types.rs @@ -0,0 +1,100 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +//! Python wrapped types from aws-smithy-types. + +use pyo3::prelude::*; + +/// Python Wrapper for [aws_smithy_types::Blob]. +#[pyclass] +#[derive(Debug, Clone, PartialEq)] +pub struct Blob(aws_smithy_types::Blob); + +impl Blob { + /// Creates a new blob from the given `input`. + pub fn new>>(input: T) -> Self { + Self(aws_smithy_types::Blob::new(input)) + } + + /// Consumes the `Blob` and returns a `Vec` with its contents. + pub fn into_inner(self) -> Vec { + self.0.into_inner() + } +} + +impl AsRef<[u8]> for Blob { + fn as_ref(&self) -> &[u8] { + self.0.as_ref() + } +} + +#[pymethods] +impl Blob { + /// Create a new Python instance of `Blob`. + #[new] + pub fn pynew(input: Vec) -> Self { + Self(aws_smithy_types::Blob::new(input)) + } + + /// Python getter for the `Blob` byte array. + #[getter(data)] + pub fn get_data(&self) -> &[u8] { + self.as_ref() + } + + /// Python setter for the `Blob` byte array. + #[setter(data)] + pub fn set_data(&mut self, data: Vec) { + *self = Self::pynew(data); + } +} + +#[cfg(test)] +mod tests { + use pyo3::py_run; + + use super::*; + + #[test] + fn blob_can_be_used_in_python_when_initialized_in_rust() { + crate::tests::initialize(); + Python::with_gil(|py| { + let blob = Blob::new("some data".as_bytes().to_vec()); + let blob = PyCell::new(py, blob).unwrap(); + py_run!( + py, + blob, + r#" + assert blob.data == b"some data" + assert len(blob.data) == 9 + blob.data = b"some other data" + assert blob.data == b"some other data" + assert len(blob.data) == 15 + "# + ); + }) + } + + #[test] + fn blob_can_be_initialized_in_python() { + crate::tests::initialize(); + Python::with_gil(|py| { + let types = PyModule::new(py, "types").unwrap(); + types.add_class::().unwrap(); + py_run!( + py, + types, + r#" + blob = types.Blob(b"some data") + assert blob.data == b"some data" + assert len(blob.data) == 9 + blob.data = b"some other data" + assert blob.data == b"some other data" + assert len(blob.data) == 15 + "# + ); + }) + } +} diff --git a/rust-runtime/aws-smithy-http-server/src/routing/into_make_service.rs b/rust-runtime/aws-smithy-http-server/src/routing/into_make_service.rs index 05a34111ae7a813731146960a7f61c06a4d5afb8..324b9d59dc40acccf56cb57130952b12f02492c7 100644 --- a/rust-runtime/aws-smithy-http-server/src/routing/into_make_service.rs +++ b/rust-runtime/aws-smithy-http-server/src/routing/into_make_service.rs @@ -48,7 +48,7 @@ pub struct IntoMakeService { } impl IntoMakeService { - pub(super) fn new(service: S) -> Self { + pub fn new(service: S) -> Self { Self { service } } } diff --git a/rust-runtime/aws-smithy-http-server/src/routing/mod.rs b/rust-runtime/aws-smithy-http-server/src/routing/mod.rs index 2f4fa6c94f1bf29ed4c5787f44c70e8b2aa2ab1c..7ce1485b49b27c7369f9c6e28b2f2ed0ece3a6fc 100644 --- a/rust-runtime/aws-smithy-http-server/src/routing/mod.rs +++ b/rust-runtime/aws-smithy-http-server/src/routing/mod.rs @@ -7,7 +7,6 @@ //! //! [Smithy specification]: https://awslabs.github.io/smithy/1.0/spec/core/http-traits.html -use self::future::RouterFuture; use self::request_spec::RequestSpec; use self::tiny_map::TinyMap; use crate::body::{boxed, Body, BoxBody, HttpBody}; @@ -34,7 +33,7 @@ pub mod request_spec; mod route; mod tiny_map; -pub use self::{into_make_service::IntoMakeService, route::Route}; +pub use self::{future::RouterFuture, into_make_service::IntoMakeService, route::Route}; /// The router is a [`tower::Service`] that routes incoming requests to other `Service`s /// based on the request's URI and HTTP method or on some specific header setting the target operation.