Unverified Commit 8e84ee2e authored by Matteo Bigoi's avatar Matteo Bigoi Committed by GitHub
Browse files

Codegenerate StructureShape, BlobShape, application, server and Python runtime (#1403) 



We are introducing code-generation for Python bindings of StructureShape, BlobShape, EnumShape, OperationShape.

This PR also add a runtime server implementation to support serving Python business logic directly and with an idiomatic experience.

Co-authored-by: default avatardavid-perez <d@vidp.dev>
parent 8911e865

codegen-server-test/python/build.gradle.kts

+1 −8
Original line number Diff line number Diff line
@@ -16,7 +16,7 @@ val defaultRustFlags: String by project 
val defaultRustDocFlags: String by project

val properties = PropertyRetriever(rootProject, project)



val pluginName = "rust-server-codegen"

val pluginName = "rust-server-codegen-python"

val workingDirUnderBuildDir = "smithyprojections/codegen-server-test-python/"



configure<software.amazon.smithy.gradle.SmithyExtension> {
@@ -39,13 +39,6 @@ dependencies { 


val allCodegenTests = listOf(

    CodegenTest("com.amazonaws.simple#SimpleService", "simple"),

    CodegenTest("aws.protocoltests.restjson#RestJson", "rest_json"),

    CodegenTest("aws.protocoltests.restjson.validation#RestJsonValidation", "rest_json_validation"),

    CodegenTest("aws.protocoltests.json10#JsonRpc10", "json_rpc10"),

    CodegenTest("aws.protocoltests.json#JsonProtocol", "json_rpc11"),

    CodegenTest("aws.protocoltests.misc#MiscService", "misc"),

    CodegenTest("com.amazonaws.ebs#Ebs", "ebs"),

    CodegenTest("com.amazonaws.s3#AmazonS3", "s3"),

    CodegenTest("com.aws.example#PokemonService", "pokemon_service_sdk")

)

codegen-server/python/src/main/kotlin/software/amazon/smithy/rust/codegen/server/python/smithy/RustCodegenServerPlugin.kt→codegen-server/python/src/main/kotlin/software/amazon/smithy/rust/codegen/server/python/smithy/PythonCodegenServerPlugin.kt

+12 −9
Original line number Diff line number Diff line
@@ -11,7 +11,6 @@ import software.amazon.smithy.codegen.core.ReservedWordSymbolProvider 
import software.amazon.smithy.model.Model

import software.amazon.smithy.model.shapes.ServiceShape

import software.amazon.smithy.rust.codegen.rustlang.RustReservedWordSymbolProvider

import software.amazon.smithy.rust.codegen.server.smithy.ServerCodegenVisitor

import software.amazon.smithy.rust.codegen.smithy.BaseSymbolMetadataProvider

import software.amazon.smithy.rust.codegen.smithy.DefaultConfig

import software.amazon.smithy.rust.codegen.smithy.EventStreamSymbolProvider
@@ -23,12 +22,13 @@ import software.amazon.smithy.rust.codegen.smithy.customize.CombinedCodegenDecor 
import java.util.logging.Level

import java.util.logging.Logger



/** Rust with Python bindings Codegen Plugin.

/**

 * Rust with Python bindings Codegen Plugin.

 * This is the entrypoint for code generation, triggered by the smithy-build plugin.

 * `resources/META-INF.services/software.amazon.smithy.build.SmithyBuildPlugin` refers to this class by name which

 * enables the smithy-build plugin to invoke `execute` with all of the Smithy plugin context + models.

 */

class RustCodegenServerPlugin : SmithyBuildPlugin {

class PythonCodegenServerPlugin : SmithyBuildPlugin {

    private val logger = Logger.getLogger(javaClass.name)



    override fun getName(): String = "rust-server-codegen-python"
@@ -43,9 +43,9 @@ class RustCodegenServerPlugin : SmithyBuildPlugin { 
        // - writer: The active RustWriter at the given location

        val codegenDecorator = CombinedCodegenDecorator.fromClasspath(context)



        // ServerCodegenVisitor is the main driver of code generation that traverses the model and generates code

        logger.info("Loaded plugin to generate Rust/Python bindings for the server SSDK")

        ServerCodegenVisitor(context, codegenDecorator).execute()

        // PythonServerCodegenVisitor is the main driver of code generation that traverses the model and generates code

        logger.info("Loaded plugin to generate Rust/Python bindings for the server SSDK for projection ${context.projectionName}")

        PythonServerCodegenVisitor(context, codegenDecorator).execute()

    }



    companion object {
@@ -61,6 +61,9 @@ class RustCodegenServerPlugin : SmithyBuildPlugin { 
            symbolVisitorConfig: SymbolVisitorConfig = DefaultConfig

        ) =

            SymbolVisitor(model, serviceShape = serviceShape, config = symbolVisitorConfig)

                // Rename a set of symbols that do not implement `PyClass` and have been wrapped in

                // `aws_smithy_http_server_python::types`.

                .let { PythonServerSymbolProvider(it) }

                // Generate different types for EventStream shapes (e.g. transcribe streaming)

                .let {

                    EventStreamSymbolProvider(symbolVisitorConfig.runtimeConfig, it, model)

codegen-server/python/src/main/kotlin/software/amazon/smithy/rust/codegen/server/python/smithy/PythonServerCargoDependency.kt

0 → 100644
+29 −0
Original line number Diff line number Diff line 
/*

 * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.

 * SPDX-License-Identifier: Apache-2.0

 */



package software.amazon.smithy.rust.codegen.server.python.smithy



import software.amazon.smithy.rust.codegen.rustlang.CargoDependency

import software.amazon.smithy.rust.codegen.rustlang.CratesIo

import software.amazon.smithy.rust.codegen.smithy.RuntimeConfig



/**

 * Object used *exclusively* in the runtime of the Python server, for separation concerns.

 * Analogous to the companion object in [CargoDependency] and [ServerCargoDependency]; see its documentation for details.

 * For a dependency that is used in the client, or in both the client and the server, use [CargoDependency] directly.

 */

object PythonServerCargoDependency {

    val PyO3: CargoDependency = CargoDependency("pyo3", CratesIo("0.16"), features = setOf("extension-module"))

    val PyO3Asyncio: CargoDependency = CargoDependency("pyo3-asyncio", CratesIo("0.16"), features = setOf("attributes", "tokio-runtime"))

    val Tokio: CargoDependency = CargoDependency("tokio", CratesIo("1.0"), features = setOf("full"))

    val Tracing: CargoDependency = CargoDependency("tracing", CratesIo("0.1"))

    val Tower: CargoDependency = CargoDependency("tower", CratesIo("0.4"))

    val TowerHttp: CargoDependency = CargoDependency("tower-http", CratesIo("0.3"), features = setOf("trace"))

    val Hyper: CargoDependency = CargoDependency("hyper", CratesIo("0.14"), features = setOf("server", "http1", "http2", "tcp", "stream"))

    val NumCpus: CargoDependency = CargoDependency("num_cpus", CratesIo("1.13"))



    fun SmithyHttpServer(runtimeConfig: RuntimeConfig) = runtimeConfig.runtimeCrate("http-server")

    fun SmithyHttpServerPython(runtimeConfig: RuntimeConfig) = runtimeConfig.runtimeCrate("http-server-python")

}

codegen-server/python/src/main/kotlin/software/amazon/smithy/rust/codegen/server/python/smithy/PythonServerCodegenVisitor.kt

0 → 100644
+131 −0
Original line number Diff line number Diff line 


/*

 * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.

 * SPDX-License-Identifier: Apache-2.0

 */



package software.amazon.smithy.rust.codegen.server.python.smithy



import software.amazon.smithy.build.PluginContext

import software.amazon.smithy.model.shapes.ServiceShape

import software.amazon.smithy.model.shapes.StringShape

import software.amazon.smithy.model.shapes.StructureShape

import software.amazon.smithy.model.traits.EnumTrait

import software.amazon.smithy.rust.codegen.server.python.smithy.generators.PythonServerEnumGenerator

import software.amazon.smithy.rust.codegen.server.python.smithy.generators.PythonServerServiceGenerator

import software.amazon.smithy.rust.codegen.server.python.smithy.generators.PythonServerStructureGenerator

import software.amazon.smithy.rust.codegen.server.smithy.ServerCodegenVisitor

import software.amazon.smithy.rust.codegen.server.smithy.protocols.ServerProtocolLoader

import software.amazon.smithy.rust.codegen.smithy.CodegenContext

import software.amazon.smithy.rust.codegen.smithy.DefaultPublicModules

import software.amazon.smithy.rust.codegen.smithy.RustCrate

import software.amazon.smithy.rust.codegen.smithy.SymbolVisitorConfig

import software.amazon.smithy.rust.codegen.smithy.customize.RustCodegenDecorator

import software.amazon.smithy.rust.codegen.smithy.generators.BuilderGenerator

import software.amazon.smithy.rust.codegen.smithy.generators.CodegenTarget

import software.amazon.smithy.rust.codegen.smithy.generators.implBlock

import software.amazon.smithy.rust.codegen.util.getTrait



/**

 * Entrypoint for Python server-side code generation. This class will walk the in-memory model and

 * generate all the needed types by calling the accept() function on the available shapes.

 *

 * This class inherits from [ServerCodegenVisitor] since it uses most of the functionlities of the super class

 * and have to override the symbol provider with [PythonServerSymbolProvider].

 */

class PythonServerCodegenVisitor(context: PluginContext, codegenDecorator: RustCodegenDecorator) :

    ServerCodegenVisitor(context, codegenDecorator) {



    init {

        val symbolVisitorConfig =

            SymbolVisitorConfig(

                runtimeConfig = settings.runtimeConfig,

                codegenConfig = settings.codegenConfig,

                handleRequired = true

            )

        val baseModel = baselineTransform(context.model)

        val service = settings.getService(baseModel)

        val (protocol, generator) =

            ServerProtocolLoader(

                codegenDecorator.protocols(

                    service.id,

                    ServerProtocolLoader.DefaultProtocols

                )

            )

                .protocolFor(context.model, service)

        protocolGeneratorFactory = generator

        model = generator.transformModel(codegenDecorator.transformModel(service, baseModel))

        val baseProvider = PythonCodegenServerPlugin.baseSymbolProvider(model, service, symbolVisitorConfig)

        // Override symbolProvider.

        symbolProvider =

            codegenDecorator.symbolProvider(generator.symbolProvider(model, baseProvider))



        // Override `codegenContext` which carries the symbolProvider.

        codegenContext = CodegenContext(model, symbolProvider, service, protocol, settings, target = CodegenTarget.SERVER)



        // Override `rustCrate` which carries the symbolProvider.

        rustCrate = RustCrate(context.fileManifest, symbolProvider, DefaultPublicModules, settings.codegenConfig)

        // Override `protocolGenerator` which carries the symbolProvider.

        protocolGenerator = protocolGeneratorFactory.buildProtocolGenerator(codegenContext)

    }



    /**

     * Structure Shape Visitor

     *

     * For each structure shape, generate:

     * - A Rust structure for the shape ([StructureGenerator]).

     * - `pyo3::PyClass` trait implementation.

     * - A builder for the shape.

     *

     * This function _does not_ generate any serializers.

     */

    override fun structureShape(shape: StructureShape) {

        logger.info("[python-server-codegen] Generating a structure $shape")

        rustCrate.useShapeWriter(shape) { writer ->

            // Use Python specific structure generator that adds the #[pyclass] attribute

            // and #[pymethods] implementation.

            PythonServerStructureGenerator(model, symbolProvider, writer, shape).render(CodegenTarget.SERVER)

            val builderGenerator =

                BuilderGenerator(codegenContext.model, codegenContext.symbolProvider, shape)

            builderGenerator.render(writer)

            writer.implBlock(shape, symbolProvider) {

                builderGenerator.renderConvenienceMethod(this)

            }

        }

    }



    /**

     * String Shape Visitor

     *

     * Although raw strings require no code generation, enums are actually [EnumTrait] applied to string shapes.

     */

    override fun stringShape(shape: StringShape) {

        logger.info("[rust-server-codegen] Generating an enum $shape")

        shape.getTrait<EnumTrait>()?.also { enum ->

            rustCrate.useShapeWriter(shape) { writer ->

                PythonServerEnumGenerator(model, symbolProvider, writer, shape, enum, codegenContext.runtimeConfig).render()

            }

        }

    }



    /**

     * Generate service-specific code for the model:

     * - Serializers

     * - Deserializers

     * - Trait implementations

     * - Protocol tests

     * - Operation structures

     * - Python operation handlers

     */

    override fun serviceShape(shape: ServiceShape) {

        logger.info("[python-server-codegen] Generating a service $shape")

        PythonServerServiceGenerator(

            rustCrate,

            protocolGenerator,

            protocolGeneratorFactory.support(),

            protocolGeneratorFactory.protocol(codegenContext).httpBindingResolver,

            codegenContext,

        )

            .render()

    }

}

codegen-server/python/src/main/kotlin/software/amazon/smithy/rust/codegen/server/python/smithy/PythonServerRuntimeType.kt

0 → 100644
+26 −0
Original line number Diff line number Diff line 
/*

 * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.

 * SPDX-License-Identifier: Apache-2.0

 */



package software.amazon.smithy.rust.codegen.server.python.smithy



import software.amazon.smithy.rust.codegen.smithy.RuntimeConfig

import software.amazon.smithy.rust.codegen.smithy.RuntimeType



/**

 * Object used *exclusively* in the runtime of the Python server, for separation concerns.

 * Analogous to the companion object in [RuntimeType] and [ServerRuntimeType]; see its documentation for details.

 * For a runtime type that is used in the client, or in both the client and the server, use [RuntimeType] directly.

 */

object PythonServerRuntimeType {



    fun SharedSocket(runtimeConfig: RuntimeConfig) =

        RuntimeType("SharedSocket", PythonServerCargoDependency.SmithyHttpServerPython(runtimeConfig), "${runtimeConfig.crateSrcPrefix}_http_server_python")



    fun Blob(runtimeConfig: RuntimeConfig) =

        RuntimeType("Blob", PythonServerCargoDependency.SmithyHttpServerPython(runtimeConfig), "${runtimeConfig.crateSrcPrefix}_http_server_python::types")



    fun PyError(runtimeConfig: RuntimeConfig) =

        RuntimeType("Error", PythonServerCargoDependency.SmithyHttpServerPython(runtimeConfig), "${runtimeConfig.crateSrcPrefix}_http_server_python")

}