Unverified Commit a2d37ad2 authored by Matteo Bigoi's avatar Matteo Bigoi Committed by GitHub
Browse files

[Python] Automatically generate stubs (#2576) 



## Motivation and Context
We want to automatically generate stubs in the codegen diff to ensure
they can be reviewed and have a simple way to generate and include the
stubs inside the Maturin wheel.

## Description
The Python example has been moved to the `examples` folder and
refactored. The refactoring
ensures the script `stubgen.py` is included in the codegeneration of the
SDK crate. The script is later used to generate stubs automatically
during testing and can be used by customers to add their own stubs
before the Maturin build

----

_By submitting this pull request, I confirm that you can use, modify,
copy, and redistribute this contribution, under the terms of your
choice._

----

_By submitting this pull request, I confirm that you can use, modify,
copy, and redistribute this contribution, under the terms of your
choice._

---------

Signed-off-by: default avatarBigo <1781140+crisidev@users.noreply.github.com>
Co-authored-by: default avatarBurak <unexge@gmail.com>
parent fc63800f

codegen-core/src/main/kotlin/software/amazon/smithy/rust/codegen/core/rustlang/RustWriter.kt

+1 −0
Original line number Diff line number Diff line
@@ -469,6 +469,7 @@ class RustWriter private constructor( 
                    devDependenciesOnly = true,

                )

                fileName == "package.json" -> rawWriter(fileName, debugMode = debugMode)

                fileName == "stubgen.sh" -> rawWriter(fileName, debugMode = debugMode)

                else -> RustWriter(fileName, namespace, debugMode = debugMode)

            }

        }

codegen-core/src/main/kotlin/software/amazon/smithy/rust/codegen/core/smithy/RuntimeType.kt

+2 −0
Original line number Diff line number Diff line
@@ -96,6 +96,8 @@ data class RuntimeConfig( 


    val crateSrcPrefix: String = cratePrefix.replace("-", "_")



    fun runtimeCratesPath(): String? = runtimeCrateLocation.path



    fun smithyRuntimeCrate(

        runtimeCrateName: String,

        optional: Boolean = false,

codegen-server-test/python/build.gradle.kts

+16 −1
Original line number Diff line number Diff line
@@ -66,7 +66,7 @@ val allCodegenTests = "../../codegen-core/common-test-models".let { commonModels 
            "rest_json_extras",

            imports = listOf("$commonModels/rest-json-extras.smithy"),

        ),

        // TODO(https://github.com/awslabs/smithy-rs/issues/2551)

        // TODO(https://github.com/awslabs/smithy-rs/issues/2477)

        // CodegenTest(

        //     "aws.protocoltests.restjson.validation#RestJsonValidation",

        //     "rest_json_validation",
@@ -104,6 +104,21 @@ project.registerGenerateSmithyBuildTask(rootProject, pluginName, allCodegenTests 
project.registerGenerateCargoWorkspaceTask(rootProject, pluginName, allCodegenTests, workingDirUnderBuildDir)

project.registerGenerateCargoConfigTomlTask(buildDir.resolve(workingDirUnderBuildDir))



tasks.register("stubs") {

    description = "Generate Python stubs for all models"

    dependsOn("assemble")



    doLast {

        allCodegenTests.forEach { test ->

            val crateDir = "$buildDir/$workingDirUnderBuildDir/${test.module}/$pluginName"

            val moduleName = test.module.replace("-", "_")

            exec {

                commandLine("bash", "$crateDir/stubgen.sh", moduleName, "$crateDir/Cargo.toml", "$crateDir/python/$moduleName")

            }

        }

    }

}



tasks["smithyBuildJar"].dependsOn("generateSmithyBuild")

tasks["assemble"].finalizedBy("generateCargoWorkspace")

codegen-server/python/src/main/kotlin/software/amazon/smithy/rust/codegen/server/python/smithy/PythonServerCargoDependency.kt

+2 −2
Original line number Diff line number Diff line
@@ -15,8 +15,8 @@ import software.amazon.smithy.rust.codegen.core.smithy.RuntimeConfig 
 * For a dependency that is used in the client, or in both the client and the server, use [CargoDependency] directly.

 */

object PythonServerCargoDependency {

    val PyO3: CargoDependency = CargoDependency("pyo3", CratesIo("0.17"))

    val PyO3Asyncio: CargoDependency = CargoDependency("pyo3-asyncio", CratesIo("0.17"), features = setOf("attributes", "tokio-runtime"))

    val PyO3: CargoDependency = CargoDependency("pyo3", CratesIo("0.18"))

    val PyO3Asyncio: CargoDependency = CargoDependency("pyo3-asyncio", CratesIo("0.18"), features = setOf("attributes", "tokio-runtime"))

    val Tokio: CargoDependency = CargoDependency("tokio", CratesIo("1.20.1"), features = setOf("full"))

    val TokioStream: CargoDependency = CargoDependency("tokio-stream", CratesIo("0.1.12"))

    val Tracing: CargoDependency = CargoDependency("tracing", CratesIo("0.1"))

codegen-server/python/src/main/kotlin/software/amazon/smithy/rust/codegen/server/python/smithy/customizations/PythonServerCodegenDecorator.kt

+28 −0
Original line number Diff line number Diff line
@@ -23,6 +23,7 @@ import software.amazon.smithy.rust.codegen.server.python.smithy.generators.Pytho 
import software.amazon.smithy.rust.codegen.server.smithy.ServerCodegenContext

import software.amazon.smithy.rust.codegen.server.smithy.customizations.AddInternalServerErrorToAllOperationsDecorator

import software.amazon.smithy.rust.codegen.server.smithy.customize.ServerCodegenDecorator

import java.io.File



/**

 * Configure the [lib] section of `Cargo.toml`.
@@ -194,6 +195,31 @@ class PyTypedMarkerDecorator : ServerCodegenDecorator { 
    }

}



/**

 * Copies the stubgen scripts to the generated crate root.

 *

 * The shell script `stubgen.sh` runs a quick build and uses `stubgen.py` to generate mypy compatibile

 * types stubs for the project.

 */

class AddStubgenScriptDecorator : ServerCodegenDecorator {

    override val name: String = "AddStubgenScriptDecorator"

    override val order: Byte = 0



    override fun extras(codegenContext: ServerCodegenContext, rustCrate: RustCrate) {

        val runtimeCratesPath = codegenContext.runtimeConfig.runtimeCratesPath()

        val stubgenPythonLocation = "$runtimeCratesPath/aws-smithy-http-server-python/stubgen.py"

        val stubgenPythonContent = File(stubgenPythonLocation).readText(Charsets.UTF_8)

        rustCrate.withFile("stubgen.py") {

            writeWithNoFormatting("$stubgenPythonContent")

        }

        val stubgenShellLocation = "$runtimeCratesPath/aws-smithy-http-server-python/stubgen.sh"

        val stubgenShellContent = File(stubgenShellLocation).readText(Charsets.UTF_8)

        rustCrate.withFile("stubgen.sh") {

            writeWithNoFormatting("$stubgenShellContent")

        }

    }

}



val DECORATORS = arrayOf(

    /**

     * Add the [InternalServerError] error to all operations.
@@ -214,4 +240,6 @@ val DECORATORS = arrayOf( 
    InitPyDecorator(),

    // Generate `py.typed` for the Python source.

    PyTypedMarkerDecorator(),

    // Generate scripts for stub generation.

    AddStubgenScriptDecorator(),

)