Add more docs to codegen (#768)

import software.amazon.smithy.rust.codegen.util.runCommand

import java.util.logging.Logger

/**

 * Base Entrypoint for Code generation

 */

class CodegenVisitor(context: PluginContext, private val codegenDecorator: RustCodegenDecorator) :

    ShapeVisitor.Default<Unit>() {

        httpGenerator = protocolGenerator.buildProtocolGenerator(codegenContext)

    }

    /**

     * Base model transformation applied to all services

     * See below for details.

     */

    private fun baselineTransform(model: Model) =

        // Add `Box<T>` to recursive shapes as necessary

        model.let(RecursiveShapeBoxer::transform)

            // Normalize the `message` field on errors when enabled in settings (default: true)

            .letIf(settings.codegenConfig.addMessageToErrors, AddErrorMessage::transform)

            // NormalizeOperations by ensuring every operation has an input & output shape

            .let(OperationNormalizer::transform)

            // Drop unsupported event stream operations from the model

            .let { RemoveEventStreamOperations.transform(it, settings) }

            // - Normalize event stream operations

            .let(EventStreamNormalizer::transform)

    /**

     * Execute code generation

     *

     * 1. Load the service from RustSettings

     * 2. Traverse every shape in the closure of the service.

     * 3. Loop through each shape and visit them (calling the override functions in this class)

     * 4. Call finalization tasks specified by decorators.

     * 5. Write the in-memory buffers out to files.

     *

     * The main work of code generation (serializers, protocols, etc.) is handled in `fn serviceShape` below.

     */

    fun execute() {

        logger.info("generating Rust client...")

        val service = settings.getService(model)

        val serviceShapes = Walker(model).walkShapes(service)

        serviceShapes.forEach { it.accept(this) }

        codegenDecorator.extras(codegenContext, rustCrate)

        // finalize actually writes files into the base directory, renders any inline functions that were used, and

        // performs finalization like generating a Cargo.toml

        rustCrate.finalize(

            settings,

            codegenDecorator.libRsCustomizations(

        logger.info("Rust Client generation complete!")

    }

    /**

     * Generate service-specific code for the model:

     * - Serializers

     * - Deserializers

     * - Fluent client

     * - Trait implementations

     * - Protocol tests

     * - Operation structures

     */

    override fun serviceShape(shape: ServiceShape) {

        ServiceGenerator(

            rustCrate,

            httpGenerator,

            protocolGenerator.support(),

            codegenContext,

            codegenDecorator

        ).render()

    }

    override fun getDefault(shape: Shape?) {

    }

    /**

     * Structure Shape Visitor

     *

     * For each structure shape, generate:

     * - A Rust structure for the shape (StructureGenerator)

     * - A builder for the shape

     *

     * This function _does not_ generate any serializers

     */

    override fun structureShape(shape: StructureShape) {

        logger.fine("generating a structure...")

        rustCrate.useShapeWriter(shape) { writer ->

        }

    }

    /**

     * String Shape Visitor

     *

     * Although raw strings require no code generation, enums are actually `EnumTrait` applied to string shapes.

     * For strings that have the enum trait attached,

     */

    override fun stringShape(shape: StringShape) {

        shape.getTrait<EnumTrait>()?.also { enum ->

            rustCrate.useShapeWriter(shape) { writer ->

        }

    }

    /**

     * Union Shape Visitor

     *

     * Generate an `enum` for union shapes.

     *

     * Note: this does not generate serializers

     */

    override fun unionShape(shape: UnionShape) {

        rustCrate.useShapeWriter(shape) {

            UnionGenerator(model, symbolProvider, it, shape).render()

        }

    }

    override fun serviceShape(shape: ServiceShape) {

        ServiceGenerator(

            rustCrate,

            httpGenerator,

            protocolGenerator.support(),

            codegenContext,

            codegenDecorator

        ).render()

    }

}

import java.util.logging.Level

import java.util.logging.Logger

/** Rust Codegen Plugin

 *  This is the entrypoint for code generation, triggered by the smithy-build plugin.

 *  `resources/META-INF.services/software.amazon.smithy.build.SmithyBuildPlugin` refers to this class by name which

 *  enables the smithy-build plugin to invoke `execute` with all of the Smithy plugin context + models.

 */

class RustCodegenPlugin : SmithyBuildPlugin {

    override fun getName(): String = "rust-codegen"

    override fun execute(context: PluginContext) {

        // Suppress extremely noisy logs about reserved words

        Logger.getLogger(ReservedWordSymbolProvider::class.java.name).level = Level.OFF

        // Discover `RustCodegenDecorators` on the classpath. `RustCodegenDectorator` return different types of

        // customization. A customization is a function of:

        // - location (eg. the mutate section of an operation)

        // - context (eg. the of the operation)

        // - writer: The active RustWriter at the given location

        val codegenDecorator = CombinedCodegenDecorator.fromClasspath(context)

        // CodegenVistor is the main driver of code generation that traverses the model and generates code

        CodegenVisitor(context, codegenDecorator).execute()

    }

    companion object {

        /** SymbolProvider

         * When generating code, smithy types need to be converted into Rust types—that is the core role of the symbol provider

         *

         * The Symbol provider is composed of a base `SymbolVisitor` which handles the core funcitonality, then is layered

         * with other symbol providers, documented inline, to handle the full scope of Smithy types.

         */

        fun baseSymbolProvider(model: Model, serviceShape: ServiceShape, symbolVisitorConfig: SymbolVisitorConfig = DefaultConfig) =

            SymbolVisitor(model, serviceShape = serviceShape, config = symbolVisitorConfig)

                // Generate different types for EventStream shapes (eg. transcribe streaming)

                .let { EventStreamSymbolProvider(symbolVisitorConfig.runtimeConfig, it, model) }

                // Generate `ByteStream` instead of `Blob` for streaming binary shapes (eg. S3 GetObject)

                .let { StreamingShapeSymbolProvider(it, model) }

                // Add Rust attributes (like `#[derive(PartialEq)]`) to generated shapes

                .let { BaseSymbolMetadataProvider(it) }

                // Streaming shapes need different derives (eg. they cannot derive Eq)

                .let { StreamingShapeMetadataProvider(it, model) }

                // Rename shapes that clash with Rust reserved words & and other SDK specific features eg. `send()` cannot

                // be the name of an operation input

                .let { RustReservedWordSymbolProvider(it) }

    }

}

import software.amazon.smithy.rust.codegen.smithy.generators.protocol.ProtocolTestGenerator

import software.amazon.smithy.rust.codegen.util.inputShape

/**

 * ServiceGenerator

 *

 * Service generator is the main codegeneration entry point for Smithy services. Individual structures and unions are

 * generated in codegen visitor, but this class handles all protocol-specific code generation.

 */

class ServiceGenerator(

    private val rustCrate: RustCrate,

    private val protocolGenerator: ProtocolGenerator,

) {

    private val index = TopDownIndex.of(config.model)

    /**

     * Render Service Specific code. Code will end up in different files via `useShapeWriter`. See `SymbolVisitor.kt`

     * which assigns a symbol location to each shape.

     *

     */

    fun render() {

        val operations = index.getContainedOperations(config.serviceShape).sortedBy { it.id }

        operations.map { operation ->

            rustCrate.useShapeWriter(operation) { operationWriter ->

                rustCrate.useShapeWriter(operation.inputShape(config.model)) { inputWriter ->

                    // Render the operation shape & serializers input `input.rs`

                    protocolGenerator.renderOperation(

                        operationWriter,

                        inputWriter,

                        operation,

                        decorator.operationCustomizations(config, operation, listOf())

                    )

                    // render protocol tests into `operation.rs` (note operationWriter vs. inputWriter)

                    ProtocolTestGenerator(config, protocolSupport, operation, operationWriter).render()

                }

            }

            // Render a service-level error enum containing every error that the service can emit

            rustCrate.withModule(RustModule.Error) { writer ->

                CombinedErrorGenerator(config.model, config.symbolProvider, operation).render(writer)

            }

package software.amazon.smithy.rust.codegen.smithy.generators.protocol

import software.amazon.smithy.model.shapes.OperationShape

import software.amazon.smithy.model.shapes.StructureShape

import software.amazon.smithy.rust.codegen.rustlang.Attribute

import software.amazon.smithy.rust.codegen.rustlang.CargoDependency

import software.amazon.smithy.rust.codegen.rustlang.RustWriter

import software.amazon.smithy.rust.codegen.smithy.protocols.Protocol

import software.amazon.smithy.rust.codegen.util.inputShape

/**

 * Request Body Generator

 *

 * **Note:** There is only one real implementation of this interface. The other implementation is test only.

 * All protocols use the same class.

 *

 * Different protocols (eg. JSON vs. XML) need to use different functionality to generate request bodies.

 */

interface ProtocolBodyGenerator {

    data class BodyMetadata(val takesOwnership: Boolean)

    /**

     * Code generation needs to handle whether or not `generateBody` takes ownership of the input for a given operation shape

     *

     * Most operations will parse the HTTP body as a reference, but for operations that will consume the entire stream later,

     * they will need to take ownership and different code needs to be generated.

     */

    fun bodyMetadata(operationShape: OperationShape): BodyMetadata

    /**

     * Write the body into [writer]

     *

     * This should be an expression that returns an `SdkBody`

     */

    fun generateBody(writer: RustWriter, self: String, operationShape: OperationShape)

}

/**

 * Protocol Trait implementation generator

 *

 * **Note:** There is only one real implementation of this interface. The other implementation is test only.

 * All protocols use the same class.

 *

 * Protocols implement one of two traits to enable parsing HTTP responses:

 * 1. `ParseHttpResponse`: Streaming binary operations

 * 2. `ParseStrictResponse`: Non-streaming operations for the body must be "strict" (as in, not lazy) where the parser

 *                           must have the complete body to return a result.

 */

interface ProtocolTraitImplGenerator {

    fun generateTraitImpls(operationWriter: RustWriter, operationShape: OperationShape)

}

 */

open class ProtocolGenerator(

    codegenContext: CodegenContext,

    /**

     * `Protocol` contains all protocol specific information. Each smithy protocol, eg. RestJson, RestXml, etc. will

     * have their own implementation of the protocol interface which defines how an input shape becomes and http::Request

     * and an output shape is build from an http::Response.

     */

    private val protocol: Protocol,

    /**

     * Operations generate a `make_operation(&config)` method to build a `smithy_http::Operation` that can be dispatched

     * This is the serializer side of request dispatch

     */

    private val makeOperationGenerator: MakeOperationGenerator,

    /**

     * Operations generate implementations of ParseHttpResponse or ParseStrictResponse.

     * This is the deserializer side of request dispatch (parsing the response)

     */

    private val traitGenerator: ProtocolTraitImplGenerator,

) {

    private val runtimeConfig = codegenContext.runtimeConfig

        "operation" to RuntimeType.operationModule(runtimeConfig),

    )

    /**

     * Render all code required for serializing requests and deserializing responses for the operation

     *

     * This primarily relies on two components:

     * 1. [traitGenerator]: Generate implementations of the `ParseHttpResponse` trait for the operations

     * 2. [makeOperationGenerator]: Generate the `make_operation()` method which is used to serialize operations

     *    to HTTP requests

     */

    fun renderOperation(

        operationWriter: RustWriter,

        inputWriter: RustWriter,

        val builderGenerator = BuilderGenerator(model, symbolProvider, operationShape.inputShape(model))

        builderGenerator.render(inputWriter)

        // TODO: One day, it should be possible for callers to invoke

        // buildOperationType* directly to get the type rather than depending

        // on these aliases.

        val operationTypeOutput = buildOperationTypeOutput(inputWriter, operationShape)

        val operationTypeRetry = buildOperationTypeRetry(inputWriter, customizations)

        val inputPrefix = symbolProvider.toSymbol(inputShape).name

        inputWriter.rust(

            """

            ##[doc(hidden)] pub type ${inputPrefix}OperationOutputAlias = $operationTypeOutput;

            ##[doc(hidden)] pub type ${inputPrefix}OperationRetryAlias = $operationTypeRetry;

            """

        )

        // generate type aliases for the fluent builders

        renderTypeAliases(inputWriter, operationShape, customizations, inputShape)

        // impl OperationInputShape { ... }

        val operationName = symbolProvider.toSymbol(operationShape).name

        traitGenerator.generateTraitImpls(operationWriter, operationShape)

    }

    private fun renderTypeAliases(

        inputWriter: RustWriter,

        operationShape: OperationShape,

        customizations: List<OperationCustomization>,

        inputShape: StructureShape

    ) {

        // TODO: One day, it should be possible for callers to invoke

        // buildOperationType* directly to get the type rather than depending

        // on these aliases.

        // These are used in fluent clients

        val operationTypeOutput = buildOperationTypeOutput(inputWriter, operationShape)

        val operationTypeRetry = buildOperationTypeRetry(inputWriter, customizations)

        val inputPrefix = symbolProvider.toSymbol(inputShape).name

        inputWriter.rust(

            """

                ##[doc(hidden)] pub type ${inputPrefix}OperationOutputAlias = $operationTypeOutput;

                ##[doc(hidden)] pub type ${inputPrefix}OperationRetryAlias = $operationTypeRetry;

                """

        )

    }

    private fun buildOperationTypeOutput(writer: RustWriter, shape: OperationShape): String =

        writer.format(symbolProvider.toSymbol(shape))

    private val model = codegenContext.model

    private val runtimeConfig = codegenContext.runtimeConfig

    private val httpBindingResolver = protocol.httpBindingResolver

    private val operationSerModule = RustModule.private("operation_ser")

    private val operationDeserModule = RustModule.private("operation_deser")

    private val codegenScope = arrayOf(