Unverified Commit 9af72f5f authored by Landon James's avatar Landon James Committed by GitHub
Browse files

Add customization to ensure S3 `Expires` field is always a `DateTime` (#3730) 

## Motivation and Context
<!--- Why is this change required? What problem does it solve? -->
<!--- If it fixes an open issue, please link to the issue here -->
At some point in the near future (after this customization is applied to
all existing SDKs) the S3 model will change the type of `Expires`
members to `String` from the current `Timestamp`. This change would
break backwards compatibility for us.

## Description
<!--- Describe your changes in detail -->
Add customization to ensure S3 `Expires` field is always a `Timestamp`
and ass a new synthetic member `ExpiresString` that persists the
un-parsed data from the `Expires` header.

## Testing
<!--- Please describe in detail how you tested your changes -->
<!--- Include details of your testing environment, and the tests you ran
to -->
<!--- see how your change affects other areas of the code, etc. -->
Added tests to ensure that the model is pre-processed correctly. Added
integration tests for S3. Considered making this more generic codegen
tests, but since this customization will almost certainly only ever
apply to S3 and I wanted to ensure that it was properly applied to the
generated S3 SDK I opted for this route.

## Checklist
<!--- If a checkbox below is not applicable, then please DELETE it
rather than leaving it unchecked -->
- [x] I have updated `CHANGELOG.next.toml` if I made changes to the AWS
SDK, generated SDK code, or SDK runtime crates

----

_By submitting this pull request, I confirm that you can use, modify,
copy, and redistribute this contribution, under the terms of your
choice._
parent 5c0baa79

CHANGELOG.next.toml

+6 −0
Original line number Diff line number Diff line
@@ -23,3 +23,9 @@ Content-Type header validation now ignores parameter portion of media types. 
references = ["smithy-rs#3471","smithy-rs#3724"]

meta = { "breaking" = false, "tada" = false, "bug" = true, target = "server" }

authors = ["djedward"]



[[aws-sdk-rust]]

message = "Add customizations for S3 Expires fields."

references = ["smithy-rs#3730"]

meta = { "breaking" = false, "tada" = false, "bug" = false }

author = "landonxjames"

aws/rust-runtime/aws-inlineable/src/lib.rs

+3 −0
Original line number Diff line number Diff line
@@ -56,6 +56,9 @@ pub mod endpoint_discovery; 
// the `presigning_interceptors` module can refer to it.

mod serialization_settings;



/// Parse the Expires and ExpiresString fields correctly

pub mod s3_expires_interceptor;



// just so docs work

#[allow(dead_code)]

/// allow docs to work

aws/rust-runtime/aws-inlineable/src/s3_expires_interceptor.rs

0 → 100644
+56 −0
Original line number Diff line number Diff line 
/*

 * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.

 * SPDX-License-Identifier: Apache-2.0

 */



use aws_smithy_runtime_api::box_error::BoxError;

use aws_smithy_runtime_api::client::interceptors::context::BeforeDeserializationInterceptorContextMut;

use aws_smithy_runtime_api::client::interceptors::Intercept;

use aws_smithy_runtime_api::client::runtime_components::RuntimeComponents;

use aws_smithy_types::config_bag::ConfigBag;

use aws_smithy_types::date_time::{DateTime, Format};



/// An interceptor to implement custom parsing logic for S3's `Expires` header. This

/// intercaptor copies the value of the `Expires` header to a (synthetically added)

/// `ExpiresString` header. It also attempts to parse the header as an `HttpDate`, if

/// that parsing fails the header is removed so the `Expires` field in the final output

/// will be `None`.

#[derive(Debug)]

pub(crate) struct S3ExpiresInterceptor;

const EXPIRES: &str = "Expires";

const EXPIRES_STRING: &str = "ExpiresString";



impl Intercept for S3ExpiresInterceptor {

    fn name(&self) -> &'static str {

        "S3ExpiresInterceptor"

    }



    fn modify_before_deserialization(

        &self,

        context: &mut BeforeDeserializationInterceptorContextMut<'_>,

        _: &RuntimeComponents,

        _: &mut ConfigBag,

    ) -> Result<(), BoxError> {

        let headers = context.response_mut().headers_mut();



        if headers.contains_key(EXPIRES) {

            let expires_header = headers.get(EXPIRES).unwrap().to_string();



            // If the Expires header fails to parse to an HttpDate we remove the header so

            // it is parsed to None. We use HttpDate since that is the SEP defined default

            // if no other format is specified in the model.

            if DateTime::from_str(&expires_header, Format::HttpDate).is_err() {

                tracing::debug!(

                    "Failed to parse the header `{EXPIRES}` = \"{expires_header}\" as an HttpDate. The raw string value can found in `{EXPIRES_STRING}`."

                );

                headers.remove(EXPIRES);

            }



            // Regardless of parsing success we copy the value of the Expires header to the

            // ExpiresString header.

            headers.insert(EXPIRES_STRING, expires_header);

        }



        Ok(())

    }

}

aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/AwsCodegenDecorator.kt

+2 −0
Original line number Diff line number Diff line
@@ -21,6 +21,7 @@ import software.amazon.smithy.rustsdk.customize.lambda.LambdaDecorator 
import software.amazon.smithy.rustsdk.customize.onlyApplyTo

import software.amazon.smithy.rustsdk.customize.route53.Route53Decorator

import software.amazon.smithy.rustsdk.customize.s3.S3Decorator

import software.amazon.smithy.rustsdk.customize.s3.S3ExpiresDecorator

import software.amazon.smithy.rustsdk.customize.s3.S3ExpressDecorator

import software.amazon.smithy.rustsdk.customize.s3.S3ExtendedRequestIdDecorator

import software.amazon.smithy.rustsdk.customize.s3control.S3ControlDecorator
@@ -79,6 +80,7 @@ val DECORATORS: List<ClientCodegenDecorator> = 
            S3ExpressDecorator(),

            S3ExtendedRequestIdDecorator(),

            IsTruncatedPaginatorDecorator(),

            S3ExpiresDecorator(),

        ),

        S3ControlDecorator().onlyApplyTo("com.amazonaws.s3control#AWSS3ControlServiceV20180820"),

        STSDecorator().onlyApplyTo("com.amazonaws.sts#AWSSecurityTokenServiceV20110615"),

aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/customize/s3/S3ExpiresDecorator.kt

0 → 100644
+149 −0
Original line number Diff line number Diff line 
/*

 * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.

 * SPDX-License-Identifier: Apache-2.0

 */



package software.amazon.smithy.rustsdk.customize.s3



import software.amazon.smithy.model.Model

import software.amazon.smithy.model.shapes.MemberShape

import software.amazon.smithy.model.shapes.OperationShape

import software.amazon.smithy.model.shapes.ServiceShape

import software.amazon.smithy.model.shapes.ShapeType

import software.amazon.smithy.model.shapes.StringShape

import software.amazon.smithy.model.shapes.StructureShape

import software.amazon.smithy.model.traits.DeprecatedTrait

import software.amazon.smithy.model.traits.DocumentationTrait

import software.amazon.smithy.model.traits.HttpHeaderTrait

import software.amazon.smithy.model.traits.OutputTrait

import software.amazon.smithy.model.transform.ModelTransformer

import software.amazon.smithy.rust.codegen.client.smithy.ClientCodegenContext

import software.amazon.smithy.rust.codegen.client.smithy.ClientRustSettings

import software.amazon.smithy.rust.codegen.client.smithy.customize.ClientCodegenDecorator

import software.amazon.smithy.rust.codegen.client.smithy.generators.OperationCustomization

import software.amazon.smithy.rust.codegen.client.smithy.generators.OperationSection

import software.amazon.smithy.rust.codegen.core.rustlang.Writable

import software.amazon.smithy.rust.codegen.core.rustlang.rustTemplate

import software.amazon.smithy.rust.codegen.core.rustlang.writable

import software.amazon.smithy.rust.codegen.core.smithy.RuntimeType

import software.amazon.smithy.rust.codegen.core.util.getTrait

import software.amazon.smithy.rust.codegen.core.util.hasTrait

import software.amazon.smithy.rust.codegen.core.util.outputShape

import software.amazon.smithy.rustsdk.InlineAwsDependency

import kotlin.streams.asSequence



/**

 * Enforces that Expires fields have the DateTime type (since in the future the model will change to model them as String),

 * and add an ExpiresString field to maintain the raw string value sent.

 */

class S3ExpiresDecorator : ClientCodegenDecorator {

    override val name: String = "S3ExpiresDecorator"

    override val order: Byte = 0

    private val expires = "Expires"

    private val expiresString = "ExpiresString"



    override fun transformModel(

        service: ServiceShape,

        model: Model,

        settings: ClientRustSettings,

    ): Model {

        val transformer = ModelTransformer.create()



        // Ensure all `Expires` shapes are timestamps

        val expiresShapeTimestampMap =

            model.shapes()

                .asSequence()

                .mapNotNull { shape ->

                    shape.members()

                        .singleOrNull { member -> member.memberName.equals(expires, ignoreCase = true) }

                        ?.target

                }

                .associateWith { ShapeType.TIMESTAMP }

        var transformedModel = transformer.changeShapeType(model, expiresShapeTimestampMap)



        // Add an `ExpiresString` string shape to the model

        val expiresStringShape = StringShape.builder().id("aws.sdk.rust.s3.synthetic#$expiresString").build()

        transformedModel = transformedModel.toBuilder().addShape(expiresStringShape).build()



        // For output shapes only, deprecate `Expires` and add a synthetic member that targets `ExpiresString`

        transformedModel =

            transformer.mapShapes(transformedModel) { shape ->

                if (shape.hasTrait<OutputTrait>() && shape.memberNames.any { it.equals(expires, ignoreCase = true) }) {

                    val builder = (shape as StructureShape).toBuilder()



                    // Deprecate `Expires`

                    val expiresMember = shape.members().single { it.memberName.equals(expires, ignoreCase = true) }



                    builder.removeMember(expiresMember.memberName)

                    val deprecatedTrait =

                        DeprecatedTrait.builder()

                            .message("Please use `expires_string` which contains the raw, unparsed value of this field.")

                            .build()



                    builder.addMember(

                        expiresMember.toBuilder()

                            .addTrait(deprecatedTrait)

                            .build(),

                    )



                    // Add a synthetic member targeting `ExpiresString`

                    val expiresStringMember = MemberShape.builder()

                    expiresStringMember.target(expiresStringShape.id)

                    expiresStringMember.id(expiresMember.id.toString() + "String") // i.e. com.amazonaws.s3.<MEMBER_NAME>$ExpiresString

                    expiresStringMember.addTrait(HttpHeaderTrait(expiresString)) // Add HttpHeaderTrait to ensure the field is deserialized

                    expiresMember.getTrait<DocumentationTrait>()?.let {

                        expiresStringMember.addTrait(it) // Copy documentation from `Expires`

                    }

                    builder.addMember(expiresStringMember.build())

                    builder.build()

                } else {

                    shape

                }

            }



        return transformedModel

    }



    override fun operationCustomizations(

        codegenContext: ClientCodegenContext,

        operation: OperationShape,

        baseCustomizations: List<OperationCustomization>,

    ): List<OperationCustomization> {

        val outputShape = operation.outputShape(codegenContext.model)



        if (outputShape.memberNames.any { it.equals(expires, ignoreCase = true) }) {

            return baseCustomizations +

                ParseExpiresFieldsCustomization(

                    codegenContext,

                )

        } else {

            return baseCustomizations

        }

    }

}



class ParseExpiresFieldsCustomization(

    private val codegenContext: ClientCodegenContext,

) : OperationCustomization() {

    override fun section(section: OperationSection): Writable =

        writable {

            when (section) {

                is OperationSection.AdditionalInterceptors -> {

                    section.registerInterceptor(codegenContext.runtimeConfig, this) {

                        val interceptor =

                            RuntimeType.forInlineDependency(

                                InlineAwsDependency.forRustFile("s3_expires_interceptor"),

                            ).resolve("S3ExpiresInterceptor")

                        rustTemplate(

                            """

                            #{S3ExpiresInterceptor}

                            """,

                            "S3ExpiresInterceptor" to interceptor,

                        )

                    }

                }



                else -> {}

            }

        }

}