Unverified Commit f3b332da authored by ysaito1001's avatar ysaito1001 Committed by GitHub
Browse files

Add support for S3 Express One Zone (#3465) 

## Motivation and Context
Allows the Rust SDK to use [S3 Express One
Zone](https://aws.amazon.com/s3/storage-classes/express-one-zone/)

## Description
The PR adds the said S3-specific functionality to the Rust SDK. The code
changes have already been reviewed by previous sub PRs, but it's worth
going through them again as a whole:
- https://github.com/smithy-lang/smithy-rs/pull/3386
- https://github.com/smithy-lang/smithy-rs/pull/3388
- https://github.com/smithy-lang/smithy-rs/pull/3390
- https://github.com/smithy-lang/smithy-rs/pull/3432
- https://github.com/smithy-lang/smithy-rs/pull/3433
- https://github.com/smithy-lang/smithy-rs/pull/3459
- https://github.com/smithy-lang/smithy-rs/pull/3457
- https://github.com/smithy-lang/smithy-rs/pull/3462

In addition to the PRs above, commit eebe8af increases the canary
lambda's memory size to 512MB from 128MB (also makes it configurable
through a command line arg for `canary-runner`). By default, lambda's
allowed memory size is 128MB but with the addition of `canary-wasm` in
main, canary lambda's memory usage will be 152MB, causing the lambda to
be killed by a signal during runtime. The commit addresses that issue.

## Testing
- Unit tests in
[aws/rust-runtime/aws-inlineable/src/s3_express.rs](https://github.com/smithy-lang/smithy-rs/blob/7f8c28b7038372927ec6196eff88384452f908dd/aws/rust-runtime/aws-inlineable/src/s3_express.rs)
- Integration tests in
[aws/sdk/integration-tests/s3/tests/express.rs](https://github.com/smithy-lang/smithy-rs/blob/7f8c28b7038372927ec6196eff88384452f908dd/aws/sdk/integration-tests/s3/tests/express.rs

)
- Canary in smithy-rs#3462

## Checklist
<!--- If a checkbox below is not applicable, then please DELETE it
rather than leaving it unchecked -->
- [x] I have updated `CHANGELOG.next.toml` if I made changes to the AWS
SDK, generated SDK code, or SDK runtime crates

----

_By submitting this pull request, I confirm that you can use, modify,
copy, and redistribute this contribution, under the terms of your
choice._

---------

Co-authored-by: default avatarJohn DiSanti <jdisanti@amazon.com>
Co-authored-by: default avatarAWS SDK Rust Bot <aws-sdk-rust-primary@amazon.com>
Co-authored-by: default avatarAWS SDK Rust Bot <97246200+aws-sdk-rust-ci@users.noreply.github.com>
Co-authored-by: default avatarZelda Hessler <zhessler@amazon.com>
Co-authored-by: default avatarRussell Cohen <rcoh@amazon.com>
parent a413b9ba

CHANGELOG.next.toml

+6 −0
Original line number Diff line number Diff line
@@ -52,3 +52,9 @@ message = "Upgrade Smithy to 1.45." 
references = ["smithy-rs#3470"]

meta = { "breaking" = false, "tada" = false, "bug" = false, "target" = "all" }

authors = ["jdisanti"]



[[aws-sdk-rust]]

message = "Add support for S3 Express One Zone. See [the user guide](https://github.com/awslabs/aws-sdk-rust/discussions/1091) for more details."

references = ["aws-sdk-rust#992", "smithy-rs#3465"]

meta = { "breaking" = false, "bug" = false, "tada" = true }

author = "ysaito1001"

aws/rust-runtime/aws-inlineable/Cargo.toml

+8 −0
Original line number Diff line number Diff line
@@ -12,6 +12,10 @@ publish = false 
repository = "https://github.com/smithy-lang/smithy-rs"



[dependencies]

# Used by lru, and this forces it to be a later version that avoids

# https://github.com/tkaitchuck/aHash/issues/200

# when built with `cargo update -Z minimal-versions`

ahash = "0.8.11"

aws-credential-types = { path = "../aws-credential-types" }

aws-runtime = { path = "../aws-runtime", features = ["http-02x"] }

aws-sigv4 = { path = "../aws-sigv4" }
@@ -22,10 +26,14 @@ aws-smithy-runtime = { path = "../../../rust-runtime/aws-smithy-runtime", featur 
aws-smithy-runtime-api = { path = "../../../rust-runtime/aws-smithy-runtime-api", features = ["client"] }

aws-smithy-types = { path = "../../../rust-runtime/aws-smithy-types", features = ["http-body-0-4-x"] }

bytes = "1"

fastrand = "2.0.0"

hex = "0.4.3"

http = "0.2.9"

http-body = "0.4.5"

hmac = "0.12"

lru = "0.12.2"

ring = "0.17.5"

sha2 = "0.10"

tokio = "1.23.1"

tracing = "0.1"

aws/rust-runtime/aws-inlineable/src/http_request_checksum.rs

+52 −2
Original line number Diff line number Diff line
@@ -60,6 +60,45 @@ impl Storable for RequestChecksumInterceptorState { 
    type Storer = StoreReplace<Self>;

}



type CustomDefaultFn = Box<

    dyn Fn(Option<ChecksumAlgorithm>, &ConfigBag) -> Option<ChecksumAlgorithm>

        + Send

        + Sync

        + 'static,

>;



pub(crate) struct DefaultRequestChecksumOverride {

    custom_default: CustomDefaultFn,

}

impl fmt::Debug for DefaultRequestChecksumOverride {

    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {

        f.debug_struct("DefaultRequestChecksumOverride").finish()

    }

}

impl Storable for DefaultRequestChecksumOverride {

    type Storer = StoreReplace<Self>;

}

impl DefaultRequestChecksumOverride {

    pub(crate) fn new<F>(custom_default: F) -> Self

    where

        F: Fn(Option<ChecksumAlgorithm>, &ConfigBag) -> Option<ChecksumAlgorithm>

            + Send

            + Sync

            + 'static,

    {

        Self {

            custom_default: Box::new(custom_default),

        }

    }

    pub(crate) fn custom_default(

        &self,

        original: Option<ChecksumAlgorithm>,

        config_bag: &ConfigBag,

    ) -> Option<ChecksumAlgorithm> {

        (self.custom_default)(original, config_bag)

    }

}



pub(crate) struct RequestChecksumInterceptor<AP> {

    algorithm_provider: AP,

}
@@ -102,7 +141,7 @@ where 
    /// Calculate a checksum and modify the request to include the checksum as a header

    /// (for in-memory request bodies) or a trailer (for streaming request bodies).

    /// Streaming bodies must be sized or this will return an error.

    fn modify_before_retry_loop(

    fn modify_before_signing(

        &self,

        context: &mut BeforeTransmitInterceptorContextMut<'_>,

        _runtime_components: &RuntimeComponents,
@@ -112,7 +151,8 @@ where 
            .load::<RequestChecksumInterceptorState>()

            .expect("set in `read_before_serialization`");



        if let Some(checksum_algorithm) = state.checksum_algorithm {

        let checksum_algorithm = incorporate_custom_default(state.checksum_algorithm, cfg);

        if let Some(checksum_algorithm) = checksum_algorithm {

            let request = context.request_mut();

            add_checksum_for_request_body(request, checksum_algorithm, cfg)?;

        }
@@ -121,6 +161,16 @@ where 
    }

}



fn incorporate_custom_default(

    checksum: Option<ChecksumAlgorithm>,

    cfg: &ConfigBag,

) -> Option<ChecksumAlgorithm> {

    match cfg.load::<DefaultRequestChecksumOverride>() {

        Some(checksum_override) => checksum_override.custom_default(checksum, cfg),

        None => checksum,

    }

}



fn add_checksum_for_request_body(

    request: &mut HttpRequest,

    checksum_algorithm: ChecksumAlgorithm,

aws/rust-runtime/aws-inlineable/src/lib.rs

+5 −0
Original line number Diff line number Diff line
@@ -31,6 +31,11 @@ pub mod presigning; 
/// Presigning interceptors

pub mod presigning_interceptors;



// This module uses module paths that assume the target crate to which it is copied, e.g.

// `crate::config::endpoint::Params`. If included into `aws-inlineable`, this module would

// fail to compile.

// pub mod s3_express;



/// Special logic for extracting request IDs from S3's responses.

#[allow(dead_code)]

pub mod s3_request_id;