From f7f037d3fda2d02b6a2ad1bce40dd84c787c6c35 Mon Sep 17 00:00:00 2001 From: Landon James Date: Tue, 14 Jan 2025 13:44:26 -0700 Subject: [PATCH] Flexible Checksums V2 (#3967) ## Motivation and Context Flexible checksums V2 will be launching this week ## Description ## Testing ## Checklist - [x] For changes to the smithy-rs codegen or runtime crates, I have created a changelog entry Markdown file in the `.changelog` directory, specifying "client," "server," or both in the `applies_to` key. - [x] For changes to the AWS SDK, generated SDK code, or SDK runtime crates, I have created a changelog entry Markdown file in the `.changelog` directory, specifying "aws-sdk-rust" in the `applies_to` key. ---- _By submitting this pull request, I confirm that you can use, modify, copy, and redistribute this contribution, under the terms of your choice._ --------- Co-authored-by: Zelda Hessler Co-authored-by: Aaron Todd --- .changelog/flexible-checksums-client.md | 10 + .changelog/flexible-checksums-s3.md | 12 + aws/rust-runtime/Cargo.lock | 188 +++- aws/rust-runtime/aws-config/Cargo.lock | 12 +- aws/rust-runtime/aws-config/Cargo.toml | 2 +- .../aws-config/external-types.toml | 2 + aws/rust-runtime/aws-config/fuzz/Cargo.toml | 3 +- .../aws-config/src/default_provider.rs | 3 + .../src/default_provider/checksums.rs | 259 +++++ aws/rust-runtime/aws-config/src/lib.rs | 23 +- .../src/http_request_checksum.rs | 181 +++- .../src/http_response_checksum.rs | 79 +- .../aws-inlineable/src/presigning.rs | 10 + .../src/presigning_interceptors.rs | 4 +- aws/rust-runtime/aws-runtime/Cargo.toml | 2 +- .../aws-runtime/src/user_agent/test_util.rs | 21 +- aws/rust-runtime/aws-sigv4/Cargo.toml | 5 +- .../src/http_request/canonical_request.rs | 7 +- aws/rust-runtime/aws-types/Cargo.toml | 2 +- .../aws-types/external-types.toml | 2 + aws/rust-runtime/aws-types/src/sdk_config.rs | 69 ++ .../rustsdk/HttpRequestChecksumDecorator.kt | 280 +++++- .../rustsdk/HttpResponseChecksumDecorator.kt | 183 +++- .../amazon/smithy/rustsdk/HttpChecksumTest.kt | 938 ++++++++++++++++++ aws/sdk/Cargo.lock | 545 ++++++---- aws/sdk/aws-models-extra/s3-tests.smithy | 4 +- .../integration-tests/s3/tests/endpoints.rs | 2 +- aws/sdk/integration-tests/s3/tests/express.rs | 6 +- .../integration-tests/s3/tests/presigning.rs | 14 +- .../s3/tests/stalled-stream-protection.rs | 3 + .../smithy/generators/OperationGenerator.kt | 8 +- rust-runtime/Cargo.lock | 341 +++++-- rust-runtime/aws-smithy-async/Cargo.toml | 2 +- rust-runtime/aws-smithy-checksums/Cargo.toml | 3 +- rust-runtime/aws-smithy-checksums/src/http.rs | 57 +- rust-runtime/aws-smithy-checksums/src/lib.rs | 77 +- .../aws-smithy-eventstream/Cargo.toml | 2 +- .../aws-smithy-eventstream/fuzz/Cargo.toml | 3 +- rust-runtime/aws-smithy-http/Cargo.toml | 2 +- rust-runtime/aws-smithy-http/fuzz/Cargo.toml | 3 +- rust-runtime/aws-smithy-json/Cargo.toml | 2 +- rust-runtime/aws-smithy-json/fuzz/Cargo.toml | 3 +- rust-runtime/aws-smithy-types/Cargo.toml | 2 +- rust-runtime/aws-smithy-types/fuzz/Cargo.toml | 3 +- .../aws-smithy-types/src/checksum_config.rs | 163 +++ rust-runtime/aws-smithy-types/src/lib.rs | 1 + 46 files changed, 3135 insertions(+), 408 deletions(-) create mode 100644 .changelog/flexible-checksums-client.md create mode 100644 .changelog/flexible-checksums-s3.md create mode 100644 aws/rust-runtime/aws-config/src/default_provider/checksums.rs create mode 100644 aws/sdk-codegen/src/test/kotlin/software/amazon/smithy/rustsdk/HttpChecksumTest.kt create mode 100644 rust-runtime/aws-smithy-types/src/checksum_config.rs diff --git a/.changelog/flexible-checksums-client.md b/.changelog/flexible-checksums-client.md new file mode 100644 index 000000000..ee69e3fd4 --- /dev/null +++ b/.changelog/flexible-checksums-client.md @@ -0,0 +1,10 @@ +--- +applies_to: ["client"] +authors: ["landonxjames"] +references: ["smithy-rs#3967"] +breaking: false +new_feature: true +bug_fix: true +--- + +Updates client generation to conform with Smithy's updates to the [httpChecksum trait](https://smithy.io/2.0/aws/aws-core.html#aws-protocols-httpchecksum-trait). diff --git a/.changelog/flexible-checksums-s3.md b/.changelog/flexible-checksums-s3.md new file mode 100644 index 000000000..0e903e9a2 --- /dev/null +++ b/.changelog/flexible-checksums-s3.md @@ -0,0 +1,12 @@ +--- +applies_to: ["client", "aws-sdk-rust"] +authors: ["landonxjames"] +references: ["smithy-rs#3845"] +breaking: false +new_feature: true +bug_fix: true +--- + +S3 client behavior is updated to always calculate a checksum by default for operations that support it (such as PutObject or UploadPart), or require it (such as DeleteObjects). The default checksum algorithm is CRC32. Checksum behavior can be configured using `when_supported` and `when_required` options - in shared config using request_checksum_calculation, or as env variable using AWS_REQUEST_CHECKSUM_CALCULATION. + +The S3 client attempts to validate response checksums for all S3 API operations that support checksums. However, if the SDK has not implemented the specified checksum algorithm then this validation is skipped. Checksum validation behavior can be configured using `when_supported` and `when_required` options - in shared config using response_checksum_validation, or as env variable using AWS_RESPONSE_CHECKSUM_VALIDATION. diff --git a/aws/rust-runtime/Cargo.lock b/aws/rust-runtime/Cargo.lock index 9d8fc4f59..95a150756 100644 --- a/aws/rust-runtime/Cargo.lock +++ b/aws/rust-runtime/Cargo.lock @@ -38,12 +38,55 @@ version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4b46cbb362ab8752921c97e041f5e366ee6297bd428a31275b9fcf1e380f7299" +[[package]] +name = "anstream" +version = "0.6.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8acc5369981196006228e28809f761875c0327210a891e941f4c683b3a99529b" +dependencies = [ + "anstyle", + "anstyle-parse", + "anstyle-query", + "anstyle-wincon", + "colorchoice", + "is_terminal_polyfill", + "utf8parse", +] + [[package]] name = "anstyle" version = "1.0.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "55cc3b69f167a1ef2e161439aa98aed94e6028e5f9a59be9a6ffb47aef1651f9" +[[package]] +name = "anstyle-parse" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b2d16507662817a6a20a9ea92df6652ee4f94f914589377d69f3b21bc5798a9" +dependencies = [ + "utf8parse", +] + +[[package]] +name = "anstyle-query" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "79947af37f4177cfead1110013d678905c37501914fba0efea834c3fe9a8d60c" +dependencies = [ + "windows-sys 0.59.0", +] + +[[package]] +name = "anstyle-wincon" +version = "3.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2109dbce0e72be3ec00bed26e6a7479ca384ad226efdd66db8fa2e3a38c83125" +dependencies = [ + "anstyle", + "windows-sys 0.59.0", +] + [[package]] name = "arbitrary" version = "1.4.1" @@ -138,7 +181,7 @@ dependencies = [ [[package]] name = "aws-runtime" -version = "1.5.3" +version = "1.5.4" dependencies = [ "arbitrary", "aws-credential-types", @@ -184,7 +227,7 @@ version = "0.60.3" [[package]] name = "aws-sigv4" -version = "1.2.6" +version = "1.2.7" dependencies = [ "aws-credential-types", "aws-smithy-eventstream", @@ -220,7 +263,7 @@ dependencies = [ [[package]] name = "aws-smithy-async" -version = "1.2.3" +version = "1.2.4" dependencies = [ "futures-util", "pin-project-lite", @@ -229,13 +272,14 @@ dependencies = [ [[package]] name = "aws-smithy-checksums" -version = "0.60.13" +version = "0.62.0" dependencies = [ "aws-smithy-http", "aws-smithy-types", "bytes", "crc32c", "crc32fast", + "crc64fast-nvme", "hex", "http 0.2.12", "http-body 0.4.6", @@ -248,7 +292,7 @@ dependencies = [ [[package]] name = "aws-smithy-eventstream" -version = "0.60.5" +version = "0.60.6" dependencies = [ "aws-smithy-types", "bytes", @@ -257,7 +301,7 @@ dependencies = [ [[package]] name = "aws-smithy-http" -version = "0.60.11" +version = "0.60.12" dependencies = [ "aws-smithy-runtime-api", "aws-smithy-types", @@ -332,7 +376,7 @@ dependencies = [ [[package]] name = "aws-smithy-types" -version = "1.2.11" +version = "1.2.12" dependencies = [ "base64-simd", "bytes", @@ -356,7 +400,7 @@ dependencies = [ [[package]] name = "aws-types" -version = "1.3.3" +version = "1.3.4" dependencies = [ "aws-credential-types", "aws-smithy-async", @@ -488,6 +532,25 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" +[[package]] +name = "cbindgen" +version = "0.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3fce8dd7fcfcbf3a0a87d8f515194b49d6135acab73e18bd380d1d93bb1a15eb" +dependencies = [ + "clap", + "heck", + "indexmap", + "log", + "proc-macro2", + "quote", + "serde", + "serde_json", + "syn 2.0.94", + "tempfile", + "toml", +] + [[package]] name = "cbor-diag" version = "0.1.12" @@ -575,8 +638,10 @@ version = "4.5.26" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "96b01801b5fc6a0a232407abc821660c9c6d25a1cafc0d4f85f29fb8d9afc121" dependencies = [ + "anstream", "anstyle", "clap_lex", + "strsim", ] [[package]] @@ -585,6 +650,12 @@ version = "0.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f46ad14479a25103f283c0f10005961cf086d8dc42205bb44c46ac563475dca6" +[[package]] +name = "colorchoice" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b63caa9aa9397e2d9480a9b13673856c78d8ac123288526c37d7839f2a86990" + [[package]] name = "const-oid" version = "0.9.6" @@ -625,6 +696,21 @@ dependencies = [ "libc", ] +[[package]] +name = "crc" +version = "3.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69e6e4d7b33a94f0991c26729976b10ebde1d34c3ee82408fb536164fa10d636" +dependencies = [ + "crc-catalog", +] + +[[package]] +name = "crc-catalog" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19d374276b40fb8bbdee95aef7c7fa6b5316ec764510eb64b8dd0e2ed0d7e7f5" + [[package]] name = "crc32c" version = "0.6.8" @@ -643,6 +729,16 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "crc64fast-nvme" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d5e2ee08013e3f228d6d2394116c4549a6df77708442c62d887d83f68ef2ee37" +dependencies = [ + "cbindgen", + "crc", +] + [[package]] name = "criterion" version = "0.5.1" @@ -1015,6 +1111,12 @@ dependencies = [ "foldhash", ] +[[package]] +name = "heck" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" + [[package]] name = "hermit-abi" version = "0.4.0" @@ -1311,6 +1413,12 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "is_terminal_polyfill" +version = "1.70.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7943c866cc5cd64cbc25b2e01621d07fa8eb2a1a23160ee81ce38704e97b8ecf" + [[package]] name = "itertools" version = "0.10.5" @@ -2106,6 +2214,15 @@ dependencies = [ "serde", ] +[[package]] +name = "serde_spanned" +version = "0.6.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87607cb1398ed59d48732e575a4c28a7a8ebf2454b964fe3f224f2afc07909e1" +dependencies = [ + "serde", +] + [[package]] name = "sha1" version = "0.10.6" @@ -2209,6 +2326,12 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3" +[[package]] +name = "strsim" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" + [[package]] name = "subtle" version = "2.6.1" @@ -2409,6 +2532,40 @@ dependencies = [ "tokio", ] +[[package]] +name = "toml" +version = "0.8.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1ed1f98e3fdc28d6d910e6737ae6ab1a93bf1985935a1193e68f93eeb68d24e" +dependencies = [ + "serde", + "serde_spanned", + "toml_datetime", + "toml_edit", +] + +[[package]] +name = "toml_datetime" +version = "0.6.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0dd7358ecb8fc2f8d014bf86f6f638ce72ba252a2c3a2572f2a795f1d23efb41" +dependencies = [ + "serde", +] + +[[package]] +name = "toml_edit" +version = "0.22.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ae48d6208a266e853d946088ed816055e556cc6028c5e8e2b84d9fa5dd7c7f5" +dependencies = [ + "indexmap", + "serde", + "serde_spanned", + "toml_datetime", + "winnow", +] + [[package]] name = "tower-service" version = "0.3.3" @@ -2556,6 +2713,12 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" +[[package]] +name = "utf8parse" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" + [[package]] name = "uuid" version = "1.12.0" @@ -2801,6 +2964,15 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" +[[package]] +name = "winnow" +version = "0.6.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39281189af81c07ec09db316b302a3e67bf9bd7cbf6c820b50e35fee9c2fa980" +dependencies = [ + "memchr", +] + [[package]] name = "write16" version = "1.0.0" diff --git a/aws/rust-runtime/aws-config/Cargo.lock b/aws/rust-runtime/aws-config/Cargo.lock index 6537c71af..bed7bac1a 100644 --- a/aws/rust-runtime/aws-config/Cargo.lock +++ b/aws/rust-runtime/aws-config/Cargo.lock @@ -45,7 +45,7 @@ checksum = "ace50bade8e6234aa140d9a2f552bbee1db4d353f69b8217bc503490fc1a9f26" [[package]] name = "aws-config" -version = "1.5.13" +version = "1.5.14" dependencies = [ "aws-credential-types", "aws-runtime", @@ -89,7 +89,7 @@ dependencies = [ [[package]] name = "aws-runtime" -version = "1.5.3" +version = "1.5.4" dependencies = [ "aws-credential-types", "aws-sigv4", @@ -174,7 +174,7 @@ dependencies = [ [[package]] name = "aws-sigv4" -version = "1.2.6" +version = "1.2.7" dependencies = [ "aws-credential-types", "aws-smithy-http", @@ -195,7 +195,7 @@ dependencies = [ [[package]] name = "aws-smithy-async" -version = "1.2.3" +version = "1.2.4" dependencies = [ "futures-util", "pin-project-lite", @@ -299,7 +299,7 @@ dependencies = [ [[package]] name = "aws-smithy-types" -version = "1.2.11" +version = "1.2.12" dependencies = [ "base64-simd", "bytes", @@ -327,7 +327,7 @@ dependencies = [ [[package]] name = "aws-types" -version = "1.3.3" +version = "1.3.4" dependencies = [ "aws-credential-types", "aws-smithy-async", diff --git a/aws/rust-runtime/aws-config/Cargo.toml b/aws/rust-runtime/aws-config/Cargo.toml index 2cb33239b..0e47958c5 100644 --- a/aws/rust-runtime/aws-config/Cargo.toml +++ b/aws/rust-runtime/aws-config/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "aws-config" -version = "1.5.13" +version = "1.5.14" authors = [ "AWS Rust SDK Team ", "Russell Cohen ", diff --git a/aws/rust-runtime/aws-config/external-types.toml b/aws/rust-runtime/aws-config/external-types.toml index 9c8d3c1b2..3e2c6726d 100644 --- a/aws/rust-runtime/aws-config/external-types.toml +++ b/aws/rust-runtime/aws-config/external-types.toml @@ -34,6 +34,8 @@ allowed_external_types = [ "aws_smithy_runtime_api::client::result::SdkError", "aws_smithy_runtime_api::client::stalled_stream_protection::StalledStreamProtectionConfig", "aws_smithy_types::body::SdkBody", + "aws_smithy_types::checksum_config::RequestChecksumCalculation", + "aws_smithy_types::checksum_config::ResponseChecksumValidation", "aws_smithy_types::retry", "aws_smithy_types::retry::*", "aws_smithy_types::timeout", diff --git a/aws/rust-runtime/aws-config/fuzz/Cargo.toml b/aws/rust-runtime/aws-config/fuzz/Cargo.toml index e6139ee80..c62f180a0 100644 --- a/aws/rust-runtime/aws-config/fuzz/Cargo.toml +++ b/aws/rust-runtime/aws-config/fuzz/Cargo.toml @@ -10,7 +10,8 @@ edition = "2021" cargo-fuzz = true [dependencies] -libfuzzer-sys = "0.4" +# Version pinned due to https://github.com/rust-fuzz/libfuzzer/issues/126 +libfuzzer-sys = "=0.4.7" aws-config = { path = ".." } [dependencies.aws-types] diff --git a/aws/rust-runtime/aws-config/src/default_provider.rs b/aws/rust-runtime/aws-config/src/default_provider.rs index fceb869fb..42e5a6173 100644 --- a/aws/rust-runtime/aws-config/src/default_provider.rs +++ b/aws/rust-runtime/aws-config/src/default_provider.rs @@ -63,3 +63,6 @@ pub mod disable_request_compression; /// Default "request minimum compression size bytes" provider chain pub mod request_min_compression_size_bytes; + +/// Default provider chains for request/response checksum configuration +pub mod checksums; diff --git a/aws/rust-runtime/aws-config/src/default_provider/checksums.rs b/aws/rust-runtime/aws-config/src/default_provider/checksums.rs new file mode 100644 index 000000000..25da664a3 --- /dev/null +++ b/aws/rust-runtime/aws-config/src/default_provider/checksums.rs @@ -0,0 +1,259 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +use crate::provider_config::ProviderConfig; +use aws_runtime::env_config::EnvConfigValue; +use aws_smithy_types::error::display::DisplayErrorContext; +use aws_types::sdk_config::{RequestChecksumCalculation, ResponseChecksumValidation}; +use std::str::FromStr; + +mod env { + pub(super) const REQUEST_CHECKSUM_CALCULATION: &str = "AWS_REQUEST_CHECKSUM_CALCULATION"; + pub(super) const RESPONSE_CHECKSUM_VALIDATION: &str = "AWS_RESPONSE_CHECKSUM_VALIDATION"; +} + +mod profile_key { + pub(super) const REQUEST_CHECKSUM_CALCULATION: &str = "request_checksum_calculation"; + pub(super) const RESPONSE_CHECKSUM_VALIDATION: &str = "response_checksum_validation"; +} + +/// Load the value for `request_checksum_calculation` +/// +/// This checks the following sources: +/// 1. The environment variable `AWS_REQUEST_CHECKSUM_CALCULATION=WHEN_SUPPORTED/WHEN_REQUIRED` +/// 2. The profile key `request_checksum_calculation=WHEN_SUPPORTED/WHEN_REQUIRED` +/// +/// If invalid values are found, the provider will return `None` and an error will be logged. +pub async fn request_checksum_calculation_provider( + provider_config: &ProviderConfig, +) -> Option { + let env = provider_config.env(); + let profiles = provider_config.profile().await; + + let loaded = EnvConfigValue::new() + .env(env::REQUEST_CHECKSUM_CALCULATION) + .profile(profile_key::REQUEST_CHECKSUM_CALCULATION) + .validate(&env, profiles, RequestChecksumCalculation::from_str) + .map_err( + |err| tracing::warn!(err = %DisplayErrorContext(&err), "invalid value for request_checksum_calculation setting"), + ) + .unwrap_or(None); + + // request_checksum_calculation should always have a non-None value and the + // default is WhenSupported + loaded.or(Some(RequestChecksumCalculation::WhenSupported)) +} + +/// Load the value for `response_checksum_validation` +/// +/// This checks the following sources: +/// 1. The environment variable `AWS_RESPONSE_CHECKSUM_VALIDATION=WHEN_SUPPORTED/WHEN_REQUIRED` +/// 2. The profile key `response_checksum_validation=WHEN_SUPPORTED/WHEN_REQUIRED` +/// +/// If invalid values are found, the provider will return `None` and an error will be logged. +pub async fn response_checksum_validation_provider( + provider_config: &ProviderConfig, +) -> Option { + let env = provider_config.env(); + let profiles = provider_config.profile().await; + + let loaded = EnvConfigValue::new() + .env(env::RESPONSE_CHECKSUM_VALIDATION) + .profile(profile_key::RESPONSE_CHECKSUM_VALIDATION) + .validate(&env, profiles, ResponseChecksumValidation::from_str) + .map_err( + |err| tracing::warn!(err = %DisplayErrorContext(&err), "invalid value for response_checksum_validation setting"), + ) + .unwrap_or(None); + + // response_checksum_validation should always have a non-None value and the + // default is WhenSupported + loaded.or(Some(ResponseChecksumValidation::WhenSupported)) +} + +#[cfg(test)] +mod test { + use crate::default_provider::checksums::{ + request_checksum_calculation_provider, response_checksum_validation_provider, + }; + #[allow(deprecated)] + use crate::profile::profile_file::{ProfileFileKind, ProfileFiles}; + use crate::provider_config::ProviderConfig; + use aws_smithy_types::checksum_config::{ + RequestChecksumCalculation, ResponseChecksumValidation, + }; + use aws_types::os_shim_internal::{Env, Fs}; + use tracing_test::traced_test; + + #[tokio::test] + #[traced_test] + async fn log_error_on_invalid_value_request() { + let conf = ProviderConfig::empty().with_env(Env::from_slice(&[( + "AWS_REQUEST_CHECKSUM_CALCULATION", + "not-a-valid-value", + )])); + assert_eq!( + request_checksum_calculation_provider(&conf).await, + Some(RequestChecksumCalculation::WhenSupported) + ); + assert!(logs_contain( + "invalid value for request_checksum_calculation setting" + )); + assert!(logs_contain("AWS_REQUEST_CHECKSUM_CALCULATION")); + } + + #[tokio::test] + #[traced_test] + async fn environment_priority_request() { + let conf = ProviderConfig::empty() + .with_env(Env::from_slice(&[( + "AWS_REQUEST_CHECKSUM_CALCULATION", + "WHEN_REQUIRED", + )])) + .with_profile_config( + Some( + #[allow(deprecated)] + ProfileFiles::builder() + .with_file( + #[allow(deprecated)] + ProfileFileKind::Config, + "conf", + ) + .build(), + ), + None, + ) + .with_fs(Fs::from_slice(&[( + "conf", + "[default]\nrequest_checksum_calculation = WHEN_SUPPORTED", + )])); + assert_eq!( + request_checksum_calculation_provider(&conf).await, + Some(RequestChecksumCalculation::WhenRequired) + ); + } + + #[tokio::test] + #[traced_test] + async fn profile_works_request() { + let conf = ProviderConfig::empty() + .with_profile_config( + Some( + #[allow(deprecated)] + ProfileFiles::builder() + .with_file( + #[allow(deprecated)] + ProfileFileKind::Config, + "conf", + ) + .build(), + ), + None, + ) + .with_fs(Fs::from_slice(&[( + "conf", + "[default]\nrequest_checksum_calculation = WHEN_REQUIRED", + )])); + assert_eq!( + request_checksum_calculation_provider(&conf).await, + Some(RequestChecksumCalculation::WhenRequired) + ); + } + + #[tokio::test] + #[traced_test] + async fn default_works_request() { + let conf = ProviderConfig::empty(); + assert_eq!( + request_checksum_calculation_provider(&conf).await, + Some(RequestChecksumCalculation::WhenSupported) + ); + } + + #[tokio::test] + #[traced_test] + async fn log_error_on_invalid_value_response() { + let conf = ProviderConfig::empty().with_env(Env::from_slice(&[( + "AWS_RESPONSE_CHECKSUM_VALIDATION", + "not-a-valid-value", + )])); + assert_eq!( + response_checksum_validation_provider(&conf).await, + Some(ResponseChecksumValidation::WhenSupported) + ); + assert!(logs_contain( + "invalid value for response_checksum_validation setting" + )); + assert!(logs_contain("AWS_RESPONSE_CHECKSUM_VALIDATION")); + } + + #[tokio::test] + #[traced_test] + async fn environment_priority_response() { + let conf = ProviderConfig::empty() + .with_env(Env::from_slice(&[( + "AWS_RESPONSE_CHECKSUM_VALIDATION", + "WHEN_SUPPORTED", + )])) + .with_profile_config( + Some( + #[allow(deprecated)] + ProfileFiles::builder() + .with_file( + #[allow(deprecated)] + ProfileFileKind::Config, + "conf", + ) + .build(), + ), + None, + ) + .with_fs(Fs::from_slice(&[( + "conf", + "[default]\response_checksum_validation = WHEN_REQUIRED", + )])); + assert_eq!( + response_checksum_validation_provider(&conf).await, + Some(ResponseChecksumValidation::WhenSupported) + ); + } + + #[tokio::test] + #[traced_test] + async fn profile_works_response() { + let conf = ProviderConfig::empty() + .with_profile_config( + Some( + #[allow(deprecated)] + ProfileFiles::builder() + .with_file( + #[allow(deprecated)] + ProfileFileKind::Config, + "conf", + ) + .build(), + ), + None, + ) + .with_fs(Fs::from_slice(&[( + "conf", + "[default]\nresponse_checksum_validation = WHEN_REQUIRED", + )])); + assert_eq!( + response_checksum_validation_provider(&conf).await, + Some(ResponseChecksumValidation::WhenRequired) + ); + } + + #[tokio::test] + #[traced_test] + async fn default_works_response() { + let conf = ProviderConfig::empty(); + assert_eq!( + response_checksum_validation_provider(&conf).await, + Some(ResponseChecksumValidation::WhenSupported) + ); + } +} diff --git a/aws/rust-runtime/aws-config/src/lib.rs b/aws/rust-runtime/aws-config/src/lib.rs index dd8732703..8650325d0 100644 --- a/aws/rust-runtime/aws-config/src/lib.rs +++ b/aws/rust-runtime/aws-config/src/lib.rs @@ -228,6 +228,9 @@ mod loader { use aws_smithy_runtime_api::client::identity::{ResolveCachedIdentity, SharedIdentityCache}; use aws_smithy_runtime_api::client::stalled_stream_protection::StalledStreamProtectionConfig; use aws_smithy_runtime_api::shared::IntoShared; + use aws_smithy_types::checksum_config::{ + RequestChecksumCalculation, ResponseChecksumValidation, + }; use aws_smithy_types::retry::RetryConfig; use aws_smithy_types::timeout::TimeoutConfig; use aws_types::app_name::AppName; @@ -238,7 +241,7 @@ mod loader { use aws_types::SdkConfig; use crate::default_provider::{ - app_name, credentials, disable_request_compression, endpoint_url, + app_name, checksums, credentials, disable_request_compression, endpoint_url, ignore_configured_endpoint_urls as ignore_ep, region, request_min_compression_size_bytes, retry_config, timeout_config, use_dual_stack, use_fips, }; @@ -289,6 +292,8 @@ mod loader { env: Option, fs: Option, behavior_version: Option, + request_checksum_calculation: Option, + response_checksum_validation: Option, } impl ConfigLoader { @@ -919,6 +924,22 @@ mod loader { Some(user_cache) => Some(user_cache), }; + let request_checksum_calculation = + if let Some(request_checksum_calculation) = self.request_checksum_calculation { + Some(request_checksum_calculation) + } else { + checksums::request_checksum_calculation_provider(&conf).await + }; + + let response_checksum_validation = + if let Some(response_checksum_validation) = self.response_checksum_validation { + Some(response_checksum_validation) + } else { + checksums::response_checksum_validation_provider(&conf).await + }; + + builder.set_request_checksum_calculation(request_checksum_calculation); + builder.set_response_checksum_validation(response_checksum_validation); builder.set_identity_cache(identity_cache); builder.set_credentials_provider(credentials_provider); builder.set_token_provider(token_provider); diff --git a/aws/rust-runtime/aws-inlineable/src/http_request_checksum.rs b/aws/rust-runtime/aws-inlineable/src/http_request_checksum.rs index 689edcc55..f278bb0b3 100644 --- a/aws/rust-runtime/aws-inlineable/src/http_request_checksum.rs +++ b/aws/rust-runtime/aws-inlineable/src/http_request_checksum.rs @@ -12,20 +12,26 @@ use aws_runtime::content_encoding::header_value::AWS_CHUNKED; use aws_runtime::content_encoding::{AwsChunkedBody, AwsChunkedBodyOptions}; use aws_smithy_checksums::ChecksumAlgorithm; use aws_smithy_checksums::{body::calculate, http::HttpChecksum}; +use aws_smithy_runtime::client::sdk_feature::SmithySdkFeature; use aws_smithy_runtime_api::box_error::BoxError; use aws_smithy_runtime_api::client::interceptors::context::{ - BeforeSerializationInterceptorContextRef, BeforeTransmitInterceptorContextMut, Input, + BeforeSerializationInterceptorContextMut, BeforeTransmitInterceptorContextMut, Input, }; use aws_smithy_runtime_api::client::interceptors::Intercept; use aws_smithy_runtime_api::client::orchestrator::HttpRequest; use aws_smithy_runtime_api::client::runtime_components::RuntimeComponents; +use aws_smithy_runtime_api::http::Request; use aws_smithy_types::body::SdkBody; +use aws_smithy_types::checksum_config::RequestChecksumCalculation; use aws_smithy_types::config_bag::{ConfigBag, Layer, Storable, StoreReplace}; use aws_smithy_types::error::operation::BuildError; use http::HeaderValue; use http_body::Body; +use std::str::FromStr; use std::{fmt, mem}; +use crate::presigning::PresigningMarker; + /// Errors related to constructing checksum-validated HTTP requests #[derive(Debug)] pub(crate) enum Error { @@ -52,9 +58,12 @@ impl fmt::Display for Error { impl std::error::Error for Error {} -#[derive(Debug)] +#[derive(Debug, Clone)] struct RequestChecksumInterceptorState { - checksum_algorithm: Option, + /// The checksum algorithm to calculate + checksum_algorithm: Option, + /// This value is set in the model on the `httpChecksum` trait + request_checksum_required: bool, } impl Storable for RequestChecksumInterceptorState { type Storer = StoreReplace; @@ -99,40 +108,49 @@ impl DefaultRequestChecksumOverride { } } -pub(crate) struct RequestChecksumInterceptor { +pub(crate) struct RequestChecksumInterceptor { algorithm_provider: AP, + checksum_mutator: CM, } -impl fmt::Debug for RequestChecksumInterceptor { +impl fmt::Debug for RequestChecksumInterceptor { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { f.debug_struct("RequestChecksumInterceptor").finish() } } -impl RequestChecksumInterceptor { - pub(crate) fn new(algorithm_provider: AP) -> Self { - Self { algorithm_provider } +impl RequestChecksumInterceptor { + pub(crate) fn new(algorithm_provider: AP, checksum_mutator: CM) -> Self { + Self { + algorithm_provider, + checksum_mutator, + } } } -impl Intercept for RequestChecksumInterceptor +impl Intercept for RequestChecksumInterceptor where - AP: Fn(&Input) -> Result, BoxError> + Send + Sync, + AP: Fn(&Input) -> (Option, bool) + Send + Sync, + CM: Fn(&mut Request, &ConfigBag) -> Result + Send + Sync, { fn name(&self) -> &'static str { "RequestChecksumInterceptor" } - fn read_before_serialization( + fn modify_before_serialization( &self, - context: &BeforeSerializationInterceptorContextRef<'_>, + context: &mut BeforeSerializationInterceptorContextMut<'_>, _runtime_components: &RuntimeComponents, cfg: &mut ConfigBag, ) -> Result<(), BoxError> { - let checksum_algorithm = (self.algorithm_provider)(context.input())?; + let (checksum_algorithm, request_checksum_required) = + (self.algorithm_provider)(context.input()); let mut layer = Layer::new("RequestChecksumInterceptor"); - layer.store_put(RequestChecksumInterceptorState { checksum_algorithm }); + layer.store_put(RequestChecksumInterceptorState { + checksum_algorithm, + request_checksum_required, + }); cfg.push_layer(layer); Ok(()) @@ -141,7 +159,7 @@ where /// Calculate a checksum and modify the request to include the checksum as a header /// (for in-memory request bodies) or a trailer (for streaming request bodies). /// Streaming bodies must be sized or this will return an error. - fn modify_before_signing( + fn modify_before_retry_loop( &self, context: &mut BeforeTransmitInterceptorContextMut<'_>, _runtime_components: &RuntimeComponents, @@ -151,14 +169,119 @@ where .load::() .expect("set in `read_before_serialization`"); - let checksum_algorithm = incorporate_custom_default(state.checksum_algorithm, cfg); - if let Some(checksum_algorithm) = checksum_algorithm { + let user_set_checksum_value = (self.checksum_mutator)(context.request_mut(), cfg) + .expect("Checksum header mutation should not fail"); + + // If the user manually set a checksum header we short circuit + if user_set_checksum_value { + return Ok(()); + } + + // This value is from the trait, but is needed for runtime logic + let request_checksum_required = state.request_checksum_required; + + // If the algorithm fails to parse it is not one we support and we error + let checksum_algorithm = state + .checksum_algorithm + .clone() + .map(|s| ChecksumAlgorithm::from_str(s.as_str())) + .transpose()?; + + // This value is set by the user on the SdkConfig to indicate their preference + // We provide a default here for users that use a client config instead of the SdkConfig + let request_checksum_calculation = cfg + .load::() + .unwrap_or(&RequestChecksumCalculation::WhenSupported); + + // Determine if we actually calculate the checksum. If the user setting is WhenSupported (the default) + // we always calculate it (because this interceptor isn't added if it isn't supported). If it is + // WhenRequired we only calculate it if the checksum is marked required on the trait. + let calculate_checksum = match request_checksum_calculation { + RequestChecksumCalculation::WhenRequired => request_checksum_required, + RequestChecksumCalculation::WhenSupported => true, + _ => true, + }; + + // Calculate the checksum if necessary + if calculate_checksum { + let is_presigned_req = cfg.load::().is_some(); + + // If this is a presigned request and the user has not set a checksum we short circuit + if is_presigned_req && checksum_algorithm.is_none() { + return Ok(()); + } + + // If a checksum override is set in the ConfigBag we use that instead (currently only used by S3Express) + // If we have made it this far without a checksum being set we set the default (currently Crc32) + let checksum_algorithm = + incorporate_custom_default(checksum_algorithm, cfg).unwrap_or_default(); + + // Set the user-agent metric for the selected checksum algorithm + match checksum_algorithm { + ChecksumAlgorithm::Crc32 => { + cfg.interceptor_state() + .store_append(SmithySdkFeature::FlexibleChecksumsReqCrc32); + } + ChecksumAlgorithm::Crc32c => { + cfg.interceptor_state() + .store_append(SmithySdkFeature::FlexibleChecksumsReqCrc32c); + } + ChecksumAlgorithm::Crc64Nvme => { + cfg.interceptor_state() + .store_append(SmithySdkFeature::FlexibleChecksumsReqCrc64); + } + #[allow(deprecated)] + ChecksumAlgorithm::Md5 => { + tracing::warn!(more_info = "Unsupported ChecksumAlgorithm MD5 set"); + } + ChecksumAlgorithm::Sha1 => { + cfg.interceptor_state() + .store_append(SmithySdkFeature::FlexibleChecksumsReqSha1); + } + ChecksumAlgorithm::Sha256 => { + cfg.interceptor_state() + .store_append(SmithySdkFeature::FlexibleChecksumsReqSha256); + } + unsupported => tracing::warn!( + more_info = "Unsupported value of ChecksumAlgorithm detected when setting user-agent metrics", + unsupported = ?unsupported), + } + let request = context.request_mut(); add_checksum_for_request_body(request, checksum_algorithm, cfg)?; } Ok(()) } + + /// Set the user-agent metrics for `RequestChecksumCalculation` here to avoid ownership issues + /// with the mutable borrow of cfg in `modify_before_signing` + fn read_after_serialization( + &self, + _context: &aws_smithy_runtime_api::client::interceptors::context::BeforeTransmitInterceptorContextRef<'_>, + _runtime_components: &RuntimeComponents, + cfg: &mut ConfigBag, + ) -> Result<(), BoxError> { + let request_checksum_calculation = cfg + .load::() + .unwrap_or(&RequestChecksumCalculation::WhenSupported); + + match request_checksum_calculation { + RequestChecksumCalculation::WhenSupported => { + cfg.interceptor_state() + .store_append(SmithySdkFeature::FlexibleChecksumsReqWhenSupported); + } + RequestChecksumCalculation::WhenRequired => { + cfg.interceptor_state() + .store_append(SmithySdkFeature::FlexibleChecksumsReqWhenRequired); + } + unsupported => tracing::warn!( + more_info = "Unsupported value of RequestChecksumCalculation when setting user-agent metrics", + unsupported = ?unsupported), + }; + + Ok(()) + } } fn incorporate_custom_default( @@ -179,13 +302,18 @@ fn add_checksum_for_request_body( match request.body().bytes() { // Body is in-memory: read it and insert the checksum as a header. Some(data) => { - tracing::debug!("applying {checksum_algorithm:?} of the request body as a header"); let mut checksum = checksum_algorithm.into_impl(); - checksum.update(data); - request - .headers_mut() - .insert(checksum.header_name(), checksum.header_value()); + // If the header has not already been set we set it. If it was already set by the user + // we do nothing and maintain their set value. + if request.headers().get(checksum.header_name()).is_none() { + tracing::debug!("applying {checksum_algorithm:?} of the request body as a header"); + checksum.update(data); + + request + .headers_mut() + .insert(checksum.header_name(), checksum.header_value()); + } } // Body is streaming: wrap the body so it will emit a checksum as a trailer. None => { @@ -202,6 +330,13 @@ fn wrap_streaming_request_body_in_checksum_calculating_body( request: &mut HttpRequest, checksum_algorithm: ChecksumAlgorithm, ) -> Result<(), BuildError> { + let checksum = checksum_algorithm.into_impl(); + + // If the user already set the header value then do nothing and return early + if request.headers().get(checksum.header_name()).is_some() { + return Ok(()); + } + let original_body_size = request .body() .size_hint() @@ -233,7 +368,7 @@ fn wrap_streaming_request_body_in_checksum_calculating_body( headers.insert( http::header::HeaderName::from_static("x-amz-trailer"), - checksum_algorithm.into_impl().header_name(), + checksum.header_name(), ); headers.insert( diff --git a/aws/rust-runtime/aws-inlineable/src/http_response_checksum.rs b/aws/rust-runtime/aws-inlineable/src/http_response_checksum.rs index 1236c9fe5..d28eca921 100644 --- a/aws/rust-runtime/aws-inlineable/src/http_response_checksum.rs +++ b/aws/rust-runtime/aws-inlineable/src/http_response_checksum.rs @@ -8,14 +8,16 @@ //! Interceptor for handling Smithy `@httpChecksum` response checksumming use aws_smithy_checksums::ChecksumAlgorithm; +use aws_smithy_runtime::client::sdk_feature::SmithySdkFeature; use aws_smithy_runtime_api::box_error::BoxError; use aws_smithy_runtime_api::client::interceptors::context::{ - BeforeDeserializationInterceptorContextMut, BeforeSerializationInterceptorContextRef, Input, + BeforeDeserializationInterceptorContextMut, BeforeSerializationInterceptorContextMut, Input, }; use aws_smithy_runtime_api::client::interceptors::Intercept; use aws_smithy_runtime_api::client::runtime_components::RuntimeComponents; use aws_smithy_runtime_api::http::Headers; use aws_smithy_types::body::SdkBody; +use aws_smithy_types::checksum_config::ResponseChecksumValidation; use aws_smithy_types::config_bag::{ConfigBag, Layer, Storable, StoreReplace}; use std::{fmt, mem}; @@ -27,12 +29,13 @@ impl Storable for ResponseChecksumInterceptorState { type Storer = StoreReplace; } -pub(crate) struct ResponseChecksumInterceptor { +pub(crate) struct ResponseChecksumInterceptor { response_algorithms: &'static [&'static str], validation_enabled: VE, + checksum_mutator: CM, } -impl fmt::Debug for ResponseChecksumInterceptor { +impl fmt::Debug for ResponseChecksumInterceptor { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { f.debug_struct("ResponseChecksumInterceptor") .field("response_algorithms", &self.response_algorithms) @@ -40,38 +43,61 @@ impl fmt::Debug for ResponseChecksumInterceptor { } } -impl ResponseChecksumInterceptor { +impl ResponseChecksumInterceptor { pub(crate) fn new( response_algorithms: &'static [&'static str], validation_enabled: VE, + checksum_mutator: CM, ) -> Self { Self { response_algorithms, validation_enabled, + checksum_mutator, } } } -impl Intercept for ResponseChecksumInterceptor +impl Intercept for ResponseChecksumInterceptor where VE: Fn(&Input) -> bool + Send + Sync, + CM: Fn(&mut Input, &ConfigBag) -> Result<(), BoxError> + Send + Sync, { fn name(&self) -> &'static str { "ResponseChecksumInterceptor" } - fn read_before_serialization( + fn modify_before_serialization( &self, - context: &BeforeSerializationInterceptorContextRef<'_>, + context: &mut BeforeSerializationInterceptorContextMut<'_>, _runtime_components: &RuntimeComponents, cfg: &mut ConfigBag, ) -> Result<(), BoxError> { + (self.checksum_mutator)(context.input_mut(), cfg)?; let validation_enabled = (self.validation_enabled)(context.input()); let mut layer = Layer::new("ResponseChecksumInterceptor"); layer.store_put(ResponseChecksumInterceptorState { validation_enabled }); cfg.push_layer(layer); + let response_checksum_validation = cfg + .load::() + .unwrap_or(&ResponseChecksumValidation::WhenSupported); + + // Set the user-agent feature metric for the response checksum config + match response_checksum_validation { + ResponseChecksumValidation::WhenSupported => { + cfg.interceptor_state() + .store_append(SmithySdkFeature::FlexibleChecksumsResWhenSupported); + } + ResponseChecksumValidation::WhenRequired => { + cfg.interceptor_state() + .store_append(SmithySdkFeature::FlexibleChecksumsResWhenRequired); + } + unsupported => tracing::warn!( + more_info = "Unsupported value of ResponseChecksumValidation when setting user-agent metrics", + unsupported = ?unsupported), + }; + Ok(()) } @@ -85,12 +111,33 @@ where .load::() .expect("set in `read_before_serialization`"); - if state.validation_enabled { + // This value is set by the user on the SdkConfig to indicate their preference + // We provide a default here for users that use a client config instead of the SdkConfig + let response_checksum_validation = cfg + .load::() + .unwrap_or(&ResponseChecksumValidation::WhenSupported); + + // If validation has not been explicitly enabled we check the ResponseChecksumValidation + // from the SdkConfig. If it is WhenSupported (or unknown) we enable validation and if it + // is WhenRequired we leave it disabled since there is no way to indicate that a response + // checksum is required. + let validation_enabled = if !state.validation_enabled { + match response_checksum_validation { + ResponseChecksumValidation::WhenRequired => false, + ResponseChecksumValidation::WhenSupported => true, + _ => true, + } + } else { + true + }; + + if validation_enabled { let response = context.response_mut(); let maybe_checksum_headers = check_headers_for_precalculated_checksum( response.headers(), self.response_algorithms, ); + if let Some((checksum_algorithm, precalculated_checksum)) = maybe_checksum_headers { let mut body = SdkBody::taken(); mem::swap(&mut body, response.body_mut()); @@ -195,8 +242,8 @@ fn is_part_level_checksum(checksum: &str) -> bool { continue; } - // Yup, it's a part-level checksum - if ch == '-' { + // We saw a number first followed by the dash, yup, it's a part-level checksum + if found_number && ch == '-' { if found_dash { // Found a second dash?? This isn't a part-level checksum. return false; @@ -267,4 +314,16 @@ mod tests { assert!(!is_part_level_checksum("abcd==--11")); assert!(!is_part_level_checksum("abcd==-AA")); } + + #[test] + fn part_level_checksum_detection_works() { + let a_real_checksum = is_part_level_checksum("C9A5A6878D97B48CC965C1E41859F034-14"); + assert!(a_real_checksum); + let close_but_not_quite = is_part_level_checksum("a4-"); + assert!(!close_but_not_quite); + let backwards = is_part_level_checksum("14-C9A5A6878D97B48CC965C1E41859F034"); + assert!(!backwards); + let double_dash = is_part_level_checksum("C9A5A6878D97B48CC965C1E41859F03-4-14"); + assert!(!double_dash); + } } diff --git a/aws/rust-runtime/aws-inlineable/src/presigning.rs b/aws/rust-runtime/aws-inlineable/src/presigning.rs index c19d160e1..c30830768 100644 --- a/aws/rust-runtime/aws-inlineable/src/presigning.rs +++ b/aws/rust-runtime/aws-inlineable/src/presigning.rs @@ -15,6 +15,7 @@ use aws_smithy_runtime_api::box_error::BoxError; use aws_smithy_runtime_api::client::orchestrator::HttpRequest; use aws_smithy_types::body::SdkBody; +use aws_smithy_types::config_bag::{Storable, StoreReplace}; use std::fmt; use std::time::{Duration, SystemTime}; @@ -265,3 +266,12 @@ impl fmt::Debug for PresignedRequest { .finish() } } + +/// A marker struct to be stored in the ConfigBag allowing other interceptors to know that +/// the current request is Presigned +#[derive(Debug)] +pub(crate) struct PresigningMarker; + +impl Storable for PresigningMarker { + type Storer = StoreReplace; +} diff --git a/aws/rust-runtime/aws-inlineable/src/presigning_interceptors.rs b/aws/rust-runtime/aws-inlineable/src/presigning_interceptors.rs index 481a8d9ca..25c674feb 100644 --- a/aws/rust-runtime/aws-inlineable/src/presigning_interceptors.rs +++ b/aws/rust-runtime/aws-inlineable/src/presigning_interceptors.rs @@ -5,7 +5,7 @@ #![allow(dead_code)] -use crate::presigning::PresigningConfig; +use crate::presigning::{PresigningConfig, PresigningMarker}; use crate::serialization_settings::HeaderSerializationSettings; use aws_runtime::auth::{HttpSignatureType, SigV4OperationSigningConfig}; use aws_runtime::invocation_id::InvocationIdInterceptor; @@ -63,6 +63,8 @@ impl Intercept for SigV4PresigningInterceptor { .omit_default_content_length() .omit_default_content_type(), ); + + cfg.interceptor_state().store_put(PresigningMarker); Ok(()) } diff --git a/aws/rust-runtime/aws-runtime/Cargo.toml b/aws/rust-runtime/aws-runtime/Cargo.toml index a30478ac5..9695833cc 100644 --- a/aws/rust-runtime/aws-runtime/Cargo.toml +++ b/aws/rust-runtime/aws-runtime/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "aws-runtime" -version = "1.5.3" +version = "1.5.4" authors = ["AWS Rust SDK Team "] description = "Runtime support code for the AWS SDK. This crate isn't intended to be used directly." edition = "2021" diff --git a/aws/rust-runtime/aws-runtime/src/user_agent/test_util.rs b/aws/rust-runtime/aws-runtime/src/user_agent/test_util.rs index a0a5679b6..de8013454 100644 --- a/aws/rust-runtime/aws-runtime/src/user_agent/test_util.rs +++ b/aws/rust-runtime/aws-runtime/src/user_agent/test_util.rs @@ -17,13 +17,8 @@ static RE: Lazy = Lazy::new(|| Regex::new(r"m/([A-Za-z0-9+/=_,-]+)").unwr /// Refer to the end of the parent module file `user_agent.rs` for the complete ABNF specification /// of `business-metrics`. pub fn assert_ua_contains_metric_values(user_agent: &str, values: &[&str]) { - match RE.find(user_agent) { - Some(matched) => { - let csv = matched - .as_str() - .strip_prefix("m/") - .expect("prefix `m/` is guaranteed to exist by regex match"); - let metrics: Vec<&str> = csv.split(',').collect(); + match extract_ua_values(user_agent) { + Some(metrics) => { let mut missed = vec![]; for value in values.iter() { @@ -43,6 +38,18 @@ pub fn assert_ua_contains_metric_values(user_agent: &str, values: &[&str]) { } } +/// Extract the metric values from the `user_agent` string +pub fn extract_ua_values(user_agent: &str) -> Option> { + RE.find(user_agent).map(|matched| { + matched + .as_str() + .strip_prefix("m/") + .expect("prefix `m/` is guaranteed to exist by regex match") + .split(',') + .collect() + }) +} + #[cfg(test)] mod tests { use super::*; diff --git a/aws/rust-runtime/aws-sigv4/Cargo.toml b/aws/rust-runtime/aws-sigv4/Cargo.toml index cc80e1590..5bda4ee04 100644 --- a/aws/rust-runtime/aws-sigv4/Cargo.toml +++ b/aws/rust-runtime/aws-sigv4/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "aws-sigv4" -version = "1.2.6" +version = "1.2.7" authors = ["AWS Rust SDK Team ", "David Barsky "] description = "SigV4 signer for HTTP requests and Event Stream messages." edition = "2021" @@ -45,7 +45,8 @@ aws-smithy-runtime-api = { path = "../../../rust-runtime/aws-smithy-runtime-api" bytes = "1" hex-literal = "0.4.1" httparse = "1.8" -libfuzzer-sys = "0.4.6" +# Version pinned due to https://github.com/rust-fuzz/libfuzzer/issues/126 +libfuzzer-sys = "=0.4.7" pretty_assertions = "1.3" proptest = "1.2" serde = "1.0.180" diff --git a/aws/rust-runtime/aws-sigv4/src/http_request/canonical_request.rs b/aws/rust-runtime/aws-sigv4/src/http_request/canonical_request.rs index 03404276d..4622f841d 100644 --- a/aws/rust-runtime/aws-sigv4/src/http_request/canonical_request.rs +++ b/aws/rust-runtime/aws-sigv4/src/http_request/canonical_request.rs @@ -31,6 +31,7 @@ pub(crate) mod header { pub(crate) const X_AMZ_DATE: &str = "x-amz-date"; pub(crate) const X_AMZ_SECURITY_TOKEN: &str = "x-amz-security-token"; pub(crate) const X_AMZ_USER_AGENT: &str = "x-amz-user-agent"; + pub(crate) const X_AMZ_CHECKSUM_MODE: &str = "x-amz-checksum-mode"; } pub(crate) mod param { @@ -293,8 +294,10 @@ impl<'a> CanonicalRequest<'a> { } if params.settings().signature_location == SignatureLocation::QueryParams { - // The X-Amz-User-Agent header should not be signed if this is for a presigned URL - if name == HeaderName::from_static(header::X_AMZ_USER_AGENT) { + // The X-Amz-User-Agent and x-amz-checksum-mode headers should not be signed if this is for a presigned URL + if name == HeaderName::from_static(header::X_AMZ_USER_AGENT) + || name == HeaderName::from_static(header::X_AMZ_CHECKSUM_MODE) + { continue; } } diff --git a/aws/rust-runtime/aws-types/Cargo.toml b/aws/rust-runtime/aws-types/Cargo.toml index 694b8676a..e13428e4b 100644 --- a/aws/rust-runtime/aws-types/Cargo.toml +++ b/aws/rust-runtime/aws-types/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "aws-types" -version = "1.3.3" +version = "1.3.4" authors = ["AWS Rust SDK Team ", "Russell Cohen "] description = "Cross-service types for the AWS SDK." edition = "2021" diff --git a/aws/rust-runtime/aws-types/external-types.toml b/aws/rust-runtime/aws-types/external-types.toml index 5dce405f8..ee28a1732 100644 --- a/aws/rust-runtime/aws-types/external-types.toml +++ b/aws/rust-runtime/aws-types/external-types.toml @@ -13,6 +13,8 @@ allowed_external_types = [ "aws_smithy_runtime_api::client::identity::SharedIdentityCache", "aws_smithy_runtime_api::client::stalled_stream_protection::StalledStreamProtectionConfig", "aws_smithy_runtime_api::http::headers::Headers", + "aws_smithy_types::checksum_config::ResponseChecksumValidation", + "aws_smithy_types::checksum_config::RequestChecksumCalculation", "aws_smithy_types::config_bag::storable::Storable", "aws_smithy_types::config_bag::storable::StoreReplace", "aws_smithy_types::config_bag::storable::Storer", diff --git a/aws/rust-runtime/aws-types/src/sdk_config.rs b/aws/rust-runtime/aws-types/src/sdk_config.rs index c26d8741b..19220fc52 100644 --- a/aws/rust-runtime/aws-types/src/sdk_config.rs +++ b/aws/rust-runtime/aws-types/src/sdk_config.rs @@ -25,6 +25,9 @@ pub use aws_smithy_runtime_api::client::http::SharedHttpClient; use aws_smithy_runtime_api::client::identity::{ResolveCachedIdentity, SharedIdentityCache}; pub use aws_smithy_runtime_api::client::stalled_stream_protection::StalledStreamProtectionConfig; use aws_smithy_runtime_api::shared::IntoShared; +pub use aws_smithy_types::checksum_config::{ + RequestChecksumCalculation, ResponseChecksumValidation, +}; pub use aws_smithy_types::retry::RetryConfig; pub use aws_smithy_types::timeout::TimeoutConfig; use std::collections::HashMap; @@ -92,6 +95,8 @@ pub struct SdkConfig { config_origins: HashMap<&'static str, Origin>, disable_request_compression: Option, request_min_compression_size_bytes: Option, + request_checksum_calculation: Option, + response_checksum_validation: Option, } /// Builder for AWS Shared Configuration @@ -120,6 +125,8 @@ pub struct Builder { config_origins: HashMap<&'static str, Origin>, disable_request_compression: Option, request_min_compression_size_bytes: Option, + request_checksum_calculation: Option, + response_checksum_validation: Option, } impl Builder { @@ -174,6 +181,54 @@ impl Builder { self } + /// Set the checksum calculation strategy to use when making requests. + /// # Examples + /// ``` + /// use aws_types::SdkConfig; + /// use aws_smithy_types::checksum_config::RequestChecksumCalculation; + /// let config = SdkConfig::builder().request_checksum_calculation(RequestChecksumCalculation::WhenSupported).build(); + /// ``` + pub fn request_checksum_calculation( + mut self, + request_checksum_calculation: RequestChecksumCalculation, + ) -> Self { + self.set_request_checksum_calculation(Some(request_checksum_calculation)); + self + } + + /// Set the checksum calculation strategy to use when making requests. + pub fn set_request_checksum_calculation( + &mut self, + request_checksum_calculation: Option, + ) -> &mut Self { + self.request_checksum_calculation = request_checksum_calculation; + self + } + + /// Set the checksum calculation strategy to use for responses. + /// # Examples + /// ``` + /// use aws_types::SdkConfig; + /// use aws_smithy_types::checksum_config::ResponseChecksumValidation; + /// let config = SdkConfig::builder().response_checksum_validation(ResponseChecksumValidation::WhenSupported).build(); + /// ``` + pub fn response_checksum_validation( + mut self, + response_checksum_validation: ResponseChecksumValidation, + ) -> Self { + self.set_response_checksum_validation(Some(response_checksum_validation)); + self + } + + /// Set the checksum calculation strategy to use for responses. + pub fn set_response_checksum_validation( + &mut self, + response_checksum_validation: Option, + ) -> &mut Self { + self.response_checksum_validation = response_checksum_validation; + self + } + /// Set the retry_config for the builder /// /// _Note:_ Retries require a sleep implementation in order to work. When enabling retry, make @@ -720,6 +775,8 @@ impl Builder { config_origins: self.config_origins, disable_request_compression: self.disable_request_compression, request_min_compression_size_bytes: self.request_min_compression_size_bytes, + request_checksum_calculation: self.request_checksum_calculation, + response_checksum_validation: self.response_checksum_validation, } } } @@ -866,6 +923,16 @@ impl SdkConfig { self.disable_request_compression } + /// Configured checksum request behavior. + pub fn request_checksum_calculation(&self) -> Option { + self.request_checksum_calculation + } + + /// Configured checksum response behavior. + pub fn response_checksum_validation(&self) -> Option { + self.response_checksum_validation + } + /// Configured minimum request compression size. pub fn request_min_compression_size_bytes(&self) -> Option { self.request_min_compression_size_bytes @@ -933,6 +1000,8 @@ impl SdkConfig { config_origins: self.config_origins, disable_request_compression: self.disable_request_compression, request_min_compression_size_bytes: self.request_min_compression_size_bytes, + request_checksum_calculation: self.request_checksum_calculation, + response_checksum_validation: self.response_checksum_validation, } } } diff --git a/aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/HttpRequestChecksumDecorator.kt b/aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/HttpRequestChecksumDecorator.kt index da5c2ac7b..f786b8194 100644 --- a/aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/HttpRequestChecksumDecorator.kt +++ b/aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/HttpRequestChecksumDecorator.kt @@ -6,11 +6,17 @@ package software.amazon.smithy.rustsdk import software.amazon.smithy.aws.traits.HttpChecksumTrait +import software.amazon.smithy.model.knowledge.TopDownIndex +import software.amazon.smithy.model.shapes.MemberShape import software.amazon.smithy.model.shapes.OperationShape +import software.amazon.smithy.model.traits.HttpHeaderTrait import software.amazon.smithy.rust.codegen.client.smithy.ClientCodegenContext +import software.amazon.smithy.rust.codegen.client.smithy.configReexport import software.amazon.smithy.rust.codegen.client.smithy.customize.ClientCodegenDecorator import software.amazon.smithy.rust.codegen.client.smithy.generators.OperationCustomization import software.amazon.smithy.rust.codegen.client.smithy.generators.OperationSection +import software.amazon.smithy.rust.codegen.client.smithy.generators.config.ConfigCustomization +import software.amazon.smithy.rust.codegen.client.smithy.generators.config.ServiceConfig import software.amazon.smithy.rust.codegen.core.rustlang.CargoDependency import software.amazon.smithy.rust.codegen.core.rustlang.Visibility import software.amazon.smithy.rust.codegen.core.rustlang.Writable @@ -20,9 +26,13 @@ import software.amazon.smithy.rust.codegen.core.rustlang.writable import software.amazon.smithy.rust.codegen.core.smithy.RuntimeConfig import software.amazon.smithy.rust.codegen.core.smithy.RuntimeType import software.amazon.smithy.rust.codegen.core.smithy.RuntimeType.Companion.preludeScope -import software.amazon.smithy.rust.codegen.core.smithy.generators.operationBuildError +import software.amazon.smithy.rust.codegen.core.smithy.customize.AdHocCustomization +import software.amazon.smithy.rust.codegen.core.smithy.customize.NamedCustomization +import software.amazon.smithy.rust.codegen.core.smithy.customize.adhocCustomization +import software.amazon.smithy.rust.codegen.core.util.dq import software.amazon.smithy.rust.codegen.core.util.expectMember import software.amazon.smithy.rust.codegen.core.util.getTrait +import software.amazon.smithy.rust.codegen.core.util.hasTrait import software.amazon.smithy.rust.codegen.core.util.inputShape import software.amazon.smithy.rust.codegen.core.util.orNull @@ -39,21 +49,49 @@ internal fun RuntimeConfig.awsInlineableHttpRequestChecksum() = CargoDependency.smithyHttp(this), CargoDependency.smithyRuntimeApiClient(this), CargoDependency.smithyTypes(this), - CargoDependency.TempFile, + AwsCargoDependency.awsSigv4(this), + CargoDependency.TempFile.toDevDependency(), ), ) class HttpRequestChecksumDecorator : ClientCodegenDecorator { override val name: String = "HttpRequestChecksum" override val order: Byte = 0 + private val defaultAlgorithm = "CRC32" override fun operationCustomizations( codegenContext: ClientCodegenContext, operation: OperationShape, baseCustomizations: List, ): List = baseCustomizations + HttpRequestChecksumCustomization(codegenContext, operation) + + override fun configCustomizations( + codegenContext: ClientCodegenContext, + baseCustomizations: List, + ): List = baseCustomizations + HttpRequestChecksumConfigCustomization(codegenContext) + + /** + * Copy the `request_checksum_calculation` value from the `SdkConfig` to the client config + */ + override fun extraSections(codegenContext: ClientCodegenContext): List = + if (!serviceHasHttpChecksumOperation(codegenContext)) { + listOf() + } else { + listOf( + adhocCustomization { section -> + rust( + """ + ${section.serviceConfigBuilder}.set_request_checksum_calculation(${section.sdkConfig}.request_checksum_calculation()); + """, + ) + }, + ) + } } +/** + * Extract the name of the operation's input member that indicates which checksum algorithm to use + */ private fun HttpChecksumTrait.requestAlgorithmMember( codegenContext: ClientCodegenContext, operationShape: OperationShape, @@ -65,6 +103,9 @@ private fun HttpChecksumTrait.requestAlgorithmMember( return codegenContext.symbolProvider.toMemberName(checksumAlgorithmMemberShape) } +/** + * Extract the name of the operation's input member that indicates which checksum algorithm to use + */ private fun HttpChecksumTrait.checksumAlgorithmToStr( codegenContext: ClientCodegenContext, operationShape: OperationShape, @@ -74,33 +115,14 @@ private fun HttpChecksumTrait.checksumAlgorithmToStr( val isRequestChecksumRequired = this.isRequestChecksumRequired return { - if (requestAlgorithmMember != null) { - if (isRequestChecksumRequired) { - // Checksums are required, fall back to MD5 - rust("""let checksum_algorithm = checksum_algorithm.map(|algorithm| algorithm.as_str()).or(Some("md5"));""") - } else { - // Checksums aren't required, don't set a fallback - rust("let checksum_algorithm = checksum_algorithm.map(|algorithm| algorithm.as_str());") - } - } else if (isRequestChecksumRequired) { - // Checksums are required but a user can't set one, so we set MD5 for them - rust("""let checksum_algorithm = Some("md5");""") + if (requestAlgorithmMember == null && isRequestChecksumRequired) { + // Checksums are required but a user can't set one, so we set crc32 for them + rust("""let checksum_algorithm = Some("crc32");""") + } else { + // Use checksum algo set by user or crc32 if one has not been set + rust("""let checksum_algorithm = checksum_algorithm.map(|algorithm| algorithm.as_str()).or(Some("crc32"));""") } - rustTemplate( - """ - let checksum_algorithm = match checksum_algorithm { - Some(algo) => Some( - algo.parse::<#{ChecksumAlgorithm}>() - .map_err(#{BuildError}::other)? - ), - None => None, - }; - """, - "BuildError" to runtimeConfig.operationBuildError(), - "ChecksumAlgorithm" to RuntimeType.smithyChecksums(runtimeConfig).resolve("ChecksumAlgorithm"), - ) - // If a request checksum is not required and there's no way to set one, do nothing // This happens when an operation only supports response checksums } @@ -108,6 +130,12 @@ private fun HttpChecksumTrait.checksumAlgorithmToStr( // This generator was implemented based on this spec: // https://smithy.io/2.0/aws/aws-core.html#http-request-checksums + +/** + * Calculate the checksum algorithm based on the input member identified by the trait's + * `requestAlgorithmMember`. Then instantiate an (inlineable) `http_request_checksum` + * interceptor with that checksum algorithm. + */ class HttpRequestChecksumCustomization( private val codegenContext: ClientCodegenContext, private val operationShape: OperationShape, @@ -117,28 +145,94 @@ class HttpRequestChecksumCustomization( override fun section(section: OperationSection): Writable = writable { // Get the `HttpChecksumTrait`, returning early if this `OperationShape` doesn't have one + // if there is no request algorithm member or if there is no httpHeader for the request algorithm val checksumTrait = operationShape.getTrait() ?: return@writable - val requestAlgorithmMember = checksumTrait.requestAlgorithmMember(codegenContext, operationShape) + val requestAlgorithmMember = + checksumTrait.requestAlgorithmMemberShape(codegenContext, operationShape) ?: return@writable + val requestAlgoHeader = + requestAlgorithmMember.getTrait()?.value ?: return@writable + val requestAlgorithmMemberName = checksumTrait.requestAlgorithmMember(codegenContext, operationShape) val inputShape = codegenContext.model.expectShape(operationShape.inputShape) + val requestChecksumRequired = checksumTrait.isRequestChecksumRequired + val operationName = codegenContext.symbolProvider.toSymbol(operationShape).name + val requestAlgorithmMemberInner = + if (requestAlgorithmMember.isOptional) { + codegenContext.model.expectShape(requestAlgorithmMember.target) + } else { + requestAlgorithmMember + } when (section) { is OperationSection.AdditionalInterceptors -> { - if (requestAlgorithmMember != null) { + if (requestAlgorithmMemberName != null) { section.registerInterceptor(runtimeConfig, this) { val runtimeApi = RuntimeType.smithyRuntimeApiClient(runtimeConfig) rustTemplate( """ - #{RequestChecksumInterceptor}::new(|input: &#{Input}| { + #{RequestChecksumInterceptor}::new( + |input: &#{Input}| { let input: &#{OperationInput} = input.downcast_ref().expect("correct type"); - let checksum_algorithm = input.$requestAlgorithmMember(); + let checksum_algorithm = input.$requestAlgorithmMemberName(); #{checksum_algorithm_to_str} - #{Result}::<_, #{BoxError}>::Ok(checksum_algorithm) - }) + (checksum_algorithm.map(|s| s.to_string()), $requestChecksumRequired) + }, + |request: &mut #{Request}, cfg: &#{ConfigBag}| { + // We check if the user has set any of the checksum values manually + let mut user_set_checksum_value = false; + let headers_to_check = request.headers().iter().filter_map(|(name, _val)| { + if name.starts_with("x-amz-checksum-") { + Some(name) + } else { + None + } + }); + for algo_header in headers_to_check { + if request.headers().get(algo_header).is_some() { + user_set_checksum_value = true; + } + } + + // We check if the user set the checksum algo manually + let user_set_checksum_algo = request.headers() + .get(${requestAlgoHeader.dq()}) + .is_some(); + + // This value is set by the user on the SdkConfig to indicate their preference + let request_checksum_calculation = cfg + .load::<#{RequestChecksumCalculation}>() + .unwrap_or(&#{RequestChecksumCalculation}::WhenSupported); + + // From the httpChecksum trait + let http_checksum_required = $requestChecksumRequired; + + // If the RequestChecksumCalculation is WhenSupported and the user has not set a checksum value or algo + // we default to Crc32. If it is WhenRequired and a checksum is required by the trait and the user has not + // set a checksum value or algo we also set the default. In all other cases we do nothing. + match ( + request_checksum_calculation, + http_checksum_required, + user_set_checksum_value, + user_set_checksum_algo + ) { + (#{RequestChecksumCalculation}::WhenSupported, _, false, false) + | (#{RequestChecksumCalculation}::WhenRequired, true, false, false) => { + request.headers_mut().insert(${requestAlgoHeader.dq()}, "CRC32"); + } + _ => {}, + } + + // We return a bool indicating if the user did set the checksum value, if they did + // we can short circuit and exit the interceptor early. + Ok(user_set_checksum_value) + } + ) """, *preludeScope, "BoxError" to RuntimeType.boxError(runtimeConfig), "Input" to runtimeApi.resolve("client::interceptors::context::Input"), + "Request" to RuntimeType.smithyRuntimeApi(runtimeConfig).resolve("http::Request"), "OperationInput" to codegenContext.symbolProvider.toSymbol(inputShape), + "ConfigBag" to RuntimeType.configBag(runtimeConfig), "RequestChecksumInterceptor" to runtimeConfig.awsInlineableHttpRequestChecksum() .resolve("RequestChecksumInterceptor"), @@ -147,11 +241,129 @@ class HttpRequestChecksumCustomization( codegenContext, operationShape, ), + "RequestChecksumCalculation" to + CargoDependency.smithyTypes(runtimeConfig).toType() + .resolve("checksum_config::RequestChecksumCalculation"), + "ChecksumAlgoShape" to + codegenContext.symbolProvider.toSymbol( + requestAlgorithmMemberInner, + ), + "OperationInputType" to + codegenContext.symbolProvider.toSymbol( + operationShape.inputShape( + codegenContext.model, + ), + ), ) } } } - else -> { } + + else -> {} } } } + +/** + * Add a `request_checksum_calculation;` field to Service config. + */ +class HttpRequestChecksumConfigCustomization(private val codegenContext: ClientCodegenContext) : + NamedCustomization() { + private val rc = codegenContext.runtimeConfig + private val codegenScope = + arrayOf( + *preludeScope, + "RequestChecksumCalculation" to + configReexport( + RuntimeType.smithyTypes(rc) + .resolve("checksum_config::RequestChecksumCalculation"), + ), + ) + + override fun section(section: ServiceConfig): Writable { + // If the service contains no operations with the httpChecksum trait we return early + if (!serviceHasHttpChecksumOperation(codegenContext)) { + return emptySection + } + + // Otherwise we write the necessary sections to the service config + return when (section) { + is ServiceConfig.ConfigImpl -> + writable { + rustTemplate( + """ + /// Return a reference to the request_checksum_calculation value contained in this config, if any. + pub fn request_checksum_calculation(&self) -> #{Option}<&#{RequestChecksumCalculation}> { + self.config.load::<#{RequestChecksumCalculation}>() + } + """, + *codegenScope, + ) + } + + is ServiceConfig.BuilderImpl -> + writable { + rustTemplate( + """ + /// Set the [`RequestChecksumCalculation`](#{RequestChecksumCalculation}) + /// to determine when a checksum will be calculated for request payloads. + pub fn request_checksum_calculation( + mut self, + request_checksum_calculation: #{RequestChecksumCalculation} + ) -> Self { + self.set_request_checksum_calculation(#{Some}(request_checksum_calculation)); + self + } + """, + *codegenScope, + ) + + rustTemplate( + """ + /// Set the [`RequestChecksumCalculation`](#{RequestChecksumCalculation}) + /// to determine when a checksum will be calculated for request payloads. + pub fn set_request_checksum_calculation( + &mut self, + request_checksum_calculation: #{Option}<#{RequestChecksumCalculation}> + ) -> &mut Self { + self.config.store_or_unset(request_checksum_calculation); + self + } + """, + *codegenScope, + ) + } + + is ServiceConfig.BuilderFromConfigBag -> + writable { + rustTemplate( + "${section.builder}.set_request_checksum_calculation(${section.configBag}.load::<#{RequestChecksumCalculation}>().cloned());", + *codegenScope, + ) + } + + else -> emptySection + } + } +} + +/** + * Determine if the current service contains any operations with the HttpChecksum trait + */ +fun serviceHasHttpChecksumOperation(codegenContext: ClientCodegenContext): Boolean { + val index = TopDownIndex.of(codegenContext.model) + val ops = index.getContainedOperations(codegenContext.serviceShape.id) + return ops.any { it.hasTrait() } +} + +/** + * Get the top-level operation input member used to opt-in to best-effort validation of a checksum returned in + * the HTTP response of the operation. + */ +fun HttpChecksumTrait.requestAlgorithmMemberShape( + codegenContext: ClientCodegenContext, + operationShape: OperationShape, +): MemberShape? { + val requestAlgorithmMember = this.requestAlgorithmMember.orNull() ?: return null + return operationShape.inputShape(codegenContext.model).expectMember(requestAlgorithmMember) +} diff --git a/aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/HttpResponseChecksumDecorator.kt b/aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/HttpResponseChecksumDecorator.kt index 16a452167..66c1402ff 100644 --- a/aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/HttpResponseChecksumDecorator.kt +++ b/aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/HttpResponseChecksumDecorator.kt @@ -10,17 +10,25 @@ import software.amazon.smithy.model.shapes.MemberShape import software.amazon.smithy.model.shapes.OperationShape import software.amazon.smithy.model.shapes.ShapeId import software.amazon.smithy.rust.codegen.client.smithy.ClientCodegenContext +import software.amazon.smithy.rust.codegen.client.smithy.configReexport import software.amazon.smithy.rust.codegen.client.smithy.customize.ClientCodegenDecorator import software.amazon.smithy.rust.codegen.client.smithy.generators.OperationCustomization import software.amazon.smithy.rust.codegen.client.smithy.generators.OperationSection +import software.amazon.smithy.rust.codegen.client.smithy.generators.config.ConfigCustomization +import software.amazon.smithy.rust.codegen.client.smithy.generators.config.ServiceConfig import software.amazon.smithy.rust.codegen.core.rustlang.CargoDependency import software.amazon.smithy.rust.codegen.core.rustlang.Visibility import software.amazon.smithy.rust.codegen.core.rustlang.Writable +import software.amazon.smithy.rust.codegen.core.rustlang.rust import software.amazon.smithy.rust.codegen.core.rustlang.rustTemplate import software.amazon.smithy.rust.codegen.core.rustlang.writable import software.amazon.smithy.rust.codegen.core.smithy.RuntimeConfig import software.amazon.smithy.rust.codegen.core.smithy.RuntimeType import software.amazon.smithy.rust.codegen.core.smithy.RuntimeType.Companion.preludeScope +import software.amazon.smithy.rust.codegen.core.smithy.customize.AdHocCustomization +import software.amazon.smithy.rust.codegen.core.smithy.customize.NamedCustomization +import software.amazon.smithy.rust.codegen.core.smithy.customize.adhocCustomization +import software.amazon.smithy.rust.codegen.core.util.dq import software.amazon.smithy.rust.codegen.core.util.expectMember import software.amazon.smithy.rust.codegen.core.util.getTrait import software.amazon.smithy.rust.codegen.core.util.inputShape @@ -42,6 +50,10 @@ private fun RuntimeConfig.awsInlineableHttpResponseChecksum() = ), ) +/** + * Get the top-level operation input member used to opt-in to best-effort validation of a checksum returned in + * the HTTP response of the operation. + */ fun HttpChecksumTrait.requestValidationModeMember( codegenContext: ClientCodegenContext, operationShape: OperationShape, @@ -65,19 +77,50 @@ class HttpResponseChecksumDecorator : ClientCodegenDecorator { baseCustomizations.letIf(applies(operation)) { it + HttpResponseChecksumCustomization(codegenContext, operation) } + + override fun configCustomizations( + codegenContext: ClientCodegenContext, + baseCustomizations: List, + ): List = baseCustomizations + HttpResponseChecksumConfigCustomization(codegenContext) + + /** + * Copy the `response_checksum_validation` value from the `SdkConfig` to the client config + */ + override fun extraSections(codegenContext: ClientCodegenContext): List = + if (serviceHasHttpChecksumOperation(codegenContext)) { + listOf( + adhocCustomization { section -> + rust( + """ + ${section.serviceConfigBuilder}.set_response_checksum_validation(${section.sdkConfig}.response_checksum_validation()); + """, + ) + }, + ) + } else { + listOf() + } } // This generator was implemented based on this spec: // https://smithy.io/2.0/aws/aws-core.html#http-request-checksums + +/** + * Calculate the checksum algorithm based on the input member identified by the trait's + * `requestAlgorithmMember`. Then instantiate an (inlineable) `http_request_checksum` + * interceptor with that checksum algorithm. + */ class HttpResponseChecksumCustomization( private val codegenContext: ClientCodegenContext, private val operationShape: OperationShape, ) : OperationCustomization() { override fun section(section: OperationSection): Writable = writable { + // Return early if this operation lacks the `httpChecksum` trait or that trait lacks a `requestValidationModeMember` val checksumTrait = operationShape.getTrait() ?: return@writable val requestValidationModeMember = checksumTrait.requestValidationModeMember(codegenContext, operationShape) ?: return@writable + val requestValidationModeName = codegenContext.symbolProvider.toSymbol(requestValidationModeMember).name val requestValidationModeMemberInner = if (requestValidationModeMember.isOptional) { codegenContext.model.expectShape(requestValidationModeMember.target) @@ -86,6 +129,7 @@ class HttpResponseChecksumCustomization( } val validationModeName = codegenContext.symbolProvider.toMemberName(requestValidationModeMember) val inputShape = codegenContext.model.expectShape(operationShape.inputShape) + val operationName = codegenContext.symbolProvider.toSymbol(operationShape).name when (section) { is OperationSection.AdditionalInterceptors -> { @@ -93,17 +137,48 @@ class HttpResponseChecksumCustomization( // CRC32, CRC32C, SHA256, SHA1 -> "crc32", "crc32c", "sha256", "sha1" val responseAlgorithms = checksumTrait.responseAlgorithms - .map { algorithm -> algorithm.lowercase() }.joinToString(", ") { algorithm -> "\"$algorithm\"" } + .map { algorithm -> algorithm.lowercase() } + .joinToString(", ") { algorithm -> algorithm.dq() } val runtimeApi = RuntimeType.smithyRuntimeApiClient(codegenContext.runtimeConfig) rustTemplate( """ #{ResponseChecksumInterceptor}::new( [$responseAlgorithms].as_slice(), |input: &#{Input}| { - ${""/* Per [the spec](https://smithy.io/2.0/aws/aws-core.html#http-response-checksums), - we check to see if it's the `ENABLED` variant */} + ${ + ""/* Per [the spec](https://smithy.io/2.0/aws/aws-core.html#http-response-checksums), + we check to see if it's the `ENABLED` variant */ + } let input: &#{OperationInput} = input.downcast_ref().expect("correct type"); matches!(input.$validationModeName(), #{Some}(#{ValidationModeShape}::Enabled)) + }, + |input: &mut #{Input}, cfg: &#{ConfigBag}| { + let input = input + .downcast_mut::<#{OperationInputType}>() + .ok_or("failed to downcast to #{OperationInputType}")?; + + let request_validation_enabled = + matches!(input.$requestValidationModeName(), Some(#{ValidationModeShape}::Enabled)); + + if !request_validation_enabled { + // This value is set by the user on the SdkConfig to indicate their preference + let response_checksum_validation = cfg + .load::<#{ResponseChecksumValidation}>() + .unwrap_or(&#{ResponseChecksumValidation}::WhenSupported); + + // If validation setting is WhenSupported (or unknown) we enable response checksum + // validation. If it is WhenRequired we do not enable (since there is no way to + // indicate that a response checksum is required). + ##[allow(clippy::wildcard_in_or_patterns)] + match response_checksum_validation { + #{ResponseChecksumValidation}::WhenRequired => {} + #{ResponseChecksumValidation}::WhenSupported | _ => { + input.$requestValidationModeName = Some(#{ValidationModeShape}::Enabled); + } + } + } + + #{Ok}(()) } ) """, @@ -113,7 +188,24 @@ class HttpResponseChecksumCustomization( .resolve("ResponseChecksumInterceptor"), "Input" to runtimeApi.resolve("client::interceptors::context::Input"), "OperationInput" to codegenContext.symbolProvider.toSymbol(inputShape), - "ValidationModeShape" to codegenContext.symbolProvider.toSymbol(requestValidationModeMemberInner), + "ValidationModeShape" to + codegenContext.symbolProvider.toSymbol( + requestValidationModeMemberInner, + ), + "OperationInputType" to + codegenContext.symbolProvider.toSymbol( + operationShape.inputShape( + codegenContext.model, + ), + ), + "ValidationModeShape" to + codegenContext.symbolProvider.toSymbol( + requestValidationModeMemberInner, + ), + "ResponseChecksumValidation" to + CargoDependency.smithyTypes(codegenContext.runtimeConfig).toType() + .resolve("checksum_config::ResponseChecksumValidation"), + "ConfigBag" to RuntimeType.configBag(codegenContext.runtimeConfig), ) } } @@ -122,3 +214,86 @@ class HttpResponseChecksumCustomization( } } } + +/** + * Add a `response_checksum_validation;` field to Service config. + */ +class HttpResponseChecksumConfigCustomization(private val codegenContext: ClientCodegenContext) : + NamedCustomization() { + private val rc = codegenContext.runtimeConfig + private val codegenScope = + arrayOf( + *preludeScope, + "ResponseChecksumValidation" to + configReexport( + RuntimeType.smithyTypes(rc) + .resolve("checksum_config::ResponseChecksumValidation"), + ), + ) + + override fun section(section: ServiceConfig): Writable { + // If the service contains no operations with the httpChecksum trait we return early + if (!serviceHasHttpChecksumOperation(codegenContext)) { + return emptySection + } + + // Otherwise we write the necessary sections to the service config + return when (section) { + is ServiceConfig.ConfigImpl -> + writable { + rustTemplate( + """ + /// Return a reference to the response_checksum_validation value contained in this config, if any. + pub fn response_checksum_validation(&self) -> #{Option}<&#{ResponseChecksumValidation}> { + self.config.load::<#{ResponseChecksumValidation}>() + } + """, + *codegenScope, + ) + } + + is ServiceConfig.BuilderImpl -> + writable { + rustTemplate( + """ + /// Set the [`ResponseChecksumValidation`](#{ResponseChecksumValidation}) + /// to determine when checksum validation will be performed on response payloads. + pub fn response_checksum_validation( + mut self, + response_checksum_validation: #{ResponseChecksumValidation} + ) -> Self { + self.set_response_checksum_validation(#{Some}(response_checksum_validation)); + self + } + """, + *codegenScope, + ) + + rustTemplate( + """ + /// Set the [`ResponseChecksumValidation`](#{ResponseChecksumValidation}) + /// to determine when checksum validation will be performed on response payloads. + pub fn set_response_checksum_validation( + &mut self, + response_checksum_validation: #{Option}<#{ResponseChecksumValidation}> + ) -> &mut Self { + self.config.store_or_unset(response_checksum_validation); + self + } + """, + *codegenScope, + ) + } + + is ServiceConfig.BuilderFromConfigBag -> + writable { + rustTemplate( + "${section.builder}.set_response_checksum_validation(${section.configBag}.load::<#{ResponseChecksumValidation}>().cloned());", + *codegenScope, + ) + } + + else -> emptySection + } + } +} diff --git a/aws/sdk-codegen/src/test/kotlin/software/amazon/smithy/rustsdk/HttpChecksumTest.kt b/aws/sdk-codegen/src/test/kotlin/software/amazon/smithy/rustsdk/HttpChecksumTest.kt new file mode 100644 index 000000000..d6d64f761 --- /dev/null +++ b/aws/sdk-codegen/src/test/kotlin/software/amazon/smithy/rustsdk/HttpChecksumTest.kt @@ -0,0 +1,938 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +package software.amazon.smithy.rustsdk + +import org.junit.jupiter.api.Test +import software.amazon.smithy.rust.codegen.client.smithy.ClientCodegenContext +import software.amazon.smithy.rust.codegen.core.rustlang.CargoDependency +import software.amazon.smithy.rust.codegen.core.rustlang.Feature +import software.amazon.smithy.rust.codegen.core.rustlang.Writable +import software.amazon.smithy.rust.codegen.core.rustlang.join +import software.amazon.smithy.rust.codegen.core.rustlang.plus +import software.amazon.smithy.rust.codegen.core.rustlang.rustTemplate +import software.amazon.smithy.rust.codegen.core.rustlang.writable +import software.amazon.smithy.rust.codegen.core.smithy.RuntimeType +import software.amazon.smithy.rust.codegen.core.smithy.RuntimeType.Companion.preludeScope +import software.amazon.smithy.rust.codegen.core.testutil.asSmithyModel +import software.amazon.smithy.rust.codegen.core.testutil.integrationTest +import software.amazon.smithy.rust.codegen.core.util.dq + +internal class HttpChecksumTest { + companion object { + private const val PREFIX = "\$version: \"2\"" + private val model = + """ + $PREFIX + namespace test + + use aws.api#service + use aws.auth#sigv4 + use aws.protocols#httpChecksum + use aws.protocols#restJson1 + use smithy.rules#endpointRuleSet + + @service(sdkId: "dontcare") + @restJson1 + @sigv4(name: "dontcare") + @auth([sigv4]) + @endpointRuleSet({ + "version": "1.0", + "rules": [{ "type": "endpoint", "conditions": [], "endpoint": { "url": "https://example.com" } }], + "parameters": { + "Region": { "required": false, "type": "String", "builtIn": "AWS::Region" }, + } + }) + service TestService { + version: "2023-01-01", + operations: [HttpChecksumOperation, HttpChecksumStreamingOperation] + } + + @http(uri: "/HttpChecksumOperation", method: "POST") + @optionalAuth + @httpChecksum( + requestChecksumRequired: true, + requestAlgorithmMember: "checksumAlgorithm", + requestValidationModeMember: "validationMode", + responseAlgorithms: ["CRC32", "CRC32C", "CRC64NVME", "SHA1", "SHA256"] + ) + operation HttpChecksumOperation { + input: SomeInput, + output: SomeOutput + } + + @input + structure SomeInput { + @httpHeader("x-amz-request-algorithm") + checksumAlgorithm: ChecksumAlgorithm + + @httpHeader("x-amz-response-validation-mode") + validationMode: ValidationMode + + @httpHeader("x-amz-checksum-crc32") + ChecksumCRC32: String + + @httpHeader("x-amz-checksum-crc32c") + ChecksumCRC32C: String + + @httpHeader("x-amz-checksum-crc64nvme") + ChecksumCRC64Nvme: String + + @httpHeader("x-amz-checksum-sha1") + ChecksumSHA1: String + + @httpHeader("x-amz-checksum-sha256") + ChecksumSHA256: String + + @httpHeader("x-amz-checksum-foo") + ChecksumFoo: String + + @httpPayload + @required + body: Blob + } + + @output + structure SomeOutput {} + + @http(uri: "/HttpChecksumStreamingOperation", method: "POST") + @optionalAuth + @httpChecksum( + requestChecksumRequired: true, + requestAlgorithmMember: "checksumAlgorithm", + requestValidationModeMember: "validationMode", + responseAlgorithms: ["CRC32", "CRC32C", "CRC64NVME", "SHA1", "SHA256"] + ) + operation HttpChecksumStreamingOperation { + input: SomeStreamingInput, + output: SomeStreamingOutput + } + + @streaming + blob StreamingBlob + + @input + structure SomeStreamingInput { + @httpHeader("x-amz-request-algorithm") + checksumAlgorithm: ChecksumAlgorithm + + @httpHeader("x-amz-response-validation-mode") + validationMode: ValidationMode + + @httpPayload + @required + body: StreamingBlob + } + + @output + structure SomeStreamingOutput {} + + enum ChecksumAlgorithm { + CRC32 + CRC32C + CRC64NVME + SHA1 + SHA256 + } + + enum ValidationMode { + ENABLED + } + """.asSmithyModel() + } + + @Test + fun requestChecksumWorks() { + awsSdkIntegrationTest(model) { context, rustCrate -> + // Allows us to use the user-agent test-utils in aws-runtime + rustCrate.mergeFeature(Feature("test-util", true, listOf("aws-runtime/test-util"))) + val rc = context.runtimeConfig + + // Create Writables for all test types + val checksumRequestTestWritables = + checksumRequestTests.map { createRequestChecksumCalculationTest(it, context) }.join("\n") + val checksumResponseSuccTestWritables = + checksumResponseSuccTests.map { createResponseChecksumValidationSuccessTest(it, context) }.join("\n") + val checksumResponseFailTestWritables = + checksumResponseFailTests.map { createResponseChecksumValidationFailureTest(it, context) }.join("\n") + val checksumStreamingRequestTestWritables = + streamingRequestTests.map { createStreamingRequestChecksumCalculationTest(it, context) }.join("\n") + val userProvidedChecksumTestWritables = + userProvidedChecksumTests.map { createUserProvidedChecksumsTest(it, context) }.join("\n") + val miscTests = createMiscellaneousTests(context) + + // Shared imports for all test types + // Note about the `//#{PresigningMarker}` below. The `RequestChecksumInterceptor` relies on the `PresigningMarker` type from + // the presigning inlineable. The decorator for that inlineable doesn't play nicely with the test model from the SEP, so we + // use this as a kind of blunt way to include the presigning inlineable without actually wiring it up with the model. + val testBase = + writable { + rustTemplate( + """ + ##![cfg(feature = "test-util")] + ##![allow(unused_imports)] + + use #{Blob}; + use #{Region}; + use #{pretty_assertions}::assert_eq; + use #{SdkBody}; + use std::io::Write; + use http_body::Body; + use #{HttpRequest}; + use #{UaAssert}; + use #{UaExtract}; + //#{PresigningMarker}; + """, + *preludeScope, + "Blob" to RuntimeType.smithyTypes(rc).resolve("Blob"), + "Region" to AwsRuntimeType.awsTypes(rc).resolve("region::Region"), + "pretty_assertions" to CargoDependency.PrettyAssertions.toType(), + "SdkBody" to RuntimeType.smithyTypes(rc).resolve("body::SdkBody"), + "HttpRequest" to RuntimeType.smithyRuntimeApi(rc).resolve("client::orchestrator::HttpRequest"), + "UaAssert" to + AwsRuntimeType.awsRuntime(rc) + .resolve("user_agent::test_util::assert_ua_contains_metric_values"), + "UaExtract" to + AwsRuntimeType.awsRuntime(rc) + .resolve("user_agent::test_util::extract_ua_values"), + "PresigningMarker" to AwsRuntimeType.presigning().resolve("PresigningMarker"), + ) + } + + // Create one integ test per test type + rustCrate.integrationTest("request_checksums") { + testBase.plus(checksumRequestTestWritables)() + } + + rustCrate.integrationTest("response_checksums_success") { + testBase.plus(checksumResponseSuccTestWritables)() + } + + rustCrate.integrationTest("response_checksums_fail") { + testBase.plus(checksumResponseFailTestWritables)() + } + + rustCrate.integrationTest("streaming_request_checksums") { + testBase.plus(checksumStreamingRequestTestWritables)() + } + + rustCrate.integrationTest("user_provided_checksums") { + testBase.plus(userProvidedChecksumTestWritables)() + } + + rustCrate.integrationTest("misc_tests") { + testBase.plus(miscTests)() + } + } + } + + /** + * Generate tests where the request checksum is calculated correctly + */ + private fun createRequestChecksumCalculationTest( + testDef: RequestChecksumCalculationTest, + context: ClientCodegenContext, + ): Writable { + val rc = context.runtimeConfig + val moduleName = context.moduleUseName() + val algoLower = testDef.checksumAlgorithm.lowercase() + // If the algo is Crc32 don't explicitly set it to test that the default is correctly set + val setChecksumAlgo = + if (testDef.checksumAlgorithm != "Crc32") { + ".checksum_algorithm($moduleName::types::ChecksumAlgorithm::${testDef.checksumAlgorithm})" + } else { + "" + } + return writable { + rustTemplate( + """ + //${testDef.docs} + ##[#{tokio}::test] + async fn ${algoLower}_request_checksums_work() { + let (http_client, rx) = #{capture_request}(None); + let config = $moduleName::Config::builder() + .region(Region::from_static("doesntmatter")) + .with_test_defaults() + .http_client(http_client) + .build(); + + let client = $moduleName::Client::from_conf(config); + let _ = client.http_checksum_operation() + .body(Blob::new(b"${testDef.requestPayload}")) + $setChecksumAlgo + .send() + .await; + let request = rx.expect_request(); + let ${algoLower}_header = request.headers() + .get("x-amz-checksum-$algoLower") + .expect("x-amz-checksum-$algoLower header should exist"); + + assert_eq!(${algoLower}_header, "${testDef.checksumHeader}"); + + let algo_header = request.headers() + .get("x-amz-request-algorithm") + .expect("algo header should exist"); + + assert_eq!(algo_header, "${testDef.algoHeader}"); + + // Check the user-agent metrics for the selected algo + assert_ua_contains_metric_values( + &request + .headers() + .get("x-amz-user-agent") + .expect("UA header should be present"), + &["${testDef.algoFeatureId}"], + ); + } + """, + *preludeScope, + "tokio" to CargoDependency.Tokio.toType(), + "capture_request" to RuntimeType.captureRequest(rc), + ) + } + } + + /** + * Generate tests where the request is streaming and checksum is calculated correctly + */ + private fun createStreamingRequestChecksumCalculationTest( + testDef: StreamingRequestChecksumCalculationTest, + context: ClientCodegenContext, + ): Writable { + val rc = context.runtimeConfig + val moduleName = context.moduleUseName() + val algoLower = testDef.checksumAlgorithm.lowercase() + // If the algo is Crc32 don't explicitly set it to test that the default is correctly set + val setChecksumAlgo = + if (testDef.checksumAlgorithm != "Crc32") { + ".checksum_algorithm($moduleName::types::ChecksumAlgorithm::${testDef.checksumAlgorithm})" + } else { + "" + } + return writable { + rustTemplate( + """ + //${testDef.docs} + ##[#{tokio}::test] + async fn ${algoLower}_request_checksums_work() { + let (http_client, rx) = #{capture_request}(None); + let config = $moduleName::Config::builder() + .region(Region::from_static("doesntmatter")) + .with_test_defaults() + .http_client(http_client) + .build(); + + let client = $moduleName::Client::from_conf(config); + + let mut file = tempfile::NamedTempFile::new().unwrap(); + file.as_file_mut() + .write_all("${testDef.requestPayload}".as_bytes()) + .unwrap(); + + let streaming_body = aws_smithy_types::byte_stream::ByteStream::read_from() + .path(&file) + .build() + .await + .unwrap(); + + let _operation = client + .http_checksum_streaming_operation() + .body(streaming_body) + $setChecksumAlgo + .send() + .await; + + let request = rx.expect_request(); + + let headers = request.headers(); + + assert_eq!( + headers.get("x-amz-trailer").unwrap(), + "x-amz-checksum-$algoLower", + ); + assert_eq!(headers.get("content-encoding").unwrap(), "aws-chunked"); + + let mut body = request.body().try_clone().expect("body is retryable"); + + let mut body_data = bytes::BytesMut::new(); + while let Some(data) = body.data().await { + body_data.extend_from_slice(&data.unwrap()) + } + + let body_string = std::str::from_utf8(&body_data).unwrap(); + assert!(body_string.contains("x-amz-checksum-$algoLower:${testDef.trailerChecksum}")); + } + """, + *preludeScope, + "tokio" to CargoDependency.Tokio.toType(), + "capture_request" to RuntimeType.captureRequest(rc), + ) + } + } + + /** + * Generate tests where the response checksum validates successfully + */ + private fun createResponseChecksumValidationSuccessTest( + testDef: ResponseChecksumValidationSuccessTest, + context: ClientCodegenContext, + ): Writable { + val rc = context.runtimeConfig + val moduleName = context.moduleUseName() + val algoLower = testDef.checksumAlgorithm.lowercase() + return writable { + rustTemplate( + """ + //${testDef.docs} + ##[::tokio::test] + async fn ${algoLower}_response_checksums_works() { + let (http_client, _rx) = #{capture_request}(Some( + http::Response::builder() + .header("x-amz-checksum-$algoLower", "${testDef.checksumHeaderValue}") + .body(SdkBody::from("${testDef.responsePayload}")) + .unwrap(), + )); + let config = $moduleName::Config::builder() + .region(Region::from_static("doesntmatter")) + .with_test_defaults() + .http_client(http_client) + .build(); + + let client = $moduleName::Client::from_conf(config); + let res = client + .http_checksum_operation() + .body(Blob::new(b"Doesn't matter.")) + .checksum_algorithm($moduleName::types::ChecksumAlgorithm::${testDef.checksumAlgorithm}) + .validation_mode($moduleName::types::ValidationMode::Enabled) + .send() + .await; + assert!(res.is_ok()) + } + """, + *preludeScope, + "tokio" to CargoDependency.Tokio.toType(), + "capture_request" to RuntimeType.captureRequest(rc), + ) + } + } + + /** + * Generate tests where the response checksum fails to validate + */ + private fun createResponseChecksumValidationFailureTest( + testDef: ResponseChecksumValidationFailureTest, + context: ClientCodegenContext, + ): Writable { + val rc = context.runtimeConfig + val moduleName = context.moduleUseName() + val algoLower = testDef.checksumAlgorithm.lowercase() + return writable { + rustTemplate( + """ + //${testDef.docs} + ##[::tokio::test] + async fn ${algoLower}_response_checksums_fail_correctly() { + let (http_client, _rx) = #{capture_request}(Some( + http::Response::builder() + .header("x-amz-checksum-$algoLower", "${testDef.checksumHeaderValue}") + .body(SdkBody::from("${testDef.responsePayload}")) + .unwrap(), + )); + let config = $moduleName::Config::builder() + .region(Region::from_static("doesntmatter")) + .with_test_defaults() + .http_client(http_client) + .build(); + + let client = $moduleName::Client::from_conf(config); + let res = client + .http_checksum_operation() + .body(Blob::new(b"Doesn't matter.")) + .checksum_algorithm($moduleName::types::ChecksumAlgorithm::${testDef.checksumAlgorithm}) + .validation_mode($moduleName::types::ValidationMode::Enabled) + .send() + .await; + + assert!(res.is_err()); + + let boxed_err = res + .unwrap_err() + .into_source() + .unwrap() + .downcast::(); + let typed_err = boxed_err.as_ref().unwrap().as_ref(); + + match typed_err { + aws_smithy_checksums::body::validate::Error::ChecksumMismatch { actual, .. } => { + let calculated_checksum = aws_smithy_types::base64::encode(actual); + assert_eq!(calculated_checksum, "${testDef.calculatedChecksum}"); + } + _ => panic!("Unknown error type in checksum validation"), + }; + } + """, + *preludeScope, + "tokio" to CargoDependency.Tokio.toType(), + "capture_request" to RuntimeType.captureRequest(rc), + ) + } + } + + /** + * Generate tests for the case where a user provides a checksum + */ + private fun createUserProvidedChecksumsTest( + testDef: UserProvidedChecksumTest, + context: ClientCodegenContext, + ): Writable { + val rc = context.runtimeConfig + val moduleName = context.moduleUseName() + val algoLower = testDef.checksumAlgorithm.lowercase() + // We treat the c after crc32c and the nvme after crc64nvme as separate words + // so this quick map helps us find the field to set + val algoFieldNames = + mapOf( + "crc32" to "checksum_crc32", + "crc32c" to "checksum_crc32_c", + "crc64nvme" to "checksum_crc64_nvme", + "foo" to "checksum_foo", + "sha1" to "checksum_sha1", + "sha256" to "checksum_sha256", + ) + + return writable { + rustTemplate( + """ + //${testDef.docs} + ##[#{tokio}::test] + async fn user_provided_${algoLower}_request_checksum_works() { + let (http_client, rx) = #{capture_request}(None); + let config = $moduleName::Config::builder() + .region(Region::from_static("doesntmatter")) + .with_test_defaults() + .http_client(http_client) + .build(); + + let client = $moduleName::Client::from_conf(config); + let _ = client.http_checksum_operation() + .body(Blob::new(b"${testDef.requestPayload}")) + .${algoFieldNames.get(algoLower)}(${testDef.checksumValue.dq()}) + .send() + .await; + let request = rx.expect_request(); + let ${algoLower}_header = request.headers() + .get("x-amz-checksum-$algoLower") + .expect("x-amz-checksum-$algoLower header should exist"); + + assert_eq!(${algoLower}_header, "${testDef.expectedHeaderValue}"); + + let algo_header = request.headers() + .get("x-amz-request-algorithm"); + + assert!(algo_header.is_none()); + } + """, + *preludeScope, + "tokio" to CargoDependency.Tokio.toType(), + "capture_request" to RuntimeType.captureRequest(rc), + ) + } + } + + /** + * Generate miscellaneous tests, currently mostly focused on the inclusion of the checksum config metrics in the + * user-agent header + */ + private fun createMiscellaneousTests(context: ClientCodegenContext): Writable { + val rc = context.runtimeConfig + val moduleName = context.moduleUseName() + return writable { + rustTemplate( + """ + // The following tests confirm that the user-agent business metrics header is correctly + // set for the request/response checksum config values + ##[::tokio::test] + async fn request_config_ua_required() { + let (http_client, rx) = #{capture_request}(None); + let config = $moduleName::Config::builder() + .region(Region::from_static("doesntmatter")) + .with_test_defaults() + .http_client(http_client) + .request_checksum_calculation( + aws_types::sdk_config::RequestChecksumCalculation::WhenRequired, + ) + .build(); + + let client = $moduleName::Client::from_conf(config); + let _ = client + .http_checksum_operation() + .body(Blob::new(b"Doesn't matter")) + .send() + .await; + let request = rx.expect_request(); + + let sdk_metrics = extract_ua_values( + &request + .headers() + .get("x-amz-user-agent") + .expect("UA header should be present"), + ).expect("UA header should be present"); + assert!(sdk_metrics.contains(&"a")); + assert!(!sdk_metrics.contains(&"Z")); + } + + ##[::tokio::test] + async fn request_config_ua_supported() { + let (http_client, rx) = #{capture_request}(None); + let config = $moduleName::Config::builder() + .region(Region::from_static("doesntmatter")) + .with_test_defaults() + .http_client(http_client) + .build(); + + let client = $moduleName::Client::from_conf(config); + let _ = client + .http_checksum_operation() + .body(Blob::new(b"Doesn't matter")) + .send() + .await; + let request = rx.expect_request(); + + let sdk_metrics = extract_ua_values( + &request + .headers() + .get("x-amz-user-agent") + .expect("UA header should be present"), + ).expect("UA header should be present"); + assert!(sdk_metrics.contains(&"Z")); + assert!(!sdk_metrics.contains(&"a")); + } + + ##[::tokio::test] + async fn response_config_ua_supported() { + let (http_client, rx) = #{capture_request}(Some( + http::Response::builder() + .header("x-amz-checksum-crc32", "i9aeUg==") + .body(SdkBody::from("Hello world")) + .unwrap(), + )); + let config = $moduleName::Config::builder() + .region(Region::from_static("doesntmatter")) + .with_test_defaults() + .http_client(http_client) + .build(); + + let client = $moduleName::Client::from_conf(config); + let _ = client + .http_checksum_operation() + .body(Blob::new(b"Doesn't matter")) + .send() + .await; + let request = rx.expect_request(); + + let sdk_metrics = extract_ua_values( + &request + .headers() + .get("x-amz-user-agent") + .expect("UA header should be present"), + ).expect("UA header should be present"); + assert!(sdk_metrics.contains(&"b")); + assert!(!sdk_metrics.contains(&"c")); + } + + ##[::tokio::test] + async fn response_config_ua_required() { + let (http_client, rx) = #{capture_request}(Some( + http::Response::builder() + .header("x-amz-checksum-crc32", "i9aeUg==") + .body(SdkBody::from("Hello world")) + .unwrap(), + )); + let config = $moduleName::Config::builder() + .region(Region::from_static("doesntmatter")) + .with_test_defaults() + .http_client(http_client) + .response_checksum_validation( + aws_types::sdk_config::ResponseChecksumValidation::WhenRequired, + ) + .build(); + + let client = $moduleName::Client::from_conf(config); + let _ = client + .http_checksum_operation() + .body(Blob::new(b"Doesn't matter")) + .send() + .await; + let request = rx.expect_request(); + + let sdk_metrics = extract_ua_values( + &request + .headers() + .get("x-amz-user-agent") + .expect("UA header should be present"), + ).expect("UA header should be present"); + assert!(sdk_metrics.contains(&"c")); + assert!(!sdk_metrics.contains(&"b")); + } + """, + *preludeScope, + "tokio" to CargoDependency.Tokio.toType(), + "capture_request" to RuntimeType.captureRequest(rc), + ) + } + } +} + +// Classes and data for test definitions + +data class RequestChecksumCalculationTest( + val docs: String, + val requestPayload: String, + val checksumAlgorithm: String, + val algoHeader: String, + val checksumHeader: String, + val algoFeatureId: String, +) + +val checksumRequestTests = + listOf( + RequestChecksumCalculationTest( + "CRC32 checksum calculation works.", + "Hello world", + "Crc32", + "CRC32", + "i9aeUg==", + "U", + ), + RequestChecksumCalculationTest( + "CRC32C checksum calculation works.", + "Hello world", + "Crc32C", + "CRC32C", + "crUfeA==", + "V", + ), + RequestChecksumCalculationTest( + "CRC64NVME checksum calculation works.", + "Hello world", + "Crc64Nvme", + "CRC64NVME", + "OOJZ0D8xKts=", + "W", + ), + RequestChecksumCalculationTest( + "SHA1 checksum calculation works.", + "Hello world", + "Sha1", + "SHA1", + "e1AsOh9IyGCa4hLN+2Od7jlnP14=", + "X", + ), + RequestChecksumCalculationTest( + "SHA256 checksum calculation works.", + "Hello world", + "Sha256", + "SHA256", + "ZOyIygCyaOW6GjVnihtTFtIS9PNmskdyMlNKiuyjfzw=", + "Y", + ), + ) + +data class StreamingRequestChecksumCalculationTest( + val docs: String, + val requestPayload: String, + val checksumAlgorithm: String, + val trailerChecksum: String, +) + +val streamingRequestTests = + listOf( + StreamingRequestChecksumCalculationTest( + "CRC32 streaming checksum calculation works.", + "Hello world", + "Crc32", + "i9aeUg==", + ), + StreamingRequestChecksumCalculationTest( + "CRC32C streaming checksum calculation works.", + "Hello world", + "Crc32C", + "crUfeA==", + ), + StreamingRequestChecksumCalculationTest( + "CRC64NVME streaming checksum calculation works.", + "Hello world", + "Crc64Nvme", + "OOJZ0D8xKts=", + ), + StreamingRequestChecksumCalculationTest( + "SHA1 streaming checksum calculation works.", + "Hello world", + "Sha1", + "e1AsOh9IyGCa4hLN+2Od7jlnP14=", + ), + StreamingRequestChecksumCalculationTest( + "SHA256 streaming checksum calculation works.", + "Hello world", + "Sha256", + "ZOyIygCyaOW6GjVnihtTFtIS9PNmskdyMlNKiuyjfzw=", + ), + ) + +data class ResponseChecksumValidationSuccessTest( + val docs: String, + val responsePayload: String, + val checksumAlgorithm: String, + val checksumHeaderValue: String, +) + +val checksumResponseSuccTests = + listOf( + ResponseChecksumValidationSuccessTest( + "Successful payload validation with CRC32 checksum.", + "Hello world", + "Crc32", + "i9aeUg==", + ), + ResponseChecksumValidationSuccessTest( + "Successful payload validation with Crc32C checksum.", + "Hello world", + "Crc32C", + "crUfeA==", + ), + ResponseChecksumValidationSuccessTest( + "Successful payload validation with Crc64Nvme checksum.", + "Hello world", + "Crc64Nvme", + "OOJZ0D8xKts=", + ), + ResponseChecksumValidationSuccessTest( + "Successful payload validation with Sha1 checksum.", + "Hello world", + "Sha1", + "e1AsOh9IyGCa4hLN+2Od7jlnP14=", + ), + ResponseChecksumValidationSuccessTest( + "Successful payload validation with Sha256 checksum.", + "Hello world", + "Sha256", + "ZOyIygCyaOW6GjVnihtTFtIS9PNmskdyMlNKiuyjfzw=", + ), + ) + +data class ResponseChecksumValidationFailureTest( + val docs: String, + val responsePayload: String, + val checksumAlgorithm: String, + val checksumHeaderValue: String, + val calculatedChecksum: String, +) + +val checksumResponseFailTests = + listOf( + ResponseChecksumValidationFailureTest( + "Failed payload validation with CRC32 checksum.", + "Hello world", + "Crc32", + "bm90LWEtY2hlY2tzdW0=", + "i9aeUg==", + ), + ResponseChecksumValidationFailureTest( + "Failed payload validation with CRC32C checksum.", + "Hello world", + "Crc32C", + "bm90LWEtY2hlY2tzdW0=", + "crUfeA==", + ), + ResponseChecksumValidationFailureTest( + "Failed payload validation with CRC64NVME checksum.", + "Hello world", + "Crc64Nvme", + "bm90LWEtY2hlY2tzdW0=", + "OOJZ0D8xKts=", + ), + ResponseChecksumValidationFailureTest( + "Failed payload validation with SHA1 checksum.", + "Hello world", + "Sha1", + "bm90LWEtY2hlY2tzdW0=", + "e1AsOh9IyGCa4hLN+2Od7jlnP14=", + ), + ResponseChecksumValidationFailureTest( + "Failed payload validation with SHA256 checksum.", + "Hello world", + "Sha256", + "bm90LWEtY2hlY2tzdW0=", + "ZOyIygCyaOW6GjVnihtTFtIS9PNmskdyMlNKiuyjfzw=", + ), + ) + +data class UserProvidedChecksumTest( + val docs: String, + val requestPayload: String, + val checksumAlgorithm: String, + val checksumValue: String, + val expectedHeaderName: String, + val expectedHeaderValue: String, + val forbidHeaderName: String, +) + +val userProvidedChecksumTests = + listOf( + UserProvidedChecksumTest( + "CRC32 checksum provided by user.", + "Hello world", + "Crc32", + "i9aeUg==", + "x-amz-checksum-crc32", + "i9aeUg==", + "x-amz-request-algorithm", + ), + UserProvidedChecksumTest( + "CRC32C checksum provided by user.", + "Hello world", + "Crc32C", + "crUfeA==", + "x-amz-checksum-crc32c", + "crUfeA==", + "x-amz-request-algorithm", + ), + UserProvidedChecksumTest( + "CRC64NVME checksum provided by user.", + "Hello world", + "Crc64Nvme", + "OOJZ0D8xKts=", + "x-amz-checksum-crc64nvme", + "OOJZ0D8xKts=", + "x-amz-request-algorithm", + ), + UserProvidedChecksumTest( + "SHA1 checksum provided by user.", + "Hello world", + "Sha1", + "e1AsOh9IyGCa4hLN+2Od7jlnP14=", + "x-amz-checksum-sha1", + "e1AsOh9IyGCa4hLN+2Od7jlnP14=", + "x-amz-request-algorithm", + ), + UserProvidedChecksumTest( + "SHA256 checksum provided by user.", + "Hello world", + "Sha256", + "ZOyIygCyaOW6GjVnihtTFtIS9PNmskdyMlNKiuyjfzw=", + "x-amz-checksum-sha256", + "ZOyIygCyaOW6GjVnihtTFtIS9PNmskdyMlNKiuyjfzw=", + "x-amz-request-algorithm", + ), + UserProvidedChecksumTest( + "Forwards compatibility, unmodeled checksum provided by user.", + "Hello world", + "Foo", + "This-is-not-a-real-checksum", + "x-amz-checksum-foo", + "This-is-not-a-real-checksum", + "x-amz-request-algorithm", + ), + ) diff --git a/aws/sdk/Cargo.lock b/aws/sdk/Cargo.lock index 6ecbdceb3..830c671ce 100644 --- a/aws/sdk/Cargo.lock +++ b/aws/sdk/Cargo.lock @@ -38,12 +38,55 @@ version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4b46cbb362ab8752921c97e041f5e366ee6297bd428a31275b9fcf1e380f7299" +[[package]] +name = "anstream" +version = "0.6.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8acc5369981196006228e28809f761875c0327210a891e941f4c683b3a99529b" +dependencies = [ + "anstyle", + "anstyle-parse", + "anstyle-query", + "anstyle-wincon", + "colorchoice", + "is_terminal_polyfill", + "utf8parse", +] + [[package]] name = "anstyle" version = "1.0.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "55cc3b69f167a1ef2e161439aa98aed94e6028e5f9a59be9a6ffb47aef1651f9" +[[package]] +name = "anstyle-parse" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b2d16507662817a6a20a9ea92df6652ee4f94f914589377d69f3b21bc5798a9" +dependencies = [ + "utf8parse", +] + +[[package]] +name = "anstyle-query" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "79947af37f4177cfead1110013d678905c37501914fba0efea834c3fe9a8d60c" +dependencies = [ + "windows-sys 0.59.0", +] + +[[package]] +name = "anstyle-wincon" +version = "3.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2109dbce0e72be3ec00bed26e6a7479ca384ad226efdd66db8fa2e3a38c83125" +dependencies = [ + "anstyle", + "windows-sys 0.59.0", +] + [[package]] name = "approx" version = "0.5.1" @@ -317,20 +360,20 @@ checksum = "ace50bade8e6234aa140d9a2f552bbee1db4d353f69b8217bc503490fc1a9f26" [[package]] name = "aws-config" -version = "1.5.13" +version = "1.5.14" dependencies = [ "aws-credential-types 1.2.1", - "aws-runtime 1.5.3", + "aws-runtime 1.5.4", "aws-sdk-sso", "aws-sdk-ssooidc", "aws-sdk-sts", - "aws-smithy-async 1.2.3", + "aws-smithy-async 1.2.4", "aws-smithy-http 0.60.11", "aws-smithy-json 0.61.1", "aws-smithy-runtime 1.7.7", "aws-smithy-runtime-api 1.7.3", - "aws-smithy-types 1.2.11", - "aws-types 1.3.3", + "aws-smithy-types 1.2.12", + "aws-types 1.3.4", "bytes", "fastrand 2.0.2", "futures-util", @@ -354,9 +397,9 @@ name = "aws-credential-types" version = "1.2.1" dependencies = [ "async-trait", - "aws-smithy-async 1.2.3", + "aws-smithy-async 1.2.4", "aws-smithy-runtime-api 1.7.3", - "aws-smithy-types 1.2.11", + "aws-smithy-types 1.2.12", "tokio", "zeroize", ] @@ -367,9 +410,9 @@ version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "60e8f6b615cb5fc60a98132268508ad104310f0cfb25a1c22eee76efdf9154da" dependencies = [ - "aws-smithy-async 1.2.3 (registry+https://github.com/rust-lang/crates.io-index)", + "aws-smithy-async 1.2.3", "aws-smithy-runtime-api 1.7.3 (registry+https://github.com/rust-lang/crates.io-index)", - "aws-smithy-types 1.2.11 (registry+https://github.com/rust-lang/crates.io-index)", + "aws-smithy-types 1.2.11", "zeroize", ] @@ -381,7 +424,7 @@ version = "0.60.3" name = "aws-http" version = "0.60.6" dependencies = [ - "aws-runtime 1.5.3", + "aws-runtime 1.5.4", ] [[package]] @@ -432,19 +475,19 @@ dependencies = [ [[package]] name = "aws-runtime" -version = "1.5.3" +version = "1.5.4" dependencies = [ "arbitrary", "aws-credential-types 1.2.1", - "aws-sigv4 1.2.6", - "aws-smithy-async 1.2.3", + "aws-sigv4 1.2.7", + "aws-smithy-async 1.2.4", "aws-smithy-eventstream 0.60.5", "aws-smithy-http 0.60.11", "aws-smithy-protocol-test 0.63.0", "aws-smithy-runtime 1.7.7", "aws-smithy-runtime-api 1.7.3", - "aws-smithy-types 1.2.11", - "aws-types 1.3.3", + "aws-smithy-types 1.2.12", + "aws-types 1.3.4", "bytes", "bytes-utils", "convert_case", @@ -504,15 +547,15 @@ version = "0.0.0-local" dependencies = [ "aws-config", "aws-credential-types 1.2.1", - "aws-runtime 1.5.3", - "aws-smithy-async 1.2.3", + "aws-runtime 1.5.4", + "aws-smithy-async 1.2.4", "aws-smithy-eventstream 0.60.5", "aws-smithy-http 0.60.11", "aws-smithy-json 0.61.1", "aws-smithy-runtime 1.7.7", "aws-smithy-runtime-api 1.7.3", - "aws-smithy-types 1.2.11", - "aws-types 1.3.3", + "aws-smithy-types 1.2.12", + "aws-types 1.3.4", "bytes", "http 0.2.12", "once_cell", @@ -527,15 +570,15 @@ version = "0.0.0-local" dependencies = [ "aws-config", "aws-credential-types 1.2.1", - "aws-runtime 1.5.3", - "aws-smithy-async 1.2.3", + "aws-runtime 1.5.4", + "aws-smithy-async 1.2.4", "aws-smithy-http 0.60.11", "aws-smithy-json 0.61.1", "aws-smithy-protocol-test 0.63.0", "aws-smithy-runtime 1.7.7", "aws-smithy-runtime-api 1.7.3", - "aws-smithy-types 1.2.11", - "aws-types 1.3.3", + "aws-smithy-types 1.2.12", + "aws-types 1.3.4", "bytes", "fastrand 2.0.2", "futures-util", @@ -554,14 +597,14 @@ version = "0.0.0-local" dependencies = [ "aws-config", "aws-credential-types 1.2.1", - "aws-runtime 1.5.3", - "aws-smithy-async 1.2.3", + "aws-runtime 1.5.4", + "aws-smithy-async 1.2.4", "aws-smithy-http 0.60.11", "aws-smithy-json 0.61.1", "aws-smithy-runtime 1.7.7", "aws-smithy-runtime-api 1.7.3", - "aws-smithy-types 1.2.11", - "aws-types 1.3.3", + "aws-smithy-types 1.2.12", + "aws-types 1.3.4", "bytes", "http 0.2.12", "once_cell", @@ -577,15 +620,15 @@ dependencies = [ "approx", "aws-config", "aws-credential-types 1.2.1", - "aws-runtime 1.5.3", - "aws-smithy-async 1.2.3", + "aws-runtime 1.5.4", + "aws-smithy-async 1.2.4", "aws-smithy-http 0.60.11", "aws-smithy-json 0.61.1", "aws-smithy-protocol-test 0.63.0", "aws-smithy-runtime 1.7.7", "aws-smithy-runtime-api 1.7.3", - "aws-smithy-types 1.2.11", - "aws-types 1.3.3", + "aws-smithy-types 1.2.12", + "aws-types 1.3.4", "bytes", "criterion", "fastrand 2.0.2", @@ -605,17 +648,17 @@ version = "0.0.0-local" dependencies = [ "aws-config", "aws-credential-types 1.2.1", - "aws-runtime 1.5.3", - "aws-smithy-async 1.2.3", + "aws-runtime 1.5.4", + "aws-smithy-async 1.2.4", "aws-smithy-http 0.60.11", "aws-smithy-json 0.61.1", "aws-smithy-protocol-test 0.63.0", "aws-smithy-query", "aws-smithy-runtime 1.7.7", "aws-smithy-runtime-api 1.7.3", - "aws-smithy-types 1.2.11", + "aws-smithy-types 1.2.12", "aws-smithy-xml 0.60.9", - "aws-types 1.3.3", + "aws-types 1.3.4", "fastrand 2.0.2", "futures-util", "http 0.2.12", @@ -633,14 +676,14 @@ version = "0.0.0-local" dependencies = [ "aws-config", "aws-credential-types 1.2.1", - "aws-runtime 1.5.3", - "aws-smithy-async 1.2.3", + "aws-runtime 1.5.4", + "aws-smithy-async 1.2.4", "aws-smithy-http 0.60.11", "aws-smithy-json 0.61.1", "aws-smithy-runtime 1.7.7", "aws-smithy-runtime-api 1.7.3", - "aws-smithy-types 1.2.11", - "aws-types 1.3.3", + "aws-smithy-types 1.2.12", + "aws-types 1.3.4", "bytes", "fastrand 2.0.2", "http 0.2.12", @@ -656,16 +699,16 @@ version = "0.0.0-local" dependencies = [ "aws-config", "aws-credential-types 1.2.1", - "aws-runtime 1.5.3", - "aws-sigv4 1.2.6", - "aws-smithy-async 1.2.3", + "aws-runtime 1.5.4", + "aws-sigv4 1.2.7", + "aws-smithy-async 1.2.4", "aws-smithy-http 0.60.11", "aws-smithy-json 0.61.1", "aws-smithy-protocol-test 0.63.0", "aws-smithy-runtime 1.7.7", "aws-smithy-runtime-api 1.7.3", - "aws-smithy-types 1.2.11", - "aws-types 1.3.3", + "aws-smithy-types 1.2.12", + "aws-types 1.3.4", "bytes", "futures-util", "hex", @@ -687,17 +730,17 @@ version = "0.0.0-local" dependencies = [ "aws-config", "aws-credential-types 1.2.1", - "aws-runtime 1.5.3", - "aws-smithy-async 1.2.3", + "aws-runtime 1.5.4", + "aws-smithy-async 1.2.4", "aws-smithy-http 0.60.11", "aws-smithy-json 0.61.1", "aws-smithy-protocol-test 0.63.0", "aws-smithy-query", "aws-smithy-runtime 1.7.7", "aws-smithy-runtime-api 1.7.3", - "aws-smithy-types 1.2.11", + "aws-smithy-types 1.2.12", "aws-smithy-xml 0.60.9", - "aws-types 1.3.3", + "aws-types 1.3.4", "futures-util", "http 0.2.12", "once_cell", @@ -714,15 +757,15 @@ version = "0.0.0-local" dependencies = [ "aws-config", "aws-credential-types 1.2.1", - "aws-runtime 1.5.3", - "aws-smithy-async 1.2.3", + "aws-runtime 1.5.4", + "aws-smithy-async 1.2.4", "aws-smithy-http 0.60.11", "aws-smithy-json 0.61.1", "aws-smithy-protocol-test 0.63.0", "aws-smithy-runtime 1.7.7", "aws-smithy-runtime-api 1.7.3", - "aws-smithy-types 1.2.11", - "aws-types 1.3.3", + "aws-smithy-types 1.2.12", + "aws-types 1.3.4", "bytes", "futures-util", "http 0.2.12", @@ -740,16 +783,16 @@ version = "0.0.0-local" dependencies = [ "aws-config", "aws-credential-types 1.2.1", - "aws-runtime 1.5.3", - "aws-smithy-async 1.2.3", + "aws-runtime 1.5.4", + "aws-smithy-async 1.2.4", "aws-smithy-eventstream 0.60.5", "aws-smithy-http 0.60.11", "aws-smithy-json 0.61.1", "aws-smithy-protocol-test 0.63.0", "aws-smithy-runtime 1.7.7", "aws-smithy-runtime-api 1.7.3", - "aws-smithy-types 1.2.11", - "aws-types 1.3.3", + "aws-smithy-types 1.2.12", + "aws-types 1.3.4", "bytes", "futures-util", "http 0.2.12", @@ -767,16 +810,16 @@ version = "0.0.0-local" dependencies = [ "aws-config", "aws-credential-types 1.2.1", - "aws-runtime 1.5.3", - "aws-sigv4 1.2.6", - "aws-smithy-async 1.2.3", + "aws-runtime 1.5.4", + "aws-sigv4 1.2.7", + "aws-smithy-async 1.2.4", "aws-smithy-http 0.60.11", "aws-smithy-json 0.61.1", "aws-smithy-protocol-test 0.63.0", "aws-smithy-runtime 1.7.7", "aws-smithy-runtime-api 1.7.3", - "aws-smithy-types 1.2.11", - "aws-types 1.3.3", + "aws-smithy-types 1.2.12", + "aws-types 1.3.4", "bytes", "futures-util", "http 0.2.12", @@ -796,15 +839,15 @@ version = "0.0.0-local" dependencies = [ "aws-config", "aws-credential-types 1.2.1", - "aws-runtime 1.5.3", - "aws-smithy-async 1.2.3", + "aws-runtime 1.5.4", + "aws-smithy-async 1.2.4", "aws-smithy-http 0.60.11", "aws-smithy-json 0.61.1", "aws-smithy-protocol-test 0.63.0", "aws-smithy-runtime 1.7.7", "aws-smithy-runtime-api 1.7.3", - "aws-smithy-types 1.2.11", - "aws-types 1.3.3", + "aws-smithy-types 1.2.12", + "aws-types 1.3.4", "bytes", "futures-util", "http 0.2.12", @@ -822,15 +865,15 @@ version = "0.0.0-local" dependencies = [ "aws-config", "aws-credential-types 1.2.1", - "aws-runtime 1.5.3", - "aws-smithy-async 1.2.3", + "aws-runtime 1.5.4", + "aws-smithy-async 1.2.4", "aws-smithy-http 0.60.11", "aws-smithy-json 0.61.1", "aws-smithy-runtime 1.7.7", "aws-smithy-runtime-api 1.7.3", - "aws-smithy-types 1.2.11", + "aws-smithy-types 1.2.12", "aws-smithy-xml 0.60.9", - "aws-types 1.3.3", + "aws-types 1.3.4", "http 0.2.12", "once_cell", "pretty_assertions", @@ -847,10 +890,10 @@ dependencies = [ "async-std", "aws-config", "aws-credential-types 1.2.1", - "aws-runtime 1.5.3", - "aws-sigv4 1.2.6", - "aws-smithy-async 1.2.3", - "aws-smithy-checksums 0.60.13", + "aws-runtime 1.5.4", + "aws-sigv4 1.2.7", + "aws-smithy-async 1.2.4", + "aws-smithy-checksums 0.62.0", "aws-smithy-eventstream 0.60.5", "aws-smithy-experimental", "aws-smithy-http 0.60.11", @@ -858,9 +901,9 @@ dependencies = [ "aws-smithy-protocol-test 0.63.0", "aws-smithy-runtime 1.7.7", "aws-smithy-runtime-api 1.7.3", - "aws-smithy-types 1.2.11", + "aws-smithy-types 1.2.12", "aws-smithy-xml 0.60.9", - "aws-types 1.3.3", + "aws-types 1.3.4", "bytes", "bytes-utils", "fastrand 2.0.2", @@ -906,9 +949,9 @@ dependencies = [ "aws-smithy-json 0.61.1 (registry+https://github.com/rust-lang/crates.io-index)", "aws-smithy-runtime 1.7.6", "aws-smithy-runtime-api 1.7.3 (registry+https://github.com/rust-lang/crates.io-index)", - "aws-smithy-types 1.2.11 (registry+https://github.com/rust-lang/crates.io-index)", + "aws-smithy-types 1.2.11", "aws-smithy-xml 0.60.9 (registry+https://github.com/rust-lang/crates.io-index)", - "aws-types 1.3.3 (registry+https://github.com/rust-lang/crates.io-index)", + "aws-types 1.3.3", "bytes", "fastrand 2.0.2", "hex", @@ -930,16 +973,16 @@ version = "0.0.0-local" dependencies = [ "aws-config", "aws-credential-types 1.2.1", - "aws-runtime 1.5.3", - "aws-smithy-async 1.2.3", + "aws-runtime 1.5.4", + "aws-smithy-async 1.2.4", "aws-smithy-http 0.60.11", "aws-smithy-json 0.61.1", "aws-smithy-protocol-test 0.63.0", "aws-smithy-runtime 1.7.7", "aws-smithy-runtime-api 1.7.3", - "aws-smithy-types 1.2.11", + "aws-smithy-types 1.2.12", "aws-smithy-xml 0.60.9", - "aws-types 1.3.3", + "aws-types 1.3.4", "fastrand 2.0.2", "futures-util", "http 0.2.12", @@ -959,14 +1002,14 @@ name = "aws-sdk-sso" version = "0.0.0-local" dependencies = [ "aws-credential-types 1.2.1", - "aws-runtime 1.5.3", - "aws-smithy-async 1.2.3", + "aws-runtime 1.5.4", + "aws-smithy-async 1.2.4", "aws-smithy-http 0.60.11", "aws-smithy-json 0.61.1", "aws-smithy-runtime 1.7.7", "aws-smithy-runtime-api 1.7.3", - "aws-smithy-types 1.2.11", - "aws-types 1.3.3", + "aws-smithy-types 1.2.12", + "aws-types 1.3.4", "bytes", "http 0.2.12", "once_cell", @@ -980,14 +1023,14 @@ name = "aws-sdk-ssooidc" version = "0.0.0-local" dependencies = [ "aws-credential-types 1.2.1", - "aws-runtime 1.5.3", - "aws-smithy-async 1.2.3", + "aws-runtime 1.5.4", + "aws-smithy-async 1.2.4", "aws-smithy-http 0.60.11", "aws-smithy-json 0.61.1", "aws-smithy-runtime 1.7.7", "aws-smithy-runtime-api 1.7.3", - "aws-smithy-types 1.2.11", - "aws-types 1.3.3", + "aws-smithy-types 1.2.12", + "aws-types 1.3.4", "bytes", "http 0.2.12", "once_cell", @@ -1001,17 +1044,17 @@ name = "aws-sdk-sts" version = "0.0.0-local" dependencies = [ "aws-credential-types 1.2.1", - "aws-runtime 1.5.3", - "aws-smithy-async 1.2.3", + "aws-runtime 1.5.4", + "aws-smithy-async 1.2.4", "aws-smithy-http 0.60.11", "aws-smithy-json 0.61.1", "aws-smithy-protocol-test 0.63.0", "aws-smithy-query", "aws-smithy-runtime 1.7.7", "aws-smithy-runtime-api 1.7.3", - "aws-smithy-types 1.2.11", + "aws-smithy-types 1.2.12", "aws-smithy-xml 0.60.9", - "aws-types 1.3.3", + "aws-types 1.3.4", "futures-util", "http 0.2.12", "once_cell", @@ -1028,15 +1071,15 @@ version = "0.0.0-local" dependencies = [ "aws-config", "aws-credential-types 1.2.1", - "aws-runtime 1.5.3", - "aws-smithy-async 1.2.3", + "aws-runtime 1.5.4", + "aws-smithy-async 1.2.4", "aws-smithy-http 0.60.11", "aws-smithy-json 0.61.1", "aws-smithy-protocol-test 0.63.0", "aws-smithy-runtime 1.7.7", "aws-smithy-runtime-api 1.7.3", - "aws-smithy-types 1.2.11", - "aws-types 1.3.3", + "aws-smithy-types 1.2.12", + "aws-types 1.3.4", "bytes", "fastrand 2.0.2", "futures-util", @@ -1055,14 +1098,14 @@ version = "0.0.0-local" dependencies = [ "aws-config", "aws-credential-types 1.2.1", - "aws-runtime 1.5.3", - "aws-smithy-async 1.2.3", + "aws-runtime 1.5.4", + "aws-smithy-async 1.2.4", "aws-smithy-http 0.60.11", "aws-smithy-json 0.61.1", "aws-smithy-runtime 1.7.7", "aws-smithy-runtime-api 1.7.3", - "aws-smithy-types 1.2.11", - "aws-types 1.3.3", + "aws-smithy-types 1.2.12", + "aws-types 1.3.4", "bytes", "fastrand 2.0.2", "http 0.2.12", @@ -1079,17 +1122,17 @@ dependencies = [ "async-stream", "aws-config", "aws-credential-types 1.2.1", - "aws-runtime 1.5.3", - "aws-sigv4 1.2.6", - "aws-smithy-async 1.2.3", + "aws-runtime 1.5.4", + "aws-sigv4 1.2.7", + "aws-smithy-async 1.2.4", "aws-smithy-eventstream 0.60.5", "aws-smithy-http 0.60.11", "aws-smithy-json 0.61.1", "aws-smithy-protocol-test 0.63.0", "aws-smithy-runtime 1.7.7", "aws-smithy-runtime-api 1.7.3", - "aws-smithy-types 1.2.11", - "aws-types 1.3.3", + "aws-smithy-types 1.2.12", + "aws-types 1.3.4", "bytes", "futures-core", "futures-util", @@ -1111,32 +1154,25 @@ version = "0.60.3" [[package]] name = "aws-sigv4" version = "1.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d3820e0c08d0737872ff3c7c1f21ebbb6693d832312d6152bf18ef50a5471c2" dependencies = [ - "aws-credential-types 1.2.1", - "aws-smithy-eventstream 0.60.5", - "aws-smithy-http 0.60.11", - "aws-smithy-runtime-api 1.7.3", + "aws-credential-types 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)", + "aws-smithy-eventstream 0.60.5 (registry+https://github.com/rust-lang/crates.io-index)", + "aws-smithy-http 0.60.11 (registry+https://github.com/rust-lang/crates.io-index)", + "aws-smithy-runtime-api 1.7.3 (registry+https://github.com/rust-lang/crates.io-index)", "aws-smithy-types 1.2.11", "bytes", - "criterion", "crypto-bigint 0.5.5", "form_urlencoded", "hex", - "hex-literal", "hmac", "http 0.2.12", "http 1.2.0", - "httparse", - "libfuzzer-sys", "once_cell", "p256", "percent-encoding", - "pretty_assertions", - "proptest", "ring", - "serde", - "serde_derive", - "serde_json", "sha2", "subtle", "time", @@ -1146,26 +1182,33 @@ dependencies = [ [[package]] name = "aws-sigv4" -version = "1.2.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d3820e0c08d0737872ff3c7c1f21ebbb6693d832312d6152bf18ef50a5471c2" +version = "1.2.7" dependencies = [ - "aws-credential-types 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)", - "aws-smithy-eventstream 0.60.5 (registry+https://github.com/rust-lang/crates.io-index)", - "aws-smithy-http 0.60.11 (registry+https://github.com/rust-lang/crates.io-index)", - "aws-smithy-runtime-api 1.7.3 (registry+https://github.com/rust-lang/crates.io-index)", - "aws-smithy-types 1.2.11 (registry+https://github.com/rust-lang/crates.io-index)", + "aws-credential-types 1.2.1", + "aws-smithy-eventstream 0.60.5", + "aws-smithy-http 0.60.11", + "aws-smithy-runtime-api 1.7.3", + "aws-smithy-types 1.2.12", "bytes", + "criterion", "crypto-bigint 0.5.5", "form_urlencoded", "hex", + "hex-literal", "hmac", "http 0.2.12", "http 1.2.0", + "httparse", + "libfuzzer-sys", "once_cell", "p256", "percent-encoding", + "pretty_assertions", + "proptest", "ring", + "serde", + "serde_derive", + "serde_json", "sha2", "subtle", "time", @@ -1176,30 +1219,30 @@ dependencies = [ [[package]] name = "aws-smithy-async" version = "1.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "427cb637d15d63d6f9aae26358e1c9a9c09d5aa490d64b09354c8217cfef0f28" dependencies = [ "futures-util", "pin-project-lite", - "pin-utils", "tokio", - "tokio-test", ] [[package]] name = "aws-smithy-async" -version = "1.2.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "427cb637d15d63d6f9aae26358e1c9a9c09d5aa490d64b09354c8217cfef0f28" +version = "1.2.4" dependencies = [ "futures-util", "pin-project-lite", + "pin-utils", "tokio", + "tokio-test", ] [[package]] name = "aws-smithy-cbor" version = "0.60.8" dependencies = [ - "aws-smithy-types 1.2.11", + "aws-smithy-types 1.2.12", "criterion", "minicbor", ] @@ -1207,11 +1250,12 @@ dependencies = [ [[package]] name = "aws-smithy-checksums" version = "0.60.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba1a71073fca26775c8b5189175ea8863afb1c9ea2cceb02a5de5ad9dfbaa795" dependencies = [ - "aws-smithy-http 0.60.11", + "aws-smithy-http 0.60.11 (registry+https://github.com/rust-lang/crates.io-index)", "aws-smithy-types 1.2.11", "bytes", - "bytes-utils", "crc32c", "crc32fast", "hex", @@ -1219,33 +1263,33 @@ dependencies = [ "http-body 0.4.6", "md-5", "pin-project-lite", - "pretty_assertions", "sha1", "sha2", - "tokio", "tracing", - "tracing-test", ] [[package]] name = "aws-smithy-checksums" -version = "0.60.13" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ba1a71073fca26775c8b5189175ea8863afb1c9ea2cceb02a5de5ad9dfbaa795" +version = "0.62.0" dependencies = [ - "aws-smithy-http 0.60.11 (registry+https://github.com/rust-lang/crates.io-index)", - "aws-smithy-types 1.2.11 (registry+https://github.com/rust-lang/crates.io-index)", + "aws-smithy-http 0.60.11", + "aws-smithy-types 1.2.12", "bytes", + "bytes-utils", "crc32c", "crc32fast", + "crc64fast-nvme", "hex", "http 0.2.12", "http-body 0.4.6", "md-5", "pin-project-lite", + "pretty_assertions", "sha1", "sha2", + "tokio", "tracing", + "tracing-test", ] [[package]] @@ -1257,7 +1301,7 @@ name = "aws-smithy-compression" version = "0.0.2" dependencies = [ "aws-smithy-runtime-api 1.7.3", - "aws-smithy-types 1.2.11", + "aws-smithy-types 1.2.12", "bytes", "bytes-utils", "flate2", @@ -1278,7 +1322,7 @@ name = "aws-smithy-eventstream" version = "0.60.5" dependencies = [ "arbitrary", - "aws-smithy-types 1.2.11", + "aws-smithy-types 1.2.12", "bytes", "bytes-utils", "crc32fast", @@ -1291,7 +1335,7 @@ version = "0.60.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cef7d0a272725f87e51ba2bf89f8c21e4df61b9e49ae1ac367a6d69916ef7c90" dependencies = [ - "aws-smithy-types 1.2.11 (registry+https://github.com/rust-lang/crates.io-index)", + "aws-smithy-types 1.2.11", "bytes", "crc32fast", ] @@ -1303,7 +1347,7 @@ dependencies = [ "aws-smithy-async 1.2.3", "aws-smithy-runtime 1.7.7", "aws-smithy-runtime-api 1.7.3", - "aws-smithy-types 1.2.11", + "aws-smithy-types 1.2.12", "h2 0.4.7", "http 1.2.0", "hyper 1.5.2", @@ -1324,7 +1368,7 @@ dependencies = [ "async-stream", "aws-smithy-eventstream 0.60.5", "aws-smithy-runtime-api 1.7.3", - "aws-smithy-types 1.2.11", + "aws-smithy-types 1.2.12", "bytes", "bytes-utils", "futures-core", @@ -1349,7 +1393,7 @@ checksum = "5c8bc3e8fdc6b8d07d976e301c02fe553f72a39b7a9fea820e023268467d7ab6" dependencies = [ "aws-smithy-eventstream 0.60.5 (registry+https://github.com/rust-lang/crates.io-index)", "aws-smithy-runtime-api 1.7.3 (registry+https://github.com/rust-lang/crates.io-index)", - "aws-smithy-types 1.2.11 (registry+https://github.com/rust-lang/crates.io-index)", + "aws-smithy-types 1.2.11", "bytes", "bytes-utils", "futures-core", @@ -1374,7 +1418,7 @@ version = "0.60.3" name = "aws-smithy-json" version = "0.61.1" dependencies = [ - "aws-smithy-types 1.2.11", + "aws-smithy-types 1.2.12", "proptest", "serde_json", ] @@ -1385,7 +1429,7 @@ version = "0.61.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ee4e69cc50921eb913c6b662f8d909131bb3e6ad6cb6090d3a39b66fc5c52095" dependencies = [ - "aws-smithy-types 1.2.11 (registry+https://github.com/rust-lang/crates.io-index)", + "aws-smithy-types 1.2.11", ] [[package]] @@ -1394,7 +1438,7 @@ version = "0.2.1" dependencies = [ "aws-sdk-s3 1.68.0", "aws-smithy-runtime-api 1.7.3", - "aws-smithy-types 1.2.11", + "aws-smithy-types 1.2.12", "tokio", ] @@ -1438,7 +1482,7 @@ dependencies = [ name = "aws-smithy-query" version = "0.60.7" dependencies = [ - "aws-smithy-types 1.2.11", + "aws-smithy-types 1.2.12", "urlencoding", ] @@ -1513,8 +1557,8 @@ dependencies = [ name = "aws-smithy-runtime-api" version = "1.7.3" dependencies = [ - "aws-smithy-async 1.2.3", - "aws-smithy-types 1.2.11", + "aws-smithy-async 1.2.4", + "aws-smithy-types 1.2.12", "bytes", "http 0.2.12", "http 1.2.0", @@ -1531,8 +1575,8 @@ version = "1.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "92165296a47a812b267b4f41032ff8069ab7ff783696d217f0994a0d7ab585cd" dependencies = [ - "aws-smithy-async 1.2.3 (registry+https://github.com/rust-lang/crates.io-index)", - "aws-smithy-types 1.2.11 (registry+https://github.com/rust-lang/crates.io-index)", + "aws-smithy-async 1.2.3", + "aws-smithy-types 1.2.11", "bytes", "http 0.2.12", "http 1.2.0", @@ -1545,60 +1589,60 @@ dependencies = [ [[package]] name = "aws-smithy-types" version = "1.2.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38ddc9bd6c28aeb303477170ddd183760a956a03e083b3902a990238a7e3792d" dependencies = [ - "base64 0.13.1", "base64-simd", "bytes", "bytes-utils", - "ciborium", - "criterion", "futures-core", "http 0.2.12", "http 1.2.0", "http-body 0.4.6", "http-body 1.0.1", "http-body-util", - "hyper 0.14.32", "itoa", - "lazy_static", "num-integer", "pin-project-lite", "pin-utils", - "proptest", - "rand", "ryu", "serde", - "serde_json", - "tempfile", "time", "tokio", - "tokio-stream", "tokio-util", ] [[package]] name = "aws-smithy-types" -version = "1.2.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "38ddc9bd6c28aeb303477170ddd183760a956a03e083b3902a990238a7e3792d" +version = "1.2.12" dependencies = [ + "base64 0.13.1", "base64-simd", "bytes", "bytes-utils", + "ciborium", + "criterion", "futures-core", "http 0.2.12", "http 1.2.0", "http-body 0.4.6", "http-body 1.0.1", "http-body-util", + "hyper 0.14.32", "itoa", + "lazy_static", "num-integer", "pin-project-lite", "pin-utils", + "proptest", + "rand", "ryu", "serde", + "serde_json", + "tempfile", "time", "tokio", + "tokio-stream", "tokio-util", ] @@ -1606,8 +1650,8 @@ dependencies = [ name = "aws-smithy-types-convert" version = "0.60.8" dependencies = [ - "aws-smithy-async 1.2.3", - "aws-smithy-types 1.2.11", + "aws-smithy-async 1.2.4", + "aws-smithy-types 1.2.12", "chrono", "futures-core", "time", @@ -1619,7 +1663,7 @@ version = "0.1.3" dependencies = [ "aws-smithy-http 0.60.11", "aws-smithy-runtime-api 1.7.3", - "aws-smithy-types 1.2.11", + "aws-smithy-types 1.2.12", "bytes", "http 1.2.0", "tracing", @@ -1648,33 +1692,34 @@ dependencies = [ [[package]] name = "aws-types" version = "1.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5221b91b3e441e6675310829fd8984801b772cb1546ef6c0e54dec9f1ac13fef" dependencies = [ - "aws-credential-types 1.2.1", + "aws-credential-types 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)", "aws-smithy-async 1.2.3", "aws-smithy-runtime 1.7.7", "aws-smithy-runtime-api 1.7.3", "aws-smithy-types 1.2.11", - "http 0.2.12", - "hyper-rustls 0.24.2", "rustc_version", - "tempfile", - "tokio", "tracing", - "tracing-test", ] [[package]] name = "aws-types" -version = "1.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5221b91b3e441e6675310829fd8984801b772cb1546ef6c0e54dec9f1ac13fef" +version = "1.3.4" dependencies = [ - "aws-credential-types 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)", - "aws-smithy-async 1.2.3 (registry+https://github.com/rust-lang/crates.io-index)", - "aws-smithy-runtime-api 1.7.3 (registry+https://github.com/rust-lang/crates.io-index)", - "aws-smithy-types 1.2.11 (registry+https://github.com/rust-lang/crates.io-index)", + "aws-credential-types 1.2.1", + "aws-smithy-async 1.2.4", + "aws-smithy-runtime 1.7.7", + "aws-smithy-runtime-api 1.7.3", + "aws-smithy-types 1.2.12", + "http 0.2.12", + "hyper-rustls 0.24.2", "rustc_version", + "tempfile", + "tokio", "tracing", + "tracing-test", ] [[package]] @@ -1841,6 +1886,25 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" +[[package]] +name = "cbindgen" +version = "0.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3fce8dd7fcfcbf3a0a87d8f515194b49d6135acab73e18bd380d1d93bb1a15eb" +dependencies = [ + "clap", + "heck", + "indexmap", + "log", + "proc-macro2", + "quote", + "serde", + "serde_json", + "syn 2.0.94", + "tempfile", + "toml", +] + [[package]] name = "cbor-diag" version = "0.1.12" @@ -1948,8 +2012,10 @@ version = "4.5.26" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "96b01801b5fc6a0a232407abc821660c9c6d25a1cafc0d4f85f29fb8d9afc121" dependencies = [ + "anstream", "anstyle", "clap_lex", + "strsim", ] [[package]] @@ -1967,6 +2033,12 @@ dependencies = [ "cc", ] +[[package]] +name = "colorchoice" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b63caa9aa9397e2d9480a9b13673856c78d8ac123288526c37d7839f2a86990" + [[package]] name = "concurrent-queue" version = "2.5.0" @@ -2026,6 +2098,21 @@ dependencies = [ "libc", ] +[[package]] +name = "crc" +version = "3.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69e6e4d7b33a94f0991c26729976b10ebde1d34c3ee82408fb536164fa10d636" +dependencies = [ + "crc-catalog", +] + +[[package]] +name = "crc-catalog" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19d374276b40fb8bbdee95aef7c7fa6b5316ec764510eb64b8dd0e2ed0d7e7f5" + [[package]] name = "crc32c" version = "0.6.8" @@ -2044,6 +2131,16 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "crc64fast-nvme" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d5e2ee08013e3f228d6d2394116c4549a6df77708442c62d887d83f68ef2ee37" +dependencies = [ + "cbindgen", + "crc", +] + [[package]] name = "criterion" version = "0.5.1" @@ -2603,6 +2700,12 @@ dependencies = [ "num-traits", ] +[[package]] +name = "heck" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" + [[package]] name = "hermit-abi" version = "0.3.9" @@ -2998,6 +3101,12 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "is_terminal_polyfill" +version = "1.70.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7943c866cc5cd64cbc25b2e01621d07fa8eb2a1a23160ee81ce38704e97b8ecf" + [[package]] name = "itertools" version = "0.10.5" @@ -4000,6 +4109,15 @@ dependencies = [ "serde", ] +[[package]] +name = "serde_spanned" +version = "0.6.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87607cb1398ed59d48732e575a4c28a7a8ebf2454b964fe3f224f2afc07909e1" +dependencies = [ + "serde", +] + [[package]] name = "sha1" version = "0.10.6" @@ -4130,6 +4248,12 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3" +[[package]] +name = "strsim" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" + [[package]] name = "subtle" version = "2.6.1" @@ -4364,6 +4488,40 @@ dependencies = [ "tokio", ] +[[package]] +name = "toml" +version = "0.8.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1ed1f98e3fdc28d6d910e6737ae6ab1a93bf1985935a1193e68f93eeb68d24e" +dependencies = [ + "serde", + "serde_spanned", + "toml_datetime", + "toml_edit", +] + +[[package]] +name = "toml_datetime" +version = "0.6.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0dd7358ecb8fc2f8d014bf86f6f638ce72ba252a2c3a2572f2a795f1d23efb41" +dependencies = [ + "serde", +] + +[[package]] +name = "toml_edit" +version = "0.22.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ae48d6208a266e853d946088ed816055e556cc6028c5e8e2b84d9fa5dd7c7f5" +dependencies = [ + "indexmap", + "serde", + "serde_spanned", + "toml_datetime", + "winnow", +] + [[package]] name = "tower" version = "0.4.13" @@ -4560,6 +4718,12 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" +[[package]] +name = "utf8parse" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" + [[package]] name = "uuid" version = "1.12.0" @@ -4917,6 +5081,15 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" +[[package]] +name = "winnow" +version = "0.6.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39281189af81c07ec09db316b302a3e67bf9bd7cbf6c820b50e35fee9c2fa980" +dependencies = [ + "memchr", +] + [[package]] name = "wit-bindgen" version = "0.19.2" diff --git a/aws/sdk/aws-models-extra/s3-tests.smithy b/aws/sdk/aws-models-extra/s3-tests.smithy index 59ced8bc1..bee98811f 100644 --- a/aws/sdk/aws-models-extra/s3-tests.smithy +++ b/aws/sdk/aws-models-extra/s3-tests.smithy @@ -111,9 +111,7 @@ apply PutBucketLifecycleConfiguration @httpRequestTests([ protocol: "aws.protocols#restXml", uri: "/", headers: { - // we can assert this, but when this test is promoted, it can't assert - // on the exact contents - "content-md5": "JP8DTuCSH6yDC8wNGg4+mA==", + "x-amz-checksum-crc32": "11+f3g==", }, bodyMediaType: "application/xml", body: """ diff --git a/aws/sdk/integration-tests/s3/tests/endpoints.rs b/aws/sdk/integration-tests/s3/tests/endpoints.rs index 6b4f88c8f..11b370ecc 100644 --- a/aws/sdk/integration-tests/s3/tests/endpoints.rs +++ b/aws/sdk/integration-tests/s3/tests/endpoints.rs @@ -80,7 +80,7 @@ async fn multi_region_access_points() { let auth_header = captured_request.headers().get("AUTHORIZATION").unwrap(); // Verifies that the sigv4a signing algorithm was used, that the signing scope doesn't include a region, and that the x-amz-region-set header was signed. let expected_start = - "AWS4-ECDSA-P256-SHA256 Credential=ANOTREAL/20090213/s3/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date;x-amz-region-set;x-amz-user-agent, Signature="; + "AWS4-ECDSA-P256-SHA256 Credential=ANOTREAL/20090213/s3/aws4_request, SignedHeaders=host;x-amz-checksum-mode;x-amz-content-sha256;x-amz-date;x-amz-region-set;x-amz-user-agent, Signature="; assert!( auth_header.starts_with(expected_start), diff --git a/aws/sdk/integration-tests/s3/tests/express.rs b/aws/sdk/integration-tests/s3/tests/express.rs index 8621b6c10..6c9fd1613 100644 --- a/aws/sdk/integration-tests/s3/tests/express.rs +++ b/aws/sdk/integration-tests/s3/tests/express.rs @@ -183,7 +183,11 @@ async fn presigning() { ][..], &query_params ); - assert_eq!(presigned.headers().count(), 0); + // Presigned request has one header and that is the x-amz-checksum-mode + // header with default value ENABLED + assert_eq!(presigned.headers().count(), 1); + let headers = presigned.headers().collect::>(); + assert_eq!(headers.get(0).unwrap(), &("x-amz-checksum-mode", "ENABLED")); } fn operation_request_with_checksum( diff --git a/aws/sdk/integration-tests/s3/tests/presigning.rs b/aws/sdk/integration-tests/s3/tests/presigning.rs index 768dd563b..f887b0643 100644 --- a/aws/sdk/integration-tests/s3/tests/presigning.rs +++ b/aws/sdk/integration-tests/s3/tests/presigning.rs @@ -100,7 +100,9 @@ async fn test_presigning() { ][..], &query_params ); - assert_eq!(presigned.headers().count(), 0); + assert_eq!(presigned.headers().count(), 1); + let headers = presigned.headers().collect::>(); + assert_eq!(headers.get("x-amz-checksum-mode").unwrap(), &"ENABLED"); } #[tokio::test] @@ -135,8 +137,8 @@ async fn test_presigning_with_payload_headers() { "X-Amz-Date=20090213T233131Z", "X-Amz-Expires=30", "X-Amz-Security-Token=notarealsessiontoken", - "X-Amz-Signature=be1d41dc392f7019750e4f5e577234fb9059dd20d15f6a99734196becce55e52", - "X-Amz-SignedHeaders=content-length%3Bcontent-type%3Bhost", + "X-Amz-Signature=9403eb5961c8af066f2473f88cb9248b39fd61f8eedc33af14ec9c2d26c22974", + "X-Amz-SignedHeaders=content-length%3Bcontent-type%3Bhost%3Bx-amz-checksum-crc32%3Bx-amz-sdk-checksum-algorithm", "x-id=PutObject" ][..], &query_params @@ -148,7 +150,9 @@ async fn test_presigning_with_payload_headers() { Some(&"application/x-test") ); assert_eq!(headers.get(CONTENT_LENGTH.as_str()), Some(&"12345")); - assert_eq!(headers.len(), 2); + assert_eq!(headers.get("x-amz-sdk-checksum-algorithm"), Some(&"CRC32")); + assert_eq!(headers.get("x-amz-checksum-crc32"), Some(&"AAAAAA==")); + assert_eq!(headers.len(), 4); } #[tokio::test] @@ -164,7 +168,7 @@ async fn test_presigned_upload_part() { }) .await; pretty_assertions::assert_eq!( - "https://bucket.s3.us-east-1.amazonaws.com/key?x-id=UploadPart&partNumber=0&uploadId=upload-id&X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=ANOTREAL%2F20090213%2Fus-east-1%2Fs3%2Faws4_request&X-Amz-Date=20090213T233131Z&X-Amz-Expires=30&X-Amz-SignedHeaders=content-length%3Bhost&X-Amz-Signature=a702867244f0bd1fb4d161e2a062520dcbefae3b9992d2e5366bcd61a60c6ddd&X-Amz-Security-Token=notarealsessiontoken", + "https://bucket.s3.us-east-1.amazonaws.com/key?x-id=UploadPart&partNumber=0&uploadId=upload-id&X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=ANOTREAL%2F20090213%2Fus-east-1%2Fs3%2Faws4_request&X-Amz-Date=20090213T233131Z&X-Amz-Expires=30&X-Amz-SignedHeaders=content-length%3Bhost%3Bx-amz-checksum-crc32%3Bx-amz-sdk-checksum-algorithm&X-Amz-Signature=0b5835e056c463d6c0963326966f6cf42c75b7a218057836274d38288e055d36&X-Amz-Security-Token=notarealsessiontoken", presigned.uri().to_string(), ); } diff --git a/aws/sdk/integration-tests/s3/tests/stalled-stream-protection.rs b/aws/sdk/integration-tests/s3/tests/stalled-stream-protection.rs index 283999d60..920c95c1d 100644 --- a/aws/sdk/integration-tests/s3/tests/stalled-stream-protection.rs +++ b/aws/sdk/integration-tests/s3/tests/stalled-stream-protection.rs @@ -96,6 +96,9 @@ async fn test_stalled_stream_protection_defaults_for_upload() { .credentials_provider(Credentials::for_tests()) .region(Region::new("us-east-1")) .endpoint_url(format!("http://{server_addr}")) + // The Body used here is odd and fails the body.size_hint().exact() check in the streaming branch of + // the `RequestChecksumInterceptor` + .request_checksum_calculation(aws_sdk_s3::config::RequestChecksumCalculation::WhenRequired) .build(); let client = Client::from_conf(conf); diff --git a/codegen-client/src/main/kotlin/software/amazon/smithy/rust/codegen/client/smithy/generators/OperationGenerator.kt b/codegen-client/src/main/kotlin/software/amazon/smithy/rust/codegen/client/smithy/generators/OperationGenerator.kt index c2200003d..bb4c69e29 100644 --- a/codegen-client/src/main/kotlin/software/amazon/smithy/rust/codegen/client/smithy/generators/OperationGenerator.kt +++ b/codegen-client/src/main/kotlin/software/amazon/smithy/rust/codegen/client/smithy/generators/OperationGenerator.kt @@ -93,7 +93,9 @@ open class OperationGenerator( *preludeScope, "Arc" to RuntimeType.Arc, "ConcreteInput" to symbolProvider.toSymbol(operationShape.inputShape(model)), - "Input" to RuntimeType.smithyRuntimeApiClient(runtimeConfig).resolve("client::interceptors::context::Input"), + "Input" to + RuntimeType.smithyRuntimeApiClient(runtimeConfig) + .resolve("client::interceptors::context::Input"), "Operation" to symbolProvider.toSymbol(operationShape), "OperationError" to errorType, "OperationOutput" to outputType, @@ -169,7 +171,9 @@ open class OperationGenerator( } """, *codegenScope, - "Error" to RuntimeType.smithyRuntimeApiClient(runtimeConfig).resolve("client::interceptors::context::Error"), + "Error" to + RuntimeType.smithyRuntimeApiClient(runtimeConfig) + .resolve("client::interceptors::context::Error"), "InterceptorContext" to RuntimeType.interceptorContext(runtimeConfig), "OrchestratorError" to RuntimeType.smithyRuntimeApiClient(runtimeConfig) diff --git a/rust-runtime/Cargo.lock b/rust-runtime/Cargo.lock index 434820db4..17b5cca1c 100644 --- a/rust-runtime/Cargo.lock +++ b/rust-runtime/Cargo.lock @@ -38,12 +38,56 @@ version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4b46cbb362ab8752921c97e041f5e366ee6297bd428a31275b9fcf1e380f7299" +[[package]] +name = "anstream" +version = "0.6.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8acc5369981196006228e28809f761875c0327210a891e941f4c683b3a99529b" +dependencies = [ + "anstyle", + "anstyle-parse", + "anstyle-query", + "anstyle-wincon", + "colorchoice", + "is_terminal_polyfill", + "utf8parse", +] + [[package]] name = "anstyle" version = "1.0.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "55cc3b69f167a1ef2e161439aa98aed94e6028e5f9a59be9a6ffb47aef1651f9" +[[package]] +name = "anstyle-parse" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b2d16507662817a6a20a9ea92df6652ee4f94f914589377d69f3b21bc5798a9" +dependencies = [ + "utf8parse", +] + +[[package]] +name = "anstyle-query" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "79947af37f4177cfead1110013d678905c37501914fba0efea834c3fe9a8d60c" +dependencies = [ + "windows-sys 0.59.0", +] + +[[package]] +name = "anstyle-wincon" +version = "3.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca3534e77181a9cc07539ad51f2141fe32f6c3ffd4df76db8ad92346b003ae4e" +dependencies = [ + "anstyle", + "once_cell", + "windows-sys 0.59.0", +] + [[package]] name = "approx" version = "0.5.1" @@ -132,9 +176,9 @@ version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "60e8f6b615cb5fc60a98132268508ad104310f0cfb25a1c22eee76efdf9154da" dependencies = [ - "aws-smithy-async 1.2.3 (registry+https://github.com/rust-lang/crates.io-index)", + "aws-smithy-async 1.2.3", "aws-smithy-runtime-api 1.7.3 (registry+https://github.com/rust-lang/crates.io-index)", - "aws-smithy-types 1.2.11 (registry+https://github.com/rust-lang/crates.io-index)", + "aws-smithy-types 1.2.11", "zeroize", ] @@ -193,7 +237,7 @@ dependencies = [ "aws-smithy-http 0.60.11 (registry+https://github.com/rust-lang/crates.io-index)", "aws-smithy-runtime 1.7.6", "aws-smithy-runtime-api 1.7.3 (registry+https://github.com/rust-lang/crates.io-index)", - "aws-smithy-types 1.2.11 (registry+https://github.com/rust-lang/crates.io-index)", + "aws-smithy-types 1.2.11", "aws-types", "bytes", "fastrand", @@ -222,7 +266,7 @@ dependencies = [ "aws-smithy-json 0.61.1 (registry+https://github.com/rust-lang/crates.io-index)", "aws-smithy-runtime 1.7.6", "aws-smithy-runtime-api 1.7.3 (registry+https://github.com/rust-lang/crates.io-index)", - "aws-smithy-types 1.2.11 (registry+https://github.com/rust-lang/crates.io-index)", + "aws-smithy-types 1.2.11", "aws-smithy-xml 0.60.9 (registry+https://github.com/rust-lang/crates.io-index)", "aws-types", "bytes", @@ -247,10 +291,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7d3820e0c08d0737872ff3c7c1f21ebbb6693d832312d6152bf18ef50a5471c2" dependencies = [ "aws-credential-types", - "aws-smithy-eventstream 0.60.5 (registry+https://github.com/rust-lang/crates.io-index)", - "aws-smithy-http 0.60.11 (registry+https://github.com/rust-lang/crates.io-index)", + "aws-smithy-eventstream 0.60.5", + "aws-smithy-http 0.60.11", "aws-smithy-runtime-api 1.7.3 (registry+https://github.com/rust-lang/crates.io-index)", - "aws-smithy-types 1.2.11 (registry+https://github.com/rust-lang/crates.io-index)", + "aws-smithy-types 1.2.11", "bytes", "crypto-bigint 0.5.5", "form_urlencoded", @@ -272,30 +316,30 @@ dependencies = [ [[package]] name = "aws-smithy-async" version = "1.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "427cb637d15d63d6f9aae26358e1c9a9c09d5aa490d64b09354c8217cfef0f28" dependencies = [ "futures-util", "pin-project-lite", - "pin-utils", "tokio", - "tokio-test", ] [[package]] name = "aws-smithy-async" -version = "1.2.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "427cb637d15d63d6f9aae26358e1c9a9c09d5aa490d64b09354c8217cfef0f28" +version = "1.2.4" dependencies = [ "futures-util", "pin-project-lite", + "pin-utils", "tokio", + "tokio-test", ] [[package]] name = "aws-smithy-cbor" version = "0.60.8" dependencies = [ - "aws-smithy-types 1.2.11", + "aws-smithy-types 1.2.12", "criterion", "minicbor", ] @@ -303,11 +347,12 @@ dependencies = [ [[package]] name = "aws-smithy-checksums" version = "0.60.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba1a71073fca26775c8b5189175ea8863afb1c9ea2cceb02a5de5ad9dfbaa795" dependencies = [ "aws-smithy-http 0.60.11", "aws-smithy-types 1.2.11", "bytes", - "bytes-utils", "crc32c", "crc32fast", "hex", @@ -315,33 +360,33 @@ dependencies = [ "http-body 0.4.6", "md-5", "pin-project-lite", - "pretty_assertions", "sha1", "sha2", - "tokio", "tracing", - "tracing-test", ] [[package]] name = "aws-smithy-checksums" -version = "0.60.13" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ba1a71073fca26775c8b5189175ea8863afb1c9ea2cceb02a5de5ad9dfbaa795" +version = "0.62.0" dependencies = [ - "aws-smithy-http 0.60.11 (registry+https://github.com/rust-lang/crates.io-index)", - "aws-smithy-types 1.2.11 (registry+https://github.com/rust-lang/crates.io-index)", + "aws-smithy-http 0.60.12", + "aws-smithy-types 1.2.12", "bytes", + "bytes-utils", "crc32c", "crc32fast", + "crc64fast-nvme", "hex", "http 0.2.12", "http-body 0.4.6", "md-5", "pin-project-lite", + "pretty_assertions", "sha1", "sha2", + "tokio", "tracing", + "tracing-test", ] [[package]] @@ -353,7 +398,7 @@ name = "aws-smithy-compression" version = "0.0.2" dependencies = [ "aws-smithy-runtime-api 1.7.3", - "aws-smithy-types 1.2.11", + "aws-smithy-types 1.2.12", "bytes", "bytes-utils", "flate2", @@ -372,24 +417,24 @@ dependencies = [ [[package]] name = "aws-smithy-eventstream" version = "0.60.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cef7d0a272725f87e51ba2bf89f8c21e4df61b9e49ae1ac367a6d69916ef7c90" dependencies = [ - "arbitrary", "aws-smithy-types 1.2.11", "bytes", - "bytes-utils", "crc32fast", - "derive_arbitrary", ] [[package]] name = "aws-smithy-eventstream" -version = "0.60.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cef7d0a272725f87e51ba2bf89f8c21e4df61b9e49ae1ac367a6d69916ef7c90" +version = "0.60.6" dependencies = [ - "aws-smithy-types 1.2.11 (registry+https://github.com/rust-lang/crates.io-index)", + "arbitrary", + "aws-smithy-types 1.2.12", "bytes", + "bytes-utils", "crc32fast", + "derive_arbitrary", ] [[package]] @@ -399,7 +444,7 @@ dependencies = [ "aws-smithy-async 1.2.3", "aws-smithy-runtime 1.7.7", "aws-smithy-runtime-api 1.7.3", - "aws-smithy-types 1.2.11", + "aws-smithy-types 1.2.12", "h2 0.4.7", "http 1.2.0", "hyper 1.5.2", @@ -416,45 +461,45 @@ dependencies = [ [[package]] name = "aws-smithy-http" version = "0.60.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c8bc3e8fdc6b8d07d976e301c02fe553f72a39b7a9fea820e023268467d7ab6" dependencies = [ - "async-stream", "aws-smithy-eventstream 0.60.5", - "aws-smithy-runtime-api 1.7.3", + "aws-smithy-runtime-api 1.7.3 (registry+https://github.com/rust-lang/crates.io-index)", "aws-smithy-types 1.2.11", "bytes", "bytes-utils", "futures-core", - "futures-util", "http 0.2.12", "http-body 0.4.6", - "hyper 0.14.32", "once_cell", "percent-encoding", "pin-project-lite", "pin-utils", - "proptest", - "tokio", "tracing", ] [[package]] name = "aws-smithy-http" -version = "0.60.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c8bc3e8fdc6b8d07d976e301c02fe553f72a39b7a9fea820e023268467d7ab6" +version = "0.60.12" dependencies = [ - "aws-smithy-eventstream 0.60.5 (registry+https://github.com/rust-lang/crates.io-index)", - "aws-smithy-runtime-api 1.7.3 (registry+https://github.com/rust-lang/crates.io-index)", - "aws-smithy-types 1.2.11 (registry+https://github.com/rust-lang/crates.io-index)", + "async-stream", + "aws-smithy-eventstream 0.60.6", + "aws-smithy-runtime-api 1.7.3", + "aws-smithy-types 1.2.12", "bytes", "bytes-utils", "futures-core", + "futures-util", "http 0.2.12", "http-body 0.4.6", + "hyper 0.14.32", "once_cell", "percent-encoding", "pin-project-lite", "pin-utils", + "proptest", + "tokio", "tracing", ] @@ -467,10 +512,10 @@ name = "aws-smithy-http-server" version = "0.63.3" dependencies = [ "aws-smithy-cbor", - "aws-smithy-http 0.60.11", - "aws-smithy-json 0.61.1", + "aws-smithy-http 0.60.12", + "aws-smithy-json 0.61.2", "aws-smithy-runtime-api 1.7.3", - "aws-smithy-types 1.2.11", + "aws-smithy-types 1.2.12", "aws-smithy-xml 0.60.9", "bytes", "futures-util", @@ -497,10 +542,10 @@ dependencies = [ name = "aws-smithy-http-server-python" version = "0.63.2" dependencies = [ - "aws-smithy-http 0.60.11", + "aws-smithy-http 0.60.12", "aws-smithy-http-server", - "aws-smithy-json 0.61.1", - "aws-smithy-types 1.2.11", + "aws-smithy-json 0.61.2", + "aws-smithy-types 1.2.12", "aws-smithy-xml 0.60.9", "bytes", "futures", @@ -539,19 +584,19 @@ version = "0.60.3" [[package]] name = "aws-smithy-json" version = "0.61.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee4e69cc50921eb913c6b662f8d909131bb3e6ad6cb6090d3a39b66fc5c52095" dependencies = [ "aws-smithy-types 1.2.11", - "proptest", - "serde_json", ] [[package]] name = "aws-smithy-json" -version = "0.61.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ee4e69cc50921eb913c6b662f8d909131bb3e6ad6cb6090d3a39b66fc5c52095" +version = "0.61.2" dependencies = [ - "aws-smithy-types 1.2.11 (registry+https://github.com/rust-lang/crates.io-index)", + "aws-smithy-types 1.2.12", + "proptest", + "serde_json", ] [[package]] @@ -560,7 +605,7 @@ version = "0.2.1" dependencies = [ "aws-sdk-s3", "aws-smithy-runtime-api 1.7.3", - "aws-smithy-types 1.2.11", + "aws-smithy-types 1.2.12", "tokio", ] @@ -604,7 +649,7 @@ dependencies = [ name = "aws-smithy-query" version = "0.60.7" dependencies = [ - "aws-smithy-types 1.2.11", + "aws-smithy-types 1.2.12", "urlencoding", ] @@ -679,8 +724,8 @@ dependencies = [ name = "aws-smithy-runtime-api" version = "1.7.3" dependencies = [ - "aws-smithy-async 1.2.3", - "aws-smithy-types 1.2.11", + "aws-smithy-async 1.2.4", + "aws-smithy-types 1.2.12", "bytes", "http 0.2.12", "http 1.2.0", @@ -697,8 +742,8 @@ version = "1.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "92165296a47a812b267b4f41032ff8069ab7ff783696d217f0994a0d7ab585cd" dependencies = [ - "aws-smithy-async 1.2.3 (registry+https://github.com/rust-lang/crates.io-index)", - "aws-smithy-types 1.2.11 (registry+https://github.com/rust-lang/crates.io-index)", + "aws-smithy-async 1.2.3", + "aws-smithy-types 1.2.11", "bytes", "http 0.2.12", "http 1.2.0", @@ -711,60 +756,60 @@ dependencies = [ [[package]] name = "aws-smithy-types" version = "1.2.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38ddc9bd6c28aeb303477170ddd183760a956a03e083b3902a990238a7e3792d" dependencies = [ - "base64 0.13.1", "base64-simd", "bytes", "bytes-utils", - "ciborium", - "criterion", "futures-core", "http 0.2.12", "http 1.2.0", "http-body 0.4.6", "http-body 1.0.1", "http-body-util", - "hyper 0.14.32", "itoa", - "lazy_static", "num-integer", "pin-project-lite", "pin-utils", - "proptest", - "rand", "ryu", "serde", - "serde_json", - "tempfile", "time", "tokio", - "tokio-stream", "tokio-util", ] [[package]] name = "aws-smithy-types" -version = "1.2.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "38ddc9bd6c28aeb303477170ddd183760a956a03e083b3902a990238a7e3792d" +version = "1.2.12" dependencies = [ + "base64 0.13.1", "base64-simd", "bytes", "bytes-utils", + "ciborium", + "criterion", "futures-core", "http 0.2.12", "http 1.2.0", "http-body 0.4.6", "http-body 1.0.1", "http-body-util", + "hyper 0.14.32", "itoa", + "lazy_static", "num-integer", "pin-project-lite", "pin-utils", + "proptest", + "rand", "ryu", "serde", + "serde_json", + "tempfile", "time", "tokio", + "tokio-stream", "tokio-util", ] @@ -772,8 +817,8 @@ dependencies = [ name = "aws-smithy-types-convert" version = "0.60.8" dependencies = [ - "aws-smithy-async 1.2.3", - "aws-smithy-types 1.2.11", + "aws-smithy-async 1.2.4", + "aws-smithy-types 1.2.12", "chrono", "futures-core", "time", @@ -783,9 +828,9 @@ dependencies = [ name = "aws-smithy-wasm" version = "0.1.3" dependencies = [ - "aws-smithy-http 0.60.11", + "aws-smithy-http 0.60.12", "aws-smithy-runtime-api 1.7.3", - "aws-smithy-types 1.2.11", + "aws-smithy-types 1.2.12", "bytes", "http 1.2.0", "tracing", @@ -818,9 +863,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5221b91b3e441e6675310829fd8984801b772cb1546ef6c0e54dec9f1ac13fef" dependencies = [ "aws-credential-types", - "aws-smithy-async 1.2.3 (registry+https://github.com/rust-lang/crates.io-index)", + "aws-smithy-async 1.2.3", "aws-smithy-runtime-api 1.7.3 (registry+https://github.com/rust-lang/crates.io-index)", - "aws-smithy-types 1.2.11 (registry+https://github.com/rust-lang/crates.io-index)", + "aws-smithy-types 1.2.11", "rustc_version", "tracing", ] @@ -1001,6 +1046,25 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" +[[package]] +name = "cbindgen" +version = "0.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3fce8dd7fcfcbf3a0a87d8f515194b49d6135acab73e18bd380d1d93bb1a15eb" +dependencies = [ + "clap 4.5.23", + "heck", + "indexmap 2.7.0", + "log", + "proc-macro2", + "quote", + "serde", + "serde_json", + "syn 2.0.94", + "tempfile", + "toml", +] + [[package]] name = "cbor-diag" version = "0.1.12" @@ -1103,7 +1167,7 @@ dependencies = [ "bitflags 1.3.2", "clap_lex 0.2.4", "indexmap 1.9.3", - "strsim", + "strsim 0.10.0", "termcolor", "textwrap", ] @@ -1123,8 +1187,10 @@ version = "4.5.26" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "96b01801b5fc6a0a232407abc821660c9c6d25a1cafc0d4f85f29fb8d9afc121" dependencies = [ + "anstream", "anstyle", "clap_lex 0.7.4", + "strsim 0.11.1", ] [[package]] @@ -1151,6 +1217,12 @@ dependencies = [ "cc", ] +[[package]] +name = "colorchoice" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b63caa9aa9397e2d9480a9b13673856c78d8ac123288526c37d7839f2a86990" + [[package]] name = "concurrent-queue" version = "2.5.0" @@ -1201,6 +1273,21 @@ dependencies = [ "libc", ] +[[package]] +name = "crc" +version = "3.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69e6e4d7b33a94f0991c26729976b10ebde1d34c3ee82408fb536164fa10d636" +dependencies = [ + "crc-catalog", +] + +[[package]] +name = "crc-catalog" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19d374276b40fb8bbdee95aef7c7fa6b5316ec764510eb64b8dd0e2ed0d7e7f5" + [[package]] name = "crc32c" version = "0.6.8" @@ -1219,6 +1306,16 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "crc64fast-nvme" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d5e2ee08013e3f228d6d2394116c4549a6df77708442c62d887d83f68ef2ee37" +dependencies = [ + "cbindgen", + "crc", +] + [[package]] name = "criterion" version = "0.5.1" @@ -1729,6 +1826,12 @@ dependencies = [ "foldhash", ] +[[package]] +name = "heck" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" + [[package]] name = "hermit-abi" version = "0.1.19" @@ -2131,7 +2234,7 @@ dependencies = [ "aws-smithy-json 0.61.1", "aws-smithy-runtime 1.7.7", "aws-smithy-runtime-api 1.7.3", - "aws-smithy-types 1.2.11", + "aws-smithy-types 1.2.12", "aws-smithy-xml 0.60.9", "bytes", "fastrand", @@ -2169,6 +2272,12 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "is_terminal_polyfill" +version = "1.70.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7943c866cc5cd64cbc25b2e01621d07fa8eb2a1a23160ee81ce38704e97b8ecf" + [[package]] name = "itertools" version = "0.10.5" @@ -2379,9 +2488,9 @@ checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" [[package]] name = "minicbor" -version = "0.24.2" +version = "0.24.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f8e213c36148d828083ae01948eed271d03f95f7e72571fa242d78184029af2" +checksum = "29be4f60e41fde478b36998b88821946aafac540e53591e76db53921a0cc225b" dependencies = [ "half", "minicbor-derive", @@ -3336,6 +3445,15 @@ dependencies = [ "serde", ] +[[package]] +name = "serde_spanned" +version = "0.6.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87607cb1398ed59d48732e575a4c28a7a8ebf2454b964fe3f224f2afc07909e1" +dependencies = [ + "serde", +] + [[package]] name = "serde_urlencoded" version = "0.7.1" @@ -3474,6 +3592,12 @@ version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" +[[package]] +name = "strsim" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" + [[package]] name = "subtle" version = "2.6.1" @@ -3743,6 +3867,40 @@ dependencies = [ "tokio", ] +[[package]] +name = "toml" +version = "0.8.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1ed1f98e3fdc28d6d910e6737ae6ab1a93bf1985935a1193e68f93eeb68d24e" +dependencies = [ + "serde", + "serde_spanned", + "toml_datetime", + "toml_edit", +] + +[[package]] +name = "toml_datetime" +version = "0.6.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0dd7358ecb8fc2f8d014bf86f6f638ce72ba252a2c3a2572f2a795f1d23efb41" +dependencies = [ + "serde", +] + +[[package]] +name = "toml_edit" +version = "0.22.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ae48d6208a266e853d946088ed816055e556cc6028c5e8e2b84d9fa5dd7c7f5" +dependencies = [ + "indexmap 2.7.0", + "serde", + "serde_spanned", + "toml_datetime", + "winnow", +] + [[package]] name = "tower" version = "0.4.13" @@ -3982,6 +4140,12 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" +[[package]] +name = "utf8parse" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" + [[package]] name = "uuid" version = "1.12.0" @@ -4246,6 +4410,15 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" +[[package]] +name = "winnow" +version = "0.6.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8d71a593cc5c42ad7876e2c1fda56f314f3754c084128833e64f1345ff8a03a" +dependencies = [ + "memchr", +] + [[package]] name = "wit-bindgen" version = "0.19.2" diff --git a/rust-runtime/aws-smithy-async/Cargo.toml b/rust-runtime/aws-smithy-async/Cargo.toml index d7c9037ef..cebeec289 100644 --- a/rust-runtime/aws-smithy-async/Cargo.toml +++ b/rust-runtime/aws-smithy-async/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "aws-smithy-async" -version = "1.2.3" +version = "1.2.4" authors = ["AWS Rust SDK Team ", "John DiSanti "] description = "Async runtime agnostic abstractions for smithy-rs." edition = "2021" diff --git a/rust-runtime/aws-smithy-checksums/Cargo.toml b/rust-runtime/aws-smithy-checksums/Cargo.toml index 088824699..6615d2fce 100644 --- a/rust-runtime/aws-smithy-checksums/Cargo.toml +++ b/rust-runtime/aws-smithy-checksums/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "aws-smithy-checksums" -version = "0.60.13" +version = "0.62.0" authors = [ "AWS Rust SDK Team ", "Zelda Hessler ", @@ -26,6 +26,7 @@ pin-project-lite = "0.2.9" sha1 = "0.10" sha2 = "0.10" tracing = "0.1" +crc64fast-nvme = "1.1.1" [dev-dependencies] bytes-utils = "0.1.2" diff --git a/rust-runtime/aws-smithy-checksums/src/http.rs b/rust-runtime/aws-smithy-checksums/src/http.rs index 015935683..7d84eaf3f 100644 --- a/rust-runtime/aws-smithy-checksums/src/http.rs +++ b/rust-runtime/aws-smithy-checksums/src/http.rs @@ -8,15 +8,17 @@ use aws_smithy_types::base64; use http::header::{HeaderMap, HeaderValue}; +use crate::Crc64Nvme; use crate::{ - Checksum, Crc32, Crc32c, Md5, Sha1, Sha256, CRC_32_C_NAME, CRC_32_NAME, SHA_1_NAME, - SHA_256_NAME, + Checksum, Crc32, Crc32c, Md5, Sha1, Sha256, CRC_32_C_NAME, CRC_32_NAME, CRC_64_NVME_NAME, + SHA_1_NAME, SHA_256_NAME, }; -pub static CRC_32_HEADER_NAME: &str = "x-amz-checksum-crc32"; -pub static CRC_32_C_HEADER_NAME: &str = "x-amz-checksum-crc32c"; -pub static SHA_1_HEADER_NAME: &str = "x-amz-checksum-sha1"; -pub static SHA_256_HEADER_NAME: &str = "x-amz-checksum-sha256"; +pub const CRC_32_HEADER_NAME: &str = "x-amz-checksum-crc32"; +pub const CRC_32_C_HEADER_NAME: &str = "x-amz-checksum-crc32c"; +pub const SHA_1_HEADER_NAME: &str = "x-amz-checksum-sha1"; +pub const SHA_256_HEADER_NAME: &str = "x-amz-checksum-sha256"; +pub const CRC_64_NVME_HEADER_NAME: &str = "x-amz-checksum-crc64nvme"; // Preserved for compatibility purposes. This should never be used by users, only within smithy-rs pub(crate) static MD5_HEADER_NAME: &str = "content-md5"; @@ -24,8 +26,13 @@ pub(crate) static MD5_HEADER_NAME: &str = "content-md5"; /// When a response has to be checksum-verified, we have to check possible headers until we find the /// header with the precalculated checksum. Because a service may send back multiple headers, we have /// to check them in order based on how fast each checksum is to calculate. -pub const CHECKSUM_ALGORITHMS_IN_PRIORITY_ORDER: [&str; 4] = - [CRC_32_C_NAME, CRC_32_NAME, SHA_1_NAME, SHA_256_NAME]; +pub const CHECKSUM_ALGORITHMS_IN_PRIORITY_ORDER: [&str; 5] = [ + CRC_64_NVME_NAME, + CRC_32_C_NAME, + CRC_32_NAME, + SHA_1_NAME, + SHA_256_NAME, +]; /// Checksum algorithms are use to validate the integrity of data. Structs that implement this trait /// can be used as checksum calculators. This trait requires Send + Sync because these checksums are @@ -81,6 +88,12 @@ impl HttpChecksum for Crc32c { } } +impl HttpChecksum for Crc64Nvme { + fn header_name(&self) -> &'static str { + CRC_64_NVME_HEADER_NAME + } +} + impl HttpChecksum for Sha1 { fn header_name(&self) -> &'static str { SHA_1_HEADER_NAME @@ -104,7 +117,9 @@ mod tests { use aws_smithy_types::base64; use bytes::Bytes; - use crate::{ChecksumAlgorithm, CRC_32_C_NAME, CRC_32_NAME, SHA_1_NAME, SHA_256_NAME}; + use crate::{ + ChecksumAlgorithm, CRC_32_C_NAME, CRC_32_NAME, CRC_64_NVME_NAME, SHA_1_NAME, SHA_256_NAME, + }; use super::HttpChecksum; @@ -156,6 +171,30 @@ mod tests { assert_eq!(expected_value, actual_value) } + #[test] + fn test_trailer_length_of_crc64nvme_checksum_body() { + let checksum = CRC_64_NVME_NAME + .parse::() + .unwrap() + .into_impl(); + let expected_size = 37; + let actual_size = HttpChecksum::size(&*checksum); + assert_eq!(expected_size, actual_size) + } + + #[test] + fn test_trailer_value_of_crc64nvme_checksum_body() { + let checksum = CRC_64_NVME_NAME + .parse::() + .unwrap() + .into_impl(); + // The CRC64NVME of an empty string is all zeroes + let expected_value = Bytes::from_static(b"\0\0\0\0\0\0\0\0"); + let expected_value = base64::encode(&expected_value); + let actual_value = checksum.header_value(); + assert_eq!(expected_value, actual_value) + } + #[test] fn test_trailer_length_of_sha1_checksum_body() { let checksum = SHA_1_NAME.parse::().unwrap().into_impl(); diff --git a/rust-runtime/aws-smithy-checksums/src/lib.rs b/rust-runtime/aws-smithy-checksums/src/lib.rs index e969918dd..c024c2b54 100644 --- a/rust-runtime/aws-smithy-checksums/src/lib.rs +++ b/rust-runtime/aws-smithy-checksums/src/lib.rs @@ -17,8 +17,9 @@ //! Checksum calculation and verification callbacks. use crate::error::UnknownChecksumAlgorithmError; + use bytes::Bytes; -use std::str::FromStr; +use std::{fmt::Debug, str::FromStr}; pub mod body; pub mod error; @@ -27,18 +28,23 @@ pub mod http; // Valid checksum algorithm names pub const CRC_32_NAME: &str = "crc32"; pub const CRC_32_C_NAME: &str = "crc32c"; +pub const CRC_64_NVME_NAME: &str = "crc64nvme"; pub const SHA_1_NAME: &str = "sha1"; pub const SHA_256_NAME: &str = "sha256"; pub const MD5_NAME: &str = "md5"; /// We only support checksum calculation and validation for these checksum algorithms. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +#[non_exhaustive] pub enum ChecksumAlgorithm { + #[default] Crc32, Crc32c, + #[deprecated] Md5, Sha1, Sha256, + Crc64Nvme, } impl FromStr for ChecksumAlgorithm { @@ -47,9 +53,9 @@ impl FromStr for ChecksumAlgorithm { /// Create a new `ChecksumAlgorithm` from an algorithm name. Valid algorithm names are: /// - "crc32" /// - "crc32c" + /// - "crc64nvme" /// - "sha1" /// - "sha256" - /// - "md5" /// /// Passing an invalid name will return an error. fn from_str(checksum_algorithm: &str) -> Result { @@ -62,7 +68,10 @@ impl FromStr for ChecksumAlgorithm { } else if checksum_algorithm.eq_ignore_ascii_case(SHA_256_NAME) { Ok(Self::Sha256) } else if checksum_algorithm.eq_ignore_ascii_case(MD5_NAME) { - Ok(Self::Md5) + // MD5 is now an alias for the default Crc32 since it is deprecated + Ok(Self::Crc32) + } else if checksum_algorithm.eq_ignore_ascii_case(CRC_64_NVME_NAME) { + Ok(Self::Crc64Nvme) } else { Err(UnknownChecksumAlgorithmError::new(checksum_algorithm)) } @@ -75,7 +84,9 @@ impl ChecksumAlgorithm { match self { Self::Crc32 => Box::::default(), Self::Crc32c => Box::::default(), - Self::Md5 => Box::::default(), + Self::Crc64Nvme => Box::::default(), + #[allow(deprecated)] + Self::Md5 => Box::::default(), Self::Sha1 => Box::::default(), Self::Sha256 => Box::::default(), } @@ -86,6 +97,8 @@ impl ChecksumAlgorithm { match self { Self::Crc32 => CRC_32_NAME, Self::Crc32c => CRC_32_C_NAME, + Self::Crc64Nvme => CRC_64_NVME_NAME, + #[allow(deprecated)] Self::Md5 => MD5_NAME, Self::Sha1 => SHA_1_NAME, Self::Sha256 => SHA_256_NAME, @@ -183,6 +196,45 @@ impl Checksum for Crc32c { } } +#[derive(Default)] +struct Crc64Nvme { + hasher: crc64fast_nvme::Digest, +} + +// crc64fast_nvme::Digest doesn't impl Debug so we can't derive the impl +impl Debug for Crc64Nvme { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("Crc64Nvme").finish() + } +} + +impl Crc64Nvme { + fn update(&mut self, bytes: &[u8]) { + self.hasher.write(bytes); + } + + fn finalize(self) -> Bytes { + Bytes::copy_from_slice(self.hasher.sum64().to_be_bytes().as_slice()) + } + + // Size of the checksum in bytes + fn size() -> u64 { + 8 + } +} + +impl Checksum for Crc64Nvme { + fn update(&mut self, bytes: &[u8]) { + Self::update(self, bytes) + } + fn finalize(self: Box) -> Bytes { + Self::finalize(*self) + } + fn size(&self) -> u64 { + Self::size() + } +} + #[derive(Debug, Default)] struct Sha1 { hasher: sha1::Sha1, @@ -301,6 +353,7 @@ mod tests { use crate::http::HttpChecksum; use crate::ChecksumAlgorithm; + use aws_smithy_types::base64; use http::HeaderValue; use pretty_assertions::assert_eq; @@ -349,6 +402,20 @@ mod tests { assert_eq!(decoded_checksum, expected_checksum); } + #[test] + fn test_crc64nvme_checksum() { + use crate::{http::CRC_64_NVME_HEADER_NAME, Crc64Nvme}; + let mut checksum = Crc64Nvme::default(); + checksum.update(TEST_DATA.as_bytes()); + let checksum_result = Box::new(checksum).headers(); + let encoded_checksum = checksum_result.get(CRC_64_NVME_HEADER_NAME).unwrap(); + let decoded_checksum = base64_encoded_checksum_to_hex_string(encoded_checksum); + + let expected_checksum = "0xAECAF3AF9C98A855"; + + assert_eq!(decoded_checksum, expected_checksum); + } + #[test] fn test_sha1_checksum() { let mut checksum = Sha1::default(); diff --git a/rust-runtime/aws-smithy-eventstream/Cargo.toml b/rust-runtime/aws-smithy-eventstream/Cargo.toml index f66347ff2..d07c5bd8a 100644 --- a/rust-runtime/aws-smithy-eventstream/Cargo.toml +++ b/rust-runtime/aws-smithy-eventstream/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "aws-smithy-eventstream" # Only patch releases can be made to this runtime crate until https://github.com/smithy-lang/smithy-rs/issues/3370 is resolved -version = "0.60.5" +version = "0.60.6" # authors = ["AWS Rust SDK Team ", "John DiSanti "] description = "Event stream logic for smithy-rs." diff --git a/rust-runtime/aws-smithy-eventstream/fuzz/Cargo.toml b/rust-runtime/aws-smithy-eventstream/fuzz/Cargo.toml index f95d49383..7fdb9c328 100644 --- a/rust-runtime/aws-smithy-eventstream/fuzz/Cargo.toml +++ b/rust-runtime/aws-smithy-eventstream/fuzz/Cargo.toml @@ -14,7 +14,8 @@ aws-smithy-types = { path = "../../aws-smithy-types" } bytes = "1" crc32fast = "1" derive_arbitrary = "1.3" -libfuzzer-sys = "0.4" +# Version pinned due to https://github.com/rust-fuzz/libfuzzer/issues/126 +libfuzzer-sys = "=0.4.7" [dependencies.aws-smithy-eventstream] features = ["derive-arbitrary"] diff --git a/rust-runtime/aws-smithy-http/Cargo.toml b/rust-runtime/aws-smithy-http/Cargo.toml index b915bc394..2136d2cc6 100644 --- a/rust-runtime/aws-smithy-http/Cargo.toml +++ b/rust-runtime/aws-smithy-http/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "aws-smithy-http" -version = "0.60.11" +version = "0.60.12" authors = [ "AWS Rust SDK Team ", "Russell Cohen ", diff --git a/rust-runtime/aws-smithy-http/fuzz/Cargo.toml b/rust-runtime/aws-smithy-http/fuzz/Cargo.toml index 6b2597277..040c40e76 100644 --- a/rust-runtime/aws-smithy-http/fuzz/Cargo.toml +++ b/rust-runtime/aws-smithy-http/fuzz/Cargo.toml @@ -9,7 +9,8 @@ edition = "2021" cargo-fuzz = true [dependencies] -libfuzzer-sys = "0.4" +# Version pinned due to https://github.com/rust-fuzz/libfuzzer/issues/126 +libfuzzer-sys = "=0.4.7" http = "0.2.3" [dependencies.aws-smithy-http] diff --git a/rust-runtime/aws-smithy-json/Cargo.toml b/rust-runtime/aws-smithy-json/Cargo.toml index c758e5da6..92105e744 100644 --- a/rust-runtime/aws-smithy-json/Cargo.toml +++ b/rust-runtime/aws-smithy-json/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "aws-smithy-json" -version = "0.61.1" +version = "0.61.2" authors = ["AWS Rust SDK Team ", "John DiSanti "] description = "Token streaming JSON parser for smithy-rs." edition = "2021" diff --git a/rust-runtime/aws-smithy-json/fuzz/Cargo.toml b/rust-runtime/aws-smithy-json/fuzz/Cargo.toml index 435cc1ab3..94cf33cfa 100644 --- a/rust-runtime/aws-smithy-json/fuzz/Cargo.toml +++ b/rust-runtime/aws-smithy-json/fuzz/Cargo.toml @@ -11,7 +11,8 @@ cargo-fuzz = true [dependencies] aws-smithy-json = { path = ".." } aws-smithy-types = { path = "../../aws-smithy-types" } -libfuzzer-sys = "0.4" +# Version pinned due to https://github.com/rust-fuzz/libfuzzer/issues/126 +libfuzzer-sys = "=0.4.7" serde_json = { version = "1", features = ["float_roundtrip"] } # Prevent this from interfering with workspaces diff --git a/rust-runtime/aws-smithy-types/Cargo.toml b/rust-runtime/aws-smithy-types/Cargo.toml index 249721369..8894e5ee7 100644 --- a/rust-runtime/aws-smithy-types/Cargo.toml +++ b/rust-runtime/aws-smithy-types/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "aws-smithy-types" -version = "1.2.11" +version = "1.2.12" authors = [ "AWS Rust SDK Team ", "Russell Cohen ", diff --git a/rust-runtime/aws-smithy-types/fuzz/Cargo.toml b/rust-runtime/aws-smithy-types/fuzz/Cargo.toml index da3fdda22..1d74f391c 100644 --- a/rust-runtime/aws-smithy-types/fuzz/Cargo.toml +++ b/rust-runtime/aws-smithy-types/fuzz/Cargo.toml @@ -9,7 +9,8 @@ edition = "2021" cargo-fuzz = true [dependencies] -libfuzzer-sys = "0.4" +# Version pinned due to https://github.com/rust-fuzz/libfuzzer/issues/126 +libfuzzer-sys = "=0.4.7" [dependencies.aws-smithy-types] path = ".." diff --git a/rust-runtime/aws-smithy-types/src/checksum_config.rs b/rust-runtime/aws-smithy-types/src/checksum_config.rs new file mode 100644 index 000000000..a44c7bd43 --- /dev/null +++ b/rust-runtime/aws-smithy-types/src/checksum_config.rs @@ -0,0 +1,163 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +//! Types that allow users to indicate their preferences for checksum calculation and validation + +// Note: These types would likely make more sense in `aws-smithy-checksums` and were originally +// added there. But we have lints protecting against exporting non-stable types from stable crates +// and the checksums crate is not yet 1.0, so these types cannot live there for now. In the future +// if we do decide to 1.0 the checksums crate we can move these types there and re-export them here +// to maintain the current behavior. + +use std::error::Error; +use std::fmt; +use std::str::FromStr; + +use crate::config_bag::{Storable, StoreReplace}; + +// Valid names for RequestChecksumCalculation and ResponseChecksumValidation +const WHEN_SUPPORTED: &str = "when_supported"; +const WHEN_REQUIRED: &str = "when_required"; + +/// Determines when a checksum will be calculated for request payloads. Values are: +/// * [RequestChecksumCalculation::WhenSupported] - (default) When set, a checksum will be +/// calculated for all request payloads of operations modeled with the +/// `httpChecksum` trait where `requestChecksumRequired` is `true` and/or a +/// `requestAlgorithmMember` is modeled. +/// * [RequestChecksumCalculation::WhenRequired] - When set, a checksum will only be calculated for +/// request payloads of operations modeled with the `httpChecksum` trait where +/// `requestChecksumRequired` is `true` or where a requestAlgorithmMember +/// is modeled and supplied. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +#[non_exhaustive] +pub enum RequestChecksumCalculation { + /// Calculate request checksums when they are supported. + #[default] + WhenSupported, + /// Caulculate request checksums only when they are required. + WhenRequired, +} + +impl Storable for RequestChecksumCalculation { + type Storer = StoreReplace; +} + +impl FromStr for RequestChecksumCalculation { + type Err = UnknownRequestChecksumCalculationError; + + fn from_str(request_checksum_calculation: &str) -> Result { + if request_checksum_calculation.eq_ignore_ascii_case(WHEN_SUPPORTED) { + Ok(Self::WhenSupported) + } else if request_checksum_calculation.eq_ignore_ascii_case(WHEN_REQUIRED) { + Ok(Self::WhenRequired) + } else { + Err(UnknownRequestChecksumCalculationError::new( + request_checksum_calculation, + )) + } + } +} + +/// Determines when checksum validation will be performed on response payloads. Values are: +/// * [ResponseChecksumValidation::WhenSupported] - (default) When set, checksum validation is performed on all +/// response payloads of operations modeled with the `httpChecksum` trait where +/// `responseAlgorithms` is modeled, except when no modeled checksum algorithms +/// are supported. +/// * [ResponseChecksumValidation::WhenRequired] - When set, checksum validation is not performed on +/// response payloads of operations unless the checksum algorithm is supported and +/// the `requestValidationModeMember` member is set to `ENABLED`. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +#[non_exhaustive] +pub enum ResponseChecksumValidation { + /// Validate response checksums when they are supported. + #[default] + WhenSupported, + /// Validate response checksums only when they are required. + WhenRequired, +} + +impl Storable for ResponseChecksumValidation { + type Storer = StoreReplace; +} + +impl FromStr for ResponseChecksumValidation { + type Err = UnknownResponseChecksumValidationError; + + fn from_str(response_checksum_validation: &str) -> Result { + if response_checksum_validation.eq_ignore_ascii_case(WHEN_SUPPORTED) { + Ok(Self::WhenSupported) + } else if response_checksum_validation.eq_ignore_ascii_case(WHEN_REQUIRED) { + Ok(Self::WhenRequired) + } else { + Err(UnknownResponseChecksumValidationError::new( + response_checksum_validation, + )) + } + } +} + +/// Unknown setting for `request_checksum_calculation` +#[derive(Debug)] +#[non_exhaustive] +pub struct UnknownRequestChecksumCalculationError { + request_checksum_calculation: String, +} + +impl UnknownRequestChecksumCalculationError { + pub(crate) fn new(request_checksum_calculation: impl Into) -> Self { + Self { + request_checksum_calculation: request_checksum_calculation.into(), + } + } + + /// The unknown value + pub fn request_checksum_calculation(&self) -> &str { + &self.request_checksum_calculation + } +} + +impl fmt::Display for UnknownRequestChecksumCalculationError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!( + f, + r#"unknown request_checksum_calculation value "{}", please pass a known name ("when_supported", "when_required")"#, + self.request_checksum_calculation + ) + } +} + +impl Error for UnknownRequestChecksumCalculationError {} + +/// Unknown setting for `response_checksum_validation` +#[derive(Debug)] +#[non_exhaustive] +pub struct UnknownResponseChecksumValidationError { + response_checksum_validation: String, +} + +impl UnknownResponseChecksumValidationError { + pub(crate) fn new(response_checksum_validation: impl Into) -> Self { + Self { + response_checksum_validation: response_checksum_validation.into(), + } + } + + /// The unknown value + pub fn response_checksum_validation(&self) -> &str { + &self.response_checksum_validation + } +} + +impl fmt::Display for UnknownResponseChecksumValidationError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!( + f, + r#"unknown response_checksum_validation value "{}", please pass a known name ("when_supported", "when_required")"#, + self.response_checksum_validation + ) + } +} + +impl Error for UnknownResponseChecksumValidationError {} diff --git a/rust-runtime/aws-smithy-types/src/lib.rs b/rust-runtime/aws-smithy-types/src/lib.rs index ed53de191..5af73e1fb 100644 --- a/rust-runtime/aws-smithy-types/src/lib.rs +++ b/rust-runtime/aws-smithy-types/src/lib.rs @@ -20,6 +20,7 @@ pub mod base64; pub mod body; pub mod byte_stream; +pub mod checksum_config; /// A typemap for storing configuration. pub mod config_bag; pub mod date_time; -- GitLab