diff --git a/.changelog/flexible-checksums-client.md b/.changelog/flexible-checksums-client.md new file mode 100644 index 0000000000000000000000000000000000000000..ee69e3fd408b0a2b47d6f97bb00876bedb2edc6d --- /dev/null +++ b/.changelog/flexible-checksums-client.md @@ -0,0 +1,10 @@ +--- +applies_to: ["client"] +authors: ["landonxjames"] +references: ["smithy-rs#3967"] +breaking: false +new_feature: true +bug_fix: true +--- + +Updates client generation to conform with Smithy's updates to the [httpChecksum trait](https://smithy.io/2.0/aws/aws-core.html#aws-protocols-httpchecksum-trait). diff --git a/.changelog/flexible-checksums-s3.md b/.changelog/flexible-checksums-s3.md new file mode 100644 index 0000000000000000000000000000000000000000..0e903e9a2adbdcbed129d2da172f00fbd7c4e1d2 --- /dev/null +++ b/.changelog/flexible-checksums-s3.md @@ -0,0 +1,12 @@ +--- +applies_to: ["client", "aws-sdk-rust"] +authors: ["landonxjames"] +references: ["smithy-rs#3845"] +breaking: false +new_feature: true +bug_fix: true +--- + +S3 client behavior is updated to always calculate a checksum by default for operations that support it (such as PutObject or UploadPart), or require it (such as DeleteObjects). The default checksum algorithm is CRC32. Checksum behavior can be configured using `when_supported` and `when_required` options - in shared config using request_checksum_calculation, or as env variable using AWS_REQUEST_CHECKSUM_CALCULATION. + +The S3 client attempts to validate response checksums for all S3 API operations that support checksums. However, if the SDK has not implemented the specified checksum algorithm then this validation is skipped. Checksum validation behavior can be configured using `when_supported` and `when_required` options - in shared config using response_checksum_validation, or as env variable using AWS_RESPONSE_CHECKSUM_VALIDATION. diff --git a/aws/rust-runtime/Cargo.lock b/aws/rust-runtime/Cargo.lock index 9d8fc4f5919503e4b2beb746013c6c700058afa6..95a15075640949d4e4a719704c19fd52ada5ff0b 100644 --- a/aws/rust-runtime/Cargo.lock +++ b/aws/rust-runtime/Cargo.lock @@ -38,12 +38,55 @@ version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4b46cbb362ab8752921c97e041f5e366ee6297bd428a31275b9fcf1e380f7299" +[[package]] +name = "anstream" +version = "0.6.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8acc5369981196006228e28809f761875c0327210a891e941f4c683b3a99529b" +dependencies = [ + "anstyle", + "anstyle-parse", + "anstyle-query", + "anstyle-wincon", + "colorchoice", + "is_terminal_polyfill", + "utf8parse", +] + [[package]] name = "anstyle" version = "1.0.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "55cc3b69f167a1ef2e161439aa98aed94e6028e5f9a59be9a6ffb47aef1651f9" +[[package]] +name = "anstyle-parse" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b2d16507662817a6a20a9ea92df6652ee4f94f914589377d69f3b21bc5798a9" +dependencies = [ + "utf8parse", +] + +[[package]] +name = "anstyle-query" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "79947af37f4177cfead1110013d678905c37501914fba0efea834c3fe9a8d60c" +dependencies = [ + "windows-sys 0.59.0", +] + +[[package]] +name = "anstyle-wincon" +version = "3.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2109dbce0e72be3ec00bed26e6a7479ca384ad226efdd66db8fa2e3a38c83125" +dependencies = [ + "anstyle", + "windows-sys 0.59.0", +] + [[package]] name = "arbitrary" version = "1.4.1" @@ -138,7 +181,7 @@ dependencies = [ [[package]] name = "aws-runtime" -version = "1.5.3" +version = "1.5.4" dependencies = [ "arbitrary", "aws-credential-types", @@ -184,7 +227,7 @@ version = "0.60.3" [[package]] name = "aws-sigv4" -version = "1.2.6" +version = "1.2.7" dependencies = [ "aws-credential-types", "aws-smithy-eventstream", @@ -220,7 +263,7 @@ dependencies = [ [[package]] name = "aws-smithy-async" -version = "1.2.3" +version = "1.2.4" dependencies = [ "futures-util", "pin-project-lite", @@ -229,13 +272,14 @@ dependencies = [ [[package]] name = "aws-smithy-checksums" -version = "0.60.13" +version = "0.62.0" dependencies = [ "aws-smithy-http", "aws-smithy-types", "bytes", "crc32c", "crc32fast", + "crc64fast-nvme", "hex", "http 0.2.12", "http-body 0.4.6", @@ -248,7 +292,7 @@ dependencies = [ [[package]] name = "aws-smithy-eventstream" -version = "0.60.5" +version = "0.60.6" dependencies = [ "aws-smithy-types", "bytes", @@ -257,7 +301,7 @@ dependencies = [ [[package]] name = "aws-smithy-http" -version = "0.60.11" +version = "0.60.12" dependencies = [ "aws-smithy-runtime-api", "aws-smithy-types", @@ -332,7 +376,7 @@ dependencies = [ [[package]] name = "aws-smithy-types" -version = "1.2.11" +version = "1.2.12" dependencies = [ "base64-simd", "bytes", @@ -356,7 +400,7 @@ dependencies = [ [[package]] name = "aws-types" -version = "1.3.3" +version = "1.3.4" dependencies = [ "aws-credential-types", "aws-smithy-async", @@ -488,6 +532,25 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" +[[package]] +name = "cbindgen" +version = "0.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3fce8dd7fcfcbf3a0a87d8f515194b49d6135acab73e18bd380d1d93bb1a15eb" +dependencies = [ + "clap", + "heck", + "indexmap", + "log", + "proc-macro2", + "quote", + "serde", + "serde_json", + "syn 2.0.94", + "tempfile", + "toml", +] + [[package]] name = "cbor-diag" version = "0.1.12" @@ -575,8 +638,10 @@ version = "4.5.26" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "96b01801b5fc6a0a232407abc821660c9c6d25a1cafc0d4f85f29fb8d9afc121" dependencies = [ + "anstream", "anstyle", "clap_lex", + "strsim", ] [[package]] @@ -585,6 +650,12 @@ version = "0.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f46ad14479a25103f283c0f10005961cf086d8dc42205bb44c46ac563475dca6" +[[package]] +name = "colorchoice" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b63caa9aa9397e2d9480a9b13673856c78d8ac123288526c37d7839f2a86990" + [[package]] name = "const-oid" version = "0.9.6" @@ -625,6 +696,21 @@ dependencies = [ "libc", ] +[[package]] +name = "crc" +version = "3.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69e6e4d7b33a94f0991c26729976b10ebde1d34c3ee82408fb536164fa10d636" +dependencies = [ + "crc-catalog", +] + +[[package]] +name = "crc-catalog" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19d374276b40fb8bbdee95aef7c7fa6b5316ec764510eb64b8dd0e2ed0d7e7f5" + [[package]] name = "crc32c" version = "0.6.8" @@ -643,6 +729,16 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "crc64fast-nvme" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d5e2ee08013e3f228d6d2394116c4549a6df77708442c62d887d83f68ef2ee37" +dependencies = [ + "cbindgen", + "crc", +] + [[package]] name = "criterion" version = "0.5.1" @@ -1015,6 +1111,12 @@ dependencies = [ "foldhash", ] +[[package]] +name = "heck" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" + [[package]] name = "hermit-abi" version = "0.4.0" @@ -1311,6 +1413,12 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "is_terminal_polyfill" +version = "1.70.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7943c866cc5cd64cbc25b2e01621d07fa8eb2a1a23160ee81ce38704e97b8ecf" + [[package]] name = "itertools" version = "0.10.5" @@ -2106,6 +2214,15 @@ dependencies = [ "serde", ] +[[package]] +name = "serde_spanned" +version = "0.6.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87607cb1398ed59d48732e575a4c28a7a8ebf2454b964fe3f224f2afc07909e1" +dependencies = [ + "serde", +] + [[package]] name = "sha1" version = "0.10.6" @@ -2209,6 +2326,12 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3" +[[package]] +name = "strsim" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" + [[package]] name = "subtle" version = "2.6.1" @@ -2409,6 +2532,40 @@ dependencies = [ "tokio", ] +[[package]] +name = "toml" +version = "0.8.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1ed1f98e3fdc28d6d910e6737ae6ab1a93bf1985935a1193e68f93eeb68d24e" +dependencies = [ + "serde", + "serde_spanned", + "toml_datetime", + "toml_edit", +] + +[[package]] +name = "toml_datetime" +version = "0.6.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0dd7358ecb8fc2f8d014bf86f6f638ce72ba252a2c3a2572f2a795f1d23efb41" +dependencies = [ + "serde", +] + +[[package]] +name = "toml_edit" +version = "0.22.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ae48d6208a266e853d946088ed816055e556cc6028c5e8e2b84d9fa5dd7c7f5" +dependencies = [ + "indexmap", + "serde", + "serde_spanned", + "toml_datetime", + "winnow", +] + [[package]] name = "tower-service" version = "0.3.3" @@ -2556,6 +2713,12 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" +[[package]] +name = "utf8parse" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" + [[package]] name = "uuid" version = "1.12.0" @@ -2801,6 +2964,15 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" +[[package]] +name = "winnow" +version = "0.6.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39281189af81c07ec09db316b302a3e67bf9bd7cbf6c820b50e35fee9c2fa980" +dependencies = [ + "memchr", +] + [[package]] name = "write16" version = "1.0.0" diff --git a/aws/rust-runtime/aws-config/Cargo.lock b/aws/rust-runtime/aws-config/Cargo.lock index 6537c71af7a89b06bf3327bb3acc337a153e48cc..bed7bac1afffac8cc9e2af7f440e8da5220748a8 100644 --- a/aws/rust-runtime/aws-config/Cargo.lock +++ b/aws/rust-runtime/aws-config/Cargo.lock @@ -45,7 +45,7 @@ checksum = "ace50bade8e6234aa140d9a2f552bbee1db4d353f69b8217bc503490fc1a9f26" [[package]] name = "aws-config" -version = "1.5.13" +version = "1.5.14" dependencies = [ "aws-credential-types", "aws-runtime", @@ -89,7 +89,7 @@ dependencies = [ [[package]] name = "aws-runtime" -version = "1.5.3" +version = "1.5.4" dependencies = [ "aws-credential-types", "aws-sigv4", @@ -174,7 +174,7 @@ dependencies = [ [[package]] name = "aws-sigv4" -version = "1.2.6" +version = "1.2.7" dependencies = [ "aws-credential-types", "aws-smithy-http", @@ -195,7 +195,7 @@ dependencies = [ [[package]] name = "aws-smithy-async" -version = "1.2.3" +version = "1.2.4" dependencies = [ "futures-util", "pin-project-lite", @@ -299,7 +299,7 @@ dependencies = [ [[package]] name = "aws-smithy-types" -version = "1.2.11" +version = "1.2.12" dependencies = [ "base64-simd", "bytes", @@ -327,7 +327,7 @@ dependencies = [ [[package]] name = "aws-types" -version = "1.3.3" +version = "1.3.4" dependencies = [ "aws-credential-types", "aws-smithy-async", diff --git a/aws/rust-runtime/aws-config/Cargo.toml b/aws/rust-runtime/aws-config/Cargo.toml index 2cb33239b4cb460b5853a753b6dae44a4f2c1cdb..0e47958c50b56a2f610eae08a4bc824c2e9fba90 100644 --- a/aws/rust-runtime/aws-config/Cargo.toml +++ b/aws/rust-runtime/aws-config/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "aws-config" -version = "1.5.13" +version = "1.5.14" authors = [ "AWS Rust SDK Team ", "Russell Cohen ", diff --git a/aws/rust-runtime/aws-config/external-types.toml b/aws/rust-runtime/aws-config/external-types.toml index 9c8d3c1b24e8e5b3b364fbfd87ef945b9678c412..3e2c6726d1e8862b7c09ccc9fcf36a7797a0946b 100644 --- a/aws/rust-runtime/aws-config/external-types.toml +++ b/aws/rust-runtime/aws-config/external-types.toml @@ -34,6 +34,8 @@ allowed_external_types = [ "aws_smithy_runtime_api::client::result::SdkError", "aws_smithy_runtime_api::client::stalled_stream_protection::StalledStreamProtectionConfig", "aws_smithy_types::body::SdkBody", + "aws_smithy_types::checksum_config::RequestChecksumCalculation", + "aws_smithy_types::checksum_config::ResponseChecksumValidation", "aws_smithy_types::retry", "aws_smithy_types::retry::*", "aws_smithy_types::timeout", diff --git a/aws/rust-runtime/aws-config/fuzz/Cargo.toml b/aws/rust-runtime/aws-config/fuzz/Cargo.toml index e6139ee805cf1b30327b58a7d7cd789692e63b7e..c62f180a0e9d4598b1d78d28407f637e74ed4843 100644 --- a/aws/rust-runtime/aws-config/fuzz/Cargo.toml +++ b/aws/rust-runtime/aws-config/fuzz/Cargo.toml @@ -10,7 +10,8 @@ edition = "2021" cargo-fuzz = true [dependencies] -libfuzzer-sys = "0.4" +# Version pinned due to https://github.com/rust-fuzz/libfuzzer/issues/126 +libfuzzer-sys = "=0.4.7" aws-config = { path = ".." } [dependencies.aws-types] diff --git a/aws/rust-runtime/aws-config/src/default_provider.rs b/aws/rust-runtime/aws-config/src/default_provider.rs index fceb869fb00b2428a55da409a78624d3f0b8bd5c..42e5a6173979a09c2a7c7b970a41d1c9c6260e99 100644 --- a/aws/rust-runtime/aws-config/src/default_provider.rs +++ b/aws/rust-runtime/aws-config/src/default_provider.rs @@ -63,3 +63,6 @@ pub mod disable_request_compression; /// Default "request minimum compression size bytes" provider chain pub mod request_min_compression_size_bytes; + +/// Default provider chains for request/response checksum configuration +pub mod checksums; diff --git a/aws/rust-runtime/aws-config/src/default_provider/checksums.rs b/aws/rust-runtime/aws-config/src/default_provider/checksums.rs new file mode 100644 index 0000000000000000000000000000000000000000..25da664a39318300b632908d6edba9379afd7130 --- /dev/null +++ b/aws/rust-runtime/aws-config/src/default_provider/checksums.rs @@ -0,0 +1,259 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +use crate::provider_config::ProviderConfig; +use aws_runtime::env_config::EnvConfigValue; +use aws_smithy_types::error::display::DisplayErrorContext; +use aws_types::sdk_config::{RequestChecksumCalculation, ResponseChecksumValidation}; +use std::str::FromStr; + +mod env { + pub(super) const REQUEST_CHECKSUM_CALCULATION: &str = "AWS_REQUEST_CHECKSUM_CALCULATION"; + pub(super) const RESPONSE_CHECKSUM_VALIDATION: &str = "AWS_RESPONSE_CHECKSUM_VALIDATION"; +} + +mod profile_key { + pub(super) const REQUEST_CHECKSUM_CALCULATION: &str = "request_checksum_calculation"; + pub(super) const RESPONSE_CHECKSUM_VALIDATION: &str = "response_checksum_validation"; +} + +/// Load the value for `request_checksum_calculation` +/// +/// This checks the following sources: +/// 1. The environment variable `AWS_REQUEST_CHECKSUM_CALCULATION=WHEN_SUPPORTED/WHEN_REQUIRED` +/// 2. The profile key `request_checksum_calculation=WHEN_SUPPORTED/WHEN_REQUIRED` +/// +/// If invalid values are found, the provider will return `None` and an error will be logged. +pub async fn request_checksum_calculation_provider( + provider_config: &ProviderConfig, +) -> Option { + let env = provider_config.env(); + let profiles = provider_config.profile().await; + + let loaded = EnvConfigValue::new() + .env(env::REQUEST_CHECKSUM_CALCULATION) + .profile(profile_key::REQUEST_CHECKSUM_CALCULATION) + .validate(&env, profiles, RequestChecksumCalculation::from_str) + .map_err( + |err| tracing::warn!(err = %DisplayErrorContext(&err), "invalid value for request_checksum_calculation setting"), + ) + .unwrap_or(None); + + // request_checksum_calculation should always have a non-None value and the + // default is WhenSupported + loaded.or(Some(RequestChecksumCalculation::WhenSupported)) +} + +/// Load the value for `response_checksum_validation` +/// +/// This checks the following sources: +/// 1. The environment variable `AWS_RESPONSE_CHECKSUM_VALIDATION=WHEN_SUPPORTED/WHEN_REQUIRED` +/// 2. The profile key `response_checksum_validation=WHEN_SUPPORTED/WHEN_REQUIRED` +/// +/// If invalid values are found, the provider will return `None` and an error will be logged. +pub async fn response_checksum_validation_provider( + provider_config: &ProviderConfig, +) -> Option { + let env = provider_config.env(); + let profiles = provider_config.profile().await; + + let loaded = EnvConfigValue::new() + .env(env::RESPONSE_CHECKSUM_VALIDATION) + .profile(profile_key::RESPONSE_CHECKSUM_VALIDATION) + .validate(&env, profiles, ResponseChecksumValidation::from_str) + .map_err( + |err| tracing::warn!(err = %DisplayErrorContext(&err), "invalid value for response_checksum_validation setting"), + ) + .unwrap_or(None); + + // response_checksum_validation should always have a non-None value and the + // default is WhenSupported + loaded.or(Some(ResponseChecksumValidation::WhenSupported)) +} + +#[cfg(test)] +mod test { + use crate::default_provider::checksums::{ + request_checksum_calculation_provider, response_checksum_validation_provider, + }; + #[allow(deprecated)] + use crate::profile::profile_file::{ProfileFileKind, ProfileFiles}; + use crate::provider_config::ProviderConfig; + use aws_smithy_types::checksum_config::{ + RequestChecksumCalculation, ResponseChecksumValidation, + }; + use aws_types::os_shim_internal::{Env, Fs}; + use tracing_test::traced_test; + + #[tokio::test] + #[traced_test] + async fn log_error_on_invalid_value_request() { + let conf = ProviderConfig::empty().with_env(Env::from_slice(&[( + "AWS_REQUEST_CHECKSUM_CALCULATION", + "not-a-valid-value", + )])); + assert_eq!( + request_checksum_calculation_provider(&conf).await, + Some(RequestChecksumCalculation::WhenSupported) + ); + assert!(logs_contain( + "invalid value for request_checksum_calculation setting" + )); + assert!(logs_contain("AWS_REQUEST_CHECKSUM_CALCULATION")); + } + + #[tokio::test] + #[traced_test] + async fn environment_priority_request() { + let conf = ProviderConfig::empty() + .with_env(Env::from_slice(&[( + "AWS_REQUEST_CHECKSUM_CALCULATION", + "WHEN_REQUIRED", + )])) + .with_profile_config( + Some( + #[allow(deprecated)] + ProfileFiles::builder() + .with_file( + #[allow(deprecated)] + ProfileFileKind::Config, + "conf", + ) + .build(), + ), + None, + ) + .with_fs(Fs::from_slice(&[( + "conf", + "[default]\nrequest_checksum_calculation = WHEN_SUPPORTED", + )])); + assert_eq!( + request_checksum_calculation_provider(&conf).await, + Some(RequestChecksumCalculation::WhenRequired) + ); + } + + #[tokio::test] + #[traced_test] + async fn profile_works_request() { + let conf = ProviderConfig::empty() + .with_profile_config( + Some( + #[allow(deprecated)] + ProfileFiles::builder() + .with_file( + #[allow(deprecated)] + ProfileFileKind::Config, + "conf", + ) + .build(), + ), + None, + ) + .with_fs(Fs::from_slice(&[( + "conf", + "[default]\nrequest_checksum_calculation = WHEN_REQUIRED", + )])); + assert_eq!( + request_checksum_calculation_provider(&conf).await, + Some(RequestChecksumCalculation::WhenRequired) + ); + } + + #[tokio::test] + #[traced_test] + async fn default_works_request() { + let conf = ProviderConfig::empty(); + assert_eq!( + request_checksum_calculation_provider(&conf).await, + Some(RequestChecksumCalculation::WhenSupported) + ); + } + + #[tokio::test] + #[traced_test] + async fn log_error_on_invalid_value_response() { + let conf = ProviderConfig::empty().with_env(Env::from_slice(&[( + "AWS_RESPONSE_CHECKSUM_VALIDATION", + "not-a-valid-value", + )])); + assert_eq!( + response_checksum_validation_provider(&conf).await, + Some(ResponseChecksumValidation::WhenSupported) + ); + assert!(logs_contain( + "invalid value for response_checksum_validation setting" + )); + assert!(logs_contain("AWS_RESPONSE_CHECKSUM_VALIDATION")); + } + + #[tokio::test] + #[traced_test] + async fn environment_priority_response() { + let conf = ProviderConfig::empty() + .with_env(Env::from_slice(&[( + "AWS_RESPONSE_CHECKSUM_VALIDATION", + "WHEN_SUPPORTED", + )])) + .with_profile_config( + Some( + #[allow(deprecated)] + ProfileFiles::builder() + .with_file( + #[allow(deprecated)] + ProfileFileKind::Config, + "conf", + ) + .build(), + ), + None, + ) + .with_fs(Fs::from_slice(&[( + "conf", + "[default]\response_checksum_validation = WHEN_REQUIRED", + )])); + assert_eq!( + response_checksum_validation_provider(&conf).await, + Some(ResponseChecksumValidation::WhenSupported) + ); + } + + #[tokio::test] + #[traced_test] + async fn profile_works_response() { + let conf = ProviderConfig::empty() + .with_profile_config( + Some( + #[allow(deprecated)] + ProfileFiles::builder() + .with_file( + #[allow(deprecated)] + ProfileFileKind::Config, + "conf", + ) + .build(), + ), + None, + ) + .with_fs(Fs::from_slice(&[( + "conf", + "[default]\nresponse_checksum_validation = WHEN_REQUIRED", + )])); + assert_eq!( + response_checksum_validation_provider(&conf).await, + Some(ResponseChecksumValidation::WhenRequired) + ); + } + + #[tokio::test] + #[traced_test] + async fn default_works_response() { + let conf = ProviderConfig::empty(); + assert_eq!( + response_checksum_validation_provider(&conf).await, + Some(ResponseChecksumValidation::WhenSupported) + ); + } +} diff --git a/aws/rust-runtime/aws-config/src/lib.rs b/aws/rust-runtime/aws-config/src/lib.rs index dd873270376672f16327e95e9333b541150d4d82..8650325d09ab8afe658d25ca7eaab0631ba4b39e 100644 --- a/aws/rust-runtime/aws-config/src/lib.rs +++ b/aws/rust-runtime/aws-config/src/lib.rs @@ -228,6 +228,9 @@ mod loader { use aws_smithy_runtime_api::client::identity::{ResolveCachedIdentity, SharedIdentityCache}; use aws_smithy_runtime_api::client::stalled_stream_protection::StalledStreamProtectionConfig; use aws_smithy_runtime_api::shared::IntoShared; + use aws_smithy_types::checksum_config::{ + RequestChecksumCalculation, ResponseChecksumValidation, + }; use aws_smithy_types::retry::RetryConfig; use aws_smithy_types::timeout::TimeoutConfig; use aws_types::app_name::AppName; @@ -238,7 +241,7 @@ mod loader { use aws_types::SdkConfig; use crate::default_provider::{ - app_name, credentials, disable_request_compression, endpoint_url, + app_name, checksums, credentials, disable_request_compression, endpoint_url, ignore_configured_endpoint_urls as ignore_ep, region, request_min_compression_size_bytes, retry_config, timeout_config, use_dual_stack, use_fips, }; @@ -289,6 +292,8 @@ mod loader { env: Option, fs: Option, behavior_version: Option, + request_checksum_calculation: Option, + response_checksum_validation: Option, } impl ConfigLoader { @@ -919,6 +924,22 @@ mod loader { Some(user_cache) => Some(user_cache), }; + let request_checksum_calculation = + if let Some(request_checksum_calculation) = self.request_checksum_calculation { + Some(request_checksum_calculation) + } else { + checksums::request_checksum_calculation_provider(&conf).await + }; + + let response_checksum_validation = + if let Some(response_checksum_validation) = self.response_checksum_validation { + Some(response_checksum_validation) + } else { + checksums::response_checksum_validation_provider(&conf).await + }; + + builder.set_request_checksum_calculation(request_checksum_calculation); + builder.set_response_checksum_validation(response_checksum_validation); builder.set_identity_cache(identity_cache); builder.set_credentials_provider(credentials_provider); builder.set_token_provider(token_provider); diff --git a/aws/rust-runtime/aws-inlineable/src/http_request_checksum.rs b/aws/rust-runtime/aws-inlineable/src/http_request_checksum.rs index 689edcc5569d505895e6b3112e3c2b0f66e95840..f278bb0b3af3f48de5f86ec606053e51ee77a046 100644 --- a/aws/rust-runtime/aws-inlineable/src/http_request_checksum.rs +++ b/aws/rust-runtime/aws-inlineable/src/http_request_checksum.rs @@ -12,20 +12,26 @@ use aws_runtime::content_encoding::header_value::AWS_CHUNKED; use aws_runtime::content_encoding::{AwsChunkedBody, AwsChunkedBodyOptions}; use aws_smithy_checksums::ChecksumAlgorithm; use aws_smithy_checksums::{body::calculate, http::HttpChecksum}; +use aws_smithy_runtime::client::sdk_feature::SmithySdkFeature; use aws_smithy_runtime_api::box_error::BoxError; use aws_smithy_runtime_api::client::interceptors::context::{ - BeforeSerializationInterceptorContextRef, BeforeTransmitInterceptorContextMut, Input, + BeforeSerializationInterceptorContextMut, BeforeTransmitInterceptorContextMut, Input, }; use aws_smithy_runtime_api::client::interceptors::Intercept; use aws_smithy_runtime_api::client::orchestrator::HttpRequest; use aws_smithy_runtime_api::client::runtime_components::RuntimeComponents; +use aws_smithy_runtime_api::http::Request; use aws_smithy_types::body::SdkBody; +use aws_smithy_types::checksum_config::RequestChecksumCalculation; use aws_smithy_types::config_bag::{ConfigBag, Layer, Storable, StoreReplace}; use aws_smithy_types::error::operation::BuildError; use http::HeaderValue; use http_body::Body; +use std::str::FromStr; use std::{fmt, mem}; +use crate::presigning::PresigningMarker; + /// Errors related to constructing checksum-validated HTTP requests #[derive(Debug)] pub(crate) enum Error { @@ -52,9 +58,12 @@ impl fmt::Display for Error { impl std::error::Error for Error {} -#[derive(Debug)] +#[derive(Debug, Clone)] struct RequestChecksumInterceptorState { - checksum_algorithm: Option, + /// The checksum algorithm to calculate + checksum_algorithm: Option, + /// This value is set in the model on the `httpChecksum` trait + request_checksum_required: bool, } impl Storable for RequestChecksumInterceptorState { type Storer = StoreReplace; @@ -99,40 +108,49 @@ impl DefaultRequestChecksumOverride { } } -pub(crate) struct RequestChecksumInterceptor { +pub(crate) struct RequestChecksumInterceptor { algorithm_provider: AP, + checksum_mutator: CM, } -impl fmt::Debug for RequestChecksumInterceptor { +impl fmt::Debug for RequestChecksumInterceptor { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { f.debug_struct("RequestChecksumInterceptor").finish() } } -impl RequestChecksumInterceptor { - pub(crate) fn new(algorithm_provider: AP) -> Self { - Self { algorithm_provider } +impl RequestChecksumInterceptor { + pub(crate) fn new(algorithm_provider: AP, checksum_mutator: CM) -> Self { + Self { + algorithm_provider, + checksum_mutator, + } } } -impl Intercept for RequestChecksumInterceptor +impl Intercept for RequestChecksumInterceptor where - AP: Fn(&Input) -> Result, BoxError> + Send + Sync, + AP: Fn(&Input) -> (Option, bool) + Send + Sync, + CM: Fn(&mut Request, &ConfigBag) -> Result + Send + Sync, { fn name(&self) -> &'static str { "RequestChecksumInterceptor" } - fn read_before_serialization( + fn modify_before_serialization( &self, - context: &BeforeSerializationInterceptorContextRef<'_>, + context: &mut BeforeSerializationInterceptorContextMut<'_>, _runtime_components: &RuntimeComponents, cfg: &mut ConfigBag, ) -> Result<(), BoxError> { - let checksum_algorithm = (self.algorithm_provider)(context.input())?; + let (checksum_algorithm, request_checksum_required) = + (self.algorithm_provider)(context.input()); let mut layer = Layer::new("RequestChecksumInterceptor"); - layer.store_put(RequestChecksumInterceptorState { checksum_algorithm }); + layer.store_put(RequestChecksumInterceptorState { + checksum_algorithm, + request_checksum_required, + }); cfg.push_layer(layer); Ok(()) @@ -141,7 +159,7 @@ where /// Calculate a checksum and modify the request to include the checksum as a header /// (for in-memory request bodies) or a trailer (for streaming request bodies). /// Streaming bodies must be sized or this will return an error. - fn modify_before_signing( + fn modify_before_retry_loop( &self, context: &mut BeforeTransmitInterceptorContextMut<'_>, _runtime_components: &RuntimeComponents, @@ -151,14 +169,119 @@ where .load::() .expect("set in `read_before_serialization`"); - let checksum_algorithm = incorporate_custom_default(state.checksum_algorithm, cfg); - if let Some(checksum_algorithm) = checksum_algorithm { + let user_set_checksum_value = (self.checksum_mutator)(context.request_mut(), cfg) + .expect("Checksum header mutation should not fail"); + + // If the user manually set a checksum header we short circuit + if user_set_checksum_value { + return Ok(()); + } + + // This value is from the trait, but is needed for runtime logic + let request_checksum_required = state.request_checksum_required; + + // If the algorithm fails to parse it is not one we support and we error + let checksum_algorithm = state + .checksum_algorithm + .clone() + .map(|s| ChecksumAlgorithm::from_str(s.as_str())) + .transpose()?; + + // This value is set by the user on the SdkConfig to indicate their preference + // We provide a default here for users that use a client config instead of the SdkConfig + let request_checksum_calculation = cfg + .load::() + .unwrap_or(&RequestChecksumCalculation::WhenSupported); + + // Determine if we actually calculate the checksum. If the user setting is WhenSupported (the default) + // we always calculate it (because this interceptor isn't added if it isn't supported). If it is + // WhenRequired we only calculate it if the checksum is marked required on the trait. + let calculate_checksum = match request_checksum_calculation { + RequestChecksumCalculation::WhenRequired => request_checksum_required, + RequestChecksumCalculation::WhenSupported => true, + _ => true, + }; + + // Calculate the checksum if necessary + if calculate_checksum { + let is_presigned_req = cfg.load::().is_some(); + + // If this is a presigned request and the user has not set a checksum we short circuit + if is_presigned_req && checksum_algorithm.is_none() { + return Ok(()); + } + + // If a checksum override is set in the ConfigBag we use that instead (currently only used by S3Express) + // If we have made it this far without a checksum being set we set the default (currently Crc32) + let checksum_algorithm = + incorporate_custom_default(checksum_algorithm, cfg).unwrap_or_default(); + + // Set the user-agent metric for the selected checksum algorithm + match checksum_algorithm { + ChecksumAlgorithm::Crc32 => { + cfg.interceptor_state() + .store_append(SmithySdkFeature::FlexibleChecksumsReqCrc32); + } + ChecksumAlgorithm::Crc32c => { + cfg.interceptor_state() + .store_append(SmithySdkFeature::FlexibleChecksumsReqCrc32c); + } + ChecksumAlgorithm::Crc64Nvme => { + cfg.interceptor_state() + .store_append(SmithySdkFeature::FlexibleChecksumsReqCrc64); + } + #[allow(deprecated)] + ChecksumAlgorithm::Md5 => { + tracing::warn!(more_info = "Unsupported ChecksumAlgorithm MD5 set"); + } + ChecksumAlgorithm::Sha1 => { + cfg.interceptor_state() + .store_append(SmithySdkFeature::FlexibleChecksumsReqSha1); + } + ChecksumAlgorithm::Sha256 => { + cfg.interceptor_state() + .store_append(SmithySdkFeature::FlexibleChecksumsReqSha256); + } + unsupported => tracing::warn!( + more_info = "Unsupported value of ChecksumAlgorithm detected when setting user-agent metrics", + unsupported = ?unsupported), + } + let request = context.request_mut(); add_checksum_for_request_body(request, checksum_algorithm, cfg)?; } Ok(()) } + + /// Set the user-agent metrics for `RequestChecksumCalculation` here to avoid ownership issues + /// with the mutable borrow of cfg in `modify_before_signing` + fn read_after_serialization( + &self, + _context: &aws_smithy_runtime_api::client::interceptors::context::BeforeTransmitInterceptorContextRef<'_>, + _runtime_components: &RuntimeComponents, + cfg: &mut ConfigBag, + ) -> Result<(), BoxError> { + let request_checksum_calculation = cfg + .load::() + .unwrap_or(&RequestChecksumCalculation::WhenSupported); + + match request_checksum_calculation { + RequestChecksumCalculation::WhenSupported => { + cfg.interceptor_state() + .store_append(SmithySdkFeature::FlexibleChecksumsReqWhenSupported); + } + RequestChecksumCalculation::WhenRequired => { + cfg.interceptor_state() + .store_append(SmithySdkFeature::FlexibleChecksumsReqWhenRequired); + } + unsupported => tracing::warn!( + more_info = "Unsupported value of RequestChecksumCalculation when setting user-agent metrics", + unsupported = ?unsupported), + }; + + Ok(()) + } } fn incorporate_custom_default( @@ -179,13 +302,18 @@ fn add_checksum_for_request_body( match request.body().bytes() { // Body is in-memory: read it and insert the checksum as a header. Some(data) => { - tracing::debug!("applying {checksum_algorithm:?} of the request body as a header"); let mut checksum = checksum_algorithm.into_impl(); - checksum.update(data); - request - .headers_mut() - .insert(checksum.header_name(), checksum.header_value()); + // If the header has not already been set we set it. If it was already set by the user + // we do nothing and maintain their set value. + if request.headers().get(checksum.header_name()).is_none() { + tracing::debug!("applying {checksum_algorithm:?} of the request body as a header"); + checksum.update(data); + + request + .headers_mut() + .insert(checksum.header_name(), checksum.header_value()); + } } // Body is streaming: wrap the body so it will emit a checksum as a trailer. None => { @@ -202,6 +330,13 @@ fn wrap_streaming_request_body_in_checksum_calculating_body( request: &mut HttpRequest, checksum_algorithm: ChecksumAlgorithm, ) -> Result<(), BuildError> { + let checksum = checksum_algorithm.into_impl(); + + // If the user already set the header value then do nothing and return early + if request.headers().get(checksum.header_name()).is_some() { + return Ok(()); + } + let original_body_size = request .body() .size_hint() @@ -233,7 +368,7 @@ fn wrap_streaming_request_body_in_checksum_calculating_body( headers.insert( http::header::HeaderName::from_static("x-amz-trailer"), - checksum_algorithm.into_impl().header_name(), + checksum.header_name(), ); headers.insert( diff --git a/aws/rust-runtime/aws-inlineable/src/http_response_checksum.rs b/aws/rust-runtime/aws-inlineable/src/http_response_checksum.rs index 1236c9fe5006400804f327809d7b33140b1f3c0c..d28eca921215b2ad33ebe077badafe555f40f437 100644 --- a/aws/rust-runtime/aws-inlineable/src/http_response_checksum.rs +++ b/aws/rust-runtime/aws-inlineable/src/http_response_checksum.rs @@ -8,14 +8,16 @@ //! Interceptor for handling Smithy `@httpChecksum` response checksumming use aws_smithy_checksums::ChecksumAlgorithm; +use aws_smithy_runtime::client::sdk_feature::SmithySdkFeature; use aws_smithy_runtime_api::box_error::BoxError; use aws_smithy_runtime_api::client::interceptors::context::{ - BeforeDeserializationInterceptorContextMut, BeforeSerializationInterceptorContextRef, Input, + BeforeDeserializationInterceptorContextMut, BeforeSerializationInterceptorContextMut, Input, }; use aws_smithy_runtime_api::client::interceptors::Intercept; use aws_smithy_runtime_api::client::runtime_components::RuntimeComponents; use aws_smithy_runtime_api::http::Headers; use aws_smithy_types::body::SdkBody; +use aws_smithy_types::checksum_config::ResponseChecksumValidation; use aws_smithy_types::config_bag::{ConfigBag, Layer, Storable, StoreReplace}; use std::{fmt, mem}; @@ -27,12 +29,13 @@ impl Storable for ResponseChecksumInterceptorState { type Storer = StoreReplace; } -pub(crate) struct ResponseChecksumInterceptor { +pub(crate) struct ResponseChecksumInterceptor { response_algorithms: &'static [&'static str], validation_enabled: VE, + checksum_mutator: CM, } -impl fmt::Debug for ResponseChecksumInterceptor { +impl fmt::Debug for ResponseChecksumInterceptor { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { f.debug_struct("ResponseChecksumInterceptor") .field("response_algorithms", &self.response_algorithms) @@ -40,38 +43,61 @@ impl fmt::Debug for ResponseChecksumInterceptor { } } -impl ResponseChecksumInterceptor { +impl ResponseChecksumInterceptor { pub(crate) fn new( response_algorithms: &'static [&'static str], validation_enabled: VE, + checksum_mutator: CM, ) -> Self { Self { response_algorithms, validation_enabled, + checksum_mutator, } } } -impl Intercept for ResponseChecksumInterceptor +impl Intercept for ResponseChecksumInterceptor where VE: Fn(&Input) -> bool + Send + Sync, + CM: Fn(&mut Input, &ConfigBag) -> Result<(), BoxError> + Send + Sync, { fn name(&self) -> &'static str { "ResponseChecksumInterceptor" } - fn read_before_serialization( + fn modify_before_serialization( &self, - context: &BeforeSerializationInterceptorContextRef<'_>, + context: &mut BeforeSerializationInterceptorContextMut<'_>, _runtime_components: &RuntimeComponents, cfg: &mut ConfigBag, ) -> Result<(), BoxError> { + (self.checksum_mutator)(context.input_mut(), cfg)?; let validation_enabled = (self.validation_enabled)(context.input()); let mut layer = Layer::new("ResponseChecksumInterceptor"); layer.store_put(ResponseChecksumInterceptorState { validation_enabled }); cfg.push_layer(layer); + let response_checksum_validation = cfg + .load::() + .unwrap_or(&ResponseChecksumValidation::WhenSupported); + + // Set the user-agent feature metric for the response checksum config + match response_checksum_validation { + ResponseChecksumValidation::WhenSupported => { + cfg.interceptor_state() + .store_append(SmithySdkFeature::FlexibleChecksumsResWhenSupported); + } + ResponseChecksumValidation::WhenRequired => { + cfg.interceptor_state() + .store_append(SmithySdkFeature::FlexibleChecksumsResWhenRequired); + } + unsupported => tracing::warn!( + more_info = "Unsupported value of ResponseChecksumValidation when setting user-agent metrics", + unsupported = ?unsupported), + }; + Ok(()) } @@ -85,12 +111,33 @@ where .load::() .expect("set in `read_before_serialization`"); - if state.validation_enabled { + // This value is set by the user on the SdkConfig to indicate their preference + // We provide a default here for users that use a client config instead of the SdkConfig + let response_checksum_validation = cfg + .load::() + .unwrap_or(&ResponseChecksumValidation::WhenSupported); + + // If validation has not been explicitly enabled we check the ResponseChecksumValidation + // from the SdkConfig. If it is WhenSupported (or unknown) we enable validation and if it + // is WhenRequired we leave it disabled since there is no way to indicate that a response + // checksum is required. + let validation_enabled = if !state.validation_enabled { + match response_checksum_validation { + ResponseChecksumValidation::WhenRequired => false, + ResponseChecksumValidation::WhenSupported => true, + _ => true, + } + } else { + true + }; + + if validation_enabled { let response = context.response_mut(); let maybe_checksum_headers = check_headers_for_precalculated_checksum( response.headers(), self.response_algorithms, ); + if let Some((checksum_algorithm, precalculated_checksum)) = maybe_checksum_headers { let mut body = SdkBody::taken(); mem::swap(&mut body, response.body_mut()); @@ -195,8 +242,8 @@ fn is_part_level_checksum(checksum: &str) -> bool { continue; } - // Yup, it's a part-level checksum - if ch == '-' { + // We saw a number first followed by the dash, yup, it's a part-level checksum + if found_number && ch == '-' { if found_dash { // Found a second dash?? This isn't a part-level checksum. return false; @@ -267,4 +314,16 @@ mod tests { assert!(!is_part_level_checksum("abcd==--11")); assert!(!is_part_level_checksum("abcd==-AA")); } + + #[test] + fn part_level_checksum_detection_works() { + let a_real_checksum = is_part_level_checksum("C9A5A6878D97B48CC965C1E41859F034-14"); + assert!(a_real_checksum); + let close_but_not_quite = is_part_level_checksum("a4-"); + assert!(!close_but_not_quite); + let backwards = is_part_level_checksum("14-C9A5A6878D97B48CC965C1E41859F034"); + assert!(!backwards); + let double_dash = is_part_level_checksum("C9A5A6878D97B48CC965C1E41859F03-4-14"); + assert!(!double_dash); + } } diff --git a/aws/rust-runtime/aws-inlineable/src/presigning.rs b/aws/rust-runtime/aws-inlineable/src/presigning.rs index c19d160e18d0920294e83fbfa5ffb49dc3698c92..c30830768505875b03e1ab4e6523f7ed360f9e13 100644 --- a/aws/rust-runtime/aws-inlineable/src/presigning.rs +++ b/aws/rust-runtime/aws-inlineable/src/presigning.rs @@ -15,6 +15,7 @@ use aws_smithy_runtime_api::box_error::BoxError; use aws_smithy_runtime_api::client::orchestrator::HttpRequest; use aws_smithy_types::body::SdkBody; +use aws_smithy_types::config_bag::{Storable, StoreReplace}; use std::fmt; use std::time::{Duration, SystemTime}; @@ -265,3 +266,12 @@ impl fmt::Debug for PresignedRequest { .finish() } } + +/// A marker struct to be stored in the ConfigBag allowing other interceptors to know that +/// the current request is Presigned +#[derive(Debug)] +pub(crate) struct PresigningMarker; + +impl Storable for PresigningMarker { + type Storer = StoreReplace; +} diff --git a/aws/rust-runtime/aws-inlineable/src/presigning_interceptors.rs b/aws/rust-runtime/aws-inlineable/src/presigning_interceptors.rs index 481a8d9cab4017d8a22f8b5fd996b70473d9b268..25c674feb2680948b1fd9770d4466959f42ab724 100644 --- a/aws/rust-runtime/aws-inlineable/src/presigning_interceptors.rs +++ b/aws/rust-runtime/aws-inlineable/src/presigning_interceptors.rs @@ -5,7 +5,7 @@ #![allow(dead_code)] -use crate::presigning::PresigningConfig; +use crate::presigning::{PresigningConfig, PresigningMarker}; use crate::serialization_settings::HeaderSerializationSettings; use aws_runtime::auth::{HttpSignatureType, SigV4OperationSigningConfig}; use aws_runtime::invocation_id::InvocationIdInterceptor; @@ -63,6 +63,8 @@ impl Intercept for SigV4PresigningInterceptor { .omit_default_content_length() .omit_default_content_type(), ); + + cfg.interceptor_state().store_put(PresigningMarker); Ok(()) } diff --git a/aws/rust-runtime/aws-runtime/Cargo.toml b/aws/rust-runtime/aws-runtime/Cargo.toml index a30478ac5c0dd9fb48ac8a84ade697239f117d0f..9695833ccbc09999777324a5c4ee950704eee100 100644 --- a/aws/rust-runtime/aws-runtime/Cargo.toml +++ b/aws/rust-runtime/aws-runtime/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "aws-runtime" -version = "1.5.3" +version = "1.5.4" authors = ["AWS Rust SDK Team "] description = "Runtime support code for the AWS SDK. This crate isn't intended to be used directly." edition = "2021" diff --git a/aws/rust-runtime/aws-runtime/src/user_agent/test_util.rs b/aws/rust-runtime/aws-runtime/src/user_agent/test_util.rs index a0a5679b63b3167587af23c2b3b2d452dd1a4f7b..de8013454feaab172bad6fa02738fb18bb311e56 100644 --- a/aws/rust-runtime/aws-runtime/src/user_agent/test_util.rs +++ b/aws/rust-runtime/aws-runtime/src/user_agent/test_util.rs @@ -17,13 +17,8 @@ static RE: Lazy = Lazy::new(|| Regex::new(r"m/([A-Za-z0-9+/=_,-]+)").unwr /// Refer to the end of the parent module file `user_agent.rs` for the complete ABNF specification /// of `business-metrics`. pub fn assert_ua_contains_metric_values(user_agent: &str, values: &[&str]) { - match RE.find(user_agent) { - Some(matched) => { - let csv = matched - .as_str() - .strip_prefix("m/") - .expect("prefix `m/` is guaranteed to exist by regex match"); - let metrics: Vec<&str> = csv.split(',').collect(); + match extract_ua_values(user_agent) { + Some(metrics) => { let mut missed = vec![]; for value in values.iter() { @@ -43,6 +38,18 @@ pub fn assert_ua_contains_metric_values(user_agent: &str, values: &[&str]) { } } +/// Extract the metric values from the `user_agent` string +pub fn extract_ua_values(user_agent: &str) -> Option> { + RE.find(user_agent).map(|matched| { + matched + .as_str() + .strip_prefix("m/") + .expect("prefix `m/` is guaranteed to exist by regex match") + .split(',') + .collect() + }) +} + #[cfg(test)] mod tests { use super::*; diff --git a/aws/rust-runtime/aws-sigv4/Cargo.toml b/aws/rust-runtime/aws-sigv4/Cargo.toml index cc80e159089806bb82f7e5f6f3d18069686f68bb..5bda4ee0414999451dfa926988ec539e2ca1224c 100644 --- a/aws/rust-runtime/aws-sigv4/Cargo.toml +++ b/aws/rust-runtime/aws-sigv4/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "aws-sigv4" -version = "1.2.6" +version = "1.2.7" authors = ["AWS Rust SDK Team ", "David Barsky "] description = "SigV4 signer for HTTP requests and Event Stream messages." edition = "2021" @@ -45,7 +45,8 @@ aws-smithy-runtime-api = { path = "../../../rust-runtime/aws-smithy-runtime-api" bytes = "1" hex-literal = "0.4.1" httparse = "1.8" -libfuzzer-sys = "0.4.6" +# Version pinned due to https://github.com/rust-fuzz/libfuzzer/issues/126 +libfuzzer-sys = "=0.4.7" pretty_assertions = "1.3" proptest = "1.2" serde = "1.0.180" diff --git a/aws/rust-runtime/aws-sigv4/src/http_request/canonical_request.rs b/aws/rust-runtime/aws-sigv4/src/http_request/canonical_request.rs index 03404276d9575432b702862df6f60efe96517536..4622f841d985fd9db5b54528915b3fb2cf5d59a4 100644 --- a/aws/rust-runtime/aws-sigv4/src/http_request/canonical_request.rs +++ b/aws/rust-runtime/aws-sigv4/src/http_request/canonical_request.rs @@ -31,6 +31,7 @@ pub(crate) mod header { pub(crate) const X_AMZ_DATE: &str = "x-amz-date"; pub(crate) const X_AMZ_SECURITY_TOKEN: &str = "x-amz-security-token"; pub(crate) const X_AMZ_USER_AGENT: &str = "x-amz-user-agent"; + pub(crate) const X_AMZ_CHECKSUM_MODE: &str = "x-amz-checksum-mode"; } pub(crate) mod param { @@ -293,8 +294,10 @@ impl<'a> CanonicalRequest<'a> { } if params.settings().signature_location == SignatureLocation::QueryParams { - // The X-Amz-User-Agent header should not be signed if this is for a presigned URL - if name == HeaderName::from_static(header::X_AMZ_USER_AGENT) { + // The X-Amz-User-Agent and x-amz-checksum-mode headers should not be signed if this is for a presigned URL + if name == HeaderName::from_static(header::X_AMZ_USER_AGENT) + || name == HeaderName::from_static(header::X_AMZ_CHECKSUM_MODE) + { continue; } } diff --git a/aws/rust-runtime/aws-types/Cargo.toml b/aws/rust-runtime/aws-types/Cargo.toml index 694b8676a50355ebafe0bbd79efd2d5a4cc3d552..e13428e4bafb3b42865cfdd1b29dbc8b4005be7f 100644 --- a/aws/rust-runtime/aws-types/Cargo.toml +++ b/aws/rust-runtime/aws-types/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "aws-types" -version = "1.3.3" +version = "1.3.4" authors = ["AWS Rust SDK Team ", "Russell Cohen "] description = "Cross-service types for the AWS SDK." edition = "2021" diff --git a/aws/rust-runtime/aws-types/external-types.toml b/aws/rust-runtime/aws-types/external-types.toml index 5dce405f84148df7809a0670e907fc1acf98dc93..ee28a1732a59416b13086a835fffc9b4d8b4d0e1 100644 --- a/aws/rust-runtime/aws-types/external-types.toml +++ b/aws/rust-runtime/aws-types/external-types.toml @@ -13,6 +13,8 @@ allowed_external_types = [ "aws_smithy_runtime_api::client::identity::SharedIdentityCache", "aws_smithy_runtime_api::client::stalled_stream_protection::StalledStreamProtectionConfig", "aws_smithy_runtime_api::http::headers::Headers", + "aws_smithy_types::checksum_config::ResponseChecksumValidation", + "aws_smithy_types::checksum_config::RequestChecksumCalculation", "aws_smithy_types::config_bag::storable::Storable", "aws_smithy_types::config_bag::storable::StoreReplace", "aws_smithy_types::config_bag::storable::Storer", diff --git a/aws/rust-runtime/aws-types/src/sdk_config.rs b/aws/rust-runtime/aws-types/src/sdk_config.rs index c26d8741be8ed54082d82c4992e65bf6a02e6939..19220fc526e27479370b645e3f55908feb498aca 100644 --- a/aws/rust-runtime/aws-types/src/sdk_config.rs +++ b/aws/rust-runtime/aws-types/src/sdk_config.rs @@ -25,6 +25,9 @@ pub use aws_smithy_runtime_api::client::http::SharedHttpClient; use aws_smithy_runtime_api::client::identity::{ResolveCachedIdentity, SharedIdentityCache}; pub use aws_smithy_runtime_api::client::stalled_stream_protection::StalledStreamProtectionConfig; use aws_smithy_runtime_api::shared::IntoShared; +pub use aws_smithy_types::checksum_config::{ + RequestChecksumCalculation, ResponseChecksumValidation, +}; pub use aws_smithy_types::retry::RetryConfig; pub use aws_smithy_types::timeout::TimeoutConfig; use std::collections::HashMap; @@ -92,6 +95,8 @@ pub struct SdkConfig { config_origins: HashMap<&'static str, Origin>, disable_request_compression: Option, request_min_compression_size_bytes: Option, + request_checksum_calculation: Option, + response_checksum_validation: Option, } /// Builder for AWS Shared Configuration @@ -120,6 +125,8 @@ pub struct Builder { config_origins: HashMap<&'static str, Origin>, disable_request_compression: Option, request_min_compression_size_bytes: Option, + request_checksum_calculation: Option, + response_checksum_validation: Option, } impl Builder { @@ -174,6 +181,54 @@ impl Builder { self } + /// Set the checksum calculation strategy to use when making requests. + /// # Examples + /// ``` + /// use aws_types::SdkConfig; + /// use aws_smithy_types::checksum_config::RequestChecksumCalculation; + /// let config = SdkConfig::builder().request_checksum_calculation(RequestChecksumCalculation::WhenSupported).build(); + /// ``` + pub fn request_checksum_calculation( + mut self, + request_checksum_calculation: RequestChecksumCalculation, + ) -> Self { + self.set_request_checksum_calculation(Some(request_checksum_calculation)); + self + } + + /// Set the checksum calculation strategy to use when making requests. + pub fn set_request_checksum_calculation( + &mut self, + request_checksum_calculation: Option, + ) -> &mut Self { + self.request_checksum_calculation = request_checksum_calculation; + self + } + + /// Set the checksum calculation strategy to use for responses. + /// # Examples + /// ``` + /// use aws_types::SdkConfig; + /// use aws_smithy_types::checksum_config::ResponseChecksumValidation; + /// let config = SdkConfig::builder().response_checksum_validation(ResponseChecksumValidation::WhenSupported).build(); + /// ``` + pub fn response_checksum_validation( + mut self, + response_checksum_validation: ResponseChecksumValidation, + ) -> Self { + self.set_response_checksum_validation(Some(response_checksum_validation)); + self + } + + /// Set the checksum calculation strategy to use for responses. + pub fn set_response_checksum_validation( + &mut self, + response_checksum_validation: Option, + ) -> &mut Self { + self.response_checksum_validation = response_checksum_validation; + self + } + /// Set the retry_config for the builder /// /// _Note:_ Retries require a sleep implementation in order to work. When enabling retry, make @@ -720,6 +775,8 @@ impl Builder { config_origins: self.config_origins, disable_request_compression: self.disable_request_compression, request_min_compression_size_bytes: self.request_min_compression_size_bytes, + request_checksum_calculation: self.request_checksum_calculation, + response_checksum_validation: self.response_checksum_validation, } } } @@ -866,6 +923,16 @@ impl SdkConfig { self.disable_request_compression } + /// Configured checksum request behavior. + pub fn request_checksum_calculation(&self) -> Option { + self.request_checksum_calculation + } + + /// Configured checksum response behavior. + pub fn response_checksum_validation(&self) -> Option { + self.response_checksum_validation + } + /// Configured minimum request compression size. pub fn request_min_compression_size_bytes(&self) -> Option { self.request_min_compression_size_bytes @@ -933,6 +1000,8 @@ impl SdkConfig { config_origins: self.config_origins, disable_request_compression: self.disable_request_compression, request_min_compression_size_bytes: self.request_min_compression_size_bytes, + request_checksum_calculation: self.request_checksum_calculation, + response_checksum_validation: self.response_checksum_validation, } } } diff --git a/aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/HttpRequestChecksumDecorator.kt b/aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/HttpRequestChecksumDecorator.kt index da5c2ac7bfd2abb23fc0f84e5de330ea24ebfee0..f786b8194378f7a71f0a8f3570f196bc447069ea 100644 --- a/aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/HttpRequestChecksumDecorator.kt +++ b/aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/HttpRequestChecksumDecorator.kt @@ -6,11 +6,17 @@ package software.amazon.smithy.rustsdk import software.amazon.smithy.aws.traits.HttpChecksumTrait +import software.amazon.smithy.model.knowledge.TopDownIndex +import software.amazon.smithy.model.shapes.MemberShape import software.amazon.smithy.model.shapes.OperationShape +import software.amazon.smithy.model.traits.HttpHeaderTrait import software.amazon.smithy.rust.codegen.client.smithy.ClientCodegenContext +import software.amazon.smithy.rust.codegen.client.smithy.configReexport import software.amazon.smithy.rust.codegen.client.smithy.customize.ClientCodegenDecorator import software.amazon.smithy.rust.codegen.client.smithy.generators.OperationCustomization import software.amazon.smithy.rust.codegen.client.smithy.generators.OperationSection +import software.amazon.smithy.rust.codegen.client.smithy.generators.config.ConfigCustomization +import software.amazon.smithy.rust.codegen.client.smithy.generators.config.ServiceConfig import software.amazon.smithy.rust.codegen.core.rustlang.CargoDependency import software.amazon.smithy.rust.codegen.core.rustlang.Visibility import software.amazon.smithy.rust.codegen.core.rustlang.Writable @@ -20,9 +26,13 @@ import software.amazon.smithy.rust.codegen.core.rustlang.writable import software.amazon.smithy.rust.codegen.core.smithy.RuntimeConfig import software.amazon.smithy.rust.codegen.core.smithy.RuntimeType import software.amazon.smithy.rust.codegen.core.smithy.RuntimeType.Companion.preludeScope -import software.amazon.smithy.rust.codegen.core.smithy.generators.operationBuildError +import software.amazon.smithy.rust.codegen.core.smithy.customize.AdHocCustomization +import software.amazon.smithy.rust.codegen.core.smithy.customize.NamedCustomization +import software.amazon.smithy.rust.codegen.core.smithy.customize.adhocCustomization +import software.amazon.smithy.rust.codegen.core.util.dq import software.amazon.smithy.rust.codegen.core.util.expectMember import software.amazon.smithy.rust.codegen.core.util.getTrait +import software.amazon.smithy.rust.codegen.core.util.hasTrait import software.amazon.smithy.rust.codegen.core.util.inputShape import software.amazon.smithy.rust.codegen.core.util.orNull @@ -39,21 +49,49 @@ internal fun RuntimeConfig.awsInlineableHttpRequestChecksum() = CargoDependency.smithyHttp(this), CargoDependency.smithyRuntimeApiClient(this), CargoDependency.smithyTypes(this), - CargoDependency.TempFile, + AwsCargoDependency.awsSigv4(this), + CargoDependency.TempFile.toDevDependency(), ), ) class HttpRequestChecksumDecorator : ClientCodegenDecorator { override val name: String = "HttpRequestChecksum" override val order: Byte = 0 + private val defaultAlgorithm = "CRC32" override fun operationCustomizations( codegenContext: ClientCodegenContext, operation: OperationShape, baseCustomizations: List, ): List = baseCustomizations + HttpRequestChecksumCustomization(codegenContext, operation) + + override fun configCustomizations( + codegenContext: ClientCodegenContext, + baseCustomizations: List, + ): List = baseCustomizations + HttpRequestChecksumConfigCustomization(codegenContext) + + /** + * Copy the `request_checksum_calculation` value from the `SdkConfig` to the client config + */ + override fun extraSections(codegenContext: ClientCodegenContext): List = + if (!serviceHasHttpChecksumOperation(codegenContext)) { + listOf() + } else { + listOf( + adhocCustomization { section -> + rust( + """ + ${section.serviceConfigBuilder}.set_request_checksum_calculation(${section.sdkConfig}.request_checksum_calculation()); + """, + ) + }, + ) + } } +/** + * Extract the name of the operation's input member that indicates which checksum algorithm to use + */ private fun HttpChecksumTrait.requestAlgorithmMember( codegenContext: ClientCodegenContext, operationShape: OperationShape, @@ -65,6 +103,9 @@ private fun HttpChecksumTrait.requestAlgorithmMember( return codegenContext.symbolProvider.toMemberName(checksumAlgorithmMemberShape) } +/** + * Extract the name of the operation's input member that indicates which checksum algorithm to use + */ private fun HttpChecksumTrait.checksumAlgorithmToStr( codegenContext: ClientCodegenContext, operationShape: OperationShape, @@ -74,33 +115,14 @@ private fun HttpChecksumTrait.checksumAlgorithmToStr( val isRequestChecksumRequired = this.isRequestChecksumRequired return { - if (requestAlgorithmMember != null) { - if (isRequestChecksumRequired) { - // Checksums are required, fall back to MD5 - rust("""let checksum_algorithm = checksum_algorithm.map(|algorithm| algorithm.as_str()).or(Some("md5"));""") - } else { - // Checksums aren't required, don't set a fallback - rust("let checksum_algorithm = checksum_algorithm.map(|algorithm| algorithm.as_str());") - } - } else if (isRequestChecksumRequired) { - // Checksums are required but a user can't set one, so we set MD5 for them - rust("""let checksum_algorithm = Some("md5");""") + if (requestAlgorithmMember == null && isRequestChecksumRequired) { + // Checksums are required but a user can't set one, so we set crc32 for them + rust("""let checksum_algorithm = Some("crc32");""") + } else { + // Use checksum algo set by user or crc32 if one has not been set + rust("""let checksum_algorithm = checksum_algorithm.map(|algorithm| algorithm.as_str()).or(Some("crc32"));""") } - rustTemplate( - """ - let checksum_algorithm = match checksum_algorithm { - Some(algo) => Some( - algo.parse::<#{ChecksumAlgorithm}>() - .map_err(#{BuildError}::other)? - ), - None => None, - }; - """, - "BuildError" to runtimeConfig.operationBuildError(), - "ChecksumAlgorithm" to RuntimeType.smithyChecksums(runtimeConfig).resolve("ChecksumAlgorithm"), - ) - // If a request checksum is not required and there's no way to set one, do nothing // This happens when an operation only supports response checksums } @@ -108,6 +130,12 @@ private fun HttpChecksumTrait.checksumAlgorithmToStr( // This generator was implemented based on this spec: // https://smithy.io/2.0/aws/aws-core.html#http-request-checksums + +/** + * Calculate the checksum algorithm based on the input member identified by the trait's + * `requestAlgorithmMember`. Then instantiate an (inlineable) `http_request_checksum` + * interceptor with that checksum algorithm. + */ class HttpRequestChecksumCustomization( private val codegenContext: ClientCodegenContext, private val operationShape: OperationShape, @@ -117,28 +145,94 @@ class HttpRequestChecksumCustomization( override fun section(section: OperationSection): Writable = writable { // Get the `HttpChecksumTrait`, returning early if this `OperationShape` doesn't have one + // if there is no request algorithm member or if there is no httpHeader for the request algorithm val checksumTrait = operationShape.getTrait() ?: return@writable - val requestAlgorithmMember = checksumTrait.requestAlgorithmMember(codegenContext, operationShape) + val requestAlgorithmMember = + checksumTrait.requestAlgorithmMemberShape(codegenContext, operationShape) ?: return@writable + val requestAlgoHeader = + requestAlgorithmMember.getTrait()?.value ?: return@writable + val requestAlgorithmMemberName = checksumTrait.requestAlgorithmMember(codegenContext, operationShape) val inputShape = codegenContext.model.expectShape(operationShape.inputShape) + val requestChecksumRequired = checksumTrait.isRequestChecksumRequired + val operationName = codegenContext.symbolProvider.toSymbol(operationShape).name + val requestAlgorithmMemberInner = + if (requestAlgorithmMember.isOptional) { + codegenContext.model.expectShape(requestAlgorithmMember.target) + } else { + requestAlgorithmMember + } when (section) { is OperationSection.AdditionalInterceptors -> { - if (requestAlgorithmMember != null) { + if (requestAlgorithmMemberName != null) { section.registerInterceptor(runtimeConfig, this) { val runtimeApi = RuntimeType.smithyRuntimeApiClient(runtimeConfig) rustTemplate( """ - #{RequestChecksumInterceptor}::new(|input: &#{Input}| { + #{RequestChecksumInterceptor}::new( + |input: &#{Input}| { let input: &#{OperationInput} = input.downcast_ref().expect("correct type"); - let checksum_algorithm = input.$requestAlgorithmMember(); + let checksum_algorithm = input.$requestAlgorithmMemberName(); #{checksum_algorithm_to_str} - #{Result}::<_, #{BoxError}>::Ok(checksum_algorithm) - }) + (checksum_algorithm.map(|s| s.to_string()), $requestChecksumRequired) + }, + |request: &mut #{Request}, cfg: &#{ConfigBag}| { + // We check if the user has set any of the checksum values manually + let mut user_set_checksum_value = false; + let headers_to_check = request.headers().iter().filter_map(|(name, _val)| { + if name.starts_with("x-amz-checksum-") { + Some(name) + } else { + None + } + }); + for algo_header in headers_to_check { + if request.headers().get(algo_header).is_some() { + user_set_checksum_value = true; + } + } + + // We check if the user set the checksum algo manually + let user_set_checksum_algo = request.headers() + .get(${requestAlgoHeader.dq()}) + .is_some(); + + // This value is set by the user on the SdkConfig to indicate their preference + let request_checksum_calculation = cfg + .load::<#{RequestChecksumCalculation}>() + .unwrap_or(&#{RequestChecksumCalculation}::WhenSupported); + + // From the httpChecksum trait + let http_checksum_required = $requestChecksumRequired; + + // If the RequestChecksumCalculation is WhenSupported and the user has not set a checksum value or algo + // we default to Crc32. If it is WhenRequired and a checksum is required by the trait and the user has not + // set a checksum value or algo we also set the default. In all other cases we do nothing. + match ( + request_checksum_calculation, + http_checksum_required, + user_set_checksum_value, + user_set_checksum_algo + ) { + (#{RequestChecksumCalculation}::WhenSupported, _, false, false) + | (#{RequestChecksumCalculation}::WhenRequired, true, false, false) => { + request.headers_mut().insert(${requestAlgoHeader.dq()}, "CRC32"); + } + _ => {}, + } + + // We return a bool indicating if the user did set the checksum value, if they did + // we can short circuit and exit the interceptor early. + Ok(user_set_checksum_value) + } + ) """, *preludeScope, "BoxError" to RuntimeType.boxError(runtimeConfig), "Input" to runtimeApi.resolve("client::interceptors::context::Input"), + "Request" to RuntimeType.smithyRuntimeApi(runtimeConfig).resolve("http::Request"), "OperationInput" to codegenContext.symbolProvider.toSymbol(inputShape), + "ConfigBag" to RuntimeType.configBag(runtimeConfig), "RequestChecksumInterceptor" to runtimeConfig.awsInlineableHttpRequestChecksum() .resolve("RequestChecksumInterceptor"), @@ -147,11 +241,129 @@ class HttpRequestChecksumCustomization( codegenContext, operationShape, ), + "RequestChecksumCalculation" to + CargoDependency.smithyTypes(runtimeConfig).toType() + .resolve("checksum_config::RequestChecksumCalculation"), + "ChecksumAlgoShape" to + codegenContext.symbolProvider.toSymbol( + requestAlgorithmMemberInner, + ), + "OperationInputType" to + codegenContext.symbolProvider.toSymbol( + operationShape.inputShape( + codegenContext.model, + ), + ), ) } } } - else -> { } + + else -> {} } } } + +/** + * Add a `request_checksum_calculation;` field to Service config. + */ +class HttpRequestChecksumConfigCustomization(private val codegenContext: ClientCodegenContext) : + NamedCustomization() { + private val rc = codegenContext.runtimeConfig + private val codegenScope = + arrayOf( + *preludeScope, + "RequestChecksumCalculation" to + configReexport( + RuntimeType.smithyTypes(rc) + .resolve("checksum_config::RequestChecksumCalculation"), + ), + ) + + override fun section(section: ServiceConfig): Writable { + // If the service contains no operations with the httpChecksum trait we return early + if (!serviceHasHttpChecksumOperation(codegenContext)) { + return emptySection + } + + // Otherwise we write the necessary sections to the service config + return when (section) { + is ServiceConfig.ConfigImpl -> + writable { + rustTemplate( + """ + /// Return a reference to the request_checksum_calculation value contained in this config, if any. + pub fn request_checksum_calculation(&self) -> #{Option}<&#{RequestChecksumCalculation}> { + self.config.load::<#{RequestChecksumCalculation}>() + } + """, + *codegenScope, + ) + } + + is ServiceConfig.BuilderImpl -> + writable { + rustTemplate( + """ + /// Set the [`RequestChecksumCalculation`](#{RequestChecksumCalculation}) + /// to determine when a checksum will be calculated for request payloads. + pub fn request_checksum_calculation( + mut self, + request_checksum_calculation: #{RequestChecksumCalculation} + ) -> Self { + self.set_request_checksum_calculation(#{Some}(request_checksum_calculation)); + self + } + """, + *codegenScope, + ) + + rustTemplate( + """ + /// Set the [`RequestChecksumCalculation`](#{RequestChecksumCalculation}) + /// to determine when a checksum will be calculated for request payloads. + pub fn set_request_checksum_calculation( + &mut self, + request_checksum_calculation: #{Option}<#{RequestChecksumCalculation}> + ) -> &mut Self { + self.config.store_or_unset(request_checksum_calculation); + self + } + """, + *codegenScope, + ) + } + + is ServiceConfig.BuilderFromConfigBag -> + writable { + rustTemplate( + "${section.builder}.set_request_checksum_calculation(${section.configBag}.load::<#{RequestChecksumCalculation}>().cloned());", + *codegenScope, + ) + } + + else -> emptySection + } + } +} + +/** + * Determine if the current service contains any operations with the HttpChecksum trait + */ +fun serviceHasHttpChecksumOperation(codegenContext: ClientCodegenContext): Boolean { + val index = TopDownIndex.of(codegenContext.model) + val ops = index.getContainedOperations(codegenContext.serviceShape.id) + return ops.any { it.hasTrait() } +} + +/** + * Get the top-level operation input member used to opt-in to best-effort validation of a checksum returned in + * the HTTP response of the operation. + */ +fun HttpChecksumTrait.requestAlgorithmMemberShape( + codegenContext: ClientCodegenContext, + operationShape: OperationShape, +): MemberShape? { + val requestAlgorithmMember = this.requestAlgorithmMember.orNull() ?: return null + return operationShape.inputShape(codegenContext.model).expectMember(requestAlgorithmMember) +} diff --git a/aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/HttpResponseChecksumDecorator.kt b/aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/HttpResponseChecksumDecorator.kt index 16a45216797d33f4763135725c92c738de951d69..66c1402ff466e5ba4609b5789e3704e58e81a6ec 100644 --- a/aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/HttpResponseChecksumDecorator.kt +++ b/aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/HttpResponseChecksumDecorator.kt @@ -10,17 +10,25 @@ import software.amazon.smithy.model.shapes.MemberShape import software.amazon.smithy.model.shapes.OperationShape import software.amazon.smithy.model.shapes.ShapeId import software.amazon.smithy.rust.codegen.client.smithy.ClientCodegenContext +import software.amazon.smithy.rust.codegen.client.smithy.configReexport import software.amazon.smithy.rust.codegen.client.smithy.customize.ClientCodegenDecorator import software.amazon.smithy.rust.codegen.client.smithy.generators.OperationCustomization import software.amazon.smithy.rust.codegen.client.smithy.generators.OperationSection +import software.amazon.smithy.rust.codegen.client.smithy.generators.config.ConfigCustomization +import software.amazon.smithy.rust.codegen.client.smithy.generators.config.ServiceConfig import software.amazon.smithy.rust.codegen.core.rustlang.CargoDependency import software.amazon.smithy.rust.codegen.core.rustlang.Visibility import software.amazon.smithy.rust.codegen.core.rustlang.Writable +import software.amazon.smithy.rust.codegen.core.rustlang.rust import software.amazon.smithy.rust.codegen.core.rustlang.rustTemplate import software.amazon.smithy.rust.codegen.core.rustlang.writable import software.amazon.smithy.rust.codegen.core.smithy.RuntimeConfig import software.amazon.smithy.rust.codegen.core.smithy.RuntimeType import software.amazon.smithy.rust.codegen.core.smithy.RuntimeType.Companion.preludeScope +import software.amazon.smithy.rust.codegen.core.smithy.customize.AdHocCustomization +import software.amazon.smithy.rust.codegen.core.smithy.customize.NamedCustomization +import software.amazon.smithy.rust.codegen.core.smithy.customize.adhocCustomization +import software.amazon.smithy.rust.codegen.core.util.dq import software.amazon.smithy.rust.codegen.core.util.expectMember import software.amazon.smithy.rust.codegen.core.util.getTrait import software.amazon.smithy.rust.codegen.core.util.inputShape @@ -42,6 +50,10 @@ private fun RuntimeConfig.awsInlineableHttpResponseChecksum() = ), ) +/** + * Get the top-level operation input member used to opt-in to best-effort validation of a checksum returned in + * the HTTP response of the operation. + */ fun HttpChecksumTrait.requestValidationModeMember( codegenContext: ClientCodegenContext, operationShape: OperationShape, @@ -65,19 +77,50 @@ class HttpResponseChecksumDecorator : ClientCodegenDecorator { baseCustomizations.letIf(applies(operation)) { it + HttpResponseChecksumCustomization(codegenContext, operation) } + + override fun configCustomizations( + codegenContext: ClientCodegenContext, + baseCustomizations: List, + ): List = baseCustomizations + HttpResponseChecksumConfigCustomization(codegenContext) + + /** + * Copy the `response_checksum_validation` value from the `SdkConfig` to the client config + */ + override fun extraSections(codegenContext: ClientCodegenContext): List = + if (serviceHasHttpChecksumOperation(codegenContext)) { + listOf( + adhocCustomization { section -> + rust( + """ + ${section.serviceConfigBuilder}.set_response_checksum_validation(${section.sdkConfig}.response_checksum_validation()); + """, + ) + }, + ) + } else { + listOf() + } } // This generator was implemented based on this spec: // https://smithy.io/2.0/aws/aws-core.html#http-request-checksums + +/** + * Calculate the checksum algorithm based on the input member identified by the trait's + * `requestAlgorithmMember`. Then instantiate an (inlineable) `http_request_checksum` + * interceptor with that checksum algorithm. + */ class HttpResponseChecksumCustomization( private val codegenContext: ClientCodegenContext, private val operationShape: OperationShape, ) : OperationCustomization() { override fun section(section: OperationSection): Writable = writable { + // Return early if this operation lacks the `httpChecksum` trait or that trait lacks a `requestValidationModeMember` val checksumTrait = operationShape.getTrait() ?: return@writable val requestValidationModeMember = checksumTrait.requestValidationModeMember(codegenContext, operationShape) ?: return@writable + val requestValidationModeName = codegenContext.symbolProvider.toSymbol(requestValidationModeMember).name val requestValidationModeMemberInner = if (requestValidationModeMember.isOptional) { codegenContext.model.expectShape(requestValidationModeMember.target) @@ -86,6 +129,7 @@ class HttpResponseChecksumCustomization( } val validationModeName = codegenContext.symbolProvider.toMemberName(requestValidationModeMember) val inputShape = codegenContext.model.expectShape(operationShape.inputShape) + val operationName = codegenContext.symbolProvider.toSymbol(operationShape).name when (section) { is OperationSection.AdditionalInterceptors -> { @@ -93,17 +137,48 @@ class HttpResponseChecksumCustomization( // CRC32, CRC32C, SHA256, SHA1 -> "crc32", "crc32c", "sha256", "sha1" val responseAlgorithms = checksumTrait.responseAlgorithms - .map { algorithm -> algorithm.lowercase() }.joinToString(", ") { algorithm -> "\"$algorithm\"" } + .map { algorithm -> algorithm.lowercase() } + .joinToString(", ") { algorithm -> algorithm.dq() } val runtimeApi = RuntimeType.smithyRuntimeApiClient(codegenContext.runtimeConfig) rustTemplate( """ #{ResponseChecksumInterceptor}::new( [$responseAlgorithms].as_slice(), |input: &#{Input}| { - ${""/* Per [the spec](https://smithy.io/2.0/aws/aws-core.html#http-response-checksums), - we check to see if it's the `ENABLED` variant */} + ${ + ""/* Per [the spec](https://smithy.io/2.0/aws/aws-core.html#http-response-checksums), + we check to see if it's the `ENABLED` variant */ + } let input: &#{OperationInput} = input.downcast_ref().expect("correct type"); matches!(input.$validationModeName(), #{Some}(#{ValidationModeShape}::Enabled)) + }, + |input: &mut #{Input}, cfg: &#{ConfigBag}| { + let input = input + .downcast_mut::<#{OperationInputType}>() + .ok_or("failed to downcast to #{OperationInputType}")?; + + let request_validation_enabled = + matches!(input.$requestValidationModeName(), Some(#{ValidationModeShape}::Enabled)); + + if !request_validation_enabled { + // This value is set by the user on the SdkConfig to indicate their preference + let response_checksum_validation = cfg + .load::<#{ResponseChecksumValidation}>() + .unwrap_or(&#{ResponseChecksumValidation}::WhenSupported); + + // If validation setting is WhenSupported (or unknown) we enable response checksum + // validation. If it is WhenRequired we do not enable (since there is no way to + // indicate that a response checksum is required). + ##[allow(clippy::wildcard_in_or_patterns)] + match response_checksum_validation { + #{ResponseChecksumValidation}::WhenRequired => {} + #{ResponseChecksumValidation}::WhenSupported | _ => { + input.$requestValidationModeName = Some(#{ValidationModeShape}::Enabled); + } + } + } + + #{Ok}(()) } ) """, @@ -113,7 +188,24 @@ class HttpResponseChecksumCustomization( .resolve("ResponseChecksumInterceptor"), "Input" to runtimeApi.resolve("client::interceptors::context::Input"), "OperationInput" to codegenContext.symbolProvider.toSymbol(inputShape), - "ValidationModeShape" to codegenContext.symbolProvider.toSymbol(requestValidationModeMemberInner), + "ValidationModeShape" to + codegenContext.symbolProvider.toSymbol( + requestValidationModeMemberInner, + ), + "OperationInputType" to + codegenContext.symbolProvider.toSymbol( + operationShape.inputShape( + codegenContext.model, + ), + ), + "ValidationModeShape" to + codegenContext.symbolProvider.toSymbol( + requestValidationModeMemberInner, + ), + "ResponseChecksumValidation" to + CargoDependency.smithyTypes(codegenContext.runtimeConfig).toType() + .resolve("checksum_config::ResponseChecksumValidation"), + "ConfigBag" to RuntimeType.configBag(codegenContext.runtimeConfig), ) } } @@ -122,3 +214,86 @@ class HttpResponseChecksumCustomization( } } } + +/** + * Add a `response_checksum_validation;` field to Service config. + */ +class HttpResponseChecksumConfigCustomization(private val codegenContext: ClientCodegenContext) : + NamedCustomization() { + private val rc = codegenContext.runtimeConfig + private val codegenScope = + arrayOf( + *preludeScope, + "ResponseChecksumValidation" to + configReexport( + RuntimeType.smithyTypes(rc) + .resolve("checksum_config::ResponseChecksumValidation"), + ), + ) + + override fun section(section: ServiceConfig): Writable { + // If the service contains no operations with the httpChecksum trait we return early + if (!serviceHasHttpChecksumOperation(codegenContext)) { + return emptySection + } + + // Otherwise we write the necessary sections to the service config + return when (section) { + is ServiceConfig.ConfigImpl -> + writable { + rustTemplate( + """ + /// Return a reference to the response_checksum_validation value contained in this config, if any. + pub fn response_checksum_validation(&self) -> #{Option}<&#{ResponseChecksumValidation}> { + self.config.load::<#{ResponseChecksumValidation}>() + } + """, + *codegenScope, + ) + } + + is ServiceConfig.BuilderImpl -> + writable { + rustTemplate( + """ + /// Set the [`ResponseChecksumValidation`](#{ResponseChecksumValidation}) + /// to determine when checksum validation will be performed on response payloads. + pub fn response_checksum_validation( + mut self, + response_checksum_validation: #{ResponseChecksumValidation} + ) -> Self { + self.set_response_checksum_validation(#{Some}(response_checksum_validation)); + self + } + """, + *codegenScope, + ) + + rustTemplate( + """ + /// Set the [`ResponseChecksumValidation`](#{ResponseChecksumValidation}) + /// to determine when checksum validation will be performed on response payloads. + pub fn set_response_checksum_validation( + &mut self, + response_checksum_validation: #{Option}<#{ResponseChecksumValidation}> + ) -> &mut Self { + self.config.store_or_unset(response_checksum_validation); + self + } + """, + *codegenScope, + ) + } + + is ServiceConfig.BuilderFromConfigBag -> + writable { + rustTemplate( + "${section.builder}.set_response_checksum_validation(${section.configBag}.load::<#{ResponseChecksumValidation}>().cloned());", + *codegenScope, + ) + } + + else -> emptySection + } + } +} diff --git a/aws/sdk-codegen/src/test/kotlin/software/amazon/smithy/rustsdk/HttpChecksumTest.kt b/aws/sdk-codegen/src/test/kotlin/software/amazon/smithy/rustsdk/HttpChecksumTest.kt new file mode 100644 index 0000000000000000000000000000000000000000..d6d64f7619aefd345af4722342820fe627907b58 --- /dev/null +++ b/aws/sdk-codegen/src/test/kotlin/software/amazon/smithy/rustsdk/HttpChecksumTest.kt @@ -0,0 +1,938 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +package software.amazon.smithy.rustsdk + +import org.junit.jupiter.api.Test +import software.amazon.smithy.rust.codegen.client.smithy.ClientCodegenContext +import software.amazon.smithy.rust.codegen.core.rustlang.CargoDependency +import software.amazon.smithy.rust.codegen.core.rustlang.Feature +import software.amazon.smithy.rust.codegen.core.rustlang.Writable +import software.amazon.smithy.rust.codegen.core.rustlang.join +import software.amazon.smithy.rust.codegen.core.rustlang.plus +import software.amazon.smithy.rust.codegen.core.rustlang.rustTemplate +import software.amazon.smithy.rust.codegen.core.rustlang.writable +import software.amazon.smithy.rust.codegen.core.smithy.RuntimeType +import software.amazon.smithy.rust.codegen.core.smithy.RuntimeType.Companion.preludeScope +import software.amazon.smithy.rust.codegen.core.testutil.asSmithyModel +import software.amazon.smithy.rust.codegen.core.testutil.integrationTest +import software.amazon.smithy.rust.codegen.core.util.dq + +internal class HttpChecksumTest { + companion object { + private const val PREFIX = "\$version: \"2\"" + private val model = + """ + $PREFIX + namespace test + + use aws.api#service + use aws.auth#sigv4 + use aws.protocols#httpChecksum + use aws.protocols#restJson1 + use smithy.rules#endpointRuleSet + + @service(sdkId: "dontcare") + @restJson1 + @sigv4(name: "dontcare") + @auth([sigv4]) + @endpointRuleSet({ + "version": "1.0", + "rules": [{ "type": "endpoint", "conditions": [], "endpoint": { "url": "https://example.com" } }], + "parameters": { + "Region": { "required": false, "type": "String", "builtIn": "AWS::Region" }, + } + }) + service TestService { + version: "2023-01-01", + operations: [HttpChecksumOperation, HttpChecksumStreamingOperation] + } + + @http(uri: "/HttpChecksumOperation", method: "POST") + @optionalAuth + @httpChecksum( + requestChecksumRequired: true, + requestAlgorithmMember: "checksumAlgorithm", + requestValidationModeMember: "validationMode", + responseAlgorithms: ["CRC32", "CRC32C", "CRC64NVME", "SHA1", "SHA256"] + ) + operation HttpChecksumOperation { + input: SomeInput, + output: SomeOutput + } + + @input + structure SomeInput { + @httpHeader("x-amz-request-algorithm") + checksumAlgorithm: ChecksumAlgorithm + + @httpHeader("x-amz-response-validation-mode") + validationMode: ValidationMode + + @httpHeader("x-amz-checksum-crc32") + ChecksumCRC32: String + + @httpHeader("x-amz-checksum-crc32c") + ChecksumCRC32C: String + + @httpHeader("x-amz-checksum-crc64nvme") + ChecksumCRC64Nvme: String + + @httpHeader("x-amz-checksum-sha1") + ChecksumSHA1: String + + @httpHeader("x-amz-checksum-sha256") + ChecksumSHA256: String + + @httpHeader("x-amz-checksum-foo") + ChecksumFoo: String + + @httpPayload + @required + body: Blob + } + + @output + structure SomeOutput {} + + @http(uri: "/HttpChecksumStreamingOperation", method: "POST") + @optionalAuth + @httpChecksum( + requestChecksumRequired: true, + requestAlgorithmMember: "checksumAlgorithm", + requestValidationModeMember: "validationMode", + responseAlgorithms: ["CRC32", "CRC32C", "CRC64NVME", "SHA1", "SHA256"] + ) + operation HttpChecksumStreamingOperation { + input: SomeStreamingInput, + output: SomeStreamingOutput + } + + @streaming + blob StreamingBlob + + @input + structure SomeStreamingInput { + @httpHeader("x-amz-request-algorithm") + checksumAlgorithm: ChecksumAlgorithm + + @httpHeader("x-amz-response-validation-mode") + validationMode: ValidationMode + + @httpPayload + @required + body: StreamingBlob + } + + @output + structure SomeStreamingOutput {} + + enum ChecksumAlgorithm { + CRC32 + CRC32C + CRC64NVME + SHA1 + SHA256 + } + + enum ValidationMode { + ENABLED + } + """.asSmithyModel() + } + + @Test + fun requestChecksumWorks() { + awsSdkIntegrationTest(model) { context, rustCrate -> + // Allows us to use the user-agent test-utils in aws-runtime + rustCrate.mergeFeature(Feature("test-util", true, listOf("aws-runtime/test-util"))) + val rc = context.runtimeConfig + + // Create Writables for all test types + val checksumRequestTestWritables = + checksumRequestTests.map { createRequestChecksumCalculationTest(it, context) }.join("\n") + val checksumResponseSuccTestWritables = + checksumResponseSuccTests.map { createResponseChecksumValidationSuccessTest(it, context) }.join("\n") + val checksumResponseFailTestWritables = + checksumResponseFailTests.map { createResponseChecksumValidationFailureTest(it, context) }.join("\n") + val checksumStreamingRequestTestWritables = + streamingRequestTests.map { createStreamingRequestChecksumCalculationTest(it, context) }.join("\n") + val userProvidedChecksumTestWritables = + userProvidedChecksumTests.map { createUserProvidedChecksumsTest(it, context) }.join("\n") + val miscTests = createMiscellaneousTests(context) + + // Shared imports for all test types + // Note about the `//#{PresigningMarker}` below. The `RequestChecksumInterceptor` relies on the `PresigningMarker` type from + // the presigning inlineable. The decorator for that inlineable doesn't play nicely with the test model from the SEP, so we + // use this as a kind of blunt way to include the presigning inlineable without actually wiring it up with the model. + val testBase = + writable { + rustTemplate( + """ + ##![cfg(feature = "test-util")] + ##![allow(unused_imports)] + + use #{Blob}; + use #{Region}; + use #{pretty_assertions}::assert_eq; + use #{SdkBody}; + use std::io::Write; + use http_body::Body; + use #{HttpRequest}; + use #{UaAssert}; + use #{UaExtract}; + //#{PresigningMarker}; + """, + *preludeScope, + "Blob" to RuntimeType.smithyTypes(rc).resolve("Blob"), + "Region" to AwsRuntimeType.awsTypes(rc).resolve("region::Region"), + "pretty_assertions" to CargoDependency.PrettyAssertions.toType(), + "SdkBody" to RuntimeType.smithyTypes(rc).resolve("body::SdkBody"), + "HttpRequest" to RuntimeType.smithyRuntimeApi(rc).resolve("client::orchestrator::HttpRequest"), + "UaAssert" to + AwsRuntimeType.awsRuntime(rc) + .resolve("user_agent::test_util::assert_ua_contains_metric_values"), + "UaExtract" to + AwsRuntimeType.awsRuntime(rc) + .resolve("user_agent::test_util::extract_ua_values"), + "PresigningMarker" to AwsRuntimeType.presigning().resolve("PresigningMarker"), + ) + } + + // Create one integ test per test type + rustCrate.integrationTest("request_checksums") { + testBase.plus(checksumRequestTestWritables)() + } + + rustCrate.integrationTest("response_checksums_success") { + testBase.plus(checksumResponseSuccTestWritables)() + } + + rustCrate.integrationTest("response_checksums_fail") { + testBase.plus(checksumResponseFailTestWritables)() + } + + rustCrate.integrationTest("streaming_request_checksums") { + testBase.plus(checksumStreamingRequestTestWritables)() + } + + rustCrate.integrationTest("user_provided_checksums") { + testBase.plus(userProvidedChecksumTestWritables)() + } + + rustCrate.integrationTest("misc_tests") { + testBase.plus(miscTests)() + } + } + } + + /** + * Generate tests where the request checksum is calculated correctly + */ + private fun createRequestChecksumCalculationTest( + testDef: RequestChecksumCalculationTest, + context: ClientCodegenContext, + ): Writable { + val rc = context.runtimeConfig + val moduleName = context.moduleUseName() + val algoLower = testDef.checksumAlgorithm.lowercase() + // If the algo is Crc32 don't explicitly set it to test that the default is correctly set + val setChecksumAlgo = + if (testDef.checksumAlgorithm != "Crc32") { + ".checksum_algorithm($moduleName::types::ChecksumAlgorithm::${testDef.checksumAlgorithm})" + } else { + "" + } + return writable { + rustTemplate( + """ + //${testDef.docs} + ##[#{tokio}::test] + async fn ${algoLower}_request_checksums_work() { + let (http_client, rx) = #{capture_request}(None); + let config = $moduleName::Config::builder() + .region(Region::from_static("doesntmatter")) + .with_test_defaults() + .http_client(http_client) + .build(); + + let client = $moduleName::Client::from_conf(config); + let _ = client.http_checksum_operation() + .body(Blob::new(b"${testDef.requestPayload}")) + $setChecksumAlgo + .send() + .await; + let request = rx.expect_request(); + let ${algoLower}_header = request.headers() + .get("x-amz-checksum-$algoLower") + .expect("x-amz-checksum-$algoLower header should exist"); + + assert_eq!(${algoLower}_header, "${testDef.checksumHeader}"); + + let algo_header = request.headers() + .get("x-amz-request-algorithm") + .expect("algo header should exist"); + + assert_eq!(algo_header, "${testDef.algoHeader}"); + + // Check the user-agent metrics for the selected algo + assert_ua_contains_metric_values( + &request + .headers() + .get("x-amz-user-agent") + .expect("UA header should be present"), + &["${testDef.algoFeatureId}"], + ); + } + """, + *preludeScope, + "tokio" to CargoDependency.Tokio.toType(), + "capture_request" to RuntimeType.captureRequest(rc), + ) + } + } + + /** + * Generate tests where the request is streaming and checksum is calculated correctly + */ + private fun createStreamingRequestChecksumCalculationTest( + testDef: StreamingRequestChecksumCalculationTest, + context: ClientCodegenContext, + ): Writable { + val rc = context.runtimeConfig + val moduleName = context.moduleUseName() + val algoLower = testDef.checksumAlgorithm.lowercase() + // If the algo is Crc32 don't explicitly set it to test that the default is correctly set + val setChecksumAlgo = + if (testDef.checksumAlgorithm != "Crc32") { + ".checksum_algorithm($moduleName::types::ChecksumAlgorithm::${testDef.checksumAlgorithm})" + } else { + "" + } + return writable { + rustTemplate( + """ + //${testDef.docs} + ##[#{tokio}::test] + async fn ${algoLower}_request_checksums_work() { + let (http_client, rx) = #{capture_request}(None); + let config = $moduleName::Config::builder() + .region(Region::from_static("doesntmatter")) + .with_test_defaults() + .http_client(http_client) + .build(); + + let client = $moduleName::Client::from_conf(config); + + let mut file = tempfile::NamedTempFile::new().unwrap(); + file.as_file_mut() + .write_all("${testDef.requestPayload}".as_bytes()) + .unwrap(); + + let streaming_body = aws_smithy_types::byte_stream::ByteStream::read_from() + .path(&file) + .build() + .await + .unwrap(); + + let _operation = client + .http_checksum_streaming_operation() + .body(streaming_body) + $setChecksumAlgo + .send() + .await; + + let request = rx.expect_request(); + + let headers = request.headers(); + + assert_eq!( + headers.get("x-amz-trailer").unwrap(), + "x-amz-checksum-$algoLower", + ); + assert_eq!(headers.get("content-encoding").unwrap(), "aws-chunked"); + + let mut body = request.body().try_clone().expect("body is retryable"); + + let mut body_data = bytes::BytesMut::new(); + while let Some(data) = body.data().await { + body_data.extend_from_slice(&data.unwrap()) + } + + let body_string = std::str::from_utf8(&body_data).unwrap(); + assert!(body_string.contains("x-amz-checksum-$algoLower:${testDef.trailerChecksum}")); + } + """, + *preludeScope, + "tokio" to CargoDependency.Tokio.toType(), + "capture_request" to RuntimeType.captureRequest(rc), + ) + } + } + + /** + * Generate tests where the response checksum validates successfully + */ + private fun createResponseChecksumValidationSuccessTest( + testDef: ResponseChecksumValidationSuccessTest, + context: ClientCodegenContext, + ): Writable { + val rc = context.runtimeConfig + val moduleName = context.moduleUseName() + val algoLower = testDef.checksumAlgorithm.lowercase() + return writable { + rustTemplate( + """ + //${testDef.docs} + ##[::tokio::test] + async fn ${algoLower}_response_checksums_works() { + let (http_client, _rx) = #{capture_request}(Some( + http::Response::builder() + .header("x-amz-checksum-$algoLower", "${testDef.checksumHeaderValue}") + .body(SdkBody::from("${testDef.responsePayload}")) + .unwrap(), + )); + let config = $moduleName::Config::builder() + .region(Region::from_static("doesntmatter")) + .with_test_defaults() + .http_client(http_client) + .build(); + + let client = $moduleName::Client::from_conf(config); + let res = client + .http_checksum_operation() + .body(Blob::new(b"Doesn't matter.")) + .checksum_algorithm($moduleName::types::ChecksumAlgorithm::${testDef.checksumAlgorithm}) + .validation_mode($moduleName::types::ValidationMode::Enabled) + .send() + .await; + assert!(res.is_ok()) + } + """, + *preludeScope, + "tokio" to CargoDependency.Tokio.toType(), + "capture_request" to RuntimeType.captureRequest(rc), + ) + } + } + + /** + * Generate tests where the response checksum fails to validate + */ + private fun createResponseChecksumValidationFailureTest( + testDef: ResponseChecksumValidationFailureTest, + context: ClientCodegenContext, + ): Writable { + val rc = context.runtimeConfig + val moduleName = context.moduleUseName() + val algoLower = testDef.checksumAlgorithm.lowercase() + return writable { + rustTemplate( + """ + //${testDef.docs} + ##[::tokio::test] + async fn ${algoLower}_response_checksums_fail_correctly() { + let (http_client, _rx) = #{capture_request}(Some( + http::Response::builder() + .header("x-amz-checksum-$algoLower", "${testDef.checksumHeaderValue}") + .body(SdkBody::from("${testDef.responsePayload}")) + .unwrap(), + )); + let config = $moduleName::Config::builder() + .region(Region::from_static("doesntmatter")) + .with_test_defaults() + .http_client(http_client) + .build(); + + let client = $moduleName::Client::from_conf(config); + let res = client + .http_checksum_operation() + .body(Blob::new(b"Doesn't matter.")) + .checksum_algorithm($moduleName::types::ChecksumAlgorithm::${testDef.checksumAlgorithm}) + .validation_mode($moduleName::types::ValidationMode::Enabled) + .send() + .await; + + assert!(res.is_err()); + + let boxed_err = res + .unwrap_err() + .into_source() + .unwrap() + .downcast::(); + let typed_err = boxed_err.as_ref().unwrap().as_ref(); + + match typed_err { + aws_smithy_checksums::body::validate::Error::ChecksumMismatch { actual, .. } => { + let calculated_checksum = aws_smithy_types::base64::encode(actual); + assert_eq!(calculated_checksum, "${testDef.calculatedChecksum}"); + } + _ => panic!("Unknown error type in checksum validation"), + }; + } + """, + *preludeScope, + "tokio" to CargoDependency.Tokio.toType(), + "capture_request" to RuntimeType.captureRequest(rc), + ) + } + } + + /** + * Generate tests for the case where a user provides a checksum + */ + private fun createUserProvidedChecksumsTest( + testDef: UserProvidedChecksumTest, + context: ClientCodegenContext, + ): Writable { + val rc = context.runtimeConfig + val moduleName = context.moduleUseName() + val algoLower = testDef.checksumAlgorithm.lowercase() + // We treat the c after crc32c and the nvme after crc64nvme as separate words + // so this quick map helps us find the field to set + val algoFieldNames = + mapOf( + "crc32" to "checksum_crc32", + "crc32c" to "checksum_crc32_c", + "crc64nvme" to "checksum_crc64_nvme", + "foo" to "checksum_foo", + "sha1" to "checksum_sha1", + "sha256" to "checksum_sha256", + ) + + return writable { + rustTemplate( + """ + //${testDef.docs} + ##[#{tokio}::test] + async fn user_provided_${algoLower}_request_checksum_works() { + let (http_client, rx) = #{capture_request}(None); + let config = $moduleName::Config::builder() + .region(Region::from_static("doesntmatter")) + .with_test_defaults() + .http_client(http_client) + .build(); + + let client = $moduleName::Client::from_conf(config); + let _ = client.http_checksum_operation() + .body(Blob::new(b"${testDef.requestPayload}")) + .${algoFieldNames.get(algoLower)}(${testDef.checksumValue.dq()}) + .send() + .await; + let request = rx.expect_request(); + let ${algoLower}_header = request.headers() + .get("x-amz-checksum-$algoLower") + .expect("x-amz-checksum-$algoLower header should exist"); + + assert_eq!(${algoLower}_header, "${testDef.expectedHeaderValue}"); + + let algo_header = request.headers() + .get("x-amz-request-algorithm"); + + assert!(algo_header.is_none()); + } + """, + *preludeScope, + "tokio" to CargoDependency.Tokio.toType(), + "capture_request" to RuntimeType.captureRequest(rc), + ) + } + } + + /** + * Generate miscellaneous tests, currently mostly focused on the inclusion of the checksum config metrics in the + * user-agent header + */ + private fun createMiscellaneousTests(context: ClientCodegenContext): Writable { + val rc = context.runtimeConfig + val moduleName = context.moduleUseName() + return writable { + rustTemplate( + """ + // The following tests confirm that the user-agent business metrics header is correctly + // set for the request/response checksum config values + ##[::tokio::test] + async fn request_config_ua_required() { + let (http_client, rx) = #{capture_request}(None); + let config = $moduleName::Config::builder() + .region(Region::from_static("doesntmatter")) + .with_test_defaults() + .http_client(http_client) + .request_checksum_calculation( + aws_types::sdk_config::RequestChecksumCalculation::WhenRequired, + ) + .build(); + + let client = $moduleName::Client::from_conf(config); + let _ = client + .http_checksum_operation() + .body(Blob::new(b"Doesn't matter")) + .send() + .await; + let request = rx.expect_request(); + + let sdk_metrics = extract_ua_values( + &request + .headers() + .get("x-amz-user-agent") + .expect("UA header should be present"), + ).expect("UA header should be present"); + assert!(sdk_metrics.contains(&"a")); + assert!(!sdk_metrics.contains(&"Z")); + } + + ##[::tokio::test] + async fn request_config_ua_supported() { + let (http_client, rx) = #{capture_request}(None); + let config = $moduleName::Config::builder() + .region(Region::from_static("doesntmatter")) + .with_test_defaults() + .http_client(http_client) + .build(); + + let client = $moduleName::Client::from_conf(config); + let _ = client + .http_checksum_operation() + .body(Blob::new(b"Doesn't matter")) + .send() + .await; + let request = rx.expect_request(); + + let sdk_metrics = extract_ua_values( + &request + .headers() + .get("x-amz-user-agent") + .expect("UA header should be present"), + ).expect("UA header should be present"); + assert!(sdk_metrics.contains(&"Z")); + assert!(!sdk_metrics.contains(&"a")); + } + + ##[::tokio::test] + async fn response_config_ua_supported() { + let (http_client, rx) = #{capture_request}(Some( + http::Response::builder() + .header("x-amz-checksum-crc32", "i9aeUg==") + .body(SdkBody::from("Hello world")) + .unwrap(), + )); + let config = $moduleName::Config::builder() + .region(Region::from_static("doesntmatter")) + .with_test_defaults() + .http_client(http_client) + .build(); + + let client = $moduleName::Client::from_conf(config); + let _ = client + .http_checksum_operation() + .body(Blob::new(b"Doesn't matter")) + .send() + .await; + let request = rx.expect_request(); + + let sdk_metrics = extract_ua_values( + &request + .headers() + .get("x-amz-user-agent") + .expect("UA header should be present"), + ).expect("UA header should be present"); + assert!(sdk_metrics.contains(&"b")); + assert!(!sdk_metrics.contains(&"c")); + } + + ##[::tokio::test] + async fn response_config_ua_required() { + let (http_client, rx) = #{capture_request}(Some( + http::Response::builder() + .header("x-amz-checksum-crc32", "i9aeUg==") + .body(SdkBody::from("Hello world")) + .unwrap(), + )); + let config = $moduleName::Config::builder() + .region(Region::from_static("doesntmatter")) + .with_test_defaults() + .http_client(http_client) + .response_checksum_validation( + aws_types::sdk_config::ResponseChecksumValidation::WhenRequired, + ) + .build(); + + let client = $moduleName::Client::from_conf(config); + let _ = client + .http_checksum_operation() + .body(Blob::new(b"Doesn't matter")) + .send() + .await; + let request = rx.expect_request(); + + let sdk_metrics = extract_ua_values( + &request + .headers() + .get("x-amz-user-agent") + .expect("UA header should be present"), + ).expect("UA header should be present"); + assert!(sdk_metrics.contains(&"c")); + assert!(!sdk_metrics.contains(&"b")); + } + """, + *preludeScope, + "tokio" to CargoDependency.Tokio.toType(), + "capture_request" to RuntimeType.captureRequest(rc), + ) + } + } +} + +// Classes and data for test definitions + +data class RequestChecksumCalculationTest( + val docs: String, + val requestPayload: String, + val checksumAlgorithm: String, + val algoHeader: String, + val checksumHeader: String, + val algoFeatureId: String, +) + +val checksumRequestTests = + listOf( + RequestChecksumCalculationTest( + "CRC32 checksum calculation works.", + "Hello world", + "Crc32", + "CRC32", + "i9aeUg==", + "U", + ), + RequestChecksumCalculationTest( + "CRC32C checksum calculation works.", + "Hello world", + "Crc32C", + "CRC32C", + "crUfeA==", + "V", + ), + RequestChecksumCalculationTest( + "CRC64NVME checksum calculation works.", + "Hello world", + "Crc64Nvme", + "CRC64NVME", + "OOJZ0D8xKts=", + "W", + ), + RequestChecksumCalculationTest( + "SHA1 checksum calculation works.", + "Hello world", + "Sha1", + "SHA1", + "e1AsOh9IyGCa4hLN+2Od7jlnP14=", + "X", + ), + RequestChecksumCalculationTest( + "SHA256 checksum calculation works.", + "Hello world", + "Sha256", + "SHA256", + "ZOyIygCyaOW6GjVnihtTFtIS9PNmskdyMlNKiuyjfzw=", + "Y", + ), + ) + +data class StreamingRequestChecksumCalculationTest( + val docs: String, + val requestPayload: String, + val checksumAlgorithm: String, + val trailerChecksum: String, +) + +val streamingRequestTests = + listOf( + StreamingRequestChecksumCalculationTest( + "CRC32 streaming checksum calculation works.", + "Hello world", + "Crc32", + "i9aeUg==", + ), + StreamingRequestChecksumCalculationTest( + "CRC32C streaming checksum calculation works.", + "Hello world", + "Crc32C", + "crUfeA==", + ), + StreamingRequestChecksumCalculationTest( + "CRC64NVME streaming checksum calculation works.", + "Hello world", + "Crc64Nvme", + "OOJZ0D8xKts=", + ), + StreamingRequestChecksumCalculationTest( + "SHA1 streaming checksum calculation works.", + "Hello world", + "Sha1", + "e1AsOh9IyGCa4hLN+2Od7jlnP14=", + ), + StreamingRequestChecksumCalculationTest( + "SHA256 streaming checksum calculation works.", + "Hello world", + "Sha256", + "ZOyIygCyaOW6GjVnihtTFtIS9PNmskdyMlNKiuyjfzw=", + ), + ) + +data class ResponseChecksumValidationSuccessTest( + val docs: String, + val responsePayload: String, + val checksumAlgorithm: String, + val checksumHeaderValue: String, +) + +val checksumResponseSuccTests = + listOf( + ResponseChecksumValidationSuccessTest( + "Successful payload validation with CRC32 checksum.", + "Hello world", + "Crc32", + "i9aeUg==", + ), + ResponseChecksumValidationSuccessTest( + "Successful payload validation with Crc32C checksum.", + "Hello world", + "Crc32C", + "crUfeA==", + ), + ResponseChecksumValidationSuccessTest( + "Successful payload validation with Crc64Nvme checksum.", + "Hello world", + "Crc64Nvme", + "OOJZ0D8xKts=", + ), + ResponseChecksumValidationSuccessTest( + "Successful payload validation with Sha1 checksum.", + "Hello world", + "Sha1", + "e1AsOh9IyGCa4hLN+2Od7jlnP14=", + ), + ResponseChecksumValidationSuccessTest( + "Successful payload validation with Sha256 checksum.", + "Hello world", + "Sha256", + "ZOyIygCyaOW6GjVnihtTFtIS9PNmskdyMlNKiuyjfzw=", + ), + ) + +data class ResponseChecksumValidationFailureTest( + val docs: String, + val responsePayload: String, + val checksumAlgorithm: String, + val checksumHeaderValue: String, + val calculatedChecksum: String, +) + +val checksumResponseFailTests = + listOf( + ResponseChecksumValidationFailureTest( + "Failed payload validation with CRC32 checksum.", + "Hello world", + "Crc32", + "bm90LWEtY2hlY2tzdW0=", + "i9aeUg==", + ), + ResponseChecksumValidationFailureTest( + "Failed payload validation with CRC32C checksum.", + "Hello world", + "Crc32C", + "bm90LWEtY2hlY2tzdW0=", + "crUfeA==", + ), + ResponseChecksumValidationFailureTest( + "Failed payload validation with CRC64NVME checksum.", + "Hello world", + "Crc64Nvme", + "bm90LWEtY2hlY2tzdW0=", + "OOJZ0D8xKts=", + ), + ResponseChecksumValidationFailureTest( + "Failed payload validation with SHA1 checksum.", + "Hello world", + "Sha1", + "bm90LWEtY2hlY2tzdW0=", + "e1AsOh9IyGCa4hLN+2Od7jlnP14=", + ), + ResponseChecksumValidationFailureTest( + "Failed payload validation with SHA256 checksum.", + "Hello world", + "Sha256", + "bm90LWEtY2hlY2tzdW0=", + "ZOyIygCyaOW6GjVnihtTFtIS9PNmskdyMlNKiuyjfzw=", + ), + ) + +data class UserProvidedChecksumTest( + val docs: String, + val requestPayload: String, + val checksumAlgorithm: String, + val checksumValue: String, + val expectedHeaderName: String, + val expectedHeaderValue: String, + val forbidHeaderName: String, +) + +val userProvidedChecksumTests = + listOf( + UserProvidedChecksumTest( + "CRC32 checksum provided by user.", + "Hello world", + "Crc32", + "i9aeUg==", + "x-amz-checksum-crc32", + "i9aeUg==", + "x-amz-request-algorithm", + ), + UserProvidedChecksumTest( + "CRC32C checksum provided by user.", + "Hello world", + "Crc32C", + "crUfeA==", + "x-amz-checksum-crc32c", + "crUfeA==", + "x-amz-request-algorithm", + ), + UserProvidedChecksumTest( + "CRC64NVME checksum provided by user.", + "Hello world", + "Crc64Nvme", + "OOJZ0D8xKts=", + "x-amz-checksum-crc64nvme", + "OOJZ0D8xKts=", + "x-amz-request-algorithm", + ), + UserProvidedChecksumTest( + "SHA1 checksum provided by user.", + "Hello world", + "Sha1", + "e1AsOh9IyGCa4hLN+2Od7jlnP14=", + "x-amz-checksum-sha1", + "e1AsOh9IyGCa4hLN+2Od7jlnP14=", + "x-amz-request-algorithm", + ), + UserProvidedChecksumTest( + "SHA256 checksum provided by user.", + "Hello world", + "Sha256", + "ZOyIygCyaOW6GjVnihtTFtIS9PNmskdyMlNKiuyjfzw=", + "x-amz-checksum-sha256", + "ZOyIygCyaOW6GjVnihtTFtIS9PNmskdyMlNKiuyjfzw=", + "x-amz-request-algorithm", + ), + UserProvidedChecksumTest( + "Forwards compatibility, unmodeled checksum provided by user.", + "Hello world", + "Foo", + "This-is-not-a-real-checksum", + "x-amz-checksum-foo", + "This-is-not-a-real-checksum", + "x-amz-request-algorithm", + ), + ) diff --git a/aws/sdk/Cargo.lock b/aws/sdk/Cargo.lock index 6ecbdceb30526e55b25412912c9ef411878f8869..830c671ce7fad6ae9b76ae628b71d412b92bb1d0 100644 --- a/aws/sdk/Cargo.lock +++ b/aws/sdk/Cargo.lock @@ -38,12 +38,55 @@ version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4b46cbb362ab8752921c97e041f5e366ee6297bd428a31275b9fcf1e380f7299" +[[package]] +name = "anstream" +version = "0.6.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8acc5369981196006228e28809f761875c0327210a891e941f4c683b3a99529b" +dependencies = [ + "anstyle", + "anstyle-parse", + "anstyle-query", + "anstyle-wincon", + "colorchoice", + "is_terminal_polyfill", + "utf8parse", +] + [[package]] name = "anstyle" version = "1.0.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "55cc3b69f167a1ef2e161439aa98aed94e6028e5f9a59be9a6ffb47aef1651f9" +[[package]] +name = "anstyle-parse" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b2d16507662817a6a20a9ea92df6652ee4f94f914589377d69f3b21bc5798a9" +dependencies = [ + "utf8parse", +] + +[[package]] +name = "anstyle-query" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "79947af37f4177cfead1110013d678905c37501914fba0efea834c3fe9a8d60c" +dependencies = [ + "windows-sys 0.59.0", +] + +[[package]] +name = "anstyle-wincon" +version = "3.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2109dbce0e72be3ec00bed26e6a7479ca384ad226efdd66db8fa2e3a38c83125" +dependencies = [ + "anstyle", + "windows-sys 0.59.0", +] + [[package]] name = "approx" version = "0.5.1" @@ -317,20 +360,20 @@ checksum = "ace50bade8e6234aa140d9a2f552bbee1db4d353f69b8217bc503490fc1a9f26" [[package]] name = "aws-config" -version = "1.5.13" +version = "1.5.14" dependencies = [ "aws-credential-types 1.2.1", - "aws-runtime 1.5.3", + "aws-runtime 1.5.4", "aws-sdk-sso", "aws-sdk-ssooidc", "aws-sdk-sts", - "aws-smithy-async 1.2.3", + "aws-smithy-async 1.2.4", "aws-smithy-http 0.60.11", "aws-smithy-json 0.61.1", "aws-smithy-runtime 1.7.7", "aws-smithy-runtime-api 1.7.3", - "aws-smithy-types 1.2.11", - "aws-types 1.3.3", + "aws-smithy-types 1.2.12", + "aws-types 1.3.4", "bytes", "fastrand 2.0.2", "futures-util", @@ -354,9 +397,9 @@ name = "aws-credential-types" version = "1.2.1" dependencies = [ "async-trait", - "aws-smithy-async 1.2.3", + "aws-smithy-async 1.2.4", "aws-smithy-runtime-api 1.7.3", - "aws-smithy-types 1.2.11", + "aws-smithy-types 1.2.12", "tokio", "zeroize", ] @@ -367,9 +410,9 @@ version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "60e8f6b615cb5fc60a98132268508ad104310f0cfb25a1c22eee76efdf9154da" dependencies = [ - "aws-smithy-async 1.2.3 (registry+https://github.com/rust-lang/crates.io-index)", + "aws-smithy-async 1.2.3", "aws-smithy-runtime-api 1.7.3 (registry+https://github.com/rust-lang/crates.io-index)", - "aws-smithy-types 1.2.11 (registry+https://github.com/rust-lang/crates.io-index)", + "aws-smithy-types 1.2.11", "zeroize", ] @@ -381,7 +424,7 @@ version = "0.60.3" name = "aws-http" version = "0.60.6" dependencies = [ - "aws-runtime 1.5.3", + "aws-runtime 1.5.4", ] [[package]] @@ -432,19 +475,19 @@ dependencies = [ [[package]] name = "aws-runtime" -version = "1.5.3" +version = "1.5.4" dependencies = [ "arbitrary", "aws-credential-types 1.2.1", - "aws-sigv4 1.2.6", - "aws-smithy-async 1.2.3", + "aws-sigv4 1.2.7", + "aws-smithy-async 1.2.4", "aws-smithy-eventstream 0.60.5", "aws-smithy-http 0.60.11", "aws-smithy-protocol-test 0.63.0", "aws-smithy-runtime 1.7.7", "aws-smithy-runtime-api 1.7.3", - "aws-smithy-types 1.2.11", - "aws-types 1.3.3", + "aws-smithy-types 1.2.12", + "aws-types 1.3.4", "bytes", "bytes-utils", "convert_case", @@ -504,15 +547,15 @@ version = "0.0.0-local" dependencies = [ "aws-config", "aws-credential-types 1.2.1", - "aws-runtime 1.5.3", - "aws-smithy-async 1.2.3", + "aws-runtime 1.5.4", + "aws-smithy-async 1.2.4", "aws-smithy-eventstream 0.60.5", "aws-smithy-http 0.60.11", "aws-smithy-json 0.61.1", "aws-smithy-runtime 1.7.7", "aws-smithy-runtime-api 1.7.3", - "aws-smithy-types 1.2.11", - "aws-types 1.3.3", + "aws-smithy-types 1.2.12", + "aws-types 1.3.4", "bytes", "http 0.2.12", "once_cell", @@ -527,15 +570,15 @@ version = "0.0.0-local" dependencies = [ "aws-config", "aws-credential-types 1.2.1", - "aws-runtime 1.5.3", - "aws-smithy-async 1.2.3", + "aws-runtime 1.5.4", + "aws-smithy-async 1.2.4", "aws-smithy-http 0.60.11", "aws-smithy-json 0.61.1", "aws-smithy-protocol-test 0.63.0", "aws-smithy-runtime 1.7.7", "aws-smithy-runtime-api 1.7.3", - "aws-smithy-types 1.2.11", - "aws-types 1.3.3", + "aws-smithy-types 1.2.12", + "aws-types 1.3.4", "bytes", "fastrand 2.0.2", "futures-util", @@ -554,14 +597,14 @@ version = "0.0.0-local" dependencies = [ "aws-config", "aws-credential-types 1.2.1", - "aws-runtime 1.5.3", - "aws-smithy-async 1.2.3", + "aws-runtime 1.5.4", + "aws-smithy-async 1.2.4", "aws-smithy-http 0.60.11", "aws-smithy-json 0.61.1", "aws-smithy-runtime 1.7.7", "aws-smithy-runtime-api 1.7.3", - "aws-smithy-types 1.2.11", - "aws-types 1.3.3", + "aws-smithy-types 1.2.12", + "aws-types 1.3.4", "bytes", "http 0.2.12", "once_cell", @@ -577,15 +620,15 @@ dependencies = [ "approx", "aws-config", "aws-credential-types 1.2.1", - "aws-runtime 1.5.3", - "aws-smithy-async 1.2.3", + "aws-runtime 1.5.4", + "aws-smithy-async 1.2.4", "aws-smithy-http 0.60.11", "aws-smithy-json 0.61.1", "aws-smithy-protocol-test 0.63.0", "aws-smithy-runtime 1.7.7", "aws-smithy-runtime-api 1.7.3", - "aws-smithy-types 1.2.11", - "aws-types 1.3.3", + "aws-smithy-types 1.2.12", + "aws-types 1.3.4", "bytes", "criterion", "fastrand 2.0.2", @@ -605,17 +648,17 @@ version = "0.0.0-local" dependencies = [ "aws-config", "aws-credential-types 1.2.1", - "aws-runtime 1.5.3", - "aws-smithy-async 1.2.3", + "aws-runtime 1.5.4", + "aws-smithy-async 1.2.4", "aws-smithy-http 0.60.11", "aws-smithy-json 0.61.1", "aws-smithy-protocol-test 0.63.0", "aws-smithy-query", "aws-smithy-runtime 1.7.7", "aws-smithy-runtime-api 1.7.3", - "aws-smithy-types 1.2.11", + "aws-smithy-types 1.2.12", "aws-smithy-xml 0.60.9", - "aws-types 1.3.3", + "aws-types 1.3.4", "fastrand 2.0.2", "futures-util", "http 0.2.12", @@ -633,14 +676,14 @@ version = "0.0.0-local" dependencies = [ "aws-config", "aws-credential-types 1.2.1", - "aws-runtime 1.5.3", - "aws-smithy-async 1.2.3", + "aws-runtime 1.5.4", + "aws-smithy-async 1.2.4", "aws-smithy-http 0.60.11", "aws-smithy-json 0.61.1", "aws-smithy-runtime 1.7.7", "aws-smithy-runtime-api 1.7.3", - "aws-smithy-types 1.2.11", - "aws-types 1.3.3", + "aws-smithy-types 1.2.12", + "aws-types 1.3.4", "bytes", "fastrand 2.0.2", "http 0.2.12", @@ -656,16 +699,16 @@ version = "0.0.0-local" dependencies = [ "aws-config", "aws-credential-types 1.2.1", - "aws-runtime 1.5.3", - "aws-sigv4 1.2.6", - "aws-smithy-async 1.2.3", + "aws-runtime 1.5.4", + "aws-sigv4 1.2.7", + "aws-smithy-async 1.2.4", "aws-smithy-http 0.60.11", "aws-smithy-json 0.61.1", "aws-smithy-protocol-test 0.63.0", "aws-smithy-runtime 1.7.7", "aws-smithy-runtime-api 1.7.3", - "aws-smithy-types 1.2.11", - "aws-types 1.3.3", + "aws-smithy-types 1.2.12", + "aws-types 1.3.4", "bytes", "futures-util", "hex", @@ -687,17 +730,17 @@ version = "0.0.0-local" dependencies = [ "aws-config", "aws-credential-types 1.2.1", - "aws-runtime 1.5.3", - "aws-smithy-async 1.2.3", + "aws-runtime 1.5.4", + "aws-smithy-async 1.2.4", "aws-smithy-http 0.60.11", "aws-smithy-json 0.61.1", "aws-smithy-protocol-test 0.63.0", "aws-smithy-query", "aws-smithy-runtime 1.7.7", "aws-smithy-runtime-api 1.7.3", - "aws-smithy-types 1.2.11", + "aws-smithy-types 1.2.12", "aws-smithy-xml 0.60.9", - "aws-types 1.3.3", + "aws-types 1.3.4", "futures-util", "http 0.2.12", "once_cell", @@ -714,15 +757,15 @@ version = "0.0.0-local" dependencies = [ "aws-config", "aws-credential-types 1.2.1", - "aws-runtime 1.5.3", - "aws-smithy-async 1.2.3", + "aws-runtime 1.5.4", + "aws-smithy-async 1.2.4", "aws-smithy-http 0.60.11", "aws-smithy-json 0.61.1", "aws-smithy-protocol-test 0.63.0", "aws-smithy-runtime 1.7.7", "aws-smithy-runtime-api 1.7.3", - "aws-smithy-types 1.2.11", - "aws-types 1.3.3", + "aws-smithy-types 1.2.12", + "aws-types 1.3.4", "bytes", "futures-util", "http 0.2.12", @@ -740,16 +783,16 @@ version = "0.0.0-local" dependencies = [ "aws-config", "aws-credential-types 1.2.1", - "aws-runtime 1.5.3", - "aws-smithy-async 1.2.3", + "aws-runtime 1.5.4", + "aws-smithy-async 1.2.4", "aws-smithy-eventstream 0.60.5", "aws-smithy-http 0.60.11", "aws-smithy-json 0.61.1", "aws-smithy-protocol-test 0.63.0", "aws-smithy-runtime 1.7.7", "aws-smithy-runtime-api 1.7.3", - "aws-smithy-types 1.2.11", - "aws-types 1.3.3", + "aws-smithy-types 1.2.12", + "aws-types 1.3.4", "bytes", "futures-util", "http 0.2.12", @@ -767,16 +810,16 @@ version = "0.0.0-local" dependencies = [ "aws-config", "aws-credential-types 1.2.1", - "aws-runtime 1.5.3", - "aws-sigv4 1.2.6", - "aws-smithy-async 1.2.3", + "aws-runtime 1.5.4", + "aws-sigv4 1.2.7", + "aws-smithy-async 1.2.4", "aws-smithy-http 0.60.11", "aws-smithy-json 0.61.1", "aws-smithy-protocol-test 0.63.0", "aws-smithy-runtime 1.7.7", "aws-smithy-runtime-api 1.7.3", - "aws-smithy-types 1.2.11", - "aws-types 1.3.3", + "aws-smithy-types 1.2.12", + "aws-types 1.3.4", "bytes", "futures-util", "http 0.2.12", @@ -796,15 +839,15 @@ version = "0.0.0-local" dependencies = [ "aws-config", "aws-credential-types 1.2.1", - "aws-runtime 1.5.3", - "aws-smithy-async 1.2.3", + "aws-runtime 1.5.4", + "aws-smithy-async 1.2.4", "aws-smithy-http 0.60.11", "aws-smithy-json 0.61.1", "aws-smithy-protocol-test 0.63.0", "aws-smithy-runtime 1.7.7", "aws-smithy-runtime-api 1.7.3", - "aws-smithy-types 1.2.11", - "aws-types 1.3.3", + "aws-smithy-types 1.2.12", + "aws-types 1.3.4", "bytes", "futures-util", "http 0.2.12", @@ -822,15 +865,15 @@ version = "0.0.0-local" dependencies = [ "aws-config", "aws-credential-types 1.2.1", - "aws-runtime 1.5.3", - "aws-smithy-async 1.2.3", + "aws-runtime 1.5.4", + "aws-smithy-async 1.2.4", "aws-smithy-http 0.60.11", "aws-smithy-json 0.61.1", "aws-smithy-runtime 1.7.7", "aws-smithy-runtime-api 1.7.3", - "aws-smithy-types 1.2.11", + "aws-smithy-types 1.2.12", "aws-smithy-xml 0.60.9", - "aws-types 1.3.3", + "aws-types 1.3.4", "http 0.2.12", "once_cell", "pretty_assertions", @@ -847,10 +890,10 @@ dependencies = [ "async-std", "aws-config", "aws-credential-types 1.2.1", - "aws-runtime 1.5.3", - "aws-sigv4 1.2.6", - "aws-smithy-async 1.2.3", - "aws-smithy-checksums 0.60.13", + "aws-runtime 1.5.4", + "aws-sigv4 1.2.7", + "aws-smithy-async 1.2.4", + "aws-smithy-checksums 0.62.0", "aws-smithy-eventstream 0.60.5", "aws-smithy-experimental", "aws-smithy-http 0.60.11", @@ -858,9 +901,9 @@ dependencies = [ "aws-smithy-protocol-test 0.63.0", "aws-smithy-runtime 1.7.7", "aws-smithy-runtime-api 1.7.3", - "aws-smithy-types 1.2.11", + "aws-smithy-types 1.2.12", "aws-smithy-xml 0.60.9", - "aws-types 1.3.3", + "aws-types 1.3.4", "bytes", "bytes-utils", "fastrand 2.0.2", @@ -906,9 +949,9 @@ dependencies = [ "aws-smithy-json 0.61.1 (registry+https://github.com/rust-lang/crates.io-index)", "aws-smithy-runtime 1.7.6", "aws-smithy-runtime-api 1.7.3 (registry+https://github.com/rust-lang/crates.io-index)", - "aws-smithy-types 1.2.11 (registry+https://github.com/rust-lang/crates.io-index)", + "aws-smithy-types 1.2.11", "aws-smithy-xml 0.60.9 (registry+https://github.com/rust-lang/crates.io-index)", - "aws-types 1.3.3 (registry+https://github.com/rust-lang/crates.io-index)", + "aws-types 1.3.3", "bytes", "fastrand 2.0.2", "hex", @@ -930,16 +973,16 @@ version = "0.0.0-local" dependencies = [ "aws-config", "aws-credential-types 1.2.1", - "aws-runtime 1.5.3", - "aws-smithy-async 1.2.3", + "aws-runtime 1.5.4", + "aws-smithy-async 1.2.4", "aws-smithy-http 0.60.11", "aws-smithy-json 0.61.1", "aws-smithy-protocol-test 0.63.0", "aws-smithy-runtime 1.7.7", "aws-smithy-runtime-api 1.7.3", - "aws-smithy-types 1.2.11", + "aws-smithy-types 1.2.12", "aws-smithy-xml 0.60.9", - "aws-types 1.3.3", + "aws-types 1.3.4", "fastrand 2.0.2", "futures-util", "http 0.2.12", @@ -959,14 +1002,14 @@ name = "aws-sdk-sso" version = "0.0.0-local" dependencies = [ "aws-credential-types 1.2.1", - "aws-runtime 1.5.3", - "aws-smithy-async 1.2.3", + "aws-runtime 1.5.4", + "aws-smithy-async 1.2.4", "aws-smithy-http 0.60.11", "aws-smithy-json 0.61.1", "aws-smithy-runtime 1.7.7", "aws-smithy-runtime-api 1.7.3", - "aws-smithy-types 1.2.11", - "aws-types 1.3.3", + "aws-smithy-types 1.2.12", + "aws-types 1.3.4", "bytes", "http 0.2.12", "once_cell", @@ -980,14 +1023,14 @@ name = "aws-sdk-ssooidc" version = "0.0.0-local" dependencies = [ "aws-credential-types 1.2.1", - "aws-runtime 1.5.3", - "aws-smithy-async 1.2.3", + "aws-runtime 1.5.4", + "aws-smithy-async 1.2.4", "aws-smithy-http 0.60.11", "aws-smithy-json 0.61.1", "aws-smithy-runtime 1.7.7", "aws-smithy-runtime-api 1.7.3", - "aws-smithy-types 1.2.11", - "aws-types 1.3.3", + "aws-smithy-types 1.2.12", + "aws-types 1.3.4", "bytes", "http 0.2.12", "once_cell", @@ -1001,17 +1044,17 @@ name = "aws-sdk-sts" version = "0.0.0-local" dependencies = [ "aws-credential-types 1.2.1", - "aws-runtime 1.5.3", - "aws-smithy-async 1.2.3", + "aws-runtime 1.5.4", + "aws-smithy-async 1.2.4", "aws-smithy-http 0.60.11", "aws-smithy-json 0.61.1", "aws-smithy-protocol-test 0.63.0", "aws-smithy-query", "aws-smithy-runtime 1.7.7", "aws-smithy-runtime-api 1.7.3", - "aws-smithy-types 1.2.11", + "aws-smithy-types 1.2.12", "aws-smithy-xml 0.60.9", - "aws-types 1.3.3", + "aws-types 1.3.4", "futures-util", "http 0.2.12", "once_cell", @@ -1028,15 +1071,15 @@ version = "0.0.0-local" dependencies = [ "aws-config", "aws-credential-types 1.2.1", - "aws-runtime 1.5.3", - "aws-smithy-async 1.2.3", + "aws-runtime 1.5.4", + "aws-smithy-async 1.2.4", "aws-smithy-http 0.60.11", "aws-smithy-json 0.61.1", "aws-smithy-protocol-test 0.63.0", "aws-smithy-runtime 1.7.7", "aws-smithy-runtime-api 1.7.3", - "aws-smithy-types 1.2.11", - "aws-types 1.3.3", + "aws-smithy-types 1.2.12", + "aws-types 1.3.4", "bytes", "fastrand 2.0.2", "futures-util", @@ -1055,14 +1098,14 @@ version = "0.0.0-local" dependencies = [ "aws-config", "aws-credential-types 1.2.1", - "aws-runtime 1.5.3", - "aws-smithy-async 1.2.3", + "aws-runtime 1.5.4", + "aws-smithy-async 1.2.4", "aws-smithy-http 0.60.11", "aws-smithy-json 0.61.1", "aws-smithy-runtime 1.7.7", "aws-smithy-runtime-api 1.7.3", - "aws-smithy-types 1.2.11", - "aws-types 1.3.3", + "aws-smithy-types 1.2.12", + "aws-types 1.3.4", "bytes", "fastrand 2.0.2", "http 0.2.12", @@ -1079,17 +1122,17 @@ dependencies = [ "async-stream", "aws-config", "aws-credential-types 1.2.1", - "aws-runtime 1.5.3", - "aws-sigv4 1.2.6", - "aws-smithy-async 1.2.3", + "aws-runtime 1.5.4", + "aws-sigv4 1.2.7", + "aws-smithy-async 1.2.4", "aws-smithy-eventstream 0.60.5", "aws-smithy-http 0.60.11", "aws-smithy-json 0.61.1", "aws-smithy-protocol-test 0.63.0", "aws-smithy-runtime 1.7.7", "aws-smithy-runtime-api 1.7.3", - "aws-smithy-types 1.2.11", - "aws-types 1.3.3", + "aws-smithy-types 1.2.12", + "aws-types 1.3.4", "bytes", "futures-core", "futures-util", @@ -1111,32 +1154,25 @@ version = "0.60.3" [[package]] name = "aws-sigv4" version = "1.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d3820e0c08d0737872ff3c7c1f21ebbb6693d832312d6152bf18ef50a5471c2" dependencies = [ - "aws-credential-types 1.2.1", - "aws-smithy-eventstream 0.60.5", - "aws-smithy-http 0.60.11", - "aws-smithy-runtime-api 1.7.3", + "aws-credential-types 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)", + "aws-smithy-eventstream 0.60.5 (registry+https://github.com/rust-lang/crates.io-index)", + "aws-smithy-http 0.60.11 (registry+https://github.com/rust-lang/crates.io-index)", + "aws-smithy-runtime-api 1.7.3 (registry+https://github.com/rust-lang/crates.io-index)", "aws-smithy-types 1.2.11", "bytes", - "criterion", "crypto-bigint 0.5.5", "form_urlencoded", "hex", - "hex-literal", "hmac", "http 0.2.12", "http 1.2.0", - "httparse", - "libfuzzer-sys", "once_cell", "p256", "percent-encoding", - "pretty_assertions", - "proptest", "ring", - "serde", - "serde_derive", - "serde_json", "sha2", "subtle", "time", @@ -1146,26 +1182,33 @@ dependencies = [ [[package]] name = "aws-sigv4" -version = "1.2.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d3820e0c08d0737872ff3c7c1f21ebbb6693d832312d6152bf18ef50a5471c2" +version = "1.2.7" dependencies = [ - "aws-credential-types 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)", - "aws-smithy-eventstream 0.60.5 (registry+https://github.com/rust-lang/crates.io-index)", - "aws-smithy-http 0.60.11 (registry+https://github.com/rust-lang/crates.io-index)", - "aws-smithy-runtime-api 1.7.3 (registry+https://github.com/rust-lang/crates.io-index)", - "aws-smithy-types 1.2.11 (registry+https://github.com/rust-lang/crates.io-index)", + "aws-credential-types 1.2.1", + "aws-smithy-eventstream 0.60.5", + "aws-smithy-http 0.60.11", + "aws-smithy-runtime-api 1.7.3", + "aws-smithy-types 1.2.12", "bytes", + "criterion", "crypto-bigint 0.5.5", "form_urlencoded", "hex", + "hex-literal", "hmac", "http 0.2.12", "http 1.2.0", + "httparse", + "libfuzzer-sys", "once_cell", "p256", "percent-encoding", + "pretty_assertions", + "proptest", "ring", + "serde", + "serde_derive", + "serde_json", "sha2", "subtle", "time", @@ -1176,30 +1219,30 @@ dependencies = [ [[package]] name = "aws-smithy-async" version = "1.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "427cb637d15d63d6f9aae26358e1c9a9c09d5aa490d64b09354c8217cfef0f28" dependencies = [ "futures-util", "pin-project-lite", - "pin-utils", "tokio", - "tokio-test", ] [[package]] name = "aws-smithy-async" -version = "1.2.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "427cb637d15d63d6f9aae26358e1c9a9c09d5aa490d64b09354c8217cfef0f28" +version = "1.2.4" dependencies = [ "futures-util", "pin-project-lite", + "pin-utils", "tokio", + "tokio-test", ] [[package]] name = "aws-smithy-cbor" version = "0.60.8" dependencies = [ - "aws-smithy-types 1.2.11", + "aws-smithy-types 1.2.12", "criterion", "minicbor", ] @@ -1207,11 +1250,12 @@ dependencies = [ [[package]] name = "aws-smithy-checksums" version = "0.60.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba1a71073fca26775c8b5189175ea8863afb1c9ea2cceb02a5de5ad9dfbaa795" dependencies = [ - "aws-smithy-http 0.60.11", + "aws-smithy-http 0.60.11 (registry+https://github.com/rust-lang/crates.io-index)", "aws-smithy-types 1.2.11", "bytes", - "bytes-utils", "crc32c", "crc32fast", "hex", @@ -1219,33 +1263,33 @@ dependencies = [ "http-body 0.4.6", "md-5", "pin-project-lite", - "pretty_assertions", "sha1", "sha2", - "tokio", "tracing", - "tracing-test", ] [[package]] name = "aws-smithy-checksums" -version = "0.60.13" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ba1a71073fca26775c8b5189175ea8863afb1c9ea2cceb02a5de5ad9dfbaa795" +version = "0.62.0" dependencies = [ - "aws-smithy-http 0.60.11 (registry+https://github.com/rust-lang/crates.io-index)", - "aws-smithy-types 1.2.11 (registry+https://github.com/rust-lang/crates.io-index)", + "aws-smithy-http 0.60.11", + "aws-smithy-types 1.2.12", "bytes", + "bytes-utils", "crc32c", "crc32fast", + "crc64fast-nvme", "hex", "http 0.2.12", "http-body 0.4.6", "md-5", "pin-project-lite", + "pretty_assertions", "sha1", "sha2", + "tokio", "tracing", + "tracing-test", ] [[package]] @@ -1257,7 +1301,7 @@ name = "aws-smithy-compression" version = "0.0.2" dependencies = [ "aws-smithy-runtime-api 1.7.3", - "aws-smithy-types 1.2.11", + "aws-smithy-types 1.2.12", "bytes", "bytes-utils", "flate2", @@ -1278,7 +1322,7 @@ name = "aws-smithy-eventstream" version = "0.60.5" dependencies = [ "arbitrary", - "aws-smithy-types 1.2.11", + "aws-smithy-types 1.2.12", "bytes", "bytes-utils", "crc32fast", @@ -1291,7 +1335,7 @@ version = "0.60.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cef7d0a272725f87e51ba2bf89f8c21e4df61b9e49ae1ac367a6d69916ef7c90" dependencies = [ - "aws-smithy-types 1.2.11 (registry+https://github.com/rust-lang/crates.io-index)", + "aws-smithy-types 1.2.11", "bytes", "crc32fast", ] @@ -1303,7 +1347,7 @@ dependencies = [ "aws-smithy-async 1.2.3", "aws-smithy-runtime 1.7.7", "aws-smithy-runtime-api 1.7.3", - "aws-smithy-types 1.2.11", + "aws-smithy-types 1.2.12", "h2 0.4.7", "http 1.2.0", "hyper 1.5.2", @@ -1324,7 +1368,7 @@ dependencies = [ "async-stream", "aws-smithy-eventstream 0.60.5", "aws-smithy-runtime-api 1.7.3", - "aws-smithy-types 1.2.11", + "aws-smithy-types 1.2.12", "bytes", "bytes-utils", "futures-core", @@ -1349,7 +1393,7 @@ checksum = "5c8bc3e8fdc6b8d07d976e301c02fe553f72a39b7a9fea820e023268467d7ab6" dependencies = [ "aws-smithy-eventstream 0.60.5 (registry+https://github.com/rust-lang/crates.io-index)", "aws-smithy-runtime-api 1.7.3 (registry+https://github.com/rust-lang/crates.io-index)", - "aws-smithy-types 1.2.11 (registry+https://github.com/rust-lang/crates.io-index)", + "aws-smithy-types 1.2.11", "bytes", "bytes-utils", "futures-core", @@ -1374,7 +1418,7 @@ version = "0.60.3" name = "aws-smithy-json" version = "0.61.1" dependencies = [ - "aws-smithy-types 1.2.11", + "aws-smithy-types 1.2.12", "proptest", "serde_json", ] @@ -1385,7 +1429,7 @@ version = "0.61.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ee4e69cc50921eb913c6b662f8d909131bb3e6ad6cb6090d3a39b66fc5c52095" dependencies = [ - "aws-smithy-types 1.2.11 (registry+https://github.com/rust-lang/crates.io-index)", + "aws-smithy-types 1.2.11", ] [[package]] @@ -1394,7 +1438,7 @@ version = "0.2.1" dependencies = [ "aws-sdk-s3 1.68.0", "aws-smithy-runtime-api 1.7.3", - "aws-smithy-types 1.2.11", + "aws-smithy-types 1.2.12", "tokio", ] @@ -1438,7 +1482,7 @@ dependencies = [ name = "aws-smithy-query" version = "0.60.7" dependencies = [ - "aws-smithy-types 1.2.11", + "aws-smithy-types 1.2.12", "urlencoding", ] @@ -1513,8 +1557,8 @@ dependencies = [ name = "aws-smithy-runtime-api" version = "1.7.3" dependencies = [ - "aws-smithy-async 1.2.3", - "aws-smithy-types 1.2.11", + "aws-smithy-async 1.2.4", + "aws-smithy-types 1.2.12", "bytes", "http 0.2.12", "http 1.2.0", @@ -1531,8 +1575,8 @@ version = "1.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "92165296a47a812b267b4f41032ff8069ab7ff783696d217f0994a0d7ab585cd" dependencies = [ - "aws-smithy-async 1.2.3 (registry+https://github.com/rust-lang/crates.io-index)", - "aws-smithy-types 1.2.11 (registry+https://github.com/rust-lang/crates.io-index)", + "aws-smithy-async 1.2.3", + "aws-smithy-types 1.2.11", "bytes", "http 0.2.12", "http 1.2.0", @@ -1545,60 +1589,60 @@ dependencies = [ [[package]] name = "aws-smithy-types" version = "1.2.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38ddc9bd6c28aeb303477170ddd183760a956a03e083b3902a990238a7e3792d" dependencies = [ - "base64 0.13.1", "base64-simd", "bytes", "bytes-utils", - "ciborium", - "criterion", "futures-core", "http 0.2.12", "http 1.2.0", "http-body 0.4.6", "http-body 1.0.1", "http-body-util", - "hyper 0.14.32", "itoa", - "lazy_static", "num-integer", "pin-project-lite", "pin-utils", - "proptest", - "rand", "ryu", "serde", - "serde_json", - "tempfile", "time", "tokio", - "tokio-stream", "tokio-util", ] [[package]] name = "aws-smithy-types" -version = "1.2.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "38ddc9bd6c28aeb303477170ddd183760a956a03e083b3902a990238a7e3792d" +version = "1.2.12" dependencies = [ + "base64 0.13.1", "base64-simd", "bytes", "bytes-utils", + "ciborium", + "criterion", "futures-core", "http 0.2.12", "http 1.2.0", "http-body 0.4.6", "http-body 1.0.1", "http-body-util", + "hyper 0.14.32", "itoa", + "lazy_static", "num-integer", "pin-project-lite", "pin-utils", + "proptest", + "rand", "ryu", "serde", + "serde_json", + "tempfile", "time", "tokio", + "tokio-stream", "tokio-util", ] @@ -1606,8 +1650,8 @@ dependencies = [ name = "aws-smithy-types-convert" version = "0.60.8" dependencies = [ - "aws-smithy-async 1.2.3", - "aws-smithy-types 1.2.11", + "aws-smithy-async 1.2.4", + "aws-smithy-types 1.2.12", "chrono", "futures-core", "time", @@ -1619,7 +1663,7 @@ version = "0.1.3" dependencies = [ "aws-smithy-http 0.60.11", "aws-smithy-runtime-api 1.7.3", - "aws-smithy-types 1.2.11", + "aws-smithy-types 1.2.12", "bytes", "http 1.2.0", "tracing", @@ -1648,33 +1692,34 @@ dependencies = [ [[package]] name = "aws-types" version = "1.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5221b91b3e441e6675310829fd8984801b772cb1546ef6c0e54dec9f1ac13fef" dependencies = [ - "aws-credential-types 1.2.1", + "aws-credential-types 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)", "aws-smithy-async 1.2.3", "aws-smithy-runtime 1.7.7", "aws-smithy-runtime-api 1.7.3", "aws-smithy-types 1.2.11", - "http 0.2.12", - "hyper-rustls 0.24.2", "rustc_version", - "tempfile", - "tokio", "tracing", - "tracing-test", ] [[package]] name = "aws-types" -version = "1.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5221b91b3e441e6675310829fd8984801b772cb1546ef6c0e54dec9f1ac13fef" +version = "1.3.4" dependencies = [ - "aws-credential-types 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)", - "aws-smithy-async 1.2.3 (registry+https://github.com/rust-lang/crates.io-index)", - "aws-smithy-runtime-api 1.7.3 (registry+https://github.com/rust-lang/crates.io-index)", - "aws-smithy-types 1.2.11 (registry+https://github.com/rust-lang/crates.io-index)", + "aws-credential-types 1.2.1", + "aws-smithy-async 1.2.4", + "aws-smithy-runtime 1.7.7", + "aws-smithy-runtime-api 1.7.3", + "aws-smithy-types 1.2.12", + "http 0.2.12", + "hyper-rustls 0.24.2", "rustc_version", + "tempfile", + "tokio", "tracing", + "tracing-test", ] [[package]] @@ -1841,6 +1886,25 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" +[[package]] +name = "cbindgen" +version = "0.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3fce8dd7fcfcbf3a0a87d8f515194b49d6135acab73e18bd380d1d93bb1a15eb" +dependencies = [ + "clap", + "heck", + "indexmap", + "log", + "proc-macro2", + "quote", + "serde", + "serde_json", + "syn 2.0.94", + "tempfile", + "toml", +] + [[package]] name = "cbor-diag" version = "0.1.12" @@ -1948,8 +2012,10 @@ version = "4.5.26" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "96b01801b5fc6a0a232407abc821660c9c6d25a1cafc0d4f85f29fb8d9afc121" dependencies = [ + "anstream", "anstyle", "clap_lex", + "strsim", ] [[package]] @@ -1967,6 +2033,12 @@ dependencies = [ "cc", ] +[[package]] +name = "colorchoice" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b63caa9aa9397e2d9480a9b13673856c78d8ac123288526c37d7839f2a86990" + [[package]] name = "concurrent-queue" version = "2.5.0" @@ -2026,6 +2098,21 @@ dependencies = [ "libc", ] +[[package]] +name = "crc" +version = "3.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69e6e4d7b33a94f0991c26729976b10ebde1d34c3ee82408fb536164fa10d636" +dependencies = [ + "crc-catalog", +] + +[[package]] +name = "crc-catalog" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19d374276b40fb8bbdee95aef7c7fa6b5316ec764510eb64b8dd0e2ed0d7e7f5" + [[package]] name = "crc32c" version = "0.6.8" @@ -2044,6 +2131,16 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "crc64fast-nvme" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d5e2ee08013e3f228d6d2394116c4549a6df77708442c62d887d83f68ef2ee37" +dependencies = [ + "cbindgen", + "crc", +] + [[package]] name = "criterion" version = "0.5.1" @@ -2603,6 +2700,12 @@ dependencies = [ "num-traits", ] +[[package]] +name = "heck" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" + [[package]] name = "hermit-abi" version = "0.3.9" @@ -2998,6 +3101,12 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "is_terminal_polyfill" +version = "1.70.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7943c866cc5cd64cbc25b2e01621d07fa8eb2a1a23160ee81ce38704e97b8ecf" + [[package]] name = "itertools" version = "0.10.5" @@ -4000,6 +4109,15 @@ dependencies = [ "serde", ] +[[package]] +name = "serde_spanned" +version = "0.6.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87607cb1398ed59d48732e575a4c28a7a8ebf2454b964fe3f224f2afc07909e1" +dependencies = [ + "serde", +] + [[package]] name = "sha1" version = "0.10.6" @@ -4130,6 +4248,12 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3" +[[package]] +name = "strsim" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" + [[package]] name = "subtle" version = "2.6.1" @@ -4364,6 +4488,40 @@ dependencies = [ "tokio", ] +[[package]] +name = "toml" +version = "0.8.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1ed1f98e3fdc28d6d910e6737ae6ab1a93bf1985935a1193e68f93eeb68d24e" +dependencies = [ + "serde", + "serde_spanned", + "toml_datetime", + "toml_edit", +] + +[[package]] +name = "toml_datetime" +version = "0.6.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0dd7358ecb8fc2f8d014bf86f6f638ce72ba252a2c3a2572f2a795f1d23efb41" +dependencies = [ + "serde", +] + +[[package]] +name = "toml_edit" +version = "0.22.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ae48d6208a266e853d946088ed816055e556cc6028c5e8e2b84d9fa5dd7c7f5" +dependencies = [ + "indexmap", + "serde", + "serde_spanned", + "toml_datetime", + "winnow", +] + [[package]] name = "tower" version = "0.4.13" @@ -4560,6 +4718,12 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" +[[package]] +name = "utf8parse" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" + [[package]] name = "uuid" version = "1.12.0" @@ -4917,6 +5081,15 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" +[[package]] +name = "winnow" +version = "0.6.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39281189af81c07ec09db316b302a3e67bf9bd7cbf6c820b50e35fee9c2fa980" +dependencies = [ + "memchr", +] + [[package]] name = "wit-bindgen" version = "0.19.2" diff --git a/aws/sdk/aws-models-extra/s3-tests.smithy b/aws/sdk/aws-models-extra/s3-tests.smithy index 59ced8bc10ead37549c9e92b7ab6dbec1ce74803..bee98811f637b34d678f953ba5d5a1507ade47fe 100644 --- a/aws/sdk/aws-models-extra/s3-tests.smithy +++ b/aws/sdk/aws-models-extra/s3-tests.smithy @@ -111,9 +111,7 @@ apply PutBucketLifecycleConfiguration @httpRequestTests([ protocol: "aws.protocols#restXml", uri: "/", headers: { - // we can assert this, but when this test is promoted, it can't assert - // on the exact contents - "content-md5": "JP8DTuCSH6yDC8wNGg4+mA==", + "x-amz-checksum-crc32": "11+f3g==", }, bodyMediaType: "application/xml", body: """ diff --git a/aws/sdk/integration-tests/s3/tests/endpoints.rs b/aws/sdk/integration-tests/s3/tests/endpoints.rs index 6b4f88c8f817d046d2bb610225a615b7ef2a2669..11b370ecc3c945725449edef47dc9b4a5418db57 100644 --- a/aws/sdk/integration-tests/s3/tests/endpoints.rs +++ b/aws/sdk/integration-tests/s3/tests/endpoints.rs @@ -80,7 +80,7 @@ async fn multi_region_access_points() { let auth_header = captured_request.headers().get("AUTHORIZATION").unwrap(); // Verifies that the sigv4a signing algorithm was used, that the signing scope doesn't include a region, and that the x-amz-region-set header was signed. let expected_start = - "AWS4-ECDSA-P256-SHA256 Credential=ANOTREAL/20090213/s3/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date;x-amz-region-set;x-amz-user-agent, Signature="; + "AWS4-ECDSA-P256-SHA256 Credential=ANOTREAL/20090213/s3/aws4_request, SignedHeaders=host;x-amz-checksum-mode;x-amz-content-sha256;x-amz-date;x-amz-region-set;x-amz-user-agent, Signature="; assert!( auth_header.starts_with(expected_start), diff --git a/aws/sdk/integration-tests/s3/tests/express.rs b/aws/sdk/integration-tests/s3/tests/express.rs index 8621b6c1088a470abdb70f522625a3573420a1b9..6c9fd16131d61441c91d7240bd369bb353668c5d 100644 --- a/aws/sdk/integration-tests/s3/tests/express.rs +++ b/aws/sdk/integration-tests/s3/tests/express.rs @@ -183,7 +183,11 @@ async fn presigning() { ][..], &query_params ); - assert_eq!(presigned.headers().count(), 0); + // Presigned request has one header and that is the x-amz-checksum-mode + // header with default value ENABLED + assert_eq!(presigned.headers().count(), 1); + let headers = presigned.headers().collect::>(); + assert_eq!(headers.get(0).unwrap(), &("x-amz-checksum-mode", "ENABLED")); } fn operation_request_with_checksum( diff --git a/aws/sdk/integration-tests/s3/tests/presigning.rs b/aws/sdk/integration-tests/s3/tests/presigning.rs index 768dd563bb588bad6dfb77911b1afc77d425b8c0..f887b0643dac6465d9fea487e110c45326784070 100644 --- a/aws/sdk/integration-tests/s3/tests/presigning.rs +++ b/aws/sdk/integration-tests/s3/tests/presigning.rs @@ -100,7 +100,9 @@ async fn test_presigning() { ][..], &query_params ); - assert_eq!(presigned.headers().count(), 0); + assert_eq!(presigned.headers().count(), 1); + let headers = presigned.headers().collect::>(); + assert_eq!(headers.get("x-amz-checksum-mode").unwrap(), &"ENABLED"); } #[tokio::test] @@ -135,8 +137,8 @@ async fn test_presigning_with_payload_headers() { "X-Amz-Date=20090213T233131Z", "X-Amz-Expires=30", "X-Amz-Security-Token=notarealsessiontoken", - "X-Amz-Signature=be1d41dc392f7019750e4f5e577234fb9059dd20d15f6a99734196becce55e52", - "X-Amz-SignedHeaders=content-length%3Bcontent-type%3Bhost", + "X-Amz-Signature=9403eb5961c8af066f2473f88cb9248b39fd61f8eedc33af14ec9c2d26c22974", + "X-Amz-SignedHeaders=content-length%3Bcontent-type%3Bhost%3Bx-amz-checksum-crc32%3Bx-amz-sdk-checksum-algorithm", "x-id=PutObject" ][..], &query_params @@ -148,7 +150,9 @@ async fn test_presigning_with_payload_headers() { Some(&"application/x-test") ); assert_eq!(headers.get(CONTENT_LENGTH.as_str()), Some(&"12345")); - assert_eq!(headers.len(), 2); + assert_eq!(headers.get("x-amz-sdk-checksum-algorithm"), Some(&"CRC32")); + assert_eq!(headers.get("x-amz-checksum-crc32"), Some(&"AAAAAA==")); + assert_eq!(headers.len(), 4); } #[tokio::test] @@ -164,7 +168,7 @@ async fn test_presigned_upload_part() { }) .await; pretty_assertions::assert_eq!( - "https://bucket.s3.us-east-1.amazonaws.com/key?x-id=UploadPart&partNumber=0&uploadId=upload-id&X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=ANOTREAL%2F20090213%2Fus-east-1%2Fs3%2Faws4_request&X-Amz-Date=20090213T233131Z&X-Amz-Expires=30&X-Amz-SignedHeaders=content-length%3Bhost&X-Amz-Signature=a702867244f0bd1fb4d161e2a062520dcbefae3b9992d2e5366bcd61a60c6ddd&X-Amz-Security-Token=notarealsessiontoken", + "https://bucket.s3.us-east-1.amazonaws.com/key?x-id=UploadPart&partNumber=0&uploadId=upload-id&X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=ANOTREAL%2F20090213%2Fus-east-1%2Fs3%2Faws4_request&X-Amz-Date=20090213T233131Z&X-Amz-Expires=30&X-Amz-SignedHeaders=content-length%3Bhost%3Bx-amz-checksum-crc32%3Bx-amz-sdk-checksum-algorithm&X-Amz-Signature=0b5835e056c463d6c0963326966f6cf42c75b7a218057836274d38288e055d36&X-Amz-Security-Token=notarealsessiontoken", presigned.uri().to_string(), ); } diff --git a/aws/sdk/integration-tests/s3/tests/stalled-stream-protection.rs b/aws/sdk/integration-tests/s3/tests/stalled-stream-protection.rs index 283999d60e4c906e663f9f64ee4c03493862e033..920c95c1d6c4ff7fe61078a13c040d6e1db274dc 100644 --- a/aws/sdk/integration-tests/s3/tests/stalled-stream-protection.rs +++ b/aws/sdk/integration-tests/s3/tests/stalled-stream-protection.rs @@ -96,6 +96,9 @@ async fn test_stalled_stream_protection_defaults_for_upload() { .credentials_provider(Credentials::for_tests()) .region(Region::new("us-east-1")) .endpoint_url(format!("http://{server_addr}")) + // The Body used here is odd and fails the body.size_hint().exact() check in the streaming branch of + // the `RequestChecksumInterceptor` + .request_checksum_calculation(aws_sdk_s3::config::RequestChecksumCalculation::WhenRequired) .build(); let client = Client::from_conf(conf); diff --git a/codegen-client/src/main/kotlin/software/amazon/smithy/rust/codegen/client/smithy/generators/OperationGenerator.kt b/codegen-client/src/main/kotlin/software/amazon/smithy/rust/codegen/client/smithy/generators/OperationGenerator.kt index c2200003db0a668a8750af2b483a443e81f93945..bb4c69e29fca875e7f3539ca64b0dbfe716ce90b 100644 --- a/codegen-client/src/main/kotlin/software/amazon/smithy/rust/codegen/client/smithy/generators/OperationGenerator.kt +++ b/codegen-client/src/main/kotlin/software/amazon/smithy/rust/codegen/client/smithy/generators/OperationGenerator.kt @@ -93,7 +93,9 @@ open class OperationGenerator( *preludeScope, "Arc" to RuntimeType.Arc, "ConcreteInput" to symbolProvider.toSymbol(operationShape.inputShape(model)), - "Input" to RuntimeType.smithyRuntimeApiClient(runtimeConfig).resolve("client::interceptors::context::Input"), + "Input" to + RuntimeType.smithyRuntimeApiClient(runtimeConfig) + .resolve("client::interceptors::context::Input"), "Operation" to symbolProvider.toSymbol(operationShape), "OperationError" to errorType, "OperationOutput" to outputType, @@ -169,7 +171,9 @@ open class OperationGenerator( } """, *codegenScope, - "Error" to RuntimeType.smithyRuntimeApiClient(runtimeConfig).resolve("client::interceptors::context::Error"), + "Error" to + RuntimeType.smithyRuntimeApiClient(runtimeConfig) + .resolve("client::interceptors::context::Error"), "InterceptorContext" to RuntimeType.interceptorContext(runtimeConfig), "OrchestratorError" to RuntimeType.smithyRuntimeApiClient(runtimeConfig) diff --git a/rust-runtime/Cargo.lock b/rust-runtime/Cargo.lock index 434820db41fe8708e5c6def71b74e83a813ae210..17b5cca1c40f0db7721b74797fb251a3fcfc7b31 100644 --- a/rust-runtime/Cargo.lock +++ b/rust-runtime/Cargo.lock @@ -38,12 +38,56 @@ version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4b46cbb362ab8752921c97e041f5e366ee6297bd428a31275b9fcf1e380f7299" +[[package]] +name = "anstream" +version = "0.6.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8acc5369981196006228e28809f761875c0327210a891e941f4c683b3a99529b" +dependencies = [ + "anstyle", + "anstyle-parse", + "anstyle-query", + "anstyle-wincon", + "colorchoice", + "is_terminal_polyfill", + "utf8parse", +] + [[package]] name = "anstyle" version = "1.0.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "55cc3b69f167a1ef2e161439aa98aed94e6028e5f9a59be9a6ffb47aef1651f9" +[[package]] +name = "anstyle-parse" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b2d16507662817a6a20a9ea92df6652ee4f94f914589377d69f3b21bc5798a9" +dependencies = [ + "utf8parse", +] + +[[package]] +name = "anstyle-query" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "79947af37f4177cfead1110013d678905c37501914fba0efea834c3fe9a8d60c" +dependencies = [ + "windows-sys 0.59.0", +] + +[[package]] +name = "anstyle-wincon" +version = "3.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca3534e77181a9cc07539ad51f2141fe32f6c3ffd4df76db8ad92346b003ae4e" +dependencies = [ + "anstyle", + "once_cell", + "windows-sys 0.59.0", +] + [[package]] name = "approx" version = "0.5.1" @@ -132,9 +176,9 @@ version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "60e8f6b615cb5fc60a98132268508ad104310f0cfb25a1c22eee76efdf9154da" dependencies = [ - "aws-smithy-async 1.2.3 (registry+https://github.com/rust-lang/crates.io-index)", + "aws-smithy-async 1.2.3", "aws-smithy-runtime-api 1.7.3 (registry+https://github.com/rust-lang/crates.io-index)", - "aws-smithy-types 1.2.11 (registry+https://github.com/rust-lang/crates.io-index)", + "aws-smithy-types 1.2.11", "zeroize", ] @@ -193,7 +237,7 @@ dependencies = [ "aws-smithy-http 0.60.11 (registry+https://github.com/rust-lang/crates.io-index)", "aws-smithy-runtime 1.7.6", "aws-smithy-runtime-api 1.7.3 (registry+https://github.com/rust-lang/crates.io-index)", - "aws-smithy-types 1.2.11 (registry+https://github.com/rust-lang/crates.io-index)", + "aws-smithy-types 1.2.11", "aws-types", "bytes", "fastrand", @@ -222,7 +266,7 @@ dependencies = [ "aws-smithy-json 0.61.1 (registry+https://github.com/rust-lang/crates.io-index)", "aws-smithy-runtime 1.7.6", "aws-smithy-runtime-api 1.7.3 (registry+https://github.com/rust-lang/crates.io-index)", - "aws-smithy-types 1.2.11 (registry+https://github.com/rust-lang/crates.io-index)", + "aws-smithy-types 1.2.11", "aws-smithy-xml 0.60.9 (registry+https://github.com/rust-lang/crates.io-index)", "aws-types", "bytes", @@ -247,10 +291,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7d3820e0c08d0737872ff3c7c1f21ebbb6693d832312d6152bf18ef50a5471c2" dependencies = [ "aws-credential-types", - "aws-smithy-eventstream 0.60.5 (registry+https://github.com/rust-lang/crates.io-index)", - "aws-smithy-http 0.60.11 (registry+https://github.com/rust-lang/crates.io-index)", + "aws-smithy-eventstream 0.60.5", + "aws-smithy-http 0.60.11", "aws-smithy-runtime-api 1.7.3 (registry+https://github.com/rust-lang/crates.io-index)", - "aws-smithy-types 1.2.11 (registry+https://github.com/rust-lang/crates.io-index)", + "aws-smithy-types 1.2.11", "bytes", "crypto-bigint 0.5.5", "form_urlencoded", @@ -272,30 +316,30 @@ dependencies = [ [[package]] name = "aws-smithy-async" version = "1.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "427cb637d15d63d6f9aae26358e1c9a9c09d5aa490d64b09354c8217cfef0f28" dependencies = [ "futures-util", "pin-project-lite", - "pin-utils", "tokio", - "tokio-test", ] [[package]] name = "aws-smithy-async" -version = "1.2.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "427cb637d15d63d6f9aae26358e1c9a9c09d5aa490d64b09354c8217cfef0f28" +version = "1.2.4" dependencies = [ "futures-util", "pin-project-lite", + "pin-utils", "tokio", + "tokio-test", ] [[package]] name = "aws-smithy-cbor" version = "0.60.8" dependencies = [ - "aws-smithy-types 1.2.11", + "aws-smithy-types 1.2.12", "criterion", "minicbor", ] @@ -303,11 +347,12 @@ dependencies = [ [[package]] name = "aws-smithy-checksums" version = "0.60.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba1a71073fca26775c8b5189175ea8863afb1c9ea2cceb02a5de5ad9dfbaa795" dependencies = [ "aws-smithy-http 0.60.11", "aws-smithy-types 1.2.11", "bytes", - "bytes-utils", "crc32c", "crc32fast", "hex", @@ -315,33 +360,33 @@ dependencies = [ "http-body 0.4.6", "md-5", "pin-project-lite", - "pretty_assertions", "sha1", "sha2", - "tokio", "tracing", - "tracing-test", ] [[package]] name = "aws-smithy-checksums" -version = "0.60.13" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ba1a71073fca26775c8b5189175ea8863afb1c9ea2cceb02a5de5ad9dfbaa795" +version = "0.62.0" dependencies = [ - "aws-smithy-http 0.60.11 (registry+https://github.com/rust-lang/crates.io-index)", - "aws-smithy-types 1.2.11 (registry+https://github.com/rust-lang/crates.io-index)", + "aws-smithy-http 0.60.12", + "aws-smithy-types 1.2.12", "bytes", + "bytes-utils", "crc32c", "crc32fast", + "crc64fast-nvme", "hex", "http 0.2.12", "http-body 0.4.6", "md-5", "pin-project-lite", + "pretty_assertions", "sha1", "sha2", + "tokio", "tracing", + "tracing-test", ] [[package]] @@ -353,7 +398,7 @@ name = "aws-smithy-compression" version = "0.0.2" dependencies = [ "aws-smithy-runtime-api 1.7.3", - "aws-smithy-types 1.2.11", + "aws-smithy-types 1.2.12", "bytes", "bytes-utils", "flate2", @@ -372,24 +417,24 @@ dependencies = [ [[package]] name = "aws-smithy-eventstream" version = "0.60.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cef7d0a272725f87e51ba2bf89f8c21e4df61b9e49ae1ac367a6d69916ef7c90" dependencies = [ - "arbitrary", "aws-smithy-types 1.2.11", "bytes", - "bytes-utils", "crc32fast", - "derive_arbitrary", ] [[package]] name = "aws-smithy-eventstream" -version = "0.60.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cef7d0a272725f87e51ba2bf89f8c21e4df61b9e49ae1ac367a6d69916ef7c90" +version = "0.60.6" dependencies = [ - "aws-smithy-types 1.2.11 (registry+https://github.com/rust-lang/crates.io-index)", + "arbitrary", + "aws-smithy-types 1.2.12", "bytes", + "bytes-utils", "crc32fast", + "derive_arbitrary", ] [[package]] @@ -399,7 +444,7 @@ dependencies = [ "aws-smithy-async 1.2.3", "aws-smithy-runtime 1.7.7", "aws-smithy-runtime-api 1.7.3", - "aws-smithy-types 1.2.11", + "aws-smithy-types 1.2.12", "h2 0.4.7", "http 1.2.0", "hyper 1.5.2", @@ -416,45 +461,45 @@ dependencies = [ [[package]] name = "aws-smithy-http" version = "0.60.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c8bc3e8fdc6b8d07d976e301c02fe553f72a39b7a9fea820e023268467d7ab6" dependencies = [ - "async-stream", "aws-smithy-eventstream 0.60.5", - "aws-smithy-runtime-api 1.7.3", + "aws-smithy-runtime-api 1.7.3 (registry+https://github.com/rust-lang/crates.io-index)", "aws-smithy-types 1.2.11", "bytes", "bytes-utils", "futures-core", - "futures-util", "http 0.2.12", "http-body 0.4.6", - "hyper 0.14.32", "once_cell", "percent-encoding", "pin-project-lite", "pin-utils", - "proptest", - "tokio", "tracing", ] [[package]] name = "aws-smithy-http" -version = "0.60.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c8bc3e8fdc6b8d07d976e301c02fe553f72a39b7a9fea820e023268467d7ab6" +version = "0.60.12" dependencies = [ - "aws-smithy-eventstream 0.60.5 (registry+https://github.com/rust-lang/crates.io-index)", - "aws-smithy-runtime-api 1.7.3 (registry+https://github.com/rust-lang/crates.io-index)", - "aws-smithy-types 1.2.11 (registry+https://github.com/rust-lang/crates.io-index)", + "async-stream", + "aws-smithy-eventstream 0.60.6", + "aws-smithy-runtime-api 1.7.3", + "aws-smithy-types 1.2.12", "bytes", "bytes-utils", "futures-core", + "futures-util", "http 0.2.12", "http-body 0.4.6", + "hyper 0.14.32", "once_cell", "percent-encoding", "pin-project-lite", "pin-utils", + "proptest", + "tokio", "tracing", ] @@ -467,10 +512,10 @@ name = "aws-smithy-http-server" version = "0.63.3" dependencies = [ "aws-smithy-cbor", - "aws-smithy-http 0.60.11", - "aws-smithy-json 0.61.1", + "aws-smithy-http 0.60.12", + "aws-smithy-json 0.61.2", "aws-smithy-runtime-api 1.7.3", - "aws-smithy-types 1.2.11", + "aws-smithy-types 1.2.12", "aws-smithy-xml 0.60.9", "bytes", "futures-util", @@ -497,10 +542,10 @@ dependencies = [ name = "aws-smithy-http-server-python" version = "0.63.2" dependencies = [ - "aws-smithy-http 0.60.11", + "aws-smithy-http 0.60.12", "aws-smithy-http-server", - "aws-smithy-json 0.61.1", - "aws-smithy-types 1.2.11", + "aws-smithy-json 0.61.2", + "aws-smithy-types 1.2.12", "aws-smithy-xml 0.60.9", "bytes", "futures", @@ -539,19 +584,19 @@ version = "0.60.3" [[package]] name = "aws-smithy-json" version = "0.61.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee4e69cc50921eb913c6b662f8d909131bb3e6ad6cb6090d3a39b66fc5c52095" dependencies = [ "aws-smithy-types 1.2.11", - "proptest", - "serde_json", ] [[package]] name = "aws-smithy-json" -version = "0.61.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ee4e69cc50921eb913c6b662f8d909131bb3e6ad6cb6090d3a39b66fc5c52095" +version = "0.61.2" dependencies = [ - "aws-smithy-types 1.2.11 (registry+https://github.com/rust-lang/crates.io-index)", + "aws-smithy-types 1.2.12", + "proptest", + "serde_json", ] [[package]] @@ -560,7 +605,7 @@ version = "0.2.1" dependencies = [ "aws-sdk-s3", "aws-smithy-runtime-api 1.7.3", - "aws-smithy-types 1.2.11", + "aws-smithy-types 1.2.12", "tokio", ] @@ -604,7 +649,7 @@ dependencies = [ name = "aws-smithy-query" version = "0.60.7" dependencies = [ - "aws-smithy-types 1.2.11", + "aws-smithy-types 1.2.12", "urlencoding", ] @@ -679,8 +724,8 @@ dependencies = [ name = "aws-smithy-runtime-api" version = "1.7.3" dependencies = [ - "aws-smithy-async 1.2.3", - "aws-smithy-types 1.2.11", + "aws-smithy-async 1.2.4", + "aws-smithy-types 1.2.12", "bytes", "http 0.2.12", "http 1.2.0", @@ -697,8 +742,8 @@ version = "1.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "92165296a47a812b267b4f41032ff8069ab7ff783696d217f0994a0d7ab585cd" dependencies = [ - "aws-smithy-async 1.2.3 (registry+https://github.com/rust-lang/crates.io-index)", - "aws-smithy-types 1.2.11 (registry+https://github.com/rust-lang/crates.io-index)", + "aws-smithy-async 1.2.3", + "aws-smithy-types 1.2.11", "bytes", "http 0.2.12", "http 1.2.0", @@ -711,60 +756,60 @@ dependencies = [ [[package]] name = "aws-smithy-types" version = "1.2.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38ddc9bd6c28aeb303477170ddd183760a956a03e083b3902a990238a7e3792d" dependencies = [ - "base64 0.13.1", "base64-simd", "bytes", "bytes-utils", - "ciborium", - "criterion", "futures-core", "http 0.2.12", "http 1.2.0", "http-body 0.4.6", "http-body 1.0.1", "http-body-util", - "hyper 0.14.32", "itoa", - "lazy_static", "num-integer", "pin-project-lite", "pin-utils", - "proptest", - "rand", "ryu", "serde", - "serde_json", - "tempfile", "time", "tokio", - "tokio-stream", "tokio-util", ] [[package]] name = "aws-smithy-types" -version = "1.2.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "38ddc9bd6c28aeb303477170ddd183760a956a03e083b3902a990238a7e3792d" +version = "1.2.12" dependencies = [ + "base64 0.13.1", "base64-simd", "bytes", "bytes-utils", + "ciborium", + "criterion", "futures-core", "http 0.2.12", "http 1.2.0", "http-body 0.4.6", "http-body 1.0.1", "http-body-util", + "hyper 0.14.32", "itoa", + "lazy_static", "num-integer", "pin-project-lite", "pin-utils", + "proptest", + "rand", "ryu", "serde", + "serde_json", + "tempfile", "time", "tokio", + "tokio-stream", "tokio-util", ] @@ -772,8 +817,8 @@ dependencies = [ name = "aws-smithy-types-convert" version = "0.60.8" dependencies = [ - "aws-smithy-async 1.2.3", - "aws-smithy-types 1.2.11", + "aws-smithy-async 1.2.4", + "aws-smithy-types 1.2.12", "chrono", "futures-core", "time", @@ -783,9 +828,9 @@ dependencies = [ name = "aws-smithy-wasm" version = "0.1.3" dependencies = [ - "aws-smithy-http 0.60.11", + "aws-smithy-http 0.60.12", "aws-smithy-runtime-api 1.7.3", - "aws-smithy-types 1.2.11", + "aws-smithy-types 1.2.12", "bytes", "http 1.2.0", "tracing", @@ -818,9 +863,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5221b91b3e441e6675310829fd8984801b772cb1546ef6c0e54dec9f1ac13fef" dependencies = [ "aws-credential-types", - "aws-smithy-async 1.2.3 (registry+https://github.com/rust-lang/crates.io-index)", + "aws-smithy-async 1.2.3", "aws-smithy-runtime-api 1.7.3 (registry+https://github.com/rust-lang/crates.io-index)", - "aws-smithy-types 1.2.11 (registry+https://github.com/rust-lang/crates.io-index)", + "aws-smithy-types 1.2.11", "rustc_version", "tracing", ] @@ -1001,6 +1046,25 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" +[[package]] +name = "cbindgen" +version = "0.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3fce8dd7fcfcbf3a0a87d8f515194b49d6135acab73e18bd380d1d93bb1a15eb" +dependencies = [ + "clap 4.5.23", + "heck", + "indexmap 2.7.0", + "log", + "proc-macro2", + "quote", + "serde", + "serde_json", + "syn 2.0.94", + "tempfile", + "toml", +] + [[package]] name = "cbor-diag" version = "0.1.12" @@ -1103,7 +1167,7 @@ dependencies = [ "bitflags 1.3.2", "clap_lex 0.2.4", "indexmap 1.9.3", - "strsim", + "strsim 0.10.0", "termcolor", "textwrap", ] @@ -1123,8 +1187,10 @@ version = "4.5.26" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "96b01801b5fc6a0a232407abc821660c9c6d25a1cafc0d4f85f29fb8d9afc121" dependencies = [ + "anstream", "anstyle", "clap_lex 0.7.4", + "strsim 0.11.1", ] [[package]] @@ -1151,6 +1217,12 @@ dependencies = [ "cc", ] +[[package]] +name = "colorchoice" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b63caa9aa9397e2d9480a9b13673856c78d8ac123288526c37d7839f2a86990" + [[package]] name = "concurrent-queue" version = "2.5.0" @@ -1201,6 +1273,21 @@ dependencies = [ "libc", ] +[[package]] +name = "crc" +version = "3.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69e6e4d7b33a94f0991c26729976b10ebde1d34c3ee82408fb536164fa10d636" +dependencies = [ + "crc-catalog", +] + +[[package]] +name = "crc-catalog" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19d374276b40fb8bbdee95aef7c7fa6b5316ec764510eb64b8dd0e2ed0d7e7f5" + [[package]] name = "crc32c" version = "0.6.8" @@ -1219,6 +1306,16 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "crc64fast-nvme" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d5e2ee08013e3f228d6d2394116c4549a6df77708442c62d887d83f68ef2ee37" +dependencies = [ + "cbindgen", + "crc", +] + [[package]] name = "criterion" version = "0.5.1" @@ -1729,6 +1826,12 @@ dependencies = [ "foldhash", ] +[[package]] +name = "heck" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" + [[package]] name = "hermit-abi" version = "0.1.19" @@ -2131,7 +2234,7 @@ dependencies = [ "aws-smithy-json 0.61.1", "aws-smithy-runtime 1.7.7", "aws-smithy-runtime-api 1.7.3", - "aws-smithy-types 1.2.11", + "aws-smithy-types 1.2.12", "aws-smithy-xml 0.60.9", "bytes", "fastrand", @@ -2169,6 +2272,12 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "is_terminal_polyfill" +version = "1.70.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7943c866cc5cd64cbc25b2e01621d07fa8eb2a1a23160ee81ce38704e97b8ecf" + [[package]] name = "itertools" version = "0.10.5" @@ -2379,9 +2488,9 @@ checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" [[package]] name = "minicbor" -version = "0.24.2" +version = "0.24.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f8e213c36148d828083ae01948eed271d03f95f7e72571fa242d78184029af2" +checksum = "29be4f60e41fde478b36998b88821946aafac540e53591e76db53921a0cc225b" dependencies = [ "half", "minicbor-derive", @@ -3336,6 +3445,15 @@ dependencies = [ "serde", ] +[[package]] +name = "serde_spanned" +version = "0.6.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87607cb1398ed59d48732e575a4c28a7a8ebf2454b964fe3f224f2afc07909e1" +dependencies = [ + "serde", +] + [[package]] name = "serde_urlencoded" version = "0.7.1" @@ -3474,6 +3592,12 @@ version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" +[[package]] +name = "strsim" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" + [[package]] name = "subtle" version = "2.6.1" @@ -3743,6 +3867,40 @@ dependencies = [ "tokio", ] +[[package]] +name = "toml" +version = "0.8.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1ed1f98e3fdc28d6d910e6737ae6ab1a93bf1985935a1193e68f93eeb68d24e" +dependencies = [ + "serde", + "serde_spanned", + "toml_datetime", + "toml_edit", +] + +[[package]] +name = "toml_datetime" +version = "0.6.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0dd7358ecb8fc2f8d014bf86f6f638ce72ba252a2c3a2572f2a795f1d23efb41" +dependencies = [ + "serde", +] + +[[package]] +name = "toml_edit" +version = "0.22.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ae48d6208a266e853d946088ed816055e556cc6028c5e8e2b84d9fa5dd7c7f5" +dependencies = [ + "indexmap 2.7.0", + "serde", + "serde_spanned", + "toml_datetime", + "winnow", +] + [[package]] name = "tower" version = "0.4.13" @@ -3982,6 +4140,12 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" +[[package]] +name = "utf8parse" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" + [[package]] name = "uuid" version = "1.12.0" @@ -4246,6 +4410,15 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" +[[package]] +name = "winnow" +version = "0.6.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8d71a593cc5c42ad7876e2c1fda56f314f3754c084128833e64f1345ff8a03a" +dependencies = [ + "memchr", +] + [[package]] name = "wit-bindgen" version = "0.19.2" diff --git a/rust-runtime/aws-smithy-async/Cargo.toml b/rust-runtime/aws-smithy-async/Cargo.toml index d7c9037eff4a3d5628b7360687e85381ce3bba28..cebeec2899858f57b29fe4ab24dc9d2f8ce3e1b8 100644 --- a/rust-runtime/aws-smithy-async/Cargo.toml +++ b/rust-runtime/aws-smithy-async/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "aws-smithy-async" -version = "1.2.3" +version = "1.2.4" authors = ["AWS Rust SDK Team ", "John DiSanti "] description = "Async runtime agnostic abstractions for smithy-rs." edition = "2021" diff --git a/rust-runtime/aws-smithy-checksums/Cargo.toml b/rust-runtime/aws-smithy-checksums/Cargo.toml index 08882469924cc8a74553c359b2c0e6c5b9cc1237..6615d2fce1111406ac7c1d4824cc53819a695d4f 100644 --- a/rust-runtime/aws-smithy-checksums/Cargo.toml +++ b/rust-runtime/aws-smithy-checksums/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "aws-smithy-checksums" -version = "0.60.13" +version = "0.62.0" authors = [ "AWS Rust SDK Team ", "Zelda Hessler ", @@ -26,6 +26,7 @@ pin-project-lite = "0.2.9" sha1 = "0.10" sha2 = "0.10" tracing = "0.1" +crc64fast-nvme = "1.1.1" [dev-dependencies] bytes-utils = "0.1.2" diff --git a/rust-runtime/aws-smithy-checksums/src/http.rs b/rust-runtime/aws-smithy-checksums/src/http.rs index 01593568380bf5db3793eb7e05595a6f15084458..7d84eaf3fd0868077cfc07f48f782d8cc83b9429 100644 --- a/rust-runtime/aws-smithy-checksums/src/http.rs +++ b/rust-runtime/aws-smithy-checksums/src/http.rs @@ -8,15 +8,17 @@ use aws_smithy_types::base64; use http::header::{HeaderMap, HeaderValue}; +use crate::Crc64Nvme; use crate::{ - Checksum, Crc32, Crc32c, Md5, Sha1, Sha256, CRC_32_C_NAME, CRC_32_NAME, SHA_1_NAME, - SHA_256_NAME, + Checksum, Crc32, Crc32c, Md5, Sha1, Sha256, CRC_32_C_NAME, CRC_32_NAME, CRC_64_NVME_NAME, + SHA_1_NAME, SHA_256_NAME, }; -pub static CRC_32_HEADER_NAME: &str = "x-amz-checksum-crc32"; -pub static CRC_32_C_HEADER_NAME: &str = "x-amz-checksum-crc32c"; -pub static SHA_1_HEADER_NAME: &str = "x-amz-checksum-sha1"; -pub static SHA_256_HEADER_NAME: &str = "x-amz-checksum-sha256"; +pub const CRC_32_HEADER_NAME: &str = "x-amz-checksum-crc32"; +pub const CRC_32_C_HEADER_NAME: &str = "x-amz-checksum-crc32c"; +pub const SHA_1_HEADER_NAME: &str = "x-amz-checksum-sha1"; +pub const SHA_256_HEADER_NAME: &str = "x-amz-checksum-sha256"; +pub const CRC_64_NVME_HEADER_NAME: &str = "x-amz-checksum-crc64nvme"; // Preserved for compatibility purposes. This should never be used by users, only within smithy-rs pub(crate) static MD5_HEADER_NAME: &str = "content-md5"; @@ -24,8 +26,13 @@ pub(crate) static MD5_HEADER_NAME: &str = "content-md5"; /// When a response has to be checksum-verified, we have to check possible headers until we find the /// header with the precalculated checksum. Because a service may send back multiple headers, we have /// to check them in order based on how fast each checksum is to calculate. -pub const CHECKSUM_ALGORITHMS_IN_PRIORITY_ORDER: [&str; 4] = - [CRC_32_C_NAME, CRC_32_NAME, SHA_1_NAME, SHA_256_NAME]; +pub const CHECKSUM_ALGORITHMS_IN_PRIORITY_ORDER: [&str; 5] = [ + CRC_64_NVME_NAME, + CRC_32_C_NAME, + CRC_32_NAME, + SHA_1_NAME, + SHA_256_NAME, +]; /// Checksum algorithms are use to validate the integrity of data. Structs that implement this trait /// can be used as checksum calculators. This trait requires Send + Sync because these checksums are @@ -81,6 +88,12 @@ impl HttpChecksum for Crc32c { } } +impl HttpChecksum for Crc64Nvme { + fn header_name(&self) -> &'static str { + CRC_64_NVME_HEADER_NAME + } +} + impl HttpChecksum for Sha1 { fn header_name(&self) -> &'static str { SHA_1_HEADER_NAME @@ -104,7 +117,9 @@ mod tests { use aws_smithy_types::base64; use bytes::Bytes; - use crate::{ChecksumAlgorithm, CRC_32_C_NAME, CRC_32_NAME, SHA_1_NAME, SHA_256_NAME}; + use crate::{ + ChecksumAlgorithm, CRC_32_C_NAME, CRC_32_NAME, CRC_64_NVME_NAME, SHA_1_NAME, SHA_256_NAME, + }; use super::HttpChecksum; @@ -156,6 +171,30 @@ mod tests { assert_eq!(expected_value, actual_value) } + #[test] + fn test_trailer_length_of_crc64nvme_checksum_body() { + let checksum = CRC_64_NVME_NAME + .parse::() + .unwrap() + .into_impl(); + let expected_size = 37; + let actual_size = HttpChecksum::size(&*checksum); + assert_eq!(expected_size, actual_size) + } + + #[test] + fn test_trailer_value_of_crc64nvme_checksum_body() { + let checksum = CRC_64_NVME_NAME + .parse::() + .unwrap() + .into_impl(); + // The CRC64NVME of an empty string is all zeroes + let expected_value = Bytes::from_static(b"\0\0\0\0\0\0\0\0"); + let expected_value = base64::encode(&expected_value); + let actual_value = checksum.header_value(); + assert_eq!(expected_value, actual_value) + } + #[test] fn test_trailer_length_of_sha1_checksum_body() { let checksum = SHA_1_NAME.parse::().unwrap().into_impl(); diff --git a/rust-runtime/aws-smithy-checksums/src/lib.rs b/rust-runtime/aws-smithy-checksums/src/lib.rs index e969918dd7c35d7ead37e8c840b371635467c448..c024c2b54d51bd4180e16d4a61f00066bf4a01a5 100644 --- a/rust-runtime/aws-smithy-checksums/src/lib.rs +++ b/rust-runtime/aws-smithy-checksums/src/lib.rs @@ -17,8 +17,9 @@ //! Checksum calculation and verification callbacks. use crate::error::UnknownChecksumAlgorithmError; + use bytes::Bytes; -use std::str::FromStr; +use std::{fmt::Debug, str::FromStr}; pub mod body; pub mod error; @@ -27,18 +28,23 @@ pub mod http; // Valid checksum algorithm names pub const CRC_32_NAME: &str = "crc32"; pub const CRC_32_C_NAME: &str = "crc32c"; +pub const CRC_64_NVME_NAME: &str = "crc64nvme"; pub const SHA_1_NAME: &str = "sha1"; pub const SHA_256_NAME: &str = "sha256"; pub const MD5_NAME: &str = "md5"; /// We only support checksum calculation and validation for these checksum algorithms. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +#[non_exhaustive] pub enum ChecksumAlgorithm { + #[default] Crc32, Crc32c, + #[deprecated] Md5, Sha1, Sha256, + Crc64Nvme, } impl FromStr for ChecksumAlgorithm { @@ -47,9 +53,9 @@ impl FromStr for ChecksumAlgorithm { /// Create a new `ChecksumAlgorithm` from an algorithm name. Valid algorithm names are: /// - "crc32" /// - "crc32c" + /// - "crc64nvme" /// - "sha1" /// - "sha256" - /// - "md5" /// /// Passing an invalid name will return an error. fn from_str(checksum_algorithm: &str) -> Result { @@ -62,7 +68,10 @@ impl FromStr for ChecksumAlgorithm { } else if checksum_algorithm.eq_ignore_ascii_case(SHA_256_NAME) { Ok(Self::Sha256) } else if checksum_algorithm.eq_ignore_ascii_case(MD5_NAME) { - Ok(Self::Md5) + // MD5 is now an alias for the default Crc32 since it is deprecated + Ok(Self::Crc32) + } else if checksum_algorithm.eq_ignore_ascii_case(CRC_64_NVME_NAME) { + Ok(Self::Crc64Nvme) } else { Err(UnknownChecksumAlgorithmError::new(checksum_algorithm)) } @@ -75,7 +84,9 @@ impl ChecksumAlgorithm { match self { Self::Crc32 => Box::::default(), Self::Crc32c => Box::::default(), - Self::Md5 => Box::::default(), + Self::Crc64Nvme => Box::::default(), + #[allow(deprecated)] + Self::Md5 => Box::::default(), Self::Sha1 => Box::::default(), Self::Sha256 => Box::::default(), } @@ -86,6 +97,8 @@ impl ChecksumAlgorithm { match self { Self::Crc32 => CRC_32_NAME, Self::Crc32c => CRC_32_C_NAME, + Self::Crc64Nvme => CRC_64_NVME_NAME, + #[allow(deprecated)] Self::Md5 => MD5_NAME, Self::Sha1 => SHA_1_NAME, Self::Sha256 => SHA_256_NAME, @@ -183,6 +196,45 @@ impl Checksum for Crc32c { } } +#[derive(Default)] +struct Crc64Nvme { + hasher: crc64fast_nvme::Digest, +} + +// crc64fast_nvme::Digest doesn't impl Debug so we can't derive the impl +impl Debug for Crc64Nvme { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("Crc64Nvme").finish() + } +} + +impl Crc64Nvme { + fn update(&mut self, bytes: &[u8]) { + self.hasher.write(bytes); + } + + fn finalize(self) -> Bytes { + Bytes::copy_from_slice(self.hasher.sum64().to_be_bytes().as_slice()) + } + + // Size of the checksum in bytes + fn size() -> u64 { + 8 + } +} + +impl Checksum for Crc64Nvme { + fn update(&mut self, bytes: &[u8]) { + Self::update(self, bytes) + } + fn finalize(self: Box) -> Bytes { + Self::finalize(*self) + } + fn size(&self) -> u64 { + Self::size() + } +} + #[derive(Debug, Default)] struct Sha1 { hasher: sha1::Sha1, @@ -301,6 +353,7 @@ mod tests { use crate::http::HttpChecksum; use crate::ChecksumAlgorithm; + use aws_smithy_types::base64; use http::HeaderValue; use pretty_assertions::assert_eq; @@ -349,6 +402,20 @@ mod tests { assert_eq!(decoded_checksum, expected_checksum); } + #[test] + fn test_crc64nvme_checksum() { + use crate::{http::CRC_64_NVME_HEADER_NAME, Crc64Nvme}; + let mut checksum = Crc64Nvme::default(); + checksum.update(TEST_DATA.as_bytes()); + let checksum_result = Box::new(checksum).headers(); + let encoded_checksum = checksum_result.get(CRC_64_NVME_HEADER_NAME).unwrap(); + let decoded_checksum = base64_encoded_checksum_to_hex_string(encoded_checksum); + + let expected_checksum = "0xAECAF3AF9C98A855"; + + assert_eq!(decoded_checksum, expected_checksum); + } + #[test] fn test_sha1_checksum() { let mut checksum = Sha1::default(); diff --git a/rust-runtime/aws-smithy-eventstream/Cargo.toml b/rust-runtime/aws-smithy-eventstream/Cargo.toml index f66347ff22371d0bc9d5819350fa7beace1c75f4..d07c5bd8a585787325dafe2b2a437b6223180e0c 100644 --- a/rust-runtime/aws-smithy-eventstream/Cargo.toml +++ b/rust-runtime/aws-smithy-eventstream/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "aws-smithy-eventstream" # Only patch releases can be made to this runtime crate until https://github.com/smithy-lang/smithy-rs/issues/3370 is resolved -version = "0.60.5" +version = "0.60.6" # authors = ["AWS Rust SDK Team ", "John DiSanti "] description = "Event stream logic for smithy-rs." diff --git a/rust-runtime/aws-smithy-eventstream/fuzz/Cargo.toml b/rust-runtime/aws-smithy-eventstream/fuzz/Cargo.toml index f95d49383573018d51e8a9e5bb8b0077d6d8fdf6..7fdb9c3281b2cef07612f56e0f78c1de1977eff4 100644 --- a/rust-runtime/aws-smithy-eventstream/fuzz/Cargo.toml +++ b/rust-runtime/aws-smithy-eventstream/fuzz/Cargo.toml @@ -14,7 +14,8 @@ aws-smithy-types = { path = "../../aws-smithy-types" } bytes = "1" crc32fast = "1" derive_arbitrary = "1.3" -libfuzzer-sys = "0.4" +# Version pinned due to https://github.com/rust-fuzz/libfuzzer/issues/126 +libfuzzer-sys = "=0.4.7" [dependencies.aws-smithy-eventstream] features = ["derive-arbitrary"] diff --git a/rust-runtime/aws-smithy-http/Cargo.toml b/rust-runtime/aws-smithy-http/Cargo.toml index b915bc3941f6b2be8b8175cfae1a42e7597232d9..2136d2cc605a543ad7a129b508793a47de9dd5e9 100644 --- a/rust-runtime/aws-smithy-http/Cargo.toml +++ b/rust-runtime/aws-smithy-http/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "aws-smithy-http" -version = "0.60.11" +version = "0.60.12" authors = [ "AWS Rust SDK Team ", "Russell Cohen ", diff --git a/rust-runtime/aws-smithy-http/fuzz/Cargo.toml b/rust-runtime/aws-smithy-http/fuzz/Cargo.toml index 6b2597277ee9f51fdedcdaec88d9192072a6fd52..040c40e76c30922c65e6421dbb63fee9168d9562 100644 --- a/rust-runtime/aws-smithy-http/fuzz/Cargo.toml +++ b/rust-runtime/aws-smithy-http/fuzz/Cargo.toml @@ -9,7 +9,8 @@ edition = "2021" cargo-fuzz = true [dependencies] -libfuzzer-sys = "0.4" +# Version pinned due to https://github.com/rust-fuzz/libfuzzer/issues/126 +libfuzzer-sys = "=0.4.7" http = "0.2.3" [dependencies.aws-smithy-http] diff --git a/rust-runtime/aws-smithy-json/Cargo.toml b/rust-runtime/aws-smithy-json/Cargo.toml index c758e5da6845ee72d0b029cdecf02732107ab11b..92105e7448d799f8368525309b9796a3af097a74 100644 --- a/rust-runtime/aws-smithy-json/Cargo.toml +++ b/rust-runtime/aws-smithy-json/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "aws-smithy-json" -version = "0.61.1" +version = "0.61.2" authors = ["AWS Rust SDK Team ", "John DiSanti "] description = "Token streaming JSON parser for smithy-rs." edition = "2021" diff --git a/rust-runtime/aws-smithy-json/fuzz/Cargo.toml b/rust-runtime/aws-smithy-json/fuzz/Cargo.toml index 435cc1ab327933e6693df9821d3d894c9aff89e4..94cf33cfa022dbccea51f7ac3ad37e827388da49 100644 --- a/rust-runtime/aws-smithy-json/fuzz/Cargo.toml +++ b/rust-runtime/aws-smithy-json/fuzz/Cargo.toml @@ -11,7 +11,8 @@ cargo-fuzz = true [dependencies] aws-smithy-json = { path = ".." } aws-smithy-types = { path = "../../aws-smithy-types" } -libfuzzer-sys = "0.4" +# Version pinned due to https://github.com/rust-fuzz/libfuzzer/issues/126 +libfuzzer-sys = "=0.4.7" serde_json = { version = "1", features = ["float_roundtrip"] } # Prevent this from interfering with workspaces diff --git a/rust-runtime/aws-smithy-types/Cargo.toml b/rust-runtime/aws-smithy-types/Cargo.toml index 249721369777734326844c24df5e0db3dff7c6c1..8894e5ee7102a4247173e0ef77f971efda78835e 100644 --- a/rust-runtime/aws-smithy-types/Cargo.toml +++ b/rust-runtime/aws-smithy-types/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "aws-smithy-types" -version = "1.2.11" +version = "1.2.12" authors = [ "AWS Rust SDK Team ", "Russell Cohen ", diff --git a/rust-runtime/aws-smithy-types/fuzz/Cargo.toml b/rust-runtime/aws-smithy-types/fuzz/Cargo.toml index da3fdda22780c627095fb4faddb7d40ad34a4e6e..1d74f391ca75b641ddab22f387b17fb90856b284 100644 --- a/rust-runtime/aws-smithy-types/fuzz/Cargo.toml +++ b/rust-runtime/aws-smithy-types/fuzz/Cargo.toml @@ -9,7 +9,8 @@ edition = "2021" cargo-fuzz = true [dependencies] -libfuzzer-sys = "0.4" +# Version pinned due to https://github.com/rust-fuzz/libfuzzer/issues/126 +libfuzzer-sys = "=0.4.7" [dependencies.aws-smithy-types] path = ".." diff --git a/rust-runtime/aws-smithy-types/src/checksum_config.rs b/rust-runtime/aws-smithy-types/src/checksum_config.rs new file mode 100644 index 0000000000000000000000000000000000000000..a44c7bd4322f9b35a77101587668953e6beb8815 --- /dev/null +++ b/rust-runtime/aws-smithy-types/src/checksum_config.rs @@ -0,0 +1,163 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +//! Types that allow users to indicate their preferences for checksum calculation and validation + +// Note: These types would likely make more sense in `aws-smithy-checksums` and were originally +// added there. But we have lints protecting against exporting non-stable types from stable crates +// and the checksums crate is not yet 1.0, so these types cannot live there for now. In the future +// if we do decide to 1.0 the checksums crate we can move these types there and re-export them here +// to maintain the current behavior. + +use std::error::Error; +use std::fmt; +use std::str::FromStr; + +use crate::config_bag::{Storable, StoreReplace}; + +// Valid names for RequestChecksumCalculation and ResponseChecksumValidation +const WHEN_SUPPORTED: &str = "when_supported"; +const WHEN_REQUIRED: &str = "when_required"; + +/// Determines when a checksum will be calculated for request payloads. Values are: +/// * [RequestChecksumCalculation::WhenSupported] - (default) When set, a checksum will be +/// calculated for all request payloads of operations modeled with the +/// `httpChecksum` trait where `requestChecksumRequired` is `true` and/or a +/// `requestAlgorithmMember` is modeled. +/// * [RequestChecksumCalculation::WhenRequired] - When set, a checksum will only be calculated for +/// request payloads of operations modeled with the `httpChecksum` trait where +/// `requestChecksumRequired` is `true` or where a requestAlgorithmMember +/// is modeled and supplied. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +#[non_exhaustive] +pub enum RequestChecksumCalculation { + /// Calculate request checksums when they are supported. + #[default] + WhenSupported, + /// Caulculate request checksums only when they are required. + WhenRequired, +} + +impl Storable for RequestChecksumCalculation { + type Storer = StoreReplace; +} + +impl FromStr for RequestChecksumCalculation { + type Err = UnknownRequestChecksumCalculationError; + + fn from_str(request_checksum_calculation: &str) -> Result { + if request_checksum_calculation.eq_ignore_ascii_case(WHEN_SUPPORTED) { + Ok(Self::WhenSupported) + } else if request_checksum_calculation.eq_ignore_ascii_case(WHEN_REQUIRED) { + Ok(Self::WhenRequired) + } else { + Err(UnknownRequestChecksumCalculationError::new( + request_checksum_calculation, + )) + } + } +} + +/// Determines when checksum validation will be performed on response payloads. Values are: +/// * [ResponseChecksumValidation::WhenSupported] - (default) When set, checksum validation is performed on all +/// response payloads of operations modeled with the `httpChecksum` trait where +/// `responseAlgorithms` is modeled, except when no modeled checksum algorithms +/// are supported. +/// * [ResponseChecksumValidation::WhenRequired] - When set, checksum validation is not performed on +/// response payloads of operations unless the checksum algorithm is supported and +/// the `requestValidationModeMember` member is set to `ENABLED`. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +#[non_exhaustive] +pub enum ResponseChecksumValidation { + /// Validate response checksums when they are supported. + #[default] + WhenSupported, + /// Validate response checksums only when they are required. + WhenRequired, +} + +impl Storable for ResponseChecksumValidation { + type Storer = StoreReplace; +} + +impl FromStr for ResponseChecksumValidation { + type Err = UnknownResponseChecksumValidationError; + + fn from_str(response_checksum_validation: &str) -> Result { + if response_checksum_validation.eq_ignore_ascii_case(WHEN_SUPPORTED) { + Ok(Self::WhenSupported) + } else if response_checksum_validation.eq_ignore_ascii_case(WHEN_REQUIRED) { + Ok(Self::WhenRequired) + } else { + Err(UnknownResponseChecksumValidationError::new( + response_checksum_validation, + )) + } + } +} + +/// Unknown setting for `request_checksum_calculation` +#[derive(Debug)] +#[non_exhaustive] +pub struct UnknownRequestChecksumCalculationError { + request_checksum_calculation: String, +} + +impl UnknownRequestChecksumCalculationError { + pub(crate) fn new(request_checksum_calculation: impl Into) -> Self { + Self { + request_checksum_calculation: request_checksum_calculation.into(), + } + } + + /// The unknown value + pub fn request_checksum_calculation(&self) -> &str { + &self.request_checksum_calculation + } +} + +impl fmt::Display for UnknownRequestChecksumCalculationError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!( + f, + r#"unknown request_checksum_calculation value "{}", please pass a known name ("when_supported", "when_required")"#, + self.request_checksum_calculation + ) + } +} + +impl Error for UnknownRequestChecksumCalculationError {} + +/// Unknown setting for `response_checksum_validation` +#[derive(Debug)] +#[non_exhaustive] +pub struct UnknownResponseChecksumValidationError { + response_checksum_validation: String, +} + +impl UnknownResponseChecksumValidationError { + pub(crate) fn new(response_checksum_validation: impl Into) -> Self { + Self { + response_checksum_validation: response_checksum_validation.into(), + } + } + + /// The unknown value + pub fn response_checksum_validation(&self) -> &str { + &self.response_checksum_validation + } +} + +impl fmt::Display for UnknownResponseChecksumValidationError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!( + f, + r#"unknown response_checksum_validation value "{}", please pass a known name ("when_supported", "when_required")"#, + self.response_checksum_validation + ) + } +} + +impl Error for UnknownResponseChecksumValidationError {} diff --git a/rust-runtime/aws-smithy-types/src/lib.rs b/rust-runtime/aws-smithy-types/src/lib.rs index ed53de191b46fee08cdbf219b2fe5d81416c1ed8..5af73e1fb182b8d630e1c78fb2b158bd5b8c4193 100644 --- a/rust-runtime/aws-smithy-types/src/lib.rs +++ b/rust-runtime/aws-smithy-types/src/lib.rs @@ -20,6 +20,7 @@ pub mod base64; pub mod body; pub mod byte_stream; +pub mod checksum_config; /// A typemap for storing configuration. pub mod config_bag; pub mod date_time;