Unverified Commit 2ad87c0e authored by Zelda Hessler's avatar Zelda Hessler Committed by GitHub
Browse files

feature: support flexible checksums (#1561) 

feature: support flexible checksums
update: s3 model
update: CHANGELOG.next.toml
parent 7d2a6595

CHANGELOG.next.toml

+57 −0
Original line number Diff line number Diff line
@@ -143,6 +143,63 @@ references = ["smithy-rs#1157"] 
meta = { "breaking" = true, "tada" = false, "bug" = false }

author = "82marbag"



[[aws-sdk-rust]]

message = """

The AWS SDK for Rust now supports [additional checksum algorithms for Amazon S3](https://aws.amazon.com/blogs/aws/new-additional-checksum-algorithms-for-amazon-s3/).

When getting and putting objects, you may now request that the request body be validated with a checksum. The supported

algorithms are SHA-1, SHA-256, CRC-32, and CRC-32C.



```rust

#[tokio::main]

async fn main() -> Result<(), Box<dyn std::error::Error>> {

    let sdk_config = aws_config::load_from_env().await;

    let s3_client = aws_sdk_s3::Client::new(&sdk_config);

    let body = aws_sdk_s3::types::ByteStream::read_from()

        .path(std::path::Path::new("./path/to/your/file.txt"))

        .build()

        .await

        .unwrap();



    let _ = s3_client

        .put_object()

        .bucket("your-bucket")

        .key("file.txt")

        .body(body)

        // When using this field, the checksum will be calculated for you

        .checksum_algorithm(aws_sdk_s3::model::ChecksumAlgorithm::Crc32C)

        .send()

        .await?;



    let body = aws_sdk_s3::types::ByteStream::read_from()

        .path(std::path::Path::new("./path/to/your/other-file.txt"))

        .build()

        .await

        .unwrap();



    let _ = s3_client

        .put_object()

        .bucket("your-bucket")

        .key("other-file.txt")

        .body(body)

        // Alternatively, you can pass a checksum that you've calculated yourself. It must be base64

        // encoded. Also, make sure that you're base64 encoding the bytes of the checksum, not its

        // string representation.

        .checksum_crc32_c(aws_smithy_types::base64::encode(&A_PRECALCULATED_CRC_32_C_CHECKSUM[..]))

        .send()

        .await?;

}

```

"""

references = ["smithy-rs#1482"]

meta = { "breaking" = false, "tada" = true, "bug" = false }

author = "Velfi"



[[smithy-rs]]

message = "Update codegen to generate support for flexible checksums."

references = ["smithy-rs#1482"]

meta = { "breaking" = false, "tada" = true, "bug" = false }

author = "Velfi"



[[aws-sdk-rust]]

message = "SDK crate READMEs now include an example of creating a client"

references = ["smithy-rs#1571", "smithy-rs#1385"]

aws/rust-runtime/aws-http/src/content_encoding.rs

+6 −2
Original line number Diff line number Diff line
@@ -44,7 +44,9 @@ impl AwsChunkedBodyOptions { 
    }



    fn total_trailer_length(&self) -> u64 {

        self.trailer_lengths.iter().sum()

        self.trailer_lengths.iter().sum::<u64>()

            // We need to account for a CRLF after each trailer name/value pair

            + (self.trailer_lengths.len() * CRLF.len()) as u64

    }



    /// Set a trailer len
@@ -519,10 +521,12 @@ mod tests { 
    }



    #[tokio::test]

    #[should_panic = "called `Result::unwrap()` on an `Err` value: ReportedTrailerLengthMismatch { actual: 42, expected: 0 }"]

    #[should_panic = "called `Result::unwrap()` on an `Err` value: ReportedTrailerLengthMismatch { actual: 44, expected: 0 }"]

    async fn test_aws_chunked_encoding_incorrect_trailer_length_panic() {

        let input_str = "Hello world";

        // Test body has no trailers, so this length is incorrect and will trigger an assert panic

        // When the panic occurs, it will actually expect a length of 44. This is because, when using

        // aws-chunked encoding, each trailer will end with a CRLF which is 2 bytes long.

        let wrong_trailer_len = 42;

        let opts = AwsChunkedBodyOptions::new(input_str.len() as u64, vec![wrong_trailer_len]);

        let mut body = AwsChunkedBody::new(SdkBody::from(input_str), opts);

aws/rust-runtime/aws-inlineable/Cargo.toml

+17 −14
Original line number Diff line number Diff line
@@ -12,29 +12,32 @@ publish = false 
repository = "https://github.com/awslabs/smithy-rs"



[dependencies]

aws-smithy-http = { path = "../../../rust-runtime/aws-smithy-http" }

aws-smithy-client = { path = "../../../rust-runtime/aws-smithy-client" }

aws-http = { path = "../aws-http" }

aws-endpoint = { path = "../aws-endpoint" }

aws-smithy-types = { path = "../../../rust-runtime/aws-smithy-types" }

aws-http = { path = "../aws-http" }

aws-sig-auth = { path = "../../rust-runtime/aws-sig-auth" }

aws-smithy-checksums = { path = "../../../rust-runtime/aws-smithy-checksums" }

aws-smithy-client = { path = "../../../rust-runtime/aws-smithy-client" }

aws-smithy-http = { path = "../../../rust-runtime/aws-smithy-http" }

aws-smithy-http-tower= { path = "../../../rust-runtime/aws-smithy-http-tower" }

aws-smithy-types = { path = "../../../rust-runtime/aws-smithy-types" }

aws-types = { path = "../../rust-runtime/aws-types" }

aws-sig-auth = { path = "../../rust-runtime/aws-sig-auth" }

bytes = "1"

bytes-utils = "0.1.1"

hex = "0.4.3"

http = "0.2.4"

tower = { version = "0.4", default-features = false }



# Checksum dependencies:

http-body = "0.4.5"

md-5 = "0.10.1"

ring = "0.16"

bytes-utils = "0.1.1"

tokio-stream = "0.1.7"

bytes = "1"

tokio = { version = "1", features = ["full"] }

hex = "0.4.3"

tokio-stream = "0.1.7"

tower = { version = "0.4", default-features = false }

tracing = "0.1"



[dev-dependencies]

temp-file = "0.1.6"

aws-smithy-http = { path = "../../../rust-runtime/aws-smithy-http", features = ["rt-tokio"] }

aws-smithy-client = { path = "../../../rust-runtime/aws-smithy-client", features = ["test-util"] }

aws-smithy-http = { path = "../../../rust-runtime/aws-smithy-http", features = ["rt-tokio"] }

tempfile = "3.2.0"

tracing-subscriber = { version = "0.3.5", features = ["env-filter"] }



[package.metadata.docs.rs]

all-features = true

aws/rust-runtime/aws-inlineable/src/glacier_checksums.rs

+1 −1
Original line number Diff line number Diff line
@@ -200,7 +200,7 @@ mod test { 
        while test_data.len() < total_size {

            test_data.extend_from_slice(base_seq)

        }

        let target = temp_file::empty();

        let target = tempfile::NamedTempFile::new().unwrap();

        tokio::fs::write(target.path(), test_data).await.unwrap();

        let body = ByteStream::from_path(target.path())

            .await

aws/rust-runtime/aws-inlineable/src/http_body_checksum.rs

0 → 100644
+345 −0
Original line number Diff line number Diff line 
/*

 * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.

 * SPDX-License-Identifier: Apache-2.0

 */



//! Functions for modifying requests and responses for the purposes of checksum validation



use http::header::HeaderName;



/// Errors related to constructing checksum-validated HTTP requests

#[derive(Debug)]

#[allow(dead_code)]

pub(crate) enum Error {

    /// Only request bodies with a known size can be checksum validated

    UnsizedRequestBody,

    ChecksumHeadersAreUnsupportedForStreamingBody,

}



impl std::fmt::Display for Error {

    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {

        match self {

            Self::UnsizedRequestBody => write!(

                f,

                "Only request bodies with a known size can be checksum validated."

            ),

            Self::ChecksumHeadersAreUnsupportedForStreamingBody => write!(

                f,

                "Checksum header insertion is only supported for non-streaming HTTP bodies. \

                 To checksum validate a streaming body, the checksums must be sent as trailers."

            ),

        }

    }

}



impl std::error::Error for Error {}



/// Given a `&mut http::request::Request` and a `aws_smithy_checksums::ChecksumAlgorithm`,

/// calculate a checksum and modify the request to include the checksum as a header

/// (for in-memory request bodies) or a trailer (for streaming request bodies.) Streaming bodies

/// must be sized or this will return an error.

#[allow(dead_code)]

pub(crate) fn add_checksum_calculation_to_request(

    request: &mut http::request::Request<aws_smithy_http::body::SdkBody>,

    property_bag: &mut aws_smithy_http::property_bag::PropertyBag,

    checksum_algorithm: aws_smithy_checksums::ChecksumAlgorithm,

) -> Result<(), aws_smithy_http::operation::BuildError> {

    match request.body().bytes() {

        // Body is in-memory: read it and insert the checksum as a header.

        Some(data) => {

            let mut checksum = checksum_algorithm.into_impl();

            checksum.update(data);



            request

                .headers_mut()

                .insert(checksum.header_name(), checksum.header_value());

        }

        // Body is streaming: wrap the body so it will emit a checksum as a trailer.

        None => {

            wrap_streaming_request_body_in_checksum_calculating_body(

                request,

                property_bag,

                checksum_algorithm,

            )?;

        }

    }



    Ok(())

}



#[allow(dead_code)]

fn wrap_streaming_request_body_in_checksum_calculating_body(

    request: &mut http::request::Request<aws_smithy_http::body::SdkBody>,

    property_bag: &mut aws_smithy_http::property_bag::PropertyBag,

    checksum_algorithm: aws_smithy_checksums::ChecksumAlgorithm,

) -> Result<(), aws_smithy_http::operation::BuildError> {

    use aws_http::content_encoding::{AwsChunkedBody, AwsChunkedBodyOptions};

    use aws_smithy_checksums::{body::calculate, http::HttpChecksum};

    use http_body::Body;



    let original_body_size = request.body().size_hint().exact().ok_or_else(|| {

        aws_smithy_http::operation::BuildError::Other(Box::new(Error::UnsizedRequestBody))

    })?;



    // Streaming request bodies with trailers require special signing

    property_bag.insert(aws_sig_auth::signer::SignableBody::StreamingUnsignedPayloadTrailer);



    let mut body = {

        let body = std::mem::replace(request.body_mut(), aws_smithy_http::body::SdkBody::taken());



        body.map(move |body| {

            let checksum = checksum_algorithm.into_impl();

            let trailer_len = HttpChecksum::size(checksum.as_ref());

            let body = calculate::ChecksumBody::new(body, checksum);

            let aws_chunked_body_options =

                AwsChunkedBodyOptions::new(original_body_size, vec![trailer_len]);



            let body = AwsChunkedBody::new(body, aws_chunked_body_options);



            aws_smithy_http::body::SdkBody::from_dyn(aws_smithy_http::body::BoxBody::new(body))

        })

    };



    let encoded_content_length = body.size_hint().exact().ok_or_else(|| {

        aws_smithy_http::operation::BuildError::Other(Box::new(Error::UnsizedRequestBody))

    })?;



    let headers = request.headers_mut();



    headers.insert(

        http::header::HeaderName::from_static("x-amz-trailer"),

        // Convert into a `HeaderName` and then into a `HeaderValue`

        http::header::HeaderName::from(checksum_algorithm).into(),

    );



    headers.insert(

        http::header::CONTENT_LENGTH,

        http::HeaderValue::from(encoded_content_length),

    );

    headers.insert(

        http::header::HeaderName::from_static("x-amz-decoded-content-length"),

        http::HeaderValue::from(original_body_size),

    );

    headers.insert(

        http::header::CONTENT_ENCODING,

        http::HeaderValue::from_str(aws_http::content_encoding::header_value::AWS_CHUNKED)

            .map_err(|err| aws_smithy_http::operation::BuildError::Other(Box::new(err)))

            .expect("\"aws-chunked\" will always be a valid HeaderValue"),

    );



    std::mem::swap(request.body_mut(), &mut body);



    Ok(())

}



/// Given an `SdkBody`, a `aws_smithy_checksums::ChecksumAlgorithm`, and a pre-calculated checksum,

/// return an `SdkBody` where the body will processed with the checksum algorithm and checked

/// against the pre-calculated checksum.

#[allow(dead_code)]

pub(crate) fn wrap_body_with_checksum_validator(

    body: aws_smithy_http::body::SdkBody,

    checksum_algorithm: aws_smithy_checksums::ChecksumAlgorithm,

    precalculated_checksum: bytes::Bytes,

) -> aws_smithy_http::body::SdkBody {

    use aws_smithy_checksums::body::validate;

    use aws_smithy_http::body::{BoxBody, SdkBody};



    body.map(move |body| {

        SdkBody::from_dyn(BoxBody::new(validate::ChecksumBody::new(

            body,

            checksum_algorithm.into_impl(),

            precalculated_checksum.clone(),

        )))

    })

}



/// Given a `HeaderMap`, extract any checksum included in the headers as `Some(Bytes)`.

/// If no checksum header is set, return `None`. If multiple checksum headers are set, the one that

/// is fastest to compute will be chosen.

#[allow(dead_code)]

pub(crate) fn check_headers_for_precalculated_checksum(

    headers: &http::HeaderMap<http::HeaderValue>,

    response_algorithms: &[&str],

) -> Option<(aws_smithy_checksums::ChecksumAlgorithm, bytes::Bytes)> {

    let checksum_algorithms_to_check =

        aws_smithy_checksums::http::CHECKSUM_ALGORITHMS_IN_PRIORITY_ORDER

            .into_iter()

            // Process list of algorithms, from fastest to slowest, that may have been used to checksum

            // the response body, ignoring any that aren't marked as supported algorithms by the model.

            .flat_map(|algo| {

                // For loop is necessary b/c the compiler doesn't infer the correct lifetimes for iter().find()

                for res_algo in response_algorithms {

                    if algo.eq_ignore_ascii_case(res_algo) {

                        return Some(algo);

                    }

                }



                None

            });



    for checksum_algorithm in checksum_algorithms_to_check {

        let checksum_algorithm: aws_smithy_checksums::ChecksumAlgorithm = checksum_algorithm.parse().expect(

            "CHECKSUM_ALGORITHMS_IN_PRIORITY_ORDER only contains valid checksum algorithm names",

        );

        if let Some(precalculated_checksum) = headers.get(HeaderName::from(checksum_algorithm)) {

            let base64_encoded_precalculated_checksum = precalculated_checksum

                .to_str()

                .expect("base64 uses ASCII characters");



            let precalculated_checksum: bytes::Bytes =

                aws_smithy_types::base64::decode(base64_encoded_precalculated_checksum)

                    .expect("services will always base64 encode the checksum value per the spec")

                    .into();



            return Some((checksum_algorithm, precalculated_checksum));

        }

    }



    None

}



#[cfg(test)]

mod tests {

    use super::wrap_body_with_checksum_validator;

    use aws_smithy_checksums::ChecksumAlgorithm;

    use aws_smithy_http::body::SdkBody;

    use aws_smithy_http::byte_stream::ByteStream;

    use bytes::{Bytes, BytesMut};

    use http_body::Body;

    use std::sync::Once;

    use tempfile::NamedTempFile;



    static INIT_LOGGER: Once = Once::new();

    fn init_logger() {

        INIT_LOGGER.call_once(|| {

            tracing_subscriber::fmt::init();

        });

    }



    #[tokio::test]

    async fn test_checksum_body_is_retryable() {

        let input_text = "Hello world";

        let precalculated_checksum = Bytes::from_static(&[0x8b, 0xd6, 0x9e, 0x52]);

        let body = SdkBody::retryable(move || SdkBody::from(input_text));



        // ensure original SdkBody is retryable

        assert!(body.try_clone().is_some());



        let body = body.map(move |sdk_body| {

            let checksum_algorithm: ChecksumAlgorithm = "crc32".parse().unwrap();

            wrap_body_with_checksum_validator(

                sdk_body,

                checksum_algorithm,

                precalculated_checksum.clone(),

            )

        });



        // ensure wrapped SdkBody is retryable

        let mut body = body.try_clone().expect("body is retryable");



        let mut validated_body = BytesMut::new();



        loop {

            match body.data().await {

                Some(Ok(data)) => validated_body.extend_from_slice(&data),

                Some(Err(err)) => panic!("{}", err),

                None => {

                    break;

                }

            }

        }



        let body = std::str::from_utf8(&validated_body).unwrap();



        // ensure that the wrapped body passes checksum validation

        assert_eq!(input_text, body);

    }



    #[tokio::test]

    async fn test_checksum_body_from_file_is_retryable() {

        use std::io::Write;

        let mut file = NamedTempFile::new().unwrap();

        let checksum_algorithm: ChecksumAlgorithm = "crc32c".parse().unwrap();

        let mut crc32c_checksum = checksum_algorithm.into_impl();



        for i in 0..10000 {

            let line = format!("This is a large file created for testing purposes {}", i);

            file.as_file_mut().write(line.as_bytes()).unwrap();

            crc32c_checksum.update(&line.as_bytes());

        }



        let body = ByteStream::read_from()

            .path(&file)

            .buffer_size(1024)

            .build()

            .await

            .unwrap();



        let precalculated_checksum = crc32c_checksum.finalize();

        let expected_checksum = precalculated_checksum.clone();



        let body = body.map(move |sdk_body| {

            wrap_body_with_checksum_validator(

                sdk_body,

                checksum_algorithm,

                precalculated_checksum.clone(),

            )

        });



        // ensure wrapped SdkBody is retryable

        let mut body = body.into_inner().try_clone().expect("body is retryable");



        let mut validated_body = BytesMut::new();



        // If this loop completes, then it means the body's checksum was valid, but let's calculate

        // a checksum again just in case.

        let mut redundant_crc32c_checksum = checksum_algorithm.into_impl();

        loop {

            match body.data().await {

                Some(Ok(data)) => {

                    redundant_crc32c_checksum.update(&data);

                    validated_body.extend_from_slice(&data);

                }

                Some(Err(err)) => panic!("{}", err),

                None => {

                    break;

                }

            }

        }



        let actual_checksum = redundant_crc32c_checksum.finalize();

        assert_eq!(expected_checksum, actual_checksum);



        // Ensure the file's checksum isn't the same as an empty checksum. This way, we'll know that

        // data was actually processed.

        let unexpected_checksum = checksum_algorithm.into_impl().finalize();

        assert_ne!(unexpected_checksum, actual_checksum);

    }



    #[tokio::test]

    async fn test_build_checksum_validated_body_works() {

        init_logger();



        let checksum_algorithm = "crc32".parse().unwrap();

        let input_text = "Hello world";

        let precalculated_checksum = Bytes::from_static(&[0x8b, 0xd6, 0x9e, 0x52]);

        let body = ByteStream::new(SdkBody::from(input_text));



        let body = body.map(move |sdk_body| {

            wrap_body_with_checksum_validator(

                sdk_body,

                checksum_algorithm,

                precalculated_checksum.clone(),

            )

        });



        let mut validated_body = Vec::new();

        if let Err(e) = tokio::io::copy(&mut body.into_async_read(), &mut validated_body).await {

            tracing::error!("{}", e);

            panic!("checksum validation has failed");

        };

        let body = std::str::from_utf8(&validated_body).unwrap();



        assert_eq!(input_text, body);

    }

}