Unverified Commit 1a550b1d authored by Nugine's avatar Nugine Committed by GitHub
Browse files

fix(s3s/sig_v4): single chunk upload (#369) 



* refactor(s3s/sig_v4): x-amz-content-sha256

* refactor(s3s/sig_v4): hashed payload

* fix(s3s/sig_v4): use UploadStream

* Update crates/s3s/src/sig_v4/upload_stream.rs

Co-authored-by: default avatarCopilot <175728472+Copilot@users.noreply.github.com>

* fix

* relax

---------

Co-authored-by: default avatarCopilot <175728472+Copilot@users.noreply.github.com>
parent f7d07d0d

crates/s3s-fs/tests/it_aws.rs

+10 −0
Original line number Diff line number Diff line
@@ -154,6 +154,8 @@ macro_rules! log_and_unwrap { 
#[tokio::test]

#[tracing::instrument]

async fn test_list_buckets() -> Result<()> {

    let _guard = serial().await;



    let c = Client::new(config());

    let response1 = log_and_unwrap!(c.list_buckets().send().await);

    drop(response1);
@@ -177,6 +179,8 @@ async fn test_list_buckets() -> Result<()> { 
#[tokio::test]

#[tracing::instrument]

async fn test_list_objects_v2() -> Result<()> {

    let _guard = serial().await;



    let c = Client::new(config());

    let bucket = format!("test-list-objects-v2-{}", Uuid::new_v4());

    let bucket_str = bucket.as_str();
@@ -219,6 +223,8 @@ async fn test_list_objects_v2() -> Result<()> { 
#[tokio::test]

#[tracing::instrument]

async fn test_list_objects_v2_with_prefixes() -> Result<()> {

    let _guard = serial().await;



    let c = Client::new(config());

    let bucket = format!("test-list-prefixes-{}", Uuid::new_v4());

    let bucket_str = bucket.as_str();
@@ -301,6 +307,8 @@ async fn test_list_objects_v2_with_prefixes() -> Result<()> { 
#[tokio::test]

#[tracing::instrument]

async fn test_list_objects_v1_with_prefixes() -> Result<()> {

    let _guard = serial().await;



    let c = Client::new(config());

    let bucket = format!("test-list-v1-prefixes-{}", Uuid::new_v4());

    let bucket_str = bucket.as_str();
@@ -342,6 +350,8 @@ async fn test_list_objects_v1_with_prefixes() -> Result<()> { 
#[tokio::test]

#[tracing::instrument]

async fn test_list_objects_v2_max_keys() -> Result<()> {

    let _guard = serial().await;



    let c = Client::new(config());

    let bucket = format!("test-max-keys-{}", Uuid::new_v4());

    let bucket_str = bucket.as_str();

crates/s3s/src/ops/signature.rs

+40 −34
Original line number Diff line number Diff line
@@ -8,10 +8,12 @@ use crate::protocol::TrailingHeaders; 
use crate::sig_v2;

use crate::sig_v2::{AuthorizationV2, PostSignatureV2, PresignedUrlV2};

use crate::sig_v4;

use crate::sig_v4::PostSignatureV4;

use crate::sig_v4::PresignedUrlV4;

use crate::sig_v4::{AmzContentSha256, AmzDate};

use crate::sig_v4::{AuthorizationV4, CredentialV4};

use crate::sig_v4::AmzContentSha256;

use crate::sig_v4::AmzDate;

use crate::sig_v4::UploadStream;

use crate::sig_v4::{AuthorizationV4, CredentialV4, PostSignatureV4, PresignedUrlV4};

use crate::stream::ByteStream as _;

use crate::utils::crypto::hex_sha256_string;

use crate::utils::is_base64_encoded;



use std::mem;
@@ -323,14 +325,7 @@ impl SignatureContext<'_> { 


        let amz_date = extract_amz_date(&self.hs)?.ok_or_else(|| invalid_request!("missing header: x-amz-date"))?;



        let is_stream = matches!(

            amz_content_sha256,

            Some(

                AmzContentSha256::MultipleChunks

                    | AmzContentSha256::MultipleChunksWithTrailer

                    | AmzContentSha256::UnsignedPayloadWithTrailer

            )

        );

        let is_stream = amz_content_sha256.is_some_and(|v| v.is_streaming());



        let signature = {

            let method = &self.req_method;
@@ -354,10 +349,10 @@ impl SignatureContext<'_> { 
            });



            let canonical_request = match amz_content_sha256 {

                Some(AmzContentSha256::MultipleChunks) => {

                Some(AmzContentSha256::StreamingAws4HmacSha256Payload) => {

                    sig_v4::create_canonical_request(method, uri_path, query_strings, &headers, sig_v4::Payload::MultipleChunks)

                }

                Some(AmzContentSha256::MultipleChunksWithTrailer) => sig_v4::create_canonical_request(

                Some(AmzContentSha256::StreamingAws4HmacSha256PayloadTrailer) => sig_v4::create_canonical_request(

                    method,

                    uri_path,

                    query_strings,
@@ -367,26 +362,25 @@ impl SignatureContext<'_> { 
                Some(AmzContentSha256::UnsignedPayload) => {

                    sig_v4::create_canonical_request(method, uri_path, query_strings, &headers, sig_v4::Payload::Unsigned)

                }

                Some(AmzContentSha256::UnsignedPayloadWithTrailer) => sig_v4::create_canonical_request(

                Some(AmzContentSha256::StreamingUnsignedPayloadTrailer) => sig_v4::create_canonical_request(

                    method,

                    uri_path,

                    query_strings,

                    &headers,

                    sig_v4::Payload::UnsignedMultipleChunksWithTrailer,

                ),

                Some(AmzContentSha256::SingleChunk { .. }) => {

                    let bytes = super::extract_full_body(self.content_length, self.req_body).await?;

                    if bytes.is_empty() {

                        sig_v4::create_canonical_request(method, uri_path, query_strings, &headers, sig_v4::Payload::Empty)

                    } else {

                        sig_v4::create_canonical_request(

                Some(AmzContentSha256::SingleChunk(payload_checksum)) => sig_v4::create_canonical_request(

                    method,

                    uri_path,

                    query_strings,

                    &headers,

                            sig_v4::Payload::SingleChunk(&bytes),

                        )

                    }

                    sig_v4::Payload::SingleChunk(payload_checksum),

                ),

                Some(

                    AmzContentSha256::StreamingAws4EcdsaP256Sha256Payload

                    | AmzContentSha256::StreamingAws4EcdsaP256Sha256PayloadTrailer,

                ) => {

                    return Err(s3_error!(NotImplemented, "AWS4-ECDSA-P256-SHA256 signing method is not implemented yet"));

                }

                None => {

                    if matches!(*self.req_method, Method::GET | Method::HEAD) {
@@ -396,12 +390,13 @@ impl SignatureContext<'_> { 
                        if bytes.is_empty() {

                            sig_v4::create_canonical_request(method, uri_path, query_strings, &headers, sig_v4::Payload::Empty)

                        } else {

                            let payload_checksum = hex_sha256_string(&bytes);

                            sig_v4::create_canonical_request(

                                method,

                                uri_path,

                                query_strings,

                                &headers,

                                sig_v4::Payload::SingleChunk(&bytes),

                                sig_v4::Payload::SingleChunk(&payload_checksum),

                            )

                        }

                    }
@@ -419,10 +414,7 @@ impl SignatureContext<'_> { 


        if is_stream {

            // For streaming with trailers, AWS requires x-amz-trailer header present.

            let has_trailer = matches!(

                amz_content_sha256,

                Some(AmzContentSha256::MultipleChunksWithTrailer | AmzContentSha256::UnsignedPayloadWithTrailer)

            );

            let has_trailer = amz_content_sha256.is_some_and(|v| v.has_trailer());

            if has_trailer && self.hs.get_unique("x-amz-trailer").is_none() {

                return Err(invalid_request!("missing header: x-amz-trailer"));

            }
@@ -430,7 +422,7 @@ impl SignatureContext<'_> { 
                .decoded_content_length

                .ok_or_else(|| s3_error!(MissingContentLength, "missing header: x-amz-decoded-content-length"))?;



            let unsigned = matches!(amz_content_sha256, Some(AmzContentSha256::UnsignedPayloadWithTrailer));

            let unsigned = matches!(amz_content_sha256, Some(AmzContentSha256::StreamingUnsignedPayloadTrailer));

            let stream = AwsChunkedStream::new(

                mem::take(self.req_body),

                signature.into(),
@@ -449,6 +441,20 @@ impl SignatureContext<'_> { 
            let trailers = stream.trailing_headers_handle();

            self.transformed_body = Some(Body::from(stream.into_byte_stream()));

            self.trailing_headers = Some(trailers);

        } else if let Some(AmzContentSha256::SingleChunk(expected_checksum)) = amz_content_sha256 {

            let length = if let Some(content_length) = self.content_length {

                usize::try_from(content_length).map_err(|_| invalid_request!("content-length exceeds platform limits"))?

            } else {

                self.req_body

                    .remaining_length()

                    .exact()

                    .ok_or_else(|| s3_error!(MissingContentLength, "missing header: content-length"))?

            };



            let body = mem::take(self.req_body);

            let stream = UploadStream::new(body, length, expected_checksum)

                .map_err(|_| invalid_request!("invalid header: x-amz-content-sha256"))?;

            *self.req_body = Body::from(stream.into_byte_stream());

        }



        Ok(CredentialsExt {

crates/s3s/src/sig_v4/amz_content_sha256.rs

+69 −29
Original line number Diff line number Diff line 
//! x-amz-content-sha256



use crate::utils::crypto::is_sha256_checksum;



/// x-amz-content-sha256

/// [x-amz-content-sha256](https://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-auth-using-authorization-header.html)

///

/// See [Common Request Headers](https://docs.aws.amazon.com/AmazonS3/latest/API/RESTCommonRequestHeaders.html)

#[derive(Debug)]

/// See also [Common Request Headers](https://docs.aws.amazon.com/AmazonS3/latest/API/RESTCommonRequestHeaders.html)

#[derive(Debug, Clone, Copy, PartialEq, Eq)]

pub enum AmzContentSha256<'a> {

    /// `STREAMING-AWS4-HMAC-SHA256-PAYLOAD`

    MultipleChunks,

    /// `STREAMING-AWS4-HMAC-SHA256-PAYLOAD-TRAILER`

    MultipleChunksWithTrailer,

    /// `STREAMING-UNSIGNED-PAYLOAD-TRAILER`

    UnsignedPayloadWithTrailer,

    /// single chunk

    SingleChunk {

        /// the checksum of single chunk payload

        #[allow(dead_code)] // TODO: check this field when calculating the payload checksum

        payload_checksum: &'a str,

    },

    /// Actual payload checksum value

    SingleChunk(&'a str),



    /// `UNSIGNED-PAYLOAD`

    UnsignedPayload,



    /// `STREAMING-UNSIGNED-PAYLOAD-TRAILER`

    StreamingUnsignedPayloadTrailer,



    /// `STREAMING-AWS4-HMAC-SHA256-PAYLOAD`

    StreamingAws4HmacSha256Payload,



    /// `STREAMING-AWS4-HMAC-SHA256-PAYLOAD-TRAILER`

    StreamingAws4HmacSha256PayloadTrailer,



    /// `STREAMING-AWS4-ECDSA-P256-SHA256-PAYLOAD`

    StreamingAws4EcdsaP256Sha256Payload,



    /// `STREAMING-AWS4-ECDSA-P256-SHA256-PAYLOAD-TRAILER`

    StreamingAws4EcdsaP256Sha256PayloadTrailer,

}



/// [`AmzContentSha256`]

#[derive(Debug, Clone, Copy, thiserror::Error)]

pub enum ParseAmzContentSha256Error {

    /// invalid checksum

    #[error("ParseAmzContentSha256Error: InvalidChecksum")]

    InvalidChecksum,

    /// unknown variant

    #[error("ParseAmzContentSha256Error: UnknownVariant")]

    UnknownVariant,

}



impl<'a> AmzContentSha256<'a> {
@@ -37,17 +41,53 @@ impl<'a> AmzContentSha256<'a> { 
    /// # Errors

    /// Returns an `Err` if the header is invalid

    pub fn parse(header: &'a str) -> Result<Self, ParseAmzContentSha256Error> {

        if is_sha256_checksum(header) {

            return Ok(Self::SingleChunk(header));

        }



        match header {

            "UNSIGNED-PAYLOAD" => Ok(Self::UnsignedPayload),

            "STREAMING-AWS4-HMAC-SHA256-PAYLOAD" => Ok(Self::MultipleChunks),

            "STREAMING-AWS4-HMAC-SHA256-PAYLOAD-TRAILER" => Ok(Self::MultipleChunksWithTrailer),

            "STREAMING-UNSIGNED-PAYLOAD-TRAILER" => Ok(Self::UnsignedPayloadWithTrailer),

            payload_checksum => {

                if !is_sha256_checksum(payload_checksum) {

                    return Err(ParseAmzContentSha256Error::InvalidChecksum);

            "STREAMING-UNSIGNED-PAYLOAD-TRAILER" => Ok(Self::StreamingUnsignedPayloadTrailer),

            "STREAMING-AWS4-HMAC-SHA256-PAYLOAD" => Ok(Self::StreamingAws4HmacSha256Payload),

            "STREAMING-AWS4-HMAC-SHA256-PAYLOAD-TRAILER" => Ok(Self::StreamingAws4HmacSha256PayloadTrailer),

            "STREAMING-AWS4-ECDSA-P256-SHA256-PAYLOAD" => Ok(Self::StreamingAws4EcdsaP256Sha256Payload),

            "STREAMING-AWS4-ECDSA-P256-SHA256-PAYLOAD-TRAILER" => Ok(Self::StreamingAws4EcdsaP256Sha256PayloadTrailer),

            _ => Err(ParseAmzContentSha256Error::UnknownVariant),

        }

                Ok(Self::SingleChunk { payload_checksum })

    }



    // pub fn to_str(self) -> &'a str {

    //     match self {

    //         AmzContentSha256::SingleChunk(checksum) => checksum,

    //         AmzContentSha256::UnsignedPayload => "UNSIGNED-PAYLOAD",

    //         AmzContentSha256::StreamingUnsignedPayloadTrailer => "STREAMING-UNSIGNED-PAYLOAD-TRAILER",

    //         AmzContentSha256::StreamingAws4HmacSha256Payload => "STREAMING-AWS4-HMAC-SHA256-PAYLOAD",

    //         AmzContentSha256::StreamingAws4HmacSha256PayloadTrailer => "STREAMING-AWS4-HMAC-SHA256-PAYLOAD-TRAILER",

    //         AmzContentSha256::StreamingAws4EcdsaP256Sha256Payload => "STREAMING-AWS4-ECDSA-P256-SHA256-PAYLOAD",

    //         AmzContentSha256::StreamingAws4EcdsaP256Sha256PayloadTrailer => "STREAMING-AWS4-ECDSA-P256-SHA256-PAYLOAD-TRAILER",

    //     }

    // }



    pub fn is_streaming(&self) -> bool {

        match self {

            AmzContentSha256::SingleChunk(_) | AmzContentSha256::UnsignedPayload => false,

            AmzContentSha256::StreamingUnsignedPayloadTrailer

            | AmzContentSha256::StreamingAws4HmacSha256Payload

            | AmzContentSha256::StreamingAws4HmacSha256PayloadTrailer

            | AmzContentSha256::StreamingAws4EcdsaP256Sha256Payload

            | AmzContentSha256::StreamingAws4EcdsaP256Sha256PayloadTrailer => true,

        }

    }



    pub fn has_trailer(&self) -> bool {

        match self {

            AmzContentSha256::SingleChunk(_)

            | AmzContentSha256::UnsignedPayload

            | AmzContentSha256::StreamingAws4HmacSha256Payload

            | AmzContentSha256::StreamingAws4EcdsaP256Sha256Payload => false,

            AmzContentSha256::StreamingUnsignedPayloadTrailer

            | AmzContentSha256::StreamingAws4HmacSha256PayloadTrailer

            | AmzContentSha256::StreamingAws4EcdsaP256Sha256PayloadTrailer => true,

        }

    }

}

crates/s3s/src/sig_v4/methods.rs

+14 −6
Original line number Diff line number Diff line
@@ -76,7 +76,7 @@ pub enum Payload<'a> { 
    /// empty

    Empty,

    /// single chunk

    SingleChunk(&'a [u8]),

    SingleChunk(&'a str),

    /// multiple chunks

    MultipleChunks,

    /// multiple chunks with trailing headers
@@ -179,7 +179,7 @@ pub fn create_canonical_request( 
        match payload {

            Payload::Unsigned => ans.push_str("UNSIGNED-PAYLOAD"),

            Payload::Empty => ans.push_str(EMPTY_STRING_SHA256_HASH),

            Payload::SingleChunk(data) => hex_sha256(data, |s| ans.push_str(s)),

            Payload::SingleChunk(checksum) => ans.push_str(checksum),

            Payload::MultipleChunks => ans.push_str("STREAMING-AWS4-HMAC-SHA256-PAYLOAD"),

            Payload::MultipleChunksWithTrailer => ans.push_str("STREAMING-AWS4-HMAC-SHA256-PAYLOAD-TRAILER"),

            Payload::UnsignedMultipleChunksWithTrailer => ans.push_str("STREAMING-UNSIGNED-PAYLOAD-TRAILER"),
@@ -436,6 +436,7 @@ mod tests { 


    use crate::http::OrderedQs;

    use crate::sig_v4::PresignedUrlV4;

    use crate::utils::crypto::hex_sha256_string;



    #[test]

    fn example_get_object() {
@@ -510,10 +511,10 @@ mod tests { 
        ]);



        let method = Method::PUT;

        let payload = "Welcome to Amazon S3.";

        let payload_checksum = &hex_sha256_string("Welcome to Amazon S3.".as_bytes());

        let qs: &[(String, String)] = &[];



        let canonical_request = create_canonical_request(&method, path, qs, &headers, Payload::SingleChunk(payload.as_bytes()));

        let canonical_request = create_canonical_request(&method, path, qs, &headers, Payload::SingleChunk(payload_checksum));



        assert_eq!(

            canonical_request,
@@ -1150,7 +1151,8 @@ mod tests { 
            "x-amz-user-agent",

        ];



        let payload = Payload::SingleChunk(b"RoleArn=arn%3Aaws%3Aiam%3A%3A%2A%3Arole%2FAdmin&RoleSessionName=console&DurationSeconds=43200&Action=AssumeRole&Version=2011-06-15");

        let body = b"RoleArn=arn%3Aaws%3Aiam%3A%3A%2A%3Arole%2FAdmin&RoleSessionName=console&DurationSeconds=43200&Action=AssumeRole&Version=2011-06-15";

        let payload_checksum = hex_sha256_string(body);

        let date = AmzDate::parse(x_amz_date).unwrap();

        let region = "cn-east-1";

        let service = "sts";
@@ -1166,7 +1168,13 @@ mod tests { 
                .unwrap()

                .find_multiple_with_on_missing(signed_header_names, |_| panic!());



            let canonical_request = create_canonical_request(req.method(), uri_path, query_strings, &signed_headers, payload);

            let canonical_request = create_canonical_request(

                req.method(),

                uri_path,

                query_strings,

                &signed_headers,

                Payload::SingleChunk(&payload_checksum),

            );



            let string_to_sign = create_string_to_sign(&canonical_request, &date, region, service);

crates/s3s/src/sig_v4/mod.rs

+3 −0
Original line number Diff line number Diff line
@@ -20,5 +20,8 @@ pub use self::amz_date::*; 
mod post_signature_v4;

pub use self::post_signature_v4::*;



mod upload_stream;

pub use self::upload_stream::*;



mod methods;

pub use self::methods::*;