Unverified Commit a9b9cbab authored by Zelda Hessler's avatar Zelda Hessler Committed by GitHub
Browse files

Add new MRAP test to S3 canary (#3109) 

This PR also replaces the `2023_09_25` canary with current release
*(`2023_10_26`)*

----

_By submitting this pull request, I confirm that you can use, modify,
copy, and redistribute this contribution, under the terms of your
choice._
parent b6841826

tools/ci-cdk/canary-lambda/README.md

+1 −0
Original line number Diff line number Diff line
@@ -34,6 +34,7 @@ invoke the canary: 


```bash

export CANARY_S3_BUCKET_NAME=<your bucket name>

export CANARY_S3_MRAP_BUCKET_ARN=<your MRAP bucket ARN>

# run with `--all-features` so you run all canaries (including canaries that don't work against older versions)

cargo run --all-features -- --local

```

tools/ci-cdk/canary-lambda/src/canary.rs

+6 −0
Original line number Diff line number Diff line
@@ -50,6 +50,7 @@ pub fn get_canaries_to_run( 


pub struct CanaryEnv {

    pub(crate) s3_bucket_name: String,

    pub(crate) s3_mrap_bucket_arn: String,

    pub(crate) expected_transcribe_result: String,

    #[allow(dead_code)]

    pub(crate) page_size: usize,
@@ -59,6 +60,7 @@ impl fmt::Debug for CanaryEnv { 
    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {

        f.debug_struct("CanaryEnv")

            .field("s3_bucket_name", &"*** redacted ***")

            .field("s3_mrap_bucket_arn", &"*** redacted ***")

            .field(

                "expected_transcribe_result",

                &self.expected_transcribe_result,
@@ -72,6 +74,9 @@ impl CanaryEnv { 
        // S3 bucket name to test against

        let s3_bucket_name =

            env::var("CANARY_S3_BUCKET_NAME").expect("CANARY_S3_BUCKET_NAME must be set");

        // S3 MRAP bucket name to test against

        let s3_mrap_bucket_arn =

            env::var("CANARY_S3_MRAP_BUCKET_ARN").expect("CANARY_S3_MRAP_BUCKET_ARN must be set");



        // Expected transcription from Amazon Transcribe from the embedded audio file.

        // This is an environment variable so that the code doesn't need to be changed if
@@ -89,6 +94,7 @@ impl CanaryEnv { 


        Self {

            s3_bucket_name,

            s3_mrap_bucket_arn,

            expected_transcribe_result,

            page_size,

        }

tools/ci-cdk/canary-lambda/src/latest/paginator_canary.rs

+2 −2
Original line number Diff line number Diff line
@@ -38,7 +38,7 @@ pub async fn paginator_canary(client: ec2::Client, page_size: usize) -> anyhow:: 
        }

        num_pages += 1;

    }

    if dbg!(num_pages) < 2 {

    if num_pages < 2 {

        bail!(

            "expected 3+ pages containing ~60 results but got {} pages",

            num_pages
@@ -59,7 +59,7 @@ pub async fn paginator_canary(client: ec2::Client, page_size: usize) -> anyhow:: 


#[cfg(test)]

mod test {

    use crate::latest::paginator_canary::paginator_canary;

    use super::paginator_canary;



    #[tokio::test]

    async fn test_paginator() {

tools/ci-cdk/canary-lambda/src/latest/s3_canary.rs

+101 −25
Original line number Diff line number Diff line
@@ -8,6 +8,7 @@ use crate::{mk_canary, CanaryEnv}; 
use anyhow::Context;

use aws_config::SdkConfig;

use aws_sdk_s3 as s3;

use s3::config::Region;

use s3::presigning::PresigningConfig;

use s3::primitives::ByteStream;

use std::time::Duration;
@@ -17,10 +18,15 @@ const METADATA_TEST_VALUE: &str = "some value"; 


mk_canary!("s3", |sdk_config: &SdkConfig, env: &CanaryEnv| s3_canary(

    s3::Client::new(sdk_config),

    env.s3_bucket_name.clone()

    env.s3_bucket_name.clone(),

    env.s3_mrap_bucket_arn.clone()

));



pub async fn s3_canary(client: s3::Client, s3_bucket_name: String) -> anyhow::Result<()> {

pub async fn s3_canary(

    client: s3::Client,

    s3_bucket_name: String,

    s3_mrap_bucket_arn: String,

) -> anyhow::Result<()> {

    let test_key = Uuid::new_v4().as_u128().to_string();



    // Look for the test object and expect that it doesn't exist
@@ -82,33 +88,34 @@ pub async fn s3_canary(client: s3::Client, s3_bucket_name: String) -> anyhow::Re 
        return Err(CanaryError(format!("presigned URL returned bad data: {:?}", response)).into());

    }



    let mut result = Ok(());

    match output.metadata() {

        Some(map) => {

            // Option::as_deref doesn't work here since the deref of &String is String

            let value = map.get("something").map(|s| s.as_str()).unwrap_or("");

            if value != METADATA_TEST_VALUE {

                result = Err(CanaryError(format!(

                    "S3 metadata was incorrect. Expected `{}` but got `{}`.",

                    METADATA_TEST_VALUE, value

                ))

                .into());

            }

        }

        None => {

            result = Err(CanaryError("S3 metadata was missing".into()).into());

        }

    }



    let metadata_value = output

        .metadata()

        .and_then(|m| m.get("something"))

        .map(String::as_str);

    let result: anyhow::Result<()> = match metadata_value {

        Some(value) => {

            if value == METADATA_TEST_VALUE {

                let payload = output

                    .body

                    .collect()

                    .await

                    .context("download s3::GetObject[2] body")?

                    .into_bytes();

    if std::str::from_utf8(payload.as_ref()).context("s3 payload")? != "test" {

        result = Err(CanaryError("S3 object body didn't match what was put there".into()).into());

                if std::str::from_utf8(payload.as_ref()).context("s3 payload")? == "test" {

                    Ok(())

                } else {

                    Err(CanaryError("S3 object body didn't match what was put there".into()).into())

                }

            } else {

                Err(CanaryError(format!(

                    "S3 metadata was incorrect. Expected `{}` but got `{}`.",

                    METADATA_TEST_VALUE, value

                ))

                .into())

            }

        }

        None => Err(CanaryError("S3 metadata was missing".into()).into()),

    };



    // Delete the test object

    client
@@ -119,12 +126,80 @@ pub async fn s3_canary(client: s3::Client, s3_bucket_name: String) -> anyhow::Re 
        .await

        .context("s3::DeleteObject")?;



    // Return early if the result is an error

    result?;



    // We deliberately use a region that doesn't exist here so that we can

    // ensure these requests are SigV4a requests. Because the current endpoint

    // resolver always resolves the wildcard region ('*') for SigV4a requests,

    // setting a fictitious region ensures that the request would fail if it was

    // a SigV4 request. Therefore, because the request doesn't fail, we can be

    // sure it's a successful Sigv4a request.

    let config_override = s3::Config::builder().region(Region::new("parts-unknown"));

    // Put the test object

    client

        .put_object()

        .bucket(&s3_mrap_bucket_arn)

        .key(&test_key)

        .body(ByteStream::from_static(b"test"))

        .metadata("something", METADATA_TEST_VALUE)

        .customize()

        .config_override(config_override.clone())

        .send()

        .await

        .context("s3::PutObject[MRAP]")?;



    // Get the test object and verify it looks correct

    let output = client

        .get_object()

        .bucket(&s3_mrap_bucket_arn)

        .key(&test_key)

        .customize()

        .config_override(config_override.clone())

        .send()

        .await

        .context("s3::GetObject[MRAP]")?;



    let metadata_value = output

        .metadata()

        .and_then(|m| m.get("something"))

        .map(String::as_str);

    let result = match metadata_value {

        Some(value) => {

            if value == METADATA_TEST_VALUE {

                Ok(())

            } else {

                Err(CanaryError(format!(

                    "S3 metadata was incorrect. Expected `{}` but got `{}`.",

                    METADATA_TEST_VALUE, value

                ))

                .into())

            }

        }

        None => Err(CanaryError("S3 metadata was missing".into()).into()),

    };



    // Delete the test object

    client

        .delete_object()

        .bucket(&s3_mrap_bucket_arn)

        .key(&test_key)

        .customize()

        .config_override(config_override)

        .send()

        .await

        .context("s3::DeleteObject")?;



    result

}



// This test runs against an actual AWS account. Comment out the `ignore` to run it.

// Be sure to set the `TEST_S3_BUCKET` environment variable to the S3 bucket to use,

// and also make sure the credential profile sets the region (or set `AWS_DEFAULT_PROFILE`).

// Be sure the following environment variables are set:

//

// - `TEST_S3_BUCKET`: The S3 bucket to use

// - `TEST_S3_MRAP_BUCKET_ARN`: The MRAP bucket ARN to use

//

// Also, make sure the correct region (likely `us-west-2`) by the credentials or explictly.

#[ignore]

#[cfg(test)]

#[tokio::test]
@@ -134,6 +209,7 @@ async fn test_s3_canary() { 
    s3_canary(

        client,

        std::env::var("TEST_S3_BUCKET").expect("TEST_S3_BUCKET must be set"),

        std::env::var("TEST_S3_MRAP_BUCKET_ARN").expect("TEST_S3_MRAP_BUCKET_ARN must be set"),

    )

    .await

    .expect("success");

tools/ci-cdk/canary-lambda/src/main.rs

+5 −5
Original line number Diff line number Diff line
@@ -26,11 +26,11 @@ mod latest; 
#[cfg(feature = "latest")]

pub(crate) use latest as current_canary;



// NOTE: This module can be deleted 3 releases after release-2023-09-25

#[cfg(feature = "release-2023-09-25")]

mod release_2023_09_25;

#[cfg(feature = "release-2023-09-25")]

pub(crate) use release_2023_09_25 as current_canary;

// NOTE: This module can be deleted 3 releases after release-2023-10-26

#[cfg(feature = "release-2023-10-26")]

mod release_2023_10_26;

#[cfg(feature = "release-2023-10-26")]

pub(crate) use release_2023_10_26 as current_canary;



#[tokio::main]

async fn main() -> Result<(), Error> {