Unverified Commit d3957a6e authored by ysaito1001's avatar ysaito1001 Committed by GitHub
Browse files

Allow canary-runner to specify expected speech text by Transcribe (#2660) 



## Motivation and Context
During a release for `aws-sdk-rust`, we've found that the canary test
received an improved speech text from Transcribe. This broke the canary
test because the test hard-coded the string that should be expected by
Transcribe. This PR updates the runner so that the expected speech text
can be passed-in from outside of the program.

## Testing
Added unit tests, mainly for argument parsing.

## Checklist
If the next release is created off of the main branch (because it's a
non-breaking release), we only need to merge this PR the main branch. If
we need to release from a release branch at some point in the future,
this PR will need to be duplicated and merged to the release branch as
well.

----

_By submitting this pull request, I confirm that you can use, modify,
copy, and redistribute this contribution, under the terms of your
choice._

---------

Co-authored-by: default avatarYuki Saito <awsaito@amazon.com>
parent 2fb21f64

tools/ci-cdk/canary-runner/src/run.rs

+92 −12
Original line number Diff line number Diff line
@@ -83,6 +83,11 @@ pub struct RunArgs { 
    #[clap(long)]

    musl: bool,



    /// Expected speech text generated by Transcribe. This needs to be passed-in

    /// because it can change as the accuracy of generated text improves over time.

    #[clap(long)]

    expected_speech_text_by_transcribe: Option<String>,



    /// File path to a CDK outputs JSON file. This can be used instead

    /// of all the --lambda... args.

    #[clap(long)]
@@ -101,12 +106,13 @@ pub struct RunArgs { 
    lambda_execution_role_arn: Option<String>,

}



#[derive(Debug)]

#[derive(Debug, Eq, PartialEq)]

struct Options {

    rust_version: Option<String>,

    sdk_release_tag: Option<ReleaseTag>,

    sdk_path: Option<PathBuf>,

    musl: bool,

    expected_speech_text_by_transcribe: Option<String>,

    lambda_code_s3_bucket_name: String,

    lambda_test_s3_bucket_name: String,

    lambda_execution_role_arn: String,
@@ -139,6 +145,7 @@ impl Options { 
                sdk_release_tag: run_opt.sdk_release_tag,

                sdk_path: run_opt.sdk_path,

                musl: run_opt.musl,

                expected_speech_text_by_transcribe: run_opt.expected_speech_text_by_transcribe,

                lambda_code_s3_bucket_name: value.inner.lambda_code_s3_bucket_name,

                lambda_test_s3_bucket_name: value.inner.lambda_test_s3_bucket_name,

                lambda_execution_role_arn: value.inner.lambda_execution_role_arn,
@@ -149,6 +156,7 @@ impl Options { 
                sdk_release_tag: run_opt.sdk_release_tag,

                sdk_path: run_opt.sdk_path,

                musl: run_opt.musl,

                expected_speech_text_by_transcribe: run_opt.expected_speech_text_by_transcribe,

                lambda_code_s3_bucket_name: run_opt.lambda_code_s3_bucket_name.expect("required"),

                lambda_test_s3_bucket_name: run_opt.lambda_test_s3_bucket_name.expect("required"),

                lambda_execution_role_arn: run_opt.lambda_execution_role_arn.expect("required"),
@@ -250,6 +258,7 @@ async fn run_canary(options: &Options, config: &aws_config::SdkConfig) -> Result 
        bundle_name,

        bundle_file_name,

        &options.lambda_execution_role_arn,

        options.expected_speech_text_by_transcribe.as_ref(),

        &options.lambda_code_s3_bucket_name,

        &options.lambda_test_s3_bucket_name,

    )
@@ -324,11 +333,27 @@ async fn create_lambda_fn( 
    bundle_name: &str,

    bundle_file_name: &str,

    execution_role: &str,

    expected_speech_text_by_transcribe: Option<&String>,

    code_s3_bucket: &str,

    test_s3_bucket: &str,

) -> Result<()> {

    use lambda::model::*;



    let env_builder = match expected_speech_text_by_transcribe {

        Some(expected_speech_text_by_transcribe) => Environment::builder()

            .variables("RUST_BACKTRACE", "1")

            .variables("RUST_LOG", "info")

            .variables("CANARY_S3_BUCKET_NAME", test_s3_bucket)

            .variables(

                "CANARY_EXPECTED_TRANSCRIBE_RESULT",

                expected_speech_text_by_transcribe,

            ),

        None => Environment::builder()

            .variables("RUST_BACKTRACE", "1")

            .variables("RUST_LOG", "info")

            .variables("CANARY_S3_BUCKET_NAME", test_s3_bucket),

    };



    lambda_client

        .create_function()

        .function_name(bundle_name)
@@ -342,17 +367,7 @@ async fn create_lambda_fn( 
                .build(),

        )

        .publish(true)

        .environment(

            Environment::builder()

                .variables("RUST_BACKTRACE", "1")

                .variables("RUST_LOG", "info")

                .variables("CANARY_S3_BUCKET_NAME", test_s3_bucket)

                .variables(

                    "CANARY_EXPECTED_TRANSCRIBE_RESULT",

                    "Good day to you transcribe. This is Polly talking to you from the Rust ST K.",

                )

                .build(),

        )

        .environment(env_builder.build())

        .timeout(60)

        .send()

        .await
@@ -419,3 +434,68 @@ async fn delete_lambda_fn(lambda_client: lambda::Client, bundle_name: &str) -> R 
        .context(here!("failed to delete Lambda"))?;

    Ok(())

}



#[cfg(test)]

mod tests {

    use crate::run::Options;

    use crate::run::RunArgs;

    use clap::Parser;



    #[test]

    fn args_parsing() {

        assert_eq!(

            RunArgs {

                rust_version: None,

                sdk_release_tag: None,

                sdk_path: Some("artifact-aws-sdk-rust/sdk".into()),

                musl: false,

                expected_speech_text_by_transcribe: Some("Good day to you transcribe.".to_owned()),

                cdk_output: Some("../cdk-outputs.json".into()),

                lambda_code_s3_bucket_name: None,

                lambda_test_s3_bucket_name: None,

                lambda_execution_role_arn: None

            },

            RunArgs::try_parse_from([

                "run",

                "--sdk-path",

                "artifact-aws-sdk-rust/sdk",

                "--expected-speech-text-by-transcribe",

                "Good day to you transcribe.",

                "--cdk-output",

                "../cdk-outputs.json",

            ])

            .unwrap()

        );

    }



    #[test]

    fn options_from_args() {

        let run_args = RunArgs::try_parse_from([

            "run",

            "--sdk-path",

            "artifact-aws-sdk-rust/sdk",

            "--expected-speech-text-by-transcribe",

            "Good day to you transcribe.",

            "--lambda-code-s3-bucket-name",

            "bucket-for-code",

            "--lambda-test-s3-bucket-name",

            "bucket-for-test",

            "--lambda-execution-role-arn",

            "arn:aws:lambda::role/exe-role",

        ])

        .unwrap();

        assert_eq!(

            Options {

                rust_version: None,

                sdk_release_tag: None,

                sdk_path: Some("artifact-aws-sdk-rust/sdk".into()),

                musl: false,

                expected_speech_text_by_transcribe: Some("Good day to you transcribe.".to_owned()),

                lambda_code_s3_bucket_name: "bucket-for-code".to_owned(),

                lambda_test_s3_bucket_name: "bucket-for-test".to_owned(),

                lambda_execution_role_arn: "arn:aws:lambda::role/exe-role".to_owned(),

            },

            Options::load_from(run_args).unwrap(),

        );

    }

}