Unverified Commit a84e61af authored by Russell Cohen's avatar Russell Cohen Committed by GitHub
Browse files

Add instructions for running the canary locally & cleanup canary code (#2072) 

* Add instructions for running the canary locally & cleanup the canary code

* Fix ec2 canary
parent 81028c65

tools/ci-cdk/canary-lambda/README.md

+25 −0
Original line number Diff line number Diff line
@@ -12,6 +12,31 @@ deployed SDK is functioning correctly. Similarly, it can be used to verify 
the previous version of the SDK continues to work after the deployment

of the new version.



Running the canary locally

--------------------------

For testing, it's helpful to be able to run the canary locally. To accomplish this, you first need to generate a

Cargo.toml:



```bash

cd ../canary-runner

# to use a version of the SDK, use `--sdk-version` instead

cargo run -- build-bundle \

  --sdk-path ../../../aws/sdk/build/aws-sdk/sdk/ \

  --canary-path ../canary-lambda \

  --manifest-only

```



Next, come back to the `canary-lambda` directory where you can use `cargo run` in `--local` mode to

invoke the canary:



> Note: if your default configuration does not provide a region, you must provide a region via the `AWS_REGION`

> environment variable.



```bash

export CANARY_S3_BUCKET_NAME=<your bucket name>

# run with `--all-features` so you run all canaries (including canaries that don't work against older versions)

cargo run --all-features -- --local

```



Building locally for Lambda from Amazon Linux 2

-----------------------------------------------

tools/ci-cdk/canary-lambda/src/canary.rs

+14 −29
Original line number Diff line number Diff line
@@ -3,34 +3,37 @@ 
 * SPDX-License-Identifier: Apache-2.0

 */



use crate::paginator_canary;

use crate::{s3_canary, transcribe_canary};

use aws_sdk_ec2 as ec2;

use aws_sdk_s3 as s3;

use aws_sdk_transcribestreaming as transcribe;

use std::env;

use std::fmt;

use std::future::Future;

use std::pin::Pin;



use aws_config::SdkConfig;

use tracing::{info_span, Instrument};



use crate::paginator_canary;

use crate::{s3_canary, transcribe_canary};



#[macro_export]

macro_rules! mk_canary {

    ($name: expr, $run_canary: expr) => {

        pub(crate) fn mk_canary(

            clients: &Clients,

            sdk_config: &aws_config::SdkConfig,

            env: &CanaryEnv,

        ) -> Option<(&'static str, $crate::canary::CanaryFuture)> {

            Some(($name, Box::pin($run_canary(clients, env))))

            Some(($name, Box::pin($run_canary(sdk_config, env))))

        }

    };

}



pub fn get_canaries_to_run(clients: Clients, env: CanaryEnv) -> Vec<(&'static str, CanaryFuture)> {

pub fn get_canaries_to_run(

    sdk_config: SdkConfig,

    env: CanaryEnv,

) -> Vec<(&'static str, CanaryFuture)> {

    let canaries = vec![

        paginator_canary::mk_canary(&clients, &env),

        s3_canary::mk_canary(&clients, &env),

        transcribe_canary::mk_canary(&clients, &env),

        paginator_canary::mk_canary(&sdk_config, &env),

        s3_canary::mk_canary(&sdk_config, &env),

        transcribe_canary::mk_canary(&sdk_config, &env),

    ];



    canaries
@@ -45,24 +48,6 @@ pub fn get_canaries_to_run(clients: Clients, env: CanaryEnv) -> Vec<(&'static st 
        .collect()

}



#[derive(Clone)]

pub struct Clients {

    pub s3: s3::Client,

    pub ec2: ec2::Client,

    pub transcribe: transcribe::Client,

}



impl Clients {

    pub async fn initialize() -> Self {

        let config = aws_config::load_from_env().await;

        Self {

            ec2: ec2::Client::new(&config),

            s3: s3::Client::new(&config),

            transcribe: transcribe::Client::new(&config),

        }

    }

}



pub struct CanaryEnv {

    pub(crate) s3_bucket_name: String,

    pub(crate) expected_transcribe_result: String,

tools/ci-cdk/canary-lambda/src/main.rs

+8 −7
Original line number Diff line number Diff line
@@ -3,6 +3,7 @@ 
 * SPDX-License-Identifier: Apache-2.0

 */



use aws_config::SdkConfig;

use canary::{get_canaries_to_run, CanaryEnv};

use lambda_runtime::{Context as LambdaContext, Error};

use serde_json::{json, Value};
@@ -29,7 +30,7 @@ macro_rules! canary_module { 
        #[cfg(not(feature = $version))]

        mod $name {

            pub(crate) fn mk_canary(

                _clients: &crate::canary::Clients,

                _clients: &aws_config::SdkConfig,

                _env: &crate::canary::CanaryEnv,

            ) -> Option<(&'static str, crate::canary::CanaryFuture)> {

                tracing::warn!(concat!(
@@ -62,7 +63,7 @@ async fn main() -> Result<(), Error> { 
    let local = env::args().any(|arg| arg == "--local");

    let main_handler = LambdaMain::new().await;

    if local {

        let result = lambda_main(main_handler.clients)

        let result = lambda_main(main_handler.sdk_config)

            .instrument(tracing_texray::examine(info_span!("run_canaries")))

            .await?;

        if result
@@ -88,13 +89,13 @@ async fn main() -> Result<(), Error> { 
// Enables us to keep the clients alive between successive Lambda executions.

// Not because we need to for this use-case, but to demonstrate how to.

struct LambdaMain {

    clients: canary::Clients,

    sdk_config: SdkConfig,

}



impl LambdaMain {

    async fn new() -> Self {

        Self {

            clients: canary::Clients::initialize().await,

            sdk_config: aws_config::load_from_env().await,

        }

    }

}
@@ -104,17 +105,17 @@ impl lambda_runtime::Handler<Value, Value> for LambdaMain { 
    type Fut = Pin<Box<dyn Future<Output = Result<Value, Error>>>>;



    fn call(&self, _: Value, _: LambdaContext) -> Self::Fut {

        Box::pin(lambda_main(self.clients.clone()))

        Box::pin(lambda_main(self.sdk_config.clone()))

    }

}



async fn lambda_main(clients: canary::Clients) -> Result<Value, Error> {

async fn lambda_main(sdk_config: SdkConfig) -> Result<Value, Error> {

    // Load necessary parameters from environment variables

    let env = CanaryEnv::from_env();

    info!("Env: {:#?}", env);



    // Get list of canaries to run and spawn them

    let canaries = get_canaries_to_run(clients, env);

    let canaries = get_canaries_to_run(sdk_config, env);

    let join_handles = canaries

        .into_iter()

        .map(|(name, future)| (name, tokio::spawn(future)))

tools/ci-cdk/canary-lambda/src/paginator_canary.rs

+6 −5
Original line number Diff line number Diff line
@@ -3,8 +3,6 @@ 
 * SPDX-License-Identifier: Apache-2.0

 */



use crate::canary::Clients;



use crate::mk_canary;

use anyhow::bail;
@@ -14,9 +12,12 @@ use aws_sdk_ec2::model::InstanceType; 
use crate::CanaryEnv;

use tokio_stream::StreamExt;



mk_canary!("ec2_paginator", |clients: &Clients, env: &CanaryEnv| {

    paginator_canary(clients.ec2.clone(), env.page_size)

});

mk_canary!(

    "ec2_paginator",

    |sdk_config: &aws_config::SdkConfig, env: &CanaryEnv| {

        paginator_canary(ec2::Client::new(sdk_config), env.page_size)

    }

);



pub async fn paginator_canary(client: ec2::Client, page_size: usize) -> anyhow::Result<()> {

    let mut history = client

tools/ci-cdk/canary-lambda/src/s3_canary.rs

+4 −3
Original line number Diff line number Diff line
@@ -3,9 +3,10 @@ 
 * SPDX-License-Identifier: Apache-2.0

 */



use crate::canary::{CanaryError, Clients};

use crate::canary::CanaryError;

use crate::{mk_canary, CanaryEnv};

use anyhow::Context;

use aws_config::SdkConfig;

use aws_sdk_s3 as s3;

use s3::error::{GetObjectError, GetObjectErrorKind};

use s3::types::ByteStream;
@@ -13,8 +14,8 @@ use uuid::Uuid; 


const METADATA_TEST_VALUE: &str = "some   value";



mk_canary!("s3", |clients: &Clients, env: &CanaryEnv| s3_canary(

    clients.s3.clone(),

mk_canary!("s3", |sdk_config: &SdkConfig, env: &CanaryEnv| s3_canary(

    s3::Client::new(sdk_config),

    env.s3_bucket_name.clone()

));