Unverified Commit 65e376d8 authored by John DiSanti's avatar John DiSanti Committed by GitHub
Browse files

Add SRA benchmark to compare the last release against the current release (#2648) 

## Motivation and Context
This PR adds a benchmark to make sure we didn't regress the old
middleware client implementation while making changes to it for the new
orchestrator implementation.

----

_By submitting this pull request, I confirm that you can use, modify,
copy, and redistribute this contribution, under the terms of your
choice._
parent b1ce5e3c

aws/sra-test/integration-tests/aws-sdk-s3/Cargo.toml

+5 −9
Original line number Diff line number Diff line
@@ -6,24 +6,20 @@ publish = false 
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html



[dependencies]

aws-credential-types = { path = "../../../rust-runtime/aws-credential-types", features = ["test-util"] }

aws-http = { path = "../../../rust-runtime/aws-http" }

aws-runtime = { path = "../../../rust-runtime/aws-runtime" }

aws-sdk-s3 = { path = "../../build/sdk/aws-sdk-s3", features = ["test-util"] }

aws-sigv4 = { path = "../../../rust-runtime/aws-sigv4" }

aws-types = { path = "../../../rust-runtime/aws-types" }

aws-smithy-async = { path = "../../../../rust-runtime/aws-smithy-async", features = ["rt-tokio"] }

aws-smithy-client = { path = "../../../../rust-runtime/aws-smithy-client", features = ["test-util", "rustls"] }

aws-smithy-types = { path = "../../../../rust-runtime/aws-smithy-types" }

aws-smithy-http = { path = "../../../../rust-runtime/aws-smithy-http" }

aws-smithy-runtime = { path = "../../../../rust-runtime/aws-smithy-runtime", features = ["test-util"] }

aws-smithy-runtime-api = { path = "../../../../rust-runtime/aws-smithy-runtime-api" }

aws-types = { path = "../../../rust-runtime/aws-types" }

criterion = { version = "0.4", features = ["async_tokio"] }

http = "0.2.3"

http-body = "0.4.5"

last-release-smithy-client = { version = "0.55", package = "aws-smithy-client", features = ["test-util", "rustls"] }

last-release-s3 = { version = "0.26", package = "aws-sdk-s3", features = ["test-util"] }

tokio = { version = "1.23.1", features = ["macros", "test-util", "rt-multi-thread"] }

tracing = "0.1.37"

tracing-subscriber = { version = "0.3.15", features = ["env-filter", "json"] }

http = "0.2.3"

http-body = "0.4.5"



[profile.release]

debug = 1

aws/sra-test/integration-tests/aws-sdk-s3/benches/middleware_vs_orchestrator.rs

+103 −64
Original line number Diff line number Diff line
@@ -6,14 +6,75 @@ 
#[macro_use]

extern crate criterion;

use aws_sdk_s3 as s3;

use aws_smithy_client::erase::DynConnector;

use aws_smithy_client::test_connection::infallible_connection_fn;

use aws_smithy_runtime_api::client::runtime_plugin::RuntimePlugin;

use aws_smithy_runtime_api::config_bag::ConfigBag;

use criterion::Criterion;

use s3::endpoint::Params;

use criterion::{BenchmarkId, Criterion};



async fn middleware(client: &s3::Client) {

macro_rules! test_connection {

    (head) => {

        test_connection!(aws_smithy_client)

    };

    (last_release) => {

        test_connection!(last_release_smithy_client)

    };

    ($package:ident) => {

        $package::test_connection::infallible_connection_fn(|req| {

            assert_eq!(

                "https://test-bucket.s3.us-east-1.amazonaws.com/?list-type=2&prefix=prefix~",

                req.uri().to_string()

            );

            assert!(req.headers().contains_key("authorization"));

            http::Response::builder()

                .status(200)

                .body(

                    r#"<?xml version="1.0" encoding="UTF-8"?>

<ListBucketResult>

    <Name>test-bucket</Name>

    <Prefix>prefix~</Prefix>

    <KeyCount>1</KeyCount>

    <MaxKeys>1000</MaxKeys>

    <IsTruncated>false</IsTruncated>

    <Contents>

        <Key>some-file.file</Key>

        <LastModified>2009-10-12T17:50:30.000Z</LastModified>

        <Size>434234</Size>

        <StorageClass>STANDARD</StorageClass>

    </Contents>

</ListBucketResult>

"#,

                )

                .unwrap()

        })

    };

}



macro_rules! create_client {

    (head) => {

        create_client!(head, aws_sdk_s3)

    };

    (last_release) => {

        create_client!(last_release, last_release_s3)

    };

    ($original:ident, $package:ident) => {{

        let conn = test_connection!($original);

        let config = $package::Config::builder()

            .credentials_provider($package::config::Credentials::for_tests())

            .region($package::config::Region::new("us-east-1"))

            .http_connector(conn.clone())

            .build();

        $package::Client::from_conf(config)

    }};

}



macro_rules! middleware_bench_fn {

    ($fn_name:ident, head) => {

        middleware_bench_fn!($fn_name, aws_sdk_s3)

    };

    ($fn_name:ident, last_release) => {

        middleware_bench_fn!($fn_name, last_release_s3)

    };

    ($fn_name:ident, $package:ident) => {

        async fn $fn_name(client: &$package::Client) {

            client

                .list_objects_v2()

                .bucket("test-bucket")
@@ -22,6 +83,8 @@ async fn middleware(client: &s3::Client) { 
                .await

                .expect("successful execution");

        }

    };

}



async fn orchestrator(client: &s3::Client) {

    struct FixupPlugin {
@@ -32,7 +95,7 @@ async fn orchestrator(client: &s3::Client) { 
            &self,

            cfg: &mut ConfigBag,

        ) -> Result<(), aws_smithy_runtime_api::client::runtime_plugin::BoxError> {

            let params_builder = Params::builder()

            let params_builder = s3::endpoint::Params::builder()

                .set_region(Some(self.region.clone()))

                .bucket("test-bucket");
@@ -55,61 +118,37 @@ async fn orchestrator(client: &s3::Client) { 
        .expect("successful execution");

}



fn test_connection() -> DynConnector {

    infallible_connection_fn(|req| {

        assert_eq!(

            "https://test-bucket.s3.us-east-1.amazonaws.com/?list-type=2&prefix=prefix~",

            req.uri().to_string()

        );

        assert!(req.headers().contains_key("authorization"));

        http::Response::builder()

            .status(200)

            .body(

                r#"<?xml version="1.0" encoding="UTF-8"?>

<ListBucketResult>

    <Name>test-bucket</Name>

    <Prefix>prefix~</Prefix>

    <KeyCount>1</KeyCount>

    <MaxKeys>1000</MaxKeys>

    <IsTruncated>false</IsTruncated>

    <Contents>

        <Key>some-file.file</Key>

        <LastModified>2009-10-12T17:50:30.000Z</LastModified>

        <Size>434234</Size>

        <StorageClass>STANDARD</StorageClass>

    </Contents>

</ListBucketResult>

"#,

            )

            .unwrap()

    })

}

fn bench(c: &mut Criterion) {

    let head_client = create_client!(head);

    middleware_bench_fn!(middleware_head, head);



fn client() -> s3::Client {

    let conn = test_connection();

    let config = s3::Config::builder()

        .credentials_provider(s3::config::Credentials::for_tests())

        .region(s3::config::Region::new("us-east-1"))

        .http_connector(conn.clone())

        .build();

    s3::Client::from_conf(config)

}

    let last_release_client = create_client!(last_release);

    middleware_bench_fn!(middleware_last_release, last_release);



fn middleware_bench(c: &mut Criterion) {

    let client = client();

    c.bench_function("middleware", move |b| {

    let mut group = c.benchmark_group("compare");

    let param = "S3 ListObjectsV2";

    group.bench_with_input(

        BenchmarkId::new("middleware (HEAD)", param),

        param,

        |b, _| {

            b.to_async(tokio::runtime::Runtime::new().unwrap())

            .iter(|| async { middleware(&client).await })

    });

}



fn orchestrator_bench(c: &mut Criterion) {

    let client = client();

    c.bench_function("orchestrator", move |b| {

                .iter(|| async { middleware_head(&head_client).await })

        },

    );

    group.bench_with_input(

        BenchmarkId::new("middleware (last_release)", param),

        param,

        |b, _| {

            b.to_async(tokio::runtime::Runtime::new().unwrap())

                .iter(|| async { middleware_last_release(&last_release_client).await })

        },

    );

    group.bench_with_input(BenchmarkId::new("orchestrator", param), param, |b, _| {

        b.to_async(tokio::runtime::Runtime::new().unwrap())

            .iter(|| async { orchestrator(&client).await })

            .iter(|| async { orchestrator(&head_client).await })

    });

    group.finish();

}



criterion_group!(benches, middleware_bench, orchestrator_bench);

criterion_group!(benches, bench);

criterion_main!(benches);