Unverified Commit c869e869 authored by John DiSanti's avatar John DiSanti Committed by GitHub
Browse files

Add benchmark that compares current with previous release (#3159)

This PR adds a new benchmark that compares a previous release with the
current generated SDK in smithy-rs main.

----

_By submitting this pull request, I confirm that you can use, modify,
copy, and redistribute this contribution, under the terms of your
choice._
parent a69c3460
Loading
Loading
Loading
Loading
+2606 −0

File added.

Preview size limit exceeded, changes collapsed.

+24 −0
Original line number Diff line number Diff line
[package]
name = "previous-release-comparison"
version = "0.1.0"
edition = "2021"

# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html

[dependencies]
aws-config = { path = "../../build/aws-sdk/sdk/aws-config" }
aws-credential-types = { path = "../../build/aws-sdk/sdk/aws-credential-types", features = ["test-util"] }
aws-sdk-s3 = { path = "../../build/aws-sdk/sdk/s3" }
aws-smithy-runtime = { path = "../../build/aws-sdk/sdk/aws-smithy-runtime", features = ["test-util"] }
criterion = { version = "0.5", features = ["async_tokio"] }
http = "0.2.3"
previous-s3 = { version = "0.35", package = "aws-sdk-s3", features = ["test-util"] }
previous-runtime = { version = "0.57.1", package = "aws-smithy-runtime", features = ["test-util"] }
tokio = { version = "1.23.1", features = ["macros", "test-util", "rt-multi-thread"] }

[profile.release]
debug = 1

[[bench]]
name = "previous_release_comparison"
harness = false
+6 −0
Original line number Diff line number Diff line
### Middleware vs. Orchestrator Benchmark

To run the benchmark:
```bash
./gradlew :aws:sdk:assemble && (cd aws/sdk/benchmarks/previous-release-comparison && cargo bench)
```
+99 −0
Original line number Diff line number Diff line
/*
 * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
 * SPDX-License-Identifier: Apache-2.0
 */

#[macro_use]
extern crate criterion;
use criterion::{BenchmarkId, Criterion};

macro_rules! test_client {
    (previous) => {
        test_client!(@internal previous_runtime)
    };
    (main) => {
        test_client!(@internal aws_smithy_runtime)
    };
    (@internal $runtime_crate:ident) => {
        $runtime_crate::client::http::test_util::infallible_client_fn(|req| {
            assert_eq!(
                "https://test-bucket.s3.us-east-1.amazonaws.com/?list-type=2&prefix=prefix~",
                req.uri().to_string()
            );
            http::Response::builder()
                .status(200)
                .body(
                    r#"<?xml version="1.0" encoding="UTF-8"?>
                    <ListBucketResult>
                        <Name>test-bucket</Name>
                        <Prefix>prefix~</Prefix>
                        <KeyCount>1</KeyCount>
                        <MaxKeys>1000</MaxKeys>
                        <IsTruncated>false</IsTruncated>
                        <Contents>
                            <Key>some-file.file</Key>
                            <LastModified>2009-10-12T17:50:30.000Z</LastModified>
                            <Size>434234</Size>
                            <StorageClass>STANDARD</StorageClass>
                        </Contents>
                    </ListBucketResult>
                    "#,
                )
                .unwrap()
        })
    };
}

macro_rules! test {
    (previous, $client:ident) => {
        test!(@internal, $client)
    };
    (main, $client:ident) => {
        test!(@internal, $client)
    };
    (@internal, $client:ident) => {
        $client
            .list_objects_v2()
            .bucket("test-bucket")
            .prefix("prefix~")
            .send()
            .await
            .expect("successful execution")
    };
}

fn bench(c: &mut Criterion) {
    let main_client = {
        let http_client = test_client!(main);
        let config = aws_sdk_s3::Config::builder()
            .credentials_provider(aws_sdk_s3::config::Credentials::for_tests())
            .region(aws_sdk_s3::config::Region::new("us-east-1"))
            .http_client(http_client)
            .build();
        aws_sdk_s3::Client::from_conf(config)
    };
    let previous_client = {
        let http_client = test_client!(previous);
        let config = previous_s3::Config::builder()
            .credentials_provider(previous_s3::config::Credentials::for_tests())
            .region(previous_s3::config::Region::new("us-east-1"))
            .http_client(http_client)
            .build();
        previous_s3::Client::from_conf(config)
    };

    let mut group = c.benchmark_group("compare");
    let param = "S3 ListObjectsV2";
    group.bench_with_input(BenchmarkId::new("previous", param), param, |b, _| {
        b.to_async(tokio::runtime::Runtime::new().unwrap())
            .iter(|| async { test!(previous, previous_client) })
    });
    group.bench_with_input(BenchmarkId::new("main", param), param, |b, _| {
        b.to_async(tokio::runtime::Runtime::new().unwrap())
            .iter(|| async { test!(main, main_client) })
    });
    group.finish();
}

criterion_group!(benches, bench);
criterion_main!(benches);