From 11a5691a4a3378e7d1367319be77d3bbccb735b3 Mon Sep 17 00:00:00 2001 From: Matteo Bigoi <1781140+crisidev@users.noreply.github.com> Date: Tue, 8 Mar 2022 17:42:26 +0000 Subject: [PATCH] Add benchmark deviation calculation from origin/main to current PR (#1230) This PR introduce a benchmarking tool that is run as part of the GitHub actions to allow to spot performance regressions in the server implementation. The "deviation" between the last and current benchmark is posted as a message in the pull request. I want to let this run for a little so we can figure out if GitHub action capacity can give us consistent results, otherwise we will have to move this to some capacity we own. Co-authored-by: david-perez --- .github/workflows/server-benchmark.yml | 100 +++++++++++++++++- CODEOWNERS | 1 + codegen-server-test/model/pokemon.smithy | 16 ++- .../examples/BENCHMARKS.md | 86 +++++++++++++++ .../examples/Cargo.toml | 3 + .../aws-smithy-http-server/examples/Makefile | 2 +- .../aws-smithy-http-server/examples/README.md | 4 +- .../examples/pokemon_service/Cargo.toml | 2 + .../examples/pokemon_service/src/lib.rs | 5 + .../examples/pokemon_service/src/main.rs | 3 +- .../pokemon_service/tests/benchmark.rs | 58 ++++++++++ .../tests/simple_integration_test.rs | 1 - 12 files changed, 270 insertions(+), 11 deletions(-) create mode 100644 rust-runtime/aws-smithy-http-server/examples/BENCHMARKS.md create mode 100644 rust-runtime/aws-smithy-http-server/examples/pokemon_service/tests/benchmark.rs diff --git a/.github/workflows/server-benchmark.yml b/.github/workflows/server-benchmark.yml index 3a4a4ce3c..4d7a9a188 100644 --- a/.github/workflows/server-benchmark.yml +++ b/.github/workflows/server-benchmark.yml @@ -9,12 +9,14 @@ on: env: java_version: 11 rust_version: 1.56.1 + rust_toolchain_components: clippy,rustfmt + apt_dependencies: libssl-dev gnuplot jq jobs: - run-end-to-end-integration-test: + run-e2e-integration-test: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - uses: actions/cache@v2 name: Gradle Cache with: @@ -24,17 +26,105 @@ jobs: key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle*') }} restore-keys: | ${{ runner.os }}-gradle- - # JDK is needed to generate code + # Pinned to the commit hash of v1.3.0 + - uses: Swatinem/rust-cache@842ef286fff290e445b90b4002cc9807c3669641 + with: + sharedKey: ${{ runner.os }}-${{ env.rust_version }}-${{ github.job }} + target-dir: ./target - name: Set up JDK uses: actions/setup-java@v1 with: java-version: ${{ env.java_version }} - # Install Rust - - uses: actions-rs/toolchain@v1 + - name: Install Rust + uses: actions-rs/toolchain@v1 with: toolchain: ${{ env.rust_version }} + components: ${{ env.rust_toolchain_components }} default: true - name: Run integration tests run: | cd rust-runtime/aws-smithy-http-server/examples && \ make && cargo test + + run-benchmark: + runs-on: ubuntu-latest + steps: + - name: Checkout PR + uses: actions/checkout@v3 + with: + path: pull-request + - name: Checkout origin/main + uses: actions/checkout@v3 + with: + repository: awslabs/smithy-rs + path: origin-main + ref: main + - name: Checkout wrk + uses: actions/checkout@v3 + with: + repository: wg/wrk + path: wrk-build + ref: 4.2.0 + - uses: actions/cache@v2 + name: Gradle Cache + with: + path: | + ~/.gradle/caches + ~/.gradle/wrapper + key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle*') }} + restore-keys: | + ${{ runner.os }}-gradle- + # Pinned to the commit hash of v1.3.0 + - name: Rust Cache + uses: Swatinem/rust-cache@842ef286fff290e445b90b4002cc9807c3669641 + with: + sharedKey: ${{ runner.os }}-${{ env.rust_version }}-${{ github.job }} + target-dir: ./target + - name: Set up JDK + uses: actions/setup-java@v1 + with: + java-version: ${{ env.java_version }} + - name: Install Rust + uses: actions-rs/toolchain@v1 + with: + toolchain: ${{ env.rust_version }} + components: ${{ env.rust_toolchain_components }} + default: true + - name: Install benchmarks dependencies + run: sudo apt-get install -y ${{ env.apt_dependencies }} + # Ubuntu 20.04 doesn't have wrk packaged, hence we need to build it 🤦 + # This will go away as soon as GitHub supports Ubuntu 21.10. + - name: Install wrk + run: cd wrk-build && make -j8 wrk && sudo cp wrk /usr/local/bin + - name: Run benchmark + id: run-benchmark + run: | + mkdir -p ~/.wrk-api-bench + # run the benchmark on origin/main + pushd origin-main/rust-runtime/aws-smithy-http-server/examples + make && RUN_BENCHMARKS=1 cargo test --release + popd + + # run the benchmark on current ref + pushd pull-request/rust-runtime/aws-smithy-http-server/examples + make && RUN_BENCHMARKS=1 cargo test --release + popd + # Uncomment this for debugging purposes. It will print out the + # content of all the benchmarks found in the cache + the last one + # produced by the current run. + # for x in ~/.wrk-api-bench/*; do echo "Benchmark $x content:"; jq . "$x"; echo; done + + # Ensure the output is available for the PR bot. + echo "::set-output name=bot-message::$(cat /tmp/smithy_rs_benchmark_deviation.txt)" + - name: Post deviation on PR + uses: actions/github-script@v5 + # NOTE: if comments on each commit become bothersome, add a check that github.event.pull_request.action == "opened" + if: ${{ github.head_ref != null }} + with: + script: | + await github.rest.issues.createComment({ + issue_number: context.issue.number, + owner: context.repo.owner, + repo: context.repo.repo, + body: '${{ steps.run-benchmark.outputs.bot-message }}' + }) diff --git a/CODEOWNERS b/CODEOWNERS index 70ced6c8a..cc71f8948 100644 --- a/CODEOWNERS +++ b/CODEOWNERS @@ -2,3 +2,4 @@ /codegen-server/ @awslabs/smithy-rs-server /codegen-server-test/ @awslabs/smithy-rs-server /rust-runtime/aws-smithy-http-server/ @awslabs/smithy-rs-server +/.github/workflows/server-benchmark.yml @awslabs/smithy-rs-server diff --git a/codegen-server-test/model/pokemon.smithy b/codegen-server-test/model/pokemon.smithy index 8c74afc76..4c9b3c626 100644 --- a/codegen-server-test/model/pokemon.smithy +++ b/codegen-server-test/model/pokemon.smithy @@ -10,7 +10,7 @@ use aws.protocols#restJson1 service PokemonService { version: "2021-12-01", resources: [PokemonSpecies], - operations: [GetServerStatistics], + operations: [GetServerStatistics, EmptyOperation], } /// A Pokémon species forms the basis for at least one Pokémon. @@ -101,6 +101,20 @@ structure FlavorText { ]) string Language +/// Empty operation, used to stress test the framework. +@readonly +@http(uri: "/empty-operation", method: "GET") +operation EmptyOperation { + input: EmptyOperationInput, + output: EmptyOperationOutput, +} + +@input +structure EmptyOperationInput { } + +@output +structure EmptyOperationOutput { } + @error("client") @httpError(404) structure ResourceNotFoundException { diff --git a/rust-runtime/aws-smithy-http-server/examples/BENCHMARKS.md b/rust-runtime/aws-smithy-http-server/examples/BENCHMARKS.md new file mode 100644 index 000000000..a5a339306 --- /dev/null +++ b/rust-runtime/aws-smithy-http-server/examples/BENCHMARKS.md @@ -0,0 +1,86 @@ +# Smithy Rust Server SDK benchmarks + +This Pokémon Service has been benchmarked on different type of EC2 instances +using [wrk](https://github.com/wg/wrk). + + + +* [2022-03-04](#2022-03-04) + * [c6i.8xlarge](#c6i.8xlarge) + * [Full result](#full-result) + * [c6g.8xlarge](#c6g.8xlarge) + * [Full result](#full-result) + + + +## [2022-03-04](https://github.com/awslabs/smithy-rs/commit/d823f61156577ab42590709627906d1dc35a5f49) + +The benchmark runs against the `empty_operation()` operation, which is just +returning an empty output and can be used to stress test the framework overhead. + +### c6i.8xlarge + +* 32 cores Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz +* 64 Gb memory +* Benchmark: + - Duration: 10 minutes + - Connections: 1024 + - Threads: 16 +* Result: + - Request/sec: 1_608_742 + * RSS[^1] memory: 72200 bytes + +#### Full result + +``` +❯❯❯ wrk -t16 -c1024 -d10m --latency http://localhost:13734/empty-operation +Running 10m test @ http://localhost:13734/empty-operation + 16 threads and 1024 connections + Thread Stats Avg Stdev Max +/- Stdev + Latency 1.03ms 1.84ms 208.10ms 92.16% + Req/Sec 101.11k 17.59k 164.78k 70.99% + Latency Distribution + 50% 475.00us + 75% 784.00us + 90% 2.12ms + 99% 9.74ms + 965396910 requests in 10.00m, 98.00GB read + Socket errors: connect 19, read 0, write 0, timeout 0 +Requests/sec: 1608742.65 +Transfer/sec: 167.23MB +``` + +### c6g.8xlarge + +* 32 cores Amazon Graviton 2 @ 2.50GHz +* 64 Gb memory +* Benchmark: + - Duration: 10 minutes + - Connections: 1024 + - Threads: 16 +* Result: + - Request/sec: 1_379_942 + - RSS[^1] memory: 70264 bytes + + +#### Full result + +``` +❯❯❯ wrk -t16 -c1024 -d10m --latency http://localhost:13734/empty-operation +Running 10m test @ http://localhost:13734/empty-operation + 16 threads and 1024 connections + Thread Stats Avg Stdev Max +/- Stdev + Latency 1.26ms 2.22ms 210.68ms 91.99% + Req/Sec 86.76k 16.46k 141.30k 68.81% + Latency Distribution + 50% 560.00us + 75% 0.93ms + 90% 2.53ms + 99% 11.95ms + 828097344 requests in 10.00m, 84.06GB read + Socket errors: connect 19, read 0, write 0, timeout 0 +Requests/sec: 1379942.45 +Transfer/sec: 143.45MB +``` + +[^1]: https://en.wikipedia.org/wiki/Resident_set_size diff --git a/rust-runtime/aws-smithy-http-server/examples/Cargo.toml b/rust-runtime/aws-smithy-http-server/examples/Cargo.toml index f3d05ef23..2102e16a7 100644 --- a/rust-runtime/aws-smithy-http-server/examples/Cargo.toml +++ b/rust-runtime/aws-smithy-http-server/examples/Cargo.toml @@ -5,3 +5,6 @@ members = [ "pokemon_service_sdk", "pokemon_service_client" ] + +[profile.release] +lto = true diff --git a/rust-runtime/aws-smithy-http-server/examples/Makefile b/rust-runtime/aws-smithy-http-server/examples/Makefile index 28edb2a97..2774cfec8 100644 --- a/rust-runtime/aws-smithy-http-server/examples/Makefile +++ b/rust-runtime/aws-smithy-http-server/examples/Makefile @@ -6,7 +6,7 @@ CLIENT_SDK_DST := $(CUR_DIR)/pokemon_service_client SERVER_SDK_SRC := $(SRC_DIR)/codegen-server-test/build/smithyprojections/codegen-server-test/pokemon_service_sdk/rust-server-codegen CLIENT_SDK_SRC := $(SRC_DIR)/codegen-test/build/smithyprojections/codegen-test/pokemon_service_client/rust-codegen -all: build +all: codegen codegen: $(GRADLE) --project-dir $(SRC_DIR) :codegen-test:assemble diff --git a/rust-runtime/aws-smithy-http-server/examples/README.md b/rust-runtime/aws-smithy-http-server/examples/README.md index 9e2e3d41f..918ca0c5e 100644 --- a/rust-runtime/aws-smithy-http-server/examples/README.md +++ b/rust-runtime/aws-smithy-http-server/examples/README.md @@ -13,7 +13,7 @@ build. Once the example has been built successfully the first time, idiomatic `cargo` can be used directly. -`make dist-clean` can be used for a complete cleanup of all artefacts. +`make distclean` can be used for a complete cleanup of all artefacts. ## Run @@ -29,4 +29,4 @@ More info can be found in the `tests` folder of `pokemon_service` package. ## Benchmarks -TBD. +Please see [BENCHMARKS.md](/rust-runtime/aws-smithy-http-server/example/BENCHMARKS.md). diff --git a/rust-runtime/aws-smithy-http-server/examples/pokemon_service/Cargo.toml b/rust-runtime/aws-smithy-http-server/examples/pokemon_service/Cargo.toml index bfb0ed8e6..ff9837ac1 100644 --- a/rust-runtime/aws-smithy-http-server/examples/pokemon_service/Cargo.toml +++ b/rust-runtime/aws-smithy-http-server/examples/pokemon_service/Cargo.toml @@ -18,6 +18,8 @@ pokemon_service_sdk = { path = "../pokemon_service_sdk/" } [dev-dependencies] assert_cmd = "2.0" +home = "0.5" +wrk-api-bench = "0.0.7" # Local paths aws-smithy-client = { path = "../../../aws-smithy-client/", features = ["rustls"] } diff --git a/rust-runtime/aws-smithy-http-server/examples/pokemon_service/src/lib.rs b/rust-runtime/aws-smithy-http-server/examples/pokemon_service/src/lib.rs index 8d8f5074f..c133bbb4e 100644 --- a/rust-runtime/aws-smithy-http-server/examples/pokemon_service/src/lib.rs +++ b/rust-runtime/aws-smithy-http-server/examples/pokemon_service/src/lib.rs @@ -183,6 +183,11 @@ pub async fn get_server_statistics( output::GetServerStatisticsOutput { calls_count } } +/// Empty operation used to benchmark the service. +pub async fn empty_operation(_input: input::EmptyOperationInput) -> output::EmptyOperationOutput { + output::EmptyOperationOutput {} +} + #[cfg(test)] mod tests { use super::*; diff --git a/rust-runtime/aws-smithy-http-server/examples/pokemon_service/src/main.rs b/rust-runtime/aws-smithy-http-server/examples/pokemon_service/src/main.rs index 4849113c4..14c2b0d13 100644 --- a/rust-runtime/aws-smithy-http-server/examples/pokemon_service/src/main.rs +++ b/rust-runtime/aws-smithy-http-server/examples/pokemon_service/src/main.rs @@ -7,7 +7,7 @@ use std::sync::Arc; use aws_smithy_http_server::{AddExtensionLayer, Router}; -use pokemon_service::{get_pokemon_species, get_server_statistics, setup_tracing, State}; +use pokemon_service::{empty_operation, get_pokemon_species, get_server_statistics, setup_tracing, State}; use pokemon_service_sdk::operation_registry::OperationRegistryBuilder; use tower::ServiceBuilder; use tower_http::trace::TraceLayer; @@ -21,6 +21,7 @@ pub async fn main() { // return the operation's output. .get_pokemon_species(get_pokemon_species) .get_server_statistics(get_server_statistics) + .empty_operation(empty_operation) .build() .expect("Unable to build operation registry") // Convert it into a router that will route requests to the matching operation diff --git a/rust-runtime/aws-smithy-http-server/examples/pokemon_service/tests/benchmark.rs b/rust-runtime/aws-smithy-http-server/examples/pokemon_service/tests/benchmark.rs new file mode 100644 index 000000000..d9139d9a9 --- /dev/null +++ b/rust-runtime/aws-smithy-http-server/examples/pokemon_service/tests/benchmark.rs @@ -0,0 +1,58 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0. + */ + +use std::{env, fs::OpenOptions, io::Write, path::Path, time::Duration}; + +use tokio::time; +use wrk_api_bench::{BenchmarkBuilder, HistoryPeriod, WrkBuilder}; + +use crate::helpers::PokemonService; + +mod helpers; + +#[tokio::test] +async fn benchmark() -> Result<(), Box> { + // Benchmarks are expensive, so they run only if the environment + // variable `RUN_BENCHMARKS` is present. + if env::var_os("RUN_BENCHMARKS").is_some() { + let _program = PokemonService::run(); + // Give PokémonService some time to start up. + time::sleep(Duration::from_millis(50)).await; + + // The history directory is cached inside GitHub actions under + // the running use home directory to allow us to recover historical + // data between runs. + let history_dir = if env::var_os("GITHUB_ACTIONS").is_some() { + home::home_dir().unwrap().join(".wrk-api-bench") + } else { + Path::new(".").join(".wrk-api-bench") + }; + + let mut wrk = WrkBuilder::default() + .url(String::from("http://localhost:13734/empty-operation")) + .history_dir(history_dir) + .build()?; + + // Run a single benchmark with 8 threads and 64 connections for 60 seconds. + let benches = vec![BenchmarkBuilder::default() + .duration(Duration::from_secs(60)) + .threads(8) + .connections(64) + .build()?]; + wrk.bench(&benches)?; + + // Calculate deviation from last run and write it to disk. + if let Ok(deviation) = wrk.deviation(HistoryPeriod::Last) { + let mut deviation_file = OpenOptions::new() + .create(true) + .write(true) + .truncate(true) + .open("/tmp/smithy_rs_benchmark_deviation.txt") + .unwrap(); + deviation_file.write_all(deviation.to_github_markdown().as_bytes())?; + } + } + Ok(()) +} diff --git a/rust-runtime/aws-smithy-http-server/examples/pokemon_service/tests/simple_integration_test.rs b/rust-runtime/aws-smithy-http-server/examples/pokemon_service/tests/simple_integration_test.rs index 3709595d7..9a3f03f8e 100644 --- a/rust-runtime/aws-smithy-http-server/examples/pokemon_service/tests/simple_integration_test.rs +++ b/rust-runtime/aws-smithy-http-server/examples/pokemon_service/tests/simple_integration_test.rs @@ -12,7 +12,6 @@ use std::time::Duration; use crate::helpers::{client, PokemonService}; use tokio::time; -#[macro_use] mod helpers; #[tokio::test] -- GitLab