diff --git a/.github/workflows/server-benchmark.yml b/.github/workflows/server-benchmark.yml index 3a4a4ce3c887cc4aae1e8a0b2edddeaf83e70ead..4d7a9a188c77068627471d3a4bb583419efaf0ec 100644 --- a/.github/workflows/server-benchmark.yml +++ b/.github/workflows/server-benchmark.yml @@ -9,12 +9,14 @@ on: env: java_version: 11 rust_version: 1.56.1 + rust_toolchain_components: clippy,rustfmt + apt_dependencies: libssl-dev gnuplot jq jobs: - run-end-to-end-integration-test: + run-e2e-integration-test: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - uses: actions/cache@v2 name: Gradle Cache with: @@ -24,17 +26,105 @@ jobs: key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle*') }} restore-keys: | ${{ runner.os }}-gradle- - # JDK is needed to generate code + # Pinned to the commit hash of v1.3.0 + - uses: Swatinem/rust-cache@842ef286fff290e445b90b4002cc9807c3669641 + with: + sharedKey: ${{ runner.os }}-${{ env.rust_version }}-${{ github.job }} + target-dir: ./target - name: Set up JDK uses: actions/setup-java@v1 with: java-version: ${{ env.java_version }} - # Install Rust - - uses: actions-rs/toolchain@v1 + - name: Install Rust + uses: actions-rs/toolchain@v1 with: toolchain: ${{ env.rust_version }} + components: ${{ env.rust_toolchain_components }} default: true - name: Run integration tests run: | cd rust-runtime/aws-smithy-http-server/examples && \ make && cargo test + + run-benchmark: + runs-on: ubuntu-latest + steps: + - name: Checkout PR + uses: actions/checkout@v3 + with: + path: pull-request + - name: Checkout origin/main + uses: actions/checkout@v3 + with: + repository: awslabs/smithy-rs + path: origin-main + ref: main + - name: Checkout wrk + uses: actions/checkout@v3 + with: + repository: wg/wrk + path: wrk-build + ref: 4.2.0 + - uses: actions/cache@v2 + name: Gradle Cache + with: + path: | + ~/.gradle/caches + ~/.gradle/wrapper + key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle*') }} + restore-keys: | + ${{ runner.os }}-gradle- + # Pinned to the commit hash of v1.3.0 + - name: Rust Cache + uses: Swatinem/rust-cache@842ef286fff290e445b90b4002cc9807c3669641 + with: + sharedKey: ${{ runner.os }}-${{ env.rust_version }}-${{ github.job }} + target-dir: ./target + - name: Set up JDK + uses: actions/setup-java@v1 + with: + java-version: ${{ env.java_version }} + - name: Install Rust + uses: actions-rs/toolchain@v1 + with: + toolchain: ${{ env.rust_version }} + components: ${{ env.rust_toolchain_components }} + default: true + - name: Install benchmarks dependencies + run: sudo apt-get install -y ${{ env.apt_dependencies }} + # Ubuntu 20.04 doesn't have wrk packaged, hence we need to build it 🤦 + # This will go away as soon as GitHub supports Ubuntu 21.10. + - name: Install wrk + run: cd wrk-build && make -j8 wrk && sudo cp wrk /usr/local/bin + - name: Run benchmark + id: run-benchmark + run: | + mkdir -p ~/.wrk-api-bench + # run the benchmark on origin/main + pushd origin-main/rust-runtime/aws-smithy-http-server/examples + make && RUN_BENCHMARKS=1 cargo test --release + popd + + # run the benchmark on current ref + pushd pull-request/rust-runtime/aws-smithy-http-server/examples + make && RUN_BENCHMARKS=1 cargo test --release + popd + # Uncomment this for debugging purposes. It will print out the + # content of all the benchmarks found in the cache + the last one + # produced by the current run. + # for x in ~/.wrk-api-bench/*; do echo "Benchmark $x content:"; jq . "$x"; echo; done + + # Ensure the output is available for the PR bot. + echo "::set-output name=bot-message::$(cat /tmp/smithy_rs_benchmark_deviation.txt)" + - name: Post deviation on PR + uses: actions/github-script@v5 + # NOTE: if comments on each commit become bothersome, add a check that github.event.pull_request.action == "opened" + if: ${{ github.head_ref != null }} + with: + script: | + await github.rest.issues.createComment({ + issue_number: context.issue.number, + owner: context.repo.owner, + repo: context.repo.repo, + body: '${{ steps.run-benchmark.outputs.bot-message }}' + }) diff --git a/CODEOWNERS b/CODEOWNERS index 70ced6c8a0f0ed8bafc8c52b43a3c0066cb75f43..cc71f8948a1ed1b30dfc92f819c4c0d04db0546c 100644 --- a/CODEOWNERS +++ b/CODEOWNERS @@ -2,3 +2,4 @@ /codegen-server/ @awslabs/smithy-rs-server /codegen-server-test/ @awslabs/smithy-rs-server /rust-runtime/aws-smithy-http-server/ @awslabs/smithy-rs-server +/.github/workflows/server-benchmark.yml @awslabs/smithy-rs-server diff --git a/codegen-server-test/model/pokemon.smithy b/codegen-server-test/model/pokemon.smithy index 8c74afc76c7f96087a141497db473048747f57f3..4c9b3c626b5ea00321be98160d361b3ed41c8ac9 100644 --- a/codegen-server-test/model/pokemon.smithy +++ b/codegen-server-test/model/pokemon.smithy @@ -10,7 +10,7 @@ use aws.protocols#restJson1 service PokemonService { version: "2021-12-01", resources: [PokemonSpecies], - operations: [GetServerStatistics], + operations: [GetServerStatistics, EmptyOperation], } /// A Pokémon species forms the basis for at least one Pokémon. @@ -101,6 +101,20 @@ structure FlavorText { ]) string Language +/// Empty operation, used to stress test the framework. +@readonly +@http(uri: "/empty-operation", method: "GET") +operation EmptyOperation { + input: EmptyOperationInput, + output: EmptyOperationOutput, +} + +@input +structure EmptyOperationInput { } + +@output +structure EmptyOperationOutput { } + @error("client") @httpError(404) structure ResourceNotFoundException { diff --git a/rust-runtime/aws-smithy-http-server/examples/BENCHMARKS.md b/rust-runtime/aws-smithy-http-server/examples/BENCHMARKS.md new file mode 100644 index 0000000000000000000000000000000000000000..a5a3393066758a3ee785b06779c0df2f236f0b24 --- /dev/null +++ b/rust-runtime/aws-smithy-http-server/examples/BENCHMARKS.md @@ -0,0 +1,86 @@ +# Smithy Rust Server SDK benchmarks + +This Pokémon Service has been benchmarked on different type of EC2 instances +using [wrk](https://github.com/wg/wrk). + + + +* [2022-03-04](#2022-03-04) + * [c6i.8xlarge](#c6i.8xlarge) + * [Full result](#full-result) + * [c6g.8xlarge](#c6g.8xlarge) + * [Full result](#full-result) + + + +## [2022-03-04](https://github.com/awslabs/smithy-rs/commit/d823f61156577ab42590709627906d1dc35a5f49) + +The benchmark runs against the `empty_operation()` operation, which is just +returning an empty output and can be used to stress test the framework overhead. + +### c6i.8xlarge + +* 32 cores Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz +* 64 Gb memory +* Benchmark: + - Duration: 10 minutes + - Connections: 1024 + - Threads: 16 +* Result: + - Request/sec: 1_608_742 + * RSS[^1] memory: 72200 bytes + +#### Full result + +``` +❯❯❯ wrk -t16 -c1024 -d10m --latency http://localhost:13734/empty-operation +Running 10m test @ http://localhost:13734/empty-operation + 16 threads and 1024 connections + Thread Stats Avg Stdev Max +/- Stdev + Latency 1.03ms 1.84ms 208.10ms 92.16% + Req/Sec 101.11k 17.59k 164.78k 70.99% + Latency Distribution + 50% 475.00us + 75% 784.00us + 90% 2.12ms + 99% 9.74ms + 965396910 requests in 10.00m, 98.00GB read + Socket errors: connect 19, read 0, write 0, timeout 0 +Requests/sec: 1608742.65 +Transfer/sec: 167.23MB +``` + +### c6g.8xlarge + +* 32 cores Amazon Graviton 2 @ 2.50GHz +* 64 Gb memory +* Benchmark: + - Duration: 10 minutes + - Connections: 1024 + - Threads: 16 +* Result: + - Request/sec: 1_379_942 + - RSS[^1] memory: 70264 bytes + + +#### Full result + +``` +❯❯❯ wrk -t16 -c1024 -d10m --latency http://localhost:13734/empty-operation +Running 10m test @ http://localhost:13734/empty-operation + 16 threads and 1024 connections + Thread Stats Avg Stdev Max +/- Stdev + Latency 1.26ms 2.22ms 210.68ms 91.99% + Req/Sec 86.76k 16.46k 141.30k 68.81% + Latency Distribution + 50% 560.00us + 75% 0.93ms + 90% 2.53ms + 99% 11.95ms + 828097344 requests in 10.00m, 84.06GB read + Socket errors: connect 19, read 0, write 0, timeout 0 +Requests/sec: 1379942.45 +Transfer/sec: 143.45MB +``` + +[^1]: https://en.wikipedia.org/wiki/Resident_set_size diff --git a/rust-runtime/aws-smithy-http-server/examples/Cargo.toml b/rust-runtime/aws-smithy-http-server/examples/Cargo.toml index f3d05ef234baa3f058cfad3747aa03d610e84540..2102e16a700eea8515cc6393e8e8e5fac64aacf0 100644 --- a/rust-runtime/aws-smithy-http-server/examples/Cargo.toml +++ b/rust-runtime/aws-smithy-http-server/examples/Cargo.toml @@ -5,3 +5,6 @@ members = [ "pokemon_service_sdk", "pokemon_service_client" ] + +[profile.release] +lto = true diff --git a/rust-runtime/aws-smithy-http-server/examples/Makefile b/rust-runtime/aws-smithy-http-server/examples/Makefile index 28edb2a97129dbf770846b8931574c8a7c32562f..2774cfec868be4b6560d5066d8c933a23245dfc6 100644 --- a/rust-runtime/aws-smithy-http-server/examples/Makefile +++ b/rust-runtime/aws-smithy-http-server/examples/Makefile @@ -6,7 +6,7 @@ CLIENT_SDK_DST := $(CUR_DIR)/pokemon_service_client SERVER_SDK_SRC := $(SRC_DIR)/codegen-server-test/build/smithyprojections/codegen-server-test/pokemon_service_sdk/rust-server-codegen CLIENT_SDK_SRC := $(SRC_DIR)/codegen-test/build/smithyprojections/codegen-test/pokemon_service_client/rust-codegen -all: build +all: codegen codegen: $(GRADLE) --project-dir $(SRC_DIR) :codegen-test:assemble diff --git a/rust-runtime/aws-smithy-http-server/examples/README.md b/rust-runtime/aws-smithy-http-server/examples/README.md index 9e2e3d41fe83e981b03ba39e38817312d676d414..918ca0c5ed3ac21b25dc5ddeef340b7d2e66b5f6 100644 --- a/rust-runtime/aws-smithy-http-server/examples/README.md +++ b/rust-runtime/aws-smithy-http-server/examples/README.md @@ -13,7 +13,7 @@ build. Once the example has been built successfully the first time, idiomatic `cargo` can be used directly. -`make dist-clean` can be used for a complete cleanup of all artefacts. +`make distclean` can be used for a complete cleanup of all artefacts. ## Run @@ -29,4 +29,4 @@ More info can be found in the `tests` folder of `pokemon_service` package. ## Benchmarks -TBD. +Please see [BENCHMARKS.md](/rust-runtime/aws-smithy-http-server/example/BENCHMARKS.md). diff --git a/rust-runtime/aws-smithy-http-server/examples/pokemon_service/Cargo.toml b/rust-runtime/aws-smithy-http-server/examples/pokemon_service/Cargo.toml index bfb0ed8e67b22c0601344f25e1250b05d9ac3b21..ff9837ac1a9837a661d153da127e3ec7a3184968 100644 --- a/rust-runtime/aws-smithy-http-server/examples/pokemon_service/Cargo.toml +++ b/rust-runtime/aws-smithy-http-server/examples/pokemon_service/Cargo.toml @@ -18,6 +18,8 @@ pokemon_service_sdk = { path = "../pokemon_service_sdk/" } [dev-dependencies] assert_cmd = "2.0" +home = "0.5" +wrk-api-bench = "0.0.7" # Local paths aws-smithy-client = { path = "../../../aws-smithy-client/", features = ["rustls"] } diff --git a/rust-runtime/aws-smithy-http-server/examples/pokemon_service/src/lib.rs b/rust-runtime/aws-smithy-http-server/examples/pokemon_service/src/lib.rs index 8d8f5074fc4709f727b73e1bcdcbce7049d833ef..c133bbb4e5385ae94730dc9e4603c25d3eafdd89 100644 --- a/rust-runtime/aws-smithy-http-server/examples/pokemon_service/src/lib.rs +++ b/rust-runtime/aws-smithy-http-server/examples/pokemon_service/src/lib.rs @@ -183,6 +183,11 @@ pub async fn get_server_statistics( output::GetServerStatisticsOutput { calls_count } } +/// Empty operation used to benchmark the service. +pub async fn empty_operation(_input: input::EmptyOperationInput) -> output::EmptyOperationOutput { + output::EmptyOperationOutput {} +} + #[cfg(test)] mod tests { use super::*; diff --git a/rust-runtime/aws-smithy-http-server/examples/pokemon_service/src/main.rs b/rust-runtime/aws-smithy-http-server/examples/pokemon_service/src/main.rs index 4849113c42ccef57c0a094e80686c42b5ac78d7f..14c2b0d13bee9f00e1ea34dff0f14685ece2e42f 100644 --- a/rust-runtime/aws-smithy-http-server/examples/pokemon_service/src/main.rs +++ b/rust-runtime/aws-smithy-http-server/examples/pokemon_service/src/main.rs @@ -7,7 +7,7 @@ use std::sync::Arc; use aws_smithy_http_server::{AddExtensionLayer, Router}; -use pokemon_service::{get_pokemon_species, get_server_statistics, setup_tracing, State}; +use pokemon_service::{empty_operation, get_pokemon_species, get_server_statistics, setup_tracing, State}; use pokemon_service_sdk::operation_registry::OperationRegistryBuilder; use tower::ServiceBuilder; use tower_http::trace::TraceLayer; @@ -21,6 +21,7 @@ pub async fn main() { // return the operation's output. .get_pokemon_species(get_pokemon_species) .get_server_statistics(get_server_statistics) + .empty_operation(empty_operation) .build() .expect("Unable to build operation registry") // Convert it into a router that will route requests to the matching operation diff --git a/rust-runtime/aws-smithy-http-server/examples/pokemon_service/tests/benchmark.rs b/rust-runtime/aws-smithy-http-server/examples/pokemon_service/tests/benchmark.rs new file mode 100644 index 0000000000000000000000000000000000000000..d9139d9a94ed4a672880fa674929f9c414314a0c --- /dev/null +++ b/rust-runtime/aws-smithy-http-server/examples/pokemon_service/tests/benchmark.rs @@ -0,0 +1,58 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0. + */ + +use std::{env, fs::OpenOptions, io::Write, path::Path, time::Duration}; + +use tokio::time; +use wrk_api_bench::{BenchmarkBuilder, HistoryPeriod, WrkBuilder}; + +use crate::helpers::PokemonService; + +mod helpers; + +#[tokio::test] +async fn benchmark() -> Result<(), Box> { + // Benchmarks are expensive, so they run only if the environment + // variable `RUN_BENCHMARKS` is present. + if env::var_os("RUN_BENCHMARKS").is_some() { + let _program = PokemonService::run(); + // Give PokémonService some time to start up. + time::sleep(Duration::from_millis(50)).await; + + // The history directory is cached inside GitHub actions under + // the running use home directory to allow us to recover historical + // data between runs. + let history_dir = if env::var_os("GITHUB_ACTIONS").is_some() { + home::home_dir().unwrap().join(".wrk-api-bench") + } else { + Path::new(".").join(".wrk-api-bench") + }; + + let mut wrk = WrkBuilder::default() + .url(String::from("http://localhost:13734/empty-operation")) + .history_dir(history_dir) + .build()?; + + // Run a single benchmark with 8 threads and 64 connections for 60 seconds. + let benches = vec![BenchmarkBuilder::default() + .duration(Duration::from_secs(60)) + .threads(8) + .connections(64) + .build()?]; + wrk.bench(&benches)?; + + // Calculate deviation from last run and write it to disk. + if let Ok(deviation) = wrk.deviation(HistoryPeriod::Last) { + let mut deviation_file = OpenOptions::new() + .create(true) + .write(true) + .truncate(true) + .open("/tmp/smithy_rs_benchmark_deviation.txt") + .unwrap(); + deviation_file.write_all(deviation.to_github_markdown().as_bytes())?; + } + } + Ok(()) +} diff --git a/rust-runtime/aws-smithy-http-server/examples/pokemon_service/tests/simple_integration_test.rs b/rust-runtime/aws-smithy-http-server/examples/pokemon_service/tests/simple_integration_test.rs index 3709595d7d88a98f00ef0c80eb40a9a3530542b9..9a3f03f8e3ad9d078601d8d6d479fb090dd5ba5b 100644 --- a/rust-runtime/aws-smithy-http-server/examples/pokemon_service/tests/simple_integration_test.rs +++ b/rust-runtime/aws-smithy-http-server/examples/pokemon_service/tests/simple_integration_test.rs @@ -12,7 +12,6 @@ use std::time::Duration; use crate::helpers::{client, PokemonService}; use tokio::time; -#[macro_use] mod helpers; #[tokio::test]