Unverified Commit 11a5691a authored by Matteo Bigoi's avatar Matteo Bigoi Committed by GitHub
Browse files

Add benchmark deviation calculation from origin/main to current PR (#1230) 



This PR introduce a benchmarking tool that is run as part of the GitHub actions to allow to spot performance regressions in the server implementation.

The "deviation" between the last and current benchmark is posted as a message in the pull request.

I want to let this run for a little so we can figure out if GitHub action capacity can give us consistent results, otherwise we will have to move this to some capacity we own.

Co-authored-by: default avatardavid-perez <d@vidp.dev>
parent 886a17bf

.github/workflows/server-benchmark.yml

+95 −5
Original line number Diff line number Diff line
@@ -9,12 +9,14 @@ on: 
env:

  java_version: 11

  rust_version: 1.56.1

  rust_toolchain_components: clippy,rustfmt

  apt_dependencies: libssl-dev gnuplot jq



jobs:

  run-end-to-end-integration-test:

  run-e2e-integration-test:

    runs-on: ubuntu-latest

    steps:

    - uses: actions/checkout@v2

    - uses: actions/checkout@v3

    - uses: actions/cache@v2

      name: Gradle Cache

      with:
@@ -24,17 +26,105 @@ jobs: 
        key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle*') }}

        restore-keys: |

          ${{ runner.os }}-gradle-

    # JDK is needed to generate code

      # Pinned to the commit hash of v1.3.0

    - uses: Swatinem/rust-cache@842ef286fff290e445b90b4002cc9807c3669641

      with:

        sharedKey: ${{ runner.os }}-${{ env.rust_version }}-${{ github.job }}

        target-dir: ./target

    - name: Set up JDK

      uses: actions/setup-java@v1

      with:

        java-version: ${{ env.java_version }}

    # Install Rust

    - uses: actions-rs/toolchain@v1

    - name: Install Rust

      uses: actions-rs/toolchain@v1

      with:

        toolchain: ${{ env.rust_version }}

        components: ${{ env.rust_toolchain_components }}

        default: true

    - name: Run integration tests

      run: |

        cd rust-runtime/aws-smithy-http-server/examples && \

          make && cargo test



  run-benchmark:

    runs-on: ubuntu-latest

    steps:

    - name: Checkout PR

      uses: actions/checkout@v3

      with:

        path: pull-request

    - name: Checkout origin/main

      uses: actions/checkout@v3

      with:

        repository: awslabs/smithy-rs

        path: origin-main

        ref: main

    - name: Checkout wrk

      uses: actions/checkout@v3

      with:

        repository: wg/wrk

        path: wrk-build

        ref: 4.2.0

    - uses: actions/cache@v2

      name: Gradle Cache

      with:

        path: |

          ~/.gradle/caches

          ~/.gradle/wrapper

        key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle*') }}

        restore-keys: |

          ${{ runner.os }}-gradle-

      # Pinned to the commit hash of v1.3.0

    - name: Rust Cache

      uses: Swatinem/rust-cache@842ef286fff290e445b90b4002cc9807c3669641

      with:

        sharedKey: ${{ runner.os }}-${{ env.rust_version }}-${{ github.job }}

        target-dir: ./target

    - name: Set up JDK

      uses: actions/setup-java@v1

      with:

        java-version: ${{ env.java_version }}

    - name: Install Rust

      uses: actions-rs/toolchain@v1

      with:

        toolchain: ${{ env.rust_version }}

        components: ${{ env.rust_toolchain_components }}

        default: true

    - name: Install benchmarks dependencies

      run: sudo apt-get install -y ${{ env.apt_dependencies }}

    # Ubuntu 20.04 doesn't have wrk packaged, hence we need to build it 🤦

    # This will go away as soon as GitHub supports Ubuntu 21.10.

    - name: Install wrk

      run: cd wrk-build && make -j8 wrk && sudo cp wrk /usr/local/bin

    - name: Run benchmark

      id: run-benchmark

      run: |

        mkdir -p ~/.wrk-api-bench

        # run the benchmark on origin/main

        pushd origin-main/rust-runtime/aws-smithy-http-server/examples

        make && RUN_BENCHMARKS=1 cargo test --release

        popd



        # run the benchmark on current ref

        pushd pull-request/rust-runtime/aws-smithy-http-server/examples

        make && RUN_BENCHMARKS=1 cargo test --release

        popd

        # Uncomment this for debugging purposes. It will print out the

        # content of all the benchmarks found in the cache + the last one

        # produced by the current run.

        # for x in ~/.wrk-api-bench/*; do echo "Benchmark $x content:"; jq . "$x"; echo; done



        # Ensure the output is available for the PR bot.

        echo "::set-output name=bot-message::$(cat /tmp/smithy_rs_benchmark_deviation.txt)"

    - name: Post deviation on PR

      uses: actions/github-script@v5

      # NOTE: if comments on each commit become bothersome, add a check that github.event.pull_request.action == "opened"

      if: ${{ github.head_ref != null }}

      with:

        script: |

          await github.rest.issues.createComment({

            issue_number: context.issue.number,

            owner: context.repo.owner,

            repo: context.repo.repo,

            body: '${{ steps.run-benchmark.outputs.bot-message }}'

          })

CODEOWNERS

+1 −0
Original line number Diff line number Diff line
@@ -2,3 +2,4 @@ 
/codegen-server/                        @awslabs/smithy-rs-server

/codegen-server-test/                   @awslabs/smithy-rs-server

/rust-runtime/aws-smithy-http-server/   @awslabs/smithy-rs-server

/.github/workflows/server-benchmark.yml @awslabs/smithy-rs-server

codegen-server-test/model/pokemon.smithy

+15 −1
Original line number Diff line number Diff line
@@ -10,7 +10,7 @@ use aws.protocols#restJson1 
service PokemonService {

    version: "2021-12-01",

    resources: [PokemonSpecies],

    operations: [GetServerStatistics],

    operations: [GetServerStatistics, EmptyOperation],

}



/// A Pokémon species forms the basis for at least one Pokémon.
@@ -101,6 +101,20 @@ structure FlavorText { 
])

string Language



/// Empty operation, used to stress test the framework.

@readonly

@http(uri: "/empty-operation", method: "GET")

operation EmptyOperation {

    input: EmptyOperationInput,

    output: EmptyOperationOutput,

}



@input

structure EmptyOperationInput { }



@output

structure EmptyOperationOutput { }



@error("client")

@httpError(404)

structure ResourceNotFoundException {

rust-runtime/aws-smithy-http-server/examples/BENCHMARKS.md

0 → 100644
+86 −0
Original line number Diff line number Diff line 
# Smithy Rust Server SDK benchmarks



This Pokémon Service has been benchmarked on different type of EC2 instances

using [wrk](https://github.com/wg/wrk).



<!-- vim-markdown-toc Marked -->



* [2022-03-04](#2022-03-04)

    * [c6i.8xlarge](#c6i.8xlarge)

        * [Full result](#full-result)

    * [c6g.8xlarge](#c6g.8xlarge)

        * [Full result](#full-result)



<!-- vim-markdown-toc -->



## [2022-03-04](https://github.com/awslabs/smithy-rs/commit/d823f61156577ab42590709627906d1dc35a5f49)



The benchmark runs against the `empty_operation()` operation, which is just

returning an empty output and can be used to stress test the framework overhead.



### c6i.8xlarge



* 32 cores Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz

* 64 Gb memory

* Benchmark:

    - Duration: 10 minutes

    - Connections: 1024

    - Threads: 16

* Result:

    - Request/sec: 1_608_742

    * RSS[^1] memory: 72200 bytes



#### Full result



```

❯❯❯ wrk -t16 -c1024 -d10m --latency http://localhost:13734/empty-operation

Running 10m test @ http://localhost:13734/empty-operation

  16 threads and 1024 connections

  Thread Stats   Avg      Stdev     Max   +/- Stdev

    Latency     1.03ms    1.84ms 208.10ms   92.16%

    Req/Sec   101.11k    17.59k  164.78k    70.99%

  Latency Distribution

     50%  475.00us

     75%  784.00us

     90%    2.12ms

     99%    9.74ms

  965396910 requests in 10.00m, 98.00GB read

  Socket errors: connect 19, read 0, write 0, timeout 0

Requests/sec: 1608742.65

Transfer/sec:    167.23MB

```



### c6g.8xlarge



* 32 cores Amazon Graviton 2 @ 2.50GHz

* 64 Gb memory

* Benchmark:

    - Duration: 10 minutes

    - Connections: 1024

    - Threads: 16

* Result:

    - Request/sec: 1_379_942

    - RSS[^1] memory: 70264 bytes





#### Full result



```

❯❯❯ wrk -t16 -c1024 -d10m --latency http://localhost:13734/empty-operation

Running 10m test @ http://localhost:13734/empty-operation

  16 threads and 1024 connections

  Thread Stats   Avg      Stdev     Max   +/- Stdev

    Latency     1.26ms    2.22ms 210.68ms   91.99%

    Req/Sec    86.76k    16.46k  141.30k    68.81%

  Latency Distribution

     50%  560.00us

     75%    0.93ms

     90%    2.53ms

     99%   11.95ms

  828097344 requests in 10.00m, 84.06GB read

  Socket errors: connect 19, read 0, write 0, timeout 0

Requests/sec: 1379942.45

Transfer/sec:    143.45MB

```



[^1]: https://en.wikipedia.org/wiki/Resident_set_size

rust-runtime/aws-smithy-http-server/examples/Cargo.toml

+3 −0
Original line number Diff line number Diff line
@@ -5,3 +5,6 @@ members = [ 
    "pokemon_service_sdk",

    "pokemon_service_client"

]



[profile.release]

lto = true