Unverified Commit a6c3aba5 authored by Thomas Cameron's avatar Thomas Cameron Committed by GitHub
Browse files

RFC30: Compile time benchmark (#2617) 



## Motivation and Context
This PR implements ci-script for bench marking compile time.
Results are turned into a markdown file and compile time is normalized
to a value relative to the compile time of S3's sdk on dev profile
without any features.

Benchmark is triggered for every PR.

I considered using AWS batch, however, I decided not to move forward
with for following reasons
- I do not have access to your AWS account, thus making the debugging
extremely difficult
- Fork's github action will always fail if you block access

## Testing
NA

## Checklist
- [ ] I have updated `CHANGELOG.next.toml` if I made changes to the
smithy-rs codegen or runtime crates
- [ ] I have updated `CHANGELOG.next.toml` if I made changes to the AWS
SDK, generated SDK code, or SDK runtime crates

----

_By submitting this pull request, I confirm that you can use, modify,
copy, and redistribute this contribution, under the terms of your
choice._

---------

Co-authored-by: default avatarZelda Hessler <zhessler@amazon.com>
parent 334891d9

.github/workflows/pull-request-bot.yml

+38 −1
Original line number Diff line number Diff line
@@ -135,10 +135,45 @@ jobs: 
      run: |

        aws s3 cp target/doc "s3://${S3_BUCKET_NAME}/docs/${{ inputs.head_revision }}" --recursive



  compiletime-benchmark:

    runs-on: ubuntu-latest

    name: Run Compiletime Benchmark

    permissions:

      id-token: write

      contents: read

      pull-requests: write

    outputs:

      compiletime-benchmark: ${{ steps.compiletime-benchmark.outputs.compiletime-benchmark }}

    steps:

    - uses: actions/checkout@v3

    - uses: actions/cache@v3

      name: Gradle Cache

      with:

        path: |

          ~/.gradle/caches

          ~/.gradle/wrapper

        key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle*') }}

        restore-keys: |

          ${{ runner.os }}-gradle-

      # JDK is needed to generate code

    - name: Set up JDK

      uses: actions/setup-java@v3

      with:

        distribution: corretto

        java-package: jdk

        java-version: ${{ env.java_version }}

    - uses: dtolnay/rust-toolchain@master

      with:

        toolchain: ${{ env.rust_version }}

    - name: run benchmark

      id: run-compiletime-benchmark

      run: bash tools/ci-scripts/compiletime-benchmark && cat /tmp/compiletime-benchmark.md >> "$GITHUB_OUTPUT"



  post-bot-comment:

    needs:

    - generate-diff

    - generate-doc-preview

    - compiletime-benchmark

    runs-on: ubuntu-latest

    name: Post bot comment

    permissions:
@@ -164,6 +199,8 @@ jobs: 
            issue_number: ${{ inputs.issue_number }},

            owner: context.repo.owner,

            repo: context.repo.repo,

            body: '${{ steps.bot-messages.outputs.codegen-diff }}\n\n' +

            body: '${{ steps.compiletime-benchmark.outputs.compiletime-benchmark }}\n\n' +

              '${{ steps.bot-messages.outputs.codegen-diff }}\n\n' +

              '${{ needs.generate-doc-preview.outputs.bot-message }}\n\n'

              

          })

tools/ci-scripts/compiletime-benchmark

0 → 100755
+33 −0
Original line number Diff line number Diff line 
#!/bin/bash

#

# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.

# SPDX-License-Identifier: Apache-2.0

#

set -eux



export TIMEFORMAT=%R



function compile() {

    cd $1 &&

        export CARGORUSTFLAG="" &&

        cargo build &>/dev/null && cargo clean && # this is for downloading crates

        \time --format="%e seconds" bash -c "cargo build  &> /dev/null" && cargo clean &&

        \time --format="%e seconds" bash -c "cargo build --release  &> /dev/null" && cargo clean &&

        export CARGORUSTFLAG="--cfg aws_sdk_unstable" &&

        \time --format="%e seconds" bash -c "cargo build --all-features &>/dev/null" && cargo clean &&

        \time --format="%e seconds" bash -c "cargo build --all-features --release &>/dev/null" && cargo clean

}



./gradlew :aws:sdk:assemble

DIR=$PWD

for variable in $(dir "aws/sdk/build/aws-sdk/sdk"); do

    if [[ $variable != *"aws-"* ]]; then

        echo "START" &>>/tmp/compiletime-benchmark.txt

        echo "$variable" &>>/tmp/compiletime-benchmark.txt

        compile "$DIR/aws/sdk/build/aws-sdk/sdk/$variable" &>>/tmp/compiletime-benchmark.txt

        echo "END" &>>/tmp/compiletime-benchmark.txt

    fi

done



cd $DIR

python3 tools/compiletime-benchmark/format.py

tools/compiletime-benchmark/format.py

0 → 100644
+40 −0
Original line number Diff line number Diff line 
#

# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.

# SPDX-License-Identifier: Apache-2.0

#



import itertools





def main():

    f = open("/tmp/compiletime-benchmark.txt", "r").read()

    iter = map(lambda x: x.split("END"), f.split("START"))

    iter = itertools.chain.from_iterable(iter)

    markdown = """

    | sdk name | dev | release | dev all features | release all features |

    | -------- | --- | ------- | ---------------- | -------------------- |

    """

    idx = 0

    for i in iter:

        idx += 1

        print("============")

        print(idx)

        print(i)

        print("============")



        lines = i.splitlines()

        if len(lines) > 1:

            continue



        sdk_name = lines[0]

        row = "\n|" + sdk_name + \

            "|".join(map(lambda x: float(x), lines[1:])) + "|"



        markdown += row



    print(markdown)

    with open("/tmp/compiletime-benchmark.md", "w") as f:

        f.write(markdown)

        f.flush()



main()