Unverified Commit 4453b656 authored by Russell Cohen's avatar Russell Cohen Committed by GitHub
Browse files

Experiment with better CI caching (#544) 

* Experiment with better CI caching

cache `aws-sdk/target`

* Update ci.yaml

try point cargo at `../target`

* setup proper restore-keys

* Delete unused cache, force incremental

* expiriment with pre-tarring

* Fix tar

* the download is actually a folder

* add untar to all tasks

* tar the artifact produced by generating a complete SDK

* Update ci.yaml

experiment with only testing s3

* Only test a subset of services
parent f241307f

.github/workflows/ci.yaml

+45 −26
Original line number Diff line number Diff line
@@ -155,29 +155,24 @@ jobs: 
      run: ./gradlew :aws:sdk:assemble

    - name: Generate the SDK...again?

      run: ./gradlew :aws:sdk:assemble

    - name: Generate a list of services with tests

      run: python aws/sdk/test-services.py > aws/sdk/build/aws-sdk/services-with-tests

    - name: Generate a name for the SDK

      id: gen-name

      run: echo "name=${GITHUB_REF##*/}" >> $GITHUB_ENV

    - name: Tar the SDK

      run: tar -cvf sdk.tar -C aws/sdk/build/aws-sdk/ .

    - uses: actions/upload-artifact@v2

      name: Upload SDK Artifact

      with:

        name: aws-sdk-${{ env.name }}-${{ github.sha }}

        path: |

          aws/sdk/build/aws-sdk/

          !aws/sdk/build/aws-sdk/target

        path: sdk.tar



  check-sdk:

    name: cargo check AWS SDK

    needs: generate-sdk

    runs-on: ubuntu-latest

    steps:

    - uses: actions/cache@v2

      with:

        path: |

          ~/.cargo/registry

          ~/.cargo/git

          target

        key: ${{ runner.os }}-cargocheck

    - uses: actions-rs/toolchain@v1

      with:

        toolchain: ${{ env.rust_version }}
@@ -190,24 +185,30 @@ jobs: 
      name: Download SDK Artifact

      with:

        name: aws-sdk-${{ env.name }}-${{ github.sha }}

        path: aws-sdk

        path: artifact

    - name: untar

      run: mkdir aws-sdk && cd aws-sdk && tar -xvf ../artifact/sdk.tar

    - uses: actions/cache@v2

      with:

        path: |

          ~/.cargo/registry

          ~/.cargo/git

          target

        key: ${{ runner.os }}-${{ env.rust_version }}-${{ github.job }}-${{ hashFiles('**/Cargo.toml') }}

        restore-keys: |

          ${{ runner.os }}-${{ env.rust_version }}-${{ github.job }}-

          ${{ runner.os }}-${{ env.rust_version }}-

    - name: Cargo Check

      run: cargo check --lib --tests --benches

      working-directory: aws-sdk

      env:

        RUSTFLAGS: -D warnings

        CARGO_TARGET_DIR: ../target

  test-sdk:

    name: cargo test AWS SDK

    needs: generate-sdk

    runs-on: ubuntu-latest

    steps:

    - uses: actions/cache@v2

      with:

        path: |

          ~/.cargo/registry

          ~/.cargo/git

          target

        key: ${{ runner.os }}-cargotest

    - uses: actions-rs/toolchain@v1

      with:

        toolchain: ${{ env.rust_version }}
@@ -220,12 +221,26 @@ jobs: 
      name: Download SDK Artifact

      with:

        name: aws-sdk-${{ env.name }}-${{ github.sha }}

        path: aws-sdk

        path: artifact

    - name: untar

      run: mkdir aws-sdk && cd aws-sdk && tar -xvf ../artifact/sdk.tar

    - uses: actions/cache@v2

      with:

        path: |

          ~/.cargo/registry

          ~/.cargo/git

          target

        key: ${{ runner.os }}-${{ env.rust_version }}-${{ github.job }}-${{ hashFiles('**/Cargo.toml') }}

        restore-keys: |

          ${{ runner.os }}-${{ env.rust_version }}-${{ github.job }}-

          ${{ runner.os }}-${{ env.rust_version }}-

    - name: Cargo Test

      run: cargo test

      run: cargo test $(cat services-with-tests)

      working-directory: aws-sdk

      env:

        RUSTC_FORCE_INCREMENTAL: 1

        RUSTFLAGS: -D warnings

        CARGO_TARGET_DIR: ../target

  docs-sdk:

    name: cargo docs AWS SDK

    needs: generate-sdk
@@ -250,7 +265,9 @@ jobs: 
      name: Download SDK Artifact

      with:

        name: aws-sdk-${{ env.name }}-${{ github.sha }}

        path: aws-sdk

        path: artifact

    - name: untar

      run: mkdir aws-sdk && cd aws-sdk && tar -xvf ../artifact/sdk.tar

    - name: Cargo Docs

      run: cargo doc --no-deps --document-private-items

      working-directory: aws-sdk
@@ -280,7 +297,9 @@ jobs: 
      name: Download SDK Artifact

      with:

        name: aws-sdk-${{ env.name }}-${{ github.sha }}

        path: aws-sdk

        path: artifact

    - name: untar

      run: mkdir aws-sdk && cd aws-sdk && tar -xvf ../artifact/sdk.tar

    - name: Cargo Clippy

      run: cargo clippy -- -D warnings

      working-directory: aws-sdk
@@ -315,10 +334,10 @@ jobs: 
    - name: Generate a name for the SDK

      id: gen-name

      run: echo "name=${GITHUB_REF##*/}" >> $GITHUB_ENV

    - name: Tar the SDK

      run: tar -cvf sdk.tar -C aws/sdk/build/aws-sdk/ .

    - uses: actions/upload-artifact@v2

      name: Upload SDK Artifact

      with:

        name: aws-sdk-all-services-${{ env.name }}-${{ github.sha }}

        path: |

          aws/sdk/build/aws-sdk/

          !aws/sdk/build/aws-sdk/target
 
        name: aws-sdk-${{ env.name }}-allservices-${{ github.sha }}

        path: sdk.tar

aws/sdk/build.gradle.kts

+7 −7
Original line number Diff line number Diff line
@@ -110,13 +110,6 @@ fun discoverServices(allServices: Boolean): List<AwsService> { 
    val models = project.file("aws-models")

    return fileTree(models).mapNotNull { file ->

        val model = Model.assembler().addImport(file.absolutePath).assemble().result.get()

        val testFile = file.parentFile.resolve(file.nameWithoutExtension + "-tests.smithy")

        val extras = if (testFile.exists()) {

            logger.warn("Discovered protocol tests for ${file.name}")

            listOf(testFile)

        } else {

            listOf()

        }

        val services: List<ServiceShape> = model.shapes(ServiceShape::class.java).sorted().toList()

        if (services.size > 1) {

            throw Exception("There must be exactly one service in each aws model file")
@@ -126,6 +119,13 @@ fun discoverServices(allServices: Boolean): List<AwsService> { 
        } else {

            val service = services[0]

            val sdkId = service.expectTrait(ServiceTrait::class.java).sdkId.toLowerCase().replace(" ", "")

            val testFile = file.parentFile.resolve("$sdkId-tests.smithy")

            val extras = if (testFile.exists()) {

                logger.warn("Discovered protocol tests for ${file.name}")

                listOf(testFile)

            } else {

                listOf()

            }

            AwsService(service = service.id.toString(), module = sdkId, modelFile = file, extraFiles = extras)

        }

    }.filterNot { disableServices.contains(it.module) }

aws/sdk/test-services.py

0 → 100644
+37 −0
Original line number Diff line number Diff line 
"""

Generate a list of services which have non-trivial unit tests



This script generates output like `-p aws-sdk-s3 -p aws-sdk-dynamodb`. It is intended to be used in conjunction with

`cargo test` to compile a subset of services when running tests:



```bash

cargo test $(python test-services.py)

```

"""

import os

from pathlib import Path





def main():

    # change working directory to `aws/sdk`:

    script_path = os.path.abspath(__file__)

    os.chdir(os.path.dirname(script_path))



    services = set()

    for service in os.listdir('integration-tests'):

        if os.path.isdir(Path('integration-tests') / service):

            services.add(service)



    for model in os.listdir('aws-models'):

        if model.endswith('-tests.smithy'):

            service = model[:-len('-tests.smithy')]

            services.add(service)



    services = sorted(list(services))

    as_arguments = [f'-p aws-sdk-{service}' for service in services]



    print(' '.join(as_arguments), end='')





if __name__ == "__main__":

    main()