Unverified Commit bed7b97f authored by John DiSanti's avatar John DiSanti Committed by GitHub
Browse files

Reduce Docker image rebuilds (#2269) 

* Move `acquire-build-image` into `.github`
* Move the `tools-hash` script into `.github`
* Upload to ECR from PRs as well
* Move build tools into `tools/ci-build/`
* Move CI scripts out of `ci-build`
* Split CI for forks/non-forks
* Remove `fetch-depth: 0` from PR workflows
parent d9c0b2e6

.github/actions/docker-build/action.yml

+4 −4
Original line number Diff line number Diff line 
# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.

# SPDX-License-Identifier: Apache-2.0



# Use this action to execute the action scripts in tools/ci-build/scripts within the Docker build image.

# Use this action to execute the action scripts in tools/ci-scripts.

name: smithy-rs Docker Build

description: Run Docker build command for smithy-rs

inputs:

  # The name of the script in tools/ci-build/scripts to run

  # The name of the script in tools/ci-scripts to run

  action:

    description: What action to run in the Docker build

    required: true
@@ -43,7 +43,7 @@ runs: 
      # from attempting to download an image from ECR since it will already exist,

      # which enables testing build image modifications as part of the pull request.

      if [[ -d smithy-rs-base-image ]]; then

        IMAGE_TAG="$(./smithy-rs/tools/ci-build/tools-hash)"

        IMAGE_TAG="$(./smithy-rs/.github/scripts/docker-image-hash)"

        docker load -i smithy-rs-base-image/smithy-rs-base-image

        docker tag "smithy-rs-base-image:${IMAGE_TAG}" "smithy-rs-base-image:local"

      fi
@@ -52,7 +52,7 @@ runs: 
      # or from ECR. We disable building the image from scratch so that any mistakes in the CI

      # configuration won't cause each individual action to build its own image, which would

      # drastically increase the total CI time. Fail fast!

      ALLOW_LOCAL_BUILD=false ./smithy-rs/tools/ci-build/acquire-build-image

      ALLOW_LOCAL_BUILD=false ./smithy-rs/.github/scripts/acquire-build-image

    # This runs the commands from the matrix strategy

  - name: Run ${{ inputs.action }}

    shell: bash

tools/ci-build/acquire-build-image→.github/scripts/acquire-build-image

+11 −10
Original line number Diff line number Diff line
@@ -45,6 +45,7 @@ class Context: 
        self.start_path = start_path

        self.script_path = script_path

        self.tools_path = tools_path

        self.docker_image_path = tools_path + "/ci-build"

        self.user_id = user_id

        self.image_tag = image_tag

        self.allow_local_build = allow_local_build
@@ -56,7 +57,7 @@ class Context: 
        script_path = os.path.dirname(os.path.realpath(__file__))

        tools_path = get_cmd_output("git rev-parse --show-toplevel", cwd=script_path)[1] + "/tools"

        user_id = get_cmd_output("id -u")[1]

        image_tag = get_cmd_output("./ci-build/tools-hash", cwd=tools_path)[1]

        image_tag = get_cmd_output("./docker-image-hash", cwd=script_path)[1]

        allow_local_build = os.getenv("ALLOW_LOCAL_BUILD") != "false"

        github_actions = os.getenv("GITHUB_ACTIONS") == "true"

        print(f"Start path: {start_path}")
@@ -118,10 +119,10 @@ class Shell: 
        run(f"docker build -t \"smithy-rs-base-image:{image_tag}\" .", cwd=path)



    # Builds the local build image

    def docker_build_build_image(self, user_id, script_path):

    def docker_build_build_image(self, user_id, docker_image_path):

        run(

            f"docker build -t smithy-rs-build-image --file add-local-user.dockerfile --build-arg=USER_ID={user_id} .",

            cwd=script_path

            cwd=docker_image_path

        )



    # Saves the Docker image named `image_name` with `image_tag` to `output_path`
@@ -188,7 +189,7 @@ def acquire_build_image(context=Context.default(), shell=Shell()): 
                return 1



            announce("Building a new image locally.")

            shell.docker_build_base_image(context.image_tag, context.tools_path)

            shell.docker_build_base_image(context.image_tag, context.docker_image_path)



            if context.github_actions:

                announce("Saving base image for use in later jobs...")
@@ -205,7 +206,7 @@ def acquire_build_image(context=Context.default(), shell=Shell()): 


    announce("Creating local build image...")

    shell.docker_tag(LOCAL_BASE_IMAGE_NAME, context.image_tag, LOCAL_BASE_IMAGE_NAME, LOCAL_TAG)

    shell.docker_build_build_image(context.user_id, context.script_path)

    shell.docker_build_build_image(context.user_id, context.docker_image_path)

    return 0
@@ -373,7 +374,7 @@ class SelfTest(unittest.TestCase): 


        shell.docker_image_exists_locally.assert_called_once()

        shell.docker_tag.assert_called_with(LOCAL_BASE_IMAGE_NAME, "someimagetag", LOCAL_BASE_IMAGE_NAME, LOCAL_TAG)

        shell.docker_build_build_image.assert_called_with("123", "/tmp/test/script-path")

        shell.docker_build_build_image.assert_called_with("123", "/tmp/test/tools-path")



    # When:

    #  - the base image doesn't exist locally
@@ -393,7 +394,7 @@ class SelfTest(unittest.TestCase): 
        shell.docker_build_base_image.assert_called_with("someimagetag", "/tmp/test/tools-path")

        shell.docker_save.assert_not_called()

        shell.docker_tag.assert_called_with(LOCAL_BASE_IMAGE_NAME, "someimagetag", LOCAL_BASE_IMAGE_NAME, LOCAL_TAG)

        shell.docker_build_build_image.assert_called_with("123", "/tmp/test/script-path")

        shell.docker_build_build_image.assert_called_with("123", "/tmp/test/tools-path")



    # When:

    #  - the base image doesn't exist locally
@@ -413,7 +414,7 @@ class SelfTest(unittest.TestCase): 
        shell.docker_build_base_image.assert_called_with("someimagetag", "/tmp/test/tools-path")

        shell.docker_save.assert_not_called()

        shell.docker_tag.assert_called_with(LOCAL_BASE_IMAGE_NAME, "someimagetag", LOCAL_BASE_IMAGE_NAME, LOCAL_TAG)

        shell.docker_build_build_image.assert_called_with("123", "/tmp/test/script-path")

        shell.docker_build_build_image.assert_called_with("123", "/tmp/test/tools-path")



    # When:

    #  - the base image doesn't exist locally
@@ -437,7 +438,7 @@ class SelfTest(unittest.TestCase): 
            "/tmp/test/start-path/smithy-rs-base-image"

        )

        shell.docker_tag.assert_called_with(LOCAL_BASE_IMAGE_NAME, "someimagetag", LOCAL_BASE_IMAGE_NAME, LOCAL_TAG)

        shell.docker_build_build_image.assert_called_with("123", "/tmp/test/script-path")

        shell.docker_build_build_image.assert_called_with("123", "/tmp/test/tools-path")



    # When:

    #  - the base image doesn't exist locally
@@ -477,7 +478,7 @@ class SelfTest(unittest.TestCase): 
            call(REMOTE_BASE_IMAGE_NAME, "someimagetag", LOCAL_BASE_IMAGE_NAME, "someimagetag"),

            call(LOCAL_BASE_IMAGE_NAME, "someimagetag", LOCAL_BASE_IMAGE_NAME, LOCAL_TAG)

        ])

        shell.docker_build_build_image.assert_called_with("123", "/tmp/test/script-path")

        shell.docker_build_build_image.assert_called_with("123", "/tmp/test/tools-path")





def main():

.github/workflows/ci-main.yml

+3 −3
Original line number Diff line number Diff line
@@ -31,8 +31,8 @@ jobs: 
      uses: actions/checkout@v3

    - name: Build image

      run: |

        IMAGE_TAG="$(./tools/ci-build/tools-hash)"

        cd tools

        IMAGE_TAG="$(./.github/scripts/docker-image-hash)"

        cd tools/ci-build

        docker build \

          -t "${{ env.ecr_repository }}:${IMAGE_TAG}" \

          -t "${{ env.ecr_repository }}:main" \
@@ -45,7 +45,7 @@ jobs: 
        aws-region: us-west-2

    - name: Upload image

      run: |

        IMAGE_TAG="$(./tools/ci-build/tools-hash)"

        IMAGE_TAG="$(./.github/scripts/docker-image-hash)"

        aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws

        docker push "${{ env.ecr_repository }}:${IMAGE_TAG}"

        docker push "${{ env.ecr_repository }}:main"

.github/workflows/ci-pr-forks.yml

0 → 100644
+45 −0
Original line number Diff line number Diff line 
# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.

# SPDX-License-Identifier: Apache-2.0



# This workflow runs CI for pull requests from forks, which can't make use of secrets.



name: CI (from fork)

on:

  pull_request:



# Allow one instance of this workflow per pull request, and cancel older runs when new changes are pushed

concurrency:

  group: ci-forks-yaml-${{ github.ref }}

  cancel-in-progress: true



jobs:

  # This job detects if the PR made changes to build tools. If it did, then it builds a new

  # build Docker image. Otherwise, it downloads a build image from Public ECR. In both cases,

  # it uploads the image as a build artifact for other jobs to download and use.

  acquire-base-image:

    name: Acquire Base Image

    if: ${{ github.event.pull_request.head.repo.full_name != 'awslabs/smithy-rs' }}

    runs-on: ubuntu-latest

    steps:

    - uses: actions/checkout@v3

      with:

        path: smithy-rs

    - name: Acquire base image

      id: acquire

      env:

        DOCKER_BUILDKIT: 1

      run: ./smithy-rs/.github/scripts/acquire-build-image

    - name: Upload base image

      uses: actions/upload-artifact@v3

      with:

        name: smithy-rs-base-image

        path: smithy-rs-base-image

        retention-days: 1



  # Run shared CI after the Docker build image has either been rebuilt or found in ECR

  ci:

    needs: acquire-base-image

    if: ${{ github.event.pull_request.head.repo.full_name != 'awslabs/smithy-rs' }}

    uses: ./.github/workflows/ci.yml

    with:

      run_sdk_examples: true