Unverified Commit 0c011d63 authored by John DiSanti's avatar John DiSanti Committed by GitHub
Browse files

Fix local CI on M1 Macs (#1346) 

The `acquire-build-image` script was coded up with the assumption that
it would only work with x86_64 build images. This commit revises that
script and the `Dockerfile` to correctly work on ARM64 architectures as
well.
parent ca849fb5

tools/Dockerfile

+20 −8
Original line number Diff line number Diff line
@@ -13,7 +13,6 @@ FROM ${base_image} AS bare_base_image 
#

FROM bare_base_image AS install_node

ARG node_version=v16.14.0

ARG node_bundle_sha256=0570b9354959f651b814e56a4ce98d4a067bf2385b9a0e6be075739bc65b0fae

ENV DEST_PATH=/opt/nodejs \

    PATH=/opt/nodejs/bin:${PATH}

RUN yum -y updateinfo && \
@@ -25,12 +24,20 @@ RUN yum -y updateinfo && \ 
    yum clean all

WORKDIR /root

RUN set -eux; \

    curl https://nodejs.org/dist/${node_version}/node-${node_version}-linux-x64.tar.xz --output node.tar.xz; \

    echo "${node_bundle_sha256}  node.tar.xz" | sha256sum --check; \

    ARCHITECTURE=""; \

    if [[ "$(uname -m)" == "aarch64" || "$(uname -m)" == "arm64" ]]; then \

        curl "https://nodejs.org/dist/${node_version}/node-${node_version}-linux-arm64.tar.xz" --output node.tar.xz; \

        echo "5a6e818c302527a4b1cdf61d3188408c8a3e4a1bbca1e3f836c93ea8469826ce  node.tar.xz" | sha256sum --check; \

        ARCHITECTURE="arm64"; \

    else \

        curl "https://nodejs.org/dist/${node_version}/node-${node_version}-linux-x64.tar.xz" --output node.tar.xz; \

        echo "0570b9354959f651b814e56a4ce98d4a067bf2385b9a0e6be075739bc65b0fae  node.tar.xz" | sha256sum --check; \

        ARCHITECTURE="x64"; \

    fi; \

    mkdir -p "${DEST_PATH}"; \

    tar -xJvf node.tar.xz -C "${DEST_PATH}"; \

    mv "${DEST_PATH}/node-${node_version}-linux-x64/"* "${DEST_PATH}"; \

    rmdir "${DEST_PATH}"/node-${node_version}-linux-x64; \

    mv "${DEST_PATH}/node-${node_version}-linux-${ARCHITECTURE}/"* "${DEST_PATH}"; \

    rmdir "${DEST_PATH}"/node-${node_version}-linux-${ARCHITECTURE}; \

    rm node.tar.xz; \

    node --version
@@ -65,8 +72,13 @@ RUN yum -y updateinfo && \ 
        pkgconfig && \

    yum clean all

RUN set -eux; \

    if [[ "$(uname -m)" == "aarch64" || "$(uname -m)" == "arm64" ]]; then \

        curl https://static.rust-lang.org/rustup/archive/1.24.3/aarch64-unknown-linux-gnu/rustup-init --output rustup-init; \

        echo "32a1532f7cef072a667bac53f1a5542c99666c4071af0c9549795bbdb2069ec1 rustup-init" | sha256sum --check; \

    else \

        curl https://static.rust-lang.org/rustup/archive/1.24.3/x86_64-unknown-linux-gnu/rustup-init --output rustup-init; \

        echo "3dc5ef50861ee18657f9db2eeb7392f9c2a6c95c90ab41e45ab4ca71476b4338 rustup-init" | sha256sum --check; \

    fi; \

    chmod +x rustup-init; \

    ./rustup-init -y --no-modify-path --profile minimal --default-toolchain ${rust_stable_version}; \

    rm rustup-init; \
@@ -121,7 +133,7 @@ COPY --chown=build:build --from=install_rust /opt/rustup /opt/rustup 
ENV PATH=/opt/cargo/bin:/opt/nodejs/bin:$PATH \

    CARGO_HOME=/opt/cargo \

    RUSTUP_HOME=/opt/rustup \

    JAVA_HOME=/usr/lib/jvm/java-11-amazon-corretto.x86_64 \

    JAVA_HOME=/usr/lib/jvm/jre-11-openjdk \

    GRADLE_USER_HOME=/home/build/.gradle \

    RUST_STABLE_VERSION=${rust_stable_version} \

    RUST_NIGHTLY_VERSION=${rust_nightly_version} \

tools/ci-build/acquire-build-image

+65 −6
Original line number Diff line number Diff line
@@ -27,10 +27,16 @@ def announce(message): 


class DockerPullResult(Enum):

    SUCCESS = 1

    ERROR_THROTTLED = 2

    RETRYABLE_ERROR = 3

    NOT_FOUND = 4

    UNKNOWN_ERROR = 5

    REMOTE_ARCHITECTURE_MISMATCH = 2

    ERROR_THROTTLED = 3

    RETRYABLE_ERROR = 4

    NOT_FOUND = 5

    UNKNOWN_ERROR = 6





class Platform(Enum):

    X86_64 = 0

    ARM_64 = 1





# Script context
@@ -65,6 +71,13 @@ class Context: 


# Mockable shell commands

class Shell:

    # Returns the platform that this script is running on

    def platform(self):

        (_, stdout, _) = get_cmd_output("uname -m")

        if stdout == "arm64":

            return Platform.ARM_64

        return Platform.X86_64



    # Returns True if the given `image_name` and `image_tag` exist locally

    def docker_image_exists_locally(self, image_name, image_tag):

        (status, _, _) = get_cmd_output(f"docker inspect \"{image_name}:{image_tag}\"", check=False)
@@ -117,6 +130,8 @@ class Shell: 


# Pulls a Docker image and retries if it gets throttled

def docker_pull_with_retry(shell, image_name, image_tag, throttle_sleep_time=45, retryable_error_sleep_time=1):

    if shell.platform() == Platform.ARM_64:

        return DockerPullResult.REMOTE_ARCHITECTURE_MISMATCH

    for attempt in range(1, 5):

        announce(f"Attempting to pull remote image {image_name}:{image_tag} (attempt {attempt})...")

        result = shell.docker_pull(image_name, image_tag)
@@ -155,15 +170,19 @@ def acquire_build_image(context=Context.default(), shell=Shell()): 
        announce("Base image not found locally.")

        pull_result = docker_pull_with_retry(shell, REMOTE_BASE_IMAGE_NAME, context.image_tag)

        if pull_result != DockerPullResult.SUCCESS:

            if pull_result == DockerPullResult.UNKNOWN_ERROR:

            if pull_result == DockerPullResult.REMOTE_ARCHITECTURE_MISMATCH:

                announce("Remote architecture is not the same as the local architecture. A local build is required.")

            elif pull_result == DockerPullResult.UNKNOWN_ERROR:

                announce("An unknown failure happened during Docker pull. This needs to be examined.")

                return 1

            else:

                announce("Failed to pull remote image, which can happen if it doesn't exist.")



            if not context.allow_local_build:

                announce("Local build turned off by ALLOW_LOCAL_BUILD env var. Aborting.")

                return 1



            announce("Failed to pull remote image, which can happen if it doesn't exist. Building a new image locally.")

            announce("Building a new image locally.")

            shell.docker_build_base_image(context.image_tag, context.tools_path)



            if context.github_actions:
@@ -199,6 +218,7 @@ class SelfTest(unittest.TestCase): 


    def mock_shell(self):

        shell = Shell()

        shell.platform = MagicMock()

        shell.docker_build_base_image = MagicMock()

        shell.docker_build_build_image = MagicMock()

        shell.docker_image_exists_locally = MagicMock()
@@ -207,6 +227,20 @@ class SelfTest(unittest.TestCase): 
        shell.docker_tag = MagicMock()

        return shell



    def test_retry_architecture_mismatch(self):

        shell = self.mock_shell()

        shell.platform.side_effect = [Platform.ARM_64]

        self.assertEqual(

            DockerPullResult.REMOTE_ARCHITECTURE_MISMATCH,

            docker_pull_with_retry(

                shell,

                "test-image",

                "test-image-tag",

                throttle_sleep_time=0,

                retryable_error_sleep_time=0

            )

        )



    def test_retry_immediate_success(self):

        shell = self.mock_shell()

        shell.docker_pull.side_effect = [DockerPullResult.SUCCESS]
@@ -327,6 +361,7 @@ class SelfTest(unittest.TestCase): 
    # It should: build a local build image using that local base image

    def test_image_exists_locally_already(self):

        shell = self.mock_shell()

        shell.platform.side_effect = [Platform.X86_64]

        shell.docker_image_exists_locally.side_effect = [True]



        self.assertEqual(0, acquire_build_image(self.test_context(), shell))
@@ -344,6 +379,7 @@ class SelfTest(unittest.TestCase): 
    def test_image_local_build(self):

        context = self.test_context(allow_local_build=True)

        shell = self.mock_shell()

        shell.platform.side_effect = [Platform.X86_64]

        shell.docker_image_exists_locally.side_effect = [False]

        shell.docker_pull.side_effect = [DockerPullResult.NOT_FOUND]
@@ -354,6 +390,26 @@ class SelfTest(unittest.TestCase): 
        shell.docker_tag.assert_called_with(LOCAL_BASE_IMAGE_NAME, "someimagetag", LOCAL_BASE_IMAGE_NAME, LOCAL_TAG)

        shell.docker_build_build_image.assert_called_with("123", "/tmp/test/script-path")



    # When:

    #  - the base image doesn't exist locally

    #  - the base image exists remotely

    #  - local builds are allowed

    #  - there is a difference in platform between local and remote

    #  - NOT running in GitHub Actions

    # It should: build a local image from scratch and NOT save it to file

    def test_image_local_build_architecture_mismatch(self):

        context = self.test_context(allow_local_build=True)

        shell = self.mock_shell()

        shell.platform.side_effect = [Platform.ARM_64]

        shell.docker_image_exists_locally.side_effect = [False]



        self.assertEqual(0, acquire_build_image(context, shell))

        shell.docker_image_exists_locally.assert_called_once()

        shell.docker_build_base_image.assert_called_with("someimagetag", "/tmp/test/tools-path")

        shell.docker_save.assert_not_called()

        shell.docker_tag.assert_called_with(LOCAL_BASE_IMAGE_NAME, "someimagetag", LOCAL_BASE_IMAGE_NAME, LOCAL_TAG)

        shell.docker_build_build_image.assert_called_with("123", "/tmp/test/script-path")



    # When:

    #  - the base image doesn't exist locally

    #  - the base image doesn't exist remotely
@@ -363,6 +419,7 @@ class SelfTest(unittest.TestCase): 
    def test_image_local_build_github_actions(self):

        context = self.test_context(allow_local_build=True, github_actions=True)

        shell = self.mock_shell()

        shell.platform.side_effect = [Platform.X86_64]

        shell.docker_image_exists_locally.side_effect = [False]

        shell.docker_pull.side_effect = [DockerPullResult.NOT_FOUND]
@@ -385,6 +442,7 @@ class SelfTest(unittest.TestCase): 
    def test_image_fail_local_build_disabled(self):

        context = self.test_context(allow_local_build=False)

        shell = self.mock_shell()

        shell.platform.side_effect = [Platform.X86_64]

        shell.docker_image_exists_locally.side_effect = [False]

        shell.docker_pull.side_effect = [DockerPullResult.NOT_FOUND]
@@ -402,6 +460,7 @@ class SelfTest(unittest.TestCase): 
    def test_pull_remote_image(self):

        context = self.test_context(allow_local_build=False)

        shell = self.mock_shell()

        shell.platform.side_effect = [Platform.X86_64]

        shell.docker_image_exists_locally.side_effect = [False]

        shell.docker_pull.side_effect = [DockerPullResult.SUCCESS]