Commit e011e5e0 authored by Nugine's avatar Nugine
Browse files

ci: refactor codegen scripts

parent a1b44c51

.github/workflows/ci.yml

+6 −4
Original line number Diff line number Diff line
@@ -7,7 +7,7 @@ on: 
      - main

  schedule: # https://docs.github.com/en/actions/reference/workflow-syntax-for-github-actions#onschedule

    - cron: '0 0 * * 0' # at midnight of each sunday



  workflow_dispatch:



name: CI
@@ -27,15 +27,17 @@ jobs: 
        with:

          toolchain: ${{ matrix.toolchain }}

          components: rustfmt, clippy

      - uses: taiki-e/install-action@v2

        with:

          tool: just

      - run: cargo fmt --all -- --check

      - run: cargo clippy -- -D warnings

      - run: cargo test --all-features

      - run: |

          ./scripts/codegen.sh

          [[ -z $(git status -s) ]] # Fail if changed. See https://stackoverflow.com/a/9393642

          just codegen

          ./scripts/assert-unchanged.sh



  cross-test:

    name: cross-test ${{ matrix.os }}

    runs-on: ${{ matrix.os }}

    strategy:

      fail-fast: false

justfile

+4 −2
Original line number Diff line number Diff line
@@ -10,10 +10,12 @@ doc: 
    RUSTDOCFLAGS="--cfg docsrs" cargo +nightly doc --open --no-deps --all-features



download-model:

    ./scripts/download-model.sh

    ./scripts/download-model.py --force codegen/s3.json



codegen:

    ./scripts/codegen.sh

    ./scripts/download-model.py codegen/s3.json

    cargo run -p s3s-codegen -- codegen/s3.json

    cargo fmt



install-s3s-fs:

    cargo install --offline --path crates/s3s-fs --features binary

scripts/assert-unchanged.sh

0 → 100755
+7 −0
Original line number Diff line number Diff line 
#!/bin/bash -ex



# TODO: cross-platform?

# FIXME: ignore CRLF differences



# Fail if changed. See https://stackoverflow.com/a/9393642

[[ -z $(git status -s) ]]

scripts/codegen.sh

deleted100755 → 0
+0 −7
Original line number Diff line number Diff line 
#!/bin/bash -e

F="codegen/s3.json"

if [ ! -f $F ]; then

    ./scripts/download-model.sh

fi

cargo run -p s3s-codegen -- $F

cargo fmt

scripts/download-model.py

0 → 100755
+51 −0
Original line number Diff line number Diff line 
#!/usr/bin/env python3

from pathlib import Path

from argparse import ArgumentParser

import urllib.request

import hashlib



# https://github.com/awslabs/aws-sdk-rust/commits/main/aws-models/s3.json



COMMIT = "bb355d940cfce6f8a44a0c7ba128d96ae9dc847a"

URL = f"https://github.com/awslabs/aws-sdk-rust/raw/{COMMIT}/aws-models/s3.json"

SHA256 = "72180d9df6b0582288e6a50c6e150126c8ae516d1799967122d66274f69c972d"





def sha256sum(arg: bytes | Path) -> str:

    if isinstance(arg, bytes):

        data = arg

    elif isinstance(arg, Path):

        with open(arg, "rb") as f:

            data = f.read()

    else:

        raise TypeError()



    return hashlib.sha256(data).hexdigest()





def download(dst: Path):

    print(f"Downloading {URL} to {dst}")

    resp = urllib.request.urlopen(URL)

    data = resp.read()



    sha256 = sha256sum(data)

    print(f"SHA256: {sha256}")



    with open(dst, "wb") as f:

        f.write(data)





if __name__ == "__main__":

    parser = ArgumentParser()

    parser.add_argument("--force", action="store_true")

    parser.add_argument("filename", type=Path)

    args = parser.parse_args()



    assert isinstance(args.force, bool)

    assert isinstance(args.filename, Path)



    dst: Path = args.filename

    needs_download: bool = args.force or (not dst.exists()) or sha256sum(dst) != SHA256



    if needs_download:

        download(dst)