From 508fee0c8d64e79380182ef5d9e9cc5db6abe3ad Mon Sep 17 00:00:00 2001 From: Yaron Date: Thu, 7 Sep 2023 21:42:16 +0700 Subject: [PATCH] Add workflow for GitHub action from master. --- .github/workflows/ci-image-dev.yml | 44 +++++++++++++++++ .github/workflows/ci-image-release.yml | 66 ++++++++++++++++++++++++++ .github/workflows/ci-image-test.yml | 63 ++++++++++++++++++++++++ docker/Dockerfile | 41 ++++++++++------ docker/Dockerfile.test | 17 +++++++ 5 files changed, 217 insertions(+), 14 deletions(-) create mode 100644 .github/workflows/ci-image-dev.yml create mode 100644 .github/workflows/ci-image-release.yml create mode 100644 .github/workflows/ci-image-test.yml create mode 100644 docker/Dockerfile.test diff --git a/.github/workflows/ci-image-dev.yml b/.github/workflows/ci-image-dev.yml new file mode 100644 index 000000000..fd73ac2c7 --- /dev/null +++ b/.github/workflows/ci-image-dev.yml @@ -0,0 +1,44 @@ +name: CI - Development + +on: + workflow_dispatch: + inputs: + commitOrTag: + description: 'Commit or tag' + required: false + default: '' + push: + branches-ignore: [] + +jobs: + push-to-ecr: + permissions: + id-token: write + contents: write + name: build + runs-on: docker + steps: + - name: Clean environment + # Prune the Docker resources created over 10 days before the current execution (change the value for a more/less aggressive cleanup). + shell: bash + run: | + docker system df + docker system prune -a -f --filter "until=168h" + docker system df + - name: 'Checkout scm ${{ inputs.commitOrTag }}' + uses: actions/checkout@v3 + with: + fetch-depth: 0 + ref: ${{ inputs.commitOrTag }} + - name: Docker build and push + uses: Zilliqa/gh-actions-workflows/actions/ci-dockerized-app-build-push@v1 + with: + file: docker/Dockerfile + push: true + tag: ${{ secrets.AWS_ACCOUNT_ID_ZILLIQA }}.dkr.ecr.${{ secrets.AWS_REGION_ZILLIQA }}.amazonaws.com/scilla + tag-length: 8 + registry: ${{ secrets.AWS_ACCOUNT_ID_ZILLIQA }}.dkr.ecr.${{ secrets.AWS_REGION_ZILLIQA }}.amazonaws.com + aws-region: ${{ secrets.AWS_REGION_ZILLIQA }} + role-to-assume: ${{ secrets.ECR_DEPLOYER_ROLE }} + oidc-role: ${{ secrets.OIDC_ROLE }} + cache-key: ${{ github.event.repository.name }} diff --git a/.github/workflows/ci-image-release.yml b/.github/workflows/ci-image-release.yml new file mode 100644 index 000000000..f112ff90d --- /dev/null +++ b/.github/workflows/ci-image-release.yml @@ -0,0 +1,66 @@ +name: CI - Release + +on: + workflow_dispatch: + inputs: + commitOrTag: + description: 'Commit or tag' + required: false + default: '' + push: + tags: + - v* + +jobs: + release-image: + permissions: + id-token: write + contents: write + runs-on: docker + steps: + - name: 'Checkout scm ${{ inputs.commitOrTag }}' + uses: actions/checkout@v3 + with: + fetch-depth: 0 + ref: ${{ inputs.commitOrTag }} + - name: Check if the commit or tag was passed manually + id: set-tag + run: | + if [[ "${{ github.event_name }}" == "workflow_dispatch" ]]; then + echo "tag=$(git rev-parse HEAD | cut -c1-8)" >> $GITHUB_OUTPUT + else + echo "tag=${{ github.ref_name }}" >> $GITHUB_OUTPUT + fi + shell: bash + - name: Check if the tag head is the same as release-v0.13.4 head + id: check-latest + if: github.event_name != 'workflow_dispatch' + run: | + if test $(git rev-parse origin/release-v0.13.4) = $(git rev-parse HEAD); then + echo "latest=true" >> $GITHUB_OUTPUT + else + echo "latest=false" >> $GITHUB_OUTPUT + fi + shell: bash + - name: Docker build and push + uses: Zilliqa/gh-actions-workflows/actions/ci-dockerized-app-build-push@v1 + with: + file: docker/Dockerfile + push: true + tag: ${{ secrets.AWS_ACCOUNT_ID_ZILLIQA }}.dkr.ecr.${{ secrets.AWS_REGION_ZILLIQA }}.amazonaws.com/scilla:${{ steps.set-tag.outputs.tag }} + tag-latest: ${{ steps.check-latest.outputs.latest }} + registry: ${{ secrets.AWS_ACCOUNT_ID_ZILLIQA }}.dkr.ecr.${{ secrets.AWS_REGION_ZILLIQA }}.amazonaws.com + aws-region: ${{ secrets.AWS_REGION_ZILLIQA }} + role-to-assume: ${{ secrets.ECR_DEPLOYER_ROLE }} + oidc-role: ${{ secrets.OIDC_ROLE }} + cache-key: ${{ github.event.repository.name }} + - name: Docker build and push (Dockerhub) + uses: Zilliqa/gh-actions-workflows/actions/ci-dockerized-app-build-push@v1 + with: + file: docker/Dockerfile + push: true + tag: Zilliqa/scilla:${{ steps.set-tag.outputs.tag }} + tag-latest: ${{ steps.check-latest.outputs.latest }} + cache-key: ${{ github.event.repository.name }} + registry-username: ${{ secrets.DOCKERHUB_USERNAME }} + registry-password: ${{ secrets.DOCKERHUB_PASSWORD }} diff --git a/.github/workflows/ci-image-test.yml b/.github/workflows/ci-image-test.yml new file mode 100644 index 000000000..6150e4836 --- /dev/null +++ b/.github/workflows/ci-image-test.yml @@ -0,0 +1,63 @@ +name: CI - Integration tests + +on: + workflow_dispatch: + inputs: + commitOrTag: + description: 'Commit or tag' + required: false + default: '' + pull_request: + branches: + - 'release-v**' + +jobs: + run-tests: + permissions: + id-token: write + contents: write + name: tests + runs-on: docker + steps: + - name: Clean environment + # Prune the Docker resources created over 10 days before the current execution (change the value for a more/less aggressive cleanup). + shell: bash + run: | + docker system df + docker system prune -a -f --filter "until=336h" + docker system df + - name: 'Checkout scm ${{ inputs.commitOrTag }}' + uses: actions/checkout@v3 + with: + fetch-depth: 0 + ref: ${{ inputs.commitOrTag }} + - name: Configure AWS Credentials + uses: Zilliqa/gh-actions-workflows/actions/configure-aws-credentials@v1 + with: + role-to-assume: ${{ secrets.ECR_DEPLOYER_ROLE }} + oidc-role: ${{ secrets.OIDC_ROLE }} + aws-region: ${{ secrets.AWS_REGION_ZILLIQA }} + - name: Login to the registry + uses: docker/login-action@v2 + with: + registry: ${{ secrets.AWS_ACCOUNT_ID_ZILLIQA }}.dkr.ecr.${{ secrets.AWS_REGION_ZILLIQA }}.amazonaws.com + - name: Build Docker images + run: | + DOCKER_BUILDKIT=1 docker build --target test_runner -t scilla:tests -f docker/Dockerfile . + shell: bash + - name: Run make test + run: | + docker run --rm -i scilla:tests bash -c 'eval $(opam env) && LD_LIBRARY_PATH=/scilla/0/vcpkg_installed/x64-linux-dynamic/lib make test' + shell: bash + - name: Run make test_server + run: | + docker run --rm -i scilla:tests bash -c 'eval $(opam env) && LD_LIBRARY_PATH=/scilla/0/vcpkg_installed/x64-linux-dynamic/lib make test_server' + shell: bash + - name: Run make coveralls + run: | + docker run --rm -i scilla:tests bash -c 'eval $(opam env) && LD_LIBRARY_PATH=/scilla/0/vcpkg_installed/x64-linux-dynamic/lib make coveralls TRAVIS_JOB_ID=${{ github.run_number }}' + shell: bash + - name: Run make lint + run: | + docker run --rm -i scilla:tests bash -c 'eval $(opam env) && LD_LIBRARY_PATH=/scilla/0/vcpkg_installed/x64-linux-dynamic/lib make lint' + shell: bash diff --git a/docker/Dockerfile b/docker/Dockerfile index 1e88c5e13..107651f15 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,7 +1,7 @@ # escape=\ # Common dependencies of the builder and runner stages. -FROM ubuntu:22.04 AS base +FROM arm64v8/ubuntu:22.04 AS base # Format guideline: one package per line and keep them alphabetically sorted RUN apt-get update -y \ && apt-get install -y software-properties-common \ @@ -46,12 +46,11 @@ RUN apt-get update -y \ FROM base AS builder ARG CMAKE_VERSION=3.25.1 -RUN wget https://github.com/Kitware/CMake/releases/download/v${CMAKE_VERSION}/cmake-${CMAKE_VERSION}-Linux-x86_64.sh \ - && echo "6598da34f0e3a0f763809e25cfdd646aa1d5e4d133c4277821e63ae5cfe09457 cmake-${CMAKE_VERSION}-Linux-x86_64.sh" | sha256sum -c \ +RUN wget https://github.com/Kitware/CMake/releases/download/v${CMAKE_VERSION}/cmake-${CMAKE_VERSION}-Linux-aarch64.sh \ && mkdir -p "${HOME}"/.local \ - && bash ./cmake-${CMAKE_VERSION}-Linux-x86_64.sh --skip-license --prefix="${HOME}"/.local/ \ + && bash ./cmake-${CMAKE_VERSION}-Linux-aarch64.sh --skip-license --prefix="${HOME}"/.local/ \ && "${HOME}"/.local/bin/cmake --version \ - && rm cmake-${CMAKE_VERSION}-Linux-x86_64.sh + && rm cmake-${CMAKE_VERSION}-Linux-aarch64.sh ENV PATH="/root/.local/bin:${PATH}" # Setup ccache @@ -68,6 +67,7 @@ RUN ccache -p && ccache -z # If COMMIT_OR_TAG is a branch name or a tag, clone a shallow copy which is # faster; if this fails, just clone the full repo and checkout the commit. +ENV VCPKG_FORCE_SYSTEM_BINARIES=1 RUN git clone https://github.com/microsoft/vcpkg ${VCPKG_ROOT} \ && git -C ${VCPKG_ROOT} checkout ${VCPKG_COMMIT_OR_TAG} \ && ${VCPKG_ROOT}/bootstrap-vcpkg.sh @@ -98,13 +98,17 @@ RUN apt update -y \ && curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key --keyring /usr/share/keyrings/cloud.google.gpg add - \ && apt update -y && apt install -y google-cloud-cli -ENV VCPKG_BINARY_SOURCES="default,readwrite;x-gcs,gs://vcpkg/ubuntu/22.04/x64-linux-dynamic/${VCPKG_COMMIT_OR_TAG}/,read" +ENV VCPKG_BINARY_SOURCES="default,readwrite;x-gcs,gs://vcpkg/ubuntu/22.04/arm64-linux-dynamic/${VCPKG_COMMIT_OR_TAG}/,read" -RUN --mount=type=cache,target=/root/.cache/vcpkg/ ${VCPKG_ROOT}/vcpkg install --triplet=x64-linux-dynamic +# Make sure vcpkg installs brings in the dependencies +RUN --mount=type=cache,target=/root/.cache/vcpkg/ \ + echo "set(VCPKG_TARGET_ARCHITECTURE arm64)\nset(VCPKG_CRT_LINKAGE dynamic)\nset(VCPKG_LIBRARY_LINKAGE dynamic)\nset(VCPKG_CMAKE_SYSTEM_NAME Linux)\nset(VCPKG_FIXUP_ELF_RPATH ON)" >> ${VCPKG_ROOT}/triplets/community/arm64-linux-dynamic.cmake \ + && ${VCPKG_ROOT}/vcpkg install --triplet=arm64-linux-dynamic -RUN update-alternatives --install /usr/bin/python python /usr/bin/python3 10 +# RUN update-alternatives --install /usr/bin/python python "${SOURCE_DIR}/vcpkg_installed/arm64-linux-dynamic/tools/python3/python3.10" 10 \ + # && update-alternatives --install /usr/bin/python3 python3 "${SOURCE_DIR}/vcpkg_installed/arm64-linux-dynamic/tools/python3/python3.10" 10 # set python3 as default instead python3 -ENV PKG_CONFIG_PATH="${SOURCE_DIR}/vcpkg_installed/x64-linux-dynamic/lib/pkgconfig" +ENV PKG_CONFIG_PATH="${SOURCE_DIR}/vcpkg_installed/arm64-linux-dynamic/lib/pkgconfig" RUN make opamdep-ci \ && echo '. ~/.opam/opam-init/init.sh > /dev/null 2> /dev/null || true ' >> ~/.bashrc \ @@ -112,14 +116,23 @@ RUN make opamdep-ci \ && make ARG BUILD_DIR="${SOURCE_DIR}/_build/default" -ARG VCPKG_INSTALL_LIB_DIR="${BUILD_DIR}/vcpkg_installed/x64-linux-dynamic/lib" +ARG VCPKG_INSTALL_LIB_DIR="${BUILD_DIR}/vcpkg_installed/arm64-linux-dynamic/lib" RUN mkdir -p ${VCPKG_INSTALL_LIB_DIR} \ - && ldd ${BUILD_DIR}/src/runners/*.exe | grep vcpkg_installed | gawk '{print $3}' | xargs -I{} cp {} ${VCPKG_INSTALL_LIB_DIR} \ - && rm -rf vcpkg_installed \ + && ldd ${BUILD_DIR}/src/runners/*.exe | grep vcpkg_installed | gawk '{print $3}' | xargs -I{} cp {} ${VCPKG_INSTALL_LIB_DIR} + +FROM builder AS test_runner + +ENV VCPKG_ROOT=/vcpkg +RUN apt update -y && apt install -y sudo +RUN ./scripts/install_shellcheck_ubuntu.sh + +FROM builder AS cleanup_vcpkg + +RUN rm -rf vcpkg_installed \ && ln -s ${BUILD_DIR}/vcpkg_installed vcpkg_installed -FROM ubuntu:22.04 +FROM arm64v8/ubuntu:22.04 AS base RUN apt-get update -y \ && apt-get install -y build-essential \ @@ -128,5 +141,5 @@ RUN apt-get update -y \ ARG SOURCE_DIR="/scilla/${MAJOR_VERSION}" -COPY --from=builder ${SOURCE_DIR} ${SOURCE_DIR} +COPY --from=cleanup_vcpkg ${SOURCE_DIR} ${SOURCE_DIR} diff --git a/docker/Dockerfile.test b/docker/Dockerfile.test new file mode 100644 index 000000000..f1e533cb7 --- /dev/null +++ b/docker/Dockerfile.test @@ -0,0 +1,17 @@ +ARG ACCOUNT_ID + +FROM ${ACCOUNT_ID}.dkr.ecr.us-west-2.amazonaws.com/scilla:429e2f9 + +ENV VCPKG_ROOT="/vcpkg" +ENV SCILLA_REPO_ROOT="/scilla/0" + +WORKDIR /scilla/0/ +COPY . /scilla/0/ + +RUN apt update \ + && apt install -y sudo + +RUN eval $(opam env) \ + && LD_LIBRARY_PATH=/scilla/0/vcpkg_installed/x64-linux-dynamic/lib opam install reason.3.8.2 --yes + +RUN ./scripts/install_shellcheck_ubuntu.sh