Skip to content

Commit

Permalink
Add workflow for GitHub action from master.
Browse files Browse the repository at this point in the history
  • Loading branch information
yaron-zilliqa committed Sep 8, 2023
1 parent 281b9d8 commit 508fee0
Show file tree
Hide file tree
Showing 5 changed files with 217 additions and 14 deletions.
44 changes: 44 additions & 0 deletions .github/workflows/ci-image-dev.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,44 @@
name: CI - Development

on:
workflow_dispatch:
inputs:
commitOrTag:
description: 'Commit or tag'
required: false
default: ''
push:
branches-ignore: []

jobs:
push-to-ecr:
permissions:
id-token: write
contents: write
name: build
runs-on: docker
steps:
- name: Clean environment
# Prune the Docker resources created over 10 days before the current execution (change the value for a more/less aggressive cleanup).
shell: bash
run: |
docker system df
docker system prune -a -f --filter "until=168h"
docker system df
- name: 'Checkout scm ${{ inputs.commitOrTag }}'
uses: actions/checkout@v3
with:
fetch-depth: 0
ref: ${{ inputs.commitOrTag }}
- name: Docker build and push
uses: Zilliqa/gh-actions-workflows/actions/ci-dockerized-app-build-push@v1
with:
file: docker/Dockerfile
push: true
tag: ${{ secrets.AWS_ACCOUNT_ID_ZILLIQA }}.dkr.ecr.${{ secrets.AWS_REGION_ZILLIQA }}.amazonaws.com/scilla
tag-length: 8
registry: ${{ secrets.AWS_ACCOUNT_ID_ZILLIQA }}.dkr.ecr.${{ secrets.AWS_REGION_ZILLIQA }}.amazonaws.com
aws-region: ${{ secrets.AWS_REGION_ZILLIQA }}
role-to-assume: ${{ secrets.ECR_DEPLOYER_ROLE }}
oidc-role: ${{ secrets.OIDC_ROLE }}
cache-key: ${{ github.event.repository.name }}
66 changes: 66 additions & 0 deletions .github/workflows/ci-image-release.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,66 @@
name: CI - Release

on:
workflow_dispatch:
inputs:
commitOrTag:
description: 'Commit or tag'
required: false
default: ''
push:
tags:
- v*

jobs:
release-image:
permissions:
id-token: write
contents: write
runs-on: docker
steps:
- name: 'Checkout scm ${{ inputs.commitOrTag }}'
uses: actions/checkout@v3
with:
fetch-depth: 0
ref: ${{ inputs.commitOrTag }}
- name: Check if the commit or tag was passed manually
id: set-tag
run: |
if [[ "${{ github.event_name }}" == "workflow_dispatch" ]]; then
echo "tag=$(git rev-parse HEAD | cut -c1-8)" >> $GITHUB_OUTPUT
else
echo "tag=${{ github.ref_name }}" >> $GITHUB_OUTPUT
fi
shell: bash
- name: Check if the tag head is the same as release-v0.13.4 head
id: check-latest
if: github.event_name != 'workflow_dispatch'
run: |
if test $(git rev-parse origin/release-v0.13.4) = $(git rev-parse HEAD); then
echo "latest=true" >> $GITHUB_OUTPUT
else
echo "latest=false" >> $GITHUB_OUTPUT
fi
shell: bash
- name: Docker build and push
uses: Zilliqa/gh-actions-workflows/actions/ci-dockerized-app-build-push@v1
with:
file: docker/Dockerfile
push: true
tag: ${{ secrets.AWS_ACCOUNT_ID_ZILLIQA }}.dkr.ecr.${{ secrets.AWS_REGION_ZILLIQA }}.amazonaws.com/scilla:${{ steps.set-tag.outputs.tag }}
tag-latest: ${{ steps.check-latest.outputs.latest }}
registry: ${{ secrets.AWS_ACCOUNT_ID_ZILLIQA }}.dkr.ecr.${{ secrets.AWS_REGION_ZILLIQA }}.amazonaws.com
aws-region: ${{ secrets.AWS_REGION_ZILLIQA }}
role-to-assume: ${{ secrets.ECR_DEPLOYER_ROLE }}
oidc-role: ${{ secrets.OIDC_ROLE }}
cache-key: ${{ github.event.repository.name }}
- name: Docker build and push (Dockerhub)
uses: Zilliqa/gh-actions-workflows/actions/ci-dockerized-app-build-push@v1
with:
file: docker/Dockerfile
push: true
tag: Zilliqa/scilla:${{ steps.set-tag.outputs.tag }}
tag-latest: ${{ steps.check-latest.outputs.latest }}
cache-key: ${{ github.event.repository.name }}
registry-username: ${{ secrets.DOCKERHUB_USERNAME }}
registry-password: ${{ secrets.DOCKERHUB_PASSWORD }}
63 changes: 63 additions & 0 deletions .github/workflows/ci-image-test.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,63 @@
name: CI - Integration tests

on:
workflow_dispatch:
inputs:
commitOrTag:
description: 'Commit or tag'
required: false
default: ''
pull_request:
branches:
- 'release-v**'

jobs:
run-tests:
permissions:
id-token: write
contents: write
name: tests
runs-on: docker
steps:
- name: Clean environment
# Prune the Docker resources created over 10 days before the current execution (change the value for a more/less aggressive cleanup).
shell: bash
run: |
docker system df
docker system prune -a -f --filter "until=336h"
docker system df
- name: 'Checkout scm ${{ inputs.commitOrTag }}'
uses: actions/checkout@v3
with:
fetch-depth: 0
ref: ${{ inputs.commitOrTag }}
- name: Configure AWS Credentials
uses: Zilliqa/gh-actions-workflows/actions/configure-aws-credentials@v1
with:
role-to-assume: ${{ secrets.ECR_DEPLOYER_ROLE }}
oidc-role: ${{ secrets.OIDC_ROLE }}
aws-region: ${{ secrets.AWS_REGION_ZILLIQA }}
- name: Login to the registry
uses: docker/login-action@v2
with:
registry: ${{ secrets.AWS_ACCOUNT_ID_ZILLIQA }}.dkr.ecr.${{ secrets.AWS_REGION_ZILLIQA }}.amazonaws.com
- name: Build Docker images
run: |
DOCKER_BUILDKIT=1 docker build --target test_runner -t scilla:tests -f docker/Dockerfile .
shell: bash
- name: Run make test
run: |
docker run --rm -i scilla:tests bash -c 'eval $(opam env) && LD_LIBRARY_PATH=/scilla/0/vcpkg_installed/x64-linux-dynamic/lib make test'
shell: bash
- name: Run make test_server
run: |
docker run --rm -i scilla:tests bash -c 'eval $(opam env) && LD_LIBRARY_PATH=/scilla/0/vcpkg_installed/x64-linux-dynamic/lib make test_server'
shell: bash
- name: Run make coveralls
run: |
docker run --rm -i scilla:tests bash -c 'eval $(opam env) && LD_LIBRARY_PATH=/scilla/0/vcpkg_installed/x64-linux-dynamic/lib make coveralls TRAVIS_JOB_ID=${{ github.run_number }}'
shell: bash
- name: Run make lint
run: |
docker run --rm -i scilla:tests bash -c 'eval $(opam env) && LD_LIBRARY_PATH=/scilla/0/vcpkg_installed/x64-linux-dynamic/lib make lint'
shell: bash
41 changes: 27 additions & 14 deletions docker/Dockerfile
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
# escape=\

# Common dependencies of the builder and runner stages.
FROM ubuntu:22.04 AS base
FROM arm64v8/ubuntu:22.04 AS base
# Format guideline: one package per line and keep them alphabetically sorted
RUN apt-get update -y \
&& apt-get install -y software-properties-common \
Expand Down Expand Up @@ -46,12 +46,11 @@ RUN apt-get update -y \
FROM base AS builder

ARG CMAKE_VERSION=3.25.1
RUN wget https://github.com/Kitware/CMake/releases/download/v${CMAKE_VERSION}/cmake-${CMAKE_VERSION}-Linux-x86_64.sh \
&& echo "6598da34f0e3a0f763809e25cfdd646aa1d5e4d133c4277821e63ae5cfe09457 cmake-${CMAKE_VERSION}-Linux-x86_64.sh" | sha256sum -c \
RUN wget https://github.com/Kitware/CMake/releases/download/v${CMAKE_VERSION}/cmake-${CMAKE_VERSION}-Linux-aarch64.sh \
&& mkdir -p "${HOME}"/.local \
&& bash ./cmake-${CMAKE_VERSION}-Linux-x86_64.sh --skip-license --prefix="${HOME}"/.local/ \
&& bash ./cmake-${CMAKE_VERSION}-Linux-aarch64.sh --skip-license --prefix="${HOME}"/.local/ \
&& "${HOME}"/.local/bin/cmake --version \
&& rm cmake-${CMAKE_VERSION}-Linux-x86_64.sh
&& rm cmake-${CMAKE_VERSION}-Linux-aarch64.sh
ENV PATH="/root/.local/bin:${PATH}"

# Setup ccache
Expand All @@ -68,6 +67,7 @@ RUN ccache -p && ccache -z

# If COMMIT_OR_TAG is a branch name or a tag, clone a shallow copy which is
# faster; if this fails, just clone the full repo and checkout the commit.
ENV VCPKG_FORCE_SYSTEM_BINARIES=1
RUN git clone https://github.com/microsoft/vcpkg ${VCPKG_ROOT} \
&& git -C ${VCPKG_ROOT} checkout ${VCPKG_COMMIT_OR_TAG} \
&& ${VCPKG_ROOT}/bootstrap-vcpkg.sh
Expand Down Expand Up @@ -98,28 +98,41 @@ RUN apt update -y \
&& curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key --keyring /usr/share/keyrings/cloud.google.gpg add - \
&& apt update -y && apt install -y google-cloud-cli

ENV VCPKG_BINARY_SOURCES="default,readwrite;x-gcs,gs://vcpkg/ubuntu/22.04/x64-linux-dynamic/${VCPKG_COMMIT_OR_TAG}/,read"
ENV VCPKG_BINARY_SOURCES="default,readwrite;x-gcs,gs://vcpkg/ubuntu/22.04/arm64-linux-dynamic/${VCPKG_COMMIT_OR_TAG}/,read"

RUN --mount=type=cache,target=/root/.cache/vcpkg/ ${VCPKG_ROOT}/vcpkg install --triplet=x64-linux-dynamic
# Make sure vcpkg installs brings in the dependencies
RUN --mount=type=cache,target=/root/.cache/vcpkg/ \
echo "set(VCPKG_TARGET_ARCHITECTURE arm64)\nset(VCPKG_CRT_LINKAGE dynamic)\nset(VCPKG_LIBRARY_LINKAGE dynamic)\nset(VCPKG_CMAKE_SYSTEM_NAME Linux)\nset(VCPKG_FIXUP_ELF_RPATH ON)" >> ${VCPKG_ROOT}/triplets/community/arm64-linux-dynamic.cmake \
&& ${VCPKG_ROOT}/vcpkg install --triplet=arm64-linux-dynamic

RUN update-alternatives --install /usr/bin/python python /usr/bin/python3 10
# RUN update-alternatives --install /usr/bin/python python "${SOURCE_DIR}/vcpkg_installed/arm64-linux-dynamic/tools/python3/python3.10" 10 \
# && update-alternatives --install /usr/bin/python3 python3 "${SOURCE_DIR}/vcpkg_installed/arm64-linux-dynamic/tools/python3/python3.10" 10 # set python3 as default instead python3

ENV PKG_CONFIG_PATH="${SOURCE_DIR}/vcpkg_installed/x64-linux-dynamic/lib/pkgconfig"
ENV PKG_CONFIG_PATH="${SOURCE_DIR}/vcpkg_installed/arm64-linux-dynamic/lib/pkgconfig"

RUN make opamdep-ci \
&& echo '. ~/.opam/opam-init/init.sh > /dev/null 2> /dev/null || true ' >> ~/.bashrc \
&& eval $(opam env) \
&& make

ARG BUILD_DIR="${SOURCE_DIR}/_build/default"
ARG VCPKG_INSTALL_LIB_DIR="${BUILD_DIR}/vcpkg_installed/x64-linux-dynamic/lib"
ARG VCPKG_INSTALL_LIB_DIR="${BUILD_DIR}/vcpkg_installed/arm64-linux-dynamic/lib"

RUN mkdir -p ${VCPKG_INSTALL_LIB_DIR} \
&& ldd ${BUILD_DIR}/src/runners/*.exe | grep vcpkg_installed | gawk '{print $3}' | xargs -I{} cp {} ${VCPKG_INSTALL_LIB_DIR} \
&& rm -rf vcpkg_installed \
&& ldd ${BUILD_DIR}/src/runners/*.exe | grep vcpkg_installed | gawk '{print $3}' | xargs -I{} cp {} ${VCPKG_INSTALL_LIB_DIR}

FROM builder AS test_runner

ENV VCPKG_ROOT=/vcpkg
RUN apt update -y && apt install -y sudo
RUN ./scripts/install_shellcheck_ubuntu.sh

FROM builder AS cleanup_vcpkg

RUN rm -rf vcpkg_installed \
&& ln -s ${BUILD_DIR}/vcpkg_installed vcpkg_installed

FROM ubuntu:22.04
FROM arm64v8/ubuntu:22.04 AS base

RUN apt-get update -y \
&& apt-get install -y build-essential \
Expand All @@ -128,5 +141,5 @@ RUN apt-get update -y \

ARG SOURCE_DIR="/scilla/${MAJOR_VERSION}"

COPY --from=builder ${SOURCE_DIR} ${SOURCE_DIR}
COPY --from=cleanup_vcpkg ${SOURCE_DIR} ${SOURCE_DIR}

17 changes: 17 additions & 0 deletions docker/Dockerfile.test
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
ARG ACCOUNT_ID

FROM ${ACCOUNT_ID}.dkr.ecr.us-west-2.amazonaws.com/scilla:429e2f9

ENV VCPKG_ROOT="/vcpkg"
ENV SCILLA_REPO_ROOT="/scilla/0"

WORKDIR /scilla/0/
COPY . /scilla/0/

RUN apt update \
&& apt install -y sudo

RUN eval $(opam env) \
&& LD_LIBRARY_PATH=/scilla/0/vcpkg_installed/x64-linux-dynamic/lib opam install reason.3.8.2 --yes

RUN ./scripts/install_shellcheck_ubuntu.sh

0 comments on commit 508fee0

Please sign in to comment.