From 6371256c4fbf716e9b5b06935ef5d5a765470297 Mon Sep 17 00:00:00 2001 From: Georgi Petrov <32372905+G-D-Petrov@users.noreply.github.com> Date: Mon, 23 Dec 2024 12:36:10 +0200 Subject: [PATCH] Create a dedicated docker image for ArcticDB development (#2086) #### Reference Issues/PRs #2081 #### What does this implement or fix? This PR aims to: - create a docker image with the dev dependencies - create a workflow to push it automatically to the github registry - remove the use of the action that installs the dependencies manually and use the new docker image Successful manual run of the benchmark flow can be seen [here](https://github.com/man-group/ArcticDB/actions/runs/12411871191). It needed to be run manually because the changes to the workflow file are not picked up correctly in the automatic run here. #### Any other comments? #### Checklist
Checklist for code changes... - [ ] Have you updated the relevant docstrings, documentation and copyright notice? - [ ] Is this contribution tested against [all ArcticDB's features](../docs/mkdocs/docs/technical/contributing.md)? - [ ] Do all exceptions introduced raise appropriate [error messages](https://docs.arcticdb.io/error_messages/)? - [ ] Are API changes highlighted in the PR description? - [ ] Is the PR labelled as enhancement or bug so it appears in autogenerated release notes?
--- .github/actions/setup_deps/action.yml | 43 ------------------------- .github/workflows/analysis_workflow.yml | 5 +++ .github/workflows/benchmark_commits.yml | 11 ++----- .github/workflows/build.yml | 6 +++- .github/workflows/build_steps.yml | 4 --- .github/workflows/dev_docker_image.yml | 43 +++++++++++++++++++++++++ docker/Dockerfile | 26 +++++++++++++++ 7 files changed, 81 insertions(+), 57 deletions(-) delete mode 100644 .github/actions/setup_deps/action.yml create mode 100644 .github/workflows/dev_docker_image.yml create mode 100644 docker/Dockerfile diff --git a/.github/actions/setup_deps/action.yml b/.github/actions/setup_deps/action.yml deleted file mode 100644 index eca34a5738..0000000000 --- a/.github/actions/setup_deps/action.yml +++ /dev/null @@ -1,43 +0,0 @@ -name: 'Install GCC related dependencies' -description: 'Installs GCC related dependencies for building ArcticDB' -runs: - using: "composite" - steps: - - name: Install deps - shell: bash -l {0} - run: | - dnf update -y - dnf remove -y 'gcc-toolset-*' - dnf install -y zip flex bison gcc-toolset-10 gcc-toolset-10-gdb gcc-toolset-10-libatomic-devel krb5-devel cyrus-sasl-devel openssl-devel \ - unzip tar epel-release jq wget libcurl-devel \ - python3.11-devel python3.11-pip perl-IPC-Cmd - - dnf groupinstall -y 'Development Tools' - - echo "CC=/opt/rh/gcc-toolset-10/root/bin/gcc" | tee -a $GITHUB_ENV - echo "CMAKE_C_COMPILER=/opt/rh/gcc-toolset-10/root/bin/gcc" | tee -a $GITHUB_ENV - echo "CXX=/opt/rh/gcc-toolset-10/root/bin/g++" | tee -a $GITHUB_ENV - echo "CMAKE_CXX_COMPILER=/opt/rh/gcc-toolset-10/root/bin/g++" | tee -a $GITHUB_ENV - echo "LD_LIBRARY_PATH=/opt/rh/gcc-toolset-10/root/usr/lib64:/opt/rh/gcc-toolset-10/root/usr/lib:/opt/rh/gcc-toolset-10/root/usr/lib64/dyninst" | tee -a $GITHUB_ENV - echo "/opt/rh/devtoolset-10/root/usr/bin:/opt/python/cp311-cp311/bin" | tee -a $GITHUB_PATH - - echo $GITHUB_ENV - - echo $GITHUB_PATH - - dnf install -y mono-complete - - dnf clean all - - # sccache - sccache_ver=`curl -sL -H "Accept: application/vnd.github+json" -H "X-GitHub-Api-Version: 2022-11-28" \ - https://api.github.com/repos/mozilla/sccache/releases/latest | jq -r ".tag_name"` - - wget -nv https://github.com/mozilla/sccache/releases/download/$sccache_ver/sccache-$sccache_ver-x86_64-unknown-linux-musl.tar.gz \ - -O /tmp/sccache.tar.gz - tar -xvf /tmp/sccache.tar.gz -C /tmp/ - cp /tmp/sccache-*/sccache /usr/local/bin/ - rm /tmp/sccache.tar.gz - - which gcc - which g++ diff --git a/.github/workflows/analysis_workflow.yml b/.github/workflows/analysis_workflow.yml index 8532149b54..0b99d8d64a 100644 --- a/.github/workflows/analysis_workflow.yml +++ b/.github/workflows/analysis_workflow.yml @@ -5,6 +5,10 @@ on: run_all_benchmarks: type: boolean default: false + dev_image_tag: + description: Tag of the ArcticCB development image to use for benchmark and code coverage flows + type: string + default: latest schedule: # Schedule the job to run at 12 a.m. daily - cron: '0 0 * * *' @@ -45,6 +49,7 @@ jobs: commit: ${{ matrix.commits }} run_all_benchmarks: ${{ inputs.run_all_benchmarks || false }} run_on_pr_head: ${{ github.event_name == 'pull_request_target' }} + dev_image_tag: ${{ inputs.dev_image_tag }} publish_benchmark_results_to_gh_pages: name: Publish benchmark results to gh-pages diff --git a/.github/workflows/benchmark_commits.yml b/.github/workflows/benchmark_commits.yml index 9a4327d1e7..d928fa91d1 100644 --- a/.github/workflows/benchmark_commits.yml +++ b/.github/workflows/benchmark_commits.yml @@ -5,6 +5,7 @@ on: run_all_benchmarks: {required: true, type: boolean, description: Run all benchmarks or just the one for the given commit} commit: {required: true, type: string, description: commit hash that will be benchmarked} run_on_pr_head: {required: false, default: false, type: boolean, description: Specifies if the benchmark should run on PR head branch} + dev_image_tag: {required: false, default: 'latest', type: string, description: Tag of the ArcticDB development image} jobs: start_ec2_runner: uses: ./.github/workflows/ec2_runner_jobs.yml @@ -19,7 +20,7 @@ jobs: always() && !cancelled() runs-on: ${{ needs.start_ec2_runner.outputs.label }} - container: quay.io/pypa/manylinux_2_28_x86_64:latest + container: ghcr.io/man-group/arcticdb-dev:${{ inputs.dev_image_tag }} env: # this is potentially overflowing the cache, so should be looked into after we address issue #1057 SCCACHE_GHA_VERSION: ${{vars.SCCACHE_GHA_VERSION || 1}} # Setting this env var enables the caching @@ -31,11 +32,6 @@ jobs: defaults: run: {shell: bash} steps: - - name: Initialize LFS - shell: bash -l {0} - run: | - dnf install -y git-lfs - - uses: actions/checkout@v3.3.0 with: lfs: 'true' @@ -48,9 +44,6 @@ jobs: uses: mozilla-actions/sccache-action@v0.0.3 with: version: "v0.4.0" - - - name: Install deps - uses: ./.github/actions/setup_deps - name: Extra envs shell: bash -l {0} diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 210a5813eb..81a4d91ec0 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -19,6 +19,10 @@ on: description: Override CMAKE preset type type: choice options: ["-", debug, release] + dev_image_tag: + description: Tag of the ArcticCB development image to use for the Linux C++ tests build + type: string + default: latest run-name: Building ${{github.ref_name}} on ${{github.event_name}} by ${{github.actor}} concurrency: group: ${{github.ref}} @@ -52,7 +56,7 @@ jobs: mongodb: image: "mongo:4.4" container: - image: quay.io/pypa/manylinux_2_28_x86_64 + image: ghcr.io/man-group/arcticdb-dev:${{inputs.dev_image_tag || 'latest'}} volumes: - /:/mnt windows_matrix: diff --git a/.github/workflows/build_steps.yml b/.github/workflows/build_steps.yml index d90e1e59c3..346c5cde97 100644 --- a/.github/workflows/build_steps.yml +++ b/.github/workflows/build_steps.yml @@ -87,10 +87,6 @@ jobs: maximum-size: 6GB disk-root: "D:" # This is also the checkout directory. Total size 12GB. continue-on-error: true - - - name: Install deps - if: matrix.os == 'linux' && inputs.job_type != 'build-python-wheels' - uses: ./.github/actions/setup_deps - name: Extra envs # This has to come after msvc-dev-cmd to overwrite the bad VCPKG_ROOT it sets diff --git a/.github/workflows/dev_docker_image.yml b/.github/workflows/dev_docker_image.yml new file mode 100644 index 0000000000..c24cbdb4c5 --- /dev/null +++ b/.github/workflows/dev_docker_image.yml @@ -0,0 +1,43 @@ +name: Docker Image for Development +on: + # should support manual trigger and PRs to the Dockerfile + push: + paths: + - 'docker/**' + - '.github/workflows/dev_docker_image.yml' + workflow_dispatch: + +jobs: + build_and_push: + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + steps: + - name: Checkout + uses: actions/checkout@v3.3.0 + + - name: Choose image tag + run: | + sha=$(git rev-parse --short HEAD) + image_ver="$(date '+%Y%m%d')-${sha}" + image_name="ghcr.io/man-group/arcticdb-dev" + echo -e "image_ver=$image_ver\nimage_name=$image_name\noutput_tag=$image_name:$image_ver" | tee -a $GITHUB_ENV + + - name: Build Docker image + run: | + docker build -t $image_name . -f docker/Dockerfile + + - name: Login to GHCR + run: docker login ghcr.io -u token -p "${{secrets.GITHUB_TOKEN}}" + + - name: Publish Docker versioned image to GHCR + run: | + docker tag $image_name $image_name:$image_ver + docker push $image_name:$image_ver + + - name: Publish Docker image to GHCR as latest + if: github.ref == 'refs/heads/master' + run: | + docker tag $image_name $image_name:latest + docker push $image_name:latest diff --git a/docker/Dockerfile b/docker/Dockerfile new file mode 100644 index 0000000000..2dfb5c324b --- /dev/null +++ b/docker/Dockerfile @@ -0,0 +1,26 @@ +FROM quay.io/pypa/manylinux_2_28_x86_64 + +RUN dnf update -y +RUN dnf remove -y 'gcc-toolset-*' +RUN dnf install -y zip flex bison gcc-toolset-10 gcc-toolset-10-gdb gcc-toolset-10-libatomic-devel krb5-devel cyrus-sasl-devel openssl-devel \ +unzip tar epel-release jq wget libcurl-devel git-lfs \ +python3.11-devel python3.11-pip perl-IPC-Cmd + +RUN dnf groupinstall -y 'Development Tools' +RUN dnf install -y mono-complete + +RUN dnf clean all + +RUN wget -nv https://github.com/mozilla/sccache/releases/download/v0.8.2/sccache-v0.8.2-x86_64-unknown-linux-musl.tar.gz +RUN tar xvf sccache*.tar.gz +RUN mv sccache-*/sccache . +RUN chmod 555 sccache + +RUN cp sccache /usr/local/bin/ + +ENV CC=/opt/rh/gcc-toolset-10/root/bin/gcc +ENV CMAKE_C_COMPILER=/opt/rh/gcc-toolset-10/root/bin/gcc +ENV CXX=/opt/rh/gcc-toolset-10/root/bin/g++ +ENV CMAKE_CXX_COMPILER=/opt/rh/gcc-toolset-10/root/bin/g++ +ENV LD_LIBRARY_PATH=/opt/rh/gcc-toolset-10/root/usr/lib64:/opt/rh/gcc-toolset-10/root/usr/lib:/opt/rh/gcc-toolset-10/root/usr/lib64/dyninst +ENV PATH=/opt/rh/devtoolset-10/root/usr/bin:/opt/python/cp311-cp311/bin:${PATH}