From cb7d27c89aca50ca507ec54a99fac0187cb4f183 Mon Sep 17 00:00:00 2001 From: Martin Nonnenmacher Date: Thu, 11 Apr 2024 13:37:26 +0200 Subject: [PATCH] ci(github): Rewrite the Docker build job Replace the `docker-ort` workflow with the new `docker-build` workflow. Instead of the old approach to build separate Docker images for the different package managers, the new workflow uses the Docker registry cache [1] to cache the layers in the GitHub container registry. This dramatically simplifies the build process, as no additional logic for building the Docker images is required anymore. The workflow first builds the full Docker image and caches it to the registry. The minimal image is built afterwards and can reuse all layers but the last one from the previously populated cache. This commit also slightly changes the way that Docker images are tagged: * All published images are now tagged with the Git revision. Previously, the revision was only visible as part of the ORT version tag, which does not include the revision for release builds. * The latest build from the main branch is now tagged with the `main` tag. [1]: https://docs.docker.com/build/cache/backends/registry/ Signed-off-by: Martin Nonnenmacher --- .github/workflows/docker-build.yml | 90 ++++++ .github/workflows/docker-ort.yml | 434 ----------------------------- 2 files changed, 90 insertions(+), 434 deletions(-) create mode 100644 .github/workflows/docker-build.yml delete mode 100644 .github/workflows/docker-ort.yml diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml new file mode 100644 index 0000000000000..6ce733ca1a8b8 --- /dev/null +++ b/.github/workflows/docker-build.yml @@ -0,0 +1,90 @@ +name: Docker Build + +on: + pull_request: + branches: + - main + push: + branches: + - main + tags: + - '*' + workflow_dispatch: + +env: + GRADLE_OPTS: -Dorg.gradle.daemon=false + REGISTRY: ghcr.io + IMAGE_CACHE_TAG: ghcr.io/mnonnenmacher/ort:cache + IMAGE_NAME: ghcr.io/mnonnenmacher/ort + +jobs: + disk_space: + name: Disk Space + runs-on: ubuntu-22.04 + steps: + - name: Checkout Repository + uses: actions/checkout@v4 + - name: Free Disk Space + uses: ./.github/actions/free-disk-space + - name: Setup Gradle + uses: gradle/actions/setup-gradle@v3 + with: + gradle-home-cache-cleanup: true + - name: Get ORT version + run: | + ORT_VERSION=$(./gradlew -q properties --property version | sed -nr "s/version: (.+)/\1/p") + echo "ORT_VERSION=${ORT_VERSION}" >> $GITHUB_ENV + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + - name: Login to GitHub Container Registry + uses: docker/login-action@v3 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Extract Docker Metadata for ort Image + id: meta-ort + uses: docker/metadata-action@v5 + with: + images: | + ${{ env.REGISTRY }}/${{ github.repository_owner }}/ort + tags: | + type=pep440,pattern={{version}} + type=raw,value=${{ env.ORT_VERSION }} + type=ref,event=branch + type=ref,event=tag + type=sha + - name: Build & Push ort Docker Image + uses: docker/build-push-action@v5 + with: + context: . + push: true + tags: ${{ steps.meta-ort.outputs.tags }} + labels: ${{ steps.meta-ort.outputs.labels }} + cache-from: type=registry,ref=${{ env.REGISTRY }}/${{ github.repository_owner }}/ort:cache + cache-to: type=registry,ref=${{ env.REGISTRY }}/${{ github.repository_owner }}/ort:cache,mode=max + build-args: ORT_VERSION=${{ env.ORT_VERSION }} + - name: Extract Docker Metadata for ort-minimal Image + id: meta-ort-minimal + uses: docker/metadata-action@v5 + with: + images: | + ${{ env.REGISTRY }}/${{ github.repository_owner }}/ort-minimal + tags: | + type=pep440,pattern={{version}} + type=raw,value=${{ env.ORT_VERSION }} + type=ref,event=branch + type=ref,event=tag + type=sha + - name: Build & Push ort-minimal Docker Image + uses: docker/build-push-action@v5 + with: + context: . + push: true + tags: ${{ steps.meta-ort-minimal.outputs.tags }} + labels: ${{ steps.meta-ort-minimal.outputs.labels }} + target: minimal + cache-from: type=registry,ref=${{ env.REGISTRY }}/${{ github.repository_owner }}/ort:cache + build-args: ORT_VERSION=${{ env.ORT_VERSION }} + - name: Print Disk Space + run: df -h diff --git a/.github/workflows/docker-ort.yml b/.github/workflows/docker-ort.yml deleted file mode 100644 index 1f1ef3026966f..0000000000000 --- a/.github/workflows/docker-ort.yml +++ /dev/null @@ -1,434 +0,0 @@ -# Copyright (C) 2023 The ORT Project Authors (see ) -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# SPDX-License-Identifier: Apache-2.0 -# License-Filename: LICENSE - -name: ORT Docker Image - -on: - workflow_dispatch: - schedule: - - cron: '0 4 * * *' - pull_request: - paths: - - '.versions' - - 'Dockerfile' - - '.github/workflows/docker-ort.yml' - push: - branches: - - 'main' - tags: - - '*' - -env: - REGISTRY: ghcr.io - -permissions: write-all - -jobs: - base_image: - name: Base image - runs-on: ubuntu-22.04 - steps: - - name: Checkout default branch - uses: actions/checkout@v4 - - name: Set environment variables - run: | - cat .versions >> $GITHUB_ENV - - name: Build base image - uses: ./.github/actions/ortdocker - with: - name: base - token: ${{ secrets.GITHUB_TOKEN }} - version: "${{ env.JAVA_VERSION }}-jdk-${{ env.UBUNTU_VERSION }}" - build-args: | - JAVA_VERSION=${{ env.JAVA_VERSION }} - UBUNTU_VERSION=${{ env.UBUNTU_VERSION }} - - nodejs_image: - name: NodeJS image - needs: [ base_image ] - runs-on: ubuntu-22.04 - steps: - - name: Checkout default branch - uses: actions/checkout@v4 - - name: Set environment variables - run: | - cat .versions >> $GITHUB_ENV - - name: Build NodeJS image - uses: ./.github/actions/ortdocker - with: - name: nodejs - token: ${{ secrets.GITHUB_TOKEN }} - version: "${{ env.NODEJS_VERSION }}" - build-args: | - NODEJS_VERSION=${{ env.NODEJS_VERSION }} - NPM_VERSION=${{ env.NPM_VERSION }} - PNPM_VERSION=${{ env.PNPM_VERSION }} - YARN_VERSION=${{ env.YARN_VERSION }} - - python_image: - name: Python image - needs: [ base_image ] - runs-on: ubuntu-22.04 - steps: - - name: Checkout default branch - uses: actions/checkout@v4 - - name: Set environment variables - run: | - cat .versions >> $GITHUB_ENV - - name: Build Python image - uses: ./.github/actions/ortdocker - with: - name: python - token: ${{ secrets.GITHUB_TOKEN }} - version: "${{ env.PYTHON_VERSION }}" - build-args: | - CONAN_VERSION=${{ env.CONAN_VERSION }} - PIPTOOL_VERSION=${{ env.PIPTOOL_VERSION }} - PYENV_GIT_TAG=${{ env.PYENV_GIT_TAG }} - PYTHON_INSPECTOR_VERSION=${{ env.PYTHON_INSPECTOR_VERSION }} - PYTHON_PIPENV_VERSION=${{ env.PYTHON_PIPENV_VERSION }} - PYTHON_POETRY_VERSION=${{ env.PYTHON_POETRY_VERSION }} - PYTHON_VERSION=${{ env.PYTHON_VERSION }} - SCANCODE_VERSION=${{ env.SCANCODE_VERSION }} - - rust_image: - name: Rust image - needs: [ base_image ] - runs-on: ubuntu-22.04 - steps: - - name: Checkout default branch - uses: actions/checkout@v4 - - name: Set environment variables - run: | - cat .versions >> $GITHUB_ENV - - name: Build Rust image - uses: ./.github/actions/ortdocker - with: - name: rust - token: ${{ secrets.GITHUB_TOKEN }} - version: "${{ env.RUST_VERSION }}" - build-args: | - RUST_VERSION=${{ env.RUST_VERSION }} - - ruby_image: - name: Ruby image - needs: [ base_image ] - runs-on: ubuntu-22.04 - steps: - - name: Checkout default branch - uses: actions/checkout@v4 - - name: Set environment variables - run: | - cat .versions >> $GITHUB_ENV - - name: Build Ruby image - uses: ./.github/actions/ortdocker - with: - name: ruby - token: ${{ secrets.GITHUB_TOKEN }} - version: "${{ env.RUBY_VERSION }}" - build-args: | - RUBY_VERSION=${{ env.RUBY_VERSION }} - COCOAPODS_VERSION=${{ env.COCOAPODS_VERSION }} - - golang_image: - name: Golang image - needs: [ base_image ] - runs-on: ubuntu-22.04 - steps: - - name: Checkout default branch - uses: actions/checkout@v4 - - name: Set environment variables - run: | - cat .versions >> $GITHUB_ENV - - name: Build Golang image - uses: ./.github/actions/ortdocker - with: - name: golang - token: ${{ secrets.GITHUB_TOKEN }} - version: "${{ env.GO_VERSION }}" - build-args: | - GO_VERSION=${{ env.GO_VERSION }} - - android_image: - name: Android image - needs: [ base_image ] - runs-on: ubuntu-22.04 - steps: - - name: Checkout default branch - uses: actions/checkout@v4 - - name: Set environment variables - run: | - cat .versions >> $GITHUB_ENV - - name: Build Android image - uses: ./.github/actions/ortdocker - with: - name: android - token: ${{ secrets.GITHUB_TOKEN }} - version: "${{ env.ANDROID_CMD_VERSION }}" - build-args: | - ANDROID_CMD_VERSION=${{ env.ANDROID_CMD_VERSION }} - - dart_image: - name: Dart image - needs: [ base_image ] - runs-on: ubuntu-22.04 - steps: - - name: Checkout default branch - uses: actions/checkout@v4 - - name: Set environment variables - run: | - cat .versions >> $GITHUB_ENV - - name: Build Dart image - uses: ./.github/actions/ortdocker - with: - name: dart - token: ${{ secrets.GITHUB_TOKEN }} - version: "${{ env.DART_VERSION }}" - build-args: | - DART_VERSION=${{ env.DART_VERSION }} - - dotnet_image: - name: Dotnet image - needs: [ base_image ] - runs-on: ubuntu-22.04 - steps: - - name: Checkout default branch - uses: actions/checkout@v4 - - name: Set environment variables - run: | - cat .versions >> $GITHUB_ENV - - name: Build Dotnet image - uses: ./.github/actions/ortdocker - with: - name: dotnet - token: ${{ secrets.GITHUB_TOKEN }} - version: "${{ env.DOTNET_VERSION }}" - build-args: | - DOTNET_VERSION=${{ env.DOTNET_VERSION }} - NUGET_INSPECTOR_VERSION=${{ env.NUGET_INSPECTOR_VERSION }} - - haskell_image: - name: Haskell image - needs: [ base_image ] - runs-on: ubuntu-22.04 - steps: - - name: Checkout default branch - uses: actions/checkout@v4 - - name: Set environment variables - run: | - cat .versions >> $GITHUB_ENV - - name: Build Haskell image - uses: ./.github/actions/ortdocker - with: - name: haskell - token: ${{ secrets.GITHUB_TOKEN }} - version: "${{ env.HASKELL_STACK_VERSION }}" - build-args: | - HASKELL_STACK_VERSION=${{ env.HASKELL_STACK_VERSION }} - - scala_image: - name: Scala image - needs: [ base_image ] - runs-on: ubuntu-22.04 - steps: - - name: Checkout default branch - uses: actions/checkout@v4 - - name: Set environment variables - run: | - cat .versions >> $GITHUB_ENV - - name: Build Scala image - uses: ./.github/actions/ortdocker - with: - name: scala - token: ${{ secrets.GITHUB_TOKEN }} - version: "${{ env.SBT_VERSION }}" - build-args: | - SBT_VERSION=${{ env.SBT_VERSION }} - - swift_image: - name: Swift image - needs: [ base_image ] - runs-on: ubuntu-22.04 - steps: - - name: Checkout default branch - uses: actions/checkout@v4 - - name: Set environment variables - run: | - cat .versions >> $GITHUB_ENV - - name: Build Swift image - uses: ./.github/actions/ortdocker - with: - name: swift - token: ${{ secrets.GITHUB_TOKEN }} - version: "${{ env.SWIFT_VERSION }}" - build-args: | - SWIFT_VERSION=${{ env.SWIFT_VERSION }} - - bazel_image: - name: Bazel image - needs: [ base_image ] - runs-on: ubuntu-22.04 - steps: - - name: Checkout default branch - uses: actions/checkout@v4 - - name: Set environment variables - run: | - cat .versions >> $GITHUB_ENV - - name: Build Bazel image - uses: ./.github/actions/ortdocker - with: - name: bazel - token: ${{ secrets.GITHUB_TOKEN }} - version: "${{ env.BAZEL_VERSION }}" - build-args: | - BAZEL_VERSION=${{ env.BAZEL_VERSION }} - - # Minimal Runtime ORT image - # ------------------------- - minimal_image: - needs: - [ base_image, nodejs_image, python_image, rust_image, ruby_image, golang_image ] - name: Build ORT minimal image - runs-on: ubuntu-22.04 - permissions: - contents: read - packages: write - - steps: - - name: Checkout default branch - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - - name: Set environment variables - run: | - cat .versions >> $GITHUB_ENV - - - name: Get ORT current version - run: | - ORT_VERSION=$(./gradlew -q properties --property version | sed -nr "s/version: (.+)/\1/p") - echo "ORT_VERSION=${ORT_VERSION}" >> $GITHUB_ENV - - - name: Set up Docker build - uses: docker/setup-buildx-action@v3 - - - name: Login to GitHub container registry - uses: docker/login-action@v3 - with: - registry: ${{ env.REGISTRY }} - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Extract components metadata (tags, labels) - id: meta-ort-minimal - uses: docker/metadata-action@v5 - with: - images: | - ${{ env.REGISTRY }}/${{ github.repository_owner }}/ort-minimal - tags: | - type=schedule,pattern={{date 'YYYYMMDD'}} - type=schedule,pattern=snapshot - type=pep440,pattern={{version}} - type=raw,value=${{ env.ORT_VERSION }} - type=ref,event=tag - - - name: Build ORT minimal image - uses: docker/build-push-action@v5 - with: - context: . - target: minimal - push: ${{ github.event_name == 'push' || github.event_name == 'workflow_dispatch' || github.event_name == 'schedule' }} - load: false - build-args: | - NODEJS_VERSION=${{ env.NODEJS_VERSION }} - ORT_VERSION=${{ env.ORT_VERSION }} - tags: | - ${{ steps.meta-ort-minimal.outputs.tags }} - labels: ${{ steps.meta.outputs.labels }} - build-contexts: | - base=docker-image://${{ env.REGISTRY }}/${{ github.repository }}/base:latest - nodejs=docker-image://${{ env.REGISTRY }}/${{ github.repository }}/nodejs:latest - python=docker-image://${{ env.REGISTRY }}/${{ github.repository }}/python:latest - rust=docker-image://${{ env.REGISTRY }}/${{ github.repository }}/rust:latest - ruby=docker-image://${{ env.REGISTRY }}/${{ github.repository }}/ruby:latest - golang=docker-image://${{ env.REGISTRY }}/${{ github.repository }}/golang:latest - - # Full Runtime ORT image - # ---------------------- - ort_image: - if: ${{ github.event_name != 'pull_request' }} - name: Build ORT image - needs: - [ minimal_image, android_image, dart_image, dotnet_image, haskell_image, scala_image, swift_image ] - runs-on: ubuntu-22.04 - permissions: - contents: read - packages: write - - steps: - - name: Checkout default branch - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - - name: Get ORT current version - run: | - ORT_VERSION=$(./gradlew -q properties --property version | sed -nr "s/version: (.+)/\1/p") - echo "ORT_VERSION=${ORT_VERSION}" >> $GITHUB_ENV - - - name: Set up Docker build - uses: docker/setup-buildx-action@v3 - - - name: Login to GitHub container registry - uses: docker/login-action@v3 - with: - registry: ${{ env.REGISTRY }} - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Extract components metadata (tags, labels) - id: meta-ort - uses: docker/metadata-action@v5 - with: - images: | - ${{ env.REGISTRY }}/${{ github.repository_owner }}/ort - tags: | - type=schedule,pattern={{date 'YYYYMMDD'}} - type=schedule,pattern=snapshot - type=pep440,pattern={{version}} - type=raw,value=${{ env.ORT_VERSION }} - type=ref,event=tag - - - name: Build ORT image - uses: docker/build-push-action@v5 - with: - context: . - target: run - push: ${{ github.event_name == 'push' || github.event_name == 'workflow_dispatch' || github.event_name == 'schedule' }} - load: false - tags: | - ${{ steps.meta-ort.outputs.tags }} - labels: ${{ steps.meta.outputs.labels }} - build-contexts: | - minimal=docker-image://${{ env.REGISTRY }}/${{ github.repository_owner }}/ort-minimal:${{ env.ORT_VERSION }} - android=docker-image://${{ env.REGISTRY }}/${{ github.repository }}/android:latest - swift=docker-image://${{ env.REGISTRY }}/${{ github.repository }}/swift:latest - scala=docker-image://${{ env.REGISTRY }}/${{ github.repository }}/scala:latest - dart=docker-image://${{ env.REGISTRY }}/${{ github.repository }}/dart:latest - dotnet=docker-image://${{ env.REGISTRY }}/${{ github.repository }}/dotnet:latest - haskell=docker-image://${{ env.REGISTRY }}/${{ github.repository }}/haskell:latest - bazel=docker-image://${{ env.REGISTRY }}/${{ github.repository }}/bazel:latest