stackhpc-container-image-build.yml

---
name: Build Kolla container images
on:
  workflow_dispatch:
    inputs:
      regexes:
        description: Space-separated list of regular expressions matching overcloud images to build
        type: string
        required: false
        default: ""
      overcloud:
        description: Build overcloud images?
        type: boolean
        required: false
        default: true
      seed:
        description: Build seed images?
        type: boolean
        required: false
        default: false
      rocky-linux-9:
        description: Build Rocky Linux 9 images?
        type: boolean
        required: false
        default: true
      ubuntu-jammy:
        description: Build Ubuntu Jammy 22.04 images?
        type: boolean
        required: false
        default: true
      push:
        description: Whether to push images
        type: boolean
        required: false
        default: true
      push-dirty:
        description: Push scanned images that have critical vulnerabilities?
        type: boolean
        required: false
        default: false

env:
  ANSIBLE_FORCE_COLOR: True
jobs:
  generate-tag:
    name: Generate container image tag
    if: github.repository == 'stackhpc/stackhpc-kayobe-config'
    runs-on: ubuntu-latest
    permissions: {}
    outputs:
      datetime_tag: ${{ steps.datetime_tag.outputs.datetime_tag }}
      matrix: ${{ steps.set-matrix.outputs.matrix }}
      openstack_release: ${{ steps.openstack_release.outputs.openstack_release }}
    steps:
      - name: Checkout
        uses: actions/checkout@v4

      - name: Determine OpenStack release
        id: openstack_release
        run: |
          BRANCH=$(awk -F'=' '/defaultbranch/ {print $2}' .gitreview)
          echo "openstack_release=${BRANCH}" | sed -E "s,(stable|unmaintained)/,," >> $GITHUB_OUTPUT

      # Generate a tag to apply to all built container images.
      # Without this, each kayobe * container image build command would use a different tag.
      - name: Generate container datetime tag
        id: datetime_tag
        run: |
          echo "datetime_tag=$(date +%Y%m%dT%H%M%S)" >> $GITHUB_OUTPUT

      # Dynamically define job matrix.
      # We need a separate matrix entry for each distribution, when the relevant input is true.
      # https://stackoverflow.com/questions/65384420/how-do-i-make-a-github-action-matrix-element-conditional
      # For now include only RL9 in aarch64
      - name: Generate build matrix
        id: set-matrix
        run: |
          echo -n "matrix={\"include\": [" >> $GITHUB_OUTPUT
          comma=""
          if [[ '${{ inputs.rocky-linux-9 }}' == 'true' ]]; then
            echo -n "$comma{\"distro\": \"rocky\", \"arch\": \"amd64\"}" >> $GITHUB_OUTPUT
            comma=", "
            echo -n "$comma{\"distro\": \"rocky\", \"arch\": \"aarch64\"}" >> $GITHUB_OUTPUT
          fi
          if [[ '${{ inputs.ubuntu-jammy }}' == 'true' ]]; then
            echo -n "$comma{\"distro\": \"ubuntu\", \"arch\": \"amd64\"}" >> $GITHUB_OUTPUT
          fi
          echo "]}" >> $GITHUB_OUTPUT

      - name: Display container datetime tag
        run: |
          echo "${{ steps.datetime_tag.outputs.datetime_tag }}"

  container-image-build:
    name: Build Kolla container images
    if: github.repository == 'stackhpc/stackhpc-kayobe-config'
    runs-on: arc-skc-container-image-builder-runner
    timeout-minutes: 720
    permissions: {}
    strategy:
      fail-fast: false
      matrix: ${{ fromJson(needs.generate-tag.outputs.matrix) }}
    needs:
      - generate-tag
    steps:
      - name: Install package dependencies
        run: |
          sudo apt update
          sudo apt install -y build-essential git unzip nodejs python3-wheel python3-pip python3-venv curl jq wget

      - name: Install gh
        run: |
          sudo mkdir -p -m 755 /etc/apt/keyrings && wget -qO- https://cli.github.com/packages/githubcli-archive-keyring.gpg | sudo tee /etc/apt/keyrings/githubcli-archive-keyring.gpg > /dev/null
          sudo chmod go+r /etc/apt/keyrings/githubcli-archive-keyring.gpg
          echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/githubcli-archive-keyring.gpg] https://cli.github.com/packages stable main" | sudo tee /etc/apt/sources.list.d/github-cli.list > /dev/null
          sudo apt update
          sudo apt install gh -y

      - name: Checkout
        uses: actions/checkout@v4
        with:
          path: src/kayobe-config

      - name: Make sure dockerd is running and test Docker
        run: |
          docker ps

      - name: Install Trivy
        run: |
          curl -sfL https://raw.githubusercontent.com/aquasecurity/trivy/main/contrib/install.sh | sudo sh -s -- -b /usr/local/bin v0.49.0

      - name: Install yq
        run: |
          curl -sL https://github.com/mikefarah/yq/releases/download/v4.42.1/yq_linux_amd64.tar.gz | tar xz && sudo mv yq_linux_amd64 /usr/bin/yq

      - name: Install Kayobe
        run: |
          mkdir -p venvs &&
          pushd venvs &&
          python3 -m venv kayobe &&
          source kayobe/bin/activate &&
          pip install -U pip &&
          pip install -r ../src/kayobe-config/requirements.txt

      # Required for Pulp auth proxy deployment and Docker registry login.
      # Normally installed during host configure.
      - name: Install Docker Python SDK
        run: |
          sudo pip install docker 'requests<2.32.0'

      - name: Get Kolla tag
        id: write-kolla-tag
        run: |
          kolla_tag="${{ needs.generate-tag.outputs.openstack_release }}-${{ matrix.distro }}-${{ needs.generate-tag.outputs.datetime_tag }}"
          if [[ "${{ matrix.distro }}" == 'rocky' ]]; then
            kolla_tag="$kolla_tag-${{ matrix.arch }}"
          fi
          echo "kolla-tag=$kolla_tag" >> $GITHUB_OUTPUT

      - name: Configure localhost as a seed
        run: |
          cat > src/kayobe-config/etc/kayobe/environments/ci-builder/inventory/hosts << EOF
          # A 'seed' host used for building images.
          # Use localhost for container image builds.
          [seed]
          localhost ansible_connection=local ansible_python_interpreter=/usr/bin/python3
          EOF

      # See etc/kayobe/ansible/roles/pulp_auth_proxy/README.md for details.
      # NOTE: We override pulp_auth_proxy_conf_path to a path shared by the
      # runner and dind containers.
      - name: Deploy an authenticating package repository mirror proxy
        run: |
          source venvs/kayobe/bin/activate &&
          source src/kayobe-config/kayobe-env --environment ci-builder &&
          kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/pulp-auth-proxy.yml -e pulp_auth_proxy_conf_path=/home/runner/_work/pulp_proxy
        env:
          KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }}

      - name: Create build logs output directory
        run: mkdir image-build-logs

      - name: Build kolla overcloud images
        id: build_overcloud_images
        continue-on-error: true
        run: |
          args="${{ inputs.regexes }}"
          if [[ "${{ matrix.arch }}" = 'aarch64' ]]; then
            args="$args -e kolla_base_arch=${{ matrix.arch }}"
          fi
          args="$args -e kolla_base_distro=${{ matrix.distro }}"
          args="$args -e kolla_tag=${{ steps.write-kolla-tag.outputs.kolla-tag }}"
          args="$args -e stackhpc_repo_mirror_auth_proxy_enabled=true"
          source venvs/kayobe/bin/activate &&
          source src/kayobe-config/kayobe-env --environment ci-builder &&
          kayobe overcloud container image build $args
        env:
          KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }}
        if: inputs.overcloud

      - name: Copy overcloud container image build logs to output directory
        run: sudo mv /var/log/kolla-build.log image-build-logs/kolla-build-overcloud.log
        if: inputs.overcloud

      - name: Copy build configs to output directory
        run: sudo cp -rnL /opt/kayobe/etc/kolla/* image-build-logs/
        if: inputs.overcloud

      - name: Get Kolla tag for seed images
        id: write-kolla-tag-seed
        run: |
          kolla_tag_seed="${{ needs.generate-tag.outputs.openstack_release }}-${{ matrix.distro }}-${{ needs.generate-tag.outputs.datetime_tag }}"
          echo "kolla-tag-seed=$kolla_tag_seed" >> $GITHUB_OUTPUT
        if: inputs.seed

      - name: Build kolla seed images
        id: build_seed_images
        continue-on-error: true
        run: |
          args="-e kolla_base_distro=${{ matrix.distro }}"
          args="$args -e kolla_tag=${{ steps.write-kolla-tag-seed.outputs.kolla-tag-seed }}"
          args="$args -e stackhpc_repo_mirror_auth_proxy_enabled=true"
          source venvs/kayobe/bin/activate &&
          source src/kayobe-config/kayobe-env --environment ci-builder &&
          kayobe seed container image build $args
        env:
          KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }}
        if: inputs.seed && matrix.arch == 'amd64'

      - name: Copy seed container image build logs to output directory
        run: sudo mv /var/log/kolla-build.log image-build-logs/kolla-build-seed.log
        if: inputs.seed

      - name: Get built container images
        run: docker image ls --filter "reference=ark.stackhpc.com/stackhpc-dev/*:${{ steps.write-kolla-tag.outputs.kolla-tag }}*" > ${{ matrix.distro }}-container-images

      - name: Fail if no images have been built
        run: if [ $(wc -l < ${{ matrix.distro }}-container-images) -le 1 ]; then exit 1; fi

      - name: Scan built container images
        run: src/kayobe-config/tools/scan-images.sh ${{ matrix.distro }} ${{ steps.write-kolla-tag.outputs.kolla-tag }}

      - name: Move image scan logs to output artifact
        run: mv image-scan-output image-build-logs/image-scan-output

      - name: Fail if no images have passed scanning
        run: if [ $(wc -l < image-build-logs/image-scan-output/critical-images.txt) -gt 0 ]; then exit 1; fi
        if: ${{ !inputs.push-dirty }}

      - name: Copy clean images to push-attempt-images list
        run: cp image-build-logs/image-scan-output/clean-images.txt image-build-logs/push-attempt-images.txt
        if: inputs.push

      # NOTE(seunghun1ee): This always appends dirty images with CVEs severity lower than critical.
      # This should be reverted when it's decided to filter high level CVEs as well.
      - name: Append dirty images to push list
        run: |
          cat image-build-logs/image-scan-output/dirty-images.txt >> image-build-logs/push-attempt-images.txt
        if: ${{ inputs.push }}

      - name: Append images with critical vulnerabilities to push list
        run: |
          cat image-build-logs/image-scan-output/critical-images.txt >> image-build-logs/push-attempt-images.txt
        if: ${{ inputs.push && inputs.push-dirty }}

      - name: Push images
        run: |
          touch image-build-logs/push-failed-images.txt
          source venvs/kayobe/bin/activate &&
          source src/kayobe-config/kayobe-env --environment ci-builder &&
          kayobe playbook run ${KAYOBE_CONFIG_PATH}/ansible/docker-registry-login.yml &&

          while read -r image; do
            # Retries!
            for i in {1..5}; do
              if docker push $image; then
                echo "Pushed $image"
                break
              elif [ $i -eq 5 ] ; then
                echo "Failed to push $image"
                echo $image >> image-build-logs/push-failed-images.txt
              else
                echo "Failed on retry $i"
                sleep 5
              fi;
            done
          done < image-build-logs/push-attempt-images.txt
        shell: bash
        env:
          KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }}
        if: inputs.push

      - name: Upload output artifact
        uses: actions/upload-artifact@v4
        with:
          name: ${{ matrix.distro }}-${{ matrix.arch }}-logs
          path: image-build-logs
          retention-days: 7
        if: ${{ !cancelled() }}

      - name: Fail when images failed to build
        run: echo "An image build failed. Check the workflow artifact for build logs" && exit 1
        if: ${{ steps.build_overcloud_images.outcome == 'failure' || steps.build_seed_images.outcome == 'failure' }}

      - name: Fail when images failed to push
        run: if [ $(wc -l < image-build-logs/push-failed-images.txt) -gt 0 ]; then cat image-build-logs/push-failed-images.txt && exit 1; fi
        if: ${{ !cancelled() }}

      # NOTE(seunghun1ee): Currently we want to mark the job fail only when critical CVEs are detected.
      # This can be used again instead of "Fail when critical vulnerabilities are found" when it's
      # decided to fail the job on detecting high CVEs as well.
      # - name: Fail when images failed scanning
      #   run: if [ $(wc -l < image-build-logs/image-scan-output/dirty-images.txt) -gt 0 ]; then cat image-build-logs/image-scan-output/dirty-images.txt && exit 1; fi
      #   if: ${{ !inputs.push-dirty && !cancelled() }}

      - name: Fail when critical vulnerabilities are found
        run: if [ $(wc -l < image-build-logs/image-scan-output/critical-images.txt) -gt 0 ]; then cat image-build-logs/image-scan-output/critical-images.txt && exit 1; fi
        if: ${{ !inputs.push-dirty && !cancelled() }}

  create-manifests:
    # Only for Rocky Linux for now
    name: Create Docker Manifests
    if: github.repository == 'stackhpc/stackhpc-kayobe-config'
    if: inputs.push
    runs-on: arc-skc-container-image-builder-runner
    permissions: {}
    needs:
      - container-image-build
    steps:
      - name: Install package dependencies
        run: |
          sudo apt update
          sudo apt install -y git unzip python3-wheel python3-pip python3-venv curl jq wget

      - name: Install gh
        run: |
          sudo mkdir -p -m 755 /etc/apt/keyrings && wget -qO- https://cli.github.com/packages/githubcli-archive-keyring.gpg | sudo tee /etc/apt/keyrings/githubcli-archive-keyring.gpg > /dev/null
          sudo chmod go+r /etc/apt/keyrings/githubcli-archive-keyring.gpg
          echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/githubcli-archive-keyring.gpg] https://cli.github.com/packages stable main" | sudo tee /etc/apt/sources.list.d/github-cli.list > /dev/null
          sudo apt update
          sudo apt install gh -y

      - name: Checkout Kayobe Config
        uses: actions/checkout@v4
        with:
          path: src/kayobe-config

      - name: Install Kayobe
        run: |
          mkdir -p venvs &&
          pushd venvs &&
          python3 -m venv kayobe &&
          source kayobe/bin/activate &&
          pip install -U pip &&
          pip install -r ../src/kayobe-config/requirements.txt

      # Required for Pulp auth proxy deployment and Docker registry login.
      # Normally installed during host configure.
      - name: Install Docker Python SDK
        run: |
          sudo pip install docker

      - name: Download artifacts
        uses: actions/download-artifact@v4

      - name: Combine pushed images lists
        run: |
          find . -name 'push-attempt-images.txt' -exec cat {} + > all-pushed-images.txt

      - name: Log in to Docker registry
        run: |
          source venvs/kayobe/bin/activate &&
          source src/kayobe-config/kayobe-env --environment ci-builder &&
          kayobe playbook run ${KAYOBE_CONFIG_PATH}/ansible/docker-registry-login.yml
        env:
          KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }}

      - name: Create and Push Docker Manifests
        run: |
          set -ex
          mkdir -p logs
          images=$(cat all-pushed-images.txt | sort | uniq)
          # Filter out Ubuntu images
          manifest_images=$(echo "$images" | grep 'rocky' | sed -E 's/-(amd64|aarch64)$//' | sort | uniq)
          for base_image in $manifest_images; do
            arch_images=""
            for arch in amd64 aarch64; do
              arch_image="${base_image}-${arch}"
              # Check if the image exists in the registry
              if docker manifest inspect "$arch_image" > /dev/null 2>&1; then
                arch_images="$arch_images $arch_image"
              fi
            done
            if [ -n "$arch_images" ]; then
              echo "Creating manifest for $base_image with images:$arch_images" | tee -a logs/manifest-creation.log
              docker manifest create "$base_image" $arch_images | tee -a logs/manifest-creation.log
              docker manifest push "$base_image" | tee -a logs/manifest-creation.log
            else
              echo "No images found for $base_image, skipping." | tee -a logs/manifest-creation.log
            fi
          done

      - name: Upload manifest logs
        uses: actions/upload-artifact@v4
        with:
          name: manifest-logs
          path: |
            all-pushed-images.txt
            logs/manifest-creation.log

      # NOTE(mgoddard): Trigger another CI workflow in the
      # stackhpc-release-train repository.
      - name: Trigger container image repository sync
        run: |
          filter='${{ inputs.regexes }}'
          if [[ -n $filter ]] && [[ ${{ inputs.seed }} == 'true' ]]; then
            filter="$filter bifrost"
          fi
          gh workflow run \
          container-sync.yml \
          --repo stackhpc/stackhpc-release-train \
          --ref main \
          -f filter="$filter" \
          -f sync-old-images=false
        env:
          GITHUB_TOKEN: ${{ secrets.STACKHPC_RELEASE_TRAIN_TOKEN }}
        if: ${{ github.repository == 'stackhpc/stackhpc-kayobe-config' && inputs.push && !cancelled() }}

      - name: Display link to container image repository sync workflows
        run: |
          echo "::notice Container image repository sync workflows: https://github.com/stackhpc/stackhpc-release-train/actions/workflows/container-sync.yml"
        if: ${{ github.repository == 'stackhpc/stackhpc-kayobe-config' && inputs.push && !cancelled() }}