Merge branch 'stackhpc/2024.1' into cross-arch-builds-2024.1 #1039
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| --- | ||
| name: Build Kolla container images | ||
| on: | ||
| workflow_dispatch: | ||
| inputs: | ||
| regexes: | ||
| description: Space-separated list of regular expressions matching overcloud images to build | ||
| type: string | ||
| required: false | ||
| default: "" | ||
| overcloud: | ||
| description: Build overcloud images? | ||
| type: boolean | ||
| required: false | ||
| default: true | ||
| seed: | ||
| description: Build seed images? | ||
| type: boolean | ||
| required: false | ||
| default: false | ||
| rocky-linux-9: | ||
| description: Build Rocky Linux 9 images? | ||
| type: boolean | ||
| required: false | ||
| default: true | ||
| ubuntu-jammy: | ||
| description: Build Ubuntu Jammy 22.04 images? | ||
| type: boolean | ||
| required: false | ||
| default: true | ||
| push: | ||
| description: Whether to push images | ||
| type: boolean | ||
| required: false | ||
| default: true | ||
| push-dirty: | ||
| description: Push scanned images that have critical vulnerabilities? | ||
| type: boolean | ||
| required: false | ||
| default: false | ||
| env: | ||
| ANSIBLE_FORCE_COLOR: True | ||
| jobs: | ||
| generate-tag: | ||
| name: Generate container image tag | ||
| if: github.repository == 'stackhpc/stackhpc-kayobe-config' | ||
| runs-on: ubuntu-latest | ||
| permissions: {} | ||
| outputs: | ||
| datetime_tag: ${{ steps.datetime_tag.outputs.datetime_tag }} | ||
| matrix: ${{ steps.set-matrix.outputs.matrix }} | ||
| openstack_release: ${{ steps.openstack_release.outputs.openstack_release }} | ||
| steps: | ||
| - name: Checkout | ||
| uses: actions/checkout@v4 | ||
| - name: Determine OpenStack release | ||
| id: openstack_release | ||
| run: | | ||
| BRANCH=$(awk -F'=' '/defaultbranch/ {print $2}' .gitreview) | ||
| echo "openstack_release=${BRANCH}" | sed -E "s,(stable|unmaintained)/,," >> $GITHUB_OUTPUT | ||
| # Generate a tag to apply to all built container images. | ||
| # Without this, each kayobe * container image build command would use a different tag. | ||
| - name: Generate container datetime tag | ||
| id: datetime_tag | ||
| run: | | ||
| echo "datetime_tag=$(date +%Y%m%dT%H%M%S)" >> $GITHUB_OUTPUT | ||
| # Dynamically define job matrix. | ||
| # We need a separate matrix entry for each distribution, when the relevant input is true. | ||
| # https://stackoverflow.com/questions/65384420/how-do-i-make-a-github-action-matrix-element-conditional | ||
| # For now include only RL9 in aarch64 | ||
| - name: Generate build matrix | ||
| id: set-matrix | ||
| run: | | ||
| echo -n "matrix={\"include\": [" >> $GITHUB_OUTPUT | ||
| comma="" | ||
| if [[ '${{ inputs.rocky-linux-9 }}' == 'true' ]]; then | ||
| echo -n "$comma{\"distro\": \"rocky\", \"arch\": \"amd64\"}" >> $GITHUB_OUTPUT | ||
| comma=", " | ||
| echo -n "$comma{\"distro\": \"rocky\", \"arch\": \"aarch64\"}" >> $GITHUB_OUTPUT | ||
| fi | ||
| if [[ '${{ inputs.ubuntu-jammy }}' == 'true' ]]; then | ||
| echo -n "$comma{\"distro\": \"ubuntu\", \"arch\": \"amd64\"}" >> $GITHUB_OUTPUT | ||
| fi | ||
| echo "]}" >> $GITHUB_OUTPUT | ||
| - name: Display container datetime tag | ||
| run: | | ||
| echo "${{ steps.datetime_tag.outputs.datetime_tag }}" | ||
| container-image-build: | ||
| name: Build Kolla container images | ||
| if: github.repository == 'stackhpc/stackhpc-kayobe-config' | ||
| runs-on: arc-skc-container-image-builder-runner | ||
| timeout-minutes: 720 | ||
| permissions: {} | ||
| strategy: | ||
| fail-fast: false | ||
| matrix: ${{ fromJson(needs.generate-tag.outputs.matrix) }} | ||
| needs: | ||
| - generate-tag | ||
| steps: | ||
| - name: Install package dependencies | ||
| run: | | ||
| sudo apt update | ||
| sudo apt install -y build-essential git unzip nodejs python3-wheel python3-pip python3-venv curl jq wget | ||
| - name: Install gh | ||
| run: | | ||
| sudo mkdir -p -m 755 /etc/apt/keyrings && wget -qO- https://cli.github.com/packages/githubcli-archive-keyring.gpg | sudo tee /etc/apt/keyrings/githubcli-archive-keyring.gpg > /dev/null | ||
| sudo chmod go+r /etc/apt/keyrings/githubcli-archive-keyring.gpg | ||
| echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/githubcli-archive-keyring.gpg] https://cli.github.com/packages stable main" | sudo tee /etc/apt/sources.list.d/github-cli.list > /dev/null | ||
| sudo apt update | ||
| sudo apt install gh -y | ||
| - name: Checkout | ||
| uses: actions/checkout@v4 | ||
| with: | ||
| path: src/kayobe-config | ||
| - name: Make sure dockerd is running and test Docker | ||
| run: | | ||
| docker ps | ||
| - name: Install Trivy | ||
| run: | | ||
| curl -sfL https://raw.githubusercontent.com/aquasecurity/trivy/main/contrib/install.sh | sudo sh -s -- -b /usr/local/bin v0.49.0 | ||
| - name: Install yq | ||
| run: | | ||
| curl -sL https://github.com/mikefarah/yq/releases/download/v4.42.1/yq_linux_amd64.tar.gz | tar xz && sudo mv yq_linux_amd64 /usr/bin/yq | ||
| - name: Install Kayobe | ||
| run: | | ||
| mkdir -p venvs && | ||
| pushd venvs && | ||
| python3 -m venv kayobe && | ||
| source kayobe/bin/activate && | ||
| pip install -U pip && | ||
| pip install -r ../src/kayobe-config/requirements.txt | ||
| # Required for Pulp auth proxy deployment and Docker registry login. | ||
| # Normally installed during host configure. | ||
| - name: Install Docker Python SDK | ||
| run: | | ||
| sudo pip install docker 'requests<2.32.0' | ||
| - name: Get Kolla tag | ||
| id: write-kolla-tag | ||
| run: | | ||
| kolla_tag="${{ needs.generate-tag.outputs.openstack_release }}-${{ matrix.distro }}-${{ needs.generate-tag.outputs.datetime_tag }}" | ||
| if [[ "${{ matrix.distro }}" == 'rocky' ]]; then | ||
| kolla_tag="$kolla_tag-${{ matrix.arch }}" | ||
| fi | ||
| echo "kolla-tag=$kolla_tag" >> $GITHUB_OUTPUT | ||
| - name: Configure localhost as a seed | ||
| run: | | ||
| cat > src/kayobe-config/etc/kayobe/environments/ci-builder/inventory/hosts << EOF | ||
| # A 'seed' host used for building images. | ||
| # Use localhost for container image builds. | ||
| [seed] | ||
| localhost ansible_connection=local ansible_python_interpreter=/usr/bin/python3 | ||
| EOF | ||
| # See etc/kayobe/ansible/roles/pulp_auth_proxy/README.md for details. | ||
| # NOTE: We override pulp_auth_proxy_conf_path to a path shared by the | ||
| # runner and dind containers. | ||
| - name: Deploy an authenticating package repository mirror proxy | ||
| run: | | ||
| source venvs/kayobe/bin/activate && | ||
| source src/kayobe-config/kayobe-env --environment ci-builder && | ||
| kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/pulp-auth-proxy.yml -e pulp_auth_proxy_conf_path=/home/runner/_work/pulp_proxy | ||
| env: | ||
| KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }} | ||
| - name: Create build logs output directory | ||
| run: mkdir image-build-logs | ||
| - name: Build kolla overcloud images | ||
| id: build_overcloud_images | ||
| continue-on-error: true | ||
| run: | | ||
| args="${{ inputs.regexes }}" | ||
| if [[ "${{ matrix.arch }}" = 'aarch64' ]]; then | ||
| args="$args -e kolla_base_arch=${{ matrix.arch }}" | ||
| fi | ||
| args="$args -e kolla_base_distro=${{ matrix.distro }}" | ||
| args="$args -e kolla_tag=${{ steps.write-kolla-tag.outputs.kolla-tag }}" | ||
| args="$args -e stackhpc_repo_mirror_auth_proxy_enabled=true" | ||
| source venvs/kayobe/bin/activate && | ||
| source src/kayobe-config/kayobe-env --environment ci-builder && | ||
| kayobe overcloud container image build $args | ||
| env: | ||
| KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }} | ||
| if: inputs.overcloud | ||
| - name: Copy overcloud container image build logs to output directory | ||
| run: sudo mv /var/log/kolla-build.log image-build-logs/kolla-build-overcloud.log | ||
| if: inputs.overcloud | ||
| - name: Copy build configs to output directory | ||
| run: sudo cp -rnL /opt/kayobe/etc/kolla/* image-build-logs/ | ||
| if: inputs.overcloud | ||
| - name: Get Kolla tag for seed images | ||
| id: write-kolla-tag-seed | ||
| run: | | ||
| kolla_tag_seed="${{ needs.generate-tag.outputs.openstack_release }}-${{ matrix.distro }}-${{ needs.generate-tag.outputs.datetime_tag }}" | ||
| echo "kolla-tag-seed=$kolla_tag_seed" >> $GITHUB_OUTPUT | ||
| if: inputs.seed | ||
| - name: Build kolla seed images | ||
| id: build_seed_images | ||
| continue-on-error: true | ||
| run: | | ||
| args="-e kolla_base_distro=${{ matrix.distro }}" | ||
| args="$args -e kolla_tag=${{ steps.write-kolla-tag-seed.outputs.kolla-tag-seed }}" | ||
| args="$args -e stackhpc_repo_mirror_auth_proxy_enabled=true" | ||
| source venvs/kayobe/bin/activate && | ||
| source src/kayobe-config/kayobe-env --environment ci-builder && | ||
| kayobe seed container image build $args | ||
| env: | ||
| KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }} | ||
| if: inputs.seed && matrix.arch == 'amd64' | ||
| - name: Copy seed container image build logs to output directory | ||
| run: sudo mv /var/log/kolla-build.log image-build-logs/kolla-build-seed.log | ||
| if: inputs.seed | ||
| - name: Get built container images | ||
| run: docker image ls --filter "reference=ark.stackhpc.com/stackhpc-dev/*:${{ steps.write-kolla-tag.outputs.kolla-tag }}*" > ${{ matrix.distro }}-container-images | ||
| - name: Fail if no images have been built | ||
| run: if [ $(wc -l < ${{ matrix.distro }}-container-images) -le 1 ]; then exit 1; fi | ||
| - name: Scan built container images | ||
| run: src/kayobe-config/tools/scan-images.sh ${{ matrix.distro }} ${{ steps.write-kolla-tag.outputs.kolla-tag }} | ||
| - name: Move image scan logs to output artifact | ||
| run: mv image-scan-output image-build-logs/image-scan-output | ||
| - name: Fail if no images have passed scanning | ||
| run: if [ $(wc -l < image-build-logs/image-scan-output/critical-images.txt) -gt 0 ]; then exit 1; fi | ||
| if: ${{ !inputs.push-dirty }} | ||
| - name: Copy clean images to push-attempt-images list | ||
| run: cp image-build-logs/image-scan-output/clean-images.txt image-build-logs/push-attempt-images.txt | ||
| if: inputs.push | ||
| # NOTE(seunghun1ee): This always appends dirty images with CVEs severity lower than critical. | ||
| # This should be reverted when it's decided to filter high level CVEs as well. | ||
| - name: Append dirty images to push list | ||
| run: | | ||
| cat image-build-logs/image-scan-output/dirty-images.txt >> image-build-logs/push-attempt-images.txt | ||
| if: ${{ inputs.push }} | ||
| - name: Append images with critical vulnerabilities to push list | ||
| run: | | ||
| cat image-build-logs/image-scan-output/critical-images.txt >> image-build-logs/push-attempt-images.txt | ||
| if: ${{ inputs.push && inputs.push-dirty }} | ||
| - name: Push images | ||
| run: | | ||
| touch image-build-logs/push-failed-images.txt | ||
| source venvs/kayobe/bin/activate && | ||
| source src/kayobe-config/kayobe-env --environment ci-builder && | ||
| kayobe playbook run ${KAYOBE_CONFIG_PATH}/ansible/docker-registry-login.yml && | ||
| while read -r image; do | ||
| # Retries! | ||
| for i in {1..5}; do | ||
| if docker push $image; then | ||
| echo "Pushed $image" | ||
| break | ||
| elif [ $i -eq 5 ] ; then | ||
| echo "Failed to push $image" | ||
| echo $image >> image-build-logs/push-failed-images.txt | ||
| else | ||
| echo "Failed on retry $i" | ||
| sleep 5 | ||
| fi; | ||
| done | ||
| done < image-build-logs/push-attempt-images.txt | ||
| shell: bash | ||
| env: | ||
| KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }} | ||
| if: inputs.push | ||
| - name: Upload output artifact | ||
| uses: actions/upload-artifact@v4 | ||
| with: | ||
| name: ${{ matrix.distro }}-${{ matrix.arch }}-logs | ||
| path: image-build-logs | ||
| retention-days: 7 | ||
| if: ${{ !cancelled() }} | ||
| - name: Fail when images failed to build | ||
| run: echo "An image build failed. Check the workflow artifact for build logs" && exit 1 | ||
| if: ${{ steps.build_overcloud_images.outcome == 'failure' || steps.build_seed_images.outcome == 'failure' }} | ||
| - name: Fail when images failed to push | ||
| run: if [ $(wc -l < image-build-logs/push-failed-images.txt) -gt 0 ]; then cat image-build-logs/push-failed-images.txt && exit 1; fi | ||
| if: ${{ !cancelled() }} | ||
| # NOTE(seunghun1ee): Currently we want to mark the job fail only when critical CVEs are detected. | ||
| # This can be used again instead of "Fail when critical vulnerabilities are found" when it's | ||
| # decided to fail the job on detecting high CVEs as well. | ||
| # - name: Fail when images failed scanning | ||
| # run: if [ $(wc -l < image-build-logs/image-scan-output/dirty-images.txt) -gt 0 ]; then cat image-build-logs/image-scan-output/dirty-images.txt && exit 1; fi | ||
| # if: ${{ !inputs.push-dirty && !cancelled() }} | ||
| - name: Fail when critical vulnerabilities are found | ||
| run: if [ $(wc -l < image-build-logs/image-scan-output/critical-images.txt) -gt 0 ]; then cat image-build-logs/image-scan-output/critical-images.txt && exit 1; fi | ||
| if: ${{ !inputs.push-dirty && !cancelled() }} | ||
| create-manifests: | ||
| # Only for Rocky Linux for now | ||
| name: Create Docker Manifests | ||
| if: github.repository == 'stackhpc/stackhpc-kayobe-config' | ||
| if: inputs.push | ||
| runs-on: arc-skc-container-image-builder-runner | ||
| permissions: {} | ||
| needs: | ||
| - container-image-build | ||
| steps: | ||
| - name: Install package dependencies | ||
| run: | | ||
| sudo apt update | ||
| sudo apt install -y git unzip python3-wheel python3-pip python3-venv curl jq wget | ||
| - name: Install gh | ||
| run: | | ||
| sudo mkdir -p -m 755 /etc/apt/keyrings && wget -qO- https://cli.github.com/packages/githubcli-archive-keyring.gpg | sudo tee /etc/apt/keyrings/githubcli-archive-keyring.gpg > /dev/null | ||
| sudo chmod go+r /etc/apt/keyrings/githubcli-archive-keyring.gpg | ||
| echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/githubcli-archive-keyring.gpg] https://cli.github.com/packages stable main" | sudo tee /etc/apt/sources.list.d/github-cli.list > /dev/null | ||
| sudo apt update | ||
| sudo apt install gh -y | ||
| - name: Checkout Kayobe Config | ||
| uses: actions/checkout@v4 | ||
| with: | ||
| path: src/kayobe-config | ||
| - name: Install Kayobe | ||
| run: | | ||
| mkdir -p venvs && | ||
| pushd venvs && | ||
| python3 -m venv kayobe && | ||
| source kayobe/bin/activate && | ||
| pip install -U pip && | ||
| pip install -r ../src/kayobe-config/requirements.txt | ||
| # Required for Pulp auth proxy deployment and Docker registry login. | ||
| # Normally installed during host configure. | ||
| - name: Install Docker Python SDK | ||
| run: | | ||
| sudo pip install docker | ||
| - name: Download artifacts | ||
| uses: actions/download-artifact@v4 | ||
| - name: Combine pushed images lists | ||
| run: | | ||
| find . -name 'push-attempt-images.txt' -exec cat {} + > all-pushed-images.txt | ||
| - name: Log in to Docker registry | ||
| run: | | ||
| source venvs/kayobe/bin/activate && | ||
| source src/kayobe-config/kayobe-env --environment ci-builder && | ||
| kayobe playbook run ${KAYOBE_CONFIG_PATH}/ansible/docker-registry-login.yml | ||
| env: | ||
| KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }} | ||
| - name: Create and Push Docker Manifests | ||
| run: | | ||
| set -ex | ||
| mkdir -p logs | ||
| images=$(cat all-pushed-images.txt | sort | uniq) | ||
| # Filter out Ubuntu images | ||
| manifest_images=$(echo "$images" | grep 'rocky' | sed -E 's/-(amd64|aarch64)$//' | sort | uniq) | ||
| for base_image in $manifest_images; do | ||
| arch_images="" | ||
| for arch in amd64 aarch64; do | ||
| arch_image="${base_image}-${arch}" | ||
| # Check if the image exists in the registry | ||
| if docker manifest inspect "$arch_image" > /dev/null 2>&1; then | ||
| arch_images="$arch_images $arch_image" | ||
| fi | ||
| done | ||
| if [ -n "$arch_images" ]; then | ||
| echo "Creating manifest for $base_image with images:$arch_images" | tee -a logs/manifest-creation.log | ||
| docker manifest create "$base_image" $arch_images | tee -a logs/manifest-creation.log | ||
| docker manifest push "$base_image" | tee -a logs/manifest-creation.log | ||
| else | ||
| echo "No images found for $base_image, skipping." | tee -a logs/manifest-creation.log | ||
| fi | ||
| done | ||
| - name: Upload manifest logs | ||
| uses: actions/upload-artifact@v4 | ||
| with: | ||
| name: manifest-logs | ||
| path: | | ||
| all-pushed-images.txt | ||
| logs/manifest-creation.log | ||
| # NOTE(mgoddard): Trigger another CI workflow in the | ||
| # stackhpc-release-train repository. | ||
| - name: Trigger container image repository sync | ||
| run: | | ||
| filter='${{ inputs.regexes }}' | ||
| if [[ -n $filter ]] && [[ ${{ inputs.seed }} == 'true' ]]; then | ||
| filter="$filter bifrost" | ||
| fi | ||
| gh workflow run \ | ||
| container-sync.yml \ | ||
| --repo stackhpc/stackhpc-release-train \ | ||
| --ref main \ | ||
| -f filter="$filter" \ | ||
| -f sync-old-images=false | ||
| env: | ||
| GITHUB_TOKEN: ${{ secrets.STACKHPC_RELEASE_TRAIN_TOKEN }} | ||
| if: ${{ github.repository == 'stackhpc/stackhpc-kayobe-config' && inputs.push && !cancelled() }} | ||
| - name: Display link to container image repository sync workflows | ||
| run: | | ||
| echo "::notice Container image repository sync workflows: https://github.com/stackhpc/stackhpc-release-train/actions/workflows/container-sync.yml" | ||
| if: ${{ github.repository == 'stackhpc/stackhpc-kayobe-config' && inputs.push && !cancelled() }} | ||
