overcloud-host-image-build.yml

---
name: Build overcloud host images
on:
  workflow_dispatch:
    inputs:
      centos:
        description: Build CentOS Stream 8
        type: boolean
        default: true
      rocky8:
        description: Build Rocky Linux 8
        type: boolean
        default: true
      rocky9:
        description: Build Rocky Linux 9
        type: boolean
        default: true
      ubuntu-focal:
        description: Build Ubuntu 20.04 Focal
        type: boolean
        default: true
      ubuntu-jammy:
        description: Build Ubuntu 22.04 Jammy
        type: boolean
        default: true
    secrets:
      KAYOBE_VAULT_PASSWORD:
        required: true
      CLOUDS_YAML:
        required: true
      OS_APPLICATION_CREDENTIAL_ID:
        required: true
      OS_APPLICATION_CREDENTIAL_SECRET:
        required: true

env:
  ANSIBLE_FORCE_COLOR: True
  KAYOBE_ENVIRONMENT: ci-builder
  KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }}
jobs:
  overcloud-host-image-build:
    name: Build overcloud host images
    if: github.repository == 'stackhpc/stackhpc-kayobe-config'
    runs-on: arc-skc-host-image-builder-runner
    permissions: {}
    steps:
      - name: Install Package
        uses: ConorMacBride/install-package@main
        with:
          apt: git unzip nodejs python3-pip python3-venv openssh-server openssh-client jq

      - name: Start the SSH service
        run: |
          sudo /etc/init.d/ssh start
        continue-on-error: true

      - name: Checkout
        uses: actions/checkout@v4
        with:
          path: src/kayobe-config

      - name: Output image tag of the builder
        id: builder_image_tag
        run: |
          echo image_tag=$(grep stackhpc_rocky_9_overcloud_host_image_version: etc/kayobe/pulp-host-image-versions.yml | awk '{print $2}') >> $GITHUB_OUTPUT

      - name: Determine OpenStack release
        id: openstack_release
        run: |
          BRANCH=$(awk -F'=' '/defaultbranch/ {print $2}' src/kayobe-config/.gitreview)
          echo "openstack_release=${BRANCH}" | sed -E "s,(stable|unmaintained)/,," >> $GITHUB_OUTPUT

      # Generate a tag to apply to all built overcloud host images.
      - name: Generate overcloud host image tag
        id: host_image_tag
        run: |
          echo "host_image_tag=$(date +${{ steps.openstack_release.outputs.openstack_release }}-%Y%m%dT%H%M%S)" >> $GITHUB_OUTPUT

      - name: Display overcloud host image tag
        run: |
          echo "${{ steps.host_image_tag.outputs.host_image_tag }}"

      - name: Clone StackHPC Kayobe repository
        uses: actions/checkout@v4
        with:
          repository: stackhpc/kayobe
          ref: refs/heads/stackhpc/${{ steps.openstack_release.outputs.openstack_release }}
          path: src/kayobe

      - name: Install Kayobe
        run: |
          mkdir -p venvs &&
          pushd venvs &&
          python3 -m venv kayobe &&
          source kayobe/bin/activate &&
          pip install -U pip &&
          pip install ../src/kayobe

      - name: Install terraform
        uses: hashicorp/setup-terraform@v2

      - name: Initialise terraform
        run: terraform init
        working-directory: ${{ github.workspace }}/src/kayobe-config/terraform/host-image-builder

      - name: Generate SSH keypair
        run: ssh-keygen -f id_rsa -N ''
        working-directory: ${{ github.workspace }}/src/kayobe-config/terraform/host-image-builder

      - name: Generate clouds.yaml
        run: |
          cat << EOF > clouds.yaml
          ${{ secrets.CLOUDS_YAML }}
          EOF
        working-directory: ${{ github.workspace }}/src/kayobe-config/terraform/host-image-builder

      - name: Add tags to tfvars file
        run: |
          echo -e '\nhost_image_builder_tags = ${{ env.VM_TAGS }}' >> terraform.tfvars
        working-directory: ${{ github.workspace }}/src/kayobe-config/terraform/host-image-builder
        env:
          VM_TAGS: '["skc-host-image-build", "ref=${{ github.ref_name }}"]'

      - name: Terraform Plan
        run: terraform plan
        working-directory: ${{ github.workspace }}/src/kayobe-config/terraform/host-image-builder
        env:
          OS_CLOUD: "openstack"
          OS_APPLICATION_CREDENTIAL_ID: ${{ secrets.OS_APPLICATION_CREDENTIAL_ID }}
          OS_APPLICATION_CREDENTIAL_SECRET: ${{ secrets.OS_APPLICATION_CREDENTIAL_SECRET }}

      - name: Terraform Apply
        run: |
          for attempt in $(seq 5); do
              if terraform apply -auto-approve; then
                  echo "Created infrastructure on attempt $attempt"
                  exit 0
              fi
              echo "Failed to create infrastructure on attempt $attempt"
              sleep 10
              terraform destroy -auto-approve
              sleep 60
          done
          echo "Failed to create infrastructure after $attempt attempts"
          exit 1
        working-directory: ${{ github.workspace }}/src/kayobe-config/terraform/host-image-builder
        env:
          OS_CLOUD: "openstack"
          OS_APPLICATION_CREDENTIAL_ID: ${{ secrets.OS_APPLICATION_CREDENTIAL_ID }}
          OS_APPLICATION_CREDENTIAL_SECRET: ${{ secrets.OS_APPLICATION_CREDENTIAL_SECRET }}

      - name: Get Terraform outputs
        id: tf_outputs
        run: |
          terraform output -json
        working-directory: ${{ github.workspace }}/src/kayobe-config/terraform/host-image-builder

      - name: Write Terraform outputs
        run: |
          cat << EOF > src/kayobe-config/etc/kayobe/environments/ci-builder/tf-outputs.yml
          ${{ steps.tf_outputs.outputs.stdout }}
          EOF

      - name: Write Terraform network config
        run: |
          cat << EOF > src/kayobe-config/etc/kayobe/environments/ci-builder/tf-network-allocation.yml
          ---
          aio_ips:
            builder: "{{ access_ip_v4.value }}"
          EOF

      - name: Write Terraform network interface config
        run: |
          mkdir -p src/kayobe-config/etc/kayobe/environments/$KAYOBE_ENVIRONMENT/inventory/group_vars/seed
          rm -f src/kayobe-config/etc/kayobe/environments/$KAYOBE_ENVIRONMENT/inventory/group_vars/seed/network-interfaces
          cat << EOF > src/kayobe-config/etc/kayobe/environments/$KAYOBE_ENVIRONMENT/inventory/group_vars/seed/network-interfaces
          admin_interface: "{{ access_interface.value }}"
          aio_interface: "{{ access_interface.value }}"
          EOF

      - name: Manage SSH keys
        run: |
          mkdir -p ~/.ssh
          touch ~/.ssh/authorized_keys
          cat src/kayobe-config/terraform/host-image-builder/id_rsa.pub >> ~/.ssh/authorized_keys
          cp src/kayobe-config/terraform/host-image-builder/id_rsa* ~/.ssh/

      - name: Update inventory
        run: |
          cat > src/kayobe-config/etc/kayobe/environments/ci-builder/inventory/hosts << EOF
          [seed]
          builder
          EOF

      - name: Bootstrap the control host
        run: |
          source venvs/kayobe/bin/activate &&
          source src/kayobe-config/kayobe-env --environment ci-builder &&
          kayobe control host bootstrap

      - name: Configure the seed host (Builder VM)
        continue-on-error: true
        run: |
          source venvs/kayobe/bin/activate &&
          source src/kayobe-config/kayobe-env --environment ci-builder &&
          kayobe seed host configure -e seed_bootstrap_user=rocky --skip-tags network

      - name: Install dependencies
        continue-on-error: true
        run: |
          source venvs/kayobe/bin/activate &&
          source src/kayobe-config/kayobe-env --environment ci-builder &&
          kayobe seed host command run \
          --command "sudo dnf config-manager --set-enabled crb && sudo dnf -y install epel-release && sudo dnf -y install zstd debootstrap kpartx cloud-init" --show-output
        env:
          KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }}

      - name: Create bifrost_httpboot Docker volume
        continue-on-error: true
        run: |
          source venvs/kayobe/bin/activate &&
          source src/kayobe-config/kayobe-env --environment ci-builder &&
          kayobe seed host command run --command "sudo mkdir -p /var/lib/docker/volumes/bifrost_httpboot/_data" --show-output
        env:
          KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }}

      - name: Build a CentOS Stream 8 overcloud host image
        id: build_centos_stream_8
        continue-on-error: true
        run: |
          source venvs/kayobe/bin/activate &&
          source src/kayobe-config/kayobe-env --environment ci-builder &&
          kayobe overcloud host image build --force-rebuild \
          -e os_distribution="centos" \
          -e os_release="8-stream" \
          -e stackhpc_overcloud_dib_name=overcloud-centos-8-stream
        env:
          KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }}
        if: inputs.centos

      - name: Show last error logs
        continue-on-error: true
        run: |
          source venvs/kayobe/bin/activate &&
          source src/kayobe-config/kayobe-env --environment ci-builder &&
          kayobe seed host command run --command "tail -200 /opt/kayobe/images/overcloud-centos-8-stream/overcloud-centos-8-stream.stdout" --show-output
        env:
          KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }}
        if: steps.build_centos_stream_8.outcome == 'failure'

      - name: Upload CentOS Stream 8 overcloud host image to Ark
        run: |
          source venvs/kayobe/bin/activate &&
          source src/kayobe-config/kayobe-env --environment ci-builder &&
          kayobe playbook run \
          src/kayobe-config/etc/kayobe/ansible/pulp-host-image-upload.yml \
          -e image_path=/opt/kayobe/images/overcloud-centos-8-stream \
          -e host_image_tag=${{ steps.host_image_tag.outputs.host_image_tag }} \
          -e os_distribution="centos" \
          -e os_release="8-stream"
        env:
          KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }}
        if: inputs.centos && steps.build_centos_stream_8.outcome == 'success'

      - name: Upload CentOS Stream 8 overcloud host image to Dev Cloud
        run: |
          source venvs/kayobe/bin/activate &&
          source src/kayobe-config/kayobe-env --environment ci-builder &&
          kayobe playbook run \
          src/kayobe-config/etc/kayobe/ansible/openstack-host-image-upload.yml \
          -e local_image_path="/opt/kayobe/images/overcloud-centos-8-stream/overcloud-centos-8-stream.qcow2" \
          -e image_name=overcloud-centos-8-stream-${{ steps.host_image_tag.outputs.host_image_tag }}
        env:
          CLOUDS_YAML: ${{ secrets.CLOUDS_YAML }}
          OS_APPLICATION_CREDENTIAL_ID: ${{ secrets.OS_APPLICATION_CREDENTIAL_ID }}
          OS_APPLICATION_CREDENTIAL_SECRET: ${{ secrets.OS_APPLICATION_CREDENTIAL_SECRET }}
        if: inputs.centos && steps.build_centos_stream_8.outcome == 'success'

      - name: Build a Rocky Linux 8 overcloud host image
        id: build_rocky_8
        continue-on-error: true
        run: |
          source venvs/kayobe/bin/activate &&
          source src/kayobe-config/kayobe-env --environment ci-builder &&
          kayobe overcloud host image build --force-rebuild \
          -e os_distribution="rocky" \
          -e os_release="8" \
          -e stackhpc_overcloud_dib_name=overcloud-rocky-8
        env:
          KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }}
        if: inputs.rocky8

      - name: Show last error logs
        continue-on-error: true
        run: |
          source venvs/kayobe/bin/activate &&
          source src/kayobe-config/kayobe-env --environment ci-builder &&
          kayobe seed host command run --command "tail -200 /opt/kayobe/images/overcloud-rocky-8/overcloud-rocky-8.stdout" --show-output
        env:
          KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }}
        if: steps.build_rocky_8.outcome == 'failure'

      - name: Upload Rocky Linux 8 overcloud host image to Ark
        run: |
          source venvs/kayobe/bin/activate &&
          source src/kayobe-config/kayobe-env --environment ci-builder &&
          kayobe playbook run \
          src/kayobe-config/etc/kayobe/ansible/pulp-host-image-upload.yml \
          -e image_path=/opt/kayobe/images/overcloud-rocky-8 \
          -e host_image_tag=${{ steps.host_image_tag.outputs.host_image_tag }} \
          -e os_distribution="rocky" \
          -e os_release="8"
        env:
          KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }}
        if: inputs.rocky8 && steps.build_rocky_8.outcome == 'success'

      - name: Upload Rocky Linux 8 overcloud host image to Dev Cloud
        run: |
          source venvs/kayobe/bin/activate &&
          source src/kayobe-config/kayobe-env --environment ci-builder &&
          kayobe playbook run \
          src/kayobe-config/etc/kayobe/ansible/openstack-host-image-upload.yml \
          -e local_image_path="/opt/kayobe/images/overcloud-rocky-8/overcloud-rocky-8.qcow2" \
          -e image_name=overcloud-rocky-8-${{ steps.host_image_tag.outputs.host_image_tag }}
        env:
          CLOUDS_YAML: ${{ secrets.CLOUDS_YAML }}
          OS_APPLICATION_CREDENTIAL_ID: ${{ secrets.OS_APPLICATION_CREDENTIAL_ID }}
          OS_APPLICATION_CREDENTIAL_SECRET: ${{ secrets.OS_APPLICATION_CREDENTIAL_SECRET }}
        if: inputs.rocky8 && steps.build_rocky_8.outcome == 'success'

      - name: Build a Rocky Linux 9 overcloud host image
        id: build_rocky_9
        continue-on-error: true
        run: |
          source venvs/kayobe/bin/activate &&
          source src/kayobe-config/kayobe-env --environment ci-builder &&
          kayobe overcloud host image build --force-rebuild \
          -e os_distribution="rocky" \
          -e os_release="9" \
          -e stackhpc_overcloud_dib_name=overcloud-rocky-9
        env:
          KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }}
        if: inputs.rocky9

      - name: Show last error logs
        continue-on-error: true
        run: |
          source venvs/kayobe/bin/activate &&
          source src/kayobe-config/kayobe-env --environment ci-builder &&
          kayobe seed host command run --command "tail -200 /opt/kayobe/images/overcloud-rocky-9/overcloud-rocky-9.stdout" --show-output
        env:
          KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }}
        if: steps.build_rocky_9.outcome == 'failure'

      - name: Upload Rocky Linux 9 overcloud host image to Ark
        run: |
          source venvs/kayobe/bin/activate &&
          source src/kayobe-config/kayobe-env --environment ci-builder &&
          kayobe playbook run \
          src/kayobe-config/etc/kayobe/ansible/pulp-host-image-upload.yml \
          -e image_path=/opt/kayobe/images/overcloud-rocky-9 \
          -e host_image_tag=${{ steps.host_image_tag.outputs.host_image_tag }} \
          -e os_distribution="rocky" \
          -e os_release="9"
        env:
          KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }}
        if: inputs.rocky9 && steps.build_rocky_9.outcome == 'success'

      - name: Upload Rocky Linux 9 overcloud host image to Dev Cloud
        run: |
          source venvs/kayobe/bin/activate &&
          source src/kayobe-config/kayobe-env --environment ci-builder &&
          kayobe playbook run \
          src/kayobe-config/etc/kayobe/ansible/openstack-host-image-upload.yml \
          -e local_image_path="/opt/kayobe/images/overcloud-rocky-9/overcloud-rocky-9.qcow2" \
          -e image_name=overcloud-rocky-9-${{ steps.host_image_tag.outputs.host_image_tag }}
        env:
          CLOUDS_YAML: ${{ secrets.CLOUDS_YAML }}
          OS_APPLICATION_CREDENTIAL_ID: ${{ secrets.OS_APPLICATION_CREDENTIAL_ID }}
          OS_APPLICATION_CREDENTIAL_SECRET: ${{ secrets.OS_APPLICATION_CREDENTIAL_SECRET }}
        if: inputs.rocky9 && steps.build_rocky_9.outcome == 'success'

      - name: Build an Ubuntu Focal 20.04 overcloud host image
        id: build_ubuntu_focal
        continue-on-error: true
        run: |
          source venvs/kayobe/bin/activate &&
          source src/kayobe-config/kayobe-env --environment ci-builder &&
          kayobe overcloud host image build --force-rebuild \
          -e os_distribution="ubuntu" \
          -e os_release="focal" \
          -e stackhpc_overcloud_dib_name=overcloud-ubuntu-focal
        env:
          KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }}
        if: inputs.ubuntu-focal

      - name: Show last error logs
        continue-on-error: true
        run: |
          source venvs/kayobe/bin/activate &&
          source src/kayobe-config/kayobe-env --environment ci-builder &&
          kayobe seed host command run --command "tail -200 /opt/kayobe/images/overcloud-ubuntu-focal/overcloud-ubuntu-focal.stdout" --show-output
        env:
          KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }}
        if: steps.build_ubuntu_focal.outcome == 'failure'

      - name: Upload Ubuntu Focal 20.04 overcloud host image to Ark
        run: |
          source venvs/kayobe/bin/activate &&
          source src/kayobe-config/kayobe-env --environment ci-builder &&
          kayobe playbook run \
          src/kayobe-config/etc/kayobe/ansible/pulp-host-image-upload.yml \
          -e image_path=/opt/kayobe/images/overcloud-ubuntu-focal \
          -e host_image_tag=${{ steps.host_image_tag.outputs.host_image_tag }} \
          -e os_distribution="ubuntu" \
          -e os_release="focal"
        env:
          KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }}
        if: inputs.ubuntu-focal && steps.build_ubuntu_focal.outcome == 'success'

      - name: Upload Ubuntu Focal overcloud host image to Dev Cloud
        run: |
          source venvs/kayobe/bin/activate &&
          source src/kayobe-config/kayobe-env --environment ci-builder &&
          kayobe playbook run \
          src/kayobe-config/etc/kayobe/ansible/openstack-host-image-upload.yml \
          -e local_image_path="/opt/kayobe/images/overcloud-ubuntu-focal/overcloud-ubuntu-focal.qcow2" \
          -e image_name=overcloud-ubuntu-focal-${{ steps.host_image_tag.outputs.host_image_tag }}
        env:
          CLOUDS_YAML: ${{ secrets.CLOUDS_YAML }}
          OS_APPLICATION_CREDENTIAL_ID: ${{ secrets.OS_APPLICATION_CREDENTIAL_ID }}
          OS_APPLICATION_CREDENTIAL_SECRET: ${{ secrets.OS_APPLICATION_CREDENTIAL_SECRET }}
        if: inputs.ubuntu-focal && steps.build_ubuntu_focal.outcome == 'success'

      - name: Build an Ubuntu Jammy 22.04 overcloud host image
        id: build_ubuntu_jammy
        continue-on-error: true
        run: |
          source venvs/kayobe/bin/activate &&
          source src/kayobe-config/kayobe-env --environment ci-builder &&
          kayobe overcloud host image build --force-rebuild \
          -e os_distribution="ubuntu" \
          -e os_release="jammy" \
          -e stackhpc_overcloud_dib_name=overcloud-ubuntu-jammy
        env:
          KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }}
        if: inputs.ubuntu-jammy

      - name: Show last error logs
        continue-on-error: true
        run: |
          source venvs/kayobe/bin/activate &&
          source src/kayobe-config/kayobe-env --environment ci-builder &&
          kayobe seed host command run --command "tail -200 /opt/kayobe/images/overcloud-ubuntu-jammy/overcloud-ubuntu-jammy.stdout" --show-output
        env:
          KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }}
        if: steps.build_ubuntu_jammy.outcome == 'failure'

      - name: Upload Ubuntu Jammy 22.04 overcloud host image to Ark
        run: |
          source venvs/kayobe/bin/activate &&
          source src/kayobe-config/kayobe-env --environment ci-builder &&
          kayobe playbook run \
          src/kayobe-config/etc/kayobe/ansible/pulp-host-image-upload.yml \
          -e image_path=/opt/kayobe/images/overcloud-ubuntu-jammy \
          -e host_image_tag=${{ steps.host_image_tag.outputs.host_image_tag }} \
          -e os_distribution="ubuntu" \
          -e os_release="jammy"
        env:
          KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }}
        if: inputs.ubuntu-jammy && steps.build_ubuntu_jammy.outcome == 'success'

      - name: Upload Ubuntu Jammy overcloud host image to Dev Cloud
        run: |
          source venvs/kayobe/bin/activate &&
          source src/kayobe-config/kayobe-env --environment ci-builder &&
          kayobe playbook run \
          src/kayobe-config/etc/kayobe/ansible/openstack-host-image-upload.yml \
          -e local_image_path="/opt/kayobe/images/overcloud-ubuntu-jammy/overcloud-ubuntu-jammy.qcow2" \
          -e image_name=overcloud-ubuntu-jammy-${{ steps.host_image_tag.outputs.host_image_tag }}
        env:
          CLOUDS_YAML: ${{ secrets.CLOUDS_YAML }}
          OS_APPLICATION_CREDENTIAL_ID: ${{ secrets.OS_APPLICATION_CREDENTIAL_ID }}
          OS_APPLICATION_CREDENTIAL_SECRET: ${{ secrets.OS_APPLICATION_CREDENTIAL_SECRET }}
        if: inputs.ubuntu-jammy && steps.build_ubuntu_jammy.outcome == 'success'

      - name: Copy logs back
        continue-on-error: true
        run: |
          mkdir logs
          scp -r rocky@$(jq -r .access_ip_v4.value src/kayobe-config/etc/kayobe/environments/ci-builder/tf-outputs.yml):/opt/kayobe/images/*/*.std* ./logs/
          scp -r rocky@$(jq -r .access_ip_v4.value src/kayobe-config/etc/kayobe/environments/ci-builder/tf-outputs.yml):/tmp/updated_images.txt ./logs/
        if: always()

      - name: Fail if any overcloud host image builds failed
        run: |
          echo "Builds failed. See workflow artifacts for details." &&
          exit 1
        if: steps.build_centos_stream_8.outcome == 'failure' ||
            steps.build_rocky_8.outcome == 'failure' ||
            steps.build_rocky_9.outcome == 'failure' ||
            steps.build_ubuntu_focal.outcome == 'failure' ||
            steps.build_ubuntu_jammy.outcome == 'failure'

      - name: Upload logs artifact
        uses: actions/upload-artifact@v4
        with:
          name: Build logs
          path: ./logs

      - name: Destroy
        run: terraform destroy -auto-approve
        working-directory: ${{ github.workspace }}/src/kayobe-config/terraform/host-image-builder
        env:
          OS_CLOUD: openstack
          OS_APPLICATION_CREDENTIAL_ID: ${{ secrets.OS_APPLICATION_CREDENTIAL_ID }}
          OS_APPLICATION_CREDENTIAL_SECRET: ${{ secrets.OS_APPLICATION_CREDENTIAL_SECRET }}
        if: always()