From 6ce7d97a369d6cde79b8bd63060a5c1356ac584c Mon Sep 17 00:00:00 2001 From: Mark Goddard Date: Wed, 24 Jan 2024 10:50:10 +0000 Subject: [PATCH] Fix host image builds on Arc runners Arc runners are kubernetes-orchestrated github runners. Host image builds do not work on these runners, so this commit adapts the host image build workflow to spin up a worker VM which executes the build. --- .../workflows/overcloud-host-image-build.yml | 402 +++++++++++------- .github/workflows/stackhpc-ci-cleanup.yml | 20 + .../ansible/openstack-host-image-upload.yml | 42 ++ etc/kayobe/ansible/pulp-host-image-upload.yml | 16 +- .../group_vars/seed/network-interfaces | 11 +- etc/kayobe/overcloud-dib.yml | 2 +- etc/kayobe/pulp-host-image-versions.yml | 10 +- terraform/host-image-builder/README.rst | 9 + terraform/host-image-builder/outputs.tf | 7 + terraform/host-image-builder/provider.tf | 14 + .../templates/userdata.cfg.tpl | 21 + terraform/host-image-builder/terraform.tfvars | 10 + terraform/host-image-builder/vm.tf | 98 +++++ 13 files changed, 476 insertions(+), 186 deletions(-) create mode 100644 etc/kayobe/ansible/openstack-host-image-upload.yml create mode 100644 terraform/host-image-builder/README.rst create mode 100644 terraform/host-image-builder/outputs.tf create mode 100644 terraform/host-image-builder/provider.tf create mode 100644 terraform/host-image-builder/templates/userdata.cfg.tpl create mode 100644 terraform/host-image-builder/terraform.tfvars create mode 100644 terraform/host-image-builder/vm.tf diff --git a/.github/workflows/overcloud-host-image-build.yml b/.github/workflows/overcloud-host-image-build.yml index cbccca23e..9af0f74cb 100644 --- a/.github/workflows/overcloud-host-image-build.yml +++ b/.github/workflows/overcloud-host-image-build.yml @@ -35,21 +35,39 @@ on: env: ANSIBLE_FORCE_COLOR: True + KAYOBE_ENVIRONMENT: ci-builder + KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }} jobs: overcloud-host-image-build: name: Build overcloud host images if: github.repository == 'stackhpc/stackhpc-kayobe-config' - runs-on: [self-hosted, stackhpc-kayobe-config-kolla-builder] + runs-on: arc-skc-host-image-builder-runner permissions: {} steps: - - uses: actions/checkout@v4 + - name: Install Package + uses: ConorMacBride/install-package@main + with: + apt: git unzip nodejs python3-pip python3-venv openssh-server openssh-client jq + + - name: Start the SSH service + run: | + sudo /etc/init.d/ssh start + continue-on-error: true + + - name: Checkout + uses: actions/checkout@v4 with: path: src/kayobe-config + - name: Output image tag of the builder + id: builder_image_tag + run: | + echo image_tag=$(grep stackhpc_rocky_9_overcloud_host_image_version: etc/kayobe/pulp-host-image-versions.yml | awk '{print $2}') >> $GITHUB_OUTPUT + - name: Determine OpenStack release id: openstack_release run: | - BRANCH=$(awk -F'=' '/defaultbranch/ {print $2}' .gitreview) + BRANCH=$(awk -F'=' '/defaultbranch/ {print $2}' src/kayobe-config/.gitreview) echo "openstack_release=${BRANCH}" | sed "s|stable/||" >> $GITHUB_OUTPUT # Generate a tag to apply to all built overcloud host images. @@ -62,10 +80,6 @@ jobs: run: | echo "${{ steps.host_image_tag.outputs.host_image_tag }}" - - name: Clean any previous build artifact - run: | - rm -f /tmp/updated_images.txt - - name: Clone StackHPC Kayobe repository uses: actions/checkout@v4 with: @@ -73,34 +87,6 @@ jobs: ref: refs/heads/stackhpc/${{ steps.openstack_release.outputs.openstack_release }} path: src/kayobe - # FIXME: Failed in kolla-ansible : Ensure the latest version of pip is installed - - name: Install dependencies - run: | - sudo dnf -y install python3-virtualenv zstd - - - name: Setup networking - run: | - if ! ip l show breth1 >/dev/null 2>&1; then - sudo ip l add breth1 type bridge - fi - sudo ip l set breth1 up - if ! ip a show breth1 | grep 192.168.33.3/24; then - sudo ip a add 192.168.33.3/24 dev breth1 - fi - if ! ip l show dummy1 >/dev/null 2>&1; then - sudo ip l add dummy1 type dummy - fi - sudo ip l set dummy1 up - sudo ip l set dummy1 master breth1 - - # FIXME: Without this workaround we see the following issue after the runner is power cycled: - # TASK [MichaelRigart.interfaces : RedHat | ensure network service is started and enabled] *** - # Unable to start service network: Job for network.service failed because the control process exited with error code. - # See \"systemctl status network.service\" and \"journalctl -xe\" for details. - - name: Kill dhclient (workaround) - run: | - (sudo killall dhclient || true) && sudo systemctl restart network - - name: Install Kayobe run: | mkdir -p venvs && @@ -110,36 +96,131 @@ jobs: pip install -U pip && pip install ../src/kayobe + - name: Install terraform + uses: hashicorp/setup-terraform@v2 + + - name: Initialise terraform + run: terraform init + working-directory: ${{ github.workspace }}/src/kayobe-config/terraform/host-image-builder + + - name: Generate SSH keypair + run: ssh-keygen -f id_rsa -N '' + working-directory: ${{ github.workspace }}/src/kayobe-config/terraform/host-image-builder + + - name: Generate clouds.yaml + run: | + cat << EOF > clouds.yaml + ${{ secrets.CLOUDS_YAML }} + EOF + working-directory: ${{ github.workspace }}/src/kayobe-config/terraform/host-image-builder + + - name: Add tags to tfvars file + run: | + echo -e '\nhost_image_builder_tags = ${{ env.VM_TAGS }}' >> terraform.tfvars + working-directory: ${{ github.workspace }}/src/kayobe-config/terraform/host-image-builder + env: + VM_TAGS: '["skc-host-image-build", "ref=${{ github.ref_name }}"]' + + - name: Terraform Plan + run: terraform plan + working-directory: ${{ github.workspace }}/src/kayobe-config/terraform/host-image-builder + env: + OS_CLOUD: "openstack" + OS_APPLICATION_CREDENTIAL_ID: ${{ secrets.OS_APPLICATION_CREDENTIAL_ID }} + OS_APPLICATION_CREDENTIAL_SECRET: ${{ secrets.OS_APPLICATION_CREDENTIAL_SECRET }} + + - name: Terraform Apply + run: | + for attempt in $(seq 5); do + if terraform apply -auto-approve; then + echo "Created infrastructure on attempt $attempt" + exit 0 + fi + echo "Failed to create infrastructure on attempt $attempt" + sleep 10 + terraform destroy -auto-approve + sleep 60 + done + echo "Failed to create infrastructure after $attempt attempts" + exit 1 + working-directory: ${{ github.workspace }}/src/kayobe-config/terraform/host-image-builder + env: + OS_CLOUD: "openstack" + OS_APPLICATION_CREDENTIAL_ID: ${{ secrets.OS_APPLICATION_CREDENTIAL_ID }} + OS_APPLICATION_CREDENTIAL_SECRET: ${{ secrets.OS_APPLICATION_CREDENTIAL_SECRET }} + + - name: Get Terraform outputs + id: tf_outputs + run: | + terraform output -json + working-directory: ${{ github.workspace }}/src/kayobe-config/terraform/host-image-builder + + - name: Write Terraform outputs + run: | + cat << EOF > src/kayobe-config/etc/kayobe/environments/ci-builder/tf-outputs.yml + ${{ steps.tf_outputs.outputs.stdout }} + EOF + + - name: Write Terraform network config + run: | + cat << EOF > src/kayobe-config/etc/kayobe/environments/ci-builder/tf-network-allocation.yml + --- + aio_ips: + builder: "{{ access_ip_v4.value }}" + EOF + + - name: Write Terraform network interface config + run: | + mkdir -p etc/kayobe/environments/$KAYOBE_ENVIRONMENT/inventory/group_vars/seed + cat << EOF > etc/kayobe/environments/$KAYOBE_ENVIRONMENT/inventory/group_vars/seed/tf-network-interfaces + admin_interface: "{{ access_interface.value }}" + EOF + + - name: Manage SSH keys + run: | + mkdir -p ~/.ssh + touch ~/.ssh/authorized_keys + cat src/kayobe-config/terraform/host-image-builder/id_rsa.pub >> ~/.ssh/authorized_keys + cp src/kayobe-config/terraform/host-image-builder/id_rsa* ~/.ssh/ + + - name: Update inventory + run: | + cat > src/kayobe-config/etc/kayobe/environments/ci-builder/inventory/hosts << EOF + [seed] + builder + EOF + - name: Bootstrap the control host run: | source venvs/kayobe/bin/activate && source src/kayobe-config/kayobe-env --environment ci-builder && kayobe control host bootstrap - - name: Configure the seed host + - name: Configure the seed host (Builder VM) + continue-on-error: true run: | source venvs/kayobe/bin/activate && source src/kayobe-config/kayobe-env --environment ci-builder && - kayobe seed host configure + kayobe seed host configure -e seed_bootstrap_user=rocky --skip-tags network + + - name: Install dependencies + continue-on-error: true + run: | + source venvs/kayobe/bin/activate && + source src/kayobe-config/kayobe-env --environment ci-builder && + kayobe seed host command run \ + --command "sudo dnf config-manager --set-enabled crb && sudo dnf -y install epel-release && sudo dnf -y install zstd debootstrap kpartx cloud-init" --show-output env: KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }} - name: Create bifrost_httpboot Docker volume - run: | - if [[ $(sudo docker volume ls -f Name=bifrost_httpboot -q | wc -l) = 0 ]]; then - sudo docker volume create bifrost_httpboot - fi - - - name: Generate clouds.yaml - run: | - cat << EOF > clouds.yaml - ${{ secrets.CLOUDS_YAML }} - EOF - - - name: Install OpenStack client + continue-on-error: true run: | source venvs/kayobe/bin/activate && - pip install python-openstackclient -c https://releases.openstack.org/constraints/upper/${{ steps.openstack_release.outputs.openstack_release }} + source src/kayobe-config/kayobe-env --environment ci-builder && + kayobe seed host command run --command "sudo mkdir -p /var/lib/docker/volumes/bifrost_httpboot/_data" --show-output + env: + KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }} - name: Build a CentOS Stream 8 overcloud host image id: build_centos_stream_8 @@ -153,7 +234,16 @@ jobs: -e stackhpc_overcloud_dib_name=overcloud-centos-8-stream env: KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }} - if: inputs.centos + + - name: Show last error logs + continue-on-error: true + run: | + source venvs/kayobe/bin/activate && + source src/kayobe-config/kayobe-env --environment ci-builder && + kayobe seed host command run --command "tail -200 /opt/kayobe/images/overcloud-centos-8-stream/overcloud-centos-8-stream.stdout" --show-output + env: + KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }} + if: steps.build_centos_stream_8.outcome == 'failure' - name: Upload CentOS Stream 8 overcloud host image to Ark run: | @@ -169,18 +259,16 @@ jobs: KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }} if: inputs.centos && steps.build_centos_stream_8.outcome == 'success' - - name: Upload CentOS Stream 8 overcloud host image to SMS + - name: Upload CentOS Stream 8 overcloud host image to Dev Cloud run: | source venvs/kayobe/bin/activate && - openstack image create \ - overcloud-centos-8-stream-${{ steps.host_image_tag.outputs.host_image_tag }} \ - --container-format bare \ - --disk-format qcow2 \ - --file /opt/kayobe/images/overcloud-centos-8-stream/overcloud-centos-8-stream.qcow2 \ - --private \ - --os-cloud sms-lab-release \ - --progress + source src/kayobe-config/kayobe-env --environment ci-builder && + kayobe playbook run \ + src/kayobe-config/etc/kayobe/ansible/openstack-host-image-upload.yml \ + -e local_image_path="/opt/kayobe/images/overcloud-centos-8-stream/overcloud-centos-8-stream.qcow2" \ + -e image_name=overcloud-centos-8-stream-${{ steps.host_image_tag.outputs.host_image_tag }} env: + CLOUDS_YAML: ${{ secrets.CLOUDS_YAML }} OS_APPLICATION_CREDENTIAL_ID: ${{ secrets.OS_APPLICATION_CREDENTIAL_ID }} OS_APPLICATION_CREDENTIAL_SECRET: ${{ secrets.OS_APPLICATION_CREDENTIAL_SECRET }} if: inputs.centos && steps.build_centos_stream_8.outcome == 'success' @@ -199,6 +287,16 @@ jobs: KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }} if: inputs.rocky8 + - name: Show last error logs + continue-on-error: true + run: | + source venvs/kayobe/bin/activate && + source src/kayobe-config/kayobe-env --environment ci-builder && + kayobe seed host command run --command "tail -200 /opt/kayobe/images/overcloud-rocky-8/overcloud-rocky-8.stdout" --show-output + env: + KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }} + if: steps.build_rocky_8.outcome == 'failure' + - name: Upload Rocky Linux 8 overcloud host image to Ark run: | source venvs/kayobe/bin/activate && @@ -212,19 +310,17 @@ jobs: env: KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }} if: inputs.rocky8 && steps.build_rocky_8.outcome == 'success' - - - name: Upload Rocky Linux 8 overcloud host image to SMS + + - name: Upload Rocky Linux 8 overcloud host image to Dev Cloud run: | source venvs/kayobe/bin/activate && - openstack image create \ - overcloud-rocky-8-${{ steps.host_image_tag.outputs.host_image_tag }} \ - --container-format bare \ - --disk-format qcow2 \ - --file /opt/kayobe/images/overcloud-rocky-8/overcloud-rocky-8.qcow2 \ - --private \ - --os-cloud sms-lab-release \ - --progress + source src/kayobe-config/kayobe-env --environment ci-builder && + kayobe playbook run \ + src/kayobe-config/etc/kayobe/ansible/openstack-host-image-upload.yml \ + -e local_image_path="/opt/kayobe/images/overcloud-rocky-8/overcloud-rocky-8.qcow2" \ + -e image_name=overcloud-rocky-8-${{ steps.host_image_tag.outputs.host_image_tag }} env: + CLOUDS_YAML: ${{ secrets.CLOUDS_YAML }} OS_APPLICATION_CREDENTIAL_ID: ${{ secrets.OS_APPLICATION_CREDENTIAL_ID }} OS_APPLICATION_CREDENTIAL_SECRET: ${{ secrets.OS_APPLICATION_CREDENTIAL_SECRET }} if: inputs.rocky8 && steps.build_rocky_8.outcome == 'success' @@ -243,6 +339,16 @@ jobs: KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }} if: inputs.rocky9 + - name: Show last error logs + continue-on-error: true + run: | + source venvs/kayobe/bin/activate && + source src/kayobe-config/kayobe-env --environment ci-builder && + kayobe seed host command run --command "tail -200 /opt/kayobe/images/overcloud-rocky-9/overcloud-rocky-9.stdout" --show-output + env: + KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }} + if: steps.build_rocky_9.outcome == 'failure' + - name: Upload Rocky Linux 9 overcloud host image to Ark run: | source venvs/kayobe/bin/activate && @@ -256,19 +362,17 @@ jobs: env: KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }} if: inputs.rocky9 && steps.build_rocky_9.outcome == 'success' - - - name: Upload Rocky Linux 9 overcloud host image to SMS + + - name: Upload Rocky Linux 9 overcloud host image to Dev Cloud run: | source venvs/kayobe/bin/activate && - openstack image create \ - overcloud-rocky-9-${{ steps.host_image_tag.outputs.host_image_tag }} \ - --container-format bare \ - --disk-format qcow2 \ - --file /opt/kayobe/images/overcloud-rocky-9/overcloud-rocky-9.qcow2 \ - --private \ - --os-cloud sms-lab-release \ - --progress + source src/kayobe-config/kayobe-env --environment ci-builder && + kayobe playbook run \ + src/kayobe-config/etc/kayobe/ansible/openstack-host-image-upload.yml \ + -e local_image_path="/opt/kayobe/images/overcloud-rocky-9/overcloud-rocky-9.qcow2" \ + -e image_name=overcloud-rocky-9-${{ steps.host_image_tag.outputs.host_image_tag }} env: + CLOUDS_YAML: ${{ secrets.CLOUDS_YAML }} OS_APPLICATION_CREDENTIAL_ID: ${{ secrets.OS_APPLICATION_CREDENTIAL_ID }} OS_APPLICATION_CREDENTIAL_SECRET: ${{ secrets.OS_APPLICATION_CREDENTIAL_SECRET }} if: inputs.rocky9 && steps.build_rocky_9.outcome == 'success' @@ -287,6 +391,16 @@ jobs: KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }} if: inputs.ubuntu-focal + - name: Show last error logs + continue-on-error: true + run: | + source venvs/kayobe/bin/activate && + source src/kayobe-config/kayobe-env --environment ci-builder && + kayobe seed host command run --command "tail -200 /opt/kayobe/images/overcloud-ubuntu-focal/overcloud-ubuntu-focal.stdout" --show-output + env: + KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }} + if: steps.build_ubuntu_focal.outcome == 'failure' + - name: Upload Ubuntu Focal 20.04 overcloud host image to Ark run: | source venvs/kayobe/bin/activate && @@ -301,18 +415,16 @@ jobs: KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }} if: inputs.ubuntu-focal && steps.build_ubuntu_focal.outcome == 'success' - - name: Upload Ubuntu Focal 20.04 overcloud host image to SMS + - name: Upload Ubuntu Focal overcloud host image to Dev Cloud run: | source venvs/kayobe/bin/activate && - openstack image create \ - overcloud-ubuntu-focal-${{ steps.host_image_tag.outputs.host_image_tag }} \ - --container-format bare \ - --disk-format qcow2 \ - --file /opt/kayobe/images/overcloud-ubuntu-focal/overcloud-ubuntu-focal.qcow2 \ - --private \ - --os-cloud sms-lab-release \ - --progress + source src/kayobe-config/kayobe-env --environment ci-builder && + kayobe playbook run \ + src/kayobe-config/etc/kayobe/ansible/openstack-host-image-upload.yml \ + -e local_image_path="/opt/kayobe/images/overcloud-ubuntu-focal/overcloud-ubuntu-focal.qcow2" \ + -e image_name=overcloud-ubuntu-focal-${{ steps.host_image_tag.outputs.host_image_tag }} env: + CLOUDS_YAML: ${{ secrets.CLOUDS_YAML }} OS_APPLICATION_CREDENTIAL_ID: ${{ secrets.OS_APPLICATION_CREDENTIAL_ID }} OS_APPLICATION_CREDENTIAL_SECRET: ${{ secrets.OS_APPLICATION_CREDENTIAL_SECRET }} if: inputs.ubuntu-focal && steps.build_ubuntu_focal.outcome == 'success' @@ -331,6 +443,16 @@ jobs: KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }} if: inputs.ubuntu-jammy + - name: Show last error logs + continue-on-error: true + run: | + source venvs/kayobe/bin/activate && + source src/kayobe-config/kayobe-env --environment ci-builder && + kayobe seed host command run --command "tail -200 /opt/kayobe/images/overcloud-ubuntu-jammy/overcloud-ubuntu-jammy.stdout" --show-output + env: + KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }} + if: steps.build_ubuntu_jammy.outcome == 'failure' + - name: Upload Ubuntu Jammy 22.04 overcloud host image to Ark run: | source venvs/kayobe/bin/activate && @@ -345,83 +467,27 @@ jobs: KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }} if: inputs.ubuntu-jammy && steps.build_ubuntu_jammy.outcome == 'success' - - name: Upload Ubuntu Jammy 22.04 overcloud host image to SMS + - name: Upload Ubuntu Jammy overcloud host image to Dev Cloud run: | source venvs/kayobe/bin/activate && - openstack image create \ - overcloud-ubuntu-jammy-${{ steps.host_image_tag.outputs.host_image_tag }} \ - --container-format bare \ - --disk-format qcow2 \ - --file /opt/kayobe/images/overcloud-ubuntu-jammy/overcloud-ubuntu-jammy.qcow2 \ - --private \ - --os-cloud sms-lab-release \ - --progress + source src/kayobe-config/kayobe-env --environment ci-builder && + kayobe playbook run \ + src/kayobe-config/etc/kayobe/ansible/openstack-host-image-upload.yml \ + -e local_image_path="/opt/kayobe/images/overcloud-ubuntu-jammy/overcloud-ubuntu-jammy.qcow2" \ + -e image_name=overcloud-ubuntu-jammy-${{ steps.host_image_tag.outputs.host_image_tag }} env: + CLOUDS_YAML: ${{ secrets.CLOUDS_YAML }} OS_APPLICATION_CREDENTIAL_ID: ${{ secrets.OS_APPLICATION_CREDENTIAL_ID }} OS_APPLICATION_CREDENTIAL_SECRET: ${{ secrets.OS_APPLICATION_CREDENTIAL_SECRET }} if: inputs.ubuntu-jammy && steps.build_ubuntu_jammy.outcome == 'success' - - name: Upload updated images artifact - uses: actions/upload-artifact@v4 - with: - name: Updated images list - path: /tmp/updated_images.txt - retention-days: 7 - if: steps.build_centos_stream_8.outcome == 'success' || - steps.build_rocky_8.outcome == 'success' || - steps.build_rocky_9.outcome == 'success' || - steps.build_ubuntu_focal.outcome == 'success' || - steps.build_ubuntu_jammy.outcome == 'success' - - - name: Upload CentOS build logs if build failed - uses: actions/upload-artifact@v4 - with: - name: CentOS build logs - path: | - /opt/kayobe/images/overcloud-centos-8-stream/overcloud-centos-8-stream.stdout - /opt/kayobe/images/overcloud-centos-8-stream/overcloud-centos-8-stream.stderr - retention-days: 7 - if: steps.build_centos_stream_8.outcome == 'failure' - - - name: Upload Rocky 8 build logs if build failed - uses: actions/upload-artifact@v4 - with: - name: Rocky 8 build logs - path: | - /opt/kayobe/images/overcloud-rocky-8/overcloud-rocky-8.stdout - /opt/kayobe/images/overcloud-rocky-8/overcloud-rocky-8.stderr - retention-days: 7 - if: steps.build_rocky_8.outcome == 'failure' - - - name: Upload Rocky 9 build logs if build failed - uses: actions/upload-artifact@v4 - with: - name: Rocky 9 build logs - path: | - /opt/kayobe/images/overcloud-rocky-9/overcloud-rocky-9.stdout - /opt/kayobe/images/overcloud-rocky-9/overcloud-rocky-9.stderr - retention-days: 7 - if: steps.build_rocky_9.outcome == 'failure' - - - name: Upload Ubuntu Focal 20.04 build logs if build failed - uses: actions/upload-artifact@v4 - with: - name: Ubuntu Focal 20.04 build logs - path: | - /opt/kayobe/images/overcloud-ubuntu-focal/overcloud-ubuntu-focal.stdout - /opt/kayobe/images/overcloud-ubuntu-focal/overcloud-ubuntu-focal.stderr - retention-days: 7 - if: steps.build_ubuntu_focal.outcome == 'failure' - - - name: Upload Ubuntu Jammy 22.04 build logs if build failed - uses: actions/upload-artifact@v4 - with: - name: Ubuntu Jammy 22.04 build logs - path: | - /opt/kayobe/images/overcloud-ubuntu-jammy/overcloud-ubuntu-jammy.stdout - /opt/kayobe/images/overcloud-ubuntu-jammy/overcloud-ubuntu-jammy.stderr - retention-days: 7 - if: steps.build_ubuntu_jammy.outcome == 'failure' + - name: Copy logs back + continue-on-error: true + run: | + mkdir logs + scp -r rocky@$(jq -r .access_ip_v4.value src/kayobe-config/etc/kayobe/environments/ci-builder/tf-outputs.yml):/opt/kayobe/images/*/*.std* ./logs/ + scp -r rocky@$(jq -r .access_ip_v4.value src/kayobe-config/etc/kayobe/environments/ci-builder/tf-outputs.yml):/tmp/updated_images.txt ./logs/ + if: always() - name: Fail if any overcloud host image builds failed run: | @@ -433,7 +499,17 @@ jobs: steps.build_ubuntu_focal.outcome == 'failure' || steps.build_ubuntu_jammy.outcome == 'failure' - - name: Clean up build artifacts - run: | - sudo rm -rf /opt/kayobe/images/ - if: always() + - name: Upload logs artifact + uses: actions/upload-artifact@v4 + with: + name: Build logs + path: ./logs + + - name: Destroy + run: terraform destroy -auto-approve + working-directory: ${{ github.workspace }}/src/kayobe-config/terraform/host-image-builder + env: + OS_CLOUD: openstack + OS_APPLICATION_CREDENTIAL_ID: ${{ secrets.OS_APPLICATION_CREDENTIAL_ID }} + OS_APPLICATION_CREDENTIAL_SECRET: ${{ secrets.OS_APPLICATION_CREDENTIAL_SECRET }} + if: always() \ No newline at end of file diff --git a/.github/workflows/stackhpc-ci-cleanup.yml b/.github/workflows/stackhpc-ci-cleanup.yml index d0da0c051..acd81627a 100644 --- a/.github/workflows/stackhpc-ci-cleanup.yml +++ b/.github/workflows/stackhpc-ci-cleanup.yml @@ -55,3 +55,23 @@ jobs: OS_CLOUD: openstack OS_APPLICATION_CREDENTIAL_ID: ${{ secrets.OS_APPLICATION_CREDENTIAL_ID }} OS_APPLICATION_CREDENTIAL_SECRET: ${{ secrets.OS_APPLICATION_CREDENTIAL_SECRET }} + + - name: Clean up host image builder instances over 5 hours old + run: | + result=0 + changes_before=$(date -Imin -d -5hours) + for status in ACTIVE BUILD ERROR SHUTOFF; do + for instance in $(openstack server list --tags skc-host-image-build --os-compute-api-version 2.66 --format value --column ID --changes-before $changes_before --status $status); do + echo "Cleaning up $status instance $instance" + openstack server show $instance + if ! openstack server delete $instance; then + echo "Failed to delete $status instance $instance" + result=1 + fi + done + done + exit $result + env: + OS_CLOUD: openstack + OS_APPLICATION_CREDENTIAL_ID: ${{ secrets.OS_APPLICATION_CREDENTIAL_ID }} + OS_APPLICATION_CREDENTIAL_SECRET: ${{ secrets.OS_APPLICATION_CREDENTIAL_SECRET }} diff --git a/etc/kayobe/ansible/openstack-host-image-upload.yml b/etc/kayobe/ansible/openstack-host-image-upload.yml new file mode 100644 index 000000000..e39aa16c0 --- /dev/null +++ b/etc/kayobe/ansible/openstack-host-image-upload.yml @@ -0,0 +1,42 @@ +--- +# This playbook is desinged to be used by the overcloud-host-image-build.yml +# GitHub workflow to upload newly-built images to a development cloud for +# testing and use in CI. +- name: Upload an OS image to Glance + hosts: seed + vars: + local_image_path: "/opt/kayobe/images/overcloud-{{ os_distribution }}-{{ os_release }}/overcloud-{{ os_distribution }}-{{ os_release }}.qcow2" + image_name: "overcloud-{{ os_distribution }}-{{ os_release }}" + tasks: + - name: Write out clouds.yml + copy: + content: "{{ lookup('ansible.builtin.env', 'CLOUDS_YAML') }}" + dest: clouds.yaml + mode: 0664 + + - name: Write out secure.yml + no_log: true + vars: + - os_secrets: + clouds: + openstack: + auth: + application_credential_id: "{{ lookup('ansible.builtin.env', 'OS_APPLICATION_CREDENTIAL_ID') }}" + application_credential_secret: "{{ lookup('ansible.builtin.env', 'OS_APPLICATION_CREDENTIAL_SECRET') }}" + copy: + content: "{{ os_secrets | to_nice_yaml }}" + dest: secure.yaml + mode: 0664 + + - name: Ensure dependencies are installed + pip: + name: openstacksdk + + - name: Upload an image to Glance + openstack.cloud.image: + cloud: openstack + name: "{{ image_name }}" + container_format: bare + disk_format: qcow2 + state: present + filename: "{{ local_image_path }}" diff --git a/etc/kayobe/ansible/pulp-host-image-upload.yml b/etc/kayobe/ansible/pulp-host-image-upload.yml index a06897d90..d3a44f133 100644 --- a/etc/kayobe/ansible/pulp-host-image-upload.yml +++ b/etc/kayobe/ansible/pulp-host-image-upload.yml @@ -1,12 +1,12 @@ --- - name: Upload and create a distribution for an image - hosts: localhost + hosts: seed vars: remote_pulp_url: "{{ stackhpc_release_pulp_url }}" remote_pulp_username: "{{ stackhpc_image_repository_username }}" remote_pulp_password: "{{ stackhpc_image_repository_password }}" repository_name: "kayobe-images-{{ openstack_release }}-{{ os_distribution }}-{{ os_release }}" - base_path: "kayobe-images/{{ openstack_release }}/{{ os_distribution }}/{{ os_release }}" + pulp_base_path: "kayobe-images/{{ openstack_release }}/{{ os_distribution }}/{{ os_release }}" tasks: - name: Print image tag debug: @@ -74,7 +74,7 @@ username: "{{ remote_pulp_username }}" password: "{{ remote_pulp_password }}" name: "{{ repository_name }}_latest" - base_path: "{{ base_path }}/latest" + base_path: "{{ pulp_base_path }}/latest" publication: "{{ publication_details.publication.pulp_href }}" content_guard: development state: present @@ -86,7 +86,7 @@ username: "{{ remote_pulp_username }}" password: "{{ remote_pulp_password }}" name: "{{ repository_name }}_{{ host_image_tag }}" - base_path: "{{ base_path }}/{{ host_image_tag }}" + base_path: "{{ pulp_base_path }}/{{ host_image_tag }}" publication: "{{ publication_details.publication.pulp_href }}" content_guard: development state: present @@ -95,26 +95,26 @@ - name: Update new images file with versioned path lineinfile: path: /tmp/updated_images.txt - line: "{{ remote_pulp_url }}/pulp/content/{{ base_path }}/\ + line: "{{ remote_pulp_url }}/pulp/content/{{ pulp_base_path }}/\ {{ host_image_tag }}/{{ found_files.files[0].path | basename }}" create: true - name: Update new images file with latest path lineinfile: path: /tmp/updated_images.txt - line: "{{ remote_pulp_url }}/pulp/content/{{ base_path }}/\ + line: "{{ remote_pulp_url }}/pulp/content/{{ pulp_base_path }}/\ latest/{{ found_files.files[0].path | basename }}" when: latest_distribution_details.changed - name: Print versioned path debug: - msg: "New versioned path: {{ remote_pulp_url }}/pulp/content/{{ base_path }}/\ + msg: "New versioned path: {{ remote_pulp_url }}/pulp/content/{{ pulp_base_path }}/\ {{ host_image_tag }}/{{ found_files.files[0].path | basename }}" when: latest_distribution_details.changed - name: Print latest path debug: - msg: "New latest path: {{ remote_pulp_url }}/pulp/content/{{ base_path }}/\ + msg: "New latest path: {{ remote_pulp_url }}/pulp/content/{{ pulp_base_path }}/\ latest/{{ found_files.files[0].path | basename }}" when: latest_distribution_details.changed diff --git a/etc/kayobe/environments/ci-builder/inventory/group_vars/seed/network-interfaces b/etc/kayobe/environments/ci-builder/inventory/group_vars/seed/network-interfaces index a583b7214..c143fe2c9 100644 --- a/etc/kayobe/environments/ci-builder/inventory/group_vars/seed/network-interfaces +++ b/etc/kayobe/environments/ci-builder/inventory/group_vars/seed/network-interfaces @@ -2,12 +2,5 @@ ############################################################################### # Network interface definitions for the controller group. -# Controller interface on all-in-one network. -aio_interface: breth1 -# Use dummy1 if it exists, otherwise the bridge will have no ports. -aio_bridge_ports: "{{ ['dummy1'] if 'dummy1' in hostvars[inventory_hostname].ansible_facts else [] }}" - -############################################################################### -# Dummy variable to allow Ansible to accept this file. -workaround_ansible_issue_8743: yes - +# Seed interface on all-in-one network. +aio_interface: eth0 diff --git a/etc/kayobe/overcloud-dib.yml b/etc/kayobe/overcloud-dib.yml index 8f59d58ef..d7f6dbd69 100644 --- a/etc/kayobe/overcloud-dib.yml +++ b/etc/kayobe/overcloud-dib.yml @@ -71,7 +71,7 @@ overcloud_dib_host_packages_extra: overcloud_dib_git_elements_extra: - repo: "https://github.com/stackhpc/stackhpc-image-elements" local: "{{ source_checkout_path }}/stackhpc-image-elements" - version: "v1.6.0" + version: "v1.6.1" elements_path: "elements" # List of git repositories containing Diskimage Builder (DIB) elements. See diff --git a/etc/kayobe/pulp-host-image-versions.yml b/etc/kayobe/pulp-host-image-versions.yml index 15ee74851..8afd0e45a 100644 --- a/etc/kayobe/pulp-host-image-versions.yml +++ b/etc/kayobe/pulp-host-image-versions.yml @@ -1,8 +1,8 @@ --- # Overcloud host image versioning tags # These images must be in SMS, since they are used by our AIO CI runners -stackhpc_centos_8_stream_overcloud_host_image_version: "yoga-20230525T095243" -stackhpc_rocky_8_overcloud_host_image_version: "yoga-20230629T135322" -stackhpc_rocky_9_overcloud_host_image_version: "yoga-20240124T094316" -stackhpc_ubuntu_focal_overcloud_host_image_version: "yoga-20230609T120720" -stackhpc_ubuntu_jammy_overcloud_host_image_version: "yoga-20231012T121552" +stackhpc_centos_8_stream_overcloud_host_image_version: "yoga-20240326T091511" +stackhpc_rocky_8_overcloud_host_image_version: "yoga-20240326T091511" +stackhpc_rocky_9_overcloud_host_image_version: "yoga-20240326T091511" +stackhpc_ubuntu_focal_overcloud_host_image_version: "yoga-20240326T091511" +stackhpc_ubuntu_jammy_overcloud_host_image_version: "yoga-20240326T091511" diff --git a/terraform/host-image-builder/README.rst b/terraform/host-image-builder/README.rst new file mode 100644 index 000000000..6a9645f15 --- /dev/null +++ b/terraform/host-image-builder/README.rst @@ -0,0 +1,9 @@ +============================ +Terraform Host Image Builder +============================ + +This Terraform configuration deploys a single VM on an OpenStack cloud, to +build overcloud host images. + +This configuration is used in the GitHub Actions overcloud-host-image-build.yml +workflow. diff --git a/terraform/host-image-builder/outputs.tf b/terraform/host-image-builder/outputs.tf new file mode 100644 index 000000000..25542945e --- /dev/null +++ b/terraform/host-image-builder/outputs.tf @@ -0,0 +1,7 @@ +output "access_ip_v4" { + value = openstack_compute_instance_v2.kayobe-host-image-builder.access_ip_v4 +} + +output "access_interface" { + value = var.host_image_builder_interface +} diff --git a/terraform/host-image-builder/provider.tf b/terraform/host-image-builder/provider.tf new file mode 100644 index 000000000..0e7cbdb05 --- /dev/null +++ b/terraform/host-image-builder/provider.tf @@ -0,0 +1,14 @@ +#provider "openstack" { +# use environment variables +#} + +terraform { + required_version = ">= 0.14" + backend "local" { + } + required_providers { + openstack = { + source = "terraform-provider-openstack/openstack" + } + } +} diff --git a/terraform/host-image-builder/templates/userdata.cfg.tpl b/terraform/host-image-builder/templates/userdata.cfg.tpl new file mode 100644 index 000000000..2c6600d51 --- /dev/null +++ b/terraform/host-image-builder/templates/userdata.cfg.tpl @@ -0,0 +1,21 @@ +#cloud-config +# Don't automatically mount ephemeral disk +mounts: + - [/dev/vdb, null] +# WORKAROUND: internal DNS missing from SMS lab. (currently unused) +runcmd: + - 'echo "10.0.0.34 pelican pelican.service.compute.sms-lab.cloud" >> /etc/hosts' + - 'echo "10.205.3.187 pulp-server pulp-server.internal.sms-cloud" >> /etc/hosts' +# Configure SSH keys here, to avoid creating an ephemeral keypair. +# This means only the instance needs to be cleaned up if the destroy fails. +ssh_authorized_keys: + - ${ssh_public_key} + +write_files: + # WORKAROUND: https://bugs.launchpad.net/kolla-ansible/+bug/1995409 + - content: | + #!/bin/bash + docker exec openvswitch_vswitchd ovs-vsctl "$@" + owner: root:root + path: /usr/bin/ovs-vsctl + permissions: '0755' diff --git a/terraform/host-image-builder/terraform.tfvars b/terraform/host-image-builder/terraform.tfvars new file mode 100644 index 000000000..a8673df19 --- /dev/null +++ b/terraform/host-image-builder/terraform.tfvars @@ -0,0 +1,10 @@ +ssh_public_key = "id_rsa.pub" +ssh_username = "rocky" +host_image_builder_name = "skc-ci-host-image-builder" +# Must be a Rocky Linux 9 host to successfully build all images +# This MUST NOT be an LVM image. It can cause confusing conficts with the built image. +host_image_builder_image = "Rocky-9-GenericCloud-Base-9.3-20231113.0.x86_64.qcow2" +host_image_builder_flavor = "en1.medium" +host_image_builder_network = "stackhpc-ci" +host_image_builder_subnet = "stackhpc-ci" +host_image_builder_interface = "eth0" \ No newline at end of file diff --git a/terraform/host-image-builder/vm.tf b/terraform/host-image-builder/vm.tf new file mode 100644 index 000000000..22b5e4b55 --- /dev/null +++ b/terraform/host-image-builder/vm.tf @@ -0,0 +1,98 @@ +variable "ssh_public_key" { + type = string +} + +variable "ssh_username" { + type = string +} + +variable "host_image_builder_name" { + type = string + default = "kayobe-host-image-builder" +} + +variable "host_image_builder_image" { + type = string + default = "Rocky-9" +} + +variable "host_image_builder_interface" { + type = string + default = "eth0" +} + +variable "host_image_builder_flavor" { + type = string +} + +variable "host_image_builder_network" { + type = string +} + +variable "host_image_builder_subnet" { + type = string +} + +variable "host_image_builder_volume_size" { + type = number + default = 150 +} + +variable "host_image_builder_tags" { + type = list(string) + default = [] +} + +locals { + image_is_uuid = length(regexall("^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$", var.host_image_builder_image)) > 0 +} + +data "openstack_images_image_v2" "image" { + name = var.host_image_builder_image + most_recent = true + count = local.image_is_uuid ? 0 : 1 +} + +data "openstack_networking_subnet_v2" "network" { + name = var.host_image_builder_subnet +} + +resource "openstack_compute_instance_v2" "kayobe-host-image-builder" { + name = var.host_image_builder_name + flavor_name = "en1.medium" + config_drive = true + user_data = templatefile("templates/userdata.cfg.tpl", {ssh_public_key = file(var.ssh_public_key)}) + network { + name = var.host_image_builder_network + } + + block_device { + uuid = local.image_is_uuid ? var.host_image_builder_image: data.openstack_images_image_v2.image[0].id + source_type = "image" + volume_size = var.host_image_builder_volume_size + boot_index = 0 + destination_type = "volume" + delete_on_termination = true + } + + tags = var.host_image_builder_tags +} + +# Wait for the instance to be accessible via SSH before progressing. +resource "null_resource" "kayobe-host-image-builder" { + provisioner "remote-exec" { + connection { + host = openstack_compute_instance_v2.kayobe-host-image-builder.access_ip_v4 + user = var.ssh_username + private_key = file("id_rsa") + # Terraform will run the start script from /tmp by default. For the + # current images, /tmp is noexec, so the path must be changed + script_path = "/home/${var.ssh_username}/start.sh" + } + + inline = [ + "#!/bin/sh", + "echo 'connected!'" + ] + } +}