From 136e67cec870bcd0b6a02adc60ac7a40bcc31608 Mon Sep 17 00:00:00 2001 From: Alex-Welsh Date: Wed, 7 Feb 2024 10:16:00 +0000 Subject: [PATCH] Add workflow to upload Ark host images to glance --- .../workflows/overcloud-host-image-upload.yml | 260 ++++++++++++++++++ .../ansible/pulp-host-image-download.yml | 36 +++ 2 files changed, 296 insertions(+) create mode 100644 .github/workflows/overcloud-host-image-upload.yml create mode 100644 etc/kayobe/ansible/pulp-host-image-download.yml diff --git a/.github/workflows/overcloud-host-image-upload.yml b/.github/workflows/overcloud-host-image-upload.yml new file mode 100644 index 000000000..365ba17f6 --- /dev/null +++ b/.github/workflows/overcloud-host-image-upload.yml @@ -0,0 +1,260 @@ +--- +name: Upload overcloud host images +on: + workflow_dispatch: + inputs: + centos: + description: Upload CentOS Stream 8 + type: boolean + default: true + rocky8: + description: Upload Rocky Linux 8 + type: boolean + default: true + rocky9: + description: Upload Rocky Linux 9 + type: boolean + default: true + ubuntu-focal: + description: Upload Ubuntu 20.04 Focal + type: boolean + default: true + ubuntu-jammy: + description: Upload Ubuntu 22.04 Jammy + type: boolean + default: true + kayobe-environment: + description: Kayobe environment to use + type: string + default: "ci-builder" + secrets: + KAYOBE_VAULT_PASSWORD: + required: true + CLOUDS_YAML: + required: true + OS_APPLICATION_CREDENTIAL_ID: + required: true + OS_APPLICATION_CREDENTIAL_SECRET: + required: true + +env: + ANSIBLE_FORCE_COLOR: True +jobs: + overcloud-host-image-upload: + name: Upload overcloud host images + if: github.repository == 'stackhpc/stackhpc-kayobe-config' + runs-on: arc-skc-host-image-builder-runner + permissions: {} + steps: + - name: Install package dependencies + run: | + sudo apt update + sudo apt install -y build-essential git unzip nodejs python3-wheel python3-pip python3-venv + + - uses: actions/checkout@v4 + with: + path: src/kayobe-config + + - name: Determine OpenStack release + id: openstack_release + run: | + BRANCH=$(awk -F'=' '/defaultbranch/ {print $2}' .gitreview) + echo "openstack_release=${BRANCH}" | sed "s|stable/||" >> $GITHUB_OUTPUT + + - name: Clone StackHPC Kayobe repository + uses: actions/checkout@v4 + with: + repository: stackhpc/kayobe + ref: refs/heads/stackhpc/${{ steps.openstack_release.outputs.openstack_release }} + path: src/kayobe + + - name: Install Kayobe + run: | + mkdir -p venvs && + pushd venvs && + python3 -m venv kayobe && + source kayobe/bin/activate && + pip install -U pip && + pip install ../src/kayobe + + - name: Bootstrap the control host + run: | + source venvs/kayobe/bin/activate && + source src/kayobe-config/kayobe-env --environment ${{ inputs.kayobe-environment }} && + kayobe control host bootstrap + + - name: Generate clouds.yaml + run: | + cat << EOF > clouds.yaml + ${{ secrets.CLOUDS_YAML }} + EOF + + - name: Install OpenStack client + run: | + source venvs/kayobe/bin/activate && + pip install python-openstackclient -c https://opendev.org/openstack/requirements/raw/branch/stable/yoga/upper-constraints.txt + + - name: Download CentOS Stream 8 overcloud host image from Ark + run: | + source venvs/kayobe/bin/activate && + source src/kayobe-config/kayobe-env --environment ${{ inputs.kayobe-environment }} && + kayobe playbook run \ + src/kayobe-config/etc/kayobe/ansible/pulp-host-image-download.yml \ + -e os_distribution="centos" \ + -e os_release="8-stream" + env: + KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }} + if: inputs.centos + + - name: Output CentOS Stream 8 image tag + id: centos_8_stream_image_tag + run: | + echo image_tag=$(grep stackhpc_centos_8_stream_overcloud_host_image_version: src/kayobe-config/etc/kayobe/pulp-host-image-versions.yml | awk '{print $2}') >> $GITHUB_OUTPUT + + - name: Upload CentOS Stream 8 overcloud host image to Cloud + run: | + source venvs/kayobe/bin/activate && + openstack image create \ + overcloud-centos-8-stream-${{ steps.centos_8_stream_image_tag.outputs.image_tag }} \ + --container-format bare \ + --disk-format qcow2 \ + --file /tmp/overcloud-centos-8-stream.qcow2 \ + --private \ + --progress + env: + OS_CLOUD: openstack + OS_APPLICATION_CREDENTIAL_ID: ${{ secrets.OS_APPLICATION_CREDENTIAL_ID }} + OS_APPLICATION_CREDENTIAL_SECRET: ${{ secrets.OS_APPLICATION_CREDENTIAL_SECRET }} + if: inputs.centos + + - name: Download Rocky Linux 8 overcloud host image from Ark + run: | + source venvs/kayobe/bin/activate && + source src/kayobe-config/kayobe-env --environment ${{ inputs.kayobe-environment }} && + kayobe playbook run \ + src/kayobe-config/etc/kayobe/ansible/pulp-host-image-upload.yml \ + -e os_distribution="rocky" \ + -e os_release="8" + env: + KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }} + if: inputs.rocky8 + + - name: Output Rocky Linux 8 image tag + id: rocky_8_image_tag + run: | + echo image_tag=$(grep stackhpc_rocky_8_overcloud_host_image_version: src/kayobe-config/etc/kayobe/pulp-host-image-versions.yml | awk '{print $2}') >> $GITHUB_OUTPUT + + - name: Upload Rocky Linux 8 overcloud host image to Cloud + run: | + source venvs/kayobe/bin/activate && + openstack image create \ + overcloud-rocky-8-${{ steps.rocky_8_image_tag.outputs.image_tag }} \ + --container-format bare \ + --disk-format qcow2 \ + --file /tmp/rocky-8.qcow2 \ + --private \ + --progress + env: + OS_CLOUD: openstack + OS_APPLICATION_CREDENTIAL_ID: ${{ secrets.OS_APPLICATION_CREDENTIAL_ID }} + OS_APPLICATION_CREDENTIAL_SECRET: ${{ secrets.OS_APPLICATION_CREDENTIAL_SECRET }} + if: inputs.rocky8 + + - name: Download Rocky Linux 9 overcloud host image from Ark + run: | + source venvs/kayobe/bin/activate && + source src/kayobe-config/kayobe-env --environment ${{ inputs.kayobe-environment }} && + kayobe playbook run \ + src/kayobe-config/etc/kayobe/ansible/pulp-host-image-upload.yml \ + -e os_distribution="rocky" \ + -e os_release="9" + env: + KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }} + if: inputs.rocky9 + + - name: Output Rocky Linux 9 image tag + id: rocky_9_image_tag + run: | + echo image_tag=$(grep stackhpc_rocky_9_overcloud_host_image_version: src/kayobe-config/etc/kayobe/pulp-host-image-versions.yml | awk '{print $2}') >> $GITHUB_OUTPUT + + - name: Upload Rocky Linux 9 overcloud host image to Cloud + run: | + source venvs/kayobe/bin/activate && + openstack image create \ + overcloud-rocky-9-${{ steps.rocky_9_image_tag.outputs.image_tag }} \ + --container-format bare \ + --disk-format qcow2 \ + --file /tmp/rocky-9.qcow2 \ + --private \ + --progress + env: + OS_CLOUD: openstack + OS_APPLICATION_CREDENTIAL_ID: ${{ secrets.OS_APPLICATION_CREDENTIAL_ID }} + OS_APPLICATION_CREDENTIAL_SECRET: ${{ secrets.OS_APPLICATION_CREDENTIAL_SECRET }} + if: inputs.rocky9 + + - name: Download Ubuntu Focal 20.04 overcloud host image from Ark + run: | + source venvs/kayobe/bin/activate && + source src/kayobe-config/kayobe-env --environment ${{ inputs.kayobe-environment }} && + kayobe playbook run \ + src/kayobe-config/etc/kayobe/ansible/pulp-host-image-upload.yml \ + -e os_distribution="ubuntu" \ + -e os_release="focal" + env: + KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }} + if: inputs.ubuntu-focal + + - name: Output Ubuntu Focal image tag + id: ubuntu_focal_image_tag + run: | + echo image_tag=$(grep stackhpc_ubuntu_focal_overcloud_host_image_version: src/kayobe-config/etc/kayobe/pulp-host-image-versions.yml | awk '{print $2}') >> $GITHUB_OUTPUT + + - name: Upload Ubuntu Focal 20.04 overcloud host image to Cloud + run: | + source venvs/kayobe/bin/activate && + openstack image create \ + overcloud-ubuntu-focal-${{ steps.ubuntu_focal_image_tag.outputs.image_tag }} \ + --container-format bare \ + --disk-format qcow2 \ + --file /tmp/ubuntu-focal.qcow2 \ + --private \ + --progress + env: + OS_CLOUD: openstack + OS_APPLICATION_CREDENTIAL_ID: ${{ secrets.OS_APPLICATION_CREDENTIAL_ID }} + OS_APPLICATION_CREDENTIAL_SECRET: ${{ secrets.OS_APPLICATION_CREDENTIAL_SECRET }} + if: inputs.ubuntu-focal + + - name: Download Ubuntu Jammy 22.04 overcloud host image from Ark + run: | + source venvs/kayobe/bin/activate && + source src/kayobe-config/kayobe-env --environment ${{ inputs.kayobe-environment }} && + kayobe playbook run \ + src/kayobe-config/etc/kayobe/ansible/pulp-host-image-upload.yml \ + -e os_distribution="ubuntu" \ + -e os_release="jammy" + env: + KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }} + if: inputs.ubuntu-jammy + + - name: Output Ubuntu Jammy image tag + id: ubuntu_jammy_image_tag + run: | + echo image_tag=$(grep stackhpc_ubuntu_jammy_overcloud_host_image_version: src/kayobe-config/etc/kayobe/pulp-host-image-versions.yml | awk '{print $2}') >> $GITHUB_OUTPUT + + - name: Upload Ubuntu Jammy 22.04 overcloud host image to Cloud + run: | + source venvs/kayobe/bin/activate && + openstack image create \ + overcloud-ubuntu-jammy-${{ steps.ubuntu_jammy_image_tag.outputs.image_tag }} \ + --container-format bare \ + --disk-format qcow2 \ + --file /tmp/ubuntu-jammy.qcow2 \ + --private \ + --progress + env: + OS_CLOUD: openstack + OS_APPLICATION_CREDENTIAL_ID: ${{ secrets.OS_APPLICATION_CREDENTIAL_ID }} + OS_APPLICATION_CREDENTIAL_SECRET: ${{ secrets.OS_APPLICATION_CREDENTIAL_SECRET }} + if: inputs.ubuntu-jammy diff --git a/etc/kayobe/ansible/pulp-host-image-download.yml b/etc/kayobe/ansible/pulp-host-image-download.yml new file mode 100644 index 000000000..6c7465ce3 --- /dev/null +++ b/etc/kayobe/ansible/pulp-host-image-download.yml @@ -0,0 +1,36 @@ +--- +- name: Download an overcloud host image from Ark + hosts: localhost + vars: + # This var is an edited version of stackhpc_overcloud_host_image_url + # without the auth credentials in it. Auth is handled by username and + # password in the get_url task of this playbook + stackhpc_overcloud_host_image_url_no_auth: >- + {{ stackhpc_release_pulp_content_url }}/kayobe-images/ + {{ openstack_release }}/{{ os_distribution }}/{{ os_release }}/ + {{ 'ofed/' if stackhpc_overcloud_host_image_is_ofed else '' }} + {{ stackhpc_overcloud_host_image_version }}/overcloud- + {{ os_distribution }}-{{ os_release }} + {{ '-ofed' if stackhpc_overcloud_host_image_is_ofed else '' }}.qcow2 + + tasks: + - name: Print image information + debug: + msg: | + OS Distribution: {{ os_distribution }} + OS Release: {{ os_release }} + Image tag: {{ stackhpc_overcloud_host_image_version }} + OFED: {{ stackhpc_overcloud_host_image_is_ofed }} + + # TODO: Add checksum support + - name: Download image artifact + get_url: + url: "{{ stackhpc_overcloud_host_image_url_no_auth }}" + username: "{{ stackhpc_release_pulp_username }}" + password: "{{ stackhpc_release_pulp_password }}" + unredirected_headers: + - "Authorization" + dest: "/tmp/{{ os_distribution }}-{{ os_release }}.qcow2" + owner: "{{ ansible_user }}" + group: "{{ ansible_user }}" + mode: "0644"