diff --git a/.github/workflows/amphora-image-build.yml b/.github/workflows/amphora-image-build.yml new file mode 100644 index 000000000..95150c302 --- /dev/null +++ b/.github/workflows/amphora-image-build.yml @@ -0,0 +1,261 @@ +--- +name: Build Amphora image +on: + workflow_dispatch: + secrets: + KAYOBE_VAULT_PASSWORD: + required: true + CLOUDS_YAML: + required: true + OS_APPLICATION_CREDENTIAL_ID: + required: true + OS_APPLICATION_CREDENTIAL_SECRET: + required: true + +env: + ANSIBLE_FORCE_COLOR: True + KAYOBE_ENVIRONMENT: ci-builder + KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }} +jobs: + amphora-image-build: + name: Build Amphora image + if: github.repository == 'stackhpc/stackhpc-kayobe-config' + runs-on: arc-skc-host-image-builder-runner + permissions: {} + steps: + + - name: Install Package + uses: ConorMacBride/install-package@main + with: + apt: git unzip nodejs python3-pip python3-venv openssh-server openssh-client jq + + - name: Start the SSH service + run: | + sudo /etc/init.d/ssh start + + - name: Checkout + uses: actions/checkout@v4 + with: + path: src/kayobe-config + + - name: Output image tag of the builder + id: builder_image_tag + run: | + echo image_tag=$(grep stackhpc_rocky_9_overcloud_host_image_version: etc/kayobe/pulp-host-image-versions.yml | awk '{print $2}') >> $GITHUB_OUTPUT + + - name: Determine OpenStack release + id: openstack_release + run: | + BRANCH=$(awk -F'=' '/defaultbranch/ {print $2}' src/kayobe-config/.gitreview) + echo "BRANCH=$BRANCH" >> $GITHUB_OUTPUT + echo "openstack_release=${BRANCH}" | sed -E "s,(stable|unmaintained)/,," >> $GITHUB_OUTPUT + + # Generate a tag to apply to all built Amphora image. + - name: Generate Amphora image tag + id: image_tag + run: | + echo "image_tag=$(date +${{ steps.openstack_release.outputs.openstack_release }}-%Y%m%dT%H%M%S)" >> $GITHUB_OUTPUT + + - name: Display Amphora image tag + run: | + echo "${{ steps.image_tag.outputs.image_tag }}" + + - name: Install Kayobe + run: | + mkdir -p venvs && + pushd venvs && + python3 -m venv kayobe && + source kayobe/bin/activate && + pip install -U pip && + pip install -r ../src/kayobe-config/requirements.txt + + - name: Install terraform + uses: hashicorp/setup-terraform@v2 + + - name: Initialise terraform + run: terraform init + working-directory: ${{ github.workspace }}/src/kayobe-config/terraform/aio + + - name: Generate SSH keypair + run: ssh-keygen -f id_rsa -N '' + working-directory: ${{ github.workspace }}/src/kayobe-config/terraform/aio + + - name: Generate clouds.yaml + run: | + cat << EOF > clouds.yaml + ${{ secrets.CLOUDS_YAML }} + EOF + working-directory: ${{ github.workspace }}/src/kayobe-config/terraform/aio + + - name: Generate terraform.tfvars + run: | + cat << EOF > terraform.tfvars + ssh_public_key = "id_rsa.pub" + ssh_username = "ubuntu" + aio_vm_name = "skc-amphora-image-builder" + # Must be an Ubuntu Jammy host to successfully build all images + # This MUST NOT be an LVM image. It can cause confusing conficts with the built image. + aio_vm_image = "Ubuntu-22.04" + aio_vm_flavor = "en1.medium" + aio_vm_network = "stackhpc-ci" + aio_vm_subnet = "stackhpc-ci" + aio_vm_interface = "ens3" + EOF + working-directory: ${{ github.workspace }}/src/kayobe-config/terraform/aio + + - name: Terraform Plan + run: terraform plan + working-directory: ${{ github.workspace }}/src/kayobe-config/terraform/aio + env: + OS_CLOUD: "openstack" + OS_APPLICATION_CREDENTIAL_ID: ${{ secrets.OS_APPLICATION_CREDENTIAL_ID }} + OS_APPLICATION_CREDENTIAL_SECRET: ${{ secrets.OS_APPLICATION_CREDENTIAL_SECRET }} + + - name: Terraform Apply + run: | + for attempt in $(seq 5); do + if terraform apply -auto-approve; then + echo "Created infrastructure on attempt $attempt" + exit 0 + fi + echo "Failed to create infrastructure on attempt $attempt" + sleep 10 + terraform destroy -auto-approve + sleep 60 + done + echo "Failed to create infrastructure after $attempt attempts" + exit 1 + working-directory: ${{ github.workspace }}/src/kayobe-config/terraform/aio + env: + OS_CLOUD: "openstack" + OS_APPLICATION_CREDENTIAL_ID: ${{ secrets.OS_APPLICATION_CREDENTIAL_ID }} + OS_APPLICATION_CREDENTIAL_SECRET: ${{ secrets.OS_APPLICATION_CREDENTIAL_SECRET }} + + - name: Get Terraform outputs + id: tf_outputs + run: | + terraform output -json + working-directory: ${{ github.workspace }}/src/kayobe-config/terraform/aio + + - name: Write Terraform outputs + run: | + cat << EOF > src/kayobe-config/etc/kayobe/environments/ci-builder/tf-outputs.yml + ${{ steps.tf_outputs.outputs.stdout }} + EOF + + - name: Write Terraform network config + run: | + cat << EOF > src/kayobe-config/etc/kayobe/environments/ci-builder/tf-network-allocation.yml + --- + aio_ips: + builder: "{{ access_ip_v4.value }}" + EOF + + - name: Write Terraform network interface config + run: | + mkdir -p src/kayobe-config/etc/kayobe/environments/$KAYOBE_ENVIRONMENT/inventory/group_vars/seed + rm -f src/kayobe-config/etc/kayobe/environments/$KAYOBE_ENVIRONMENT/inventory/group_vars/seed/network-interfaces + cat << EOF > src/kayobe-config/etc/kayobe/environments/$KAYOBE_ENVIRONMENT/inventory/group_vars/seed/network-interfaces + admin_interface: "{{ access_interface.value }}" + aio_interface: "{{ access_interface.value }}" + EOF + + - name: Manage SSH keys + run: | + mkdir -p ~/.ssh + touch ~/.ssh/authorized_keys + cat src/kayobe-config/terraform/aio/id_rsa.pub >> ~/.ssh/authorized_keys + cp src/kayobe-config/terraform/aio/id_rsa* ~/.ssh/ + + - name: Bootstrap the control host + run: | + source venvs/kayobe/bin/activate && + source src/kayobe-config/kayobe-env --environment ci-builder && + kayobe control host bootstrap + + - name: Configure the seed host (Builder VM) + run: | + source venvs/kayobe/bin/activate && + source src/kayobe-config/kayobe-env --environment ci-builder && + kayobe seed host configure -e seed_bootstrap_user=ubuntu --skip-tags network + + - name: Install dependencies + run: | + source venvs/kayobe/bin/activate && + source src/kayobe-config/kayobe-env --environment ci-builder && + kayobe seed host command run \ + --command "sudo apt update && sudo apt -y install gcc git libffi-dev python3-dev python-is-python3 python3-venv" --show-output + env: + KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }} + + - name: Create Amphora image output directory + run: | + source venvs/kayobe/bin/activate && + source src/kayobe-config/kayobe-env --environment ci-builder && + kayobe seed host command run \ + --command "mkdir -p /opt/kayobe/images/amphora" --show-output + env: + KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }} + + - name: Build Octavia Amphora image + id: build_amphora + run: | + source venvs/kayobe/bin/activate && + source src/kayobe-config/kayobe-env --environment ci-builder && + kayobe playbook run src/kayobe-config/etc/kayobe/ansible/octavia-amphora-image-build.yml -e amphora_image_dest=/opt/kayobe/images/amphora/amphora-x64-haproxy.qcow2 + env: + KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }} + + - name: Show last error logs + continue-on-error: true + run: | + source venvs/kayobe/bin/activate && + source src/kayobe-config/kayobe-env --environment ci-builder && + kayobe seed host command run --command "tail -200 /var/log/octavia-amphora-image-build.log" --show-output + env: + KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }} + if: steps.build_amphora.outcome == 'failure' + + - name: Upload Octavia Amphora image to Ark + run: | + source venvs/kayobe/bin/activate && + source src/kayobe-config/kayobe-env --environment ci-builder && + kayobe playbook run \ + src/kayobe-config/etc/kayobe/ansible/pulp-artifact-upload.yml \ + -e artifact_path=/opt/kayobe/images/amphora \ + -e artifact_tag=${{ steps.image_tag.outputs.image_tag }} \ + -e file_regex="*.qcow2" \ + -e repository_name="amphora-images-${{ steps.openstack_release.outputs.openstack_release }}" \ + -e pulp_base_path="amphora-images/${{ steps.openstack_release.outputs.openstack_release }}" + env: + KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }} + if: steps.build_amphora.outcome == 'success' + + - name: Copy logback + continue-on-error: true + run: | + mkdir artifact + scp stack@$(jq -r .access_ip_v4.value src/kayobe-config/etc/kayobe/environments/ci-builder/tf-outputs.yml):/var/log/octavia-amphora-image-build.log ./artifact + if: always() + + - name: Fail if Amphora image builds failed + run: | + echo "Builds failed. See workflow artifacts for details." && + exit 1 + if: steps.build_amphora.outcome == 'failure' + + - name: Upload logs & image artifact + uses: actions/upload-artifact@v4 + with: + name: amphora-image-build-log + path: ./artifact + if: always() + + - name: Destroy + run: terraform destroy -auto-approve + working-directory: ${{ github.workspace }}/src/kayobe-config/terraform/aio + env: + OS_CLOUD: openstack + OS_APPLICATION_CREDENTIAL_ID: ${{ secrets.OS_APPLICATION_CREDENTIAL_ID }} + OS_APPLICATION_CREDENTIAL_SECRET: ${{ secrets.OS_APPLICATION_CREDENTIAL_SECRET }} + if: always() diff --git a/.github/workflows/amphora-image-promote.yml b/.github/workflows/amphora-image-promote.yml new file mode 100644 index 000000000..f590e8bd0 --- /dev/null +++ b/.github/workflows/amphora-image-promote.yml @@ -0,0 +1,69 @@ +--- +name: Promote Amphora image +on: + workflow_dispatch: + inputs: + image_tag: + description: Tag to promote + type: string + required: true +env: + ANSIBLE_FORCE_COLOR: True +jobs: + amphora-image-promote: + name: Promote Amphora image + if: github.repository == 'stackhpc/stackhpc-kayobe-config' + runs-on: ubuntu-22.04 + steps: + - uses: actions/checkout@v4 + with: + path: src/kayobe-config + + - name: Determine OpenStack release + id: openstack_release + run: | + BRANCH=$(awk -F'=' '/defaultbranch/ {print $2}' .gitreview) + echo "openstack_release=${BRANCH}" | sed -E "s,(stable|unmaintained)/,," >> $GITHUB_OUTPUT + working-directory: src/kayobe-config + + - name: Setup networking + run: | + if ! ip l show breth1 >/dev/null 2>&1; then + sudo ip l add breth1 type bridge + fi + sudo ip l set breth1 up + if ! ip a show breth1 | grep 192.168.33.3/24; then + sudo ip a add 192.168.33.3/24 dev breth1 + fi + if ! ip l show dummy1 >/dev/null 2>&1; then + sudo ip l add dummy1 type dummy + fi + sudo ip l set dummy1 up + sudo ip l set dummy1 master breth1 + + - name: Install Kayobe + run: | + mkdir -p venvs && + pushd venvs && + python3 -m venv kayobe && + source kayobe/bin/activate && + pip install -U pip && + pip install -r ../src/kayobe-config/requirements.txt + + - name: Bootstrap the control host + run: | + source venvs/kayobe/bin/activate && + source src/kayobe-config/kayobe-env --environment ci-builder && + kayobe control host bootstrap + + - name: Promote Amphora image artifact + run: | + source venvs/kayobe/bin/activate && + source src/kayobe-config/kayobe-env --environment ci-builder && + kayobe playbook run \ + src/kayobe-config/etc/kayobe/ansible/pulp-artifact-promote.yml \ + -e repository_name="amphora-${{ steps.openstack_release.outputs.openstack_release }}" \ + -e pulp_base_path="amphora/${{ steps.openstack_release.outputs.openstack_release }}" + env: + ARTIFACT_TAG: ${{ inputs.image_tag }} + KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }} diff --git a/etc/kayobe/ansible/octavia-amphora-image-build.yml b/etc/kayobe/ansible/octavia-amphora-image-build.yml index 3d880d21f..d7beba6a1 100644 --- a/etc/kayobe/ansible/octavia-amphora-image-build.yml +++ b/etc/kayobe/ansible/octavia-amphora-image-build.yml @@ -3,6 +3,7 @@ hosts: "{{ amphora_builder_group | default('seed') }}" vars: amphora_dib_upper_constraints_file: "{{ pip_upper_constraints_file }}" + amphora_image_dest: "{{ image_cache_path }}/amphora-x64-haproxy-{{ openstack_release }}.qcow2" tasks: - name: Install EPEL package: @@ -15,21 +16,22 @@ - name: Ensure packages are installed become: true vars: + packages_common: + - debootstrap + - git + - python3-venv + - python3-diskimage-builder packages_for_os_family: RedHat: - - debootstrap - qemu-img - - git - e2fsprogs - policycoreutils-python-utils - yum-utils Debian: - - debootstrap - qemu-utils - - git - kpartx package: - name: "{{ packages_for_os_family[ansible_facts.os_family] }}" + name: "{{ packages_common + packages_for_os_family[ansible_facts.os_family] }}" - name: Create a temporary directory tempfile: @@ -40,7 +42,6 @@ - name: Check whether the image cache directory exists stat: path: "{{ image_cache_path }}" - get_md5: False get_checksum: False mime: False register: image_cache_stat @@ -88,14 +89,14 @@ - name: Create the Amphora image shell: - cmd: "source {{ venv_path }}/bin/activate && ./diskimage-create.sh -i ubuntu-minimal -s 3 -g {{ openstack_branch }} >> /var/log/octavia-amphora-image-build.log 2>&1" + cmd: ". {{ venv_path }}/bin/activate && ./diskimage-create.sh -i ubuntu-minimal -s 3 -g {{ openstack_branch }} >> /var/log/octavia-amphora-image-build.log 2>&1" chdir: "{{ src_path }}/diskimage-create" changed_when: true - name: Copy image to image store copy: src: "{{ src_path }}/diskimage-create/amphora-x64-haproxy.qcow2" - dest: "{{ image_cache_path }}/amphora-x64-haproxy-{{ openstack_release }}.qcow2" + dest: "{{ amphora_image_dest }}" remote_src: true always: - name: Remove temporary files diff --git a/etc/kayobe/ansible/octavia-amphora-image-register.yml b/etc/kayobe/ansible/octavia-amphora-image-register.yml index d0da1b55c..6ab6ba1a4 100644 --- a/etc/kayobe/ansible/octavia-amphora-image-register.yml +++ b/etc/kayobe/ansible/octavia-amphora-image-register.yml @@ -1,9 +1,7 @@ --- -- name: Register an Octavia Amphora image in Glance +- name: Octavia credentials precheck gather_facts: yes - hosts: "{{ amphora_builder_group | default('seed') }}" - vars: - venv: "{{ virtualenv_path }}/octavia-amphora" + hosts: localhost tasks: - name: Fail if not using octavia user and service project fail: @@ -13,21 +11,53 @@ lookup('env', 'OS_USERNAME') != 'octavia' or lookup('env', 'OS_PROJECT_NAME') != 'service' +- name: Download Amphora image from Ark + ansible.builtin.import_playbook: pulp-amphora-image-download.yml + when: download_from_ark | default(true) | bool + +- name: Register an Octavia Amphora image in Glance + gather_facts: yes + hosts: localhost + vars: + venv: "{{ virtualenv_path }}/octavia-amphora" + image_path: "/tmp/amphora-x64-haproxy.qcow2" + tasks: - name: Get image checksum + vars: + ansible_host: "{{ hostvars[groups['controllers'][0]].ansible_host }}" stat: - path: "{{ image_cache_path }}/amphora-x64-haproxy-{{ openstack_release }}.qcow2" + path: "{{ image_path }}" checksum_algorithm: md5 changed_when: false register: image_checksum + delegate_to: "{{ groups['controllers'][0] }}" + become: true - name: Assert that Amphora image exists + vars: + ansible_host: "{{ hostvars[groups['controllers'][0]].ansible_host }}" assert: that: image_checksum.stat.exists fail_msg: | - The amphora image: {{ image_cache_path }}/amphora-x64-haproxy-{{ openstack_release }}.qcow2 - does not exist. Did you build the image? + The amphora image: {{ image_path }} + does not exist. Did you download or build the image? + delegate_to: "{{ groups['controllers'][0] }}" + + - name: Ensure packages for python-openstackclient are installed + become: true + vars: + required_packages: + - python3-venv + - python3-dev + - gcc + ansible_host: "{{ hostvars[groups['controllers'][0]].ansible_host }}" + package: + name: "{{ required_packages }}" + delegate_to: "{{ groups['controllers'][0] }}" - name: Set up openstack virtualenv + vars: + ansible_host: "{{ hostvars[groups['controllers'][0]].ansible_host }}" pip: virtualenv: "{{ venv }}" virtualenv_command: python3 -m venv @@ -36,10 +66,12 @@ - python-openstackclient state: latest extra_args: "{% if openstacksdk_upper_constraints_file %}-c {{ openstacksdk_upper_constraints_file }}{% endif %}" + delegate_to: "{{ groups['controllers'][0] }}" - name: Query Octavia Amphora image vars: ansible_python_interpreter: "{{ venv }}/bin/python" + ansible_host: "{{ hostvars[groups['controllers'][0]].ansible_host }}" os_image_info: auth_type: password auth: "{{ openstack_auth }}" @@ -47,40 +79,40 @@ interface: "{{ openstack_interface }}" image: amphora-x64-haproxy register: image_info + delegate_to: "{{ groups['controllers'][0] }}" - - name: Ensure Octavia Amphora image is renamed + - name: Ensure old Octavia Amphora image is renamed vars: ansible_python_interpreter: "{{ venv }}/bin/python" + ansible_host: "{{ hostvars[groups['controllers'][0]].ansible_host }}" shell: cmd: >- {{ venv }}/bin/openstack image set amphora-x64-haproxy --name amphora-x64-haproxy-{{ ansible_facts.date_time.iso8601_basic_short }} when: - - image_info.image - - image_info.image.checksum != image_checksum.stat.checksum + - image_info.images | length != 0 + - image_info.images[0].checksum != image_checksum.stat.checksum changed_when: true environment: "{{ openstack_auth_env }}" + delegate_to: "{{ groups['controllers'][0] }}" - - name: Ensure Octavia Amphora image is registered + - name: Ensure new Octavia Amphora image is registered vars: ansible_python_interpreter: "{{ venv }}/bin/python" + ansible_host: "{{ hostvars[groups['controllers'][0]].ansible_host }}" os_image: auth_type: password auth: "{{ openstack_auth }}" ca_cert: "{{ openstack_cacert }}" interface: "{{ openstack_interface }}" name: amphora-x64-haproxy + tags: ["amphora"] container_format: bare disk_format: qcow2 is_public: no - filename: "{{ image_cache_path }}/amphora-x64-haproxy-{{ openstack_release }}.qcow2" + filename: "{{ image_path }}" properties: hw_architecture: x86_64 hw_rng_model: virtio - - # FIXME: Use 'tags' parameter of os_image module available from - # openstack.cloud.image 1.5.0. - - name: Ensure Octavia Amphora image is tagged - shell: - cmd: >- - {{ venv }}/bin/openstack image set amphora-x64-haproxy --tag amphora - environment: "{{ openstack_auth_env }}" + when: image_info.images | length == 0 or (image_info.images | length != 0 and image_info.images[0].checksum != image_checksum.stat.checksum) + become: true + delegate_to: "{{ groups['controllers'][0] }}" diff --git a/etc/kayobe/ansible/pulp-amphora-image-download.yml b/etc/kayobe/ansible/pulp-amphora-image-download.yml new file mode 100644 index 000000000..ee2c8ee82 --- /dev/null +++ b/etc/kayobe/ansible/pulp-amphora-image-download.yml @@ -0,0 +1,35 @@ +--- +- name: Download an amphora image from Ark + hosts: controllers[0] + vars: + # The auth is handled by username and + # password in the get_url task of this playbook + stackhpc_amphora_image_url_no_auth: "{{ stackhpc_release_pulp_content_url }}/amphora-images/\ + {{ openstack_release }}/\ + {{ stackhpc_amphora_image_version }}/\ + amphora-x64-haproxy.qcow2" + download_destination: /tmp/amphora-x64-haproxy.qcow2 + + tasks: + - name: Print image information + ansible.builtin.debug: + msg: | + Image tag: {{ stackhpc_amphora_image_version }} + + # TODO: Add checksum support + - name: Download image artifact + ansible.builtin.get_url: + url: "{{ stackhpc_amphora_image_url_no_auth }}" + username: "{{ stackhpc_release_pulp_username }}" + password: "{{ stackhpc_release_pulp_password }}" + force_basic_auth: true + unredirected_headers: + - "Authorization" + dest: "{{ download_destination }}" + mode: "0644" + owner: stack + group: stack + register: image_download_result + until: image_download_result.status_code >= 200 and image_download_result.status_code <= 399 + retries: 3 + delay: 60 diff --git a/etc/kayobe/stackhpc.yml b/etc/kayobe/stackhpc.yml index b1955c02b..dd04bf466 100644 --- a/etc/kayobe/stackhpc.yml +++ b/etc/kayobe/stackhpc.yml @@ -166,3 +166,7 @@ stackhpc_docker_registry_password: "{{ pulp_password }}" # Whether or not to run CIS benchmark hardening playbooks. Default is false. #stackhpc_enable_cis_benchmark_hardening_hook: + +############################################################################### +# Octavia Amphora image version +stackhpc_amphora_image_version: "placeholder" # TODO(seunghun1ee): Test the image