diff --git a/.github/workflows/overcloud-host-image-build.yml b/.github/workflows/overcloud-host-image-build.yml index b36d3d9ca..bb4c5fa48 100644 --- a/.github/workflows/overcloud-host-image-build.yml +++ b/.github/workflows/overcloud-host-image-build.yml @@ -35,6 +35,8 @@ on: env: ANSIBLE_FORCE_COLOR: True + KAYOBE_ENVIRONMENT: ci-builder + KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }} jobs: overcloud-host-image-build: name: Build overcloud host images @@ -42,16 +44,26 @@ jobs: runs-on: arc-skc-host-image-builder-runner permissions: {} steps: - - name: Install package dependencies + - name: Install Package + uses: ConorMacBride/install-package@main + with: + apt: git unzip nodejs python3-pip python3-venv openssh-server openssh-client jq + + - name: Start the SSH service run: | - sudo apt update - sudo apt install -y debootstrap gdisk zstd build-essential git unzip nodejs python3-wheel python3-pip python3-venv rpm dnf + sudo /etc/init.d/ssh start + continue-on-error: true - name: Checkout uses: actions/checkout@v4 with: path: src/kayobe-config + - name: Output image tag of the builder + id: builder_image_tag + run: | + echo image_tag=$(grep stackhpc_rocky_9_overcloud_host_image_version: etc/kayobe/pulp-host-image-versions.yml | awk '{print $2}') >> $GITHUB_OUTPUT + - name: Determine OpenStack release id: openstack_release run: | @@ -84,31 +96,124 @@ jobs: pip install -U pip && pip install ../src/kayobe - - name: Bootstrap the control host - run: | - source venvs/kayobe/bin/activate && - source src/kayobe-config/kayobe-env --environment ci-builder && - kayobe control host bootstrap --skip-tags bootstrap + - name: Install terraform + uses: hashicorp/setup-terraform@v2 - - name: Create bifrost_httpboot Docker volume - run: | - sudo mkdir -p /var/lib/docker/volumes/bifrost_httpboot/_data + - name: Initialise terraform + run: terraform init + working-directory: ${{ github.workspace }}/src/kayobe-config/terraform/host-image-builder + + - name: Generate SSH keypair + run: ssh-keygen -f id_rsa -N '' + working-directory: ${{ github.workspace }}/src/kayobe-config/terraform/host-image-builder - name: Generate clouds.yaml run: | cat << EOF > clouds.yaml ${{ secrets.CLOUDS_YAML }} EOF + working-directory: ${{ github.workspace }}/src/kayobe-config/terraform/host-image-builder + + - name: Terraform Plan + run: terraform plan + working-directory: ${{ github.workspace }}/src/kayobe-config/terraform/host-image-builder + env: + OS_CLOUD: "openstack" + OS_APPLICATION_CREDENTIAL_ID: ${{ secrets.OS_APPLICATION_CREDENTIAL_ID }} + OS_APPLICATION_CREDENTIAL_SECRET: ${{ secrets.OS_APPLICATION_CREDENTIAL_SECRET }} + + - name: Terraform Apply + run: | + for attempt in $(seq 5); do + if terraform apply -auto-approve; then + echo "Created infrastructure on attempt $attempt" + exit 0 + fi + echo "Failed to create infrastructure on attempt $attempt" + sleep 10 + terraform destroy -auto-approve + sleep 60 + done + echo "Failed to create infrastructure after $attempt attempts" + exit 1 + working-directory: ${{ github.workspace }}/src/kayobe-config/terraform/host-image-builder + env: + OS_CLOUD: "openstack" + OS_APPLICATION_CREDENTIAL_ID: ${{ secrets.OS_APPLICATION_CREDENTIAL_ID }} + OS_APPLICATION_CREDENTIAL_SECRET: ${{ secrets.OS_APPLICATION_CREDENTIAL_SECRET }} + + - name: Get Terraform outputs + id: tf_outputs + run: | + terraform output -json + working-directory: ${{ github.workspace }}/src/kayobe-config/terraform/host-image-builder + + - name: Write Terraform outputs + run: | + cat << EOF > src/kayobe-config/etc/kayobe/environments/ci-builder/tf-outputs.yml + ${{ steps.tf_outputs.outputs.stdout }} + EOF + + - name: Write Terraform network config + run: | + cat << EOF > src/kayobe-config/etc/kayobe/environments/ci-builder/tf-network-allocation.yml + --- + aio_ips: + builder: "{{ access_ip_v4.value }}" + EOF + + - name: Write Terraform network interface config + run: | + mkdir -p etc/kayobe/environments/$KAYOBE_ENVIRONMENT/inventory/group_vars/seed + cat << EOF > etc/kayobe/environments/$KAYOBE_ENVIRONMENT/inventory/group_vars/seed/tf-network-interfaces + admin_interface: "{{ access_interface.value }}" + EOF + + - name: Copy SSH keys + run: | + mkdir -p ~/.ssh + touch ~/.ssh/authorized_keys + cat src/kayobe-config/terraform/host-image-builder/id_rsa.pub >> ~/.ssh/authorized_keys + cp src/kayobe-config/terraform/host-image-builder/id_rsa* ~/.ssh/ + + - name: Update inventory + run: | + cat > src/kayobe-config/etc/kayobe/environments/ci-builder/inventory/hosts << EOF + [seed] + builder + EOF + + - name: Bootstrap the control host + run: | + source venvs/kayobe/bin/activate && + source src/kayobe-config/kayobe-env --environment ci-builder && + kayobe control host bootstrap + + - name: Configure the seed host (Builder VM) + continue-on-error: true + run: | + source venvs/kayobe/bin/activate && + source src/kayobe-config/kayobe-env --environment ci-builder && + kayobe seed host configure -e seed_bootstrap_user=rocky --skip-tags network - - name: Install OpenStack client + - name: Install dependencies + continue-on-error: true run: | source venvs/kayobe/bin/activate && - pip install python-openstackclient -c https://releases.openstack.org/constraints/upper/${{ steps.openstack_release.outputs.openstack_release }} + source src/kayobe-config/kayobe-env --environment ci-builder && + kayobe seed host command run \ + --command "sudo dnf config-manager --set-enabled crb && sudo dnf -y install epel-release && sudo dnf -y install zstd debootstrap kpartx cloud-init" --show-output + env: + KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }} - - name: Testo + - name: Create bifrost_httpboot Docker volume + continue-on-error: true run: | - mkdir -p /tmp - mkdir -p /dev + source venvs/kayobe/bin/activate && + source src/kayobe-config/kayobe-env --environment ci-builder && + kayobe seed host command run --command "sudo mkdir -p /var/lib/docker/volumes/bifrost_httpboot/_data" --show-output + env: + KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }} - name: Build a CentOS Stream 8 overcloud host image id: build_centos_stream_8 @@ -122,7 +227,16 @@ jobs: -e stackhpc_overcloud_dib_name=overcloud-centos-8-stream env: KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }} - if: inputs.centos + + - name: Show last error logs + continue-on-error: true + run: | + source venvs/kayobe/bin/activate && + source src/kayobe-config/kayobe-env --environment ci-builder && + kayobe seed host command run --command "tail -200 /opt/kayobe/images/overcloud-centos-8-stream/overcloud-centos-8-stream.stdout" --show-output + env: + KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }} + if: steps.build_centos_stream_8.outcome == 'failure' - name: Upload CentOS Stream 8 overcloud host image to Ark run: | @@ -138,22 +252,23 @@ jobs: KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }} if: inputs.centos && steps.build_centos_stream_8.outcome == 'success' - - name: Upload CentOS Stream 8 overcloud host image to Cloud + - name: Upload CentOS Stream 8 overcloud host image to Dev Cloud run: | source venvs/kayobe/bin/activate && - openstack image create \ - overcloud-centos-8-stream-${{ steps.host_image_tag.outputs.host_image_tag }} \ - --container-format bare \ - --disk-format qcow2 \ - --file /opt/kayobe/images/overcloud-centos-8-stream/overcloud-centos-8-stream.qcow2 \ - --shared \ - --os-cloud openstack \ - --progress + source src/kayobe-config/kayobe-env --environment ci-builder && + kayobe playbook run \ + src/kayobe-config/etc/kayobe/ansible/openstack-host-image-upload.yml \ + -e local_image_path="/opt/kayobe/images/overcloud-centos-8-stream/overcloud-centos-8-stream.qcow2" + -e image_name=overcloud-centos-8-stream-${{ steps.host_image_tag.outputs.host_image_tag }} env: + CLOUDS_YAML: ${{ secrets.CLOUDS_YAML }} OS_APPLICATION_CREDENTIAL_ID: ${{ secrets.OS_APPLICATION_CREDENTIAL_ID }} OS_APPLICATION_CREDENTIAL_SECRET: ${{ secrets.OS_APPLICATION_CREDENTIAL_SECRET }} if: inputs.centos && steps.build_centos_stream_8.outcome == 'success' + - name: exit 1 + run: exit 1 + - name: Build a Rocky Linux 8 overcloud host image id: build_rocky_8 continue-on-error: true @@ -168,6 +283,16 @@ jobs: KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }} if: inputs.rocky8 + - name: Show last error logs + continue-on-error: true + run: | + source venvs/kayobe/bin/activate && + source src/kayobe-config/kayobe-env --environment ci-builder && + kayobe seed host command run --command "tail -200 /opt/kayobe/images/overcloud-rocky-8/overcloud-rocky-8.stdout" --show-output + env: + KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }} + if: steps.build_rocky_8.outcome == 'failure' + - name: Upload Rocky Linux 8 overcloud host image to Ark run: | source venvs/kayobe/bin/activate && @@ -182,21 +307,21 @@ jobs: KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }} if: inputs.rocky8 && steps.build_rocky_8.outcome == 'success' - - name: Upload Rocky Linux 8 overcloud host image to Cloud - run: | - source venvs/kayobe/bin/activate && - openstack image create \ - overcloud-rocky-8-${{ steps.host_image_tag.outputs.host_image_tag }} \ - --container-format bare \ - --disk-format qcow2 \ - --file /opt/kayobe/images/overcloud-rocky-8/overcloud-rocky-8.qcow2 \ - --shared \ - --os-cloud openstack \ - --progress - env: - OS_APPLICATION_CREDENTIAL_ID: ${{ secrets.OS_APPLICATION_CREDENTIAL_ID }} - OS_APPLICATION_CREDENTIAL_SECRET: ${{ secrets.OS_APPLICATION_CREDENTIAL_SECRET }} - if: inputs.rocky8 && steps.build_rocky_8.outcome == 'success' + # - name: Upload Rocky Linux 8 overcloud host image to Dev Cloud + # run: | + # source venvs/kayobe/bin/activate && + # openstack image create \ + # overcloud-rocky-8-${{ steps.host_image_tag.outputs.host_image_tag }} \ + # --container-format bare \ + # --disk-format qcow2 \ + # --file /opt/kayobe/images/overcloud-rocky-8/overcloud-rocky-8.qcow2 \ + # --shared \ + # --os-cloud openstack \ + # --progress + # env: + # OS_APPLICATION_CREDENTIAL_ID: ${{ secrets.OS_APPLICATION_CREDENTIAL_ID }} + # OS_APPLICATION_CREDENTIAL_SECRET: ${{ secrets.OS_APPLICATION_CREDENTIAL_SECRET }} + # if: inputs.rocky8 && steps.build_rocky_8.outcome == 'success' - name: Build a Rocky Linux 9 overcloud host image id: build_rocky_9 @@ -212,6 +337,16 @@ jobs: KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }} if: inputs.rocky9 + - name: Show last error logs + continue-on-error: true + run: | + source venvs/kayobe/bin/activate && + source src/kayobe-config/kayobe-env --environment ci-builder && + kayobe seed host command run --command "tail -200 /opt/kayobe/images/overcloud-rocky-9/overcloud-rocky-9.stdout" --show-output + env: + KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }} + if: steps.build_rocky_9.outcome == 'failure' + - name: Upload Rocky Linux 9 overcloud host image to Ark run: | source venvs/kayobe/bin/activate && @@ -226,21 +361,21 @@ jobs: KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }} if: inputs.rocky9 && steps.build_rocky_9.outcome == 'success' - - name: Upload Rocky Linux 9 overcloud host image to Cloud - run: | - source venvs/kayobe/bin/activate && - openstack image create \ - overcloud-rocky-9-${{ steps.host_image_tag.outputs.host_image_tag }} \ - --container-format bare \ - --disk-format qcow2 \ - --file /opt/kayobe/images/overcloud-rocky-9/overcloud-rocky-9.qcow2 \ - --shared \ - --os-cloud openstack \ - --progress - env: - OS_APPLICATION_CREDENTIAL_ID: ${{ secrets.OS_APPLICATION_CREDENTIAL_ID }} - OS_APPLICATION_CREDENTIAL_SECRET: ${{ secrets.OS_APPLICATION_CREDENTIAL_SECRET }} - if: inputs.rocky9 && steps.build_rocky_9.outcome == 'success' + # - name: Upload Rocky Linux 9 overcloud host image to Dev Cloud + # run: | + # source venvs/kayobe/bin/activate && + # openstack image create \ + # overcloud-rocky-9-${{ steps.host_image_tag.outputs.host_image_tag }} \ + # --container-format bare \ + # --disk-format qcow2 \ + # --file /opt/kayobe/images/overcloud-rocky-9/overcloud-rocky-9.qcow2 \ + # --shared \ + # --os-cloud openstack \ + # --progress + # env: + # OS_APPLICATION_CREDENTIAL_ID: ${{ secrets.OS_APPLICATION_CREDENTIAL_ID }} + # OS_APPLICATION_CREDENTIAL_SECRET: ${{ secrets.OS_APPLICATION_CREDENTIAL_SECRET }} + # if: inputs.rocky9 && steps.build_rocky_9.outcome == 'success' - name: Build an Ubuntu Focal 20.04 overcloud host image id: build_ubuntu_focal @@ -256,6 +391,16 @@ jobs: KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }} if: inputs.ubuntu-focal + - name: Show last error logs + continue-on-error: true + run: | + source venvs/kayobe/bin/activate && + source src/kayobe-config/kayobe-env --environment ci-builder && + kayobe seed host command run --command "tail -200 /opt/kayobe/images/overcloud-ubuntu-focal/overcloud-ubuntu-focal.stdout" --show-output + env: + KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }} + if: steps.build_ubuntu_focal.outcome == 'failure' + - name: Upload Ubuntu Focal 20.04 overcloud host image to Ark run: | source venvs/kayobe/bin/activate && @@ -270,21 +415,21 @@ jobs: KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }} if: inputs.ubuntu-focal && steps.build_ubuntu_focal.outcome == 'success' - - name: Upload Ubuntu Focal 20.04 overcloud host image to Cloud - run: | - source venvs/kayobe/bin/activate && - openstack image create \ - overcloud-ubuntu-focal-${{ steps.host_image_tag.outputs.host_image_tag }} \ - --container-format bare \ - --disk-format qcow2 \ - --file /opt/kayobe/images/overcloud-ubuntu-focal/overcloud-ubuntu-focal.qcow2 \ - --shared \ - --os-cloud openstack \ - --progress - env: - OS_APPLICATION_CREDENTIAL_ID: ${{ secrets.OS_APPLICATION_CREDENTIAL_ID }} - OS_APPLICATION_CREDENTIAL_SECRET: ${{ secrets.OS_APPLICATION_CREDENTIAL_SECRET }} - if: inputs.ubuntu-focal && steps.build_ubuntu_focal.outcome == 'success' + # - name: Upload Ubuntu Focal 20.04 overcloud host image to Dev Cloud + # run: | + # source venvs/kayobe/bin/activate && + # openstack image create \ + # overcloud-ubuntu-focal-${{ steps.host_image_tag.outputs.host_image_tag }} \ + # --container-format bare \ + # --disk-format qcow2 \ + # --file /opt/kayobe/images/overcloud-ubuntu-focal/overcloud-ubuntu-focal.qcow2 \ + # --shared \ + # --os-cloud openstack \ + # --progress + # env: + # OS_APPLICATION_CREDENTIAL_ID: ${{ secrets.OS_APPLICATION_CREDENTIAL_ID }} + # OS_APPLICATION_CREDENTIAL_SECRET: ${{ secrets.OS_APPLICATION_CREDENTIAL_SECRET }} + # if: inputs.ubuntu-focal && steps.build_ubuntu_focal.outcome == 'success' - name: Build an Ubuntu Jammy 22.04 overcloud host image id: build_ubuntu_jammy @@ -300,6 +445,16 @@ jobs: KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }} if: inputs.ubuntu-jammy + - name: Show last error logs + continue-on-error: true + run: | + source venvs/kayobe/bin/activate && + source src/kayobe-config/kayobe-env --environment ci-builder && + kayobe seed host command run --command "tail -200 /opt/kayobe/images/overcloud-ubuntu-jammy/overcloud-ubuntu-jammy.stdout" --show-output + env: + KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }} + if: steps.build_ubuntu_jammy.outcome == 'failure' + - name: Upload Ubuntu Jammy 22.04 overcloud host image to Ark run: | source venvs/kayobe/bin/activate && @@ -314,21 +469,29 @@ jobs: KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }} if: inputs.ubuntu-jammy && steps.build_ubuntu_jammy.outcome == 'success' - - name: Upload Ubuntu Jammy 22.04 overcloud host image to Cloud + # - name: Upload Ubuntu Jammy 22.04 overcloud host image to Dev Cloud + # run: | + # source venvs/kayobe/bin/activate && + # openstack image create \ + # overcloud-ubuntu-jammy-${{ steps.host_image_tag.outputs.host_image_tag }} \ + # --container-format bare \ + # --disk-format qcow2 \ + # --file /opt/kayobe/images/overcloud-ubuntu-jammy/overcloud-ubuntu-jammy.qcow2 \ + # --shared \ + # --os-cloud openstack \ + # --progress + # env: + # OS_APPLICATION_CREDENTIAL_ID: ${{ secrets.OS_APPLICATION_CREDENTIAL_ID }} + # OS_APPLICATION_CREDENTIAL_SECRET: ${{ secrets.OS_APPLICATION_CREDENTIAL_SECRET }} + # if: inputs.ubuntu-jammy && steps.build_ubuntu_jammy.outcome == 'success' + + - name: Copy logs back + continue-on-error: true run: | - source venvs/kayobe/bin/activate && - openstack image create \ - overcloud-ubuntu-jammy-${{ steps.host_image_tag.outputs.host_image_tag }} \ - --container-format bare \ - --disk-format qcow2 \ - --file /opt/kayobe/images/overcloud-ubuntu-jammy/overcloud-ubuntu-jammy.qcow2 \ - --shared \ - --os-cloud openstack \ - --progress - env: - OS_APPLICATION_CREDENTIAL_ID: ${{ secrets.OS_APPLICATION_CREDENTIAL_ID }} - OS_APPLICATION_CREDENTIAL_SECRET: ${{ secrets.OS_APPLICATION_CREDENTIAL_SECRET }} - if: inputs.ubuntu-jammy && steps.build_ubuntu_jammy.outcome == 'success' + mkdir logs + scp -r rocky@$(jq -r .access_ip_v4.value src/kayobe-config/etc/kayobe/environments/ci-builder/tf-outputs.yml):/opt/kayobe/images/*/*.std* ./logs/ + scp -r rocky@$(jq -r .access_ip_v4.value src/kayobe-config/etc/kayobe/environments/ci-builder/tf-outputs.yml):updated_images.txt ./logs/ + if: always() - name: Upload updated images artifact uses: actions/upload-artifact@v4 @@ -342,56 +505,6 @@ jobs: steps.build_ubuntu_focal.outcome == 'success' || steps.build_ubuntu_jammy.outcome == 'success' - - name: Upload CentOS build logs if build failed - uses: actions/upload-artifact@v4 - with: - name: CentOS build logs - path: | - /opt/kayobe/images/overcloud-centos-8-stream/overcloud-centos-8-stream.stdout - /opt/kayobe/images/overcloud-centos-8-stream/overcloud-centos-8-stream.stderr - retention-days: 7 - if: steps.build_centos_stream_8.outcome == 'failure' - - - name: Upload Rocky 8 build logs if build failed - uses: actions/upload-artifact@v4 - with: - name: Rocky 8 build logs - path: | - /opt/kayobe/images/overcloud-rocky-8/overcloud-rocky-8.stdout - /opt/kayobe/images/overcloud-rocky-8/overcloud-rocky-8.stderr - retention-days: 7 - if: steps.build_rocky_8.outcome == 'failure' - - - name: Upload Rocky 9 build logs if build failed - uses: actions/upload-artifact@v4 - with: - name: Rocky 9 build logs - path: | - /opt/kayobe/images/overcloud-rocky-9/overcloud-rocky-9.stdout - /opt/kayobe/images/overcloud-rocky-9/overcloud-rocky-9.stderr - retention-days: 7 - if: steps.build_rocky_9.outcome == 'failure' - - - name: Upload Ubuntu Focal 20.04 build logs if build failed - uses: actions/upload-artifact@v4 - with: - name: Ubuntu Focal 20.04 build logs - path: | - /opt/kayobe/images/overcloud-ubuntu-focal/overcloud-ubuntu-focal.stdout - /opt/kayobe/images/overcloud-ubuntu-focal/overcloud-ubuntu-focal.stderr - retention-days: 7 - if: steps.build_ubuntu_focal.outcome == 'failure' - - - name: Upload Ubuntu Jammy 22.04 build logs if build failed - uses: actions/upload-artifact@v4 - with: - name: Ubuntu Jammy 22.04 build logs - path: | - /opt/kayobe/images/overcloud-ubuntu-jammy/overcloud-ubuntu-jammy.stdout - /opt/kayobe/images/overcloud-ubuntu-jammy/overcloud-ubuntu-jammy.stderr - retention-days: 7 - if: steps.build_ubuntu_jammy.outcome == 'failure' - - name: Fail if any overcloud host image builds failed run: | echo "Builds failed. See workflow artifacts for details." && @@ -401,3 +514,18 @@ jobs: steps.build_rocky_9.outcome == 'failure' || steps.build_ubuntu_focal.outcome == 'failure' || steps.build_ubuntu_jammy.outcome == 'failure' + + - name: Upload logs artifact + uses: actions/upload-artifact@v4 + with: + name: Build logs + path: ./logs + + - name: Destroy + run: terraform destroy -auto-approve + working-directory: ${{ github.workspace }}/src/kayobe-config/terraform/host-image-builder + env: + OS_CLOUD: openstack + OS_APPLICATION_CREDENTIAL_ID: ${{ secrets.OS_APPLICATION_CREDENTIAL_ID }} + OS_APPLICATION_CREDENTIAL_SECRET: ${{ secrets.OS_APPLICATION_CREDENTIAL_SECRET }} + if: always() \ No newline at end of file diff --git a/etc/kayobe/ansible/openstack-host-image-upload.yml b/etc/kayobe/ansible/openstack-host-image-upload.yml new file mode 100644 index 000000000..312dba6f1 --- /dev/null +++ b/etc/kayobe/ansible/openstack-host-image-upload.yml @@ -0,0 +1,19 @@ +--- +- name: Upload and create a distribution for an image + hosts: seed + vars: + local_image_path: "/opt/kayobe/images/overcloud-{{ os_distribution }}-{{ os_release }}/overcloud-{{ os_distribution }}-{{ os_release }}.qcow2" + image_name: "overcloud-{{ os_distribution }}-{{ os_release }}" + tasks: + + - name: Upload an image to Glance + openstack.cloud.image: + cloud: "{{ lookup('env', 'CLOUDS_YAML')['openstack'] }}" + name: "{{ image_name }}" + container_format: bare + disk_format: qcow2 + state: present + filename: "{{ local_image_path }}" + environment: + OS_APPLICATION_CREDENTIAL_ID: lookup('ansible.builtin.env', 'OS_APPLICATION_CREDENTIAL_ID') }}' + OS_APPLICATION_CREDENTIAL_SECRET: lookup('ansible.builtin.env', 'OS_APPLICATION_CREDENTIAL_SECRET') }}' \ No newline at end of file diff --git a/etc/kayobe/ansible/pulp-host-image-upload.yml b/etc/kayobe/ansible/pulp-host-image-upload.yml index a06897d90..d3a44f133 100644 --- a/etc/kayobe/ansible/pulp-host-image-upload.yml +++ b/etc/kayobe/ansible/pulp-host-image-upload.yml @@ -1,12 +1,12 @@ --- - name: Upload and create a distribution for an image - hosts: localhost + hosts: seed vars: remote_pulp_url: "{{ stackhpc_release_pulp_url }}" remote_pulp_username: "{{ stackhpc_image_repository_username }}" remote_pulp_password: "{{ stackhpc_image_repository_password }}" repository_name: "kayobe-images-{{ openstack_release }}-{{ os_distribution }}-{{ os_release }}" - base_path: "kayobe-images/{{ openstack_release }}/{{ os_distribution }}/{{ os_release }}" + pulp_base_path: "kayobe-images/{{ openstack_release }}/{{ os_distribution }}/{{ os_release }}" tasks: - name: Print image tag debug: @@ -74,7 +74,7 @@ username: "{{ remote_pulp_username }}" password: "{{ remote_pulp_password }}" name: "{{ repository_name }}_latest" - base_path: "{{ base_path }}/latest" + base_path: "{{ pulp_base_path }}/latest" publication: "{{ publication_details.publication.pulp_href }}" content_guard: development state: present @@ -86,7 +86,7 @@ username: "{{ remote_pulp_username }}" password: "{{ remote_pulp_password }}" name: "{{ repository_name }}_{{ host_image_tag }}" - base_path: "{{ base_path }}/{{ host_image_tag }}" + base_path: "{{ pulp_base_path }}/{{ host_image_tag }}" publication: "{{ publication_details.publication.pulp_href }}" content_guard: development state: present @@ -95,26 +95,26 @@ - name: Update new images file with versioned path lineinfile: path: /tmp/updated_images.txt - line: "{{ remote_pulp_url }}/pulp/content/{{ base_path }}/\ + line: "{{ remote_pulp_url }}/pulp/content/{{ pulp_base_path }}/\ {{ host_image_tag }}/{{ found_files.files[0].path | basename }}" create: true - name: Update new images file with latest path lineinfile: path: /tmp/updated_images.txt - line: "{{ remote_pulp_url }}/pulp/content/{{ base_path }}/\ + line: "{{ remote_pulp_url }}/pulp/content/{{ pulp_base_path }}/\ latest/{{ found_files.files[0].path | basename }}" when: latest_distribution_details.changed - name: Print versioned path debug: - msg: "New versioned path: {{ remote_pulp_url }}/pulp/content/{{ base_path }}/\ + msg: "New versioned path: {{ remote_pulp_url }}/pulp/content/{{ pulp_base_path }}/\ {{ host_image_tag }}/{{ found_files.files[0].path | basename }}" when: latest_distribution_details.changed - name: Print latest path debug: - msg: "New latest path: {{ remote_pulp_url }}/pulp/content/{{ base_path }}/\ + msg: "New latest path: {{ remote_pulp_url }}/pulp/content/{{ pulp_base_path }}/\ latest/{{ found_files.files[0].path | basename }}" when: latest_distribution_details.changed diff --git a/etc/kayobe/environments/ci-aio/inventory/group_vars/controllers/network-interfaces b/etc/kayobe/environments/ci-aio/inventory/group_vars/controllers/network-interfaces index 2f8d30103..4f9ce15c6 100644 --- a/etc/kayobe/environments/ci-aio/inventory/group_vars/controllers/network-interfaces +++ b/etc/kayobe/environments/ci-aio/inventory/group_vars/controllers/network-interfaces @@ -3,9 +3,9 @@ # Network interface definitions for the controller group. # Controller interface on all-in-one network. -aio_interface: breth1 +# aio_interface: breth1 # Use dummy1 if it exists, otherwise the bridge will have no ports. -aio_bridge_ports: "{{ ['dummy1'] if ('dummy1' in hostvars[inventory_hostname].ansible_facts) or (os_distribution == 'rocky' and os_release == '9') else [] }}" +# aio_bridge_ports: "{{ ['dummy1'] if ('dummy1' in hostvars[inventory_hostname].ansible_facts) or (os_distribution == 'rocky' and os_release == '9') else [] }}" ############################################################################### # Dummy variable to allow Ansible to accept this file. diff --git a/etc/kayobe/environments/ci-builder/inventory/group_vars/seed/network-interfaces b/etc/kayobe/environments/ci-builder/inventory/group_vars/seed/network-interfaces index a583b7214..c143fe2c9 100644 --- a/etc/kayobe/environments/ci-builder/inventory/group_vars/seed/network-interfaces +++ b/etc/kayobe/environments/ci-builder/inventory/group_vars/seed/network-interfaces @@ -2,12 +2,5 @@ ############################################################################### # Network interface definitions for the controller group. -# Controller interface on all-in-one network. -aio_interface: breth1 -# Use dummy1 if it exists, otherwise the bridge will have no ports. -aio_bridge_ports: "{{ ['dummy1'] if 'dummy1' in hostvars[inventory_hostname].ansible_facts else [] }}" - -############################################################################### -# Dummy variable to allow Ansible to accept this file. -workaround_ansible_issue_8743: yes - +# Seed interface on all-in-one network. +aio_interface: eth0 diff --git a/etc/kayobe/overcloud-dib.yml b/etc/kayobe/overcloud-dib.yml index 8f59d58ef..d7f6dbd69 100644 --- a/etc/kayobe/overcloud-dib.yml +++ b/etc/kayobe/overcloud-dib.yml @@ -71,7 +71,7 @@ overcloud_dib_host_packages_extra: overcloud_dib_git_elements_extra: - repo: "https://github.com/stackhpc/stackhpc-image-elements" local: "{{ source_checkout_path }}/stackhpc-image-elements" - version: "v1.6.0" + version: "v1.6.1" elements_path: "elements" # List of git repositories containing Diskimage Builder (DIB) elements. See diff --git a/terraform/host-image-builder/README.rst b/terraform/host-image-builder/README.rst new file mode 100644 index 000000000..908b7af15 --- /dev/null +++ b/terraform/host-image-builder/README.rst @@ -0,0 +1,191 @@ +========================== +Terraform All in one (aio) +========================== + +This Terraform configuration deploys a single VM on an OpenStack cloud, to be +used as an all-in-one Kayobe test environment. + +This configuration is used in the GitHub Actions all-in-one.yml workflow for CI +testing. + +Usage +===== + +These instructions show how to use this Terraform configuration manually. They +assume you are running an Ubuntu host that will be used to run Terraform. The +machine should have network access to the VM that will be created by this +configuration. + +Install Terraform: + +.. code-block:: console + + wget -qO - terraform.gpg https://apt.releases.hashicorp.com/gpg | sudo gpg --dearmor -o /usr/share/keyrings/terraform-archive-keyring.gpg + sudo echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/terraform-archive-keyring.gpg] https://apt.releases.hashicorp.com $(lsb_release -cs) main" | sudo tee /etc/apt/sources.list.d/terraform.list + sudo apt update + sudo apt install docker.io terraform + +Clone and initialise the Kayobe config: + +.. code-block:: console + + git clone https://github.com/stackhpc/stackhpc-kayobe-config + cd stackhpc-kayobe-config + git submodule init + git submodule update + +Change to the terraform/aio directory: + +.. code-block:: console + + cd terraform/aio + +Initialise Terraform: + +.. code-block:: console + + terraform init + +Generate an SSH keypair: + +.. code-block:: console + + ssh-keygen -f id_rsa -N '' + +Create an OpenStack clouds.yaml file with your credentials to access an +OpenStack cloud. Alternatively, download one from Horizon. + +.. code-block:: console + + cat << EOF > clouds.yaml + --- + clouds: + sms-lab: + auth: + auth_url: https://api.sms-lab.cloud:5000 + username: + project_name: + domain_name: default + interface: public + EOF + +Export environment variables to use the correct cloud and provide a password: + +.. code-block:: console + + export OS_CLOUD=sms-lab + read -p OS_PASSWORD -s OS_PASSWORD + export OS_PASSWORD + +Generate Terraform variables: + +.. code-block:: console + + cat << EOF > terraform.tfvars + ssh_public_key = "id_rsa.pub" + aio_vm_name = "kayobe-aio" + aio_vm_image = "overcloud-centos-8-stream-yoga-20230525T095243" + aio_vm_flavor = "general.v1.medium" + aio_vm_network = "stackhpc-ipv4-geneve" + aio_vm_subnet = "stackhpc-ipv4-geneve-subnet" + EOF + +Generate a plan: + +.. code-block:: console + + terraform plan + +Apply the changes: + +.. code-block:: console + + terraform apply -auto-approve + +Write Terraform outputs to a Kayobe config file: + +.. code-block:: console + + terraform output -json > ../../etc/kayobe/environments/$KAYOBE_ENVIRONMENT/tf-outputs.yml + +Change to the repository root: + +.. code-block:: console + + cd ../../ + +Write Terraform network config: + +.. code-block:: console + + cat << EOF > etc/kayobe/environments/$KAYOBE_ENVIRONMENT/tf-networks.yml + + admin_oc_net_name: admin + admin_cidr: "{{ access_cidr.value }}" + admin_allocation_pool_start: 0.0.0.0 + admin_allocation_pool_end: 0.0.0.0 + admin_gateway: "{{ access_gw.value }}" + admin_bootproto: dhcp + admin_ips: + controller0: "{{ access_ip_v4.value }}" + EOF + +Write Terraform network interface config: + +.. code-block:: console + + cat << EOF > etc/kayobe/environments/$KAYOBE_ENVIRONMENT/inventory/group_vars/controllers/tf-network-interfaces + admin_interface: "{{ access_interface.value }}" + EOF + +Build a Kayobe image: + +.. code-block:: console + + sudo DOCKER_BUILDKIT=1 docker build --file .automation/docker/kayobe/Dockerfile --tag kayobe:latest . + +Use the ci-aio environment: + +.. code-block:: console + + export KAYOBE_ENVIRONMENT=ci-aio + +Set the Kayobe Vault password env var: + +.. code-block:: console + + read -p KAYOBE_VAULT_PASSWORD -s KAYOBE_VAULT_PASSWORD + export KAYOBE_VAULT_PASSWORD + +Set the Kayobe SSH private key env var: + +.. code-block:: console + + export KAYOBE_AUTOMATION_SSH_PRIVATE_KEY=$(cat terraform/aio/id_rsa) + +Host configure: + +.. code-block:: console + + sudo -E docker run -it --rm -v $(pwd):/stack/kayobe-automation-env/src/kayobe-config -e KAYOBE_ENVIRONMENT -e KAYOBE_VAULT_PASSWORD -e KAYOBE_AUTOMATION_SSH_PRIVATE_KEY kayobe:latest /stack/kayobe-automation-env/src/kayobe-config/.automation/pipeline/overcloud-host-configure.sh + +Service deploy: + +.. code-block:: console + + sudo -E docker run -it --rm -v $(pwd):/stack/kayobe-automation-env/src/kayobe-config -e KAYOBE_ENVIRONMENT -e KAYOBE_VAULT_PASSWORD -e KAYOBE_AUTOMATION_SSH_PRIVATE_KEY kayobe:latest /stack/kayobe-automation-env/src/kayobe-config/.automation/pipeline/overcloud-service-deploy.sh + +Configure aio resources: + +.. code-block:: console + + sudo -E docker run -it --rm -v $(pwd):/stack/kayobe-automation-env/src/kayobe-config -e KAYOBE_ENVIRONMENT -e KAYOBE_VAULT_PASSWORD -e KAYOBE_AUTOMATION_SSH_PRIVATE_KEY kayobe:latest /stack/kayobe-automation-env/src/kayobe-config/.automation/pipeline/playbook-run.sh etc/kayobe/ansible/configure-aio-resources.yml + +Run Tempest: + +.. code-block:: console + + mkdir -p tempest-artifacts + sudo -E docker run -it --rm -v $(pwd):/stack/kayobe-automation-env/src/kayobe-config -v $(pwd)/tempest-artifacts:/stack/tempest-artifacts -e KAYOBE_ENVIRONMENT -e KAYOBE_VAULT_PASSWORD -e KAYOBE_AUTOMATION_SSH_PRIVATE_KEY kayobe:latest /stack/kayobe-automation-env/src/kayobe-config/.automation/pipeline/tempest.sh -e ansible_user=stack + +Tempest results are in tempest-artifacts. diff --git a/terraform/host-image-builder/outputs.tf b/terraform/host-image-builder/outputs.tf new file mode 100644 index 000000000..9b01cfd86 --- /dev/null +++ b/terraform/host-image-builder/outputs.tf @@ -0,0 +1,15 @@ +output "access_ip_v4" { + value = openstack_compute_instance_v2.kayobe-host-image-builder.access_ip_v4 +} + +output "access_cidr" { + value = data.openstack_networking_subnet_v2.network.cidr +} + +output "access_gw" { + value = data.openstack_networking_subnet_v2.network.gateway_ip +} + +output "access_interface" { + value = var.host_image_builder_interface +} diff --git a/terraform/host-image-builder/provider.tf b/terraform/host-image-builder/provider.tf new file mode 100644 index 000000000..0e7cbdb05 --- /dev/null +++ b/terraform/host-image-builder/provider.tf @@ -0,0 +1,14 @@ +#provider "openstack" { +# use environment variables +#} + +terraform { + required_version = ">= 0.14" + backend "local" { + } + required_providers { + openstack = { + source = "terraform-provider-openstack/openstack" + } + } +} diff --git a/terraform/host-image-builder/templates/userdata.cfg.tpl b/terraform/host-image-builder/templates/userdata.cfg.tpl new file mode 100644 index 000000000..aebd19229 --- /dev/null +++ b/terraform/host-image-builder/templates/userdata.cfg.tpl @@ -0,0 +1,21 @@ +#cloud-config +# Don't automatically mount ephemeral disk +mounts: + - [/dev/vdb, null] +# WORKAROUND: internal DNS missing from SMS lab. +runcmd: + - 'echo "10.0.0.34 pelican pelican.service.compute.sms-lab.cloud" >> /etc/hosts' + - 'echo "10.205.3.187 pulp-server pulp-server.internal.sms-cloud" >> /etc/hosts' +# Configure SSH keys here, to avoid creating an ephemeral keypair. +# This means only the instance needs to be cleaned up if the destroy fails. +ssh_authorized_keys: + - ${ssh_public_key} + +write_files: + # WORKAROUND: https://bugs.launchpad.net/kolla-ansible/+bug/1995409 + - content: | + #!/bin/bash + docker exec openvswitch_vswitchd ovs-vsctl "$@" + owner: root:root + path: /usr/bin/ovs-vsctl + permissions: '0755' diff --git a/terraform/host-image-builder/terraform.tfvars b/terraform/host-image-builder/terraform.tfvars new file mode 100644 index 000000000..614e2eeb3 --- /dev/null +++ b/terraform/host-image-builder/terraform.tfvars @@ -0,0 +1,9 @@ +ssh_public_key = "id_rsa.pub" +ssh_username = "rocky" +host_image_builder_name = "skc-ci-host-image-builder" +host_image_builder_image = "Rocky-9-GenericCloud-Base-9.3-20231113.0.x86_64.qcow2" +host_image_builder_flavor = "en1.medium" +host_image_builder_network = "stackhpc-ci" +host_image_builder_subnet = "stackhpc-ci" +host_image_builder_tags = ["skc-ci-host-image-builder", "alex-testo"] +host_image_builder_interface = "eth0" \ No newline at end of file diff --git a/terraform/host-image-builder/vm.tf b/terraform/host-image-builder/vm.tf new file mode 100644 index 000000000..ec938ad33 --- /dev/null +++ b/terraform/host-image-builder/vm.tf @@ -0,0 +1,98 @@ +variable "ssh_public_key" { + type = string +} + +variable "ssh_username" { + type = string +} + +variable "host_image_builder_name" { + type = string + default = "kayobe-host-image-builder" +} + +variable "host_image_builder_image" { + type = string + default = "CentOS-stream8" +} + +variable "host_image_builder_interface" { + type = string + default = "eth0" +} + +variable "host_image_builder_flavor" { + type = string +} + +variable "host_image_builder_network" { + type = string +} + +variable "host_image_builder_subnet" { + type = string +} + +variable "host_image_builder_volume_size" { + type = number + default = 150 +} + +variable "host_image_builder_tags" { + type = list(string) + default = [] +} + +locals { + image_is_uuid = length(regexall("^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$", var.host_image_builder_image)) > 0 +} + +data "openstack_images_image_v2" "image" { + name = var.host_image_builder_image + most_recent = true + count = local.image_is_uuid ? 0 : 1 +} + +data "openstack_networking_subnet_v2" "network" { + name = var.host_image_builder_subnet +} + +resource "openstack_compute_instance_v2" "kayobe-host-image-builder" { + name = var.host_image_builder_name + flavor_name = "en1.medium" + config_drive = true + user_data = templatefile("templates/userdata.cfg.tpl", {ssh_public_key = file(var.ssh_public_key)}) + network { + name = var.host_image_builder_network + } + + block_device { + uuid = local.image_is_uuid ? var.host_image_builder_image: data.openstack_images_image_v2.image[0].id + source_type = "image" + volume_size = var.host_image_builder_volume_size + boot_index = 0 + destination_type = "volume" + delete_on_termination = true + } + + tags = var.host_image_builder_tags +} + +# Wait for the instance to be accessible via SSH before progressing. +resource "null_resource" "kayobe-host-image-builder" { + provisioner "remote-exec" { + connection { + host = openstack_compute_instance_v2.kayobe-host-image-builder.access_ip_v4 + user = var.ssh_username + private_key = file("id_rsa") + # Terraform will run the start script from /tmp by default. For the + # current images, /tmp is noexec, so the path must be changed + script_path = "/home/${var.ssh_username}/start.sh" + } + + inline = [ + "#!/bin/sh", + "echo 'connected!'" + ] + } +}