testo

Change-Id: If1750f2495168e0f2d12747c7d907302fde7c6f9

etc/kayobe/ansible/openstack-host-image-upload.yml

@@ -0,0 +1,19 @@
+---
+- name: Upload and create a distribution for an image
+  hosts: seed
+  vars:
+    local_image_path: "/opt/kayobe/images/overcloud-{{ os_distribution }}-{{ os_release }}/overcloud-{{ os_distribution }}-{{ os_release }}.qcow2"
+    image_name: "overcloud-{{ os_distribution }}-{{ os_release }}"
+  tasks:
+
+    - name: Upload an image to Glance
+      openstack.cloud.image:
+        cloud: "{{ lookup('env', 'CLOUDS_YAML')['openstack'] }}"
+        name: "{{ image_name }}"
+        container_format: bare
+        disk_format: qcow2
+        state: present
+        filename: "{{ local_image_path }}"
+      environment:
+        OS_APPLICATION_CREDENTIAL_ID: lookup('ansible.builtin.env', 'OS_APPLICATION_CREDENTIAL_ID') }}'
+        OS_APPLICATION_CREDENTIAL_SECRET: lookup('ansible.builtin.env', 'OS_APPLICATION_CREDENTIAL_SECRET') }}'

etc/kayobe/ansible/pulp-host-image-upload.yml

@@ -1,12 +1,12 @@
 ---
 - name: Upload and create a distribution for an image
-  hosts: localhost
+  hosts: seed
   vars:
     remote_pulp_url: "{{ stackhpc_release_pulp_url }}"
     remote_pulp_username: "{{ stackhpc_image_repository_username }}"
     remote_pulp_password: "{{ stackhpc_image_repository_password }}"
     repository_name: "kayobe-images-{{ openstack_release }}-{{ os_distribution }}-{{ os_release }}"
-    base_path: "kayobe-images/{{ openstack_release }}/{{ os_distribution }}/{{ os_release }}"
+    pulp_base_path: "kayobe-images/{{ openstack_release }}/{{ os_distribution }}/{{ os_release }}"
   tasks:
     - name: Print image tag
       debug:
@@ -74,7 +74,7 @@
         username: "{{ remote_pulp_username }}"
         password: "{{ remote_pulp_password }}"
         name: "{{ repository_name }}_latest"
-        base_path: "{{ base_path }}/latest"
+        base_path: "{{ pulp_base_path }}/latest"
         publication: "{{ publication_details.publication.pulp_href }}"
         content_guard: development
         state: present
@@ -86,7 +86,7 @@
         username: "{{ remote_pulp_username }}"
         password: "{{ remote_pulp_password }}"
         name: "{{ repository_name }}_{{ host_image_tag }}"
-        base_path: "{{ base_path }}/{{ host_image_tag }}"
+        base_path: "{{ pulp_base_path }}/{{ host_image_tag }}"
         publication: "{{ publication_details.publication.pulp_href }}"
         content_guard: development
         state: present
@@ -95,26 +95,26 @@
     - name: Update new images file with versioned path
       lineinfile:
         path: /tmp/updated_images.txt
-        line: "{{ remote_pulp_url }}/pulp/content/{{ base_path }}/\
+        line: "{{ remote_pulp_url }}/pulp/content/{{ pulp_base_path }}/\
         {{ host_image_tag }}/{{ found_files.files[0].path | basename }}"
         create: true
 
     - name: Update new images file with latest path
       lineinfile:
         path: /tmp/updated_images.txt
-        line: "{{ remote_pulp_url }}/pulp/content/{{ base_path }}/\
+        line: "{{ remote_pulp_url }}/pulp/content/{{ pulp_base_path }}/\
         latest/{{ found_files.files[0].path | basename }}"
       when: latest_distribution_details.changed
 
     - name: Print versioned path
       debug:
-        msg: "New versioned path: {{ remote_pulp_url }}/pulp/content/{{ base_path }}/\
+        msg: "New versioned path: {{ remote_pulp_url }}/pulp/content/{{ pulp_base_path }}/\
         {{ host_image_tag }}/{{ found_files.files[0].path | basename }}"
       when: latest_distribution_details.changed
 
     - name: Print latest path
       debug:
-        msg: "New latest path: {{ remote_pulp_url }}/pulp/content/{{ base_path }}/\
+        msg: "New latest path: {{ remote_pulp_url }}/pulp/content/{{ pulp_base_path }}/\
         latest/{{ found_files.files[0].path | basename }}"
       when: latest_distribution_details.changed
 

etc/kayobe/environments/ci-aio/inventory/group_vars/controllers/network-interfaces

@@ -3,9 +3,9 @@
 # Network interface definitions for the controller group.
 
 # Controller interface on all-in-one network.
-aio_interface: breth1
+# aio_interface: breth1
 # Use dummy1 if it exists, otherwise the bridge will have no ports.
-aio_bridge_ports: "{{ ['dummy1'] if ('dummy1' in hostvars[inventory_hostname].ansible_facts) or (os_distribution == 'rocky' and os_release == '9') else [] }}"
+# aio_bridge_ports: "{{ ['dummy1'] if ('dummy1' in hostvars[inventory_hostname].ansible_facts) or (os_distribution == 'rocky' and os_release == '9') else [] }}"
 
 ###############################################################################
 # Dummy variable to allow Ansible to accept this file.

etc/kayobe/environments/ci-builder/inventory/group_vars/seed/network-interfaces

@@ -2,12 +2,5 @@
 ###############################################################################
 # Network interface definitions for the controller group.
 
-# Controller interface on all-in-one network.
-aio_interface: breth1
-# Use dummy1 if it exists, otherwise the bridge will have no ports.
-aio_bridge_ports: "{{ ['dummy1'] if 'dummy1' in hostvars[inventory_hostname].ansible_facts else [] }}"
-
-###############################################################################
-# Dummy variable to allow Ansible to accept this file.
-workaround_ansible_issue_8743: yes
-
+# Seed interface on all-in-one network.
+aio_interface: eth0

etc/kayobe/overcloud-dib.yml

@@ -71,7 +71,7 @@ overcloud_dib_host_packages_extra:
 overcloud_dib_git_elements_extra:
   - repo: "https://github.com/stackhpc/stackhpc-image-elements"
     local: "{{ source_checkout_path }}/stackhpc-image-elements"
-    version: "v1.6.0"
+    version: "v1.6.1"
     elements_path: "elements"
 
 # List of git repositories containing Diskimage Builder (DIB) elements. See

terraform/host-image-builder/README.rst

@@ -0,0 +1,191 @@
+==========================
+Terraform All in one (aio)
+==========================
+
+This Terraform configuration deploys a single VM on an OpenStack cloud, to be
+used as an all-in-one Kayobe test environment.
+
+This configuration is used in the GitHub Actions all-in-one.yml workflow for CI
+testing.
+
+Usage
+=====
+
+These instructions show how to use this Terraform configuration manually. They
+assume you are running an Ubuntu host that will be used to run Terraform. The
+machine should have network access to the VM that will be created by this
+configuration.
+
+Install Terraform:
+
+.. code-block:: console
+
+   wget -qO - terraform.gpg https://apt.releases.hashicorp.com/gpg | sudo gpg --dearmor -o /usr/share/keyrings/terraform-archive-keyring.gpg
+   sudo echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/terraform-archive-keyring.gpg] https://apt.releases.hashicorp.com $(lsb_release -cs) main" | sudo tee /etc/apt/sources.list.d/terraform.list
+   sudo apt update
+   sudo apt install docker.io terraform
+
+Clone and initialise the Kayobe config:
+
+.. code-block:: console
+
+   git clone https://github.com/stackhpc/stackhpc-kayobe-config
+   cd stackhpc-kayobe-config
+   git submodule init
+   git submodule update
+
+Change to the terraform/aio directory:
+
+.. code-block:: console
+
+   cd terraform/aio
+
+Initialise Terraform:
+
+.. code-block:: console
+
+   terraform init
+
+Generate an SSH keypair:
+
+.. code-block:: console
+
+   ssh-keygen -f id_rsa -N ''
+
+Create an OpenStack clouds.yaml file with your credentials to access an
+OpenStack cloud. Alternatively, download one from Horizon.
+
+.. code-block:: console
+
+   cat << EOF > clouds.yaml
+   ---
+   clouds:
+     sms-lab:
+       auth:
+         auth_url: https://api.sms-lab.cloud:5000
+         username: <username>
+         project_name: <project>
+         domain_name: default
+       interface: public
+   EOF
+
+Export environment variables to use the correct cloud and provide a password:
+
+.. code-block:: console
+
+   export OS_CLOUD=sms-lab
+   read -p OS_PASSWORD -s OS_PASSWORD
+   export OS_PASSWORD
+
+Generate Terraform variables:
+
+.. code-block:: console
+
+   cat << EOF > terraform.tfvars
+   ssh_public_key = "id_rsa.pub"
+   aio_vm_name = "kayobe-aio"
+   aio_vm_image = "overcloud-centos-8-stream-yoga-20230525T095243"
+   aio_vm_flavor = "general.v1.medium"
+   aio_vm_network = "stackhpc-ipv4-geneve"
+   aio_vm_subnet = "stackhpc-ipv4-geneve-subnet"
+   EOF
+
+Generate a plan:
+
+.. code-block:: console
+
+   terraform plan
+
+Apply the changes:
+
+.. code-block:: console
+
+   terraform apply -auto-approve
+
+Write Terraform outputs to a Kayobe config file:
+
+.. code-block:: console
+
+   terraform output -json > ../../etc/kayobe/environments/$KAYOBE_ENVIRONMENT/tf-outputs.yml
+
+Change to the repository root:
+
+.. code-block:: console
+
+   cd ../../
+
+Write Terraform network config:
+
+.. code-block:: console
+
+   cat << EOF > etc/kayobe/environments/$KAYOBE_ENVIRONMENT/tf-networks.yml
+
+   admin_oc_net_name: admin
+   admin_cidr: "{{ access_cidr.value }}"
+   admin_allocation_pool_start: 0.0.0.0
+   admin_allocation_pool_end: 0.0.0.0
+   admin_gateway: "{{ access_gw.value }}"
+   admin_bootproto: dhcp
+   admin_ips:
+     controller0: "{{ access_ip_v4.value }}"
+   EOF
+
+Write Terraform network interface config:
+
+.. code-block:: console
+
+   cat << EOF > etc/kayobe/environments/$KAYOBE_ENVIRONMENT/inventory/group_vars/controllers/tf-network-interfaces
+   admin_interface: "{{ access_interface.value }}"
+   EOF
+
+Build a Kayobe image:
+
+.. code-block:: console
+
+   sudo DOCKER_BUILDKIT=1 docker build --file .automation/docker/kayobe/Dockerfile --tag kayobe:latest .
+
+Use the ci-aio environment:
+
+.. code-block:: console
+
+   export KAYOBE_ENVIRONMENT=ci-aio
+
+Set the Kayobe Vault password env var:
+
+.. code-block:: console
+
+   read -p KAYOBE_VAULT_PASSWORD -s KAYOBE_VAULT_PASSWORD
+   export KAYOBE_VAULT_PASSWORD
+
+Set the Kayobe SSH private key env var:
+
+.. code-block:: console
+
+   export KAYOBE_AUTOMATION_SSH_PRIVATE_KEY=$(cat terraform/aio/id_rsa)
+
+Host configure:
+
+.. code-block:: console
+
+   sudo -E docker run -it --rm -v $(pwd):/stack/kayobe-automation-env/src/kayobe-config -e KAYOBE_ENVIRONMENT -e KAYOBE_VAULT_PASSWORD -e KAYOBE_AUTOMATION_SSH_PRIVATE_KEY kayobe:latest /stack/kayobe-automation-env/src/kayobe-config/.automation/pipeline/overcloud-host-configure.sh
+
+Service deploy:
+
+.. code-block:: console
+
+   sudo -E docker run -it --rm -v $(pwd):/stack/kayobe-automation-env/src/kayobe-config -e KAYOBE_ENVIRONMENT -e KAYOBE_VAULT_PASSWORD -e KAYOBE_AUTOMATION_SSH_PRIVATE_KEY kayobe:latest /stack/kayobe-automation-env/src/kayobe-config/.automation/pipeline/overcloud-service-deploy.sh
+
+Configure aio resources:
+
+.. code-block:: console
+
+   sudo -E docker run -it --rm -v $(pwd):/stack/kayobe-automation-env/src/kayobe-config -e KAYOBE_ENVIRONMENT -e KAYOBE_VAULT_PASSWORD -e KAYOBE_AUTOMATION_SSH_PRIVATE_KEY kayobe:latest /stack/kayobe-automation-env/src/kayobe-config/.automation/pipeline/playbook-run.sh etc/kayobe/ansible/configure-aio-resources.yml
+
+Run Tempest:
+
+.. code-block:: console
+
+   mkdir -p tempest-artifacts
+   sudo -E docker run -it --rm -v $(pwd):/stack/kayobe-automation-env/src/kayobe-config -v $(pwd)/tempest-artifacts:/stack/tempest-artifacts -e KAYOBE_ENVIRONMENT -e KAYOBE_VAULT_PASSWORD -e KAYOBE_AUTOMATION_SSH_PRIVATE_KEY kayobe:latest /stack/kayobe-automation-env/src/kayobe-config/.automation/pipeline/tempest.sh -e ansible_user=stack
+
+Tempest results are in tempest-artifacts.

terraform/host-image-builder/outputs.tf

@@ -0,0 +1,15 @@
+output "access_ip_v4" {
+  value = openstack_compute_instance_v2.kayobe-host-image-builder.access_ip_v4
+}
+
+output "access_cidr" {
+  value = data.openstack_networking_subnet_v2.network.cidr
+}
+
+output "access_gw" {
+  value = data.openstack_networking_subnet_v2.network.gateway_ip
+}
+
+output "access_interface" {
+  value = var.host_image_builder_interface
+}

terraform/host-image-builder/provider.tf

@@ -0,0 +1,14 @@
+#provider "openstack" {
+# use environment variables
+#}
+
+terraform {
+  required_version = ">= 0.14"
+  backend "local" {
+  }
+  required_providers {
+    openstack = {
+      source = "terraform-provider-openstack/openstack"
+    }
+  }
+}

terraform/host-image-builder/templates/userdata.cfg.tpl

@@ -0,0 +1,21 @@
+#cloud-config
+# Don't automatically mount ephemeral disk
+mounts:
+  - [/dev/vdb, null]
+# WORKAROUND: internal DNS missing from SMS lab.
+runcmd:
+  - 'echo "10.0.0.34 pelican pelican.service.compute.sms-lab.cloud" >> /etc/hosts'
+  - 'echo "10.205.3.187 pulp-server pulp-server.internal.sms-cloud" >> /etc/hosts'
+# Configure SSH keys here, to avoid creating an ephemeral keypair.
+# This means only the instance needs to be cleaned up if the destroy fails.
+ssh_authorized_keys:
+  - ${ssh_public_key}
+
+write_files:
+  # WORKAROUND: https://bugs.launchpad.net/kolla-ansible/+bug/1995409
+  - content: |
+      #!/bin/bash
+      docker exec openvswitch_vswitchd ovs-vsctl "$@"
+    owner: root:root
+    path: /usr/bin/ovs-vsctl
+    permissions: '0755'

terraform/host-image-builder/terraform.tfvars

@@ -0,0 +1,9 @@
+ssh_public_key = "id_rsa.pub"
+ssh_username = "rocky"
+host_image_builder_name = "skc-ci-host-image-builder"
+host_image_builder_image = "Rocky-9-GenericCloud-Base-9.3-20231113.0.x86_64.qcow2"
+host_image_builder_flavor = "en1.medium"
+host_image_builder_network = "stackhpc-ci"
+host_image_builder_subnet = "stackhpc-ci"
+host_image_builder_tags = ["skc-ci-host-image-builder", "alex-testo"]
+host_image_builder_interface = "eth0"