Feature/tempest tests (#287)

Add a playbook to install openstack tempest, and to template a configuration file from the site-config.

WARNING: Don't run this against a production site, you will impact your users and/or lose date.
Use it only for development sites, or for acceptance tests prior to letting users on.

docs/Developing-Openstack-Services.md

@@ -150,3 +150,47 @@ Configuration files are applied in the following order, with more specific repla
 * `node_custom_config/service/hostname/service.conf`
 
 See the [kolla-ansible docs](https://docs.openstack.org/kolla-ansible/latest/admin/advanced-configuration.html#openstack-service-configuration-in-kolla) for more. 
+
+
+## Running end-to-end functional tests with Tempest
+
+Openstack Tempest is a framework to exercise the API of a deployed Openstack site. You can utilize this to make sure that your changes have not broken the API's compatibility, and generally that tasks such as a creating a network or launching an instance work as expected.
+
+CHI-in-a-box includes a playbook to install and configure tempest to run against the local development site. 
+
+WARNING: Do NOT run this against a production site, or one with user instances/data present. Although tempest seeks to isolate the resources it creates/deletes in ephemerial Openstack Projects, it can easily consume all networks/floating ips/other instance resources on your site. Addtionally, cleaning up after tempest is not always straightforward, and significant care must be taken to avoid deleting user created resources.
+
+For this reason, we currently recommend running these tests only against a development site, or before releasing a site to end-users. 
+We are working on a "safe subset" configuration, but it's not ready yet.
+
+
+After setting your site up, including `post-deploy`, set the following parameters in your defaults.yml:
+
+```
+tempest_fixed_network_name: "sharednet1"
+tempest_public_network_uuid:  <uuid of public neutron network>
+tempest_compute_image_uuid:  <uuid of glance image to test instances with>
+tempest_baremetal_flavor_uuid: <uuid of the barmetal flavor>
+```
+
+If updating an existing site to add this feature, you'll need to add the following ansible group to your `inventory/hosts` file.
+
+Then, run `cc-ansible --playbook playbooks/tempest.yml` to install the tempest config file and tools. They will be installed into `~/tempest` by default, but you can override this by setting `tempest_install_dir` in your defaults.yml.
+
+After installing, run the following:
+```
+cd ~/tempest
+source .venv/bin/activate
+
+tempest run 
+  --workspace local \
+  --concurrency 1 \
+  --smoke
+```
+This will run a subset of important tests against your dev site.
+
+Concurrency is set to 1 to avoid running out of baremetal nodes, you can turn this up if more nodes are available. To run all tests, instead of the most critical subset, remove `--smoke`
+
+For an example of how to exclude specific tests, append the argument:
+`--exclude-list /opt/chi-in-a-box/roles/tempest/templates/exclude-list.conf`
+This list excludes tests which are known to fail if only `flat` networks are availalable, namely anything to do with user creation/update/delete of tenant networks.

playbooks/tempest.yml

@@ -0,0 +1,3 @@
+- hosts: deployment
+  roles:
+    - role: tempest

roles/post_ironic/defaults/main.yml

@@ -17,3 +17,7 @@ ironic_glance_images:
     container_format: aki
     disk_format: aki
     url: "https://tarballs.opendev.org/openstack/ironic-python-agent/dib/files/ipa-centos8-stable-{{ openstack_release }}.initramfs"
+  - name: cirros-0.6.2
+    container_format: bare
+    disk_format: qcow2
+    url: "http://download.cirros-cloud.net/0.6.2/cirros-0.6.2-x86_64-disk.img"

roles/tempest/defaults/main.yml

@@ -0,0 +1,13 @@
+tempest_install_dir: ~/tempest
+tempest_git_repo: https://github.com/openstack/tempest
+
+tempest_workspace_root: "{{ tempest_install_dir }}/workspaces"
+local_site_workspace: "{{ tempest_workspace_root }}/local"
+
+
+# TODO: these 4 parameters should be fetched from openstack API, since the values are
+# set dynamically during post-deploy
+tempest_fixed_network_name: "sharednet1"
+tempest_public_network_uuid:
+tempest_compute_image_uuid:
+tempest_baremetal_flavor_uuid:

roles/tempest/tasks/main.yml

@@ -0,0 +1,43 @@
+---
+- name: clone tempest git repo
+  ansible.builtin.git:
+    repo: "{{ tempest_git_repo }}"
+    dest: "{{ tempest_install_dir }}"
+
+- name: create venv and install build deps
+  ansible.builtin.pip:
+    virtualenv: "{{ tempest_install_dir }}/.venv/"
+    state: latest
+    name:
+      - pip
+      - wheel
+
+- name: pip install tempest from the git checkout
+  ansible.builtin.pip:
+    name: "{{ tempest_install_dir }}/"
+    virtualenv: "{{ tempest_install_dir }}/.venv/"
+
+- name: template tempest.conf from site-config
+  template:
+    src: "tempest.conf.j2"
+    dest: "~/.tempest/etc/tempest.conf"
+
+- name: initialize tempest workspace
+  ansible.builtin.command:
+    chdir: "{{ tempest_install_dir }}"
+    argv: 
+      - .venv/bin/tempest
+      -  init 
+      - --name 
+      - local
+      - "{{ local_site_workspace}}"
+    creates: "{{ local_site_workspace }}/etc/tempest.conf"
+
+- name: initialize tempest saved state
+  ansible.builtin.command:
+    chdir: "{{ local_site_workspace }}"
+    argv: 
+      - ../../.venv/bin/tempest
+      - cleanup
+      - --init-saved-state
+    creates: "{{ local_site_workspace }}/saved_state.json"

roles/tempest/templates/exclude-list.conf

@@ -0,0 +1,16 @@
+# tempest.scenario.test_server_basic_ops.TestServerBasicOps.test_server_basic_ops
+
+# tempest.api.compute.servers.test_create_server.ServersTestJSON
+# tempest.api.compute.servers.test_create_server.ServersTestManualDisk
+# tempest.api.compute.servers.test_server_actions.ServerActionsTestJSON
+# tempest.api.compute.servers.test_server_addresses.ServerAddressesTestJSON
+
+tempest.api.network.test_floating_ips.FloatingIPTestJSON
+tempest.api.network.test_networks.BulkNetworkOpsTest.test_bulk_create_delete_network
+tempest.api.network.test_networks.BulkNetworkOpsTest.test_bulk_create_delete_port
+tempest.api.network.test_networks.BulkNetworkOpsTest.test_bulk_create_delete_subnet
+tempest.api.network.test_networks.NetworksTest
+tempest.api.network.test_ports.PortsTestJSON
+tempest.api.network.test_routers.RoutersTest.test_add_multiple_router_interfaces
+tempest.api.network.test_routers.RoutersTest.test_add_remove_router_interface_with_port_id
+tempest.api.network.test_routers.RoutersTest.test_add_remove_router_interface_with_subnet_id

roles/tempest/templates/tempest.conf.j2

@@ -0,0 +1,88 @@
+[auth]
+
+# disabled due to flat networks in dev-in-a-box
+create_isolated_networks = false
+
+# don't use in prod!!
+use_dynamic_credentials = true
+
+admin_username = "{{ keystone_admin_user }}"
+admin_project_name = "{{ keystone_admin_project }}"
+admin_password = "{{ keystone_admin_password }}"
+admin_domain_name = Default
+admin_user_domain_name = Default
+admin_project_domain_name = Default
+
+[identity]
+
+uri_v3 = "{{ keystone_public_url }}/v3"
+auth_version = v3
+region = "{{ openstack_region_name }}"
+
+[service_available]
+cinder = "{{ enable_cinder }}"
+neutron = true
+glance = true
+swift = false
+nova = true
+horizon = true
+
+[compute]
+
+image_ref = "{{ tempest_compute_image_uuid }}"
+image_ref_alt = "{{ tempest_compute_image_uuid }}"
+flavor_ref = "{{ tempest_baremetal_flavor_uuid }}"
+
+fixed_network_name = "{{ tempest_fixed_network_name }}"
+
+[compute-feature-enabled]
+
+# must match value in nova.conf
+scheduler_enabled_filters = ComputeFilter,ComputeCapabilitiesFilter,ImagePropertiesFilter,BlazarFilter
+
+metadata_service = true
+interface_attach = true
+config_drive = true
+serial_console = true
+
+# not currently working with our ironic config
+rescue = false
+
+# Do these work??
+#disk_config = true
+#enable_instance_password = false
+#change_password = false
+#personality = false
+#unified_limits = false
+
+## VM only features, not applicable for barmetal
+snapshot = false
+resize = false
+pause = false
+shelve = false
+shelve_migrate = false
+suspend = false
+cold_migration = false
+live_migration = false
+live_migrate_back_and_forth = false
+vnc_console = false
+attach_encrypted_volume = false
+swap_volume = false
+volume_backed_live_migration = false
+volume_multiattach = false
+
+[network]
+project_networks_reachable = false
+public_network_id = "{{ tempest_public_network_uuid }}"
+
+project_network_cidr = 10.100.0.0/16
+project_network_mask_bits = 28
+
+[network-feature-enabled]
+ipv6 = false
+
+api_extensions = address-group,address-scope,router-admin-state-down-before-update,agent,agent-resources-synced,allowed-address-pairs,auto-allocated-topology,availability_zone,availability_zone_filter,default-subnetpools,dhcp_agent_scheduler,dvr,empty-string-filtering,external-net,extra_dhcp_opt,extraroute,extraroute-atomic,filter-validation,fip-port-details,flavors,floatingip-pools,ip-substring-filtering,router,ext-gw-mode,l3-ha,l3-flavors,l3-port-ip-change-not-allowed,l3_agent_scheduler,multi-provider,net-mtu,net-mtu-writable,network_availability_zone,network-ip-availability,network-segment-range,pagination,port-device-profile,port-mac-address-regenerate,port-numa-affinity-policy,binding,binding-extended,port-security,project-id,provider,quotas,quota_details,rbac-policies,rbac-address-group,rbac-address-scope,rbac-security-groups,rbac-subnetpool,revision-if-match,standard-attr-revisions,router_availability_zone,security-groups-normalized-cidr,port-security-groups-filtering,security-groups-remote-address-group,security-group,service-type,sorting,standard-attr-description,stateful-security-group,subnet_onboard,subnet-service-types,subnet_allocation,subnetpool-prefix-ops,standard-attr-tag,standard-attr-timestamp
+
+[dashboard]
+
+dashboard_url = "{{ horizon_public_endpoint }}"

site-config.example/inventory/hosts

@@ -67,6 +67,10 @@ keycloak
 [keycloak_tas_sync:children]
 keycloak
 
+# used for executing Tempest functional tests against a site
+[tempest:children]
+deployment
+
 # This group is used to ensure that the defaults.yml group_vars apply to
 # all nodes, including the deploy host, without running other roles.
 [defaults:children]