Added k8s worker taint configuration options

The options to add to site-config are:
- doni_enable_worker_taint: true/false, by default set to false (enables worker tainting through
  the k8s worker on doni devices)
- zun_enforce_worker_taint: true/false, by default set to false (enables taint enforcement by
  adding tolerations to worker pods)
- k8s_worker_taint
  - key: taint key, by default set to "worker-node"
  - value: taint value, by default set to "true"
  - effect: taint effect, by default set to "NoSchedule"

kolla/defaults.yml

@@ -92,6 +92,7 @@ enable_cinder: no
 
 # Doni
 enable_doni: yes
+doni_enforce_worker_taints: false
 
 # Glance
 enable_glance: yes
@@ -345,6 +346,10 @@ enable_k3s: no
 enable_zun: no
 enable_zun_compute_k8s: "{{ enable_zun | bool and enable_k3s | bool }}"
 blazar_enable_device_plugin_k8s: "{{ enable_k3s | bool }}"
+# K8S worker tainting
+zun_enforce_worker_taint: false
+doni_enable_worker_taint: false
+worker_taint: "{{ k8s_worker_taint.key | default('worker-node') }}={{ k8s_worker_taint.value | default('true') }}:{{ k8s_worker_taint.effect | default('NoSchedule') }}"
 # If not running K3s, can run standard Zun deploy w/ etcd and kuryr
 enable_etcd: "{{ enable_zun | bool and not enable_k3s | bool }}"
 enable_kuryr: "{{ enable_zun | bool and not enable_k3s | bool }}"

kolla/node_custom_config/doni.conf

@@ -37,6 +37,10 @@ device_fleet_mapping = raspberrypi3-64:chi-edge-workers,raspberrypi4-64:chi-edge
 
 {% if enable_k3s | bool %}
 [k8s]
+enable_worker_taint = {{ doni_enable_worker_taint }}
+worker_taint_key = {{ worker_taint.key }}
+worker_taint_value = {{ worker_taint.value }}
+worker_taint_effect = {{ worker_taint.effect }}
 expected_labels_index_property = machine_name
 expected_labels = raspberrypi4-64:smarter-device-manager=enabled|smarter-device-manager/configmap=rpi,
                   raspberrypi3-64:smarter-device-manager=enabled|smarter-device-manager/configmap=rpi,

kolla/node_custom_config/zun.conf

@@ -37,6 +37,10 @@ allow_without_reservation = False
 nvidia_require_jetpack = csv-mounts=all
 nvidia_visible_devices = all
 nvidia_driver_capabilities = all
+enable_worker_taint = {{ zun_enforce_worker_taint }}
+worker_taint_key = {{ worker_taint.key }}
+worker_taint_value = {{ worker_taint.value }}
+worker_taint_effect = {{ worker_taint.effect }}
 {% if enable_neutron | bool %}
 neutron_network = caliconet
 {% endif %}