[stable/2023.1] Improve CI reliability (#1411)

Signed-off-by: Mohammed Naser <[email protected]>
Co-authored-by: Mohammed Naser <[email protected]>

build/lint-jobs.py

@@ -0,0 +1,42 @@
+import sys
+import glob
+import yaml
+
+
+def main():
+    passed = True
+
+    for file in glob.glob("zuul.d/container-images/*.yaml"):
+        with open(file, "r") as file:
+            configs = yaml.safe_load(file)
+
+        for config in configs:
+            if "job" in config:
+                job = config["job"]
+
+                # Check if build or upload jobs are missing 'atmosphere-buildset-registry' dependency
+                if (
+                    "build-container-image-" in job["name"]
+                    or "upload-container-image-" in job["name"]
+                ):
+                    deps = job.get("dependencies", [])
+                    if not any(
+                        dep.get("name") == "atmosphere-buildset-registry"
+                        for dep in deps
+                    ):
+                        print(
+                            f"Job '{job['name']}' is missing 'atmosphere-buildset-registry' dependency."
+                        )
+                        passed = False
+
+    if passed:
+        print(
+            "All build and upload jobs have 'atmosphere-buildset-registry' dependency."
+        )
+    else:
+        print("Jobs missing 'atmosphere-buildset-registry' dependency.")
+        sys.exit(1)
+
+
+if __name__ == "__main__":
+    main()

roles/barbican/tasks/main.yml

@@ -55,3 +55,6 @@
   register: _octavia_implied_role_create
   changed_when: _octavia_implied_role_create.rc == 0
   failed_when: _octavia_implied_role_create.rc != 0 and 'Duplicate entry.' not in _octavia_implied_role_create.stderr
+  retries: 10
+  delay: 1
+  until: _octavia_implied_role_create.rc == 0 or 'Duplicate entry.' in _octavia_implied_role_create.stderr

roles/kube_prometheus_stack/tasks/main.yml

@@ -12,6 +12,18 @@
 # License for the specific language governing permissions and limitations
 # under the License.
 
+- name: Wait until Keycloak service is ready
+  kubernetes.core.k8s_info:
+    api_version: apps/v1
+    kind: StatefulSet
+    name: keycloak
+    namespace: auth-system
+  register: kube_prometheus_stack_keycloak_service
+  retries: 120
+  delay: 5
+  until:
+    - kube_prometheus_stack_keycloak_service.resources[0].status.replicas == kube_prometheus_stack_keycloak_service.resources[0].status.readyReplicas # noqa: yaml[line-length]
+
 - name: Create Keycloak realm
   no_log: true
   run_once: true

roles/octavia/tasks/generate_resources.yml

@@ -95,6 +95,11 @@
   environment:
     OS_CLOUD: atmosphere
   loop: "{{ groups['controllers'] }}"
+  register: _set_binding_for_ports
+  retries: 10
+  delay: 1
+  until: _set_binding_for_ports.rc == 0
+  failed_when: _set_binding_for_ports.rc != 0
 
 - name: Get health manager networking ports
   openstack.cloud.port_info:

roles/octavia/tasks/main.yml

@@ -126,6 +126,9 @@
   register: _octavia_implied_role_create
   changed_when: _octavia_implied_role_create.rc == 0
   failed_when: _octavia_implied_role_create.rc != 0 and 'Duplicate entry.' not in _octavia_implied_role_create.stderr
+  retries: 10
+  delay: 1
+  until: _octavia_implied_role_create.rc == 0 or 'Duplicate entry.' in _octavia_implied_role_create.stderr
 
 - name: Create Ingress
   ansible.builtin.include_role:

tox.ini

@@ -50,6 +50,7 @@ deps =
 commands =
   pre-commit run --all-files --show-diff-on-failure
   python3 {toxinidir}/build/sync-charts.py --check
+  python3 {toxinidir}/build/lint-jobs.py
 
 [testenv:molecule-keycloak]
 commands =

zuul.d/container-images/cluster-api-provider-openstack.yaml

@@ -45,6 +45,9 @@
 - job:
     name: atmosphere-upload-container-image-cluster-api-provider-openstack
     parent: atmosphere-upload-container-image
+    dependencies:
+      - name: atmosphere-buildset-registry
+        soft: false
     vars: *container_image_vars
     files: *container_image_files
 

zuul.d/container-images/kubernetes-entrypoint.yaml

@@ -45,6 +45,9 @@
 - job:
     name: atmosphere-upload-container-image-kubernetes-entrypoint
     parent: atmosphere-upload-container-image
+    dependencies:
+      - name: atmosphere-buildset-registry
+        soft: false
     vars: *container_image_vars
     files: *container_image_files
 

zuul.d/container-images/openvswitch.yaml

@@ -45,6 +45,9 @@
 - job:
     name: atmosphere-upload-container-image-openvswitch
     parent: atmosphere-upload-container-image
+    dependencies:
+      - name: atmosphere-buildset-registry
+        soft: false
     vars: *container_image_vars
     files: *container_image_files
 

zuul.d/container-images/python-openstackclient.yaml

@@ -60,6 +60,8 @@
     name: atmosphere-upload-container-image-python-openstackclient
     parent: atmosphere-upload-container-image
     dependencies:
+      - name: atmosphere-buildset-registry
+        soft: false
       - name: atmosphere-upload-container-image-ubuntu
         soft: true
       - name: atmosphere-upload-container-image-ubuntu-cloud-archive

zuul.d/container-images/ubuntu.yaml

@@ -47,6 +47,9 @@
 - job:
     name: atmosphere-upload-container-image-ubuntu
     parent: atmosphere-upload-container-image
+    dependencies:
+      - name: atmosphere-buildset-registry
+        soft: false
     vars: *container_image_vars
     files: *container_image_files