From 3edc4f008cbbf537eba6c09e696575bcf7ccc8e7 Mon Sep 17 00:00:00 2001
From: Giacomo Leidi <giacomo.leidi@suse.com>
Date: Thu, 31 Oct 2024 14:21:23 +0100
Subject: [PATCH 1/6] osclib/list_command.py: Show details also for requests
 targeted to packages not in rings.

This commit implements the ability to show for packages that do not
belong to any ring the same details that are shown for packages that do
live in rings.  Due to concerns about compatibility and volume of the
output the feature is hidden behind a new --adi-details command
line flag.
---
 osc-staging.py         |  6 ++--
 osclib/list_command.py | 65 ++++++++++++++++++++++++------------------
 2 files changed, 41 insertions(+), 30 deletions(-)

diff --git a/osc-staging.py b/osc-staging.py
index e9cab5363..d164fae51 100644
--- a/osc-staging.py
+++ b/osc-staging.py
@@ -93,6 +93,8 @@ def clean_args(args):
               help='split the requests into individual groups')
 @cmdln.option('--supersede', action='store_true',
               help='replace staged requests when superseded')
+@cmdln.option('--adi-details', action='store_true',
+              help='show detailed summary for packages that are not in any ring')
 @cmdln.option('--filter-from', metavar='STAGING',
               help='filter request list to only those from a specific staging')
 @cmdln.option('-p', '--project', dest='project', metavar='PROJECT',
@@ -307,7 +309,7 @@ def do_staging(self, subcmd, opts, *args):
         osc staging frozenage [STAGING...]
         osc staging ignore [-m MESSAGE] REQUEST...
         osc staging unignore [--cleanup] [REQUEST...|all]
-        osc staging list [--supersede]
+        osc staging list [--supersede] [--adi-details]
         osc staging lock [-m MESSAGE]
         osc staging select [--no-freeze] [--remove-exclusion] [--move [--filter-from STAGING]]
             STAGING REQUEST...
@@ -578,7 +580,7 @@ def do_staging(self, subcmd, opts, *args):
         elif cmd == 'unignore':
             UnignoreCommand(api).perform(args[1:], opts.cleanup)
         elif cmd == 'list':
-            ListCommand(api).perform(supersede=opts.supersede)
+            ListCommand(api).perform(supersede=opts.supersede, adi_details=opts.adi_details)
         elif cmd == 'lock':
             lock.hold(opts.message)
         elif cmd == 'adi':
diff --git a/osclib/list_command.py b/osclib/list_command.py
index ec55a2c20..9af760c16 100644
--- a/osclib/list_command.py
+++ b/osclib/list_command.py
@@ -18,7 +18,32 @@ class ListCommand:
     def __init__(self, api):
         self.api = api
 
-    def perform(self, supersede=False):
+    def print_request(self, request):
+        hide_source = self.api.project == 'openSUSE:Factory'
+        request_id = int(request.get('id'))
+        action = request.find('action')
+        target_package = action.find('target').get('package')
+        ring = action.find('target').get('ring', None)
+
+        line = f"{request_id} {Fore.CYAN}{target_package:<30}{Fore.RESET}"
+        if ring:
+            ring_color = Fore.MAGENTA if ring.startswith('0') else ''
+            line += f" -> {ring_color}{ring:<12}{Fore.RESET}"
+
+        if not hide_source and action.find('source') is not None:
+            source_project = action.find('source').get('project')
+            source_project = self.project_strip(source_project)
+            line += f' ({Fore.YELLOW + source_project + Fore.RESET})'
+        if action.get('type') == 'delete':
+            line += ' (' + Fore.RED + 'delete request' + Fore.RESET + ')'
+
+        message = self.api.ignore_format(request_id)
+        if message:
+            line += '\n' + Fore.WHITE + message + Fore.RESET
+
+        print(' ', line)
+
+    def perform(self, supersede=False, adi_details=False):
         """
         Perform the list command
         """
@@ -34,39 +59,23 @@ def perform(self, supersede=False):
         splitter.group_by('./action/target/@devel_project')
         splitter.split()
 
-        hide_source = self.api.project == 'openSUSE:Factory'
         for group in sorted(splitter.grouped.keys()):
             print(Fore.YELLOW + group)
 
             for request in splitter.grouped[group]['requests']:
-                request_id = int(request.get('id'))
-                action = request.find('action')
-                target_package = action.find('target').get('package')
-                ring = action.find('target').get('ring')
-                ring_color = Fore.MAGENTA if ring.startswith('0') else ''
-
-                line = '{} {}{:<30}{} -> {}{:<12}{}'.format(
-                    request_id, Fore.CYAN, target_package, Fore.RESET,
-                    ring_color, ring, Fore.RESET)
-
-                if not hide_source and action.find('source') is not None:
-                    source_project = action.find('source').get('project')
-                    source_project = self.project_strip(source_project)
-                    line += f' ({Fore.YELLOW + source_project + Fore.RESET})'
-                if action.get('type') == 'delete':
-                    line += ' (' + Fore.RED + 'delete request' + Fore.RESET + ')'
-
-                message = self.api.ignore_format(request_id)
-                if message:
-                    line += '\n' + Fore.WHITE + message + Fore.RESET
-
-                print(' ', line)
+                self.print_request(request)
 
         if len(splitter.other):
-            non_ring_packages = []
-            for request in splitter.other:
-                non_ring_packages.append(request.find('./action/target').get('package'))
-            print('Not in a ring: ' + ' '.join(sorted(non_ring_packages)))
+            non_ring_requests = splitter.other
+            if adi_details:
+                print('Not in a ring: ')
+                for request in non_ring_requests:
+                    self.print_request(request)
+            else:
+                non_ring_packages = sorted(
+                    request.find('./action/target').get('package')
+                    for request in non_ring_requests)
+                print('Not in a ring: ' + ' '.join(non_ring_packages))
 
         # Print requests not handled by staging process to highlight them.
         splitter.stageable = False

From cdfda0be5c95e3d966808685af366d46886b7565 Mon Sep 17 00:00:00 2001
From: Giacomo Leidi <giacomo.leidi@suse.com>
Date: Thu, 7 Nov 2024 14:29:12 +0100
Subject: [PATCH 2/6] Add 180 minutes timeout to SLFO product building
 pipelines.

---
 gocd/slfo-stagings.gocd.yaml     | 110 +++++++++++++++++++++++++++++++
 gocd/slfo-stagings.gocd.yaml.erb |  10 +++
 2 files changed, 120 insertions(+)

diff --git a/gocd/slfo-stagings.gocd.yaml b/gocd/slfo-stagings.gocd.yaml
index 36a7c07b7..1cf43be5e 100644
--- a/gocd/slfo-stagings.gocd.yaml
+++ b/gocd/slfo-stagings.gocd.yaml
@@ -122,14 +122,19 @@ pipelines:
           - staging-bot
         tasks:
           - script: |-
+              minutes=0
               osc -A $STAGING_API api -X POST "/source/$STAGING_PROJECT?cmd=remove_flag&repository=product&flag=build"
               export PYTHONPATH=$PWD/scripts
               while ./scripts/gocd/verify-repo-built-successful.py -A $STAGING_API -p $STAGING_PROJECT -r product; ret=$?; [ ${ret} -ne 0 ]; do
                 if [ ${ret} -eq 2 ]; then
                     echo "product repository not found. Project configuration issue?" >&2
                     exit 1
+                elif [ ${minutes} -gt 180 ]; then
+                    echo "Product is still failing after timeout, exiting..." >&2
+                    exit 1
                 fi
                 sleep 60
+                minutes=$(expr $minutes + 1)
               done
 
     - Enable.images.repo:
@@ -206,14 +211,19 @@ pipelines:
           - staging-bot
         tasks:
           - script: |-
+              minutes=0
               osc -A $STAGING_API api -X POST "/source/$STAGING_PROJECT?cmd=remove_flag&repository=product&flag=build"
               export PYTHONPATH=$PWD/scripts
               while ./scripts/gocd/verify-repo-built-successful.py -A $STAGING_API -p $STAGING_PROJECT -r product; ret=$?; [ ${ret} -ne 0 ]; do
                 if [ ${ret} -eq 2 ]; then
                     echo "product repository not found. Project configuration issue?" >&2
                     exit 1
+                elif [ ${minutes} -gt 180 ]; then
+                    echo "Product is still failing after timeout, exiting..." >&2
+                    exit 1
                 fi
                 sleep 60
+                minutes=$(expr $minutes + 1)
               done
 
     - Enable.images.repo:
@@ -290,14 +300,19 @@ pipelines:
           - staging-bot
         tasks:
           - script: |-
+              minutes=0
               osc -A $STAGING_API api -X POST "/source/$STAGING_PROJECT?cmd=remove_flag&repository=product&flag=build"
               export PYTHONPATH=$PWD/scripts
               while ./scripts/gocd/verify-repo-built-successful.py -A $STAGING_API -p $STAGING_PROJECT -r product; ret=$?; [ ${ret} -ne 0 ]; do
                 if [ ${ret} -eq 2 ]; then
                     echo "product repository not found. Project configuration issue?" >&2
                     exit 1
+                elif [ ${minutes} -gt 180 ]; then
+                    echo "Product is still failing after timeout, exiting..." >&2
+                    exit 1
                 fi
                 sleep 60
+                minutes=$(expr $minutes + 1)
               done
 
     - Enable.images.repo:
@@ -374,14 +389,19 @@ pipelines:
           - staging-bot
         tasks:
           - script: |-
+              minutes=0
               osc -A $STAGING_API api -X POST "/source/$STAGING_PROJECT?cmd=remove_flag&repository=product&flag=build"
               export PYTHONPATH=$PWD/scripts
               while ./scripts/gocd/verify-repo-built-successful.py -A $STAGING_API -p $STAGING_PROJECT -r product; ret=$?; [ ${ret} -ne 0 ]; do
                 if [ ${ret} -eq 2 ]; then
                     echo "product repository not found. Project configuration issue?" >&2
                     exit 1
+                elif [ ${minutes} -gt 180 ]; then
+                    echo "Product is still failing after timeout, exiting..." >&2
+                    exit 1
                 fi
                 sleep 60
+                minutes=$(expr $minutes + 1)
               done
 
     - Enable.images.repo:
@@ -458,14 +478,19 @@ pipelines:
           - staging-bot
         tasks:
           - script: |-
+              minutes=0
               osc -A $STAGING_API api -X POST "/source/$STAGING_PROJECT?cmd=remove_flag&repository=product&flag=build"
               export PYTHONPATH=$PWD/scripts
               while ./scripts/gocd/verify-repo-built-successful.py -A $STAGING_API -p $STAGING_PROJECT -r product; ret=$?; [ ${ret} -ne 0 ]; do
                 if [ ${ret} -eq 2 ]; then
                     echo "product repository not found. Project configuration issue?" >&2
                     exit 1
+                elif [ ${minutes} -gt 180 ]; then
+                    echo "Product is still failing after timeout, exiting..." >&2
+                    exit 1
                 fi
                 sleep 60
+                minutes=$(expr $minutes + 1)
               done
 
     - Enable.images.repo:
@@ -542,14 +567,19 @@ pipelines:
           - staging-bot
         tasks:
           - script: |-
+              minutes=0
               osc -A $STAGING_API api -X POST "/source/$STAGING_PROJECT?cmd=remove_flag&repository=product&flag=build"
               export PYTHONPATH=$PWD/scripts
               while ./scripts/gocd/verify-repo-built-successful.py -A $STAGING_API -p $STAGING_PROJECT -r product; ret=$?; [ ${ret} -ne 0 ]; do
                 if [ ${ret} -eq 2 ]; then
                     echo "product repository not found. Project configuration issue?" >&2
                     exit 1
+                elif [ ${minutes} -gt 180 ]; then
+                    echo "Product is still failing after timeout, exiting..." >&2
+                    exit 1
                 fi
                 sleep 60
+                minutes=$(expr $minutes + 1)
               done
 
     - Enable.images.repo:
@@ -626,14 +656,19 @@ pipelines:
           - staging-bot
         tasks:
           - script: |-
+              minutes=0
               osc -A $STAGING_API api -X POST "/source/$STAGING_PROJECT?cmd=remove_flag&repository=product&flag=build"
               export PYTHONPATH=$PWD/scripts
               while ./scripts/gocd/verify-repo-built-successful.py -A $STAGING_API -p $STAGING_PROJECT -r product; ret=$?; [ ${ret} -ne 0 ]; do
                 if [ ${ret} -eq 2 ]; then
                     echo "product repository not found. Project configuration issue?" >&2
                     exit 1
+                elif [ ${minutes} -gt 180 ]; then
+                    echo "Product is still failing after timeout, exiting..." >&2
+                    exit 1
                 fi
                 sleep 60
+                minutes=$(expr $minutes + 1)
               done
 
     - Enable.images.repo:
@@ -710,14 +745,19 @@ pipelines:
           - staging-bot
         tasks:
           - script: |-
+              minutes=0
               osc -A $STAGING_API api -X POST "/source/$STAGING_PROJECT?cmd=remove_flag&repository=product&flag=build"
               export PYTHONPATH=$PWD/scripts
               while ./scripts/gocd/verify-repo-built-successful.py -A $STAGING_API -p $STAGING_PROJECT -r product; ret=$?; [ ${ret} -ne 0 ]; do
                 if [ ${ret} -eq 2 ]; then
                     echo "product repository not found. Project configuration issue?" >&2
                     exit 1
+                elif [ ${minutes} -gt 180 ]; then
+                    echo "Product is still failing after timeout, exiting..." >&2
+                    exit 1
                 fi
                 sleep 60
+                minutes=$(expr $minutes + 1)
               done
 
     - Enable.images.repo:
@@ -794,14 +834,19 @@ pipelines:
           - staging-bot
         tasks:
           - script: |-
+              minutes=0
               osc -A $STAGING_API api -X POST "/source/$STAGING_PROJECT?cmd=remove_flag&repository=product&flag=build"
               export PYTHONPATH=$PWD/scripts
               while ./scripts/gocd/verify-repo-built-successful.py -A $STAGING_API -p $STAGING_PROJECT -r product; ret=$?; [ ${ret} -ne 0 ]; do
                 if [ ${ret} -eq 2 ]; then
                     echo "product repository not found. Project configuration issue?" >&2
                     exit 1
+                elif [ ${minutes} -gt 180 ]; then
+                    echo "Product is still failing after timeout, exiting..." >&2
+                    exit 1
                 fi
                 sleep 60
+                minutes=$(expr $minutes + 1)
               done
 
     - Enable.images.repo:
@@ -878,14 +923,19 @@ pipelines:
           - staging-bot
         tasks:
           - script: |-
+              minutes=0
               osc -A $STAGING_API api -X POST "/source/$STAGING_PROJECT?cmd=remove_flag&repository=product&flag=build"
               export PYTHONPATH=$PWD/scripts
               while ./scripts/gocd/verify-repo-built-successful.py -A $STAGING_API -p $STAGING_PROJECT -r product; ret=$?; [ ${ret} -ne 0 ]; do
                 if [ ${ret} -eq 2 ]; then
                     echo "product repository not found. Project configuration issue?" >&2
                     exit 1
+                elif [ ${minutes} -gt 180 ]; then
+                    echo "Product is still failing after timeout, exiting..." >&2
+                    exit 1
                 fi
                 sleep 60
+                minutes=$(expr $minutes + 1)
               done
 
     - Enable.images.repo:
@@ -962,14 +1012,19 @@ pipelines:
           - staging-bot
         tasks:
           - script: |-
+              minutes=0
               osc -A $STAGING_API api -X POST "/source/$STAGING_PROJECT?cmd=remove_flag&repository=product&flag=build"
               export PYTHONPATH=$PWD/scripts
               while ./scripts/gocd/verify-repo-built-successful.py -A $STAGING_API -p $STAGING_PROJECT -r product; ret=$?; [ ${ret} -ne 0 ]; do
                 if [ ${ret} -eq 2 ]; then
                     echo "product repository not found. Project configuration issue?" >&2
                     exit 1
+                elif [ ${minutes} -gt 180 ]; then
+                    echo "Product is still failing after timeout, exiting..." >&2
+                    exit 1
                 fi
                 sleep 60
+                minutes=$(expr $minutes + 1)
               done
 
     - Enable.images.repo:
@@ -1105,14 +1160,19 @@ pipelines:
           - staging-bot
         tasks:
           - script: |-
+              minutes=0
               osc -A $STAGING_API api -X POST "/source/$STAGING_PROJECT?cmd=remove_flag&repository=product&flag=build"
               export PYTHONPATH=$PWD/scripts
               while ./scripts/gocd/verify-repo-built-successful.py -A $STAGING_API -p $STAGING_PROJECT -r product; ret=$?; [ ${ret} -ne 0 ]; do
                 if [ ${ret} -eq 2 ]; then
                     echo "product repository not found. Project configuration issue?" >&2
                     exit 1
+                elif [ ${minutes} -gt 180 ]; then
+                    echo "Product is still failing after timeout, exiting..." >&2
+                    exit 1
                 fi
                 sleep 60
+                minutes=$(expr $minutes + 1)
               done
 
     - Enable.images.repo:
@@ -1190,14 +1250,19 @@ pipelines:
           - staging-bot
         tasks:
           - script: |-
+              minutes=0
               osc -A $STAGING_API api -X POST "/source/$STAGING_PROJECT?cmd=remove_flag&repository=product&flag=build"
               export PYTHONPATH=$PWD/scripts
               while ./scripts/gocd/verify-repo-built-successful.py -A $STAGING_API -p $STAGING_PROJECT -r product; ret=$?; [ ${ret} -ne 0 ]; do
                 if [ ${ret} -eq 2 ]; then
                     echo "product repository not found. Project configuration issue?" >&2
                     exit 1
+                elif [ ${minutes} -gt 180 ]; then
+                    echo "Product is still failing after timeout, exiting..." >&2
+                    exit 1
                 fi
                 sleep 60
+                minutes=$(expr $minutes + 1)
               done
 
     - Enable.images.repo:
@@ -1275,14 +1340,19 @@ pipelines:
           - staging-bot
         tasks:
           - script: |-
+              minutes=0
               osc -A $STAGING_API api -X POST "/source/$STAGING_PROJECT?cmd=remove_flag&repository=product&flag=build"
               export PYTHONPATH=$PWD/scripts
               while ./scripts/gocd/verify-repo-built-successful.py -A $STAGING_API -p $STAGING_PROJECT -r product; ret=$?; [ ${ret} -ne 0 ]; do
                 if [ ${ret} -eq 2 ]; then
                     echo "product repository not found. Project configuration issue?" >&2
                     exit 1
+                elif [ ${minutes} -gt 180 ]; then
+                    echo "Product is still failing after timeout, exiting..." >&2
+                    exit 1
                 fi
                 sleep 60
+                minutes=$(expr $minutes + 1)
               done
 
     - Enable.images.repo:
@@ -1360,14 +1430,19 @@ pipelines:
           - staging-bot
         tasks:
           - script: |-
+              minutes=0
               osc -A $STAGING_API api -X POST "/source/$STAGING_PROJECT?cmd=remove_flag&repository=product&flag=build"
               export PYTHONPATH=$PWD/scripts
               while ./scripts/gocd/verify-repo-built-successful.py -A $STAGING_API -p $STAGING_PROJECT -r product; ret=$?; [ ${ret} -ne 0 ]; do
                 if [ ${ret} -eq 2 ]; then
                     echo "product repository not found. Project configuration issue?" >&2
                     exit 1
+                elif [ ${minutes} -gt 180 ]; then
+                    echo "Product is still failing after timeout, exiting..." >&2
+                    exit 1
                 fi
                 sleep 60
+                minutes=$(expr $minutes + 1)
               done
 
     - Enable.images.repo:
@@ -1445,14 +1520,19 @@ pipelines:
           - staging-bot
         tasks:
           - script: |-
+              minutes=0
               osc -A $STAGING_API api -X POST "/source/$STAGING_PROJECT?cmd=remove_flag&repository=product&flag=build"
               export PYTHONPATH=$PWD/scripts
               while ./scripts/gocd/verify-repo-built-successful.py -A $STAGING_API -p $STAGING_PROJECT -r product; ret=$?; [ ${ret} -ne 0 ]; do
                 if [ ${ret} -eq 2 ]; then
                     echo "product repository not found. Project configuration issue?" >&2
                     exit 1
+                elif [ ${minutes} -gt 180 ]; then
+                    echo "Product is still failing after timeout, exiting..." >&2
+                    exit 1
                 fi
                 sleep 60
+                minutes=$(expr $minutes + 1)
               done
 
     - Enable.images.repo:
@@ -1530,14 +1610,19 @@ pipelines:
           - staging-bot
         tasks:
           - script: |-
+              minutes=0
               osc -A $STAGING_API api -X POST "/source/$STAGING_PROJECT?cmd=remove_flag&repository=product&flag=build"
               export PYTHONPATH=$PWD/scripts
               while ./scripts/gocd/verify-repo-built-successful.py -A $STAGING_API -p $STAGING_PROJECT -r product; ret=$?; [ ${ret} -ne 0 ]; do
                 if [ ${ret} -eq 2 ]; then
                     echo "product repository not found. Project configuration issue?" >&2
                     exit 1
+                elif [ ${minutes} -gt 180 ]; then
+                    echo "Product is still failing after timeout, exiting..." >&2
+                    exit 1
                 fi
                 sleep 60
+                minutes=$(expr $minutes + 1)
               done
 
     - Enable.images.repo:
@@ -1615,14 +1700,19 @@ pipelines:
           - staging-bot
         tasks:
           - script: |-
+              minutes=0
               osc -A $STAGING_API api -X POST "/source/$STAGING_PROJECT?cmd=remove_flag&repository=product&flag=build"
               export PYTHONPATH=$PWD/scripts
               while ./scripts/gocd/verify-repo-built-successful.py -A $STAGING_API -p $STAGING_PROJECT -r product; ret=$?; [ ${ret} -ne 0 ]; do
                 if [ ${ret} -eq 2 ]; then
                     echo "product repository not found. Project configuration issue?" >&2
                     exit 1
+                elif [ ${minutes} -gt 180 ]; then
+                    echo "Product is still failing after timeout, exiting..." >&2
+                    exit 1
                 fi
                 sleep 60
+                minutes=$(expr $minutes + 1)
               done
 
     - Enable.images.repo:
@@ -1700,14 +1790,19 @@ pipelines:
           - staging-bot
         tasks:
           - script: |-
+              minutes=0
               osc -A $STAGING_API api -X POST "/source/$STAGING_PROJECT?cmd=remove_flag&repository=product&flag=build"
               export PYTHONPATH=$PWD/scripts
               while ./scripts/gocd/verify-repo-built-successful.py -A $STAGING_API -p $STAGING_PROJECT -r product; ret=$?; [ ${ret} -ne 0 ]; do
                 if [ ${ret} -eq 2 ]; then
                     echo "product repository not found. Project configuration issue?" >&2
                     exit 1
+                elif [ ${minutes} -gt 180 ]; then
+                    echo "Product is still failing after timeout, exiting..." >&2
+                    exit 1
                 fi
                 sleep 60
+                minutes=$(expr $minutes + 1)
               done
 
     - Enable.images.repo:
@@ -1785,14 +1880,19 @@ pipelines:
           - staging-bot
         tasks:
           - script: |-
+              minutes=0
               osc -A $STAGING_API api -X POST "/source/$STAGING_PROJECT?cmd=remove_flag&repository=product&flag=build"
               export PYTHONPATH=$PWD/scripts
               while ./scripts/gocd/verify-repo-built-successful.py -A $STAGING_API -p $STAGING_PROJECT -r product; ret=$?; [ ${ret} -ne 0 ]; do
                 if [ ${ret} -eq 2 ]; then
                     echo "product repository not found. Project configuration issue?" >&2
                     exit 1
+                elif [ ${minutes} -gt 180 ]; then
+                    echo "Product is still failing after timeout, exiting..." >&2
+                    exit 1
                 fi
                 sleep 60
+                minutes=$(expr $minutes + 1)
               done
 
     - Enable.images.repo:
@@ -1870,14 +1970,19 @@ pipelines:
           - staging-bot
         tasks:
           - script: |-
+              minutes=0
               osc -A $STAGING_API api -X POST "/source/$STAGING_PROJECT?cmd=remove_flag&repository=product&flag=build"
               export PYTHONPATH=$PWD/scripts
               while ./scripts/gocd/verify-repo-built-successful.py -A $STAGING_API -p $STAGING_PROJECT -r product; ret=$?; [ ${ret} -ne 0 ]; do
                 if [ ${ret} -eq 2 ]; then
                     echo "product repository not found. Project configuration issue?" >&2
                     exit 1
+                elif [ ${minutes} -gt 180 ]; then
+                    echo "Product is still failing after timeout, exiting..." >&2
+                    exit 1
                 fi
                 sleep 60
+                minutes=$(expr $minutes + 1)
               done
 
     - Enable.images.repo:
@@ -1955,14 +2060,19 @@ pipelines:
           - staging-bot
         tasks:
           - script: |-
+              minutes=0
               osc -A $STAGING_API api -X POST "/source/$STAGING_PROJECT?cmd=remove_flag&repository=product&flag=build"
               export PYTHONPATH=$PWD/scripts
               while ./scripts/gocd/verify-repo-built-successful.py -A $STAGING_API -p $STAGING_PROJECT -r product; ret=$?; [ ${ret} -ne 0 ]; do
                 if [ ${ret} -eq 2 ]; then
                     echo "product repository not found. Project configuration issue?" >&2
                     exit 1
+                elif [ ${minutes} -gt 180 ]; then
+                    echo "Product is still failing after timeout, exiting..." >&2
+                    exit 1
                 fi
                 sleep 60
+                minutes=$(expr $minutes + 1)
               done
 
     - Enable.images.repo:
diff --git a/gocd/slfo-stagings.gocd.yaml.erb b/gocd/slfo-stagings.gocd.yaml.erb
index ed3652ad6..7ba1be4dd 100644
--- a/gocd/slfo-stagings.gocd.yaml.erb
+++ b/gocd/slfo-stagings.gocd.yaml.erb
@@ -123,14 +123,19 @@ pipelines:
           - staging-bot
         tasks:
           - script: |-
+              minutes=0
               osc -A $STAGING_API api -X POST "/source/$STAGING_PROJECT?cmd=remove_flag&repository=product&flag=build"
               export PYTHONPATH=$PWD/scripts
               while ./scripts/gocd/verify-repo-built-successful.py -A $STAGING_API -p $STAGING_PROJECT -r product; ret=$?; [ ${ret} -ne 0 ]; do
                 if [ ${ret} -eq 2 ]; then
                     echo "product repository not found. Project configuration issue?" >&2
                     exit 1
+                elif [ ${minutes} -gt 180 ]; then
+                    echo "Product is still failing after timeout, exiting..." >&2
+                    exit 1
                 fi
                 sleep 60
+                minutes=$(expr $minutes + 1)
               done
 
     - Enable.images.repo:
@@ -268,14 +273,19 @@ pipelines:
           - staging-bot
         tasks:
           - script: |-
+              minutes=0
               osc -A $STAGING_API api -X POST "/source/$STAGING_PROJECT?cmd=remove_flag&repository=product&flag=build"
               export PYTHONPATH=$PWD/scripts
               while ./scripts/gocd/verify-repo-built-successful.py -A $STAGING_API -p $STAGING_PROJECT -r product; ret=$?; [ ${ret} -ne 0 ]; do
                 if [ ${ret} -eq 2 ]; then
                     echo "product repository not found. Project configuration issue?" >&2
                     exit 1
+                elif [ ${minutes} -gt 180 ]; then
+                    echo "Product is still failing after timeout, exiting..." >&2
+                    exit 1
                 fi
                 sleep 60
+                minutes=$(expr $minutes + 1)
               done
 
     - Enable.images.repo:

From ca997498e8f85db7b2e2c1faf34fed2102fd7d93 Mon Sep 17 00:00:00 2001
From: Max Lin <mlin@suse.com>
Date: Mon, 11 Nov 2024 17:34:43 +0800
Subject: [PATCH 3/6] Fix makeurl() usage in is_repo_dirty()

---
 osclib/stagingapi.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/osclib/stagingapi.py b/osclib/stagingapi.py
index 9b52cd867..1cb8b4cb3 100644
--- a/osclib/stagingapi.py
+++ b/osclib/stagingapi.py
@@ -1289,7 +1289,11 @@ def get_prj_results(self, prj, arch):
         return results
 
     def is_repo_dirty(self, project, repository):
-        url = self.makeurl(['build', project, f'_result?code=broken&repository={repository}'])
+        query = {
+            'code': 'broken',
+            'repository': repository
+        }
+        url = self.makeurl(['build', project, '_result'], query=query)
         root = ET.parse(http_GET(url)).getroot()
         for repo in root.findall('result'):
             repostate = repo.get('state', 'missing')

From 4c1d7be5d10457fa4a1545cebef79275d09345cf Mon Sep 17 00:00:00 2001
From: Eugenio Paolantonio <eugenio.paolantonio@suse.com>
Date: Tue, 12 Nov 2024 12:32:32 +0100
Subject: [PATCH 4/6] gocd: slfo-stagings: fail on broken/failed products

Rather than using verify-repo-built-successful.py, check with
the API directly whether the product is blocked, scheduled or
building.

Since it might happen for a product being broken/failed (even after
a successful pkglistgen run), just fail in that case.

Signed-off-by: Eugenio Paolantonio <eugenio.paolantonio@suse.com>
---
 gocd/slfo-stagings.gocd.yaml     | 198 +++++++++++++++++++------------
 gocd/slfo-stagings.gocd.yaml.erb |  18 +--
 2 files changed, 132 insertions(+), 84 deletions(-)

diff --git a/gocd/slfo-stagings.gocd.yaml b/gocd/slfo-stagings.gocd.yaml
index 1cf43be5e..daa834099 100644
--- a/gocd/slfo-stagings.gocd.yaml
+++ b/gocd/slfo-stagings.gocd.yaml
@@ -1160,20 +1160,24 @@ pipelines:
           - staging-bot
         tasks:
           - script: |-
-              minutes=0
+              minutes=1
               osc -A $STAGING_API api -X POST "/source/$STAGING_PROJECT?cmd=remove_flag&repository=product&flag=build"
               export PYTHONPATH=$PWD/scripts
-              while ./scripts/gocd/verify-repo-built-successful.py -A $STAGING_API -p $STAGING_PROJECT -r product; ret=$?; [ ${ret} -ne 0 ]; do
-                if [ ${ret} -eq 2 ]; then
-                    echo "product repository not found. Project configuration issue?" >&2
-                    exit 1
-                elif [ ${minutes} -gt 180 ]; then
-                    echo "Product is still failing after timeout, exiting..." >&2
+              # Let the scheduler warm up first
+              sleep 60
+              while osc -A $STAGING_API api -X GET "/build/$STAGING_PROJECT/_result?view=summary&repository=product&arch=local" | grep -E 'statuscount code="(blocked|scheduled|building)"'; do
+                if [ ${minutes} -gt 180 ]; then
+                    echo "Product is still building after timeout, exiting..." >&2
                     exit 1
                 fi
                 sleep 60
                 minutes=$(expr $minutes + 1)
               done
+              # Always fail on broken/failed products
+              if osc -A $STAGING_API api -X GET "/build/$STAGING_PROJECT/_result?view=summary&repository=product&arch=local" | grep -qE 'statuscount code="(broken|failed)"'; then
+                    echo "Some products failed to build, exiting..." >&2
+                    exit 1
+              fi
 
     - Enable.images.repo:
         resources:
@@ -1250,20 +1254,24 @@ pipelines:
           - staging-bot
         tasks:
           - script: |-
-              minutes=0
+              minutes=1
               osc -A $STAGING_API api -X POST "/source/$STAGING_PROJECT?cmd=remove_flag&repository=product&flag=build"
               export PYTHONPATH=$PWD/scripts
-              while ./scripts/gocd/verify-repo-built-successful.py -A $STAGING_API -p $STAGING_PROJECT -r product; ret=$?; [ ${ret} -ne 0 ]; do
-                if [ ${ret} -eq 2 ]; then
-                    echo "product repository not found. Project configuration issue?" >&2
-                    exit 1
-                elif [ ${minutes} -gt 180 ]; then
-                    echo "Product is still failing after timeout, exiting..." >&2
+              # Let the scheduler warm up first
+              sleep 60
+              while osc -A $STAGING_API api -X GET "/build/$STAGING_PROJECT/_result?view=summary&repository=product&arch=local" | grep -E 'statuscount code="(blocked|scheduled|building)"'; do
+                if [ ${minutes} -gt 180 ]; then
+                    echo "Product is still building after timeout, exiting..." >&2
                     exit 1
                 fi
                 sleep 60
                 minutes=$(expr $minutes + 1)
               done
+              # Always fail on broken/failed products
+              if osc -A $STAGING_API api -X GET "/build/$STAGING_PROJECT/_result?view=summary&repository=product&arch=local" | grep -qE 'statuscount code="(broken|failed)"'; then
+                    echo "Some products failed to build, exiting..." >&2
+                    exit 1
+              fi
 
     - Enable.images.repo:
         resources:
@@ -1340,20 +1348,24 @@ pipelines:
           - staging-bot
         tasks:
           - script: |-
-              minutes=0
+              minutes=1
               osc -A $STAGING_API api -X POST "/source/$STAGING_PROJECT?cmd=remove_flag&repository=product&flag=build"
               export PYTHONPATH=$PWD/scripts
-              while ./scripts/gocd/verify-repo-built-successful.py -A $STAGING_API -p $STAGING_PROJECT -r product; ret=$?; [ ${ret} -ne 0 ]; do
-                if [ ${ret} -eq 2 ]; then
-                    echo "product repository not found. Project configuration issue?" >&2
-                    exit 1
-                elif [ ${minutes} -gt 180 ]; then
-                    echo "Product is still failing after timeout, exiting..." >&2
+              # Let the scheduler warm up first
+              sleep 60
+              while osc -A $STAGING_API api -X GET "/build/$STAGING_PROJECT/_result?view=summary&repository=product&arch=local" | grep -E 'statuscount code="(blocked|scheduled|building)"'; do
+                if [ ${minutes} -gt 180 ]; then
+                    echo "Product is still building after timeout, exiting..." >&2
                     exit 1
                 fi
                 sleep 60
                 minutes=$(expr $minutes + 1)
               done
+              # Always fail on broken/failed products
+              if osc -A $STAGING_API api -X GET "/build/$STAGING_PROJECT/_result?view=summary&repository=product&arch=local" | grep -qE 'statuscount code="(broken|failed)"'; then
+                    echo "Some products failed to build, exiting..." >&2
+                    exit 1
+              fi
 
     - Enable.images.repo:
         resources:
@@ -1430,20 +1442,24 @@ pipelines:
           - staging-bot
         tasks:
           - script: |-
-              minutes=0
+              minutes=1
               osc -A $STAGING_API api -X POST "/source/$STAGING_PROJECT?cmd=remove_flag&repository=product&flag=build"
               export PYTHONPATH=$PWD/scripts
-              while ./scripts/gocd/verify-repo-built-successful.py -A $STAGING_API -p $STAGING_PROJECT -r product; ret=$?; [ ${ret} -ne 0 ]; do
-                if [ ${ret} -eq 2 ]; then
-                    echo "product repository not found. Project configuration issue?" >&2
-                    exit 1
-                elif [ ${minutes} -gt 180 ]; then
-                    echo "Product is still failing after timeout, exiting..." >&2
+              # Let the scheduler warm up first
+              sleep 60
+              while osc -A $STAGING_API api -X GET "/build/$STAGING_PROJECT/_result?view=summary&repository=product&arch=local" | grep -E 'statuscount code="(blocked|scheduled|building)"'; do
+                if [ ${minutes} -gt 180 ]; then
+                    echo "Product is still building after timeout, exiting..." >&2
                     exit 1
                 fi
                 sleep 60
                 minutes=$(expr $minutes + 1)
               done
+              # Always fail on broken/failed products
+              if osc -A $STAGING_API api -X GET "/build/$STAGING_PROJECT/_result?view=summary&repository=product&arch=local" | grep -qE 'statuscount code="(broken|failed)"'; then
+                    echo "Some products failed to build, exiting..." >&2
+                    exit 1
+              fi
 
     - Enable.images.repo:
         resources:
@@ -1520,20 +1536,24 @@ pipelines:
           - staging-bot
         tasks:
           - script: |-
-              minutes=0
+              minutes=1
               osc -A $STAGING_API api -X POST "/source/$STAGING_PROJECT?cmd=remove_flag&repository=product&flag=build"
               export PYTHONPATH=$PWD/scripts
-              while ./scripts/gocd/verify-repo-built-successful.py -A $STAGING_API -p $STAGING_PROJECT -r product; ret=$?; [ ${ret} -ne 0 ]; do
-                if [ ${ret} -eq 2 ]; then
-                    echo "product repository not found. Project configuration issue?" >&2
-                    exit 1
-                elif [ ${minutes} -gt 180 ]; then
-                    echo "Product is still failing after timeout, exiting..." >&2
+              # Let the scheduler warm up first
+              sleep 60
+              while osc -A $STAGING_API api -X GET "/build/$STAGING_PROJECT/_result?view=summary&repository=product&arch=local" | grep -E 'statuscount code="(blocked|scheduled|building)"'; do
+                if [ ${minutes} -gt 180 ]; then
+                    echo "Product is still building after timeout, exiting..." >&2
                     exit 1
                 fi
                 sleep 60
                 minutes=$(expr $minutes + 1)
               done
+              # Always fail on broken/failed products
+              if osc -A $STAGING_API api -X GET "/build/$STAGING_PROJECT/_result?view=summary&repository=product&arch=local" | grep -qE 'statuscount code="(broken|failed)"'; then
+                    echo "Some products failed to build, exiting..." >&2
+                    exit 1
+              fi
 
     - Enable.images.repo:
         resources:
@@ -1610,20 +1630,24 @@ pipelines:
           - staging-bot
         tasks:
           - script: |-
-              minutes=0
+              minutes=1
               osc -A $STAGING_API api -X POST "/source/$STAGING_PROJECT?cmd=remove_flag&repository=product&flag=build"
               export PYTHONPATH=$PWD/scripts
-              while ./scripts/gocd/verify-repo-built-successful.py -A $STAGING_API -p $STAGING_PROJECT -r product; ret=$?; [ ${ret} -ne 0 ]; do
-                if [ ${ret} -eq 2 ]; then
-                    echo "product repository not found. Project configuration issue?" >&2
-                    exit 1
-                elif [ ${minutes} -gt 180 ]; then
-                    echo "Product is still failing after timeout, exiting..." >&2
+              # Let the scheduler warm up first
+              sleep 60
+              while osc -A $STAGING_API api -X GET "/build/$STAGING_PROJECT/_result?view=summary&repository=product&arch=local" | grep -E 'statuscount code="(blocked|scheduled|building)"'; do
+                if [ ${minutes} -gt 180 ]; then
+                    echo "Product is still building after timeout, exiting..." >&2
                     exit 1
                 fi
                 sleep 60
                 minutes=$(expr $minutes + 1)
               done
+              # Always fail on broken/failed products
+              if osc -A $STAGING_API api -X GET "/build/$STAGING_PROJECT/_result?view=summary&repository=product&arch=local" | grep -qE 'statuscount code="(broken|failed)"'; then
+                    echo "Some products failed to build, exiting..." >&2
+                    exit 1
+              fi
 
     - Enable.images.repo:
         resources:
@@ -1700,20 +1724,24 @@ pipelines:
           - staging-bot
         tasks:
           - script: |-
-              minutes=0
+              minutes=1
               osc -A $STAGING_API api -X POST "/source/$STAGING_PROJECT?cmd=remove_flag&repository=product&flag=build"
               export PYTHONPATH=$PWD/scripts
-              while ./scripts/gocd/verify-repo-built-successful.py -A $STAGING_API -p $STAGING_PROJECT -r product; ret=$?; [ ${ret} -ne 0 ]; do
-                if [ ${ret} -eq 2 ]; then
-                    echo "product repository not found. Project configuration issue?" >&2
-                    exit 1
-                elif [ ${minutes} -gt 180 ]; then
-                    echo "Product is still failing after timeout, exiting..." >&2
+              # Let the scheduler warm up first
+              sleep 60
+              while osc -A $STAGING_API api -X GET "/build/$STAGING_PROJECT/_result?view=summary&repository=product&arch=local" | grep -E 'statuscount code="(blocked|scheduled|building)"'; do
+                if [ ${minutes} -gt 180 ]; then
+                    echo "Product is still building after timeout, exiting..." >&2
                     exit 1
                 fi
                 sleep 60
                 minutes=$(expr $minutes + 1)
               done
+              # Always fail on broken/failed products
+              if osc -A $STAGING_API api -X GET "/build/$STAGING_PROJECT/_result?view=summary&repository=product&arch=local" | grep -qE 'statuscount code="(broken|failed)"'; then
+                    echo "Some products failed to build, exiting..." >&2
+                    exit 1
+              fi
 
     - Enable.images.repo:
         resources:
@@ -1790,20 +1818,24 @@ pipelines:
           - staging-bot
         tasks:
           - script: |-
-              minutes=0
+              minutes=1
               osc -A $STAGING_API api -X POST "/source/$STAGING_PROJECT?cmd=remove_flag&repository=product&flag=build"
               export PYTHONPATH=$PWD/scripts
-              while ./scripts/gocd/verify-repo-built-successful.py -A $STAGING_API -p $STAGING_PROJECT -r product; ret=$?; [ ${ret} -ne 0 ]; do
-                if [ ${ret} -eq 2 ]; then
-                    echo "product repository not found. Project configuration issue?" >&2
-                    exit 1
-                elif [ ${minutes} -gt 180 ]; then
-                    echo "Product is still failing after timeout, exiting..." >&2
+              # Let the scheduler warm up first
+              sleep 60
+              while osc -A $STAGING_API api -X GET "/build/$STAGING_PROJECT/_result?view=summary&repository=product&arch=local" | grep -E 'statuscount code="(blocked|scheduled|building)"'; do
+                if [ ${minutes} -gt 180 ]; then
+                    echo "Product is still building after timeout, exiting..." >&2
                     exit 1
                 fi
                 sleep 60
                 minutes=$(expr $minutes + 1)
               done
+              # Always fail on broken/failed products
+              if osc -A $STAGING_API api -X GET "/build/$STAGING_PROJECT/_result?view=summary&repository=product&arch=local" | grep -qE 'statuscount code="(broken|failed)"'; then
+                    echo "Some products failed to build, exiting..." >&2
+                    exit 1
+              fi
 
     - Enable.images.repo:
         resources:
@@ -1880,20 +1912,24 @@ pipelines:
           - staging-bot
         tasks:
           - script: |-
-              minutes=0
+              minutes=1
               osc -A $STAGING_API api -X POST "/source/$STAGING_PROJECT?cmd=remove_flag&repository=product&flag=build"
               export PYTHONPATH=$PWD/scripts
-              while ./scripts/gocd/verify-repo-built-successful.py -A $STAGING_API -p $STAGING_PROJECT -r product; ret=$?; [ ${ret} -ne 0 ]; do
-                if [ ${ret} -eq 2 ]; then
-                    echo "product repository not found. Project configuration issue?" >&2
-                    exit 1
-                elif [ ${minutes} -gt 180 ]; then
-                    echo "Product is still failing after timeout, exiting..." >&2
+              # Let the scheduler warm up first
+              sleep 60
+              while osc -A $STAGING_API api -X GET "/build/$STAGING_PROJECT/_result?view=summary&repository=product&arch=local" | grep -E 'statuscount code="(blocked|scheduled|building)"'; do
+                if [ ${minutes} -gt 180 ]; then
+                    echo "Product is still building after timeout, exiting..." >&2
                     exit 1
                 fi
                 sleep 60
                 minutes=$(expr $minutes + 1)
               done
+              # Always fail on broken/failed products
+              if osc -A $STAGING_API api -X GET "/build/$STAGING_PROJECT/_result?view=summary&repository=product&arch=local" | grep -qE 'statuscount code="(broken|failed)"'; then
+                    echo "Some products failed to build, exiting..." >&2
+                    exit 1
+              fi
 
     - Enable.images.repo:
         resources:
@@ -1970,20 +2006,24 @@ pipelines:
           - staging-bot
         tasks:
           - script: |-
-              minutes=0
+              minutes=1
               osc -A $STAGING_API api -X POST "/source/$STAGING_PROJECT?cmd=remove_flag&repository=product&flag=build"
               export PYTHONPATH=$PWD/scripts
-              while ./scripts/gocd/verify-repo-built-successful.py -A $STAGING_API -p $STAGING_PROJECT -r product; ret=$?; [ ${ret} -ne 0 ]; do
-                if [ ${ret} -eq 2 ]; then
-                    echo "product repository not found. Project configuration issue?" >&2
-                    exit 1
-                elif [ ${minutes} -gt 180 ]; then
-                    echo "Product is still failing after timeout, exiting..." >&2
+              # Let the scheduler warm up first
+              sleep 60
+              while osc -A $STAGING_API api -X GET "/build/$STAGING_PROJECT/_result?view=summary&repository=product&arch=local" | grep -E 'statuscount code="(blocked|scheduled|building)"'; do
+                if [ ${minutes} -gt 180 ]; then
+                    echo "Product is still building after timeout, exiting..." >&2
                     exit 1
                 fi
                 sleep 60
                 minutes=$(expr $minutes + 1)
               done
+              # Always fail on broken/failed products
+              if osc -A $STAGING_API api -X GET "/build/$STAGING_PROJECT/_result?view=summary&repository=product&arch=local" | grep -qE 'statuscount code="(broken|failed)"'; then
+                    echo "Some products failed to build, exiting..." >&2
+                    exit 1
+              fi
 
     - Enable.images.repo:
         resources:
@@ -2060,20 +2100,24 @@ pipelines:
           - staging-bot
         tasks:
           - script: |-
-              minutes=0
+              minutes=1
               osc -A $STAGING_API api -X POST "/source/$STAGING_PROJECT?cmd=remove_flag&repository=product&flag=build"
               export PYTHONPATH=$PWD/scripts
-              while ./scripts/gocd/verify-repo-built-successful.py -A $STAGING_API -p $STAGING_PROJECT -r product; ret=$?; [ ${ret} -ne 0 ]; do
-                if [ ${ret} -eq 2 ]; then
-                    echo "product repository not found. Project configuration issue?" >&2
-                    exit 1
-                elif [ ${minutes} -gt 180 ]; then
-                    echo "Product is still failing after timeout, exiting..." >&2
+              # Let the scheduler warm up first
+              sleep 60
+              while osc -A $STAGING_API api -X GET "/build/$STAGING_PROJECT/_result?view=summary&repository=product&arch=local" | grep -E 'statuscount code="(blocked|scheduled|building)"'; do
+                if [ ${minutes} -gt 180 ]; then
+                    echo "Product is still building after timeout, exiting..." >&2
                     exit 1
                 fi
                 sleep 60
                 minutes=$(expr $minutes + 1)
               done
+              # Always fail on broken/failed products
+              if osc -A $STAGING_API api -X GET "/build/$STAGING_PROJECT/_result?view=summary&repository=product&arch=local" | grep -qE 'statuscount code="(broken|failed)"'; then
+                    echo "Some products failed to build, exiting..." >&2
+                    exit 1
+              fi
 
     - Enable.images.repo:
         resources:
diff --git a/gocd/slfo-stagings.gocd.yaml.erb b/gocd/slfo-stagings.gocd.yaml.erb
index 7ba1be4dd..606a95e34 100644
--- a/gocd/slfo-stagings.gocd.yaml.erb
+++ b/gocd/slfo-stagings.gocd.yaml.erb
@@ -273,20 +273,24 @@ pipelines:
           - staging-bot
         tasks:
           - script: |-
-              minutes=0
+              minutes=1
               osc -A $STAGING_API api -X POST "/source/$STAGING_PROJECT?cmd=remove_flag&repository=product&flag=build"
               export PYTHONPATH=$PWD/scripts
-              while ./scripts/gocd/verify-repo-built-successful.py -A $STAGING_API -p $STAGING_PROJECT -r product; ret=$?; [ ${ret} -ne 0 ]; do
-                if [ ${ret} -eq 2 ]; then
-                    echo "product repository not found. Project configuration issue?" >&2
-                    exit 1
-                elif [ ${minutes} -gt 180 ]; then
-                    echo "Product is still failing after timeout, exiting..." >&2
+              # Let the scheduler warm up first
+              sleep 60
+              while osc -A $STAGING_API api -X GET "/build/$STAGING_PROJECT/_result?view=summary&repository=product&arch=local" | grep -E 'statuscount code="(blocked|scheduled|building)"'; do
+                if [ ${minutes} -gt 180 ]; then
+                    echo "Product is still building after timeout, exiting..." >&2
                     exit 1
                 fi
                 sleep 60
                 minutes=$(expr $minutes + 1)
               done
+              # Always fail on broken/failed products
+              if osc -A $STAGING_API api -X GET "/build/$STAGING_PROJECT/_result?view=summary&repository=product&arch=local" | grep -qE 'statuscount code="(broken|failed)"'; then
+                    echo "Some products failed to build, exiting..." >&2
+                    exit 1
+              fi
 
     - Enable.images.repo:
         resources:

From acb6b3ee37ff05b80556702e1862ade982e83f4a Mon Sep 17 00:00:00 2001
From: Eugenio Paolantonio <eugenio.paolantonio@suse.com>
Date: Tue, 12 Nov 2024 14:42:28 +0100
Subject: [PATCH 5/6] gocd: slfo-stagings: properly escape comments

Follow up to 4c1d7be5d10457fa4a1545cebef79275d09345cf

Signed-off-by: Eugenio Paolantonio <eugenio.paolantonio@suse.com>
---
 gocd/slfo-stagings.gocd.yaml     | 44 ++++++++++++++++----------------
 gocd/slfo-stagings.gocd.yaml.erb |  4 +--
 2 files changed, 24 insertions(+), 24 deletions(-)

diff --git a/gocd/slfo-stagings.gocd.yaml b/gocd/slfo-stagings.gocd.yaml
index daa834099..5d15bbaec 100644
--- a/gocd/slfo-stagings.gocd.yaml
+++ b/gocd/slfo-stagings.gocd.yaml
@@ -1163,7 +1163,7 @@ pipelines:
               minutes=1
               osc -A $STAGING_API api -X POST "/source/$STAGING_PROJECT?cmd=remove_flag&repository=product&flag=build"
               export PYTHONPATH=$PWD/scripts
-              # Let the scheduler warm up first
+              ## Let the scheduler warm up first
               sleep 60
               while osc -A $STAGING_API api -X GET "/build/$STAGING_PROJECT/_result?view=summary&repository=product&arch=local" | grep -E 'statuscount code="(blocked|scheduled|building)"'; do
                 if [ ${minutes} -gt 180 ]; then
@@ -1173,7 +1173,7 @@ pipelines:
                 sleep 60
                 minutes=$(expr $minutes + 1)
               done
-              # Always fail on broken/failed products
+              ## Always fail on broken/failed products
               if osc -A $STAGING_API api -X GET "/build/$STAGING_PROJECT/_result?view=summary&repository=product&arch=local" | grep -qE 'statuscount code="(broken|failed)"'; then
                     echo "Some products failed to build, exiting..." >&2
                     exit 1
@@ -1257,7 +1257,7 @@ pipelines:
               minutes=1
               osc -A $STAGING_API api -X POST "/source/$STAGING_PROJECT?cmd=remove_flag&repository=product&flag=build"
               export PYTHONPATH=$PWD/scripts
-              # Let the scheduler warm up first
+              ## Let the scheduler warm up first
               sleep 60
               while osc -A $STAGING_API api -X GET "/build/$STAGING_PROJECT/_result?view=summary&repository=product&arch=local" | grep -E 'statuscount code="(blocked|scheduled|building)"'; do
                 if [ ${minutes} -gt 180 ]; then
@@ -1267,7 +1267,7 @@ pipelines:
                 sleep 60
                 minutes=$(expr $minutes + 1)
               done
-              # Always fail on broken/failed products
+              ## Always fail on broken/failed products
               if osc -A $STAGING_API api -X GET "/build/$STAGING_PROJECT/_result?view=summary&repository=product&arch=local" | grep -qE 'statuscount code="(broken|failed)"'; then
                     echo "Some products failed to build, exiting..." >&2
                     exit 1
@@ -1351,7 +1351,7 @@ pipelines:
               minutes=1
               osc -A $STAGING_API api -X POST "/source/$STAGING_PROJECT?cmd=remove_flag&repository=product&flag=build"
               export PYTHONPATH=$PWD/scripts
-              # Let the scheduler warm up first
+              ## Let the scheduler warm up first
               sleep 60
               while osc -A $STAGING_API api -X GET "/build/$STAGING_PROJECT/_result?view=summary&repository=product&arch=local" | grep -E 'statuscount code="(blocked|scheduled|building)"'; do
                 if [ ${minutes} -gt 180 ]; then
@@ -1361,7 +1361,7 @@ pipelines:
                 sleep 60
                 minutes=$(expr $minutes + 1)
               done
-              # Always fail on broken/failed products
+              ## Always fail on broken/failed products
               if osc -A $STAGING_API api -X GET "/build/$STAGING_PROJECT/_result?view=summary&repository=product&arch=local" | grep -qE 'statuscount code="(broken|failed)"'; then
                     echo "Some products failed to build, exiting..." >&2
                     exit 1
@@ -1445,7 +1445,7 @@ pipelines:
               minutes=1
               osc -A $STAGING_API api -X POST "/source/$STAGING_PROJECT?cmd=remove_flag&repository=product&flag=build"
               export PYTHONPATH=$PWD/scripts
-              # Let the scheduler warm up first
+              ## Let the scheduler warm up first
               sleep 60
               while osc -A $STAGING_API api -X GET "/build/$STAGING_PROJECT/_result?view=summary&repository=product&arch=local" | grep -E 'statuscount code="(blocked|scheduled|building)"'; do
                 if [ ${minutes} -gt 180 ]; then
@@ -1455,7 +1455,7 @@ pipelines:
                 sleep 60
                 minutes=$(expr $minutes + 1)
               done
-              # Always fail on broken/failed products
+              ## Always fail on broken/failed products
               if osc -A $STAGING_API api -X GET "/build/$STAGING_PROJECT/_result?view=summary&repository=product&arch=local" | grep -qE 'statuscount code="(broken|failed)"'; then
                     echo "Some products failed to build, exiting..." >&2
                     exit 1
@@ -1539,7 +1539,7 @@ pipelines:
               minutes=1
               osc -A $STAGING_API api -X POST "/source/$STAGING_PROJECT?cmd=remove_flag&repository=product&flag=build"
               export PYTHONPATH=$PWD/scripts
-              # Let the scheduler warm up first
+              ## Let the scheduler warm up first
               sleep 60
               while osc -A $STAGING_API api -X GET "/build/$STAGING_PROJECT/_result?view=summary&repository=product&arch=local" | grep -E 'statuscount code="(blocked|scheduled|building)"'; do
                 if [ ${minutes} -gt 180 ]; then
@@ -1549,7 +1549,7 @@ pipelines:
                 sleep 60
                 minutes=$(expr $minutes + 1)
               done
-              # Always fail on broken/failed products
+              ## Always fail on broken/failed products
               if osc -A $STAGING_API api -X GET "/build/$STAGING_PROJECT/_result?view=summary&repository=product&arch=local" | grep -qE 'statuscount code="(broken|failed)"'; then
                     echo "Some products failed to build, exiting..." >&2
                     exit 1
@@ -1633,7 +1633,7 @@ pipelines:
               minutes=1
               osc -A $STAGING_API api -X POST "/source/$STAGING_PROJECT?cmd=remove_flag&repository=product&flag=build"
               export PYTHONPATH=$PWD/scripts
-              # Let the scheduler warm up first
+              ## Let the scheduler warm up first
               sleep 60
               while osc -A $STAGING_API api -X GET "/build/$STAGING_PROJECT/_result?view=summary&repository=product&arch=local" | grep -E 'statuscount code="(blocked|scheduled|building)"'; do
                 if [ ${minutes} -gt 180 ]; then
@@ -1643,7 +1643,7 @@ pipelines:
                 sleep 60
                 minutes=$(expr $minutes + 1)
               done
-              # Always fail on broken/failed products
+              ## Always fail on broken/failed products
               if osc -A $STAGING_API api -X GET "/build/$STAGING_PROJECT/_result?view=summary&repository=product&arch=local" | grep -qE 'statuscount code="(broken|failed)"'; then
                     echo "Some products failed to build, exiting..." >&2
                     exit 1
@@ -1727,7 +1727,7 @@ pipelines:
               minutes=1
               osc -A $STAGING_API api -X POST "/source/$STAGING_PROJECT?cmd=remove_flag&repository=product&flag=build"
               export PYTHONPATH=$PWD/scripts
-              # Let the scheduler warm up first
+              ## Let the scheduler warm up first
               sleep 60
               while osc -A $STAGING_API api -X GET "/build/$STAGING_PROJECT/_result?view=summary&repository=product&arch=local" | grep -E 'statuscount code="(blocked|scheduled|building)"'; do
                 if [ ${minutes} -gt 180 ]; then
@@ -1737,7 +1737,7 @@ pipelines:
                 sleep 60
                 minutes=$(expr $minutes + 1)
               done
-              # Always fail on broken/failed products
+              ## Always fail on broken/failed products
               if osc -A $STAGING_API api -X GET "/build/$STAGING_PROJECT/_result?view=summary&repository=product&arch=local" | grep -qE 'statuscount code="(broken|failed)"'; then
                     echo "Some products failed to build, exiting..." >&2
                     exit 1
@@ -1821,7 +1821,7 @@ pipelines:
               minutes=1
               osc -A $STAGING_API api -X POST "/source/$STAGING_PROJECT?cmd=remove_flag&repository=product&flag=build"
               export PYTHONPATH=$PWD/scripts
-              # Let the scheduler warm up first
+              ## Let the scheduler warm up first
               sleep 60
               while osc -A $STAGING_API api -X GET "/build/$STAGING_PROJECT/_result?view=summary&repository=product&arch=local" | grep -E 'statuscount code="(blocked|scheduled|building)"'; do
                 if [ ${minutes} -gt 180 ]; then
@@ -1831,7 +1831,7 @@ pipelines:
                 sleep 60
                 minutes=$(expr $minutes + 1)
               done
-              # Always fail on broken/failed products
+              ## Always fail on broken/failed products
               if osc -A $STAGING_API api -X GET "/build/$STAGING_PROJECT/_result?view=summary&repository=product&arch=local" | grep -qE 'statuscount code="(broken|failed)"'; then
                     echo "Some products failed to build, exiting..." >&2
                     exit 1
@@ -1915,7 +1915,7 @@ pipelines:
               minutes=1
               osc -A $STAGING_API api -X POST "/source/$STAGING_PROJECT?cmd=remove_flag&repository=product&flag=build"
               export PYTHONPATH=$PWD/scripts
-              # Let the scheduler warm up first
+              ## Let the scheduler warm up first
               sleep 60
               while osc -A $STAGING_API api -X GET "/build/$STAGING_PROJECT/_result?view=summary&repository=product&arch=local" | grep -E 'statuscount code="(blocked|scheduled|building)"'; do
                 if [ ${minutes} -gt 180 ]; then
@@ -1925,7 +1925,7 @@ pipelines:
                 sleep 60
                 minutes=$(expr $minutes + 1)
               done
-              # Always fail on broken/failed products
+              ## Always fail on broken/failed products
               if osc -A $STAGING_API api -X GET "/build/$STAGING_PROJECT/_result?view=summary&repository=product&arch=local" | grep -qE 'statuscount code="(broken|failed)"'; then
                     echo "Some products failed to build, exiting..." >&2
                     exit 1
@@ -2009,7 +2009,7 @@ pipelines:
               minutes=1
               osc -A $STAGING_API api -X POST "/source/$STAGING_PROJECT?cmd=remove_flag&repository=product&flag=build"
               export PYTHONPATH=$PWD/scripts
-              # Let the scheduler warm up first
+              ## Let the scheduler warm up first
               sleep 60
               while osc -A $STAGING_API api -X GET "/build/$STAGING_PROJECT/_result?view=summary&repository=product&arch=local" | grep -E 'statuscount code="(blocked|scheduled|building)"'; do
                 if [ ${minutes} -gt 180 ]; then
@@ -2019,7 +2019,7 @@ pipelines:
                 sleep 60
                 minutes=$(expr $minutes + 1)
               done
-              # Always fail on broken/failed products
+              ## Always fail on broken/failed products
               if osc -A $STAGING_API api -X GET "/build/$STAGING_PROJECT/_result?view=summary&repository=product&arch=local" | grep -qE 'statuscount code="(broken|failed)"'; then
                     echo "Some products failed to build, exiting..." >&2
                     exit 1
@@ -2103,7 +2103,7 @@ pipelines:
               minutes=1
               osc -A $STAGING_API api -X POST "/source/$STAGING_PROJECT?cmd=remove_flag&repository=product&flag=build"
               export PYTHONPATH=$PWD/scripts
-              # Let the scheduler warm up first
+              ## Let the scheduler warm up first
               sleep 60
               while osc -A $STAGING_API api -X GET "/build/$STAGING_PROJECT/_result?view=summary&repository=product&arch=local" | grep -E 'statuscount code="(blocked|scheduled|building)"'; do
                 if [ ${minutes} -gt 180 ]; then
@@ -2113,7 +2113,7 @@ pipelines:
                 sleep 60
                 minutes=$(expr $minutes + 1)
               done
-              # Always fail on broken/failed products
+              ## Always fail on broken/failed products
               if osc -A $STAGING_API api -X GET "/build/$STAGING_PROJECT/_result?view=summary&repository=product&arch=local" | grep -qE 'statuscount code="(broken|failed)"'; then
                     echo "Some products failed to build, exiting..." >&2
                     exit 1
diff --git a/gocd/slfo-stagings.gocd.yaml.erb b/gocd/slfo-stagings.gocd.yaml.erb
index 606a95e34..5684b9664 100644
--- a/gocd/slfo-stagings.gocd.yaml.erb
+++ b/gocd/slfo-stagings.gocd.yaml.erb
@@ -276,7 +276,7 @@ pipelines:
               minutes=1
               osc -A $STAGING_API api -X POST "/source/$STAGING_PROJECT?cmd=remove_flag&repository=product&flag=build"
               export PYTHONPATH=$PWD/scripts
-              # Let the scheduler warm up first
+              ## Let the scheduler warm up first
               sleep 60
               while osc -A $STAGING_API api -X GET "/build/$STAGING_PROJECT/_result?view=summary&repository=product&arch=local" | grep -E 'statuscount code="(blocked|scheduled|building)"'; do
                 if [ ${minutes} -gt 180 ]; then
@@ -286,7 +286,7 @@ pipelines:
                 sleep 60
                 minutes=$(expr $minutes + 1)
               done
-              # Always fail on broken/failed products
+              ## Always fail on broken/failed products
               if osc -A $STAGING_API api -X GET "/build/$STAGING_PROJECT/_result?view=summary&repository=product&arch=local" | grep -qE 'statuscount code="(broken|failed)"'; then
                     echo "Some products failed to build, exiting..." >&2
                     exit 1

From 441d065418ad87d55862768876089969dcb57d94 Mon Sep 17 00:00:00 2001
From: Eugenio Paolantonio <eugenio.paolantonio@suse.com>
Date: Tue, 12 Nov 2024 15:38:25 +0100
Subject: [PATCH 6/6] gocd: slfo-stagings: slfo-stagings: fail on broken/failed
 products (SLFO:Main)

Rather than using verify-repo-built-successful.py, check with
the API directly whether the product is blocked, scheduled or
building.

Since it might happen for a product being broken/failed (even after
a successful pkglistgen run), just fail in that case.

This is #3191, but for SUSE:SLFO:Main rather than 1.1.

Signed-off-by: Eugenio Paolantonio <eugenio.paolantonio@suse.com>
---
 gocd/slfo-stagings.gocd.yaml     | 198 +++++++++++++++++++------------
 gocd/slfo-stagings.gocd.yaml.erb |  18 +--
 2 files changed, 132 insertions(+), 84 deletions(-)

diff --git a/gocd/slfo-stagings.gocd.yaml b/gocd/slfo-stagings.gocd.yaml
index 5d15bbaec..f13d017b2 100644
--- a/gocd/slfo-stagings.gocd.yaml
+++ b/gocd/slfo-stagings.gocd.yaml
@@ -122,20 +122,24 @@ pipelines:
           - staging-bot
         tasks:
           - script: |-
-              minutes=0
+              minutes=1
               osc -A $STAGING_API api -X POST "/source/$STAGING_PROJECT?cmd=remove_flag&repository=product&flag=build"
               export PYTHONPATH=$PWD/scripts
-              while ./scripts/gocd/verify-repo-built-successful.py -A $STAGING_API -p $STAGING_PROJECT -r product; ret=$?; [ ${ret} -ne 0 ]; do
-                if [ ${ret} -eq 2 ]; then
-                    echo "product repository not found. Project configuration issue?" >&2
-                    exit 1
-                elif [ ${minutes} -gt 180 ]; then
-                    echo "Product is still failing after timeout, exiting..." >&2
+              ## Let the scheduler warm up first
+              sleep 60
+              while osc -A $STAGING_API api -X GET "/build/$STAGING_PROJECT/_result?view=summary&repository=product&arch=local" | grep -E 'statuscount code="(blocked|scheduled|building)"'; do
+                if [ ${minutes} -gt 180 ]; then
+                    echo "Product is still building after timeout, exiting..." >&2
                     exit 1
                 fi
                 sleep 60
                 minutes=$(expr $minutes + 1)
               done
+              ## Always fail on broken/failed products
+              if osc -A $STAGING_API api -X GET "/build/$STAGING_PROJECT/_result?view=summary&repository=product&arch=local" | grep -qE 'statuscount code="(broken|failed)"'; then
+                    echo "Some products failed to build, exiting..." >&2
+                    exit 1
+              fi
 
     - Enable.images.repo:
         resources:
@@ -211,20 +215,24 @@ pipelines:
           - staging-bot
         tasks:
           - script: |-
-              minutes=0
+              minutes=1
               osc -A $STAGING_API api -X POST "/source/$STAGING_PROJECT?cmd=remove_flag&repository=product&flag=build"
               export PYTHONPATH=$PWD/scripts
-              while ./scripts/gocd/verify-repo-built-successful.py -A $STAGING_API -p $STAGING_PROJECT -r product; ret=$?; [ ${ret} -ne 0 ]; do
-                if [ ${ret} -eq 2 ]; then
-                    echo "product repository not found. Project configuration issue?" >&2
-                    exit 1
-                elif [ ${minutes} -gt 180 ]; then
-                    echo "Product is still failing after timeout, exiting..." >&2
+              ## Let the scheduler warm up first
+              sleep 60
+              while osc -A $STAGING_API api -X GET "/build/$STAGING_PROJECT/_result?view=summary&repository=product&arch=local" | grep -E 'statuscount code="(blocked|scheduled|building)"'; do
+                if [ ${minutes} -gt 180 ]; then
+                    echo "Product is still building after timeout, exiting..." >&2
                     exit 1
                 fi
                 sleep 60
                 minutes=$(expr $minutes + 1)
               done
+              ## Always fail on broken/failed products
+              if osc -A $STAGING_API api -X GET "/build/$STAGING_PROJECT/_result?view=summary&repository=product&arch=local" | grep -qE 'statuscount code="(broken|failed)"'; then
+                    echo "Some products failed to build, exiting..." >&2
+                    exit 1
+              fi
 
     - Enable.images.repo:
         resources:
@@ -300,20 +308,24 @@ pipelines:
           - staging-bot
         tasks:
           - script: |-
-              minutes=0
+              minutes=1
               osc -A $STAGING_API api -X POST "/source/$STAGING_PROJECT?cmd=remove_flag&repository=product&flag=build"
               export PYTHONPATH=$PWD/scripts
-              while ./scripts/gocd/verify-repo-built-successful.py -A $STAGING_API -p $STAGING_PROJECT -r product; ret=$?; [ ${ret} -ne 0 ]; do
-                if [ ${ret} -eq 2 ]; then
-                    echo "product repository not found. Project configuration issue?" >&2
-                    exit 1
-                elif [ ${minutes} -gt 180 ]; then
-                    echo "Product is still failing after timeout, exiting..." >&2
+              ## Let the scheduler warm up first
+              sleep 60
+              while osc -A $STAGING_API api -X GET "/build/$STAGING_PROJECT/_result?view=summary&repository=product&arch=local" | grep -E 'statuscount code="(blocked|scheduled|building)"'; do
+                if [ ${minutes} -gt 180 ]; then
+                    echo "Product is still building after timeout, exiting..." >&2
                     exit 1
                 fi
                 sleep 60
                 minutes=$(expr $minutes + 1)
               done
+              ## Always fail on broken/failed products
+              if osc -A $STAGING_API api -X GET "/build/$STAGING_PROJECT/_result?view=summary&repository=product&arch=local" | grep -qE 'statuscount code="(broken|failed)"'; then
+                    echo "Some products failed to build, exiting..." >&2
+                    exit 1
+              fi
 
     - Enable.images.repo:
         resources:
@@ -389,20 +401,24 @@ pipelines:
           - staging-bot
         tasks:
           - script: |-
-              minutes=0
+              minutes=1
               osc -A $STAGING_API api -X POST "/source/$STAGING_PROJECT?cmd=remove_flag&repository=product&flag=build"
               export PYTHONPATH=$PWD/scripts
-              while ./scripts/gocd/verify-repo-built-successful.py -A $STAGING_API -p $STAGING_PROJECT -r product; ret=$?; [ ${ret} -ne 0 ]; do
-                if [ ${ret} -eq 2 ]; then
-                    echo "product repository not found. Project configuration issue?" >&2
-                    exit 1
-                elif [ ${minutes} -gt 180 ]; then
-                    echo "Product is still failing after timeout, exiting..." >&2
+              ## Let the scheduler warm up first
+              sleep 60
+              while osc -A $STAGING_API api -X GET "/build/$STAGING_PROJECT/_result?view=summary&repository=product&arch=local" | grep -E 'statuscount code="(blocked|scheduled|building)"'; do
+                if [ ${minutes} -gt 180 ]; then
+                    echo "Product is still building after timeout, exiting..." >&2
                     exit 1
                 fi
                 sleep 60
                 minutes=$(expr $minutes + 1)
               done
+              ## Always fail on broken/failed products
+              if osc -A $STAGING_API api -X GET "/build/$STAGING_PROJECT/_result?view=summary&repository=product&arch=local" | grep -qE 'statuscount code="(broken|failed)"'; then
+                    echo "Some products failed to build, exiting..." >&2
+                    exit 1
+              fi
 
     - Enable.images.repo:
         resources:
@@ -478,20 +494,24 @@ pipelines:
           - staging-bot
         tasks:
           - script: |-
-              minutes=0
+              minutes=1
               osc -A $STAGING_API api -X POST "/source/$STAGING_PROJECT?cmd=remove_flag&repository=product&flag=build"
               export PYTHONPATH=$PWD/scripts
-              while ./scripts/gocd/verify-repo-built-successful.py -A $STAGING_API -p $STAGING_PROJECT -r product; ret=$?; [ ${ret} -ne 0 ]; do
-                if [ ${ret} -eq 2 ]; then
-                    echo "product repository not found. Project configuration issue?" >&2
-                    exit 1
-                elif [ ${minutes} -gt 180 ]; then
-                    echo "Product is still failing after timeout, exiting..." >&2
+              ## Let the scheduler warm up first
+              sleep 60
+              while osc -A $STAGING_API api -X GET "/build/$STAGING_PROJECT/_result?view=summary&repository=product&arch=local" | grep -E 'statuscount code="(blocked|scheduled|building)"'; do
+                if [ ${minutes} -gt 180 ]; then
+                    echo "Product is still building after timeout, exiting..." >&2
                     exit 1
                 fi
                 sleep 60
                 minutes=$(expr $minutes + 1)
               done
+              ## Always fail on broken/failed products
+              if osc -A $STAGING_API api -X GET "/build/$STAGING_PROJECT/_result?view=summary&repository=product&arch=local" | grep -qE 'statuscount code="(broken|failed)"'; then
+                    echo "Some products failed to build, exiting..." >&2
+                    exit 1
+              fi
 
     - Enable.images.repo:
         resources:
@@ -567,20 +587,24 @@ pipelines:
           - staging-bot
         tasks:
           - script: |-
-              minutes=0
+              minutes=1
               osc -A $STAGING_API api -X POST "/source/$STAGING_PROJECT?cmd=remove_flag&repository=product&flag=build"
               export PYTHONPATH=$PWD/scripts
-              while ./scripts/gocd/verify-repo-built-successful.py -A $STAGING_API -p $STAGING_PROJECT -r product; ret=$?; [ ${ret} -ne 0 ]; do
-                if [ ${ret} -eq 2 ]; then
-                    echo "product repository not found. Project configuration issue?" >&2
-                    exit 1
-                elif [ ${minutes} -gt 180 ]; then
-                    echo "Product is still failing after timeout, exiting..." >&2
+              ## Let the scheduler warm up first
+              sleep 60
+              while osc -A $STAGING_API api -X GET "/build/$STAGING_PROJECT/_result?view=summary&repository=product&arch=local" | grep -E 'statuscount code="(blocked|scheduled|building)"'; do
+                if [ ${minutes} -gt 180 ]; then
+                    echo "Product is still building after timeout, exiting..." >&2
                     exit 1
                 fi
                 sleep 60
                 minutes=$(expr $minutes + 1)
               done
+              ## Always fail on broken/failed products
+              if osc -A $STAGING_API api -X GET "/build/$STAGING_PROJECT/_result?view=summary&repository=product&arch=local" | grep -qE 'statuscount code="(broken|failed)"'; then
+                    echo "Some products failed to build, exiting..." >&2
+                    exit 1
+              fi
 
     - Enable.images.repo:
         resources:
@@ -656,20 +680,24 @@ pipelines:
           - staging-bot
         tasks:
           - script: |-
-              minutes=0
+              minutes=1
               osc -A $STAGING_API api -X POST "/source/$STAGING_PROJECT?cmd=remove_flag&repository=product&flag=build"
               export PYTHONPATH=$PWD/scripts
-              while ./scripts/gocd/verify-repo-built-successful.py -A $STAGING_API -p $STAGING_PROJECT -r product; ret=$?; [ ${ret} -ne 0 ]; do
-                if [ ${ret} -eq 2 ]; then
-                    echo "product repository not found. Project configuration issue?" >&2
-                    exit 1
-                elif [ ${minutes} -gt 180 ]; then
-                    echo "Product is still failing after timeout, exiting..." >&2
+              ## Let the scheduler warm up first
+              sleep 60
+              while osc -A $STAGING_API api -X GET "/build/$STAGING_PROJECT/_result?view=summary&repository=product&arch=local" | grep -E 'statuscount code="(blocked|scheduled|building)"'; do
+                if [ ${minutes} -gt 180 ]; then
+                    echo "Product is still building after timeout, exiting..." >&2
                     exit 1
                 fi
                 sleep 60
                 minutes=$(expr $minutes + 1)
               done
+              ## Always fail on broken/failed products
+              if osc -A $STAGING_API api -X GET "/build/$STAGING_PROJECT/_result?view=summary&repository=product&arch=local" | grep -qE 'statuscount code="(broken|failed)"'; then
+                    echo "Some products failed to build, exiting..." >&2
+                    exit 1
+              fi
 
     - Enable.images.repo:
         resources:
@@ -745,20 +773,24 @@ pipelines:
           - staging-bot
         tasks:
           - script: |-
-              minutes=0
+              minutes=1
               osc -A $STAGING_API api -X POST "/source/$STAGING_PROJECT?cmd=remove_flag&repository=product&flag=build"
               export PYTHONPATH=$PWD/scripts
-              while ./scripts/gocd/verify-repo-built-successful.py -A $STAGING_API -p $STAGING_PROJECT -r product; ret=$?; [ ${ret} -ne 0 ]; do
-                if [ ${ret} -eq 2 ]; then
-                    echo "product repository not found. Project configuration issue?" >&2
-                    exit 1
-                elif [ ${minutes} -gt 180 ]; then
-                    echo "Product is still failing after timeout, exiting..." >&2
+              ## Let the scheduler warm up first
+              sleep 60
+              while osc -A $STAGING_API api -X GET "/build/$STAGING_PROJECT/_result?view=summary&repository=product&arch=local" | grep -E 'statuscount code="(blocked|scheduled|building)"'; do
+                if [ ${minutes} -gt 180 ]; then
+                    echo "Product is still building after timeout, exiting..." >&2
                     exit 1
                 fi
                 sleep 60
                 minutes=$(expr $minutes + 1)
               done
+              ## Always fail on broken/failed products
+              if osc -A $STAGING_API api -X GET "/build/$STAGING_PROJECT/_result?view=summary&repository=product&arch=local" | grep -qE 'statuscount code="(broken|failed)"'; then
+                    echo "Some products failed to build, exiting..." >&2
+                    exit 1
+              fi
 
     - Enable.images.repo:
         resources:
@@ -834,20 +866,24 @@ pipelines:
           - staging-bot
         tasks:
           - script: |-
-              minutes=0
+              minutes=1
               osc -A $STAGING_API api -X POST "/source/$STAGING_PROJECT?cmd=remove_flag&repository=product&flag=build"
               export PYTHONPATH=$PWD/scripts
-              while ./scripts/gocd/verify-repo-built-successful.py -A $STAGING_API -p $STAGING_PROJECT -r product; ret=$?; [ ${ret} -ne 0 ]; do
-                if [ ${ret} -eq 2 ]; then
-                    echo "product repository not found. Project configuration issue?" >&2
-                    exit 1
-                elif [ ${minutes} -gt 180 ]; then
-                    echo "Product is still failing after timeout, exiting..." >&2
+              ## Let the scheduler warm up first
+              sleep 60
+              while osc -A $STAGING_API api -X GET "/build/$STAGING_PROJECT/_result?view=summary&repository=product&arch=local" | grep -E 'statuscount code="(blocked|scheduled|building)"'; do
+                if [ ${minutes} -gt 180 ]; then
+                    echo "Product is still building after timeout, exiting..." >&2
                     exit 1
                 fi
                 sleep 60
                 minutes=$(expr $minutes + 1)
               done
+              ## Always fail on broken/failed products
+              if osc -A $STAGING_API api -X GET "/build/$STAGING_PROJECT/_result?view=summary&repository=product&arch=local" | grep -qE 'statuscount code="(broken|failed)"'; then
+                    echo "Some products failed to build, exiting..." >&2
+                    exit 1
+              fi
 
     - Enable.images.repo:
         resources:
@@ -923,20 +959,24 @@ pipelines:
           - staging-bot
         tasks:
           - script: |-
-              minutes=0
+              minutes=1
               osc -A $STAGING_API api -X POST "/source/$STAGING_PROJECT?cmd=remove_flag&repository=product&flag=build"
               export PYTHONPATH=$PWD/scripts
-              while ./scripts/gocd/verify-repo-built-successful.py -A $STAGING_API -p $STAGING_PROJECT -r product; ret=$?; [ ${ret} -ne 0 ]; do
-                if [ ${ret} -eq 2 ]; then
-                    echo "product repository not found. Project configuration issue?" >&2
-                    exit 1
-                elif [ ${minutes} -gt 180 ]; then
-                    echo "Product is still failing after timeout, exiting..." >&2
+              ## Let the scheduler warm up first
+              sleep 60
+              while osc -A $STAGING_API api -X GET "/build/$STAGING_PROJECT/_result?view=summary&repository=product&arch=local" | grep -E 'statuscount code="(blocked|scheduled|building)"'; do
+                if [ ${minutes} -gt 180 ]; then
+                    echo "Product is still building after timeout, exiting..." >&2
                     exit 1
                 fi
                 sleep 60
                 minutes=$(expr $minutes + 1)
               done
+              ## Always fail on broken/failed products
+              if osc -A $STAGING_API api -X GET "/build/$STAGING_PROJECT/_result?view=summary&repository=product&arch=local" | grep -qE 'statuscount code="(broken|failed)"'; then
+                    echo "Some products failed to build, exiting..." >&2
+                    exit 1
+              fi
 
     - Enable.images.repo:
         resources:
@@ -1012,20 +1052,24 @@ pipelines:
           - staging-bot
         tasks:
           - script: |-
-              minutes=0
+              minutes=1
               osc -A $STAGING_API api -X POST "/source/$STAGING_PROJECT?cmd=remove_flag&repository=product&flag=build"
               export PYTHONPATH=$PWD/scripts
-              while ./scripts/gocd/verify-repo-built-successful.py -A $STAGING_API -p $STAGING_PROJECT -r product; ret=$?; [ ${ret} -ne 0 ]; do
-                if [ ${ret} -eq 2 ]; then
-                    echo "product repository not found. Project configuration issue?" >&2
-                    exit 1
-                elif [ ${minutes} -gt 180 ]; then
-                    echo "Product is still failing after timeout, exiting..." >&2
+              ## Let the scheduler warm up first
+              sleep 60
+              while osc -A $STAGING_API api -X GET "/build/$STAGING_PROJECT/_result?view=summary&repository=product&arch=local" | grep -E 'statuscount code="(blocked|scheduled|building)"'; do
+                if [ ${minutes} -gt 180 ]; then
+                    echo "Product is still building after timeout, exiting..." >&2
                     exit 1
                 fi
                 sleep 60
                 minutes=$(expr $minutes + 1)
               done
+              ## Always fail on broken/failed products
+              if osc -A $STAGING_API api -X GET "/build/$STAGING_PROJECT/_result?view=summary&repository=product&arch=local" | grep -qE 'statuscount code="(broken|failed)"'; then
+                    echo "Some products failed to build, exiting..." >&2
+                    exit 1
+              fi
 
     - Enable.images.repo:
         resources:
diff --git a/gocd/slfo-stagings.gocd.yaml.erb b/gocd/slfo-stagings.gocd.yaml.erb
index 5684b9664..7df80517b 100644
--- a/gocd/slfo-stagings.gocd.yaml.erb
+++ b/gocd/slfo-stagings.gocd.yaml.erb
@@ -123,20 +123,24 @@ pipelines:
           - staging-bot
         tasks:
           - script: |-
-              minutes=0
+              minutes=1
               osc -A $STAGING_API api -X POST "/source/$STAGING_PROJECT?cmd=remove_flag&repository=product&flag=build"
               export PYTHONPATH=$PWD/scripts
-              while ./scripts/gocd/verify-repo-built-successful.py -A $STAGING_API -p $STAGING_PROJECT -r product; ret=$?; [ ${ret} -ne 0 ]; do
-                if [ ${ret} -eq 2 ]; then
-                    echo "product repository not found. Project configuration issue?" >&2
-                    exit 1
-                elif [ ${minutes} -gt 180 ]; then
-                    echo "Product is still failing after timeout, exiting..." >&2
+              ## Let the scheduler warm up first
+              sleep 60
+              while osc -A $STAGING_API api -X GET "/build/$STAGING_PROJECT/_result?view=summary&repository=product&arch=local" | grep -E 'statuscount code="(blocked|scheduled|building)"'; do
+                if [ ${minutes} -gt 180 ]; then
+                    echo "Product is still building after timeout, exiting..." >&2
                     exit 1
                 fi
                 sleep 60
                 minutes=$(expr $minutes + 1)
               done
+              ## Always fail on broken/failed products
+              if osc -A $STAGING_API api -X GET "/build/$STAGING_PROJECT/_result?view=summary&repository=product&arch=local" | grep -qE 'statuscount code="(broken|failed)"'; then
+                    echo "Some products failed to build, exiting..." >&2
+                    exit 1
+              fi
 
     - Enable.images.repo:
         resources: