Avoid submitting packages to Factory that we haven't actually tested yet

The packages on `devel:openQA` might change while openQA-in-openQA tests
are executing. So far we submit packages directly from `devel:openQA`
after openQA-in-openQA have passed. This means we don't neccassarily submit
the version of the packages we have actually tested.

With this change we "release" a snapshot of `devel:openQA` to the staging
repository `devel:openQA:testing` when triggering tests and use that within
openQA-in-openQA tests (see
os-autoinst/os-autoinst-distri-openQA#215). We then
only submit from this staging repository when tests have passed. This way
we only submit what we've tested despite changes on `devel:openQA`.

Packages in the staging repository are cleaned up when openQA-in-openQA
tests fail or after packages have been submitted. This trigger script will
only release a new version into the staging project if it has been cleaned
up ensuring its contents stay the same while tests/submissions are pending.

This commit also adds the new script `cleanup-obs-project`. It conveniently
deletes all packages in an OBS project without deleting the project itself.
We might want to add it on Jenkins as post-failure action for monitoring
and submissions to avoid leftovers. (It is also useful to test the deletion
code on its own.)

Related ticket: https://progress.opensuse.org/issues/167395

_common

@@ -345,3 +345,17 @@ get-dependencies-ajax() {
     local tid=$1
     openqa-cli "${client_args[@]}" --apibase '' --json tests/"$tid"/dependencies_ajax
 }
+
+list_packages() {
+    local obs_project=$1
+    osc ls "$obs_project" | grep -v '\-test$'
+}
+
+delete_packages_from_obs_project() {
+    local obs_project=$1
+    local packages
+    packages=$(list_packages "$obs_project") || true # osc ls returns non-zero return code for empty projects
+    for package in $packages; do
+        osc rdelete -m "Cleaning up $package to allow triggering new jobs" "$obs_project" "$package"
+    done
+}

cleanup-obs-project

@@ -0,0 +1,18 @@
+#!/bin/bash -e
+
+# Cleans up a project on OBS
+
+# read/validate arguments
+obs_project=$1 confirmation=$2
+[[ $obs_project ]] || exit 1
+if [[ $confirmation != 'I am sure' ]]; then
+    echo "Skipping, pass 'I am sure' as 2nd argument to confirm"
+    exit 2
+fi
+
+set -euo pipefail
+
+# shellcheck source=/dev/null
+. "$(dirname "$0")"/_common
+
+delete_packages_from_obs_project "$obs_project"

monitor-openqa_job

@@ -16,6 +16,7 @@ host="${host:-"https://openqa.opensuse.org"}"
 openqa_groupid="${openqa_groupid:-"24"}"
 obs_component="${obs_component:-"package"}"
 obs_package_name="${1:-""}"
+staging_project=${dst_project:-devel:openQA:testing}
 comment_on_obs=${comment_on_obs:-}
 
 # shellcheck source=/dev/null
@@ -41,6 +42,11 @@ for job_id in $(job_ids job_post_response); do
 done
 
 [[ ${failed_jobs[*]} ]] || exit 0
+
+# delete packages from staging project in error case as we will not continue submitting those packages
+delete_packages_from_obs_project "$staging_project"
+
+# comment on failed jobs
 [[ $comment_on_obs ]] || exit 1
 
 osc api "/comments/$obs_component/$obs_package_name" | grep id= | sed -n 's/.*id="\([^"]*\)">.*test.* failed.*/\1/p' | while read -r id; do

os-autoinst-obs-auto-submit

@@ -3,8 +3,9 @@
 set -euo pipefail
 
 export LC_ALL=C
-src_project="${src_project:-"devel:openQA"}"
-dst_project="${dst_project:-"${src_project}:tested"}"
+src_project=${src_project:-devel:openQA}
+dst_project=${dst_project:-${src_project}:tested}
+staging_project=${staging_project:-${src_project}:testing}
 submit_target="${submit_target:-"openSUSE:Factory"}"
 submit_target_escaped=$(echo -n "$submit_target" | sed -e 's|\:|_|g') # avoid, e.g. "repoid 'openSUSE:Factory' is illegal"
 dry_run="${dry_run:-"0"}"
@@ -13,6 +14,9 @@ osc_build_start_poll_tries="${osc_build_start_poll_tries:-30}"
 XMLSTARLET=$(command -v xmlstarlet || true)
 [[ -n $XMLSTARLET ]] || (echo "Need xmlstarlet" && exit 1)
 
+# shellcheck source=/dev/null
+. "$(dirname "$0")"/_common
+
 factory_request() {
     local package=$1
     # writing xpath in url encoding is not for beginners, so don't stare at it too long :)
@@ -161,12 +165,18 @@ handle_auto_submit() {
         cd "$dst_project"/"$package"
         update_package "$package"
     )
+
+    # delete package from staging project
+    [[ $from_staging ]] && $osc rdelete -m "Cleaning up $package from $staging_project for next submission" "$staging_project" "$package"
 }
 
 prefix="${prefix:-""}"
 [ "$dry_run" = "1" ] && prefix="echo"
 osc="${osc:-"$prefix retry -e -- osc"}"
 
+# submit from staging project if specified
+[[ $staging_project && $staging_project != none ]] && src_project=$staging_project from_staging=1
+
 TMPDIR=${TMPDIR:-$(mktemp -d -t os-autoinst-obs-auto-submit-XXXX)}
 trap 'rm -rf "$TMPDIR"' EXIT
 

trigger-openqa_in_openqa

@@ -22,12 +22,20 @@ build_tag=${BUILD_TAG:-}
 client_prefix=${client_prefix:-}
 full_run=${FULL:-}
 group_id="${group_id:-"openQA"}"
+osc=${osc:-osc}
+src_project=${src_project:-devel:openQA}
+staging_project=${staging_project:-devel:openQA:testing}
+dst_project=${dst_project:-devel:openQA:tested}
+
+# shellcheck source=/dev/null
+. "$(dirname "$0")"/_common
 
 main() {
-    [ "$dry_run" = "1" ] && client_prefix="echo"
+    [[ $dry_run == 1 ]] && client_prefix="echo" osc="echo $osc"
     local qcow build
     download_scenario
     download_latest_published_tumbleweed_image
+    create_devel_openqa_snapshot
     trigger
 }
 
@@ -48,6 +56,21 @@ download_scenario() {
     curl https://raw.githubusercontent.com/os-autoinst/os-autoinst-distri-openQA/master/scenario-definitions.yaml -o /var/tmp/sd.yaml
 }
 
+create_devel_openqa_snapshot() {
+    local auto_submit_packages staged_packages
+    auto_submit_packages=$(list_packages "$dst_project")
+    staged_packages=$(list_packages "$staging_project") || true # osc ls returns non-zero return code for empty projects
+    if [[ $staged_packages ]]; then
+        echo "Skipping, there are still openQA-in-openQA tests pending"
+        exit 0
+    fi
+    echo "Creating snapshots under $staging_project"
+    for package in $auto_submit_packages; do
+        echo "Creating snapshots of $package"
+        $osc release --no-delay --target-project "$staging_project" --target-repository=openSUSE_Tumbleweed -r openSUSE_Tumbleweed -a x86_64 "$src_project" "$package"
+    done
+}
+
 trigger() {
     # prevent host access problem when running within o3 infrastructure
     # where o3 is not reachable over https
@@ -66,6 +89,7 @@ trigger() {
         VERSION=Tumbleweed \
         DISTRI=openqa FLAVOR=dev ARCH="${arch}" \
         HDD_1="$qcow" BUILD="${build}" _GROUP="${group_id}" \
+        OPENQA_OBS_PROJECT="$staging_project" \
         "${ARGS[@]}" \
         | tee job_post_response
 }