From af3b461bf56624494db7c1da0a137788ab1e3dee Mon Sep 17 00:00:00 2001 From: Marius Kittler Date: Fri, 15 Nov 2024 14:48:06 +0100 Subject: [PATCH] Avoid submitting packages to Factory that we haven't actually tested yet The packages on `devel:openQA` might change while openQA-in-openQA tests are executing. So far we submit packages directly from `devel:openQA` after openQA-in-openQA have passed. This means we don't neccassarily submit the version of the packages we have actually tested. With this change we "release" a snapshot of `devel:openQA` to the staging repository `devel:openQA:testing` when triggering tests and use that within openQA-in-openQA tests (see https://github.com/os-autoinst/os-autoinst-distri-openQA/pull/215). We then only submit from this staging repository when tests have passed. This way we only submit what we've tested despite changes on `devel:openQA`. Packages in the staging repository are cleaned up when openQA-in-openQA tests fail or after packages have been submitted. This trigger script will only release a new version into the staging project if it has been cleaned up ensuring its contents stay the same while tests/submissions are pending. This commit also adds the new script `cleanup-obs-project`. It conveniently deletes all packages in an OBS project without deleting the project itself. We might want to add it on Jenkins as post-failure action for monitoring and submissions to avoid leftovers. (It is also useful to test the deletion code on its own.) Related ticket: https://progress.opensuse.org/issues/167395 --- _common | 14 ++++++++++++++ cleanup-obs-project | 18 ++++++++++++++++++ monitor-openqa_job | 6 ++++++ os-autoinst-obs-auto-submit | 14 ++++++++++++-- trigger-openqa_in_openqa | 26 +++++++++++++++++++++++++- 5 files changed, 75 insertions(+), 3 deletions(-) create mode 100755 cleanup-obs-project diff --git a/_common b/_common index c91814bd..7baef488 100644 --- a/_common +++ b/_common @@ -345,3 +345,17 @@ get-dependencies-ajax() { local tid=$1 openqa-cli "${client_args[@]}" --apibase '' --json tests/"$tid"/dependencies_ajax } + +list_packages() { + local obs_project=$1 + osc ls "$obs_project" | grep -v '\-test$' +} + +delete_packages_from_obs_project() { + local obs_project=$1 + local packages + packages=$(list_packages "$obs_project") || true # osc ls returns non-zero return code for empty projects + for package in $packages; do + osc rdelete -m "Cleaning up $package to allow triggering new jobs" "$obs_project" "$package" + done +} diff --git a/cleanup-obs-project b/cleanup-obs-project new file mode 100755 index 00000000..c6ed914f --- /dev/null +++ b/cleanup-obs-project @@ -0,0 +1,18 @@ +#!/bin/bash -e + +# Cleans up a project on OBS + +# read/validate arguments +obs_project=$1 confirmation=$2 +[[ $obs_project ]] || exit 1 +if [[ $confirmation != 'I am sure' ]]; then + echo "Skipping, pass 'I am sure' as 2nd argument to confirm" + exit 2 +fi + +set -euo pipefail + +# shellcheck source=/dev/null +. "$(dirname "$0")"/_common + +delete_packages_from_obs_project "$obs_project" diff --git a/monitor-openqa_job b/monitor-openqa_job index 8a82040d..4c95b7c5 100755 --- a/monitor-openqa_job +++ b/monitor-openqa_job @@ -16,6 +16,7 @@ host="${host:-"https://openqa.opensuse.org"}" openqa_groupid="${openqa_groupid:-"24"}" obs_component="${obs_component:-"package"}" obs_package_name="${1:-""}" +staging_project=${staging_project:-devel:openQA:testing} comment_on_obs=${comment_on_obs:-} # shellcheck source=/dev/null @@ -41,6 +42,11 @@ for job_id in $(job_ids job_post_response); do done [[ ${failed_jobs[*]} ]] || exit 0 + +# delete packages from staging project in error case as we will not continue submitting those packages +delete_packages_from_obs_project "$staging_project" + +# comment on failed jobs [[ $comment_on_obs ]] || exit 1 osc api "/comments/$obs_component/$obs_package_name" | grep id= | sed -n 's/.*id="\([^"]*\)">.*test.* failed.*/\1/p' | while read -r id; do diff --git a/os-autoinst-obs-auto-submit b/os-autoinst-obs-auto-submit index 84df5c02..5e72db88 100644 --- a/os-autoinst-obs-auto-submit +++ b/os-autoinst-obs-auto-submit @@ -3,8 +3,9 @@ set -euo pipefail export LC_ALL=C -src_project="${src_project:-"devel:openQA"}" -dst_project="${dst_project:-"${src_project}:tested"}" +src_project=${src_project:-devel:openQA} +dst_project=${dst_project:-${src_project}:tested} +staging_project=${staging_project:-${src_project}:testing} submit_target="${submit_target:-"openSUSE:Factory"}" submit_target_escaped=$(echo -n "$submit_target" | sed -e 's|\:|_|g') # avoid, e.g. "repoid 'openSUSE:Factory' is illegal" dry_run="${dry_run:-"0"}" @@ -13,6 +14,9 @@ osc_build_start_poll_tries="${osc_build_start_poll_tries:-30}" XMLSTARLET=$(command -v xmlstarlet || true) [[ -n $XMLSTARLET ]] || (echo "Need xmlstarlet" && exit 1) +# shellcheck source=/dev/null +. "$(dirname "$0")"/_common + factory_request() { local package=$1 # writing xpath in url encoding is not for beginners, so don't stare at it too long :) @@ -161,12 +165,18 @@ handle_auto_submit() { cd "$dst_project"/"$package" update_package "$package" ) + + # delete package from staging project + [[ $from_staging ]] && $osc rdelete -m "Cleaning up $package from $staging_project for next submission" "$staging_project" "$package" } prefix="${prefix:-""}" [ "$dry_run" = "1" ] && prefix="echo" osc="${osc:-"$prefix retry -e -- osc"}" +# submit from staging project if specified +[[ $staging_project && $staging_project != none ]] && src_project=$staging_project from_staging=1 + TMPDIR=${TMPDIR:-$(mktemp -d -t os-autoinst-obs-auto-submit-XXXX)} trap 'rm -rf "$TMPDIR"' EXIT diff --git a/trigger-openqa_in_openqa b/trigger-openqa_in_openqa index 094bdd79..f2897add 100755 --- a/trigger-openqa_in_openqa +++ b/trigger-openqa_in_openqa @@ -22,12 +22,20 @@ build_tag=${BUILD_TAG:-} client_prefix=${client_prefix:-} full_run=${FULL:-} group_id="${group_id:-"openQA"}" +osc=${osc:-osc} +src_project=${src_project:-devel:openQA} +staging_project=${staging_project:-devel:openQA:testing} +dst_project=${dst_project:-devel:openQA:tested} + +# shellcheck source=/dev/null +. "$(dirname "$0")"/_common main() { - [ "$dry_run" = "1" ] && client_prefix="echo" + [[ $dry_run == 1 ]] && client_prefix="echo" osc="echo $osc" local qcow build download_scenario download_latest_published_tumbleweed_image + create_devel_openqa_snapshot trigger } @@ -48,6 +56,21 @@ download_scenario() { curl https://raw.githubusercontent.com/os-autoinst/os-autoinst-distri-openQA/master/scenario-definitions.yaml -o /var/tmp/sd.yaml } +create_devel_openqa_snapshot() { + local auto_submit_packages staged_packages + auto_submit_packages=$(list_packages "$dst_project") + staged_packages=$(list_packages "$staging_project") || true # osc ls returns non-zero return code for empty projects + if [[ $staged_packages ]]; then + echo "Skipping, there are still openQA-in-openQA tests pending" + exit 0 + fi + echo "Creating snapshots under $staging_project" + for package in $auto_submit_packages; do + echo "Creating snapshots of $package" + $osc release --no-delay --target-project "$staging_project" --target-repository=openSUSE_Tumbleweed -r openSUSE_Tumbleweed -a x86_64 "$src_project" "$package" + done +} + trigger() { # prevent host access problem when running within o3 infrastructure # where o3 is not reachable over https @@ -66,6 +89,7 @@ trigger() { VERSION=Tumbleweed \ DISTRI=openqa FLAVOR=dev ARCH="${arch}" \ HDD_1="$qcow" BUILD="${build}" _GROUP="${group_id}" \ + OPENQA_OBS_PROJECT="$staging_project" \ "${ARGS[@]}" \ | tee job_post_response }