CI: Enable tests with Packit + Fedora Testing Farm

- Behave tests now remove dnf5 by default first as the dnf5 packgae is
  installed by default with `dnf install mock` (brings in weak
  dependencies).

- Build with Packit also into the stable Fedora, so we can later test on
  stable fedora.

- When run with TMT (not only in Testing Farm), we don't generate the
  Mock package with Tito anymore, but we download it pre-built from
  Copr.

- Packit config uses 'merge_pr_in_ci: false' because we don't want to
  have "git sha" mismatch like in
  packit/packit-service#2329

- Packit: move the `actions:` configuration top-level.

- Apply work-around for test failure #1300.

- Packit: start building also for pushes.

Relates: https://gitlab.com/testing-farm/tests/-/issues/2
Fixes: #1258

.fmf/version

@@ -0,0 +1 @@
+1

.packit.yaml

@@ -2,42 +2,59 @@
 # https://packit.dev/docs/configuration/
 
 ---
+actions:
+  create-archive:
+    - bash -c "tito build --tgz --test -o ."
+    - bash -c "ls -1t ./*.tar.gz | head -n 1"
+  get-current-version:
+    - bash -c "grep ^Version \"$PACKIT_CONFIG_PACKAGE_NAME.spec\" | awk '{ print $2 }'"
+
+# We need to keep the pushed SHA, it is later exported
+# as $PACKIT_COMMIT_SHA for Testing Farm jobs.
+merge_pr_in_ci: false
+
 packages:
   mock:
     specfile_path: mock.spec
     paths:
       - ./mock
-    actions:
-      create-archive:
-        - bash -c "tito build --tgz --test -o ."
-        - bash -c "ls -1t ./*.tar.gz | head -n 1"
-      get-current-version:
-        - bash -c "git describe --match mock-[0-9]* --abbrev=0 HEAD | egrep -o
-          [0-9]+\.[0-9]+"
     downstream_package_name: mock
     upstream_tag_template: mock-{version}
   mock-core-configs:
     specfile_path: mock-core-configs.spec
     paths:
       - ./mock-core-configs
-    actions:
-      create-archive:
-        - bash -c "tito build --tgz --test -o ."
-        - bash -c "ls -1t ./*.tar.gz | head -n 1"
-      get-current-version:
-        - bash -c "git describe --match mock-core-configs-[0-9]* --abbrev=0 HEAD
-          | egrep -o [0-9]+\.[0-9]+"
     downstream_package_name: mock-core-configs
     upstream_tag_template: mock-core-configs-{version}
 srpm_build_deps:
   - tito
   - git
+
 jobs:
-  - job: copr_build
-    trigger: pull_request
-    packages:
-      - mock
-      - mock-core-configs
-    metadata:
+  - <<: &pr_build
+      job: copr_build
+      trigger: pull_request
+      packages:
+        - mock
+        - mock-core-configs
       targets:
         - fedora-rawhide
+        - fedora-latest-stable
+
+  - <<: *pr_build
+    trigger: commit
+    owner: "@mock"
+    project: "mock"
+    branch: main
+
+  - <<: &pr_test
+      job: tests
+      trigger: pull_request
+      targets:
+        - fedora-latest-stable
+      # We don't want to run two test runs; so we limit this to a single package
+      packages:
+        - mock
+
+  - <<: *pr_test
+    trigger: commit

behave/features/steps/other.py

@@ -22,6 +22,28 @@
 # pylint: disable=missing-function-docstring,function-redefined
 
 
+def _first_int(string, max_lines=20):
+    for line in string.split("\n")[:max_lines]:
+        if not line:
+            continue
+        first_word = line.split()[0]
+        if first_word.isdigit():
+            return first_word
+    raise Exception("unexpected dnf history output")
+
+
+def add_cleanup_last_transaction(context):
+    # TODO: DNF5 support https://github.com/rpm-software-management/dnf5/issues/140
+    dnf = ["sudo", "/usr/bin/dnf-3", "-y", "history"]
+    _, out, _ = run(dnf + ["list"])
+    transaction_id = _first_int(out)
+
+    def _revert_transaction(_context):
+        cmd = dnf + ["undo", transaction_id]
+        assert_that(run(cmd)[0], equal_to(0))
+
+    context.add_cleanup(_revert_transaction, context)
+
 @given(u'an unique mock namespace')
 def step_impl(context):
     print("using uniqueext {}".format(context.uniqueext))
@@ -39,7 +61,14 @@ def step_impl(context, package, state):
     is_installed = bool(not is_installed)
 
     if "not" in state:
-        assert_that(is_installed, equal_to(False))
+        if not is_installed:
+            return  # nothing to do
+
+        # Remove the package and schedule its removal
+        cmd = ["sudo", "dnf", "-y", "remove", package]
+        assert_that(run(cmd)[0], equal_to(0))
+        # schedule removal
+        add_cleanup_last_transaction(context)
         return
 
     if is_installed:

mock/integration-tests/runtests.sh

@@ -1,4 +1,4 @@
-#!/bin/sh
+#!/bin/sh -x
 # vim:tw=0:ts=4:sw=4
 
 # this is a test script to run everything through its paces before you do a
@@ -32,6 +32,18 @@ trap '$MOCKCMD --clean; exit 1' INT HUP QUIT TERM
 # pre-populate yum cache for the rest of the commands below
 #
 
+# With TMT, we have the deps pre-installed by ansible.
+if test -z "$TMT_VERSION"; then
+    MOCKSRPM=$(
+        cd "$TOPDIR" || exit 1
+        tito build --srpm --offline | grep Wrote | grep src.rpm | awk '{ print $2 }'
+    )
+    test -e "$MOCKSRPM" || exit 1
+else
+    set -- /tmp/mock-test-srpms/mock-*.src.rpm
+    MOCKSRPM=$1
+fi
+
 if [ -e /usr/bin/dnf ]; then
     header "pre-populating the cache (DNF)"
     runcmd "$MOCKCMD --init --dnf"

mock/integration-tests/setup-playbook/files/install-copr-packages

@@ -0,0 +1,106 @@
+#! /bin/bash
+
+# Install RPM packages (and dependencies) identified by the (a) package names,
+# (b) upstream commmit and (c) Copr directory.  The short variant of the
+# upstream commit must be part of the packages' Release in NVR.
+#
+# ARG1: coprowner/projectname:dir
+# ARG2: upstream-commit
+# ARGN: packagename1 packagename2 ...
+
+# TODO: DNF5/YUM compat?
+DNF=/usr/bin/dnf-3
+REPOQUERY=( "$DNF" repoquery )
+DOWNLOAD=( "$DNF" download )
+
+copr_dir=$1 ; shift
+commit=$1 ; shift
+copr_uri=https://download.copr.fedorainfracloud.org/results
+
+copr_chroot() (
+    # mimic: https://github.com/rpm-software-management/dnf5/blob/c6edcd75260accf7070f261e5b406fcf1f5db328/dnf5-plugins/copr_plugin/copr_config.cpp#L71-L80
+    . /etc/os-release
+    name=$ID
+    version=$VERSION_ID
+    arch=$(rpm --eval %_arch)
+    if test "$name" = fedora; then
+        if test "$REDHAT_SUPPORT_PRODUCT_VERSION" = rawhide; then
+            version=rawhide
+        fi
+    fi
+    echo "$name-$version-$arch"
+)
+
+repo=$copr_uri/$copr_dir/$(copr_chroot)
+
+echo >&2 "using repo: $repo"
+
+repoid=xyztest
+
+export clean_cache=true
+
+
+repos=( "--repofrompath=$repoid,$repo"  )
+
+_repoquery() {
+    local opts=( "${repos[@]}" --disablerepo='*' --enablerepo "$repoid" )
+    if ${clean_cache-true}; then
+        opts+=( "--setopt=$repoid.metadata_expire=1" )
+    fi
+    local cmd=( "${REPOQUERY[@]}" "${opts[@]}" "$@" )
+    info "Executing: ${cmd[*]}"
+    "${cmd[@]}" 2>/dev/null
+}
+
+info() { echo >&2 "INFO: $*" ;  }
+error() { echo >&2 "ERROR: $*" ; false; }
+die() { echo >&2 "FATAL: $*" ; exit 1 ; }
+
+find_nvr() {
+    # ARGS: $1=pkg $2=commit
+    # RETURN: $find_nvr_result
+    # STATUS: true if found
+    local _pkgname=$1 _commit=${2:0:7} _found=false
+    while read -r name version release; do
+        test -z "$name" && continue
+        test "$name" = "$_pkgname" || continue
+        case $release in
+            *$_commit*)
+                find_nvr_result=$name-$version-$release
+                $_found && error "second occurence of $name-$version-$release"
+                _found=true
+                ;;
+            *)
+                continue
+                ;;
+        esac
+    done < <( _repoquery --qf='%{NAME} %{VERSION} %{RELEASE}\n' )
+    $_found || error "$_pkgname with $commit in release not found"
+}
+
+nvrs=()
+SECONDS=0
+TIMEOUT=${TIMEOUT-1200} # 20 minutes by default
+for pkg; do
+    while true; do
+        if find_nvr "$pkg" "$commit"; then
+            nvrs+=( "$find_nvr_result" )
+            clean_cache=false
+            break
+        fi
+        test "$SECONDS" -gt "$TIMEOUT" && die "The timeout ${TIMEOUT}s left"
+        clean_cache=true
+        sleep 5
+    done
+done
+
+if test -n "$SRPM_DOWNLOAD_DIR"; then
+    mkdir -p "$SRPM_DOWNLOAD_DIR"
+    cmd=( "${DOWNLOAD[@]}" "${repos[@]}"  '--disablerepo=*' --enablerepo xyztest
+          "${nvrs[@]}" --source --downloaddir "$SRPM_DOWNLOAD_DIR" )
+else
+    cmd=( "$DNF" -y install "${nvrs[@]}" "${repos[@]}" --nogpgcheck )
+fi
+
+info "Running: ${cmd[*]}"
+"${cmd[@]}"

mock/integration-tests/setup-playbook/files/install-mock-packages-built-by-packit

@@ -0,0 +1,23 @@
+#! /bin/bash -x
+
+# Install specific Mock RPM packages (and dependencies) from a Copr repository
+# created by Packit PR or PUSH workflow.  Pick the precise package version built
+# from the latest commit.
+
+_d=$(dirname "$(readlink -f "$0")")
+
+if test -n "$PACKIT_PR_ID"; then
+    PROJECT=packit/rpm-software-management-mock-$PACKIT_PR_ID
+    COMMIT=$PACKIT_COMMIT_SHA
+elif test -n "$PACKIT_COPR_PROJECT"; then
+    PROJECT=$PACKIT_COPR_PROJECT
+    COMMIT=$PACKIT_COMMIT_SHA
+elif test -n "$TEST_GIT_MAIN"; then
+    PROJECT=@mock/mock
+    COMMIT=$(curl -H "Accept: application/vnd.github.VERSION.sha"  https://api.github.com/repos/rpm-software-management/mock/commits/main)
+else
+    echo >&2 "Can't decide where to install packages from"
+    exit 1
+fi
+
+"$_d/install-copr-packages" "$PROJECT" "$COMMIT" "$@"

mock/integration-tests/setup-playbook/play-tf.yml

@@ -0,0 +1,25 @@
+---
+- name: Prepare the testing machine for running Mock tests
+  hosts: all
+  user: "root"
+  vars:
+    mock_test_username: mockbuild
+    mock_test_workdir: /home/mock/mock-testing
+    mock_gpg_dir: "/home/mockbuild/gpg"
+    mock_gpg_wrapper: "/home/mockbuild/gpg-mock"
+    mock_test_rpmmacros: /home/mockbuild/.rpmmacros
+    mock_lvm_volume: /test-lvm-disk
+    mock_clone: /home/mockbuild/mock
+    no_subscription_management: true
+
+  tasks:
+    - include_tasks: tasks/main.yml
+
+    - name: upload the "install copr package" script
+      copy:
+        src: "{{ item }}"
+        dest: "/usr/bin/{{ item }}"
+        mode: '0755'
+      loop:
+        - install-copr-packages
+        - install-mock-packages-built-by-packit

mock/integration-tests/setup-playbook/tasks/main.yml

@@ -112,6 +112,8 @@
     username: "{{ mock_rhn_user }}"
     password: "{{ mock_rhn_pass }}"
     pool: "Red Hat Enterprise Linux for Virtual Datacenters, Standard"
+  when:
+    - no_subscription_management is not defined or no_subscription_management is false
 
 - name: Clone Mock repo
   git:

mock/integration-tests/testenvironment

@@ -1,6 +1,5 @@
 TESTDIR=$(cd $(dirname $0); pwd; cd ..)
 TOPDIR=$(dirname $TESTDIR)
-MOCKSRPM=$( (cd $TOPDIR; tito build --srpm --offline |grep Wrote | grep src.rpm | awk '{ print $2}' ))
 SIMPLESRPM=$TESTDIR/test-C-1.1-0.src.rpm
 
 #VERBOSE=

testing-farm/plans/behave.fmf

@@ -0,0 +1,12 @@
+---
+summary: run the behave tests in Fedora Testing Farm
+discover:
+  - how: fmf
+    filter: "tag: behave"
+
+prepare:
+  - how: ansible
+    playbook: mock/integration-tests/setup-playbook/play-tf.yml
+
+execute:
+  - how: tmt

testing-farm/plans/old-testsuite.fmf

@@ -0,0 +1,12 @@
+---
+summary: run the old testsuite in Fedora Testing Farm
+discover:
+  - how: fmf
+    filter: "tag: old_testsuite"
+
+prepare:
+  - how: ansible
+    playbook: mock/integration-tests/setup-playbook/play-tf.yml
+
+execute:
+  - how: tmt

testing-farm/tests/behave/main.fmf

@@ -0,0 +1,9 @@
+---
+summary: Execute the behave test
+test: ./test.sh
+tag: behave
+duration: 2h
+
+require:
+  - type: file
+    pattern: /behave

testing-farm/tests/behave/test.sh

@@ -0,0 +1,5 @@
+#!/bin/sh -eux
+
+cd ../../../behave
+install-mock-packages-built-by-packit mock-core-configs mock
+behave

testing-farm/tests/old-testsuite/main.fmf

@@ -0,0 +1,10 @@
+---
+summary: Execute the old `make check` test-suite
+test: ./test.sh
+tag: old_testsuite
+duration: 3h
+
+require:
+  - type: file
+    # copy-paste the whole directory
+    pattern: /