[skip-ci] TMT: system tests

This commit introduces TMT test jobs triggered via packit to run system
tests on testing-farm infrastructure. Tests are run for all active
Fedora environments for aarch64 and x86_64. The same
test plan will be reused by Fedora for bodhi, zuul and fedora-ci gating
tests. Packit will handle syncing of test plan and sources from upstream
to downstream.

Jobs will be selectively triggered for `main` v/s release branches based
on the `release` PR label. The GHA for labeling PRs has also been
updated in this commit.

Test jobs on the main branch will include builds from the
`rhcontainerbot/podman-next` copr while tests on release branches will
only include official distro packages.

Signed-off-by: Lokesh Mandvekar <[email protected]>

.github/labeler.yml

@@ -6,3 +6,5 @@ kind/api-change:
 machine:
   - changed-files:
     - any-glob-to-any-file: pkg/machine/**
+release:
+  - base-branch: [^v?(0|[1-9]\d*).(0|[1-9]\d*)$]

.packit.yaml

@@ -5,6 +5,23 @@
 downstream_package_name: podman
 upstream_tag_template: v{version}
 
+# These files get synced from upstream to downstream (Fedora / CentOS Stream) on every
+# propose-downstream job. This is done so tests maintained upstream can be run
+# downstream in Zuul CI and Bodhi.
+# Ref: https://packit.dev/docs/configuration#files_to_sync
+files_to_sync:
+  - src: rpm/gating.yaml
+    dest: gating.yaml
+    delete: true
+  - src: plans/
+    dest: plans/
+    delete: true
+    mkpath: true
+  - src: .fmf/
+    dest: .fmf/
+    delete: true
+  - .packit.yaml
+
 packages:
   podman-fedora:
     pkg_tool: fedpkg
@@ -31,7 +48,7 @@ jobs:
       failure_comment:
         message: "Ephemeral COPR build failed. @containers/packit-build please check."
     enable_net: true
-    targets:
+    targets: &fedora_copr_targets
       - fedora-all-x86_64
       - fedora-all-aarch64
 
@@ -53,7 +70,7 @@ jobs:
     packages: [podman-centos]
     notifications: *packit_build_failure_notification
     enable_net: true
-    targets:
+    targets: &centos_copr_targets
       - centos-stream-9-x86_64
       - centos-stream-9-aarch64
       - centos-stream-10-x86_64
@@ -71,6 +88,51 @@ jobs:
     project: podman-next
     enable_net: true
 
+  - job: tests
+    trigger: pull_request
+    packages: [podman-fedora]
+    targets: *fedora_copr_targets
+    require: &dev_tests
+      label:
+        absent:
+          - release
+    tf_extra_params:
+      environments:
+        - artifacts:
+          - type: repository-file
+            id: https://copr.fedorainfracloud.org/coprs/rhcontainerbot/podman-next/repo/fedora-$releasever/rhcontainerbot-podman-next-fedora-$releasever.repo
+    identifier: "dev"
+
+  - job: tests
+    trigger: pull_request
+    packages: [podman-fedora]
+    targets: *fedora_copr_targets
+    require: &release_tests
+      label:
+        present:
+          - release
+    preserve_project: true
+    identifier: "release"
+
+  - job: tests
+    trigger: pull_request
+    packages: [podman-centos]
+    targets: *centos_copr_targets
+    require: *dev_tests
+    tf_extra_params:
+      environments:
+        - artifacts:
+          - type: repository-file
+            id: https://copr.fedorainfracloud.org/coprs/rhcontainerbot/podman-next/repo/centos-stream-$releasever/rhcontainerbot-podman-next-centos-stream-$releasever.repo
+    identifier: "dev"
+
+  - job: tests
+    trigger: pull_request
+    packages: [podman-centos]
+    targets: *centos_copr_targets
+    require: *release_tests
+    identifier: "release"
+
   - job: tests
     identifier: cockpit-revdeps
     trigger: pull_request

Makefile

@@ -78,6 +78,9 @@ SED=sed
 GREP=grep
 MAN_L=	man -l
 endif
+PODMAN ?= $(CURDIR)/bin/podman
+PODMAN_REMOTE ?= $(CURDIR)/bin/podman-remote
+QUADLET ?= $(CURDIR)/bin/quadlet
 
 # This isn't what we actually build; it's a superset, used for target
 # dependencies. Basically: all *.go and *.c files, except *_test.go,
@@ -706,8 +709,8 @@ localmachine:
 localsystem:
 	# Wipe existing config, database, and cache: start with clean slate.
 	$(RM) -rf ${HOME}/.local/share/containers ${HOME}/.config/containers
-	PODMAN=$(CURDIR)/bin/podman QUADLET=$(CURDIR)/bin/quadlet bats -T --filter-tags '!ci:parallel' test/system/
-	PODMAN=$(CURDIR)/bin/podman QUADLET=$(CURDIR)/bin/quadlet bats -T --filter-tags ci:parallel -j $$(nproc) test/system/
+	PODMAN=$(PODMAN) QUADLET=$(QUADLET) bats -T --filter-tags '!ci:parallel' test/system/
+	PODMAN=$(PODMAN) QUADLET=$(QUADLET) bats -T --filter-tags ci:parallel -j $$(nproc) test/system/
 
 .PHONY: remotesystem
 remotesystem:
@@ -720,26 +723,26 @@ remotesystem:
 	# . Stop server.
 	rc=0;\
 	if timeout -v 1 true; then \
-		if ./bin/podman-remote info; then \
+		if $(PODMAN_REMOTE) info; then \
 			echo "Error: podman system service (not ours) is already running" >&2;\
 			exit 1;\
 		fi;\
-		./bin/podman system service --timeout=0 > $(if $(PODMAN_SERVER_LOG),$(PODMAN_SERVER_LOG),/dev/null) 2>&1 & \
+		$(PODMAN) system service --timeout=0 > $(if $(PODMAN_SERVER_LOG),$(PODMAN_SERVER_LOG),/dev/null) 2>&1 & \
 		retry=5;\
 		while [ $$retry -ge 0 ]; do\
 			echo Waiting for server...;\
 			sleep 1;\
-			./bin/podman-remote info >/dev/null 2>&1 && break;\
+			$(PODMAN_REMOTE) info >/dev/null 2>&1 && break;\
 			retry=$$(expr $$retry - 1);\
 		done;\
 		if [ $$retry -lt 0 ]; then\
-			echo "Error: ./bin/podman system service did not come up" >&2;\
+			echo "Error: $(PODMAN) system service did not come up" >&2;\
 			exit 1;\
 		fi;\
-		env PODMAN="$(CURDIR)/bin/podman-remote" bats -T --filter-tags '!ci:parallel' test/system/ ;\
+		env PODMAN="$(PODMAN_REMOTE)" bats -T --filter-tags '!ci:parallel' test/system/ ;\
 		rc=$$?; \
 		if [ $$rc -eq 0 ]; then \
-		   env PODMAN="$(CURDIR)/bin/podman-remote" bats -T --filter-tags ci:parallel -j $$(nproc) test/system/ ;\
+		   env PODMAN="$(PODMAN_REMOTE)" bats -T --filter-tags ci:parallel -j $$(nproc) test/system/ ;\
 		   rc=$$?;\
 		fi; \
 		kill %1;\

plans/podman.fmf

@@ -0,0 +1,45 @@
+discover:
+    how: fmf
+execute:
+    how: tmt
+environment:
+    PODMAN: /usr/bin/podman
+    PODMAN_REMOTE: /usr/bin/podman-remote
+    PODMAN_TESTING: /usr/bin/podman-testing
+    QUADLET: /usr/libexec/podman/quadlet
+    ROOTLESS_USER: fedora
+adjust+:
+    environment+:
+        ROOTLESS_USER: ec2-user
+    when: distro == centos-stream
+
+prepare:
+    how: shell
+    script: |
+        rm -f /etc/yum.repos.d/tag-repository.repo
+        loginctl enable-linger fedora
+
+/upstream:
+    summary: Run tests on upstream PRs
+    discover+:
+        filter: tag:upstream
+    adjust+:
+        enabled: false
+        when: revdeps == yes and initiator is not defined or initiator != packit
+    prepare+:
+        how: shell
+        script: export PODMAN_SOURCE_DIR=$(eval $TMT_TREE)
+
+/downstream:
+    summary: Run tests on bodhi / errata and dist-git PRs
+    discover+:
+        filter: tag:downstream
+        dist-git-install-builddeps: true
+        dist-git-source: true
+        dist-git-remove-fmf-root: true
+    adjust+:
+        enabled: false
+        when: initiator == packit
+    prepare+:
+        how: shell
+        script: export PODMAN_SOURCE_DIR=$(eval $TMT_SOURCE_DIR)

plans/tmt-envvars

@@ -0,0 +1,22 @@
+PODMAN
+PODMAN_REMOTE
+PODMAN_SOURCE_DIR
+PODMAN_TESTING
+QUADLET
+ROOTLESS_USER
+TMT_TREE
+TMT_PLAN_DATA
+TMT_VERSION
+TMT_TEST_NAME
+TMT_TEST_DATA
+TMT_TEST_SERIAL_NUMBER
+TMT_TEST_ITERATION_ID
+TMT_TEST_METADATA
+TMT_SOURCE_DIR
+TMT_REBOOT_COUNT
+TMT_TEST_RESTART_COUNT
+TMT_TOPOLOGY_BASH
+TMT_TOPOLOGY_YAML
+TMT_TEST_PIDFILE
+TMT_TEST_PIDFILE_LOCK
+TMT_TEST_PIDFILE_ROOT

rpm/gating.yaml

@@ -0,0 +1,17 @@
+--- !Policy
+product_versions:
+  - fedora-*
+decision_contexts:
+  - bodhi_update_push_stable
+  - bodhi_update_push_testing
+subject_type: koji_build
+rules:
+  - !PassingTestCaseRule {test_case_name: fedora-ci.koji-build.tier0.functional}
+
+# recipients: jnovy, lsm5, santiago
+--- !Policy
+product_versions:
+  - rhel-*
+decision_context: osci_compose_gate
+rules:
+  - !PassingTestCaseRule {test_case_name: osci.brew-build.tier0.functional}

rpm/podman.spec

@@ -24,6 +24,8 @@
 
 %if %{defined fedora}
 %define build_with_btrfs 1
+# No machine stuff for rhel
+%define machine 1
 %endif
 
 %if %{defined copr_username}

test/tmt/system-test.fmf

@@ -0,0 +1,32 @@
+require:
+    - bats
+    - golang
+    - make
+    - podman
+    - podman-remote
+    - podman-tests
+    - slirp4netns
+
+/local-root:
+    tag: [ downstream, upstream ]
+    summary: local rootful test
+    test: bash $PODMAN_SOURCE_DIR/test/tmt/test.sh localsystem
+    duration: 60m
+
+/local-rootless:
+    tag: [ downstream, upstream ]
+    summary: rootless test
+    test: rm -rf /home/$ROOTLESS_USER/.local/share/containers && su --whitelist-environment=$(cat $PODMAN_SOURCE_DIR/plans/tmt-envvars | tr '\n' ',') - $ROOTLESS_USER -c 'bash $PODMAN_SOURCE_DIR/test/tmt/test.sh localsystem'
+    duration: 60m
+
+/remote-root:
+    tag: [ downstream, upstream ]
+    summary: remote rootful test
+    test: bash $PODMAN_SOURCE_DIR/test/tmt/test.sh remotesystem
+    duration: 60m
+
+/remote-rootless:
+    tag: [ downstream, upstream ]
+    summary: remote rootless test
+    test: rm -rf /home/$ROOTLESS_USER/.local/share/containers && su --whitelist-environment=$(cat $PODMAN_SOURCE_DIR/plans/tmt-envvars | tr '\n' ',') - $ROOTLESS_USER -c 'bash $PODMAN_SOURCE_DIR/test/tmt/test.sh remotesystem'
+    duration: 60m

test/tmt/test.sh

@@ -0,0 +1,11 @@
+#!/usr/bin/env bash
+
+set -exo pipefail
+
+whoami
+
+uname -r
+
+rpm -q aardvark-dns buildah conmon container-selinux containers-common crun netavark passt podman skopeo slirp4netns systemd
+
+make -C $PODMAN_SOURCE_DIR $1

test/tmt/toolbox-test.sh

@@ -0,0 +1,13 @@
+#!/usr/bin/env bash
+
+set -exo pipefail
+
+whoami
+
+uname -r
+
+rpm -q toolbox
+
+cd /usr/share/toolbox/test/system
+
+bats .

test/tmt/toolbox.fmf

@@ -0,0 +1,8 @@
+require:
+    - toolbox-tests
+
+/rootless:
+    tag: [ downstream ]
+    summary: rootless test
+    test: rm -rf /home/$ROOTLESS_USER/.local/share/containers && su --whitelist-environment=$(cat $PODMAN_SOURCE_DIR/plans/tmt-envvars | tr '\n' ',') - $ROOTLESS_USER -c 'bash $PODMAN_SOURCE_DIR/test/tmt/toolbox-test.sh'
+    duration: 60m