[skip-ci] Packit/TMT: Run gating tests

This commit will run Skopeo's gating tests using Packit and TMT.

Signed-off-by: Lokesh Mandvekar <[email protected]>

.fmf/version

@@ -0,0 +1 @@
+1

.packit.yaml

@@ -33,13 +33,13 @@ jobs:
         message: "Ephemeral COPR build failed. @containers/packit-build please check."
     targets:
       - fedora-development-x86_64
-      - fedora-development-aarch64
-      - fedora-latest-x86_64
-      - fedora-latest-aarch64
-      - fedora-latest-stable-x86_64
-      - fedora-latest-stable-aarch64
-      - fedora-40-x86_64
-      - fedora-40-aarch64
+      #- fedora-development-aarch64
+      #- fedora-latest-x86_64
+      #- fedora-latest-aarch64
+      #- fedora-latest-stable-x86_64
+      #- fedora-latest-stable-aarch64
+      #- fedora-40-x86_64
+      #- fedora-40-aarch64
     enable_net: true
 
   - job: copr_build
@@ -56,10 +56,10 @@ jobs:
     enable_net: true
 
   - job: copr_build
-    trigger: pull_request
+    trigger: ignore
     packages: [skopeo-centos]
     notifications: *copr_build_failure_notification
-    targets:
+    targets: &centos_targets
       - centos-stream-9-x86_64
       - centos-stream-9-aarch64
       - centos-stream-10-x86_64
@@ -88,6 +88,73 @@ jobs:
     project: podman-next
     enable_net: true
 
+  # ostree tests on Fedora
+  - job: tests
+    trigger: pull_request
+    identifier: ostree
+    packages: [skopeo-fedora]
+      #notifications:
+      #failure_comment:
+      #  message: "skopeo system tests failed on RHEL. @containers/packit-build please check."
+    targets:
+      - fedora-latest-stable
+    skip_build: true
+    tmt_plan: /plans/ostree
+    tf_extra_params:
+      environments:
+        - os: null
+
+  # Tests on Fedora
+  - job: tests
+    trigger: pull_request
+    packages: [skopeo-fedora]
+      #notifications:
+      #failure_comment:
+      #  message: "skopeo system tests failed on RHEL. @containers/packit-build please check."
+    tmt_plan: /plans/upstream
+    targets:
+      - fedora-rawhide
+    tf_extra_params:
+      environments:
+        - artifacts:
+          - type: repository-file
+            id: https://copr.fedorainfracloud.org/coprs/rhcontainerbot/podman-next/repo/fedora-$releasever/rhcontainerbot-podman-next-fedora-$releasever.repo
+
+  # Tests on CentOS Stream
+  - job: tests
+    trigger: ignore
+    packages: [skopeo-centos]
+      #notifications:
+      #failure_comment:
+      #  message: "skopeo system tests failed on RHEL. @containers/packit-build please check."
+    targets: *centos_targets
+    tf_extra_params:
+      environments:
+        - artifacts:
+          - type: repository-file
+            id: https://copr.fedorainfracloud.org/coprs/rhcontainerbot/podman-next/repo/centos-stream-$releasever/rhcontainerbot-podman-next-centos-stream-$releasever.repo
+
+  # System tests on RHEL
+  - job: tests
+    trigger: ignore
+    packages: [skopeo-rhel]
+    use_internal_tf: true
+      #notifications:
+      #failure_comment:
+      #  message: "skopeo system tests failed on RHEL. @containers/packit-build please check."
+    targets:
+      epel-9-aarch64:
+        distros: [RHEL-9-Nightly,RHEL-9.4.0-Nightly]
+      epel-9-x86_64:
+        distros: [RHEL-9-Nightly,RHEL-9.4.0-Nightly]
+    tf_extra_params:
+      environments:
+        - artifacts:
+          - type: repository-file
+            id: https://copr.fedorainfracloud.org/coprs/rhcontainerbot/podman-next/repo/epel-$releasever/rhcontainerbot-podman-next-epel-$releasever.repo
+          - type: repository-file
+            id: https://src.fedoraproject.org/rpms/epel-release/raw/epel9/f/epel.repo
+
   # Sync to Fedora
   - job: propose_downstream
     trigger: release

Makefile

@@ -202,7 +202,12 @@ test-integration:
 
 
 # Intended for CI, assumed to be running in quay.io/libpod/skopeo_cidev container.
+ifdef SKOPEO_BINARY
+$(info Skipping build as SKOPEO_BINARY is specified)
+test-integration-local:
+else
 test-integration-local: bin/skopeo
+endif
 	hack/warn-destructive-tests.sh
 	hack/test-integration.sh
 
@@ -218,8 +223,14 @@ test-system:
 	$(CONTAINER_RUNTIME) unshare rm -rf $$DTEMP; # This probably doesn't work with Docker, oh well, better than nothing... \
 	exit $$rc
 
+# The SKOPEO_BINARY envvar is used for running system tests with a prebuilt skopeo binary.
+ifdef SKOPEO_BINARY
+$(info Skipping build as SKOPEO_BINARY is specified)
+test-system-local:
+else
 # Intended for CI, assumed to already be running in quay.io/libpod/skopeo_cidev container.
 test-system-local: bin/skopeo
+endif
 	hack/warn-destructive-tests.sh
 	hack/test-system.sh
 
@@ -228,7 +239,7 @@ test-unit:
 	$(CONTAINER_RUN) $(MAKE) test-unit-local
 
 validate:
-	$(CONTAINER_RUN) $(MAKE) validate-local
+	$(CONTAINER_RUN) $(MAKE) tools validate-local
 
 # This target is only intended for development, e.g. executing it from an IDE. Use (make test) for CI or pre-release testing.
 test-all-local: validate-local validate-docs test-unit-local

hack/test-integration.sh

@@ -1,8 +1,10 @@
 #!/bin/bash
-set -e
+set -exo pipefail
 
-make PREFIX=/usr install
+if [[ ! -f /usr/bin/skopeo ]]; then
+    make PREFIX=/usr install
+fi
 
 echo "cd ./integration;" go test $TESTFLAGS ${BUILDTAGS:+-tags "$BUILDTAGS"}
 cd ./integration
-go test $TESTFLAGS ${BUILDTAGS:+-tags "$BUILDTAGS"}
+go test $TESTFLAGS ${BUILDTAGS:+-tags "$BUILDTAGS"}

hack/test-system.sh

@@ -38,7 +38,9 @@ EOF
 fi
 
 # Build skopeo, install into /usr/bin
-make PREFIX=/usr install
+if [[ ! -f /usr/bin/skopeo ]]; then
+    make PREFIX=/usr install
+fi
 
 # Run tests
 SKOPEO_BINARY=/usr/bin/skopeo bats --tap systemtest

hack/tmt/ostree/ostree-rs-ext.sh

@@ -0,0 +1,18 @@
+#!/usr/bin/env bash
+
+set -exo pipefail
+
+cat /etc/os-release
+
+cd ../../..
+dnf builddep -y rpm/skopeo.spec
+make
+make install
+
+EXT_REPO_NAME=ostree-rs-ext
+EXT_REPO_HOME=$(mktemp -d)/$EXT_REPO_NAME
+EXT_REPO=https://github.com/ostreedev/${EXT_REPO_NAME}.git
+git clone --depth 1 $EXT_REPO $EXT_REPO_HOME
+cd $EXT_REPO_HOME
+cargo test --no-run
+RUST_BACKTRACE=1 cargo test -- --nocapture --quiet

hack/tmt/ostree/tests.fmf

@@ -0,0 +1,9 @@
+require:
+    - cargo
+    - git-core
+    - make
+
+tag: ostree
+summary: Run ostree-rs-ext test
+test: bash ostree-rs-ext.sh
+duration: 30m

hack/tmt/tests.fmf

@@ -0,0 +1,13 @@
+require:
+    - make
+    - podman-docker
+
+/unit:
+    tag: upstream
+    summary: Run unit test
+    test: make -C ../.. test-unit
+
+/validate:
+    tag: upstream
+    summary: Run validate test
+    test: make -C ../.. tools validate

integration/tmt/tests.fmf

@@ -0,0 +1,12 @@
+require:
+    - bats
+    - golang
+    - make
+    - podman
+    - skopeo
+
+duration: 30m
+
+tag: [ upstream, downstream ]
+summary: Run integration tests
+test: make -C ../.. test-integration

plans/main.fmf

@@ -0,0 +1,31 @@
+discover:
+    how: fmf
+execute:
+    how: tmt
+
+/upstream:
+    summary: Run tests on upstream PRs
+    discover+:
+        filter: tag:upstream
+    adjust+:
+        enabled: false
+        when: initiator is not defined or initiator != packit
+
+/downstream:
+    summary: Run tests on bodhi / errata and dist-git PRs
+    discover+:
+        filter: tag:downstream
+    adjust+:
+        enabled: false
+        when: initiator == packit
+
+/ostree:
+    summary: Run ostree tests
+    discover+:
+        filter: tag:ostree
+    provision:
+        how: container
+        image: quay.io/coreos-assembler/fcos-buildroot:testing-devel
+    adjust+:
+        enabled: false
+        when: initiator is not defined or initiator != packit

rpm/skopeo.spec

@@ -71,7 +71,7 @@ BuildRequires: ostree-devel
 BuildRequires: glib2-devel
 BuildRequires: make
 BuildRequires: shadow-utils-subid-devel
-Requires: containers-common >= 4:1-21
+Requires: containers-common
 
 %description
 Command line utility to inspect images and repositories directly on Docker

systemtest/helpers.bash

@@ -301,11 +301,11 @@ start_registry() {
     # it's fixed. It's simply a workaround, forcing podman to use runc,
     # which might work great for skopeo CI but breaks Fedora gating tests.
     # Instead of always forcing runc, do so only when under cgroups v1:
-    local runtime=
-    cgroup_type=$(stat -f -c %T /sys/fs/cgroup)
-    if [[ $cgroup_type == "tmpfs" ]]; then
-        runtime="--runtime runc"
-    fi
+    #local runtime=
+    #cgroup_type=$(stat -f -c %T /sys/fs/cgroup)
+    #if [[ $cgroup_type == "tmpfs" ]]; then
+    #    runtime="--runtime runc"
+    #fi
 
     # cgroup option necessary under podman-in-podman (CI tests),
     # and doesn't seem to do any harm otherwise.

systemtest/tmt/systemtest.sh

@@ -0,0 +1,14 @@
+#!/usr/bin/env bash
+
+set -exo pipefail
+
+cat /etc/redhat-release
+
+# Remove testing-farm repos if they exist as these interfere with the packages
+# we want to install, especially when podman-next copr is involved
+rm -f /etc/yum.repos.d/tag-repository.repo
+
+# Print versions of distro and installed packages
+rpm -q bats container-selinux golang netavark podman selinux-policy skopeo
+
+make -C ../.. SKOPEO_BINARY=/usr/bin/skopeo test-system

systemtest/tmt/tests.fmf

@@ -0,0 +1,12 @@
+require:
+    - bats
+    - golang
+    - make
+    - podman
+    - skopeo
+
+duration: 30m
+
+tag: [ upstream, downstream ]
+summary: Run system tests
+test: bash systemtest.sh