From 2e8c07423470e08dfc7d5c73096665781d962f51 Mon Sep 17 00:00:00 2001
From: Ed Santiago <santiago@redhat.com>
Date: Thu, 6 Jun 2024 12:48:59 -0600
Subject: [PATCH 1/3] CI: use local registry, part 1 of 3: setup

As of https://github.com/containers/automation_images/pull/357
our CI VMs include a local registry preloaded with all(*)
images used in tests.

 * where "all" means "most".

This commit installs a new registries.conf that redirects docker
and quay to the new local registry. The hope is that this will
reduce CI flakes.

Since tests change over time, and new tests may require new
images, this commit also adds a mechanism for pulling in
remote images at test run time. Obviously this negates
the purpose of the cache, since it introduces a flake
pain point. The idea is: DO NOT DO THIS UNLESS ABSOLUTELY
NECESSARY, and then, if we have to do this, hurry up and
spin new CI VMs that include the new image(s).

Signed-off-by: Ed Santiago <santiago@redhat.com>
---
 .cirrus.yml                         |  2 +-
 contrib/cirrus/lib.sh               |  6 ++++-
 contrib/cirrus/setup_environment.sh | 16 +++++++++++
 test/NEW-IMAGES                     | 14 ++++++++++
 test/registries-cached.conf         | 41 +++++++++++++++++++++++++++++
 test/registries.conf                |  7 +++++
 6 files changed, 84 insertions(+), 2 deletions(-)
 create mode 100644 test/NEW-IMAGES
 create mode 100644 test/registries-cached.conf

diff --git a/.cirrus.yml b/.cirrus.yml
index ec8230a919..2138bcb57f 100644
--- a/.cirrus.yml
+++ b/.cirrus.yml
@@ -33,7 +33,7 @@ env:
     DEBIAN_NAME: "debian-13"
 
     # Image identifiers
-    IMAGE_SUFFIX: "c20240702t180702z-f40f39d13"
+    IMAGE_SUFFIX: "c20240708t152000z-f40f39d13"
 
     # EC2 images
     FEDORA_AMI: "fedora-aws-${IMAGE_SUFFIX}"
diff --git a/contrib/cirrus/lib.sh b/contrib/cirrus/lib.sh
index 3c93306cec..6a0d893ae4 100644
--- a/contrib/cirrus/lib.sh
+++ b/contrib/cirrus/lib.sh
@@ -72,6 +72,9 @@ CIRRUS_CI="${CIRRUS_CI:-false}"
 CONTINUOUS_INTEGRATION="${CONTINUOUS_INTEGRATION:-false}"
 CIRRUS_REPO_NAME=${CIRRUS_REPO_NAME:-podman}
 
+# All CI jobs use a local registry
+export CI_USE_REGISTRY_CACHE=true
+
 # shellcheck disable=SC2154
 if [[ -n "$CIRRUS_PR" ]] && [[ -z "$PR_BASE_SHA" ]]; then
     # shellcheck disable=SC2154
@@ -193,7 +196,8 @@ setup_rootless() {
 
 install_test_configs() {
     msg "Installing ./test/registries.conf system-wide."
-    install -v -D -m 644 ./test/registries.conf /etc/containers/
+    # All CI VMs run with a local registry
+    install -v -D -m 644 ./test/registries-cached.conf /etc/containers/registries.conf
 }
 
 # Remove all files provided by the distro version of podman.
diff --git a/contrib/cirrus/setup_environment.sh b/contrib/cirrus/setup_environment.sh
index eaaf22a3d7..f45e5b2f82 100755
--- a/contrib/cirrus/setup_environment.sh
+++ b/contrib/cirrus/setup_environment.sh
@@ -335,6 +335,18 @@ case "$PODBIN_NAME" in
     *) die_unknown PODBIN_NAME
 esac
 
+# As of July 2024, CI VMs come built-in with a registry.
+LCR=/var/cache/local-registry/local-cache-registry
+if [[ -x $LCR ]]; then
+    # Images in cache registry are prepopulated at the time
+    # VMs are built. If any PR adds a dependency on new images,
+    # those must be fetched now, at VM start time. This should
+    # be rare, and must be fixed in next automation_images build.
+    while read new_image; do
+        $LCR cache $new_image
+    done < <(grep '^[^#]' test/NEW-IMAGES || true)
+fi
+
 # Required to be defined by caller: The primary type of testing that will be performed
 # shellcheck disable=SC2154
 showrun echo "about to set up for TEST_FLAVOR [=$TEST_FLAVOR]"
@@ -426,6 +438,10 @@ case "$TEST_FLAVOR" in
         fi
         remove_packaged_podman_files
         showrun make install PREFIX=/usr ETCDIR=/etc
+        # machine-os image changes too frequently, can't be precached
+        # FIXME: I don't think we can use version.go, because of chicken-egg
+        # problem when that gets bumped. Ideas welcome.
+        $LCR cache podman/machine-os:5.2
         install_test_configs
         ;;
     swagger)
diff --git a/test/NEW-IMAGES b/test/NEW-IMAGES
new file mode 100644
index 0000000000..2a69de8436
--- /dev/null
+++ b/test/NEW-IMAGES
@@ -0,0 +1,14 @@
+#
+# As of July 2024, all CI VMs include a local registry prepopulated
+# with all container images used in tests:
+#
+#     https://github.com/containers/automation_images/pull/357
+#     https://github.com/containers/podman/pull/22726
+#
+# From time to time -- infrequently, we hope! -- existing images are
+# updated, or tests are added that require new images. Those must be
+# prefetched on each CI job, at least until new VMs are built. This
+# file contains those images.
+#
+# Format is one FQIN per line. Enumerate them below:
+#
diff --git a/test/registries-cached.conf b/test/registries-cached.conf
new file mode 100644
index 0000000000..552f13ebea
--- /dev/null
+++ b/test/registries-cached.conf
@@ -0,0 +1,41 @@
+#####
+##### CAUTION! All changes to this file must also be
+##### CAUTION!   mirrored to registries.conf !
+#####
+##### This is the registries.conf file used IN DEVELOPER ENVIRONMENT.
+##### For the one used in CI, see registries-cached.conf
+#####
+# Note that changing the order here may break tests.
+unqualified-search-registries = ['docker.io', 'quay.io', 'registry.fedoraproject.org']
+
+[[registry]]
+# In Nov. 2020, Docker rate-limits image pulling.  To avoid hitting these
+# limits while testing, always use the google mirror for qualified and
+# unqualified `docker.io` images.
+# Ref: https://cloud.google.com/container-registry/docs/pulling-cached-images
+prefix="docker.io"
+location="127.0.0.1:60333"
+insecure=true
+
+[[registry]]
+prefix="quay.io"
+location="127.0.0.1:60333"
+insecure=true
+
+# 2020-10-27 a number of images are not present in gcr.io, and podman
+# barfs spectacularly when trying to fetch them. We've hand-copied
+# those to quay, using skopeo copy --all ...
+[[registry]]
+prefix="docker.io/library"
+location="127.0.0.1:60333/libpod"
+insecure=true
+
+# For testing #11933 to make sure that registries.conf is consulted unless
+# --tls-verify is used during container creation.
+[[registry]]
+location="localhost:5000"
+insecure=true
+
+# Alias used in tests.
+[aliases]
+  "podman-desktop-test123"="florent.fr/will/like"
diff --git a/test/registries.conf b/test/registries.conf
index 6569be833f..8b83ed1e25 100644
--- a/test/registries.conf
+++ b/test/registries.conf
@@ -1,3 +1,10 @@
+#####
+##### CAUTION! All changes to this file must also be
+##### CAUTION!   mirrored to registries-cached.conf !
+#####
+##### This is the registries.conf file used IN DEVELOPER ENVIRONMENT.
+##### For the one used in CI, see registries-cached.conf
+#####
 # Note that changing the order here may break tests.
 unqualified-search-registries = ['docker.io', 'quay.io', 'registry.fedoraproject.org']
 

From dd1bcabae90f43dc958561322d995519f1dd2548 Mon Sep 17 00:00:00 2001
From: Ed Santiago <santiago@redhat.com>
Date: Thu, 6 Jun 2024 13:15:52 -0600
Subject: [PATCH 2/3] CI: use local registry, part 2 of 3: fix tests

This commit gets tests working under the new local-registry system:

  * amend a few image names, mostly just sticking to a consistent
    list of those images in our registry cache. Mostly minor
    tag updates.

  * trickier: pull_test: change some error messages, and remove
    a test that's now a NOP. Basically, with a local (unprotected)
    registry we always get "404 manifest unknown"; with a real
    registry we'll get "403 I can't tell you".

  * trickiest: seccomp_test: build our own images at run time,
    with our desired labels. Until now we've been pulling
    prebuilt images, but those will not copy to the local
    cache registry. Something about v1? Anyhow, I gave up
    trying to cache them, and the workaround is straightforward.

Also took the liberty of strengthening a few error-message checks

Signed-off-by: Ed Santiago <santiago@redhat.com>
---
 contrib/cirrus/runner.sh             |  1 +
 hack/podman-registry                 |  2 +-
 pkg/bindings/test/images_test.go     |  2 +-
 test/apiv2/10-images.at              | 46 +++++++++++++++--------
 test/apiv2/20-containers.at          | 12 +++---
 test/apiv2/test-apiv2                | 20 +++++++---
 test/e2e/common_test.go              |  4 ++
 test/e2e/config.go                   |  9 -----
 test/e2e/info_test.go                | 10 ++---
 test/e2e/libpod_suite_remote_test.go |  8 +++-
 test/e2e/libpod_suite_test.go        |  8 +++-
 test/e2e/load_test.go                |  5 ++-
 test/e2e/pull_test.go                | 23 +++++-------
 test/e2e/run_seccomp_test.go         | 56 ++++++++++++++++------------
 test/system/helpers.registry.bash    |  2 +-
 15 files changed, 119 insertions(+), 89 deletions(-)

diff --git a/contrib/cirrus/runner.sh b/contrib/cirrus/runner.sh
index cf37a6daed..88f9abfcec 100755
--- a/contrib/cirrus/runner.sh
+++ b/contrib/cirrus/runner.sh
@@ -124,6 +124,7 @@ exec_container() {
 
     # VM Images and Container images are built using (nearly) identical operations.
     set -x
+    env CONTAINERS_REGISTRIES_CONF=/dev/null bin/podman pull -q $CTR_FQIN
     # shellcheck disable=SC2154
     exec bin/podman run --rm --privileged --net=host --cgroupns=host \
         -v `mktemp -d -p /var/tmp`:/var/tmp:Z \
diff --git a/hack/podman-registry b/hack/podman-registry
index c539a5b6e9..8316c2e5c8 100755
--- a/hack/podman-registry
+++ b/hack/podman-registry
@@ -7,7 +7,7 @@ ME=$(basename $0)
 ###############################################################################
 # BEGIN defaults
 
-PODMAN_REGISTRY_IMAGE=quay.io/libpod/registry:2.8
+PODMAN_REGISTRY_IMAGE=quay.io/libpod/registry:2.8.2
 
 PODMAN_REGISTRY_USER=
 PODMAN_REGISTRY_PASS=
diff --git a/pkg/bindings/test/images_test.go b/pkg/bindings/test/images_test.go
index 1b0a23c63f..42a1dc2db6 100644
--- a/pkg/bindings/test/images_test.go
+++ b/pkg/bindings/test/images_test.go
@@ -177,7 +177,7 @@ var _ = Describe("Podman images", func() {
 
 		// Adding one more image. There Should be no errors in the response.
 		// And the count should be three now.
-		bt.Pull("testimage:20200929")
+		bt.Pull("testimage:20221018")
 		imageSummary, err = images.List(bt.conn, nil)
 		Expect(err).ToNot(HaveOccurred())
 		Expect(len(imageSummary)).To(BeNumerically(">=", 2))
diff --git a/test/apiv2/10-images.at b/test/apiv2/10-images.at
index 8cd7ed2234..375a9f984e 100644
--- a/test/apiv2/10-images.at
+++ b/test/apiv2/10-images.at
@@ -7,7 +7,9 @@
 podman pull -q $IMAGE
 
 t GET libpod/images/json 200 \
-  .[0].Id~[0-9a-f]\\{64\\}
+  length=1 \
+  .[0].Id~[0-9a-f]\\{64\\} \
+  .[0].Names[0]="$IMAGE"
 iid=$(jq -r '.[0].Id' <<<"$output")
 
 # Create an empty manifest and make sure it is not listed
@@ -18,7 +20,7 @@ t GET images/json 200 length=1
 t GET libpod/images/json 200 length=2
 
 t GET libpod/images/$iid/exists                     204
-t GET libpod/images/$PODMAN_TEST_IMAGE_NAME/exists  204
+t GET libpod/images/$PODMAN_TEST_IMAGE_NAME:$PODMAN_TEST_IMAGE_TAG/exists  204
 t GET libpod/images/${iid}abcdef/exists  404 \
   .cause="failed to find image ${iid}abcdef"
 
@@ -51,11 +53,13 @@ t GET images/$iid/json 200 \
 t POST "images/create?fromImage=alpine" 200 .error~null .status~".*Download complete.*"
 t POST "libpod/images/pull?reference=alpine&compatMode=true" 200 .error~null .status~".*Download complete.*"
 
-t POST "images/create?fromImage=alpine&tag=latest" 200
+t POST "images/create?fromImage=alpine&tag=latest" 200 \
+  .status~"Already exists"
 
 # 10977 - handle platform parameter correctly
-t POST "images/create?fromImage=quay.io/libpod/testimage:20240123&platform=linux/arm64" 200
-t GET  "images/testimage:20240123/json" 200 \
+# THIS IMAGE MUST NOT BE THE SAME AS $IMAGE
+t POST "images/create?fromImage=quay.io/libpod/testimage:20221018&platform=linux/arm64" 200
+t GET  "images/testimage:20221018/json" 200 \
   .Architecture=arm64
 
 # Make sure that new images are pulled
@@ -78,34 +82,42 @@ t POST /images/create?fromImage=busybox:invalidtag123 404
 # Display the image history
 t GET libpod/images/nonesuch/history 404
 
-for i in $iid ${iid:0:12} $PODMAN_TEST_IMAGE_NAME; do
+for i in $iid ${iid:0:12} $PODMAN_TEST_IMAGE_NAME:$PODMAN_TEST_IMAGE_TAG; do
   t GET libpod/images/$i/history 200 \
     .[0].Id=$iid \
     .[0].Created~[0-9]\\{10\\} \
     .[0].Tags[0]="$IMAGE" \
-    .[0].Size=0 \
-    .[0].Comment=
+    .[0].Size=1024 \
+    .[0].Comment="FROM localhost/interim-image:latest"
 done
 
-for i in $iid ${iid:0:12} $PODMAN_TEST_IMAGE_NAME; do
+for i in $iid ${iid:0:12} $PODMAN_TEST_IMAGE_NAME:$PODMAN_TEST_IMAGE_TAG; do
   t GET images/$i/history 200 \
     .[0].Id="sha256:"$iid \
     .[0].Created~[0-9]\\{10\\} \
     .[0].Tags[0]="$IMAGE" \
-    .[0].Size=0 \
-    .[0].Comment=
+    .[0].Size=1024 \
+    .[0].Comment="FROM localhost/interim-image:latest"
 done
 
 # compat api pull image unauthorized message error
-t POST "/images/create?fromImage=quay.io/idonotexist/idonotexist:dummy" 401 \
-  .message="unauthorized: access to the requested resource is not authorized"
+# This depends on whether we're using local cache registry or real quay
+expect_code=401
+expect_msg="unauthorized: access to the requested resource is not authorized"
+if [[ -n "$CI_USE_REGISTRY_CACHE" ]]; then
+    # local registry has no auth, so it can return 404
+    expect_code=404
+    expect_msg="manifest unknown: manifest unknown"
+fi
+t POST "/images/create?fromImage=quay.io/idonotexist/idonotexist:dummy" $expect_code \
+  .message="$expect_msg"
 
 # Export an image on the local
 t GET libpod/images/nonesuch/get 404
 t GET libpod/images/$iid/get?format=foo 500
 t GET libpod/images/$PODMAN_TEST_IMAGE_NAME/get?compress=bar 400
 
-for i in $iid ${iid:0:12} $PODMAN_TEST_IMAGE_NAME; do
+for i in $iid ${iid:0:12} $PODMAN_TEST_IMAGE_NAME:$PODMAN_TEST_IMAGE_TAG; do
   t GET "libpod/images/$i/get"                200 '[POSIX tar archive]'
   t GET "libpod/images/$i/get?compress=true"  200 '[POSIX tar archive]'
   t GET "libpod/images/$i/get?compress=false" 200 '[POSIX tar archive]'
@@ -296,10 +308,12 @@ t DELETE libpod/images/test:test 200
 # This sleep seems to avoid the race.
 # If it fails and begins to flake, investigate a retry loop.
 sleep 1
+# FIXME 2024-05-30 #22726: when running with a local cache registry, DELETE
+# sometimes produces 5-6 events instead of the desired only-one.
 t GET "libpod/events?stream=false&since=$START"  200  \
-  'select(.status | contains("remove")).Action=remove'
+  'select(.status | contains("remove")).Actor.Attributes.name~.*localhost/test:test'
 t GET "events?stream=false&since=$START"  200  \
-  'select(.status | contains("delete")).Action=delete'
+  'select(.status | contains("delete")).Actor.Attributes.name~.*localhost/test:test'
 
 # Test image removal with `noprune={true,false}`
 podman create --name c_test1 $IMAGE true
diff --git a/test/apiv2/20-containers.at b/test/apiv2/20-containers.at
index d73187650b..7aca66797f 100644
--- a/test/apiv2/20-containers.at
+++ b/test/apiv2/20-containers.at
@@ -4,12 +4,10 @@
 #
 
 # WORKDIR=/data
-ENV_WORKDIR_IMG=quay.io/libpod/testimage:20240123
 MultiTagName=localhost/test/testformultitag:tag
 
 podman pull $IMAGE &>/dev/null
 podman tag $IMAGE $MultiTagName
-podman pull $ENV_WORKDIR_IMG &>/dev/null
 # Unimplemented
 #t POST libpod/containers/create '' 201 'sdf'
 
@@ -281,7 +279,7 @@ t DELETE libpod/containers/bogus 404
 
 
 # test apiv2 create container with correct entrypoint and cmd
-# --data '{"Image":"quay.io/libpod/alpine_labels:latest","Entrypoint":["echo"],"Cmd":["param1","param2"]}'
+# --data '{"Image":"quay.io/libpod/some:thing","Entrypoint":["echo"],"Cmd":["param1","param2"]}'
 t POST containers/create \
   Image=$IMAGE \
   Entrypoint='["echo"]' \
@@ -352,7 +350,7 @@ t DELETE containers/$cid_top 204
 
 # test the WORKDIR and StopSignal
 t POST containers/create \
-  Image=$ENV_WORKDIR_IMG \
+  Image=$IMAGE \
   WorkingDir=/dataDir \
   StopSignal=\"9\" \
   201 \
@@ -406,7 +404,7 @@ t GET containers/json 200 \
 podman stop bar
 
 #compat api list containers sanity checks
-podman run -d --rm --name labelcontainer_with $ENV_WORKDIR_IMG top
+podman run -d --rm --name labelcontainer_with --label slartibart=fast $IMAGE top
 podman run -d --rm --name labelcontainer_without $IMAGE top
 
 t GET containers/json?filters='garb1age}' 500 \
@@ -436,10 +434,10 @@ t POST containers/prune?filters='{"label":["tes' 500 \
 t POST libpod/containers/prune?filters='{"label":["tes' 500 \
     .cause="unexpected end of JSON input"
 
-t GET libpod/containers/json?filters='{"label":["created_by"]}' 200 \
+t GET libpod/containers/json?filters='{"label":["slartibart"]}' 200 \
   length=1 \
   .[0].Names[0]="labelcontainer_with"
-t GET libpod/containers/json?filters='{"label!":["created_by"]}' 200 \
+t GET libpod/containers/json?filters='{"label!":["slartibart"]}' 200 \
   length=1 \
   .[0].Names[0]="labelcontainer_without"
 t GET libpod/containers/json?filters='{"label!":["testlabel"]}' 200 length=2
diff --git a/test/apiv2/test-apiv2 b/test/apiv2/test-apiv2
index 14227ea8fe..b92ada2bd4 100755
--- a/test/apiv2/test-apiv2
+++ b/test/apiv2/test-apiv2
@@ -11,13 +11,13 @@ ME=$(basename $0)
 
 PODMAN_TEST_IMAGE_REGISTRY=${PODMAN_TEST_IMAGE_REGISTRY:-"quay.io"}
 PODMAN_TEST_IMAGE_USER=${PODMAN_TEST_IMAGE_USER:-"libpod"}
-PODMAN_TEST_IMAGE_NAME=${PODMAN_TEST_IMAGE_NAME:-"alpine_labels"}
-PODMAN_TEST_IMAGE_TAG=${PODMAN_TEST_IMAGE_TAG:-"latest"}
+PODMAN_TEST_IMAGE_NAME=${PODMAN_TEST_IMAGE_NAME:-"testimage"}
+PODMAN_TEST_IMAGE_TAG=${PODMAN_TEST_IMAGE_TAG:-"20240123"}
 PODMAN_TEST_IMAGE_FQN="$PODMAN_TEST_IMAGE_REGISTRY/$PODMAN_TEST_IMAGE_USER/$PODMAN_TEST_IMAGE_NAME:$PODMAN_TEST_IMAGE_TAG"
 
 IMAGE=$PODMAN_TEST_IMAGE_FQN
 
-REGISTRY_IMAGE="${PODMAN_TEST_IMAGE_REGISTRY}/${PODMAN_TEST_IMAGE_USER}/registry:2.7"
+REGISTRY_IMAGE="${PODMAN_TEST_IMAGE_REGISTRY}/${PODMAN_TEST_IMAGE_USER}/registry:2.8.2"
 
 # END   stuff you can but probably shouldn't customize
 ###############################################################################
@@ -54,6 +54,14 @@ fi
 # Path to podman binary
 PODMAN_BIN=${PODMAN:-${CONTAINERS_HELPER_BINARY_DIR}/podman}
 
+# Path to registries.conf file. CI uses a locally-running cache registry,
+# but we can't expect developers to have that in their local environment.
+registries_conf_file=registries.conf
+if [[ -n "$CI_USE_REGISTRY_CACHE" ]]; then
+    registries_conf_file=registries-cached.conf
+fi
+registries_conf_path=$TESTS_DIR/../$registries_conf_file
+
 # Cleanup handlers
 clean_up_server() {
     if [ -n "$service_pid" ]; then
@@ -474,7 +482,7 @@ function start_service() {
         $PODMAN_BIN unshare true
     fi
 
-    CONTAINERS_REGISTRIES_CONF=$TESTS_DIR/../registries.conf \
+    CONTAINERS_REGISTRIES_CONF=$registries_conf_path \
         $PODMAN_BIN \
         --root $WORKDIR/server_root --syslog=true \
         system service \
@@ -526,7 +534,7 @@ function start_registry() {
     mkdir -p $AUTHDIR
 
     mkdir -p ${REGDIR}/{root,runroot}
-    local PODMAN_REGISTRY_ARGS="--root ${REGDIR}/root --runroot ${REGDIR}/runroot"
+    local PODMAN_REGISTRY_ARGS="--root ${REGDIR}/root --runroot ${REGDIR}/runroot --tmpdir ${REGDIR}/tmp"
 
     # Give it three tries, to compensate for network flakes
     podman ${PODMAN_REGISTRY_ARGS} pull $REGISTRY_IMAGE ||
@@ -644,7 +652,7 @@ function wait_for_port() {
 ############
 function podman() {
     echo "\$ $PODMAN_BIN $*"                           >>$WORKDIR/output.log
-#    env CONTAINERS_REGISTRIES_CONF=$TESTS_DIR/../registries.conf \
+    env CONTAINERS_REGISTRIES_CONF=$registries_conf_path \
         $PODMAN_BIN --root $WORKDIR/server_root "$@"   >>$WORKDIR/output.log 2>&1
 }
 
diff --git a/test/e2e/common_test.go b/test/e2e/common_test.go
index b7b7e57113..a3c9c72df4 100644
--- a/test/e2e/common_test.go
+++ b/test/e2e/common_test.go
@@ -1493,3 +1493,7 @@ func CopySymLink(source, dest string) error {
 	}
 	return os.Symlink(link, dest)
 }
+
+func UsingCacheRegistry() bool {
+	return os.Getenv("CI_USE_REGISTRY_CACHE") != ""
+}
diff --git a/test/e2e/config.go b/test/e2e/config.go
index 4867734247..6cadae2a14 100644
--- a/test/e2e/config.go
+++ b/test/e2e/config.go
@@ -15,15 +15,6 @@ var (
 	HEALTHCHECK_IMAGE = "quay.io/libpod/alpine_healthcheck:latest" //nolint:revive,stylecheck
 	volumeTest        = "quay.io/libpod/volume-plugin-test-img:20220623"
 
-	// This image has seccomp profiles that blocks all syscalls.
-	// The intention behind blocking all syscalls is to prevent
-	// regressions in the future.  The required syscalls can vary
-	// depending on which runtime we're using.
-	alpineSeccomp = "quay.io/libpod/alpine-with-seccomp:label"
-	// This image has a bogus/invalid seccomp profile which should
-	// yield a json error when being read.
-	alpineBogusSeccomp = "quay.io/libpod/alpine-with-bogus-seccomp:label"
-
 	// ImageCacheDir is initialized at runtime.
 	// e.g., filepath.Join(os.TempDir(), "imagecachedir")
 	// This directory should be used by per-user.
diff --git a/test/e2e/info_test.go b/test/e2e/info_test.go
index 25b9ca5a30..6e5ae8722e 100644
--- a/test/e2e/info_test.go
+++ b/test/e2e/info_test.go
@@ -117,13 +117,11 @@ var _ = Describe("Podman Info", func() {
 			Expect(session.OutputToString()).To(Equal("false"))
 		}
 
-		session = podmanTest.Podman([]string{"info", "--format", "{{.Host.RemoteSocket.Exists}}"})
-		session.WaitWithDefaultTimeout()
-		Expect(session).Should(ExitCleanly())
 		if IsRemote() {
-			Expect(session.OutputToString()).To(ContainSubstring("true"))
-		} else {
-			Expect(session.OutputToString()).To(ContainSubstring("false"))
+			session = podmanTest.Podman([]string{"info", "--format", "{{.Host.RemoteSocket.Exists}}"})
+			session.WaitWithDefaultTimeout()
+			Expect(session).Should(ExitCleanly())
+			Expect(session.OutputToString()).To(Equal("true"))
 		}
 
 	})
diff --git a/test/e2e/libpod_suite_remote_test.go b/test/e2e/libpod_suite_remote_test.go
index 6d5cdc08ff..4c7a7c96f1 100644
--- a/test/e2e/libpod_suite_remote_test.go
+++ b/test/e2e/libpod_suite_remote_test.go
@@ -49,8 +49,12 @@ func (p *PodmanTestIntegration) PodmanExtraFiles(args []string, extraFiles []*os
 }
 
 func (p *PodmanTestIntegration) setDefaultRegistriesConfigEnv() {
-	defaultFile := filepath.Join(INTEGRATION_ROOT, "test/registries.conf")
-	os.Setenv("CONTAINERS_REGISTRIES_CONF", defaultFile)
+	defaultFile := "registries.conf"
+	if UsingCacheRegistry() {
+		defaultFile = "registries-cached.conf"
+	}
+	defaultPath := filepath.Join(INTEGRATION_ROOT, "test", defaultFile)
+	os.Setenv("CONTAINERS_REGISTRIES_CONF", defaultPath)
 }
 
 func (p *PodmanTestIntegration) setRegistriesConfigEnv(b []byte) {
diff --git a/test/e2e/libpod_suite_test.go b/test/e2e/libpod_suite_test.go
index 2e99206ebb..549777e796 100644
--- a/test/e2e/libpod_suite_test.go
+++ b/test/e2e/libpod_suite_test.go
@@ -37,8 +37,12 @@ func (p *PodmanTestIntegration) PodmanExtraFiles(args []string, extraFiles []*os
 }
 
 func (p *PodmanTestIntegration) setDefaultRegistriesConfigEnv() {
-	defaultFile := filepath.Join(INTEGRATION_ROOT, "test/registries.conf")
-	err := os.Setenv("CONTAINERS_REGISTRIES_CONF", defaultFile)
+	defaultFile := "registries.conf"
+	if UsingCacheRegistry() {
+		defaultFile = "registries-cached.conf"
+	}
+	defaultPath := filepath.Join(INTEGRATION_ROOT, "test", defaultFile)
+	err := os.Setenv("CONTAINERS_REGISTRIES_CONF", defaultPath)
 	Expect(err).ToNot(HaveOccurred())
 }
 
diff --git a/test/e2e/load_test.go b/test/e2e/load_test.go
index e4e190c30f..9cade0294e 100644
--- a/test/e2e/load_test.go
+++ b/test/e2e/load_test.go
@@ -148,7 +148,7 @@ var _ = Describe("Podman load", func() {
 			Skip("skip on ppc64le")
 		}
 		outfile := filepath.Join(podmanTest.TempDir, "alpine.tar")
-		alpVersion := "quay.io/libpod/alpine:3.2"
+		alpVersion := "quay.io/libpod/alpine:3.10.2"
 
 		pull := podmanTest.Podman([]string{"pull", "-q", alpVersion})
 		pull.WaitWithDefaultTimeout()
@@ -169,9 +169,12 @@ var _ = Describe("Podman load", func() {
 		inspect := podmanTest.Podman([]string{"inspect", ALPINE})
 		inspect.WaitWithDefaultTimeout()
 		Expect(result).Should(ExitCleanly())
+		Expect(result.OutputToString()).To(ContainSubstring(alpVersion))
+
 		inspect = podmanTest.Podman([]string{"inspect", alpVersion})
 		inspect.WaitWithDefaultTimeout()
 		Expect(result).Should(ExitCleanly())
+		Expect(result.OutputToString()).To(ContainSubstring(alpVersion))
 	})
 
 	It("podman load localhost registry from scratch", func() {
diff --git a/test/e2e/pull_test.go b/test/e2e/pull_test.go
index a38305acf3..523856e5ab 100644
--- a/test/e2e/pull_test.go
+++ b/test/e2e/pull_test.go
@@ -21,26 +21,21 @@ var _ = Describe("Podman pull", func() {
 
 		session = podmanTest.Podman([]string{"pull", "busybox:latest", "docker.io/library/ibetthisdoesnotexistfr:random", "alpine"})
 		session.WaitWithDefaultTimeout()
-		Expect(session).Should(ExitWithError(125, "initializing source docker://ibetthisdoesnotexistfr:random: reading manifest random in quay.io/libpod/ibetthisdoesnotexistfr:"))
+
+		// As of 2024-06 all Cirrus tests run using a local registry where
+		// we get 404. When running in dev environment, though, we still
+		// test against real registry, which returns 401
+		expect := "quay.io/libpod/ibetthisdoesnotexistfr: unauthorized: access to the requested resource is not authorized"
+		if UsingCacheRegistry() {
+			expect = "127.0.0.1:60333/libpod/ibetthisdoesnotexistfr: manifest unknown"
+		}
+		Expect(session).Should(ExitWithError(125, "initializing source docker://ibetthisdoesnotexistfr:random: reading manifest random in "+expect))
 
 		session = podmanTest.Podman([]string{"rmi", "busybox:musl", "alpine", "quay.io/libpod/cirros", "testdigest_v2s2@sha256:755f4d90b3716e2bf57060d249e2cd61c9ac089b1233465c5c2cb2d7ee550fdb"})
 		session.WaitWithDefaultTimeout()
 		Expect(session).Should(ExitCleanly())
 	})
 
-	It("podman pull bogus image", func() {
-		// This is a NOP in CI; but in a developer environment, if user
-		// has a valid login to quay.io, pull fails with "repository not found"
-		defer func() {
-			os.Unsetenv("REGISTRY_AUTH_FILE")
-		}()
-		os.Setenv("REGISTRY_AUTH_FILE", "/tmp/this/does/not/exist")
-
-		session := podmanTest.Podman([]string{"pull", "quay.io/libpod/ibetthisdoesntexist:there"})
-		session.WaitWithDefaultTimeout()
-		Expect(session).To(ExitWithError(125, "nitializing source docker://quay.io/libpod/ibetthisdoesntexist:there: reading manifest there in quay.io/libpod/ibetthisdoesntexist: unauthorized: access to the requested resource is not authorized"))
-	})
-
 	It("podman pull with tag --quiet", func() {
 		session := podmanTest.Podman([]string{"pull", "-q", "quay.io/libpod/testdigest_v2s2:20200210"})
 		session.WaitWithDefaultTimeout()
diff --git a/test/e2e/run_seccomp_test.go b/test/e2e/run_seccomp_test.go
index fd28ff7182..c55a8033e2 100644
--- a/test/e2e/run_seccomp_test.go
+++ b/test/e2e/run_seccomp_test.go
@@ -2,59 +2,69 @@ package integration
 
 import (
 	"fmt"
-	"path/filepath"
 
 	. "github.com/containers/podman/v5/test/utils"
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
 )
 
+// Create an image with the given seccomp label
+func makeLabeledImage(seccompLabel string) string {
+	ctrName := "temp-working-container"
+	imgName := "workingimage"
+	session := podmanTest.Podman([]string{"run", "--name", ctrName, CITEST_IMAGE, "true"})
+	session.WaitWithDefaultTimeout()
+	Expect(session).To(ExitCleanly())
+
+	session = podmanTest.Podman([]string{"commit", "-q", "--change", "LABEL io.containers.seccomp.profile=" + seccompLabel, ctrName, imgName})
+	session.WaitWithDefaultTimeout()
+	Expect(session).To(ExitCleanly())
+
+	return imgName
+}
+
 var _ = Describe("Podman run", func() {
 
 	It("podman run --seccomp-policy default", func() {
-		session := podmanTest.Podman([]string{"run", "-q", "--seccomp-policy", "default", alpineSeccomp, "ls"})
+		session := podmanTest.Podman([]string{"run", "-q", "--seccomp-policy", "default", CITEST_IMAGE, "ls"})
 		session.WaitWithDefaultTimeout()
 		Expect(session).Should(ExitCleanly())
 	})
 
 	It("podman run --seccomp-policy ''", func() {
 		// Empty string is interpreted as "default".
-		session := podmanTest.Podman([]string{"run", "-q", "--seccomp-policy", "", alpineSeccomp, "ls"})
+		session := podmanTest.Podman([]string{"run", "-q", "--seccomp-policy", "", CITEST_IMAGE, "ls"})
 		session.WaitWithDefaultTimeout()
 		Expect(session).Should(ExitCleanly())
 	})
 
 	It("podman run --seccomp-policy invalid", func() {
-		session := podmanTest.Podman([]string{"run", "--seccomp-policy", "invalid", alpineSeccomp, "ls"})
+		session := podmanTest.Podman([]string{"run", "--seccomp-policy", "invalid", CITEST_IMAGE, "ls"})
 		session.WaitWithDefaultTimeout()
 		Expect(session).To(ExitWithError(125, `invalid seccomp policy "invalid": valid policies are ["default" "image"]`))
 	})
 
 	It("podman run --seccomp-policy image (block all syscalls)", func() {
-		session := podmanTest.Podman([]string{"run", "--seccomp-policy", "image", alpineSeccomp, "ls"})
+		// This image has seccomp profiles that blocks all syscalls.
+		// The intention behind blocking all syscalls is to prevent
+		// regressions in the future.  The required syscalls can vary
+		// depending on which runtime we're using.
+		img := makeLabeledImage(`'{"defaultAction":"SCMP_ACT_ERRNO"}'`)
+		session := podmanTest.Podman([]string{"run", "--seccomp-policy", "image", img, "ls"})
 		session.WaitWithDefaultTimeout()
-		// TODO: we're getting a "cannot start a container that has
-		//       stopped" error which seems surprising.  Investigate
-		//       why that is so.
-		base := filepath.Base(podmanTest.OCIRuntime)
-		if base == "runc" {
-			// TODO: worse than that. With runc, we get two alternating failures:
-			//   126 + cannot start a container that has stopped
-			//   127 + failed to connect to container's attach socket ... ENOENT
-			Expect(session.ExitCode()).To(BeNumerically(">=", 126), "Exit status using runc")
-		} else if base == "crun" {
-			expect := fmt.Sprintf("OCI runtime error: %s: read from the init process", podmanTest.OCIRuntime)
-			if IsRemote() {
-				expect = fmt.Sprintf("for attach: %s: read from the init process: OCI runtime error", podmanTest.OCIRuntime)
-			}
-			Expect(session).To(ExitWithError(126, expect))
-		} else {
-			Skip("Not valid with the current OCI runtime")
+
+		expect := fmt.Sprintf("OCI runtime error: %s: read from the init process", podmanTest.OCIRuntime)
+		if IsRemote() {
+			expect = fmt.Sprintf("for attach: %s: read from the init process: OCI runtime error", podmanTest.OCIRuntime)
 		}
+		Expect(session).To(ExitWithError(126, expect))
 	})
 
 	It("podman run --seccomp-policy image (bogus profile)", func() {
-		session := podmanTest.Podman([]string{"run", "--seccomp-policy", "image", alpineBogusSeccomp, "ls"})
+		// This image has a bogus/invalid seccomp profile which should
+		// yield a json error when being read.
+		img := makeLabeledImage(`'BOGUS - this should yield an error'`)
+		session := podmanTest.Podman([]string{"run", "--seccomp-policy", "image", img, "ls"})
 		session.WaitWithDefaultTimeout()
 		Expect(session).Should(ExitWithError(125, "loading seccomp profile failed: decoding seccomp profile failed: invalid character 'B' looking for beginning of value"))
 	})
diff --git a/test/system/helpers.registry.bash b/test/system/helpers.registry.bash
index 605f4bd34f..643a0e5151 100644
--- a/test/system/helpers.registry.bash
+++ b/test/system/helpers.registry.bash
@@ -34,7 +34,7 @@ function start_registry() {
     mkdir -p $AUTHDIR
 
     # Registry image; copy of docker.io, but on our own registry
-    local REGISTRY_IMAGE="$PODMAN_TEST_IMAGE_REGISTRY/$PODMAN_TEST_IMAGE_USER/registry:2.8"
+    local REGISTRY_IMAGE="$PODMAN_TEST_IMAGE_REGISTRY/$PODMAN_TEST_IMAGE_USER/registry:2.8.2"
 
     # Pull registry image, but into a separate container storage and DB and everything
     PODMAN_LOGIN_ARGS="--storage-driver vfs $(podman_isolation_opts ${PODMAN_LOGIN_WORKDIR})"

From 07b69943a472fa5cfda4cdca00cac160b004478c Mon Sep 17 00:00:00 2001
From: Ed Santiago <santiago@redhat.com>
Date: Thu, 6 Jun 2024 13:24:46 -0600
Subject: [PATCH 3/3] CI: use local registry, part 3 of 3: for developers

New tool, get-local-registry-script, intended for developers
to get a local registry running in their environment. This is
not necessary for any tests, but may be desirable for performance
reasons and/or to recreate the CI environment.

Signed-off-by: Ed Santiago <santiago@redhat.com>
---
 contrib/cirrus/get-local-registry-script | 163 +++++++++++++++++++++++
 1 file changed, 163 insertions(+)
 create mode 100755 contrib/cirrus/get-local-registry-script

diff --git a/contrib/cirrus/get-local-registry-script b/contrib/cirrus/get-local-registry-script
new file mode 100755
index 0000000000..6e9d709973
--- /dev/null
+++ b/contrib/cirrus/get-local-registry-script
@@ -0,0 +1,163 @@
+#!/bin/bash
+#
+# get-local-registry-script - fetches local-registry script from automation_images repo
+#
+ME=$(basename $0)
+
+###############################################################################
+# BEGIN global settings
+
+SCRIPTNAME=cache_images/local-cache-registry
+
+REPO=containers/automation_images
+
+GITHUB_GRAPHQL=https://api.github.com/graphql
+
+# END   global settings
+###############################################################################
+
+usage="Usage: $ME [--help] [-v|--verbose] [IMG_SFX]
+
+--help          display usage message
+-v, --verbose   verbose output
+
+$ME fetches the $SCRIPTNAME tool
+from the github $REPO repo.
+If successful, the script will be left in cwd.
+
+It is up to you to invoke it:
+
+    \$ sudo ./$(basename $SCRIPTNAME) initialize
+
+This will set up a local registry and prepopulate it with
+images used in tests. Initialization may take tens of minutes
+on a slow connection.
+
+To use this registry in e2e or apiv2 tests:
+
+    export CI_USE_REGISTRY_CACHE=1
+
+To use in system tests, you need to overwrite /etc/containers/registries.conf
+because system tests, by definition, use system settings. This is left as
+an exercise for the reader.
+
+Reasons for doing this:
+  1) Consistency with CI environment
+  2) Speedier (and less flaky) tests
+"
+
+verbose=
+for i; do
+    value=`expr "$i" : '[^=]*=\(.*\)'`
+    case "$i" in
+    -h*|--help)	echo "$usage"; exit 0;;
+    -v|--verbose)	verbose=$i; shift;;
+    -*)	echo "$ME: unrecognized option $i" >&2
+	echo "$usage" >&2
+	exit 1;;
+    *)	break;;
+    esac
+done
+
+###############################################################################
+# BEGIN helper functions
+
+function die() {
+    echo "$ME: $*" >&2
+    exit 1
+}
+
+function get_imgsfx() {
+    test -e .cirrus.yml || die ".cirrus.yml does not exist; please run me from top of repo"
+
+    imgsfx=$(sed -ne 's/^ *IMAGE_SUFFIX:.*"c\(202.*\)"/\1/p' <.cirrus.yml)
+    if [[ -z "$imgsfx" ]]; then
+        die "Did not find 'IMAGE_SUFFIX:.*c202.*' in .cirrus.yml"
+    fi
+
+    expr "$imgsfx" : "^20[0-9]\{6\}t[0-9]\{6\}z-" &> /dev/null || die "IMAGE_SUFFIX '$imgsfx' (from .cirrus.yml) does not match expected YYYYMMDDtHHMMSS-* form"
+
+    echo $imgsfx
+}
+
+function get_script() {
+    local tag=$1
+
+    local tmpfile=$(mktemp --tmpdir --suffix=.sh $ME.fetched-script.XXXXXXX)
+
+    local url="https://raw.githubusercontent.com/${REPO}/$tag/$SCRIPTNAME"
+    test -n "$verbose" && echo "[ $url ]" >&2
+
+    curl -s -f -o $tmpfile $url
+    if [[ $? -eq 0 ]]; then
+        chmod 755 $tmpfile
+        mv -f $tmpfile $SCRIPTNAME
+        echo $SCRIPTNAME
+        return
+    fi
+    rm -f $tmpfile
+}
+
+# END   helper functions
+###############################################################################
+# BEGIN code
+
+if [[ -n "$1" ]]; then
+    imgsfx=$1
+    shift
+    if [[ -n "$*" ]]; then
+        die "Too many arguments; run $ME --help for help"
+    fi
+else
+    imgsfx=$(get_imgsfx)
+fi
+test -n "$verbose" && echo "[ imgsfx=$imgsfx ]" >&2
+
+# Easy case: this is a tagged automation_images build
+fetched_script=$(get_script $imgsfx)
+if [[ -n "$fetched_script" ]]; then
+    echo $fetched_script
+    exit 0
+fi
+
+# Nope, not a tagged build. Look for imgsfx in PR
+query="
+{
+    \"query\": \"{
+        search(query: \\\"repo:$REPO is:pr c$imgsfx\\\", type:ISSUE, first:10) {
+            edges {
+               node { ... on PullRequest {
+                       number
+                       title
+                       commits(last:30)  { nodes { commit { committedDate, oid } } }
+                       comments(last:30) { nodes { createdAt author { login } body } }
+                   }
+               }
+            }
+        }
+    }\"
+}
+"
+
+query_clean=$(tr -d \\012 <<<"$query")
+
+jsontmp=$(mktemp --tmpdir --suffix=.json $ME.graphql.XXXXXXX)
+
+curl -s -S -H "Authorization: bearer $GITHUB_TOKEN" -X POST --data "$query_clean" $GITHUB_GRAPHQL > $jsontmp
+
+# There should be one and exactly one comment from github-actions
+# that mentions the given imgsfx. Find its timestamp
+img_timestamp=$(jq -C -r '.data.search.edges.[].node.comments.nodes.[] | select(.author.login="github-actions") | select(.body | contains("c'${imgsfx}'")).createdAt' < $jsontmp)
+if [[ -z "$img_timestamp" ]]; then
+    die "Did not find a github-actions comment mentioning '$imgsfx'"
+fi
+
+# Some time prior to that comment, there was a commit pushed to github
+# that resulted in that build. Find that commit's SHA (OID).
+push_sha=$(jq -C -r '.data.search.edges.[].node.commits.nodes.[].commit | select(.committedDate < "'${img_timestamp}'").oid' < $jsontmp)
+
+# Got all the info we need. Clean up tmpfile
+rm -f $jsontmp
+
+# This must succeed.
+get_script $push_sha