CI: safer podman-stop tests

A number of tests start a container then immediately run podman stop. This frequently flakes with: StopSignal SIGTERM failed to stop [...] in 10 seconds, resorting to SIGKILL Likely reason: container is still initializing, and its process has not yet set up its signal handlers. Solution: if possible (containers running "top"), wait for "Mem:" to indicate that top is running. If not possible (pods / catatonit), sleep half a second. Intended to fix some of the flakes cataloged in containers#20196 but I'm leaving that open in case we see more. These are hard to identify just by looking in the code. Signed-off-by: Ed Santiago <[email protected]>
rhatdan · Dec 14, 2023 · 1c59a9c · 1c59a9c
1 parent b82e336
commit 1c59a9c
Show file tree

Hide file tree

Showing 3 changed files with 17 additions and 8 deletions.
diff --git a/test/e2e/common_test.go b/test/e2e/common_test.go
@@ -508,8 +508,17 @@ func (p *PodmanTestIntegration) RunTopContainerWithArgs(name string, args []stri
 		podmanArgs = append(podmanArgs, "--name", name)
 	}
 	podmanArgs = append(podmanArgs, args...)
-	podmanArgs = append(podmanArgs, "-d", ALPINE, "top")
-	return p.Podman(podmanArgs)
+	podmanArgs = append(podmanArgs, "-d", ALPINE, "top", "-b")
+	session := p.Podman(podmanArgs)
+	session.WaitWithDefaultTimeout()
+	Expect(session).To(ExitCleanly())
+	cid := session.OutputToString()
+	// Output indicates that top is running, which means it's safe
+	// for our caller to invoke `podman stop`
+	if !WaitContainerReady(p, cid, "Mem:", 20, 1) {
+		Fail("Could not start a top container")
+	}
+	return session
 }
 
 // RunLsContainer runs a simple container in the background that

diff --git a/test/e2e/prune_test.go b/test/e2e/prune_test.go
@@ -220,8 +220,7 @@ var _ = Describe("Podman prune", func() {
 		session = podmanTest.Podman([]string{"pod", "start", podid1})
 		session.WaitWithDefaultTimeout()
 		Expect(session).Should(ExitCleanly())
-
-		session = podmanTest.Podman([]string{"pod", "stop", podid1})
+		session = podmanTest.Podman([]string{"pod", "stop", "-t0", podid1})
 		session.WaitWithDefaultTimeout()
 		Expect(session).Should(ExitCleanly())
 
@@ -294,8 +293,7 @@ var _ = Describe("Podman prune", func() {
 		session = podmanTest.Podman([]string{"pod", "start", podid1})
 		session.WaitWithDefaultTimeout()
 		Expect(session).Should(ExitCleanly())
-
-		session = podmanTest.Podman([]string{"pod", "stop", podid1})
+		session = podmanTest.Podman([]string{"pod", "stop", "-t0", podid1})
 		session.WaitWithDefaultTimeout()
 		Expect(session).Should(ExitCleanly())
 
@@ -327,8 +325,7 @@ var _ = Describe("Podman prune", func() {
 		session = podmanTest.Podman([]string{"pod", "start", podid1})
 		session.WaitWithDefaultTimeout()
 		Expect(session).Should(ExitCleanly())
-
-		session = podmanTest.Podman([]string{"pod", "stop", podid1})
+		session = podmanTest.Podman([]string{"pod", "stop", "-t0", podid1})
 		session.WaitWithDefaultTimeout()
 		Expect(session).Should(ExitCleanly())
 

diff --git a/test/system/050-stop.bats b/test/system/050-stop.bats
@@ -77,12 +77,15 @@ load helpers
     # stop -a must print the IDs
     run_podman run -d $IMAGE top
     ctrID="$output"
+    # Output means container has set up its signal handlers
+    wait_for_output "Mem:" $ctrID
     run_podman stop --all
     is "$output" "$ctrID"
 
     # stop $input must print $input
     cname=$(random_string)
     run_podman run -d --name $cname $IMAGE top
+    wait_for_output "Mem:" $cname
     run_podman stop $cname
     is "$output" $cname