From 1c59a9ced1a625a14a3f0f69b23442a1d23fb328 Mon Sep 17 00:00:00 2001 From: Ed Santiago Date: Wed, 6 Dec 2023 15:06:03 -0700 Subject: [PATCH] CI: safer podman-stop tests A number of tests start a container then immediately run podman stop. This frequently flakes with: StopSignal SIGTERM failed to stop [...] in 10 seconds, resorting to SIGKILL Likely reason: container is still initializing, and its process has not yet set up its signal handlers. Solution: if possible (containers running "top"), wait for "Mem:" to indicate that top is running. If not possible (pods / catatonit), sleep half a second. Intended to fix some of the flakes cataloged in #20196 but I'm leaving that open in case we see more. These are hard to identify just by looking in the code. Signed-off-by: Ed Santiago --- test/e2e/common_test.go | 13 +++++++++++-- test/e2e/prune_test.go | 9 +++------ test/system/050-stop.bats | 3 +++ 3 files changed, 17 insertions(+), 8 deletions(-) diff --git a/test/e2e/common_test.go b/test/e2e/common_test.go index dd6af3f40d..6d0ed0d5f8 100644 --- a/test/e2e/common_test.go +++ b/test/e2e/common_test.go @@ -508,8 +508,17 @@ func (p *PodmanTestIntegration) RunTopContainerWithArgs(name string, args []stri podmanArgs = append(podmanArgs, "--name", name) } podmanArgs = append(podmanArgs, args...) - podmanArgs = append(podmanArgs, "-d", ALPINE, "top") - return p.Podman(podmanArgs) + podmanArgs = append(podmanArgs, "-d", ALPINE, "top", "-b") + session := p.Podman(podmanArgs) + session.WaitWithDefaultTimeout() + Expect(session).To(ExitCleanly()) + cid := session.OutputToString() + // Output indicates that top is running, which means it's safe + // for our caller to invoke `podman stop` + if !WaitContainerReady(p, cid, "Mem:", 20, 1) { + Fail("Could not start a top container") + } + return session } // RunLsContainer runs a simple container in the background that diff --git a/test/e2e/prune_test.go b/test/e2e/prune_test.go index 01e8484785..9fbbe0500d 100644 --- a/test/e2e/prune_test.go +++ b/test/e2e/prune_test.go @@ -220,8 +220,7 @@ var _ = Describe("Podman prune", func() { session = podmanTest.Podman([]string{"pod", "start", podid1}) session.WaitWithDefaultTimeout() Expect(session).Should(ExitCleanly()) - - session = podmanTest.Podman([]string{"pod", "stop", podid1}) + session = podmanTest.Podman([]string{"pod", "stop", "-t0", podid1}) session.WaitWithDefaultTimeout() Expect(session).Should(ExitCleanly()) @@ -294,8 +293,7 @@ var _ = Describe("Podman prune", func() { session = podmanTest.Podman([]string{"pod", "start", podid1}) session.WaitWithDefaultTimeout() Expect(session).Should(ExitCleanly()) - - session = podmanTest.Podman([]string{"pod", "stop", podid1}) + session = podmanTest.Podman([]string{"pod", "stop", "-t0", podid1}) session.WaitWithDefaultTimeout() Expect(session).Should(ExitCleanly()) @@ -327,8 +325,7 @@ var _ = Describe("Podman prune", func() { session = podmanTest.Podman([]string{"pod", "start", podid1}) session.WaitWithDefaultTimeout() Expect(session).Should(ExitCleanly()) - - session = podmanTest.Podman([]string{"pod", "stop", podid1}) + session = podmanTest.Podman([]string{"pod", "stop", "-t0", podid1}) session.WaitWithDefaultTimeout() Expect(session).Should(ExitCleanly()) diff --git a/test/system/050-stop.bats b/test/system/050-stop.bats index 3d7bb73901..fe02eafebd 100644 --- a/test/system/050-stop.bats +++ b/test/system/050-stop.bats @@ -77,12 +77,15 @@ load helpers # stop -a must print the IDs run_podman run -d $IMAGE top ctrID="$output" + # Output means container has set up its signal handlers + wait_for_output "Mem:" $ctrID run_podman stop --all is "$output" "$ctrID" # stop $input must print $input cname=$(random_string) run_podman run -d --name $cname $IMAGE top + wait_for_output "Mem:" $cname run_podman stop $cname is "$output" $cname