test: Add enough TestHistoryMetrics.testEvents wiggle room for PCP 6.3.2

That version changes the interpretation of existing data somewhat, and
thus events slightly move around. Make the assertions flexible enough to work
with old and new versions.

There was not much science in the original assertions anyway -- I simply
recorded some stuff and then pinned down the status quo.

We can make this stricter again once all supported OSes have PCP ≥ 6.3.2 (but
that'll be a few years).

Addresses the non-pixel parts of cockpit-project#21237

test/verify/check-metrics

@@ -303,7 +303,7 @@ class TestHistoryMetrics(testlib.MachineCase):
 
         self.assertGreaterEqual(getCompressedMinuteValue(test=self, g_type="cpu", saturation=True, hour=1598950800000, minute=7), 0.3)
         self.assertLessEqual(getCompressedMinuteValue(test=self, g_type="cpu", saturation=True, hour=1598950800000, minute=7), 0.4)
-        self.assertGreaterEqual(getCompressedMinuteValue(test=self, g_type="cpu", saturation=True, hour=1598950800000, minute=8), 0.3)
+        self.assertGreaterEqual(getCompressedMinuteValue(test=self, g_type="cpu", saturation=True, hour=1598950800000, minute=8), 0.1)
         self.assertLessEqual(getCompressedMinuteValue(test=self, g_type="cpu", saturation=True, hour=1598950800000, minute=8), 0.4)
 
         self.assertNotIn("Load", events_at(1598950800000, 2))
@@ -324,7 +324,7 @@ class TestHistoryMetrics(testlib.MachineCase):
         b.wait_in_text(".metrics-heading", "CPU")
 
         # basic RAM consumption after boot; it's still a network spike, thus event+SVG
-        self.assertLessEqual(getMaximumSpike(test=self, g_type="memory", saturation=False, hour=1600236000000, minute=44), 0.3)
+        self.assertLessEqual(getMaximumSpike(test=self, g_type="memory", saturation=False, hour=1600236000000, minute=44), 0.4)
         self.assertNotIn("Memory", events_at(1600236000000, 44))
         if have_swap:
             self.assertAlmostEqual(getMaximumSpike(test=self, g_type="memory", saturation=True, hour=1600236000000, minute=44), 0)
@@ -342,12 +342,15 @@ class TestHistoryMetrics(testlib.MachineCase):
             self.assertNotIn(b.text(".metrics-heading"), "Swap")
             b.wait_not_present(".metrics-data-memory .saturation")
 
-        # memory spike in :47
+        # memory spike in :46 or :47 (depends on PCP version)
         self.assertGreater(getMaximumSpike(test=self, g_type="memory", saturation=False, hour=1600236000000, minute=47), 0.6)
-        self.assertIn("Memory", events_at(1600236000000, 47))
+        try:
+            self.assertIn("Memory", events_at(1600236000000, 46))
+        except AssertionError:
+            self.assertIn("Memory", events_at(1600236000000, 47))
 
         # at :54 the machine is loaded to ~80% so no event even if elevated
-        self.assertGreater(getCompressedMinuteValue(test=self, g_type="memory", saturation=False, hour=1600236000000, minute=54), 0.8)
+        self.assertGreater(getCompressedMinuteValue(test=self, g_type="memory", saturation=False, hour=1600236000000, minute=54), 0.3)
         b.wait_not_present("#metrics-hour-1600236000000 div.metrics-minute[data-minute='54'] .metrics-events")
         if have_swap:
             self.assertAlmostEqual(getCompressedMinuteValue(test=self, g_type="memory", saturation=True, hour=1600236000000, minute=54), 0.0)
@@ -369,8 +372,13 @@ class TestHistoryMetrics(testlib.MachineCase):
         b.wait_text("#date-picker-select-toggle .pf-v5-c-select__toggle-text", "Today")
 
         b.select_PF_deprecated("#date-picker-select-toggle", "Wednesday, September 16, 2020")
-        self.assertGreater(getMaximumSpike(test=self, g_type="memory", saturation=False, hour=1600236000000, minute=51), 0.5)
-        self.assertIn("Memory", events_at(1600236000000, 51))
+        # memory spike at :50 or :51 (depends on PCP version)
+        try:
+            self.assertGreater(getMaximumSpike(test=self, g_type="memory", saturation=False, hour=1600236000000, minute=50), 0.3)
+            self.assertIn("Memory", events_at(1600236000000, 50))
+        except (AttributeError, AssertionError):
+            self.assertGreater(getMaximumSpike(test=self, g_type="memory", saturation=False, hour=1600236000000, minute=51), 0.3)
+            self.assertIn("Memory", events_at(1600236000000, 51))
 
         # Reload should keep the filters intact
         b.reload()
@@ -412,7 +420,14 @@ class TestHistoryMetrics(testlib.MachineCase):
         self.login_and_go("/metrics")
         b.wait_in_text(".metrics-heading", "CPU")
         b.click("#metrics-hour-1615197600000 button.metrics-events-expander")
-        b.wait_in_text("#metrics-hour-1615197600000 div.metrics-minute[data-minute='39'] .metrics-events span.spikes_info", "Load")
+        b.wait_in_text("#metrics-hour-1615197600000", "Load")
+        # Load event at :38 or :39 (depending on PCP version)
+        for load_minute in [38, 39]:
+            load_minute_sel = f"#metrics-hour-1615197600000 div.metrics-minute[data-minute='{load_minute}']"
+            if "Load" in b.text(load_minute_sel):
+                break
+        else:
+            self.fail("no Load event found at either :38 or :39")
 
         # Now add the journal
         # Older systemds get a slightly wrong log window with --since/until, so only run on newer ones
@@ -425,29 +440,33 @@ class TestHistoryMetrics(testlib.MachineCase):
         b.reload()
         b.enter_page("/metrics")
 
-        # details for load event at 10:39
+        # details for above load event
         b.wait_in_text(".metrics-heading", "CPU")
         b.click("#metrics-hour-1615197600000 button.metrics-events-expander")
-        b.click("#metrics-hour-1615197600000 div.metrics-minute[data-minute='39'] .metrics-events button.spikes_info")
-        b.wait_in_text(".cockpit-log-panel", "load-hog.service: Succeeded.")
-        b.wait_in_text(".cockpit-log-panel", "Stopping /usr/bin/sh -ec for i in `seq 500`")
-        b.click(".cockpit-logline:contains('Stopping /usr/bin/sh -ec for i in `seq 500`') .cockpit-log-message")
+        b.click(f"{load_minute_sel} .metrics-events button.spikes_info")
+        b.wait_in_text(".cockpit-log-panel", "load-hog")
+        action = "Stopping" if load_minute == 39 else "Started"
+        b.wait_in_text(".cockpit-log-panel", f"{action} /usr/bin/sh -ec for i in `seq 500`")
+        b.click(f".cockpit-logline:contains('{action} /usr/bin/sh -ec for i in `seq 500`') .cockpit-log-message")
         b.enter_page("/system/logs")
         b.wait_in_text(".pf-v5-c-card__header", "load-hog.service")
-        b.wait_in_text(".pf-v5-c-card__title", "Stopping /usr/bin/sh -ec")
+        b.wait_in_text(".pf-v5-c-card__title", "/usr/bin/sh -ec")
         b.click("li:contains('Logs')")
-        b.wait_visible(".cockpit-log-message:contains('Stopping /usr/bin/sh -ec')")
+        b.wait_in_text(".cockpit-log-panel", "/usr/bin/sh -ec")
 
         b.go("/metrics")
         b.enter_page("/metrics")
         # logs exist, should show tight range
         b.click("button:contains('View detailed logs')")
         b.enter_page("/system/logs")
-        b.wait_visible(".cockpit-log-message:contains('load-hog.service: Succeeded.')")
+        b.wait_in_text(".cockpit-log-panel", "load-hog")
         url = b.eval_js('window.location.hash')
         self.assertIn("priority=info", url)
-        self.assertIn("since=2021-3-8%2010%3A39%3A0", url)
-        self.assertIn("until=2021-3-8%2010%3A39%3A45", url)
+        self.assertIn(f"since=2021-3-8%2010%3A{load_minute}%3A", url)
+        try:
+            self.assertIn(f"until=2021-3-8%2010%3A{load_minute}%3A", url)
+        except AssertionError:
+            self.assertIn(f"until=2021-3-8%2010%3A{load_minute + 1}%3A", url)
 
     @testlib.nondestructive
     def testNoDataEnable(self):