fix(ci): ensure timeout steps also notify (#18962)

ci/scripts/notify.py

@@ -11,6 +11,7 @@
 MAIN_CRON_TEST_MAP = {
     "test-notify": ["noelkwan", "noelkwan"],
     "test-notify-2": ["noelkwan", "noelkwan"],
+    "test-notify-timeout": ["noelkwan", "noelkwan"],
     "docslt": ["tianxiao"],
     "e2e-test-release": ["zhi", "Eric"],
     "e2e-meta-backup-test-release": ["zhi", "Eric"],
@@ -110,7 +111,15 @@ def get_failed_tests(get_test_status, test_map):
     failed_test_map = {}
     for test in test_map.keys():
         test_status = get_test_status(test)
-        if test_status == "hard_failed" or test_status == "soft_failed":
+        if test_status == "hard_failed" or test_status == "soft_failed" or test_status == "errored":
+            print(f"{test} failed with outcome: {test_status}")
+            failed_test_map[test] = test_map[test]
+        elif test_status == "passed":
+            print(f"{test} passed with outcome: {test_status}")
+        elif test_status is None or test_status == "":
+            print(f"{test} no outcome, skipping")
+        else:
+            print(f"{test} failed with unknown outcome: {test_status}")
             failed_test_map[test] = test_map[test]
     return failed_test_map
 
@@ -188,14 +197,14 @@ def run_test_1():
 
 def main():
     test_map = get_test_map()
+    print("--- Getting failed tests")
     failed_test_map = get_failed_tests(get_buildkite_test_status, test_map)
     message = generate_test_status_message(failed_test_map)
     if message == "":
-        print("All tests passed, no need to notify")
+        print("--- Tests passed, no need to notify")
         return
     else:
-        print("Some tests failed, notify users")
-        print(message)
+        print("--- Some tests failed, notify users")
         cmd = format_cmd(message)
         print(cmd)
         subprocess.run(cmd, shell=True)

ci/workflows/main-cron.yml

@@ -1179,6 +1179,13 @@ steps:
     command: |
       bash -c 'echo test && exit -1'
 
+  # Notification test.
+  - key: "test-notify-timeout"
+    if: build.pull_request.labels includes "ci/main-cron/test-notify" || build.env("CI_STEPS") =~ /(^|,)test_notify(,|$$)/
+    command: |
+      bash -c 'echo test && sleep 300'
+    timeout_in_minutes: 1
+
   - wait: true
     continue_on_failure: true
     allow_dependency_failure: true