ci: retry flakey tests 3 times

.buildkite/ci.mjs

@@ -29,6 +29,9 @@ import {
   writeFile,
 } from "../scripts/utils.mjs";
 
+/** Number of times to retry tests before failing. Helps with flakey tests. */
+const FLAKEY_RETRY_COUNT = 3;
+
 /**
  * @typedef {"linux" | "darwin" | "windows"} Os
  * @typedef {"aarch64" | "x64"} Arch
@@ -529,7 +532,7 @@ function getTestBunStep(platform, options, testOptions = {}) {
   const { os } = platform;
   const { buildId, unifiedTests, testFiles } = testOptions;
 
-  const args = [`--step=${getTargetKey(platform)}-build-bun`];
+  const args = [`--step=${getTargetKey(platform)}-build-bun`, `--flakey=${FLAKEY_RETRY_COUNT}`];
   if (buildId) {
     args.push(`--build-id=${buildId}`);
   }

scripts/runner.node.mjs

@@ -18,6 +18,7 @@ import {
   appendFileSync,
   readdirSync,
 } from "node:fs";
+import assert from "node:assert";
 import { spawn, spawnSync } from "node:child_process";
 import { join, basename, dirname, relative, sep } from "node:path";
 import { parseArgs } from "node:util";
@@ -49,6 +50,18 @@ const spawnTimeout = 5_000;
 const testTimeout = 3 * 60_000;
 const integrationTimeout = 5 * 60_000;
 
+const settings = {
+  /** Controls behavior for retrying flakey tests. */
+  flakey: {
+    /** Time (MS) to sleep between retries of flakey tests. */
+    initialSleepTime: 1,
+    /** Factor to increase sleep time by after each retry. */
+    sleepFactor: 1.25,
+    /** Max time to sleep between retries. */
+    maxSleepTime: 100,
+  },
+};
+
 const { values: options, positionals: filters } = parseArgs({
   allowPositionals: true,
   options: {
@@ -60,6 +73,13 @@ const { values: options, positionals: filters } = parseArgs({
       type: "string",
       default: "bun",
     },
+    // --flakey <n> - Flakey test recovery. Retry failed tests `n` times. `n`
+    // must be a positive int.
+    ["flakey"]: {
+      type: "string",
+      default: undefined,
+    },
+
     ["step"]: {
       type: "string",
       default: undefined,
@@ -114,6 +134,7 @@ if (options["quiet"]) {
  * @returns {Promise<TestResult[]>}
  */
 async function runTests() {
+  console.log("Running tests...");
   let execPath;
   if (options["step"]) {
     execPath = await getExecPathFromBuildKite(options["step"], options["build-id"]);
@@ -142,6 +163,8 @@ async function runTests() {
   let i = 0;
   let total = vendorTotal + tests.length + 2;
   const results = [];
+  /** number of times to retry (_not_ total number of times to run a test). 0 to only run once. */
+  const retryLimit = int(options["flakey"]);
 
   /**
    * @param {string} title
@@ -150,25 +173,93 @@ async function runTests() {
    */
   const runTest = async (title, fn) => {
     const label = `${getAnsi("gray")}[${++i}/${total}]${getAnsi("reset")} ${title}`;
-    const result = await startGroup(label, fn);
+    const result = await runTestWithRetry(label, fn);
     results.push(result);
 
-    if (isBuildkite) {
-      const { ok, error, stdoutPreview } = result;
-      if (title.startsWith("vendor")) {
-        const markdown = formatTestToMarkdown({ ...result, testPath: title });
-        if (markdown) {
-          reportAnnotationToBuildKite({ label: title, content: markdown, style: "warning", priority: 5 });
-        }
-      } else {
-        const markdown = formatTestToMarkdown(result);
-        if (markdown) {
-          reportAnnotationToBuildKite({ label: title, content: markdown, style: "error" });
+    const isVendor = title.startsWith("vendor");
+    reportResult(
+      result,
+      title,
+      "error",
+      /* buildkite */ isVendor ? { testPath: title } : {},
+      /* fmt to md */ isVendor ? { priority: isVendor } : {},
+    );
+
+    if (options["bail"] && !result.ok) {
+      process.exit(getExitCode("fail"));
+    }
+
+    return result;
+  };
+
+  /**
+   * @param {() => TestResult | Promise<TestResult>} fn
+   * @returns {Promise<TestResult>}
+   */
+  const runTestWithRetry = async (label, fn) => {
+    const { maxSleepTime, initialSleepTime, sleepFactor } = settings.flakey;
+    /** @type {(retry: number) => number} retry */
+    const sleepTime = retry => Math.min(maxSleepTime, initialSleepTime * Math.pow(sleepFactor, retry));
+    const sleep = ms => new Promise(resolves => setTimeout(resolves, ms));
+
+    if (!retryLimit) return startGroup(label, fn); // short circuit early
+
+    /** @type {TestResult | Error} */
+    let result = undefined;
+    /** @type {TestResult | undefined} */
+    let lastFailedResult = undefined;
+
+    for (let attempt = 0; attempt < retryLimit; attempt++) {
+      // NOTE: one-indexed attempt number [1 of 2]. Do not include attempt suffix on first try.
+      const attemptLabel = attempt == 0 ? label : `${label} [retry #${attempt + 1}]`;
+      try {
+        result = await startGroup(attemptLabel, fn);
+        if (result.ok) {
+          // we only warn about the last flakey test to avoid log spam.
+          if (lastFailedResult) {
+            reportResult(result, label, "warning", { testPath: label });
+          }
+          break;
         }
+
+        lastFailedResult = result;
+        // Wait a bit between retries. e.g. flake may happen b/c disk is busy.
+        if (attempt < retryLimit - 1) await sleep(sleepTime(attempt));
+      } catch (error) {
+        // NOTE: since we don't store this to lastFailedResult, flakes that
+        // throw an error won't be warned about if they succeed on retry.
+        result = error;
       }
+    }
+
+    if (result instanceof Error) {
+      throw result;
+    } else {
+      return result;
+    }
+  };
+
+  /**
+   * @param {TestResult} result
+   * @param {string} title
+   * @param {"error" | "warning"} level what kind of diagnostic to report when `result` fails.
+   * @param {object} buildkiteOptions add/override options passed to {@link reportAnnotationToBuildKite}
+   * @param {object} markdownOptions add/override options passed to {@link formatTestToMarkdown}
+   */
+  const reportResult = (result, title, level = "error", buildkiteOptions = {}, markdownOptions = {}) => {
+    if (isBuildkite) {
+      const { ok, error, stdoutPreview } = result;
+      const markdown = formatTestToMarkdown(Object.assign(result, markdownOptions));
+      reportAnnotationToBuildKite({ label: title, content: markdown, style: level, ...buildkiteOptions });
+
+      const color =
+        {
+          error: "red",
+          warning: "yellow",
+        }[level] || "white";
 
       if (!ok) {
-        const label = `${getAnsi("red")}[${i}/${total}] ${title} - ${error}${getAnsi("reset")}`;
+        const label = `${getAnsi(color)}[${i}/${total}] ${title} - ${error}${getAnsi("reset")}`;
         startGroup(label, () => {
           process.stderr.write(stdoutPreview);
         });
@@ -177,19 +268,8 @@ async function runTests() {
 
     if (isGithubAction) {
       const summaryPath = process.env["GITHUB_STEP_SUMMARY"];
-      if (summaryPath) {
-        const longMarkdown = formatTestToMarkdown(result);
-        appendFileSync(summaryPath, longMarkdown);
-      }
-      const shortMarkdown = formatTestToMarkdown(result, true);
-      appendFileSync("comment.md", shortMarkdown);
-    }
-
-    if (options["bail"] && !result.ok) {
-      process.exit(getExitCode("fail"));
+      appendFileSync(summaryPath || "comment.md", formatTestToMarkdown(result, !!summaryPath));
     }
-
-    return result;
   };
 
   if (!isQuiet) {
@@ -1291,6 +1371,25 @@ function listArtifactsFromBuildKite(glob, step) {
   return [];
 }
 
+/**
+ * Parse a string to a integer that is >= 0.
+ * - empty strings and nullish values -> default value
+ * - floats and NaN -> panic
+ *
+ * @param {string | undefined} value the integer value to parse
+ * @param {number} [defaultValue] the default value to return if the input is undefined. Defaults to 0.
+ *
+ * @param {string | undefined} value the integer value to parse
+ * @returns {number} the parsed integer
+ */
+const int = (value, defaultValue = 0) => {
+  if (!value) return defaultValue;
+  const i = parseInt(value, 10);
+  assert(!isNaN(i), `Expected an integer, but got: ${value}`);
+  assert(i >= 0);
+  return i;
+};
+
 /**
  * @typedef {object} BuildkiteAnnotation
  * @property {string} label