[JS] Experiment Name Retry on Conflict (#939)

js/src/evaluation/_runner.ts

@@ -15,6 +15,7 @@ import {
   RunEvaluator,
   runEvaluator,
 } from "./evaluator.js";
+import { LangSmithConflictError } from "../utils/error.js";
 import { v4 as uuidv4 } from "uuid";
 
 // eslint-disable-next-line @typescript-eslint/no-explicit-any
@@ -140,7 +141,7 @@ interface ExperimentResultRow {
  * Supports lazily running predictions and evaluations in parallel to facilitate
  * result streaming and early debugging.
  */
-class _ExperimentManager {
+export class _ExperimentManager {
   _data?: DataT;
 
   _runs?: AsyncGenerator<Run>;
@@ -312,17 +313,41 @@ class _ExperimentManager {
     return projectMetadata;
   }
 
-  async _getProject(firstExample: Example): Promise<TracerSession> {
+  async _createProject(firstExample: Example, projectMetadata: KVMap) {
+    // Create the project, updating the experimentName until we find a unique one.
     let project: TracerSession;
-    if (!this._experiment) {
+    const originalExperimentName = this._experimentName;
+    for (let i = 0; i < 10; i++) {
       try {
-        const projectMetadata = await this._getExperimentMetadata();
         project = await this.client.createProject({
-          projectName: this.experimentName,
+          projectName: this._experimentName,
           referenceDatasetId: firstExample.dataset_id,
           metadata: projectMetadata,
           description: this._description,
         });
+        return project;
+      } catch (e) {
+        // Naming collision
+        if ((e as LangSmithConflictError)?.name === "LangSmithConflictError") {
+          const ent = uuidv4().slice(0, 6);
+          this._experimentName = `${originalExperimentName}-${ent}`;
+        } else {
+          throw e;
+        }
+      }
+    }
+    throw new Error(
+      "Could not generate a unique experiment name within 10 attempts." +
+        " Please try again with a different name."
+    );
+  }
+
+  async _getProject(firstExample: Example): Promise<TracerSession> {
+    let project: TracerSession;
+    if (!this._experiment) {
+      try {
+        const projectMetadata = await this._getExperimentMetadata();
+        project = await this._createProject(firstExample, projectMetadata);
         this._experiment = project;
       } catch (e) {
         if (String(e).includes("already exists")) {

js/src/tests/evaluate_comparative.int.test.ts

@@ -2,8 +2,9 @@ import { evaluate } from "../evaluation/_runner.js";
 import { evaluateComparative } from "../evaluation/evaluate_comparative.js";
 import { Client } from "../index.js";
 import { waitUntilRunFound } from "./utils.js";
+import { v4 as uuidv4 } from "uuid";
 
-const TESTING_DATASET_NAME = "test_evaluate_comparative_js";
+const TESTING_DATASET_NAME = `test_evaluate_comparative_js_${uuidv4()}`;
 
 beforeAll(async () => {
   const client = new Client();

js/src/tests/experiment_manager.int.test.ts

@@ -0,0 +1,45 @@
+import { _ExperimentManager } from "../evaluation/_runner.js";
+import { Client } from "../index.js";
+import { v4 as uuidv4 } from "uuid";
+
+const TESTING_DATASET_NAME = `test_experiment_manager_${uuidv4()}`;
+
+beforeAll(async () => {
+  const client = new Client();
+
+  if (!(await client.hasDataset({ datasetName: TESTING_DATASET_NAME }))) {
+    await client.createDataset(TESTING_DATASET_NAME, {
+      description: "For testing pruposes",
+    });
+
+    await client.createExamples({
+      inputs: [{ input: 1 }, { input: 2 }],
+      outputs: [{ output: 2 }, { output: 3 }],
+      datasetName: TESTING_DATASET_NAME,
+    });
+  }
+});
+
+afterAll(async () => {
+  const client = new Client();
+  await client.deleteDataset({ datasetName: TESTING_DATASET_NAME });
+});
+
+describe("experiment manager", () => {
+  test("can recover from collisions", async () => {
+    const client = new Client();
+    const ds = await client.readDataset({ datasetName: TESTING_DATASET_NAME });
+    const manager = await new _ExperimentManager({
+      data: TESTING_DATASET_NAME,
+      client,
+      numRepetitions: 1,
+    });
+    const experimentName = manager._experimentName;
+    await client.createProject({
+      projectName: experimentName,
+      referenceDatasetId: ds.id,
+    });
+    await manager.start();
+    expect(manager._experimentName).not.toEqual(experimentName);
+  });
+});