Release w/ updates to listRuns (#440)

.github/workflows/integration_tests.yml

@@ -4,11 +4,28 @@ on:
   push:
     branches:
       - main
+  pull_request:
+    branches:
+      - main
+    types: [opened, synchronize, reopened, labeled, unlabeled]
   workflow_dispatch:
+    inputs:
+      run-python-tests:
+        description: "Run Python integration tests"
+        default: "true"
+        required: false
+      run-js-tests:
+        description: "Run JS integration tests"
+        default: "true"
+        required: false
 
 jobs:
   python_integration_test:
     name: Python Integration Test
+    if: >
+      github.event_name == 'push' ||
+      (github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.name, 'release')) ||
+      (github.event_name == 'workflow_dispatch' && ${{ github.event.inputs.run-python-tests == 'true' }})
     runs-on: ubuntu-20.04
     defaults:
       run:
@@ -33,6 +50,10 @@ jobs:
 
   js_integration_test:
     name: JS Integration Test
+    if: >
+      github.event_name == 'push' ||
+      (github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.name, 'release')) ||
+      (github.event_name == 'workflow_dispatch' && ${{ github.event.inputs.run-js-tests == 'true' }})
     runs-on: ubuntu-20.04
     defaults:
       run:

.github/workflows/release_js.yml

@@ -31,7 +31,17 @@ jobs:
         run: cd js && yarn run build
       - name: Check version
         run: cd js && yarn run check-version
+      - name: Check NPM version
+        id: check_npm_version
+        run: |
+          cd js
+          if yarn run check-npm-version; then
+            echo "::set-output name=should_publish::true"
+          else
+            echo "::set-output name=should_publish::false"
+          fi
       - name: Publish package to NPM
+        if: steps.check_npm_version.outputs.should_publish == 'true'
         run: |
           cd js
           echo "//registry.npmjs.org/:_authToken=${{ secrets.NPM_TOKEN }}" > .npmrc

js/package.json

@@ -1,6 +1,6 @@
 {
   "name": "langsmith",
-  "version": "0.0.69",
+  "version": "0.0.70",
   "description": "Client library to connect to the LangSmith LLM Tracing and Evaluation Platform.",
   "packageManager": "[email protected]",
   "files": [
@@ -31,6 +31,7 @@
     "build": "yarn clean && yarn build:esm && yarn build:cjs && node scripts/create-entrypoints.js && node scripts/create-cli.js",
     "bump-version": "node scripts/bump-version.js",
     "check-version": "node scripts/check-version.js",
+    "check-npm-version": "node scripts/check-npm-version.js",
     "clean": "rm -rf dist/ && node scripts/create-entrypoints.js clean",
     "build:esm": "tsc --outDir dist/ && rm -rf dist/tests dist/**/tests",
     "build:cjs": "tsc --outDir dist-cjs/ -p tsconfig.cjs.json && node scripts/move-cjs-to-dist.js && rm -r dist-cjs",
@@ -122,4 +123,4 @@
     },
     "./package.json": "./package.json"
   }
-}
+}

js/scripts/check-npm-version.js

@@ -0,0 +1,27 @@
+import { execSync } from 'child_process';
+import fs from 'fs';
+import { fileURLToPath } from 'url';
+import path from 'path';
+
+// Convert the URL to a file path
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = path.dirname(__filename);
+
+// Adjust the path to your package.json as necessary
+const packageJsonPath = path.join(__dirname, '../package.json');
+const packageJson = JSON.parse(fs.readFileSync(packageJsonPath, { encoding: 'utf-8' }));
+const { version } = packageJson;
+const { name: packageName } = packageJson;
+
+try {
+    const npmVersion = execSync(`npm view ${packageName} version`, { encoding: 'utf-8' }).trim();
+    if (npmVersion && version <= npmVersion) {
+        console.error(`Current version ${version} is not greater than npm version ${npmVersion}.`);
+        process.exit(1); // Exit with error
+    } else {
+        console.log(`Current version ${version} is greater than npm version ${npmVersion}. Proceeding with publish.`);
+    }
+} catch (error) {
+    console.error('Error checking version:', error);
+    process.exit(1); // Exit with error if the check fails
+}

js/scripts/check-version.js

@@ -1,5 +1,4 @@
 import { readFileSync } from "fs";
-import { join } from "path";
 const indexFilePath = "src/index.ts";
 const packageJson = JSON.parse(readFileSync("package.json"));
 let indexFileContent = readFileSync(indexFilePath, "utf-8");

js/src/client.ts

@@ -47,6 +47,7 @@ interface ClientConfig {
 interface ListRunsParams {
   projectId?: string | string[];
   projectName?: string | string[];
+  traceId?: string;
   executionOrder?: number;
   parentRunId?: string;
   referenceExampleId?: string;
@@ -808,6 +809,7 @@ export class Client {
     projectId,
     projectName,
     parentRunId,
+    traceId,
     referenceExampleId,
     startTime,
     executionOrder,
@@ -845,6 +847,7 @@ export class Client {
       error,
       id,
       limit,
+      trace: traceId,
     };
 
     for await (const runs of this._getCursorPaginatedList<Run>(

js/src/index.ts

@@ -5,4 +5,4 @@ export { Dataset, Example, TracerSession, Run, Feedback } from "./schemas.js";
 export { RunTree, RunTreeConfig } from "./run_trees.js";
 
 // Update using yarn bump-version
-export const __version__ = "0.0.69";
+export const __version__ = "0.0.70";

js/src/tests/client.int.test.ts

@@ -145,7 +145,7 @@ test.concurrent(
   "test create dataset",
   async () => {
     const langchainClient = new Client({ autoBatchTracing: false });
-    const datasetName = "__test_create_dataset";
+    const datasetName = "__test_create_dataset JS";
     const datasets = await toArray(
       langchainClient.listDatasets({ datasetName })
     );
@@ -202,8 +202,8 @@ test.concurrent(
   "Test list datasets",
   async () => {
     const langchainClient = new Client({ autoBatchTracing: false });
-    const datasetName1 = "___TEST dataset1";
-    const datasetName2 = "___TEST dataset2";
+    const datasetName1 = "___TEST dataset1 JS";
+    const datasetName2 = "___TEST dataset2 JS";
     await deleteDataset(langchainClient, datasetName1);
     await deleteDataset(langchainClient, datasetName2);
     // Create two new datasets
@@ -247,7 +247,7 @@ test.concurrent(
   "Test create feedback with source run",
   async () => {
     const langchainClient = new Client({ autoBatchTracing: false });
-    const projectName = "__test_create_feedback_with_source_run";
+    const projectName = "__test_create_feedback_with_source_run JS";
     await deleteProject(langchainClient, projectName);
     const runId = uuidv4();
     await langchainClient.createRun({
@@ -290,7 +290,7 @@ test.concurrent(
       hideOutputs: true,
       autoBatchTracing: false,
     });
-    const projectName = "__test_create_run_with_masked_inputs_outputs";
+    const projectName = "__test_create_run_with_masked_inputs_outputs JS";
     await deleteProject(langchainClient, projectName);
     const runId = uuidv4();
     await langchainClient.createRun({
@@ -344,7 +344,7 @@ test.concurrent(
     process.env.LANGCHAIN_OTHER_FIELD = "test_other_field";
     // eslint-disable-next-line no-process-env
     process.env.LANGCHAIN_OTHER_KEY = "test_other_key";
-    const projectName = "__test_create_run_with_revision_id";
+    const projectName = "__test_create_run_with_revision_id JS";
     await deleteProject(langchainClient, projectName);
     const runId = uuidv4();
     await langchainClient.createRun({
@@ -396,7 +396,7 @@ describe("createChatExample", () => {
   it("should convert LangChainBaseMessage objects to examples", async () => {
     const langchainClient = new Client({ autoBatchTracing: false });
 
-    const datasetName = "__createChatExample-test-dataset";
+    const datasetName = "__createChatExample-test-dataset JS";
     await deleteDataset(langchainClient, datasetName);
     const dataset = await langchainClient.createDataset(datasetName);
 
@@ -477,7 +477,7 @@ test.concurrent(
   "Examples CRUD",
   async () => {
     const client = new Client({ autoBatchTracing: false });
-    const datasetName = "__test_examples_crud";
+    const datasetName = "__test_examples_crud JS";
     await deleteDataset(client, datasetName);
     const dataset = await client.createDataset(datasetName);
     const example = await client.createExample(

js/src/tests/run_trees.int.test.ts

@@ -145,6 +145,20 @@ test.concurrent(
       runMap.get("parent_run")?.id
     );
     expect(runMap.get("parent_run")?.parent_run_id).toBeNull();
+
+    const traceRunsIter = langchainClient.listRuns({
+      traceId: runs[0].trace_id,
+    });
+    const traceRuns = await toArray(traceRunsIter);
+    expect(traceRuns.length).toEqual(5);
+    // Sort by dotted order and assert runs lists are equal
+    const sortedRuns = runs.sort((a, b) =>
+      (a?.dotted_order ?? "").localeCompare(b?.dotted_order ?? "")
+    );
+    const sortedTraceRuns = traceRuns.sort((a, b) =>
+      (a?.dotted_order ?? "").localeCompare(b?.dotted_order ?? "")
+    );
+    expect(sortedRuns).toEqual(sortedTraceRuns);
     await langchainClient.deleteProject({ projectName });
   },
   120_000

python/langsmith/client.py

@@ -1242,6 +1242,7 @@ def list_runs(
         project_id: Optional[Union[ID_TYPE, Sequence[ID_TYPE]]] = None,
         project_name: Optional[Union[str, Sequence[str]]] = None,
         run_type: Optional[str] = None,
+        trace_id: Optional[ID_TYPE] = None,
         reference_example_id: Optional[ID_TYPE] = None,
         query: Optional[str] = None,
         filter: Optional[str] = None,
@@ -1262,6 +1263,8 @@ def list_runs(
             The name(s) of the project to filter by.
         run_type : str or None, default=None
             The type of the runs to filter by.
+        trace_id : UUID or None, default=None
+            The ID of the trace to filter by.
         reference_example_id : UUID or None, default=None
             The ID of the reference example to filter by.
         query : str or None, default=None
@@ -1313,6 +1316,7 @@ def list_runs(
             "start_time": start_time.isoformat() if start_time else None,
             "error": error,
             "id": run_ids,
+            "trace": trace_id,
             **kwargs,
         }
         body_query = {k: v for k, v in body_query.items() if v is not None}

python/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "langsmith"
-version = "0.0.91"
+version = "0.0.92"
 description = "Client library to connect to the LangSmith LLM Tracing and Evaluation Platform."
 authors = ["LangChain <[email protected]>"]
 license = "MIT"

python/tests/integration_tests/test_runs.py

@@ -224,12 +224,18 @@ async def my_chain_run(text: str, run_tree: RunTree):
     executor.shutdown(wait=True)
     poll_runs_until_count(langchain_client, project_name, 17)
     runs = list(langchain_client.list_runs(project_name=project_name))
+    trace_runs = list(langchain_client.list_runs(trace_id=runs[0].trace_id))
+    assert len(trace_runs) == 17
     assert len(runs) == 17
     assert sum([run.run_type == "llm" for run in runs]) == 8
     assert sum([run.name == "async_llm" for run in runs]) == 6
     assert sum([run.name == "my_llm_run" for run in runs]) == 2
     assert sum([run.run_type == "tool" for run in runs]) == 6
     assert sum([run.run_type == "chain" for run in runs]) == 3
+    # sort by dotted_order
+    runs = sorted(runs, key=lambda run: run.dotted_order)
+    trace_runs = sorted(trace_runs, key=lambda run: run.dotted_order)
+    assert runs == trace_runs
     # Check that all instances of async_llm have a parent with
     # the same name (my_tool_run)
     name_to_ids_map = defaultdict(list)