diff --git a/.github/workflows/integration_tests.yml b/.github/workflows/integration_tests.yml index 6f0943ddd..b8ff3f553 100644 --- a/.github/workflows/integration_tests.yml +++ b/.github/workflows/integration_tests.yml @@ -4,11 +4,28 @@ on: push: branches: - main + pull_request: + branches: + - main + types: [opened, synchronize, reopened, labeled, unlabeled] workflow_dispatch: + inputs: + run-python-tests: + description: "Run Python integration tests" + default: "true" + required: false + run-js-tests: + description: "Run JS integration tests" + default: "true" + required: false jobs: python_integration_test: name: Python Integration Test + if: > + github.event_name == 'push' || + (github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.name, 'release')) || + (github.event_name == 'workflow_dispatch' && ${{ github.event.inputs.run-python-tests == 'true' }}) runs-on: ubuntu-20.04 defaults: run: @@ -33,6 +50,10 @@ jobs: js_integration_test: name: JS Integration Test + if: > + github.event_name == 'push' || + (github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.name, 'release')) || + (github.event_name == 'workflow_dispatch' && ${{ github.event.inputs.run-js-tests == 'true' }}) runs-on: ubuntu-20.04 defaults: run: diff --git a/.github/workflows/release_js.yml b/.github/workflows/release_js.yml index 7083786f1..778b5c948 100644 --- a/.github/workflows/release_js.yml +++ b/.github/workflows/release_js.yml @@ -31,7 +31,17 @@ jobs: run: cd js && yarn run build - name: Check version run: cd js && yarn run check-version + - name: Check NPM version + id: check_npm_version + run: | + cd js + if yarn run check-npm-version; then + echo "::set-output name=should_publish::true" + else + echo "::set-output name=should_publish::false" + fi - name: Publish package to NPM + if: steps.check_npm_version.outputs.should_publish == 'true' run: | cd js echo "//registry.npmjs.org/:_authToken=${{ secrets.NPM_TOKEN }}" > .npmrc diff --git a/js/package.json b/js/package.json index da4695470..a2a9ce837 100644 --- a/js/package.json +++ b/js/package.json @@ -1,6 +1,6 @@ { "name": "langsmith", - "version": "0.0.69", + "version": "0.0.70", "description": "Client library to connect to the LangSmith LLM Tracing and Evaluation Platform.", "packageManager": "yarn@1.22.19", "files": [ @@ -31,6 +31,7 @@ "build": "yarn clean && yarn build:esm && yarn build:cjs && node scripts/create-entrypoints.js && node scripts/create-cli.js", "bump-version": "node scripts/bump-version.js", "check-version": "node scripts/check-version.js", + "check-npm-version": "node scripts/check-npm-version.js", "clean": "rm -rf dist/ && node scripts/create-entrypoints.js clean", "build:esm": "tsc --outDir dist/ && rm -rf dist/tests dist/**/tests", "build:cjs": "tsc --outDir dist-cjs/ -p tsconfig.cjs.json && node scripts/move-cjs-to-dist.js && rm -r dist-cjs", @@ -122,4 +123,4 @@ }, "./package.json": "./package.json" } -} +} \ No newline at end of file diff --git a/js/scripts/check-npm-version.js b/js/scripts/check-npm-version.js new file mode 100644 index 000000000..14c4c1f99 --- /dev/null +++ b/js/scripts/check-npm-version.js @@ -0,0 +1,27 @@ +import { execSync } from 'child_process'; +import fs from 'fs'; +import { fileURLToPath } from 'url'; +import path from 'path'; + +// Convert the URL to a file path +const __filename = fileURLToPath(import.meta.url); +const __dirname = path.dirname(__filename); + +// Adjust the path to your package.json as necessary +const packageJsonPath = path.join(__dirname, '../package.json'); +const packageJson = JSON.parse(fs.readFileSync(packageJsonPath, { encoding: 'utf-8' })); +const { version } = packageJson; +const { name: packageName } = packageJson; + +try { + const npmVersion = execSync(`npm view ${packageName} version`, { encoding: 'utf-8' }).trim(); + if (npmVersion && version <= npmVersion) { + console.error(`Current version ${version} is not greater than npm version ${npmVersion}.`); + process.exit(1); // Exit with error + } else { + console.log(`Current version ${version} is greater than npm version ${npmVersion}. Proceeding with publish.`); + } +} catch (error) { + console.error('Error checking version:', error); + process.exit(1); // Exit with error if the check fails +} diff --git a/js/scripts/check-version.js b/js/scripts/check-version.js index 70642b576..7b81c7f0e 100644 --- a/js/scripts/check-version.js +++ b/js/scripts/check-version.js @@ -1,5 +1,4 @@ import { readFileSync } from "fs"; -import { join } from "path"; const indexFilePath = "src/index.ts"; const packageJson = JSON.parse(readFileSync("package.json")); let indexFileContent = readFileSync(indexFilePath, "utf-8"); diff --git a/js/src/client.ts b/js/src/client.ts index dcf0fbb7f..743899c0d 100644 --- a/js/src/client.ts +++ b/js/src/client.ts @@ -47,6 +47,7 @@ interface ClientConfig { interface ListRunsParams { projectId?: string | string[]; projectName?: string | string[]; + traceId?: string; executionOrder?: number; parentRunId?: string; referenceExampleId?: string; @@ -808,6 +809,7 @@ export class Client { projectId, projectName, parentRunId, + traceId, referenceExampleId, startTime, executionOrder, @@ -845,6 +847,7 @@ export class Client { error, id, limit, + trace: traceId, }; for await (const runs of this._getCursorPaginatedList( diff --git a/js/src/index.ts b/js/src/index.ts index 0a9df5dae..013cbac9c 100644 --- a/js/src/index.ts +++ b/js/src/index.ts @@ -5,4 +5,4 @@ export { Dataset, Example, TracerSession, Run, Feedback } from "./schemas.js"; export { RunTree, RunTreeConfig } from "./run_trees.js"; // Update using yarn bump-version -export const __version__ = "0.0.69"; +export const __version__ = "0.0.70"; diff --git a/js/src/tests/client.int.test.ts b/js/src/tests/client.int.test.ts index c52b54d54..bc5a97154 100644 --- a/js/src/tests/client.int.test.ts +++ b/js/src/tests/client.int.test.ts @@ -145,7 +145,7 @@ test.concurrent( "test create dataset", async () => { const langchainClient = new Client({ autoBatchTracing: false }); - const datasetName = "__test_create_dataset"; + const datasetName = "__test_create_dataset JS"; const datasets = await toArray( langchainClient.listDatasets({ datasetName }) ); @@ -202,8 +202,8 @@ test.concurrent( "Test list datasets", async () => { const langchainClient = new Client({ autoBatchTracing: false }); - const datasetName1 = "___TEST dataset1"; - const datasetName2 = "___TEST dataset2"; + const datasetName1 = "___TEST dataset1 JS"; + const datasetName2 = "___TEST dataset2 JS"; await deleteDataset(langchainClient, datasetName1); await deleteDataset(langchainClient, datasetName2); // Create two new datasets @@ -247,7 +247,7 @@ test.concurrent( "Test create feedback with source run", async () => { const langchainClient = new Client({ autoBatchTracing: false }); - const projectName = "__test_create_feedback_with_source_run"; + const projectName = "__test_create_feedback_with_source_run JS"; await deleteProject(langchainClient, projectName); const runId = uuidv4(); await langchainClient.createRun({ @@ -290,7 +290,7 @@ test.concurrent( hideOutputs: true, autoBatchTracing: false, }); - const projectName = "__test_create_run_with_masked_inputs_outputs"; + const projectName = "__test_create_run_with_masked_inputs_outputs JS"; await deleteProject(langchainClient, projectName); const runId = uuidv4(); await langchainClient.createRun({ @@ -344,7 +344,7 @@ test.concurrent( process.env.LANGCHAIN_OTHER_FIELD = "test_other_field"; // eslint-disable-next-line no-process-env process.env.LANGCHAIN_OTHER_KEY = "test_other_key"; - const projectName = "__test_create_run_with_revision_id"; + const projectName = "__test_create_run_with_revision_id JS"; await deleteProject(langchainClient, projectName); const runId = uuidv4(); await langchainClient.createRun({ @@ -396,7 +396,7 @@ describe("createChatExample", () => { it("should convert LangChainBaseMessage objects to examples", async () => { const langchainClient = new Client({ autoBatchTracing: false }); - const datasetName = "__createChatExample-test-dataset"; + const datasetName = "__createChatExample-test-dataset JS"; await deleteDataset(langchainClient, datasetName); const dataset = await langchainClient.createDataset(datasetName); @@ -477,7 +477,7 @@ test.concurrent( "Examples CRUD", async () => { const client = new Client({ autoBatchTracing: false }); - const datasetName = "__test_examples_crud"; + const datasetName = "__test_examples_crud JS"; await deleteDataset(client, datasetName); const dataset = await client.createDataset(datasetName); const example = await client.createExample( diff --git a/js/src/tests/run_trees.int.test.ts b/js/src/tests/run_trees.int.test.ts index af9572802..749ebe2d7 100644 --- a/js/src/tests/run_trees.int.test.ts +++ b/js/src/tests/run_trees.int.test.ts @@ -145,6 +145,20 @@ test.concurrent( runMap.get("parent_run")?.id ); expect(runMap.get("parent_run")?.parent_run_id).toBeNull(); + + const traceRunsIter = langchainClient.listRuns({ + traceId: runs[0].trace_id, + }); + const traceRuns = await toArray(traceRunsIter); + expect(traceRuns.length).toEqual(5); + // Sort by dotted order and assert runs lists are equal + const sortedRuns = runs.sort((a, b) => + (a?.dotted_order ?? "").localeCompare(b?.dotted_order ?? "") + ); + const sortedTraceRuns = traceRuns.sort((a, b) => + (a?.dotted_order ?? "").localeCompare(b?.dotted_order ?? "") + ); + expect(sortedRuns).toEqual(sortedTraceRuns); await langchainClient.deleteProject({ projectName }); }, 120_000 diff --git a/python/langsmith/client.py b/python/langsmith/client.py index 169e995b6..62ee0f31c 100644 --- a/python/langsmith/client.py +++ b/python/langsmith/client.py @@ -1242,6 +1242,7 @@ def list_runs( project_id: Optional[Union[ID_TYPE, Sequence[ID_TYPE]]] = None, project_name: Optional[Union[str, Sequence[str]]] = None, run_type: Optional[str] = None, + trace_id: Optional[ID_TYPE] = None, reference_example_id: Optional[ID_TYPE] = None, query: Optional[str] = None, filter: Optional[str] = None, @@ -1262,6 +1263,8 @@ def list_runs( The name(s) of the project to filter by. run_type : str or None, default=None The type of the runs to filter by. + trace_id : UUID or None, default=None + The ID of the trace to filter by. reference_example_id : UUID or None, default=None The ID of the reference example to filter by. query : str or None, default=None @@ -1313,6 +1316,7 @@ def list_runs( "start_time": start_time.isoformat() if start_time else None, "error": error, "id": run_ids, + "trace": trace_id, **kwargs, } body_query = {k: v for k, v in body_query.items() if v is not None} diff --git a/python/pyproject.toml b/python/pyproject.toml index 9a9c77260..1bd0cf6cf 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "langsmith" -version = "0.0.91" +version = "0.0.92" description = "Client library to connect to the LangSmith LLM Tracing and Evaluation Platform." authors = ["LangChain "] license = "MIT" diff --git a/python/tests/integration_tests/test_runs.py b/python/tests/integration_tests/test_runs.py index d04512271..e63b83401 100644 --- a/python/tests/integration_tests/test_runs.py +++ b/python/tests/integration_tests/test_runs.py @@ -224,12 +224,18 @@ async def my_chain_run(text: str, run_tree: RunTree): executor.shutdown(wait=True) poll_runs_until_count(langchain_client, project_name, 17) runs = list(langchain_client.list_runs(project_name=project_name)) + trace_runs = list(langchain_client.list_runs(trace_id=runs[0].trace_id)) + assert len(trace_runs) == 17 assert len(runs) == 17 assert sum([run.run_type == "llm" for run in runs]) == 8 assert sum([run.name == "async_llm" for run in runs]) == 6 assert sum([run.name == "my_llm_run" for run in runs]) == 2 assert sum([run.run_type == "tool" for run in runs]) == 6 assert sum([run.run_type == "chain" for run in runs]) == 3 + # sort by dotted_order + runs = sorted(runs, key=lambda run: run.dotted_order) + trace_runs = sorted(trace_runs, key=lambda run: run.dotted_order) + assert runs == trace_runs # Check that all instances of async_llm have a parent with # the same name (my_tool_run) name_to_ids_map = defaultdict(list)