Skip to content

Commit

Permalink
resolve conflict
Browse files Browse the repository at this point in the history
  • Loading branch information
@angus-langchain
angus-langchain committed Dec 11, 2024
2 parents e7dc6bc + 8c8afaf commit 3ab6b39
Show file tree
Hide file tree
Showing 21 changed files with 3,344 additions and 115 deletions.
221 changes: 218 additions & 3 deletions js/src/client.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,12 @@ import {
AnnotationQueue,
RunWithAnnotationQueueInfo,
Attachments,
ExampleUploadWithAttachments,
UploadExamplesResponse,
ExampleUpdateWithAttachments,
UpdateExamplesResponse,
RawExample,
AttachmentInfo,
} from "./schemas.js";
import {
convertLangChainMessageToExample,
Expand Down Expand Up @@ -760,6 +766,13 @@ export class Client implements LangSmithTracingClientInterface {
);
}

private async _getMultiPartSupport(): Promise<boolean> {
const serverInfo = await this._ensureServerInfo();
return (
serverInfo.instance_flags?.dataset_examples_multipart_enabled ?? false
);
}

private drainAutoBatchQueue(batchSizeLimit: number) {
while (this.autoBatchQueue.items.length > 0) {
const [batch, done] = this.autoBatchQueue.pop(batchSizeLimit);
Expand Down Expand Up @@ -2715,7 +2728,22 @@ export class Client implements LangSmithTracingClientInterface {
public async readExample(exampleId: string): Promise<Example> {
assertUuid(exampleId);
const path = `/examples/${exampleId}`;
return await this._get<Example>(path);
const rawExample: RawExample = await this._get(path);
const { attachment_urls, ...rest } = rawExample;
const example: Example = rest;
if (attachment_urls) {
// add attachments back to the example
example.attachments = Object.entries(attachment_urls).reduce(
(acc, [key, value]) => {
acc[key] = {
presigned_url: value.presigned_url,
};
return acc;
},
{} as Record<string, AttachmentInfo>
);
}
return example;
}

public async *listExamples({
Expand All @@ -2729,6 +2757,7 @@ export class Client implements LangSmithTracingClientInterface {
limit,
offset,
filter,
includeAttachments,
}: {
datasetId?: string;
datasetName?: string;
Expand All @@ -2740,6 +2769,7 @@ export class Client implements LangSmithTracingClientInterface {
limit?: number;
offset?: number;
filter?: string;
includeAttachments?: boolean;
} = {}): AsyncIterable<Example> {
let datasetId_;
if (datasetId !== undefined && datasetName !== undefined) {
Expand Down Expand Up @@ -2786,12 +2816,30 @@ export class Client implements LangSmithTracingClientInterface {
if (filter !== undefined) {
params.append("filter", filter);
}
if (includeAttachments === true) {
["attachment_urls", "outputs", "metadata"].forEach((field) =>
params.append("select", field)
);
}
let i = 0;
for await (const examples of this._getPaginated<Example>(
for await (const rawExamples of this._getPaginated<RawExample>(
"/examples",
params
)) {
for (const example of examples) {
for (const rawExample of rawExamples) {
const { attachment_urls, ...rest } = rawExample;
const example: Example = rest;
if (attachment_urls) {
example.attachments = Object.entries(attachment_urls).reduce(
(acc, [key, value]) => {
acc[key] = {
presigned_url: value.presigned_url,
};
return acc;
},
{} as Record<string, AttachmentInfo>
);
}
yield example;
i++;
}
Expand Down Expand Up @@ -3847,6 +3895,173 @@ export class Client implements LangSmithTracingClientInterface {
);
}

/**
* Update examples with attachments using multipart form data.
* @param updates List of ExampleUpdateWithAttachments objects to upsert
* @returns Promise with the update response
*/
public async updateExamplesMultipart(
datasetId: string,
updates: ExampleUpdateWithAttachments[] = []
): Promise<UpdateExamplesResponse> {
if (!(await this._getMultiPartSupport())) {
throw new Error(
"Your LangSmith version does not allow using the multipart examples endpoint, please update to the latest version."
);
}
const formData = new FormData();

for (const example of updates) {
const exampleId = example.id;

// Prepare the main example body
const exampleBody = {
...(example.metadata && { metadata: example.metadata }),
...(example.split && { split: example.split }),
};

// Add main example data
const stringifiedExample = stringifyForTracing(exampleBody);
const exampleBlob = new Blob([stringifiedExample], {
type: "application/json",
});
formData.append(exampleId, exampleBlob);

// Add inputs
if (example.inputs) {
const stringifiedInputs = stringifyForTracing(example.inputs);
const inputsBlob = new Blob([stringifiedInputs], {
type: "application/json",
});
formData.append(`${exampleId}.inputs`, inputsBlob);
}

// Add outputs if present
if (example.outputs) {
const stringifiedOutputs = stringifyForTracing(example.outputs);
const outputsBlob = new Blob([stringifiedOutputs], {
type: "application/json",
});
formData.append(`${exampleId}.outputs`, outputsBlob);
}

// Add attachments if present
if (example.attachments) {
for (const [name, [mimeType, data]] of Object.entries(
example.attachments
)) {
const attachmentBlob = new Blob([data], {
type: `${mimeType}; length=${data.byteLength}`,
});
formData.append(`${exampleId}.attachment.${name}`, attachmentBlob);
}
}

if (example.attachments_operations) {
const stringifiedAttachmentsOperations = stringifyForTracing(
example.attachments_operations
);
const attachmentsOperationsBlob = new Blob(
[stringifiedAttachmentsOperations],
{
type: "application/json",
}
);
formData.append(
`${exampleId}.attachments_operations`,
attachmentsOperationsBlob
);
}
}

const response = await this.caller.call(
_getFetchImplementation(),
`${this.apiUrl}/v1/platform/datasets/${datasetId}/examples`,
{
method: "PATCH",
headers: this.headers,
body: formData,
}
);
const result = await response.json();
return result;
}

/**
* Upload examples with attachments using multipart form data.
* @param uploads List of ExampleUploadWithAttachments objects to upload
* @returns Promise with the upload response
*/
public async uploadExamplesMultipart(
datasetId: string,
uploads: ExampleUploadWithAttachments[] = []
): Promise<UploadExamplesResponse> {
if (!(await this._getMultiPartSupport())) {
throw new Error(
"Your LangSmith version does not allow using the multipart examples endpoint, please update to the latest version."
);
}
const formData = new FormData();

for (const example of uploads) {
const exampleId = (example.id ?? uuid.v4()).toString();

// Prepare the main example body
const exampleBody = {
created_at: example.created_at,
...(example.metadata && { metadata: example.metadata }),
...(example.split && { split: example.split }),
};

// Add main example data
const stringifiedExample = stringifyForTracing(exampleBody);
const exampleBlob = new Blob([stringifiedExample], {
type: "application/json",
});
formData.append(exampleId, exampleBlob);

// Add inputs
const stringifiedInputs = stringifyForTracing(example.inputs);
const inputsBlob = new Blob([stringifiedInputs], {
type: "application/json",
});
formData.append(`${exampleId}.inputs`, inputsBlob);

// Add outputs if present
if (example.outputs) {
const stringifiedOutputs = stringifyForTracing(example.outputs);
const outputsBlob = new Blob([stringifiedOutputs], {
type: "application/json",
});
formData.append(`${exampleId}.outputs`, outputsBlob);
}

// Add attachments if present
if (example.attachments) {
for (const [name, [mimeType, data]] of Object.entries(
example.attachments
)) {
const attachmentBlob = new Blob([data], {
type: `${mimeType}; length=${data.byteLength}`,
});
formData.append(`${exampleId}.attachment.${name}`, attachmentBlob);
}
}
}

const response = await this.caller.call(
_getFetchImplementation(),
`${this.apiUrl}/v1/platform/datasets/${datasetId}/examples`,
{
method: "POST",
headers: this.headers,
body: formData,
}
);
const result = await response.json();
return result;
}

public async updatePrompt(
promptIdentifier: string,
options?: {
Expand Down
Loading

0 comments on commit 3ab6b39

Please sign in to comment.