From 49ee3833e5f2fc423ca8c059670157b20acc8ac6 Mon Sep 17 00:00:00 2001 From: dosco <832235+dosco@users.noreply.github.com> Date: Sat, 15 Jun 2024 11:10:28 -0700 Subject: [PATCH] feat: support for google vertex fix: bun support --- README.md | 5 ++-- package.json | 2 +- src/ai/base.ts | 3 ++- src/ai/google-gemini/api.ts | 12 ++++++++- src/ai/google-gemini/types.ts | 2 +- src/ai/groq/api.ts | 2 +- src/ai/openai/api.ts | 6 +++-- src/util/apicall.ts | 9 ++++++- src/util/stream.ts | 50 +++++++++++++++++++++++++++++++++++ 9 files changed, 80 insertions(+), 11 deletions(-) create mode 100644 src/util/stream.ts diff --git a/README.md b/README.md index 8c6c16e3..1e5e3573 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ -# LLMClient - Typescript/JS Library to build with LLMs +# LLMClient - Build LLMs Powered Agents (Typescript) -JS/TS library to make to easy to build with LLMs. Full support for various LLMs and VectorDBs, Agents, Function Calling, Chain-of-Thought, RAG, Semantic Router and more. Based on the popular Stanford DSP paper. Create and compose efficient prompts using prompt signatures. 🌵 🦙 🔥 ❤️ 🖖🏼 +JS/TS library to make to easy to build with Agents and agentic workflows with LLMs. Full support for various LLMs and VectorDBs, Function Calling, Chain-of-Thought, RAG, Semantic Router and more. Based on the popular Stanford DSP paper. Build agents or teams or agents to solve complex problems 🌵 🦙 🔥 ❤️ 🖖🏼 [![NPM Package](https://img.shields.io/npm/v/llmclient?style=for-the-badge&color=green)](https://www.npmjs.com/package/llmclient) [![Twitter](https://img.shields.io/twitter/follow/dosco?style=for-the-badge&color=red)](https://twitter.com/dosco) @@ -422,7 +422,6 @@ OPENAI_APIKEY=openai_key npm run tsx ./src/examples/marketing.ts | streaming1.ts | Output fields validation while streaming | | streaming2.ts | Per output field validation while streaming | - ## Built-in Functions | Function | Description | diff --git a/package.json b/package.json index 784e71ed..aff50495 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "llmclient", - "version": "8.1.21", + "version": "8.1.22", "type": "module", "description": "The best library to work with LLMs", "typings": "build/module/src/index.d.ts", diff --git a/src/ai/base.ts b/src/ai/base.ts index aa24e3ec..8b1db560 100644 --- a/src/ai/base.ts +++ b/src/ai/base.ts @@ -106,7 +106,8 @@ export class BaseAI< throw new Error('No model defined'); } - this.modelInfo = modelInfo.filter((v) => v.name === models.model).at(0) ?? { + const mname = models.model.replace(/-0\d+$|-\d{2,}$/, ''); + this.modelInfo = modelInfo.filter((v) => v.name === mname).at(0) ?? { name: models.model, currency: 'usd', promptTokenCostPer1M: 0, diff --git a/src/ai/google-gemini/api.ts b/src/ai/google-gemini/api.ts index 1eb67b66..adeb1cec 100644 --- a/src/ai/google-gemini/api.ts +++ b/src/ai/google-gemini/api.ts @@ -73,6 +73,8 @@ export const GoogleGeminiDefaultCreativeConfig = (): GoogleGeminiConfig => export interface GoogleGeminiArgs { apiKey: string; + projectId?: string; + region?: string; config: Readonly; options?: Readonly; } @@ -93,6 +95,8 @@ export class GoogleGemini extends BaseAI< constructor({ apiKey, + projectId, + region, config = GoogleGeminiDefaultConfig(), options }: Readonly) { @@ -100,9 +104,15 @@ export class GoogleGemini extends BaseAI< throw new Error('GoogleGemini AI API key not set'); } + let apiURL = 'https://generativelanguage.googleapis.com/v1beta'; + + if (projectId && region) { + apiURL = `POST https://${region}-aiplatform.googleapis.com/v1/projects/${projectId}/locations/{REGION}/publishers/google/`; + } + super({ name: 'GoogleGeminiAI', - apiURL: 'https://generativelanguage.googleapis.com/v1beta', + apiURL, headers: {}, modelInfo: modelInfoGoogleGemini, models: { model: config.model, embedModel: config.embedModel }, diff --git a/src/ai/google-gemini/types.ts b/src/ai/google-gemini/types.ts index 1ad78e06..21292ada 100644 --- a/src/ai/google-gemini/types.ts +++ b/src/ai/google-gemini/types.ts @@ -149,7 +149,7 @@ export type GoogleGeminiChatResponseDelta = GoogleGeminiChatResponse; * @export */ export type GoogleGeminiConfig = TextModelConfig & { - model: GoogleGeminiModel; + model: GoogleGeminiModel | string; embedModel: GoogleGeminiEmbedModels; safetySettings?: GoogleGeminiSafetySettings; }; diff --git a/src/ai/groq/api.ts b/src/ai/groq/api.ts index 9631cee8..0e02a71f 100644 --- a/src/ai/groq/api.ts +++ b/src/ai/groq/api.ts @@ -37,7 +37,7 @@ export class Groq extends OpenAI { super({ apiKey, config, - options, + options: { ...options, streamingUsage: false }, apiURL: 'https://api.groq.com/openai/v1' }); diff --git a/src/ai/openai/api.ts b/src/ai/openai/api.ts index 8f5385ee..7bf95af4 100644 --- a/src/ai/openai/api.ts +++ b/src/ai/openai/api.ts @@ -76,7 +76,7 @@ export interface OpenAIArgs { apiKey: string; apiURL?: string; config?: Readonly; - options?: Readonly; + options?: Readonly; } /** @@ -91,6 +91,7 @@ export class OpenAI extends BaseAI< OpenAIEmbedResponse > { private config: OpenAIConfig; + private streamingUsage: boolean; constructor({ apiKey, @@ -111,6 +112,7 @@ export class OpenAI extends BaseAI< supportFor: { functions: true, streaming: true } }); this.config = config; + this.streamingUsage = options?.streamingUsage ?? true; } override getModelConfig(): TextModelConfig { @@ -219,7 +221,7 @@ export class OpenAI extends BaseAI< user: req.identity?.user ?? this.config.user, organization: req.identity?.organization, ...(frequencyPenalty ? { frequency_penalty: frequencyPenalty } : {}), - ...(stream + ...(stream && this.streamingUsage ? { stream: true, stream_options: { include_usage: true } } : {}) }; diff --git a/src/util/apicall.ts b/src/util/apicall.ts index f0ff24d9..a5ed66ab 100644 --- a/src/util/apicall.ts +++ b/src/util/apicall.ts @@ -1,9 +1,14 @@ import path from 'path'; -import { type ReadableStream, TextDecoderStream } from 'stream/web'; +import { + type ReadableStream, + TextDecoderStream as TextDecoderStreamNative +} from 'stream/web'; import type { Span } from '../trace/index.js'; +import { TextDecoderStreamPolyfill } from './stream.js'; import { JSONStringifyStream } from './transform.js'; + /** * Util: API details * @export @@ -14,6 +19,8 @@ export type API = { put?: boolean; }; +const TextDecoderStream = TextDecoderStreamNative ?? TextDecoderStreamPolyfill; + export const apiCall = async ( api: Readonly< API & { diff --git a/src/util/stream.ts b/src/util/stream.ts new file mode 100644 index 00000000..957b9f99 --- /dev/null +++ b/src/util/stream.ts @@ -0,0 +1,50 @@ +import { + type Transformer, + TransformStream, + type TransformStreamDefaultController +} from 'stream/web'; + +export interface TextDecoderCommon { + readonly encoding: string; + readonly fatal: boolean; + readonly ignoreBOM: boolean; +} + +class TextDecodeTransformer + implements Transformer +{ + private decoder; + + constructor() { + this.decoder = new TextDecoder(); + } + + transform( + chunk: ArrayBuffer | Uint8Array, + controller: TransformStreamDefaultController + ) { + if (!(chunk instanceof ArrayBuffer || ArrayBuffer.isView(chunk))) { + throw new TypeError('Input data must be a BufferSource'); + } + const text = this.decoder.decode(chunk, { stream: true }); + if (text.length !== 0) { + controller.enqueue(text); + } + } + + flush(controller: TransformStreamDefaultController) { + const text = this.decoder.decode(); + if (text.length !== 0) { + controller.enqueue(text); + } + } +} + +export class TextDecoderStreamPolyfill extends TransformStream< + ArrayBuffer | Uint8Array, + string +> { + constructor() { + super(new TextDecodeTransformer()); + } +}