From 60235a4078cb3a6c9a22db941afbdb8d1e9a4609 Mon Sep 17 00:00:00 2001 From: Louis Date: Fri, 8 Nov 2024 17:59:09 +0700 Subject: [PATCH 1/4] chore: proxies Jan APIs to cortex.cpp --- core/src/node/api/restful/common.ts | 10 ++- core/src/node/api/restful/helper/builder.ts | 81 ++++++++++----------- core/src/node/api/restful/helper/consts.ts | 2 +- 3 files changed, 48 insertions(+), 45 deletions(-) diff --git a/core/src/node/api/restful/common.ts b/core/src/node/api/restful/common.ts index c8061c34a2..4705bea2f9 100644 --- a/core/src/node/api/restful/common.ts +++ b/core/src/node/api/restful/common.ts @@ -10,6 +10,7 @@ import { getMessages, retrieveMessage, updateThread, + getModels, } from './helper/builder' import { JanApiRouteConfiguration } from './helper/configuration' @@ -26,9 +27,12 @@ export const commonRouter = async (app: HttpServer) => { // Common Routes // Read & Delete :: Threads | Models | Assistants Object.keys(JanApiRouteConfiguration).forEach((key) => { - app.get(`/${key}`, async (_request) => - getBuilder(JanApiRouteConfiguration[key]).then(normalizeData) - ) + app.get(`/${key}`, async (_req, _res) => { + if (key === 'models') { + return getModels(_req, _res) + } + return getBuilder(JanApiRouteConfiguration[key]).then(normalizeData) + }) app.get(`/${key}/:id`, async (request: any) => retrieveBuilder(JanApiRouteConfiguration[key], request.params.id) diff --git a/core/src/node/api/restful/helper/builder.ts b/core/src/node/api/restful/helper/builder.ts index da33808dc6..455b489d2b 100644 --- a/core/src/node/api/restful/helper/builder.ts +++ b/core/src/node/api/restful/helper/builder.ts @@ -10,9 +10,9 @@ import { } from 'fs' import { JanApiRouteConfiguration, RouteConfiguration } from './configuration' import { join } from 'path' -import { ContentType, MessageStatus, Model, ThreadMessage } from '../../../../types' -import { getEngineConfiguration, getJanDataFolderPath } from '../../../helper' -import { DEFAULT_CHAT_COMPLETION_URL } from './consts' +import { ContentType, InferenceEngine, MessageStatus, ThreadMessage } from '../../../../types' +import { getJanDataFolderPath } from '../../../helper' +import { CORTEX_API_URL } from './consts' // TODO: Refactor these export const getBuilder = async (configuration: RouteConfiguration) => { @@ -297,57 +297,56 @@ export const downloadModel = async ( } } -export const chatCompletions = async (request: any, reply: any) => { - const modelList = await getBuilder(JanApiRouteConfiguration.models) - const modelId = request.body.model - - const matchedModels = modelList.filter((model: Model) => model.id === modelId) - if (matchedModels.length === 0) { - const error = { - error: { - message: `The model ${request.body.model} does not exist`, - type: 'invalid_request_error', - param: null, - code: 'model_not_found', - }, - } - reply.code(404).send(error) - return +/** + * Proxy /models to cortex + * @param request + * @param reply + */ +export const getModels = async (request: any, reply: any) => { + const fetch = require('node-fetch') + const headers: Record = { + 'Content-Type': 'application/json', } - const requestedModel = matchedModels[0] - - const engineConfiguration = await getEngineConfiguration(requestedModel.engine) - - let apiKey: string | undefined = undefined - let apiUrl: string = DEFAULT_CHAT_COMPLETION_URL + const response = await fetch(`${CORTEX_API_URL}/models`, { + method: 'GET', + headers: headers, + body: JSON.stringify(request.body), + }) - if (engineConfiguration) { - apiKey = engineConfiguration.api_key - apiUrl = engineConfiguration.full_url ?? DEFAULT_CHAT_COMPLETION_URL + if (response.status !== 200) { + // Forward the error response to client via reply + const responseBody = await response.text() + const responseHeaders = Object.fromEntries(response.headers) + reply.code(response.status).headers(responseHeaders).send(responseBody) + } else { + reply.raw.writeHead(200, { + 'Content-Type': 'application/json', + 'Cache-Control': 'no-cache', + 'Connection': 'keep-alive', + 'Access-Control-Allow-Origin': '*', + }) + response.body.pipe(reply.raw) } +} +/** + * Proxy chat completions + * @param request + * @param reply + */ +export const chatCompletions = async (request: any, reply: any) => { const headers: Record = { 'Content-Type': 'application/json', } - if (apiKey) { - headers['Authorization'] = `Bearer ${apiKey}` - headers['api-key'] = apiKey - } - - if (requestedModel.engine === 'openai' && request.body.stop) { - // openai only allows max 4 stop words - request.body.stop = request.body.stop.slice(0, 4) - } - // add engine for new cortex cpp engine - if (requestedModel.engine === 'nitro') { - request.body.engine = 'llama-cpp' + if (request.body.engine === InferenceEngine.nitro) { + request.body.engine = InferenceEngine.cortex_llamacpp } const fetch = require('node-fetch') - const response = await fetch(apiUrl, { + const response = await fetch(`${CORTEX_API_URL}/chat/completions`, { method: 'POST', headers: headers, body: JSON.stringify(request.body), diff --git a/core/src/node/api/restful/helper/consts.ts b/core/src/node/api/restful/helper/consts.ts index 0f57bb5ff1..e81b8f8d80 100644 --- a/core/src/node/api/restful/helper/consts.ts +++ b/core/src/node/api/restful/helper/consts.ts @@ -6,4 +6,4 @@ export const LOCAL_HOST = '127.0.0.1' export const SUPPORTED_MODEL_FORMAT = '.gguf' -export const DEFAULT_CHAT_COMPLETION_URL = `http://${LOCAL_HOST}:${CORTEX_DEFAULT_PORT}/v1/chat/completions` // default nitro url +export const CORTEX_API_URL = `http://${LOCAL_HOST}:${CORTEX_DEFAULT_PORT}/v1` // default nitro url From 81015bab3c042d5bef10381e5f40cd67e8d69f3d Mon Sep 17 00:00:00 2001 From: Louis Date: Fri, 8 Nov 2024 18:03:37 +0700 Subject: [PATCH 2/4] chore: clean nitro stuffs --- core/src/node/api/restful/helper/consts.ts | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/core/src/node/api/restful/helper/consts.ts b/core/src/node/api/restful/helper/consts.ts index e81b8f8d80..412d304eef 100644 --- a/core/src/node/api/restful/helper/consts.ts +++ b/core/src/node/api/restful/helper/consts.ts @@ -1,9 +1,7 @@ -// The PORT to use for the Nitro subprocess export const CORTEX_DEFAULT_PORT = 39291 -// The HOST address to use for the Nitro subprocess export const LOCAL_HOST = '127.0.0.1' export const SUPPORTED_MODEL_FORMAT = '.gguf' -export const CORTEX_API_URL = `http://${LOCAL_HOST}:${CORTEX_DEFAULT_PORT}/v1` // default nitro url +export const CORTEX_API_URL = `http://${LOCAL_HOST}:${CORTEX_DEFAULT_PORT}/v1` From 4a414bf5e9db9412e98035872af106ba72b7f82c Mon Sep 17 00:00:00 2001 From: Louis Date: Fri, 8 Nov 2024 18:16:09 +0700 Subject: [PATCH 3/4] chore: proxy all /models methods to cortex.cpp --- core/src/node/api/restful/common.ts | 4 ++-- core/src/node/api/restful/helper/builder.ts | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/core/src/node/api/restful/common.ts b/core/src/node/api/restful/common.ts index 4705bea2f9..39f7b8d8b5 100644 --- a/core/src/node/api/restful/common.ts +++ b/core/src/node/api/restful/common.ts @@ -10,7 +10,7 @@ import { getMessages, retrieveMessage, updateThread, - getModels, + models, } from './helper/builder' import { JanApiRouteConfiguration } from './helper/configuration' @@ -29,7 +29,7 @@ export const commonRouter = async (app: HttpServer) => { Object.keys(JanApiRouteConfiguration).forEach((key) => { app.get(`/${key}`, async (_req, _res) => { if (key === 'models') { - return getModels(_req, _res) + return models(_req, _res) } return getBuilder(JanApiRouteConfiguration[key]).then(normalizeData) }) diff --git a/core/src/node/api/restful/helper/builder.ts b/core/src/node/api/restful/helper/builder.ts index 455b489d2b..c3493a8be7 100644 --- a/core/src/node/api/restful/helper/builder.ts +++ b/core/src/node/api/restful/helper/builder.ts @@ -302,14 +302,14 @@ export const downloadModel = async ( * @param request * @param reply */ -export const getModels = async (request: any, reply: any) => { +export const models = async (request: any, reply: any) => { const fetch = require('node-fetch') const headers: Record = { 'Content-Type': 'application/json', } const response = await fetch(`${CORTEX_API_URL}/models`, { - method: 'GET', + method: request.method, headers: headers, body: JSON.stringify(request.body), }) From 487fd279121a1eb1a96d35e201f6a1bea8c3e9c6 Mon Sep 17 00:00:00 2001 From: Louis Date: Sat, 9 Nov 2024 12:21:51 +0700 Subject: [PATCH 4/4] test: remove outdated test --- core/src/node/api/restful/helper/builder.test.ts | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/core/src/node/api/restful/helper/builder.test.ts b/core/src/node/api/restful/helper/builder.test.ts index eb21e9401b..f212570980 100644 --- a/core/src/node/api/restful/helper/builder.test.ts +++ b/core/src/node/api/restful/helper/builder.test.ts @@ -220,22 +220,6 @@ describe('builder helper functions', () => { }) describe('chatCompletions', () => { - it('should return an error if model is not found', async () => { - const request = { body: { model: 'nonexistentModel' } } - const reply = { code: jest.fn().mockReturnThis(), send: jest.fn() } - - await chatCompletions(request, reply) - expect(reply.code).toHaveBeenCalledWith(404) - expect(reply.send).toHaveBeenCalledWith({ - error: { - message: 'The model nonexistentModel does not exist', - type: 'invalid_request_error', - param: null, - code: 'model_not_found', - }, - }) - }) - it('should return the error on status not ok', async () => { const request = { body: { model: 'model1' } } const mockSend = jest.fn()