diff --git a/docs/static/openapi/cortex.json b/docs/static/openapi/cortex.json index fdb5c4ed2..9747a1830 100644 --- a/docs/static/openapi/cortex.json +++ b/docs/static/openapi/cortex.json @@ -22,9 +22,7 @@ "description": "The assistant has been successfully created." } }, - "tags": [ - "Assistants" - ] + "tags": ["Assistants"] }, "get": { "operationId": "AssistantsController_findAll", @@ -83,9 +81,7 @@ } } }, - "tags": [ - "Assistants" - ] + "tags": ["Assistants"] } }, "/v1/assistants/{id}": { @@ -116,9 +112,7 @@ } } }, - "tags": [ - "Assistants" - ] + "tags": ["Assistants"] }, "delete": { "operationId": "AssistantsController_remove", @@ -147,9 +141,7 @@ } } }, - "tags": [ - "Assistants" - ] + "tags": ["Assistants"] } }, "/healthz": { @@ -166,9 +158,7 @@ } } }, - "tags": [ - "Server" - ] + "tags": ["Server"] } }, "/processManager/destroy": { @@ -185,9 +175,7 @@ } } }, - "tags": [ - "Server" - ] + "tags": ["Server"] } }, "/v1/embeddings": { @@ -242,17 +230,11 @@ "encoding_format": { "type": "string", "description": "The format to return the embeddings in.", - "enum": [ - "float", - "base64" - ], + "enum": ["float", "base64"], "default": "float" } }, - "required": [ - "input", - "model" - ] + "required": ["input", "model"] } } } @@ -295,9 +277,7 @@ } } }, - "tags": [ - "Embeddings" - ] + "tags": ["Embeddings"] } }, "/v1/chat/completions": { @@ -337,9 +317,7 @@ } } }, - "tags": [ - "Chat" - ] + "tags": ["Chat"] } }, "/v1/models/pull": { @@ -438,14 +416,10 @@ } } }, - "tags": [ - "Pulling Models" - ] + "tags": ["Pulling Models"] }, "delete": { - "tags": [ - "Pulling Models" - ], + "tags": ["Pulling Models"], "summary": "Stop model download", "description": "Stops the download of a model with the corresponding taskId provided in the request body", "operationId": "ModelsController_stopModelDownload", @@ -461,9 +435,7 @@ "description": "The unique identifier of the download task to be stopped" } }, - "required": [ - "taskId" - ] + "required": ["taskId"] } } } @@ -558,9 +530,7 @@ } } }, - "tags": [ - "Running Models" - ] + "tags": ["Running Models"] } }, "/v1/models/start": { @@ -593,9 +563,7 @@ } } }, - "tags": [ - "Running Models" - ] + "tags": ["Running Models"] } }, "/v1/models/stop": { @@ -628,9 +596,7 @@ } } }, - "tags": [ - "Running Models" - ] + "tags": ["Running Models"] } }, "/v1/models/{id}": { @@ -661,9 +627,7 @@ } } }, - "tags": [ - "Running Models" - ] + "tags": ["Running Models"] }, "delete": { "operationId": "ModelsController_remove", @@ -692,9 +656,7 @@ } } }, - "tags": [ - "Running Models" - ] + "tags": ["Running Models"] } }, "/v1/models/{model}": { @@ -734,9 +696,7 @@ } } }, - "tags": [ - "Running Models" - ] + "tags": ["Running Models"] } }, "/v1/models/import": { @@ -777,18 +737,14 @@ } } }, - "tags": [ - "Pulling Models" - ] + "tags": ["Pulling Models"] } }, "/v1/threads": { "post": { "operationId": "ThreadsController_create", "summary": "Create thread", - "tags": [ - "Threads" - ], + "tags": ["Threads"], "description": "Creates a new thread.", "parameters": [], "requestBody": { @@ -817,9 +773,7 @@ "get": { "operationId": "ThreadsController_findAll", "summary": "List threads", - "tags": [ - "Threads" - ], + "tags": ["Threads"], "description": "Lists all the available threads along with its configurations.", "parameters": [], "responses": { @@ -843,9 +797,7 @@ "get": { "operationId": "ThreadsController_retrieveMessage", "summary": "Retrieve message", - "tags": [ - "Messages" - ], + "tags": ["Messages"], "description": "Retrieves a message.", "parameters": [ { @@ -881,9 +833,7 @@ "post": { "operationId": "ThreadsController_updateMessage", "summary": "Modify message", - "tags": [ - "Messages" - ], + "tags": ["Messages"], "description": "Modifies a message.", "responses": { "201": { @@ -930,9 +880,7 @@ "operationId": "ThreadsController_deleteMessage", "summary": "Delete message", "description": "Deletes a message.", - "tags": [ - "Messages" - ], + "tags": ["Messages"], "parameters": [ { "name": "thread_id", @@ -969,9 +917,7 @@ "get": { "operationId": "ThreadsController_getMessagesOfThread", "summary": "List messages", - "tags": [ - "Messages" - ], + "tags": ["Messages"], "description": "Returns a list of messages for a given thread.", "parameters": [ { @@ -1039,9 +985,7 @@ "post": { "operationId": "ThreadsController_createMessageInThread", "summary": "Create message", - "tags": [ - "Messages" - ], + "tags": ["Messages"], "description": "Create a message.", "responses": { "201": { @@ -1082,9 +1026,7 @@ "operationId": "ThreadsController_cleanThread", "summary": "Clean thread", "description": "Deletes all messages in a thread.", - "tags": [ - "Threads" - ], + "tags": ["Threads"], "parameters": [ { "name": "thread_id", @@ -1106,9 +1048,7 @@ "get": { "operationId": "ThreadsController_retrieveThread", "summary": "Retrieve thread", - "tags": [ - "Threads" - ], + "tags": ["Threads"], "description": "Retrieves a thread.", "parameters": [ { @@ -1136,9 +1076,7 @@ "post": { "operationId": "ThreadsController_modifyThread", "summary": "Modify thread", - "tags": [ - "Threads" - ], + "tags": ["Threads"], "description": "Modifies a thread.", "parameters": [ { @@ -1179,9 +1117,7 @@ "delete": { "operationId": "ThreadsController_remove", "summary": "Delete thread", - "tags": [ - "Threads" - ], + "tags": ["Threads"], "description": "Deletes a specific thread defined by a thread `id` .", "parameters": [ { @@ -1218,9 +1154,7 @@ "description": "" } }, - "tags": [ - "System" - ] + "tags": ["System"] }, "get": { "operationId": "SystemController_get", @@ -1232,9 +1166,7 @@ "description": "Ok" } }, - "tags": [ - "System" - ] + "tags": ["System"] } }, "/v1/system/events/download": { @@ -1255,9 +1187,7 @@ } } }, - "tags": [ - "System" - ] + "tags": ["System"] } }, "/v1/system/events/model": { @@ -1278,9 +1208,7 @@ } } }, - "tags": [ - "System" - ] + "tags": ["System"] } }, "/v1/system/events/resources": { @@ -1301,9 +1229,7 @@ } } }, - "tags": [ - "System" - ] + "tags": ["System"] } }, "/v1/engines/{name}": { @@ -1318,11 +1244,7 @@ "required": true, "schema": { "type": "string", - "enum": [ - "llama-cpp", - "onnxruntime", - "tensorrt-llm" - ], + "enum": ["llama-cpp", "onnxruntime", "tensorrt-llm"], "default": "llama-cpp" }, "description": "The type of engine" @@ -1369,9 +1291,7 @@ } } }, - "tags": [ - "Engines" - ] + "tags": ["Engines"] }, "post": { "summary": "Install an engine", @@ -1383,11 +1303,7 @@ "required": true, "schema": { "type": "string", - "enum": [ - "llama-cpp", - "onnxruntime", - "tensorrt-llm" - ], + "enum": ["llama-cpp", "onnxruntime", "tensorrt-llm"], "default": "llama-cpp" }, "description": "The type of engine" @@ -1421,9 +1337,7 @@ } } }, - "tags": [ - "Engines" - ] + "tags": ["Engines"] }, "delete": { "summary": "Uninstall an engine", @@ -1435,11 +1349,7 @@ "required": true, "schema": { "type": "string", - "enum": [ - "llama-cpp", - "onnxruntime", - "tensorrt-llm" - ], + "enum": ["llama-cpp", "onnxruntime", "tensorrt-llm"], "default": "llama-cpp" }, "description": "The type of engine" @@ -1516,9 +1426,7 @@ } } }, - "tags": [ - "Engines" - ] + "tags": ["Engines"] } }, "/v1/engines/{name}/default": { @@ -1532,11 +1440,7 @@ "required": true, "schema": { "type": "string", - "enum": [ - "llama-cpp", - "onnxruntime", - "tensorrt-llm" - ], + "enum": ["llama-cpp", "onnxruntime", "tensorrt-llm"], "default": "llama-cpp" }, "description": "The type of engine" @@ -1568,9 +1472,7 @@ } } }, - "tags": [ - "Engines" - ] + "tags": ["Engines"] }, "post": { "summary": "Set default engine variant", @@ -1582,11 +1484,7 @@ "required": true, "schema": { "type": "string", - "enum": [ - "llama-cpp", - "onnxruntime", - "tensorrt-llm" - ], + "enum": ["llama-cpp", "onnxruntime", "tensorrt-llm"], "default": "llama-cpp" }, "description": "The type of engine" @@ -1628,9 +1526,7 @@ } } }, - "tags": [ - "Engines" - ] + "tags": ["Engines"] } }, "/v1/engines/{name}/load": { @@ -1644,11 +1540,7 @@ "required": true, "schema": { "type": "string", - "enum": [ - "llama-cpp", - "onnxruntime", - "tensorrt-llm" - ], + "enum": ["llama-cpp", "onnxruntime", "tensorrt-llm"], "default": "llama-cpp" }, "description": "The name of the engine to update" @@ -1672,9 +1564,7 @@ } } }, - "tags": [ - "Engines" - ] + "tags": ["Engines"] }, "delete": { "summary": "Unload engine", @@ -1686,11 +1576,7 @@ "required": true, "schema": { "type": "string", - "enum": [ - "llama-cpp", - "onnxruntime", - "tensorrt-llm" - ], + "enum": ["llama-cpp", "onnxruntime", "tensorrt-llm"], "default": "llama-cpp" }, "description": "The name of the engine to update" @@ -1714,9 +1600,7 @@ } } }, - "tags": [ - "Engines" - ] + "tags": ["Engines"] } }, "/v1/engines/{name}/update": { @@ -1730,11 +1614,7 @@ "required": true, "schema": { "type": "string", - "enum": [ - "llama-cpp", - "onnxruntime", - "tensorrt-llm" - ], + "enum": ["llama-cpp", "onnxruntime", "tensorrt-llm"], "default": "llama-cpp" }, "description": "The name of the engine to update" @@ -1758,9 +1638,7 @@ } } }, - "tags": [ - "Engines" - ] + "tags": ["Engines"] } }, "/v1/hardware": { @@ -1802,9 +1680,7 @@ } } }, - "tags": [ - "Hardware" - ] + "tags": ["Hardware"] } }, "/v1/hardware/activate": { @@ -1823,17 +1699,11 @@ "items": { "type": "integer" }, - "example": [ - 0, - 1, - 2 - ], + "example": [0, 1, 2], "description": "An array of GPU indices to activate." } }, - "required": [ - "gpus" - ] + "required": ["gpus"] } } } @@ -1856,11 +1726,7 @@ "items": { "type": "integer" }, - "example": [ - 0, - 1, - 2 - ], + "example": [0, 1, 2], "description": "List of GPU indices that were activated." } } @@ -1886,9 +1752,7 @@ } } }, - "tags": [ - "Hardware" - ] + "tags": ["Hardware"] } }, "/v1/configs": { @@ -1908,14 +1772,44 @@ "items": { "type": "string" }, - "example": [ - "http://localhost:39281", - "https://cortex.so" - ] + "example": ["http://localhost:39281", "https://cortex.so"] }, "cors": { "type": "boolean", "example": false + }, + "proxy_username": { + "type": "string", + "example": "username" + }, + "proxy_password": { + "type": "string", + "example": "password" + }, + "proxy_url": { + "type": "string", + "example": "http://proxy.example.com:8080" + }, + "verify_proxy_ssl": { + "type": "boolean", + "description": "test", + "example": false + }, + "verify_proxy_host_ssl": { + "type": "boolean", + "example": false + }, + "verify_peer_ssl": { + "type": "boolean", + "example": false + }, + "verify_host_ssl": { + "type": "boolean", + "example": false + }, + "no_proxy": { + "type": "string", + "example": "localhost" } } }, @@ -1924,20 +1818,24 @@ "http://localhost:39281", "https://cortex.so" ], - "cors": false + "cors": false, + "proxy_username": "username", + "proxy_password": "password", + "proxy_url": "http://proxy.example.com:8080", + "verify_proxy_ssl": false, + "verify_proxy_host_ssl": false, + "verify_peer_ssl": false, + "verify_host_ssl": false, + "no_proxy": "localhost" } } } } }, - "tags": [ - "Configurations" - ] + "tags": ["Configurations"] }, "patch": { - "tags": [ - "Configurations" - ], + "tags": ["Configurations"], "summary": "Update configuration settings", "requestBody": { "required": true, @@ -1957,10 +1855,47 @@ "type": "string" }, "description": "List of allowed origins.", - "example": [ - "http://localhost:39281", - "https://cortex.so" - ] + "example": ["http://localhost:39281", "https://cortex.so"] + }, + "proxy_username": { + "type": "string", + "description": "Username for the proxy server.", + "example": "username" + }, + "proxy_password": { + "type": "string", + "description": "Password for the proxy server.", + "example": "password" + }, + "proxy_url": { + "type": "string", + "description": "URL for the proxy server.", + "example": "http://proxy.example.com:8080" + }, + "verify_proxy_ssl": { + "type": "boolean", + "description": "Indicates whether to verify the SSL certificate of the proxy server.", + "example": false + }, + "verify_proxy_host_ssl": { + "type": "boolean", + "description": "Indicates whether to verify the SSL certificate of the proxy server host.", + "example": false + }, + "verify_peer_ssl": { + "type": "boolean", + "description": "Indicates whether to verify the SSL certificate of the peer.", + "example": false + }, + "verify_host_ssl": { + "type": "boolean", + "description": "Indicates whether to verify the SSL certificate of the host.", + "example": false + }, + "no_proxy": { + "type": "string", + "description": "List of hosts that should not be proxied.", + "example": "localhost" } } } @@ -1991,6 +1926,38 @@ "cors": { "type": "boolean", "example": false + }, + "proxy_username": { + "type": "string", + "example": "username" + }, + "proxy_password": { + "type": "string", + "example": "password" + }, + "proxy_url": { + "type": "string", + "example": "http://proxy.example.com:8080" + }, + "verify_proxy_ssl": { + "type": "boolean", + "example": false + }, + "verify_proxy_host_ssl": { + "type": "boolean", + "example": false + }, + "verify_peer_ssl": { + "type": "boolean", + "example": false + }, + "verify_host_ssl": { + "type": "boolean", + "example": false + }, + "no_proxy": { + "type": "string", + "example": "localhost" } } }, @@ -2252,11 +2219,7 @@ "description": "Indicates whether the assistant was successfully deleted." } }, - "required": [ - "id", - "object", - "deleted" - ] + "required": ["id", "object", "deleted"] }, "Message": { "type": "object", @@ -2273,21 +2236,14 @@ "properties": { "role": { "type": "string", - "enum": [ - "system", - "user", - "assistant", - "tool" - ] + "enum": ["system", "user", "assistant", "tool"] }, "name": { "type": "string", "description": "An optional name for the participant. Provides the model information to differentiate between participants of the same role." } }, - "required": [ - "role" - ] + "required": ["role"] }, "SystemMessage": { "allOf": [ @@ -2316,10 +2272,7 @@ "description": "An optional name for the participant. Provides the model information to differentiate between participants of the same role." } }, - "required": [ - "content", - "role" - ] + "required": ["content", "role"] } ] }, @@ -2370,10 +2323,7 @@ "description": "An optional name for the participant. Provides the model information to differentiate between participants of the same role." } }, - "required": [ - "content", - "role" - ] + "required": ["content", "role"] } ] }, @@ -2485,10 +2435,7 @@ "type": "string" } }, - "required": [ - "content", - "tool_call_id" - ] + "required": ["content", "tool_call_id"] } ] }, @@ -2505,36 +2452,26 @@ "properties": { "type": { "type": "string", - "enum": [ - "text" - ] + "enum": ["text"] }, "text": { "type": "string" } }, - "required": [ - "type", - "text" - ] + "required": ["type", "text"] }, "ImageContentPart": { "type": "object", "properties": { "type": { "type": "string", - "enum": [ - "image_url" - ] + "enum": ["image_url"] }, "image_url": { "$ref": "#/components/schemas/ImageUrl" } }, - "required": [ - "type", - "image_url" - ] + "required": ["type", "image_url"] }, "AudioContentPart": { "type": "object", @@ -2547,10 +2484,7 @@ "$ref": "#/components/schemas/InputAudio" } }, - "required": [ - "type", - "input_audio" - ] + "required": ["type", "input_audio"] }, "RefusalContentPart": { "type": "object", @@ -2562,10 +2496,7 @@ "type": "string" } }, - "required": [ - "type", - "refusal" - ] + "required": ["type", "refusal"] }, "ImageUrl": { "type": "object", @@ -2580,9 +2511,7 @@ "description": "Specifies the detail level of the image. Defaults to `auto`." } }, - "required": [ - "url" - ] + "required": ["url"] }, "InputAudio": { "type": "object", @@ -2593,17 +2522,11 @@ }, "format": { "type": "string", - "enum": [ - "wav", - "mp3" - ], + "enum": ["wav", "mp3"], "description": "The format of the encoded audio data. Currently supports `wav` and `mp3`." } }, - "required": [ - "data", - "format" - ] + "required": ["data", "format"] }, "Audio": { "type": "object", @@ -2614,9 +2537,7 @@ "description": "Unique identifier for a previous audio response from the model." } }, - "required": [ - "id" - ] + "required": ["id"] }, "ToolCall": { "type": "object", @@ -2631,11 +2552,7 @@ "$ref": "#/components/schemas/FunctionCall" } }, - "required": [ - "id", - "type", - "function" - ] + "required": ["id", "type", "function"] }, "FunctionCall": { "type": "object", @@ -2647,10 +2564,7 @@ "type": "string" } }, - "required": [ - "name", - "arguments" - ] + "required": ["name", "arguments"] }, "CreateChatCompletionDto": { "type": "object", @@ -2704,9 +2618,7 @@ }, "stop": { "description": "Defines specific tokens or phrases that signal the model to stop producing further output.", - "example": [ - "End" - ], + "example": ["End"], "type": "array", "items": { "type": "string" @@ -2736,15 +2648,10 @@ "type": "array", "items": { "type": "string", - "enum": [ - "text", - "audio" - ] + "enum": ["text", "audio"] }, "description": "Specifies the modalities (types of input) supported by the model. Currently, cortex only support text modalities. We are actively working on this feature to bring cortex as fully OpenAI compatible platform. Planning and roadmap for this feature can be found [**here**](https://github.com/janhq/cortex.cpp/issues/1582).", - "example": [ - "text" - ] + "example": ["text"] }, "audio": { "description": "Parameters for audio output. Required when audio output is requested with `modalities: ['audio']`. We are actively working on this feature to bring cortex as fully OpenAI compatible platform. Planning and roadmap for this feature can be found [**here**](https://github.com/janhq/cortex.cpp/issues/1582).", @@ -2757,19 +2664,10 @@ "format": { "type": "string", "description": "Specifies the output audio format. Must be one of `wav`, `mp3`, `flac`, `opus`, or `pcm16`.", - "enum": [ - "mp3", - "wav", - "flac", - "opus", - "pcm16" - ] + "enum": ["mp3", "wav", "flac", "opus", "pcm16"] } }, - "required": [ - "voice", - "format" - ] + "required": ["voice", "format"] }, "store": { "type": "boolean", @@ -2816,16 +2714,10 @@ "type": { "type": "string", "description": "The format of the generated output. Must be one of `text`, `json_schema` or `json_object`.", - "enum": [ - "text", - "json_object", - "json_schema" - ] + "enum": ["text", "json_object", "json_schema"] } }, - "required": [ - "type" - ] + "required": ["type"] }, "seed": { "type": "number", @@ -2855,37 +2747,26 @@ "properties": { "type": { "type": "string", - "enum": [ - "function" - ] + "enum": ["function"] }, "function": { "$ref": "#/components/schemas/Function" } }, - "required": [ - "type", - "function" - ] + "required": ["type", "function"] }, "tool_choice": { "anyOf": [ { "type": "string", - "enum": [ - "none", - "auto", - "required" - ] + "enum": ["none", "auto", "required"] }, { "type": "object", "properties": { "type": { "type": "string", - "enum": [ - "function" - ] + "enum": ["function"] }, "function": { "type": "object", @@ -2894,15 +2775,10 @@ "type": "string" } }, - "required": [ - "name" - ] + "required": ["name"] } }, - "required": [ - "type", - "function" - ] + "required": ["type", "function"] } ] }, @@ -2977,10 +2853,7 @@ "description": "Minimum number of tokens to keep. This parameter only supported by `llama-cpp` engine." } }, - "required": [ - "messages", - "model" - ] + "required": ["messages", "model"] }, "Function": { "type": "object", @@ -3000,9 +2873,7 @@ "default": false } }, - "required": [ - "name" - ] + "required": ["name"] }, "MessageDto": { "type": "object", @@ -3016,10 +2887,7 @@ "description": "The role of the participant in the chat, such as 'user' or 'system', indicating who is the sender of the message." } }, - "required": [ - "content", - "role" - ] + "required": ["content", "role"] }, "ChoiceDto": { "type": "object", @@ -3041,11 +2909,7 @@ ] } }, - "required": [ - "finish_reason", - "index", - "message" - ] + "required": ["finish_reason", "index", "message"] }, "UsageDto": { "type": "object", @@ -3063,11 +2927,7 @@ "description": "The total number of tokens used in both the prompt and the completion, summarizing the entire token count of the chat operation." } }, - "required": [ - "completion_tokens", - "prompt_tokens", - "total_tokens" - ] + "required": ["completion_tokens", "prompt_tokens", "total_tokens"] }, "ChatCompletionResponseDto": { "type": "object", @@ -3094,17 +2954,11 @@ "type": "object", "properties": { "content": { - "type": [ - "string", - "null" - ], + "type": ["string", "null"], "description": "The contents of the message." }, "refusal": { - "type": [ - "string", - "null" - ], + "type": ["string", "null"], "description": "The refusal message generated by the model." }, "tool_calls": { @@ -3133,17 +2987,10 @@ "description": "The arguments to call the function with, as generated by the model in JSON format. Note that the model does not always generate valid JSON, and may hallucinate parameters not defined by your function schema. Validate the arguments in your code before calling your function." } }, - "required": [ - "name", - "arguments" - ] + "required": ["name", "arguments"] } }, - "required": [ - "id", - "type", - "function" - ] + "required": ["id", "type", "function"] } }, "role": { @@ -3164,10 +3011,7 @@ "description": "The name of the function to call." } }, - "required": [ - "arguments", - "name" - ] + "required": ["arguments", "name"] }, "audio": { "type": "object", @@ -3190,27 +3034,17 @@ "description": "Transcript of the audio generated by the model." } }, - "required": [ - "id", - "expires_at", - "data", - "transcript" - ] + "required": ["id", "expires_at", "data", "transcript"] } }, - "required": [ - "role" - ] + "required": ["role"] }, "logprobs": { "type": "object", "description": "Log probability information for the choice.", "properties": { "content": { - "type": [ - "array", - "null" - ], + "type": ["array", "null"], "description": "A list of message content tokens with log probability information.", "items": { "type": "object", @@ -3224,17 +3058,11 @@ "description": "The log probability of this token, if it is within the top 20 most likely tokens. Otherwise, the value -9999.0 is used to signify that the token is very unlikely." }, "bytes": { - "type": [ - "array", - "null" - ], + "type": ["array", "null"], "description": "A list of integers representing the UTF-8 bytes representation of the token. Useful in instances where characters are represented by multiple tokens and their byte representations must be combined to generate the correct text representation. Can be null if there is no bytes representation for the token." } }, - "required": [ - "token", - "logprob" - ] + "required": ["token", "logprob"] } }, "top_logprobs": { @@ -3252,24 +3080,15 @@ "description": "The log probability of this token, if it is within the top 20 most likely tokens. Otherwise, the value -9999.0 is used to signify that the token is very unlikely." }, "bytes": { - "type": [ - "array", - "null" - ], + "type": ["array", "null"], "description": "A list of integers representing the UTF-8 bytes representation of the token. Useful in instances where characters are represented by multiple tokens and their byte representations must be combined to generate the correct text representation. Can be null if there is no bytes representation for the token." } }, - "required": [ - "token", - "logprob" - ] + "required": ["token", "logprob"] } }, "refusal": { - "type": [ - "array", - "null" - ], + "type": ["array", "null"], "description": "A list of message refusal tokens with log probability information.", "items": { "type": "object", @@ -3283,27 +3102,17 @@ "description": "The log probability of this token, if it is within the top 20 most likely tokens. Otherwise, the value -9999.0 is used to signify that the token is very unlikely." }, "bytes": { - "type": [ - "array", - "null" - ], + "type": ["array", "null"], "description": "A list of integers representing the UTF-8 bytes representation of the token. Useful in instances where characters are represented by multiple tokens and their byte representations must be combined to generate the correct text representation. Can be null if there is no bytes representation for the token." } }, - "required": [ - "token", - "logprob" - ] + "required": ["token", "logprob"] } } } } }, - "required": [ - "finish_reason", - "index", - "message" - ] + "required": ["finish_reason", "index", "message"] } }, "created": { @@ -3315,10 +3124,7 @@ "description": "The model used for the chat completion." }, "service_tier": { - "type": [ - "string", - "null" - ], + "type": ["string", "null"], "description": "The service tier used for processing the request. This field is only included if the service_tier parameter is specified in the request." }, "system_fingerprint": { @@ -3358,10 +3164,7 @@ "description": "Tokens generated by the model for reasoning." } }, - "required": [ - "audio_tokens", - "reasoning_tokens" - ] + "required": ["audio_tokens", "reasoning_tokens"] }, "prompt_tokens_details": { "type": "object", @@ -3376,10 +3179,7 @@ "description": "Cached tokens present in the prompt." } }, - "required": [ - "audio_tokens", - "cached_tokens" - ] + "required": ["audio_tokens", "cached_tokens"] } }, "required": [ @@ -3419,10 +3219,7 @@ "description": "A chat completion delta generated by streamed model responses.", "properties": { "content": { - "type": [ - "string", - "null" - ], + "type": ["string", "null"], "description": "The contents of the chunk message." }, "function_call": { @@ -3460,18 +3257,10 @@ "description": "The arguments to call the function with, as generated by the model in JSON format. Note that the model does not always generate valid JSON, and may hallucinate parameters not defined by your function schema. Validate the arguments in your code before calling your function." } }, - "required": [ - "name", - "arguments" - ] + "required": ["name", "arguments"] } }, - "required": [ - "index", - "id", - "type", - "function" - ] + "required": ["index", "id", "type", "function"] } }, "role": { @@ -3479,10 +3268,7 @@ "description": "The role of the author of this message." }, "refusal": { - "type": [ - "string", - "null" - ], + "type": ["string", "null"], "description": "The refusal message generated by the model." } } @@ -3492,10 +3278,7 @@ "description": "Log probability information for the choice.", "properties": { "content": { - "type": [ - "array", - "null" - ], + "type": ["array", "null"], "description": "A list of message content tokens with log probability information.", "items": { "type": "object", @@ -3509,17 +3292,11 @@ "description": "The log probability of this token, if it is within the top 20 most likely tokens. Otherwise, the value -9999.0 is used to signify that the token is very unlikely." }, "bytes": { - "type": [ - "array", - "null" - ], + "type": ["array", "null"], "description": "A list of integers representing the UTF-8 bytes representation of the token. Useful in instances where characters are represented by multiple tokens and their byte representations must be combined to generate the correct text representation. Can be null if there is no bytes representation for the token." } }, - "required": [ - "token", - "logprob" - ] + "required": ["token", "logprob"] } }, "top_logprobs": { @@ -3537,24 +3314,15 @@ "description": "The log probability of this token, if it is within the top 20 most likely tokens. Otherwise, the value -9999.0 is used to signify that the token is very unlikely." }, "bytes": { - "type": [ - "array", - "null" - ], + "type": ["array", "null"], "description": "A list of integers representing the UTF-8 bytes representation of the token. Useful in instances where characters are represented by multiple tokens and their byte representations must be combined to generate the correct text representation. Can be null if there is no bytes representation for the token." } }, - "required": [ - "token", - "logprob" - ] + "required": ["token", "logprob"] } }, "refusal": { - "type": [ - "array", - "null" - ], + "type": ["array", "null"], "description": "A list of message refusal tokens with log probability information.", "items": { "type": "object", @@ -3568,26 +3336,17 @@ "description": "The log probability of this token, if it is within the top 20 most likely tokens. Otherwise, the value -9999.0 is used to signify that the token is very unlikely." }, "bytes": { - "type": [ - "array", - "null" - ], + "type": ["array", "null"], "description": "A list of integers representing the UTF-8 bytes representation of the token. Useful in instances where characters are represented by multiple tokens and their byte representations must be combined to generate the correct text representation. Can be null if there is no bytes representation for the token." } }, - "required": [ - "token", - "logprob" - ] + "required": ["token", "logprob"] } } } }, "finish_reason": { - "type": [ - "string", - "null" - ], + "type": ["string", "null"], "description": "The reason the model stopped generating tokens. This will be stop if the model hit a natural stop point or a provided stop sequence, length if the maximum number of tokens specified in the request was reached, content_filter if content was omitted due to a flag from our content filters, tool_calls if the model called a tool, or function_call (deprecated) if the model called a function." }, "index": { @@ -3595,10 +3354,7 @@ "description": "The index of the choice in the list of choices." } }, - "required": [ - "delta", - "index" - ] + "required": ["delta", "index"] } }, "created": { @@ -3610,10 +3366,7 @@ "description": "The model used to generate the completion." }, "service_tier": { - "type": [ - "string", - "null" - ], + "type": ["string", "null"], "description": "The service tier used for processing the request. This field is only included if the service_tier parameter is specified in the request." }, "system_fingerprint": { @@ -3641,11 +3394,7 @@ "description": "Total number of tokens used in the request (prompt + completion)." } }, - "required": [ - "completion_tokens", - "prompt_tokens", - "total_tokens" - ] + "required": ["completion_tokens", "prompt_tokens", "total_tokens"] } }, "required": [ @@ -3666,9 +3415,7 @@ "description": "The name of the embedding model to be used." }, "input": { - "example": [ - "Hello World" - ], + "example": ["Hello World"], "description": "The text or token array(s) to be embedded. This can be a single string, an array of strings, or an array of token arrays to embed multiple inputs in one request.", "type": "array", "items": { @@ -3686,10 +3433,7 @@ "description": "Defines the number of dimensions for the output embeddings. This feature is supported by certain models only. This field is optional." } }, - "required": [ - "model", - "input" - ] + "required": ["model", "input"] }, "EmbeddingsResponseDto": { "type": "object", @@ -3718,18 +3462,11 @@ ] } }, - "required": [ - "object", - "model", - "embedding", - "usage" - ] + "required": ["object", "model", "embedding", "usage"] }, "PullModelRequest": { "type": "object", - "required": [ - "model" - ], + "required": ["model"], "properties": { "model": { "type": "string", @@ -3775,9 +3512,7 @@ }, "files": { "description": "The URL sources from which the model downloaded or accessed.", - "example": [ - "https://huggingface.co/cortexso/mistral/tree/gguf" - ], + "example": ["https://huggingface.co/cortexso/mistral/tree/gguf"], "oneOf": [ { "type": "array", @@ -3797,9 +3532,7 @@ }, "stop": { "description": "Defines specific tokens or phrases that signal the model to stop producing further output.", - "example": [ - "End" - ], + "example": ["End"], "type": "array", "items": { "type": "string" @@ -3869,10 +3602,7 @@ "default": "" } }, - "required": [ - "model", - "files" - ] + "required": ["model", "files"] }, "StartModelSuccessDto": { "type": "object", @@ -3886,10 +3616,7 @@ "description": "The unique identifier of the model." } }, - "required": [ - "message", - "modelId" - ] + "required": ["message", "modelId"] }, "ModelStartDto": { "type": "object", @@ -3936,9 +3663,7 @@ "example": "/tmp/model.gguf" } }, - "required": [ - "model" - ] + "required": ["model"] }, "ModelStopDto": { "type": "object", @@ -3949,9 +3674,7 @@ "description": "A downloaded model name." } }, - "required": [ - "model" - ] + "required": ["model"] }, "ImportModelRequest": { "type": "object", @@ -3971,16 +3694,10 @@ "option": { "type": "string", "description": "Import options such as symlink or copy.", - "enum": [ - "symlink", - "copy" - ] + "enum": ["symlink", "copy"] } }, - "required": [ - "model", - "modelPath" - ] + "required": ["model", "modelPath"] }, "ImportModelResponse": { "type": "object", @@ -3999,11 +3716,7 @@ "example": "OK" } }, - "required": [ - "message", - "modelHandle", - "result" - ] + "required": ["message", "modelHandle", "result"] }, "CommonResponseDto": { "type": "object", @@ -4013,9 +3726,7 @@ "description": "The response success or error message." } }, - "required": [ - "message" - ] + "required": ["message"] }, "EngineUninstallationResponseDto": { "type": "object", @@ -4071,11 +3782,7 @@ "example": "OK" } }, - "required": [ - "data", - "object", - "result" - ] + "required": ["data", "object", "result"] }, "Engine": { "type": "object", @@ -4105,12 +3812,7 @@ "example": "0.1.34" } }, - "required": [ - "description", - "name", - "productName", - "status" - ] + "required": ["description", "name", "productName", "status"] }, "ModelDto": { "type": "object", @@ -4126,9 +3828,7 @@ "description": "A predefined text or framework that guides the AI model's response generation." }, "stop": { - "example": [ - "End" - ], + "example": ["End"], "description": "Defines specific tokens or phrases that signal the model to stop producing further output.", "type": "array", "items": { @@ -4240,9 +3940,7 @@ "example": "llamacpp" } }, - "required": [ - "id" - ] + "required": ["id"] }, "ListModelsResponseDto": { "type": "object", @@ -4250,9 +3948,7 @@ "object": { "type": "string", "example": "list", - "enum": [ - "list" - ] + "enum": ["list"] }, "data": { "description": "List of models", @@ -4262,10 +3958,7 @@ } } }, - "required": [ - "object", - "data" - ] + "required": ["object", "data"] }, "UpdateModelDto": { "type": "object", @@ -4284,9 +3977,7 @@ "items": { "type": "string" }, - "example": [ - "" - ] + "example": [""] }, "stream": { "type": "boolean", @@ -4445,11 +4136,7 @@ "description": "Indicates whether the model was successfully deleted." } }, - "required": [ - "id", - "object", - "deleted" - ] + "required": ["id", "object", "deleted"] }, "CreateThreadAssistantDto": { "type": "object", @@ -4539,10 +4226,7 @@ "tool_resources": { "type": "object", "example": { - "resources": [ - "database1", - "database2" - ] + "resources": ["database1", "database2"] }, "description": "Tool resources for the assistant." } @@ -4570,9 +4254,7 @@ } } }, - "required": [ - "assistants" - ] + "required": ["assistants"] }, "ContentDto": { "type": "object", @@ -4591,10 +4273,7 @@ "description": "Text content of the message along with any annotations." } }, - "required": [ - "type", - "text" - ] + "required": ["type", "text"] }, "GetMessageResponseDto": { "type": "object", @@ -4768,13 +4447,7 @@ "description": "Indicates whether there are more messages to retrieve." } }, - "required": [ - "object", - "data", - "first_id", - "last_id", - "has_more" - ] + "required": ["object", "data", "first_id", "last_id", "has_more"] }, "CreateMessageDto": { "type": "object", @@ -4790,10 +4463,7 @@ "description": "The text contents of the message." } }, - "required": [ - "role", - "content" - ] + "required": ["role", "content"] }, "UpdateMessageDto": { "type": "object", @@ -4819,11 +4489,7 @@ "description": "Indicates whether the message was successfully deleted." } }, - "required": [ - "id", - "object", - "deleted" - ] + "required": ["id", "object", "deleted"] }, "GetThreadResponseDto": { "type": "object", @@ -4844,9 +4510,7 @@ "description": "Unix timestamp representing the creation time of the thread." }, "assistants": { - "example": [ - "assistant-001" - ], + "example": ["assistant-001"], "description": "List of assistants involved in the thread.", "type": "array", "items": { @@ -4900,11 +4564,7 @@ "description": "Indicates whether the thread was successfully deleted." } }, - "required": [ - "id", - "object", - "deleted" - ] + "required": ["id", "object", "deleted"] }, "CPUDto": { "type": "object", @@ -4947,12 +4607,7 @@ "description": "The model name of the CPU." } }, - "required": [ - "arch", - "cores", - "instructions", - "model" - ] + "required": ["arch", "cores", "instructions", "model"] }, "GPUDto": { "type": "object", @@ -4976,10 +4631,7 @@ "description": "The version of the installed driver." } }, - "required": [ - "compute_cap", - "driver_version" - ] + "required": ["compute_cap", "driver_version"] }, "free_vram": { "type": "integer", @@ -5037,10 +4689,7 @@ "description": "The version of the operating system." } }, - "required": [ - "name", - "version" - ] + "required": ["name", "version"] }, "PowerDto": { "type": "object", @@ -5061,11 +4710,7 @@ "description": "Indicates if the power-saving mode is enabled." } }, - "required": [ - "battery_life", - "charging_status", - "is_power_saving" - ] + "required": ["battery_life", "charging_status", "is_power_saving"] }, "RAMDto": { "type": "object", @@ -5086,11 +4731,7 @@ "description": "The type of RAM." } }, - "required": [ - "available", - "total", - "type" - ] + "required": ["available", "total", "type"] }, "Storage": { "type": "object", @@ -5111,12 +4752,8 @@ "description": "The type of storage." } }, - "required": [ - "available", - "total", - "type" - ] + "required": ["available", "total", "type"] } } } -} \ No newline at end of file +} diff --git a/engine/cli/CMakeLists.txt b/engine/cli/CMakeLists.txt index 758a51dc8..ce6f254ca 100644 --- a/engine/cli/CMakeLists.txt +++ b/engine/cli/CMakeLists.txt @@ -73,6 +73,7 @@ add_executable(${TARGET_NAME} main.cc ${CMAKE_CURRENT_SOURCE_DIR}/../utils/cpuid/cpu_info.cc ${CMAKE_CURRENT_SOURCE_DIR}/../utils/file_logger.cc ${CMAKE_CURRENT_SOURCE_DIR}/command_line_parser.cc + ${CMAKE_CURRENT_SOURCE_DIR}/../services/config_service.cc ${CMAKE_CURRENT_SOURCE_DIR}/../services/download_service.cc ${CMAKE_CURRENT_SOURCE_DIR}/../services/engine_service.cc ${CMAKE_CURRENT_SOURCE_DIR}/../services/model_service.cc diff --git a/engine/cli/command_line_parser.cc b/engine/cli/command_line_parser.cc index 136aebdbc..2996401fc 100644 --- a/engine/cli/command_line_parser.cc +++ b/engine/cli/command_line_parser.cc @@ -326,14 +326,8 @@ void CommandLineParser::SetupModelCommands() { void CommandLineParser::SetupConfigsCommands() { auto config_cmd = app_.add_subcommand("config", "Subcommands for managing configurations"); - config_cmd->usage( - "Usage:\n" + commands::GetCortexBinary() + - " config status for listing all API server configuration.\n" + - commands::GetCortexBinary() + - " config --cors [on/off] to toggle CORS.\n" + - commands::GetCortexBinary() + - " config --allowed_origins [comma separated origin] to set a list of " - "allowed origin"); + config_cmd->usage("Usage:\n" + commands::GetCortexBinary() + + " config [option] [value]"); config_cmd->group(kConfigGroup); auto config_status_cmd = config_cmd->add_subcommand("status", "Print all configurations"); @@ -344,18 +338,18 @@ void CommandLineParser::SetupConfigsCommands() { std::stoi(cml_data_.config.apiServerPort)); }); - // TODO: this can be improved - std::vector avai_opts{"cors", "allowed_origins"}; - std::unordered_map description{ - {"cors", "[on/off] Toggling CORS."}, - {"allowed_origins", - "Allowed origins for CORS. Comma separated. E.g. " - "http://localhost,https://cortex.so"}}; - for (const auto& opt : avai_opts) { - std::string option = "--" + opt; - config_cmd->add_option(option, config_update_opts_[opt], description[opt]) - ->expected(0, 1) - ->default_str("*"); + for (const auto& [key, opt] : CONFIGURATIONS) { + std::string option = "--" + opt.name; + auto option_cmd = + config_cmd->add_option(option, config_update_opts_[opt.name], opt.desc) + ->group(opt.group) + ->default_str(opt.default_value); + + if (opt.allow_empty) { + option_cmd->expected(0, 1); + } else { + option_cmd->expected(1); + } } config_cmd->callback([this, config_cmd] { diff --git a/engine/cli/commands/config_upd_cmd.cc b/engine/cli/commands/config_upd_cmd.cc index ad1211f36..58bedb2e5 100644 --- a/engine/cli/commands/config_upd_cmd.cc +++ b/engine/cli/commands/config_upd_cmd.cc @@ -1,35 +1,39 @@ #include "config_upd_cmd.h" #include "commands/server_start_cmd.h" +#include "common/api_server_configuration.h" #include "utils/curl_utils.h" #include "utils/logging_utils.h" #include "utils/string_utils.h" #include "utils/url_parser.h" namespace { -const std::vector config_keys{"cors", "allowed_origins"}; - inline Json::Value NormalizeJson( const std::unordered_map options) { Json::Value root; for (const auto& [key, value] : options) { - if (std::find(config_keys.begin(), config_keys.end(), key) == - config_keys.end()) { + if (CONFIGURATIONS.find(key) == CONFIGURATIONS.end()) { continue; } + auto config = CONFIGURATIONS.at(key); - if (key == "cors") { + if (config.accept_value == "[on|off]") { if (string_utils::EqualsIgnoreCase("on", value)) { - root["cors"] = true; + root[key] = true; } else if (string_utils::EqualsIgnoreCase("off", value)) { - root["cors"] = false; + root[key] = false; } - } else if (key == "allowed_origins") { + } else if (config.accept_value == "comma separated") { auto origins = string_utils::SplitBy(value, ","); Json::Value origin_array(Json::arrayValue); for (const auto& origin : origins) { origin_array.append(origin); } root[key] = origin_array; + } else if (config.accept_value == "string") { + root[key] = value; + } else { + CTL_ERR("Not support configuration type: " << config.accept_value + << " for config key: " << key); } } @@ -50,13 +54,21 @@ void commands::ConfigUpdCmd::Exec( } } + auto non_null_opts = std::unordered_map(); + for (const auto& [key, value] : options) { + if (value.empty()) { + continue; + } + non_null_opts[key] = value; + } + auto url = url_parser::Url{ .protocol = "http", .host = host + ":" + std::to_string(port), .pathParams = {"v1", "configs"}, }; - auto json = NormalizeJson(options); + auto json = NormalizeJson(non_null_opts); if (json.empty()) { CLI_LOG_ERROR("Invalid configuration options provided"); return; diff --git a/engine/common/api_server_configuration.h b/engine/common/api_server_configuration.h index 9d261231f..5bfcbbdc5 100644 --- a/engine/common/api_server_configuration.h +++ b/engine/common/api_server_configuration.h @@ -5,15 +5,128 @@ #include #include +// current only support basic auth +enum class ProxyAuthMethod { + Basic, + Digest, + DigestIe, + Bearer, + Negotiate, + Ntlm, + NtlmWb, + Any, + AnySafe, + AuthOnly, + AwsSigV4 +}; + +struct ApiConfigurationMetadata { + std::string name; + std::string desc; + std::string group; + std::string accept_value; + std::string default_value; + + bool allow_empty = false; +}; + +static const std::unordered_map + CONFIGURATIONS = { + {"cors", + ApiConfigurationMetadata{ + .name = "cors", + .desc = "Cross-Origin Resource Sharing configuration.", + .group = "CORS", + .accept_value = "[on|off]", + .default_value = "on"}}, + {"allowed_origins", + ApiConfigurationMetadata{ + .name = "allowed_origins", + .desc = "Allowed origins for CORS. Comma separated. E.g. " + "http://localhost,https://cortex.so", + .group = "CORS", + .accept_value = "comma separated", + .default_value = "*", + .allow_empty = true}}, + {"proxy_url", ApiConfigurationMetadata{.name = "proxy_url", + .desc = "Proxy URL", + .group = "Proxy", + .accept_value = "string", + .default_value = ""}}, + {"proxy_username", ApiConfigurationMetadata{.name = "proxy_username", + .desc = "Proxy Username", + .group = "Proxy", + .accept_value = "string", + .default_value = ""}}, + {"proxy_password", ApiConfigurationMetadata{.name = "proxy_password", + .desc = "Proxy Password", + .group = "Proxy", + .accept_value = "string", + .default_value = ""}}, + {"verify_proxy_ssl", + ApiConfigurationMetadata{.name = "verify_proxy_ssl", + .desc = "Verify SSL for proxy", + .group = "Proxy", + .accept_value = "[on|off]", + .default_value = "on"}}, + {"verify_proxy_host_ssl", + ApiConfigurationMetadata{.name = "verify_proxy_host_ssl", + .desc = "Verify SSL for proxy", + .group = "Proxy", + .accept_value = "[on|off]", + .default_value = "on"}}, + {"no_proxy", ApiConfigurationMetadata{.name = "no_proxy", + .desc = "No proxy for hosts", + .group = "Proxy", + .accept_value = "string", + .default_value = ""}}, + {"verify_peer_ssl", ApiConfigurationMetadata{.name = "verify_peer_ssl", + .desc = "Verify peer SSL", + .group = "Proxy", + .accept_value = "[on|off]", + .default_value = "on"}}, + {"verify_host_ssl", ApiConfigurationMetadata{.name = "verify_host_ssl", + .desc = "Verify host SSL", + .group = "Proxy", + .accept_value = "[on|off]", + .default_value = "on"}}, +}; + class ApiServerConfiguration { public: - ApiServerConfiguration(bool cors = true, - std::vector allowed_origins = {}) - : cors{cors}, allowed_origins{allowed_origins} {} + ApiServerConfiguration( + bool cors = true, std::vector allowed_origins = {}, + bool verify_proxy_ssl = true, bool verify_proxy_host_ssl = true, + const std::string& proxy_url = "", const std::string& proxy_username = "", + const std::string& proxy_password = "", const std::string& no_proxy = "", + bool verify_peer_ssl = true, bool verify_host_ssl = true) + : cors{cors}, + allowed_origins{allowed_origins}, + verify_proxy_ssl{verify_proxy_ssl}, + verify_proxy_host_ssl{verify_proxy_host_ssl}, + proxy_url{proxy_url}, + proxy_username{proxy_username}, + proxy_password{proxy_password}, + no_proxy{no_proxy}, + verify_peer_ssl{verify_peer_ssl}, + verify_host_ssl{verify_host_ssl} {} + // cors bool cors{true}; std::vector allowed_origins; + // proxy + bool verify_proxy_ssl{true}; + bool verify_proxy_host_ssl{true}; + ProxyAuthMethod proxy_auth_method{ProxyAuthMethod::Basic}; + std::string proxy_url{""}; + std::string proxy_username{""}; + std::string proxy_password{""}; + std::string no_proxy{""}; + + bool verify_peer_ssl{true}; + bool verify_host_ssl{true}; + Json::Value ToJson() const { Json::Value root; root["cors"] = cors; @@ -21,6 +134,15 @@ class ApiServerConfiguration { for (const auto& origin : allowed_origins) { root["allowed_origins"].append(origin); } + root["verify_proxy_ssl"] = verify_proxy_ssl; + root["verify_proxy_host_ssl"] = verify_proxy_host_ssl; + root["proxy_url"] = proxy_url; + root["proxy_username"] = proxy_username; + root["proxy_password"] = proxy_password; + root["no_proxy"] = no_proxy; + root["verify_peer_ssl"] = verify_peer_ssl; + root["verify_host_ssl"] = verify_host_ssl; + return root; } @@ -31,6 +153,78 @@ class ApiServerConfiguration { const std::unordered_map> field_updater{ + {"verify_peer_ssl", + [this](const Json::Value& value) -> bool { + if (!value.isBool()) { + return false; + } + verify_peer_ssl = value.asBool(); + return true; + }}, + + {"verify_host_ssl", + [this](const Json::Value& value) -> bool { + if (!value.isBool()) { + return false; + } + verify_host_ssl = value.asBool(); + return true; + }}, + + {"verify_proxy_host_ssl", + [this](const Json::Value& value) -> bool { + if (!value.isBool()) { + return false; + } + verify_proxy_host_ssl = value.asBool(); + return true; + }}, + + {"verify_proxy_ssl", + [this](const Json::Value& value) -> bool { + if (!value.isBool()) { + return false; + } + verify_proxy_ssl = value.asBool(); + return true; + }}, + + {"no_proxy", + [this](const Json::Value& value) -> bool { + if (!value.isString()) { + return false; + } + no_proxy = value.asString(); + return true; + }}, + + {"proxy_url", + [this](const Json::Value& value) -> bool { + if (!value.isString()) { + return false; + } + proxy_url = value.asString(); + return true; + }}, + + {"proxy_username", + [this](const Json::Value& value) -> bool { + if (!value.isString()) { + return false; + } + proxy_username = value.asString(); + return true; + }}, + + {"proxy_password", + [this](const Json::Value& value) -> bool { + if (!value.isString()) { + return false; + } + proxy_password = value.asString(); + return true; + }}, + {"cors", [this](const Json::Value& value) -> bool { if (!value.isBool()) { diff --git a/engine/config/yaml_config.cc b/engine/config/yaml_config.cc index e4932c9c3..0b3c23d8b 100644 --- a/engine/config/yaml_config.cc +++ b/engine/config/yaml_config.cc @@ -8,6 +8,7 @@ #include "utils/file_manager_utils.h" #include "utils/format_utils.h" #include "yaml_config.h" + namespace config { // Method to read YAML file void YamlHandler::Reset() { @@ -44,6 +45,7 @@ void YamlHandler::ReadYamlFile(const std::string& file_path) { throw; } } + void YamlHandler::SplitPromptTemplate(ModelConfig& mc) { if (mc.prompt_template.size() > 0) { auto& pt = mc.prompt_template; @@ -220,7 +222,7 @@ void YamlHandler::UpdateModelConfig(ModelConfig new_model_config) { yaml_node_["ngl"] = model_config_.ngl; if (!std::isnan(static_cast(model_config_.ctx_len))) yaml_node_["ctx_len"] = model_config_.ctx_len; - if (!std::isnan(static_cast(model_config_.n_parallel))) + if (!std::isnan(static_cast(model_config_.n_parallel))) yaml_node_["n_parallel"] = model_config_.n_parallel; if (!std::isnan(static_cast(model_config_.tp))) yaml_node_["tp"] = model_config_.tp; @@ -377,7 +379,8 @@ void YamlHandler::WriteYamlFile(const std::string& file_path) const { outFile << format_utils::writeKeyValue( "ctx_len", yaml_node_["ctx_len"], "llama.context_length | 0 or undefined = loaded from model"); - outFile << format_utils::writeKeyValue("n_parallel", yaml_node_["n_parallel"]); + outFile << format_utils::writeKeyValue("n_parallel", + yaml_node_["n_parallel"]); outFile << format_utils::writeKeyValue("ngl", yaml_node_["ngl"], "Undefined = loaded from model"); outFile << "# END OPTIONAL\n"; diff --git a/engine/main.cc b/engine/main.cc index e723a8fc7..5fdd69d6c 100644 --- a/engine/main.cc +++ b/engine/main.cc @@ -106,13 +106,14 @@ void RunServer(std::optional port, bool ignore_cout) { auto event_queue_ptr = std::make_shared(); cortex::event::EventProcessor event_processor(event_queue_ptr); - auto download_service = std::make_shared(event_queue_ptr); + auto config_service = std::make_shared(); + auto download_service = + std::make_shared(event_queue_ptr, config_service); auto engine_service = std::make_shared(download_service); auto inference_svc = std::make_shared(engine_service); auto model_service = std::make_shared( download_service, inference_svc, engine_service); - auto config_service = std::make_shared(); // initialize custom controllers auto engine_ctl = std::make_shared(engine_service); diff --git a/engine/services/config_service.cc b/engine/services/config_service.cc index 9f1589887..9c794bb38 100644 --- a/engine/services/config_service.cc +++ b/engine/services/config_service.cc @@ -5,8 +5,12 @@ cpp::result ConfigService::UpdateApiServerConfiguration(const Json::Value& json) { auto config = file_manager_utils::GetCortexConfig(); - ApiServerConfiguration api_server_config{config.enableCors, - config.allowedOrigins}; + ApiServerConfiguration api_server_config{ + config.enableCors, config.allowedOrigins, config.verifyProxySsl, + config.verifyProxyHostSsl, config.proxyUrl, config.proxyUsername, + config.proxyPassword, config.noProxy, config.verifyPeerSsl, + config.verifyHostSsl}; + std::vector updated_fields; std::vector invalid_fields; std::vector unknown_fields; @@ -20,13 +24,27 @@ ConfigService::UpdateApiServerConfiguration(const Json::Value& json) { config.enableCors = api_server_config.cors; config.allowedOrigins = api_server_config.allowed_origins; + config.verifyProxySsl = api_server_config.verify_proxy_ssl; + config.verifyProxyHostSsl = api_server_config.verify_proxy_host_ssl; + + config.proxyUrl = api_server_config.proxy_url; + config.proxyUsername = api_server_config.proxy_username; + config.proxyPassword = api_server_config.proxy_password; + config.noProxy = api_server_config.no_proxy; + + config.verifyPeerSsl = api_server_config.verify_peer_ssl; + config.verifyHostSsl = api_server_config.verify_host_ssl; auto result = file_manager_utils::UpdateCortexConfig(config); return api_server_config; } cpp::result -ConfigService::GetApiServerConfiguration() const { +ConfigService::GetApiServerConfiguration() { auto config = file_manager_utils::GetCortexConfig(); - return ApiServerConfiguration{config.enableCors, config.allowedOrigins}; + return ApiServerConfiguration{ + config.enableCors, config.allowedOrigins, config.verifyProxySsl, + config.verifyProxyHostSsl, config.proxyUrl, config.proxyUsername, + config.proxyPassword, config.noProxy, config.verifyPeerSsl, + config.verifyHostSsl}; } diff --git a/engine/services/config_service.h b/engine/services/config_service.h index 3e9b9ec6e..5bebca292 100644 --- a/engine/services/config_service.h +++ b/engine/services/config_service.h @@ -8,6 +8,5 @@ class ConfigService { cpp::result UpdateApiServerConfiguration( const Json::Value& json); - cpp::result GetApiServerConfiguration() - const; + cpp::result GetApiServerConfiguration(); }; diff --git a/engine/services/download_service.cc b/engine/services/download_service.cc index b8f39595a..c58935fe3 100644 --- a/engine/services/download_service.cc +++ b/engine/services/download_service.cc @@ -52,6 +52,48 @@ cpp::result ProcessCompletedTransfers(CURLM* multi_handle) { } return {}; } + +void SetUpProxy(CURL* handle, std::shared_ptr config_service) { + auto configuration = config_service->GetApiServerConfiguration(); + if (configuration.has_value()) { + if (!configuration->proxy_url.empty()) { + auto proxy_url = configuration->proxy_url; + auto verify_proxy_ssl = configuration->verify_proxy_ssl; + auto verify_proxy_host_ssl = configuration->verify_proxy_host_ssl; + + auto verify_ssl = configuration->verify_peer_ssl; + auto verify_host_ssl = configuration->verify_host_ssl; + + auto proxy_username = configuration->proxy_username; + auto proxy_password = configuration->proxy_password; + + CTL_INF("=== Proxy configuration ==="); + CTL_INF("Proxy url: " << proxy_url); + CTL_INF("Verify proxy ssl: " << verify_proxy_ssl); + CTL_INF("Verify proxy host ssl: " << verify_proxy_host_ssl); + CTL_INF("Verify ssl: " << verify_ssl); + CTL_INF("Verify host ssl: " << verify_host_ssl); + + curl_easy_setopt(handle, CURLOPT_PROXY, proxy_url.c_str()); + curl_easy_setopt(handle, CURLOPT_SSL_VERIFYPEER, verify_ssl ? 1L : 0L); + curl_easy_setopt(handle, CURLOPT_SSL_VERIFYHOST, + verify_host_ssl ? 2L : 0L); + + curl_easy_setopt(handle, CURLOPT_PROXY_SSL_VERIFYPEER, + verify_proxy_ssl ? 1L : 0L); + curl_easy_setopt(handle, CURLOPT_PROXY_SSL_VERIFYHOST, + verify_proxy_host_ssl ? 2L : 0L); + + auto proxy_auth = proxy_username + ":" + proxy_password; + curl_easy_setopt(handle, CURLOPT_PROXYUSERPWD, proxy_auth.c_str()); + + curl_easy_setopt(handle, CURLOPT_NOPROXY, + configuration->no_proxy.c_str()); + } + } else { + CTL_ERR("Failed to get configuration"); + } +} } // namespace cpp::result DownloadService::AddDownloadTask( @@ -87,6 +129,7 @@ cpp::result DownloadService::GetFileSize( return cpp::fail(static_cast("Failed to init CURL")); } + SetUpProxy(curl, config_service_); curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L); curl_easy_setopt(curl, CURLOPT_NOBODY, 1L); curl_easy_setopt(curl, CURLOPT_URL, url.c_str()); @@ -189,6 +232,8 @@ cpp::result DownloadService::Download( curl_easy_setopt(curl, CURLOPT_HTTPHEADER, curl_headers); } + + SetUpProxy(curl, config_service_); curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, &WriteCallback); curl_easy_setopt(curl, CURLOPT_WRITEDATA, file); curl_easy_setopt(curl, CURLOPT_NOPROGRESS, 0L); @@ -407,6 +452,7 @@ cpp::result DownloadService::ProcessMultiDownload( void DownloadService::SetUpCurlHandle(CURL* handle, const DownloadItem& item, FILE* file, DownloadingData* dl_data) { + SetUpProxy(handle, config_service_); curl_easy_setopt(handle, CURLOPT_URL, item.downloadUrl.c_str()); curl_easy_setopt(handle, CURLOPT_WRITEFUNCTION, WriteCallback); curl_easy_setopt(handle, CURLOPT_WRITEDATA, file); diff --git a/engine/services/download_service.h b/engine/services/download_service.h index 47d351fdf..b915c9814 100644 --- a/engine/services/download_service.h +++ b/engine/services/download_service.h @@ -8,6 +8,7 @@ #include #include "common/download_task_queue.h" #include "common/event.h" +#include "services/config_service.h" #include "utils/result.hpp" struct ProcessDownloadFailed { @@ -20,6 +21,8 @@ class DownloadService { private: static constexpr int MAX_CONCURRENT_TASKS = 4; + std::shared_ptr config_service_; + struct DownloadingData { std::string task_id; std::string item_id; @@ -82,8 +85,9 @@ class DownloadService { explicit DownloadService() = default; - explicit DownloadService(std::shared_ptr event_queue) - : event_queue_{event_queue} { + explicit DownloadService(std::shared_ptr event_queue, + std::shared_ptr config_service) + : event_queue_{event_queue}, config_service_{config_service} { InitializeWorkers(); }; diff --git a/engine/templates/linux/install.sh b/engine/templates/linux/install.sh new file mode 100644 index 000000000..e907150a8 --- /dev/null +++ b/engine/templates/linux/install.sh @@ -0,0 +1,281 @@ +#!/bin/bash -e + +# Check for root privileges +if [ "$(id -u)" != "0" ]; then + echo "This script must be run as root. Please run again with sudo." + exit 1 +fi + +# Determine the home directory based on the user +USER_TO_RUN_AS=${SUDO_USER:-$(whoami)} +if [ "$USER_TO_RUN_AS" = "root" ]; then + USER_HOME="/root" +else + USER_HOME="/home/$USER_TO_RUN_AS" +fi + +# Check and suggest installing jq and tar if not present +check_install_jq_tar() { + RED='\033[0;31m' + GREEN='\033[0;32m' + NC='\033[0m' # No Color + + if ! command -v jq &> /dev/null; then + echo -e "${RED}jq could not be found ...${NC}" + echo -e "${GREEN}Please install jq then rerun this script${NC}" + exit 1 + fi + + if ! command -v tar &> /dev/null; then + echo -e "${RED}tar could not be found ...${NC}" + echo -e "${GREEN}Please install tar then rerun this script${NC}" + exit 1 + fi +} + +# Function to fetch the latest version based on channel +get_latest_version() { + local channel=$1 + local tag_name + case $channel in + stable) + tag_name=$(curl -s "https://api.github.com/repos/janhq/cortex.cpp/releases/latest" | grep -oP '"tag_name": "\K(.*)(?=")') + ;; + beta) + tag_name=$(curl -s "https://api.github.com/repos/janhq/cortex.cpp/releases" | jq -r '.[] | select(.prerelease) | .tag_name' | head -n 1) + ;; + nightly) + tag_name=$(curl -s "https://delta.jan.ai/cortex/latest/version.json" | jq -r '.tag_name') + ;; + *) + echo "Invalid channel specified." + exit 1 + ;; + esac + echo "${tag_name#v}" +} + +# Default values +CHANNEL="stable" +VERSION="" +IS_UPDATE="false" +DEB_LOCAL="false" + +# Function to parse command-line arguments +parse_args() { + while [[ "$#" -gt 0 ]]; do + case $1 in + --channel) + CHANNEL="$2" + shift 2 + ;; + --version) + VERSION="$2" + shift 2 + ;; + --deb_local) + DEB_LOCAL="true" + shift 1 + ;; + --is_update) + IS_UPDATE="true" + shift 1 + ;; + *) + echo "Unknown option: $1" + exit 1 + ;; + esac + done +} + +# Call parse_args function to handle options +parse_args "$@" + +# Check if VERSION is empty and fetch latest if necessary +if [ -z "$VERSION" ]; then + VERSION=$(get_latest_version $CHANNEL) +fi + +# Set paths based on channel +case $CHANNEL in + stable) + CLI_BINARY_NAME="cortex" + SERVER_BINARY_NAME="cortex-server" + DATA_DIR="$USER_HOME/cortexcpp" + UNINSTALL_SCRIPT="/usr/bin/cortex-uninstall.sh" + CONFIGURATION_FILE="$USER_HOME/.cortexrc" + DEB_APP_NAME="cortexcpp" + ;; + beta) + CLI_BINARY_NAME="cortex-beta" + SERVER_BINARY_NAME="cortex-server-beta" + DATA_DIR="$USER_HOME/cortexcpp-beta" + UNINSTALL_SCRIPT="/usr/bin/cortex-beta-uninstall.sh" + CONFIGURATION_FILE="$USER_HOME/.cortexrc-beta" + DEB_APP_NAME="cortexcpp-beta" + ;; + nightly) + CLI_BINARY_NAME="cortex-nightly" + SERVER_BINARY_NAME="cortex-server-nightly" + DATA_DIR="$USER_HOME/cortexcpp-nightly" + UNINSTALL_SCRIPT="/usr/bin/cortex-nightly-uninstall.sh" + CONFIGURATION_FILE="$USER_HOME/.cortexrc-nightly" + DEB_APP_NAME="cortexcpp-nightly" + ;; + *) + echo "Invalid channel specified." + exit 1 + ;; +esac + +INSTALL_DIR="/usr/bin" + +# Function to download and extract cortex +install_cortex() { + local channel=$1 + local version=$2 + local is_deb=$3 + local url_binary="" + local url_deb_local="" + local url_deb_network="" + + case $channel in + stable) + url_binary="https://github.com/janhq/cortex.cpp/releases/download/v${version}/cortex-${version}-linux-amd64.tar.gz" + url_deb_local="https://github.com/janhq/cortex.cpp/releases/download/v${version}/cortex-${version}-linux-amd64-local-installer.deb" + url_deb_network="https://github.com/janhq/cortex.cpp/releases/download/v${version}/cortex-${version}-linux-amd64-network-installer.deb" + ;; + beta) + url_binary="https://github.com/janhq/cortex.cpp/releases/download/v${version}/cortex-${version}-linux-amd64.tar.gz" + url_deb_local="https://github.com/janhq/cortex.cpp/releases/download/v${version}/cortex-${version}-linux-amd64-local-installer.deb" + url_deb_network="https://github.com/janhq/cortex.cpp/releases/download/v${version}/cortex-${version}-linux-amd64-network-installer.deb" + ;; + nightly) + url_binary="https://delta.jan.ai/cortex/v${version}/linux-amd64/cortex-nightly.tar.gz" + url_deb_local="https://delta.jan.ai/cortex/v${version}/linux-amd64/cortex-${version}-linux-amd64-local-installer.deb" + url_deb_network="https://delta.jan.ai/cortex/v${version}/linux-amd64/cortex-${version}-linux-amd64-network-installer.deb" + ;; + esac + + if [ "$is_deb" = "true" ]; then + # Download the deb package + if [ "$DEB_LOCAL" = "true" ]; then + echo "Downloading cortex $channel version $version from $url_deb_local" + curl -L $url_deb_local -o /tmp/cortex.deb + else + echo "Downloading cortex $channel version $version from $url_deb_network" + curl -L $url_deb_network -o /tmp/cortex.deb + fi + + # Install the deb package + if [ "$IS_UPDATE" = "false" ]; then + apt-get install -y /tmp/cortex.deb + else + echo -e "n\n" | SKIP_POSTINSTALL=true apt-get install -y --allow-downgrades /tmp/cortex.deb + fi + rm -f /tmp/cortex.deb + else + echo "Downloading cortex $channel version $version from $url_binary" + curl -L $url_binary -o /tmp/cortex.tar.gz + tar -xzvf /tmp/cortex.tar.gz -C /tmp + chmod +x /tmp/cortex/* + cp /tmp/cortex/* /usr/bin/ + # Check is update or not + if [ "$IS_UPDATE" = "false" ]; then + su -c "$INSTALL_DIR/$CLI_BINARY_NAME engines install llama-cpp" $USER_TO_RUN_AS + su -c "$INSTALL_DIR/$CLI_BINARY_NAME stop > /dev/null 2>&1" $USER_TO_RUN_AS + fi + rm -rf /tmp/cortex + rm -f /tmp/cortex.tar.gz + fi +} + +# Function to create uninstall script +create_uninstall_script() { + local is_deb=$1 + if [ "$is_deb" = "false" ]; then + cat << EOF > $UNINSTALL_SCRIPT +#!/bin/bash +# Check for root privileges +if [ "\$(id -u)" != "0" ]; then + echo "This script must be run as root. Please run again with sudo." + exit 1 +fi + +echo "Stopping cortex..." +su -c "$INSTALL_DIR/$CLI_BINARY_NAME stop > /dev/null 2>&1" $USER_TO_RUN_AS +rm -f $INSTALL_DIR/$CLI_BINARY_NAME +rm -f $INSTALL_DIR/$SERVER_BINARY_NAME +rm -f $UNINSTALL_SCRIPT + +echo "Do you want to delete the $DATA_DIR data folder and file $CONFIGURATION_FILE? (yes/no) [default: no]" +read -r answer +while true; do + case "\$answer" in + [yY][eE][sS]|[yY]) + echo "Deleting cortex data folders..." + if [ -d "$DATA_DIR" ]; then + echo "Removing $DATA_DIR" + rm -rf "$DATA_DIR" > /dev/null 2>&1 + fi + if [ -f "$CONFIGURATION_FILE" ]; then + echo "Removing $CONFIGURATION_FILE" + rm -f "$CONFIGURATION_FILE" > /dev/null 2>&1 + fi + break + ;; + [nN][oO]|[nN]|"") + echo "Keeping the 'cortex' data folders." + break + ;; + *) + echo "Invalid response. Please type 'yes', 'no', 'y', or 'n' (case-insensitive)." + read -r answer + ;; + esac +done + +EOF + + else + cat << EOF > $UNINSTALL_SCRIPT +#!/bin/bash +# Check for root privileges +if [ "\$(id -u)" != "0" ]; then + echo "This script must be run as root. Please run again with sudo." + exit 1 +fi + +apt-get remove -y $DEB_APP_NAME +rm -f $UNINSTALL_SCRIPT +EOF + fi + + chmod +x $UNINSTALL_SCRIPT + echo "Uninstall script created at $UNINSTALL_SCRIPT" +} + +# Run installation +check_install_jq_tar + +IS_DEB="false" + +# Check if apt-get command is available +if command -v apt-get &> /dev/null; then + if [ "$IS_UPDATE" = "true" ]; then + # check if cortexcpp deb package is installed + if dpkg -l | grep -q $DEB_APP_NAME; then + IS_DEB="true" + else + IS_DEB="false" + fi + else + IS_DEB="true" + fi +fi + +install_cortex $CHANNEL $VERSION $IS_DEB +create_uninstall_script $IS_DEB + +echo "Installation complete. Run cortex-uninstall.sh to uninstall." \ No newline at end of file diff --git a/engine/test/components/CMakeLists.txt b/engine/test/components/CMakeLists.txt index 1c1e84de7..b92770a65 100644 --- a/engine/test/components/CMakeLists.txt +++ b/engine/test/components/CMakeLists.txt @@ -9,6 +9,7 @@ add_executable(${PROJECT_NAME} ${CMAKE_CURRENT_SOURCE_DIR}/../../config/gguf_parser.cc ${CMAKE_CURRENT_SOURCE_DIR}/../../cli/commands/cortex_upd_cmd.cc ${CMAKE_CURRENT_SOURCE_DIR}/../../cli/commands/server_stop_cmd.cc + ${CMAKE_CURRENT_SOURCE_DIR}/../../services/config_service.cc ${CMAKE_CURRENT_SOURCE_DIR}/../../services/download_service.cc ${CMAKE_CURRENT_SOURCE_DIR}/../../database/models.cc ) diff --git a/engine/utils/config_yaml_utils.h b/engine/utils/config_yaml_utils.h index cd634ee43..3dc6bc61f 100644 --- a/engine/utils/config_yaml_utils.h +++ b/engine/utils/config_yaml_utils.h @@ -1,4 +1,5 @@ #pragma once + #include #include #include @@ -34,6 +35,16 @@ struct CortexConfig { bool enableCors; std::vector allowedOrigins; + + std::string proxyUrl; + bool verifyProxySsl; + bool verifyProxyHostSsl; + std::string proxyUsername; + std::string proxyPassword; + std::string noProxy; + + bool verifyPeerSsl; + bool verifyHostSsl; }; const std::string kDefaultHost{"127.0.0.1"}; @@ -46,6 +57,7 @@ constexpr const auto kDefaultLatestLlamacppRelease = ""; constexpr const auto kDefaultCorsEnabled = true; const std::vector kDefaultEnabledOrigins{ "http://localhost:39281", "http://127.0.0.1:39281", "http://0.0.0.0:39281"}; +constexpr const auto kDefaultNoProxy = "localhost,127.0.0.1"; inline cpp::result DumpYamlConfig(const CortexConfig& config, const std::string& path) { @@ -76,6 +88,14 @@ inline cpp::result DumpYamlConfig(const CortexConfig& config, node["llamacppVersion"] = config.llamacppVersion; node["enableCors"] = config.enableCors; node["allowedOrigins"] = config.allowedOrigins; + node["proxyUrl"] = config.proxyUrl; + node["verifyProxySsl"] = config.verifyProxySsl; + node["verifyProxyHostSsl"] = config.verifyProxyHostSsl; + node["proxyUsername"] = config.proxyUsername; + node["proxyPassword"] = config.proxyPassword; + node["noProxy"] = config.noProxy; + node["verifyPeerSsl"] = config.verifyPeerSsl; + node["verifyHostSsl"] = config.verifyHostSsl; out_file << node; out_file.close(); @@ -105,7 +125,11 @@ inline CortexConfig FromYaml(const std::string& path, !node["huggingFaceToken"] || !node["gitHubUserAgent"] || !node["gitHubToken"] || !node["llamacppVariant"] || !node["llamacppVersion"] || !node["enableCors"] || - !node["allowedOrigins"]); + !node["allowedOrigins"] || !node["proxyUrl"] || + !node["proxyUsername"] || !node["proxyPassword"] || + !node["verifyPeerSsl"] || !node["verifyHostSsl"] || + !node["verifyProxySsl"] || !node["verifyProxyHostSsl"] || + !node["noProxy"]); CortexConfig config = { .logFolderPath = node["logFolderPath"] @@ -147,23 +171,48 @@ inline CortexConfig FromYaml(const std::string& path, : default_cfg.latestLlamacppRelease, .huggingFaceToken = node["huggingFaceToken"] ? node["huggingFaceToken"].as() - : "", + : default_cfg.huggingFaceToken, .gitHubUserAgent = node["gitHubUserAgent"] ? node["gitHubUserAgent"].as() - : "", - .gitHubToken = - node["gitHubToken"] ? node["gitHubToken"].as() : "", + : default_cfg.gitHubUserAgent, + .gitHubToken = node["gitHubToken"] + ? node["gitHubToken"].as() + : default_cfg.gitHubToken, .llamacppVariant = node["llamacppVariant"] ? node["llamacppVariant"].as() - : "", + : default_cfg.llamacppVariant, .llamacppVersion = node["llamacppVersion"] ? node["llamacppVersion"].as() - : "", - .enableCors = node["enableCors"] ? node["enableCors"].as() : true, + : default_cfg.llamacppVersion, + .enableCors = node["enableCors"] ? node["enableCors"].as() + : default_cfg.enableCors, .allowedOrigins = node["allowedOrigins"] ? node["allowedOrigins"].as>() - : std::vector{}}; + : default_cfg.allowedOrigins, + .proxyUrl = node["proxyUrl"] ? node["proxyUrl"].as() + : default_cfg.proxyUrl, + .verifyProxySsl = node["verifyProxySsl"] + ? node["verifyProxySsl"].as() + : default_cfg.verifyProxySsl, + .verifyProxyHostSsl = node["verifyProxyHostSsl"] + ? node["verifyProxyHostSsl"].as() + : default_cfg.verifyProxyHostSsl, + .proxyUsername = node["proxyUsername"] + ? node["proxyUsername"].as() + : default_cfg.proxyUsername, + .proxyPassword = node["proxyPassword"] + ? node["proxyPassword"].as() + : default_cfg.proxyPassword, + .noProxy = node["noProxy"] ? node["noProxy"].as() + : default_cfg.noProxy, + .verifyPeerSsl = node["verifyPeerSsl"] + ? node["verifyPeerSsl"].as() + : default_cfg.verifyPeerSsl, + .verifyHostSsl = node["verifyHostSsl"] + ? node["verifyHostSsl"].as() + : default_cfg.verifyHostSsl, + }; if (should_update_config) { auto result = DumpYamlConfig(config, path); if (result.has_error()) { diff --git a/engine/utils/curl_utils.h b/engine/utils/curl_utils.h index 55a824c6c..34fa634b5 100644 --- a/engine/utils/curl_utils.h +++ b/engine/utils/curl_utils.h @@ -23,6 +23,43 @@ size_t WriteCallback(void* contents, size_t size, size_t nmemb, output->append((char*)contents, totalSize); return totalSize; } + +void SetUpProxy(CURL* handle) { + auto config = file_manager_utils::GetCortexConfig(); + if (!config.proxyUrl.empty()) { + auto proxy_url = config.proxyUrl; + auto verify_proxy_ssl = config.verifyProxySsl; + auto verify_proxy_host_ssl = config.verifyProxyHostSsl; + + auto verify_ssl = config.verifyPeerSsl; + auto verify_host_ssl = config.verifyHostSsl; + + auto proxy_username = config.proxyUsername; + auto proxy_password = config.proxyPassword; + auto no_proxy = config.noProxy; + + CTL_INF("=== Proxy configuration ==="); + CTL_INF("Proxy url: " << proxy_url); + CTL_INF("Verify proxy ssl: " << verify_proxy_ssl); + CTL_INF("Verify proxy host ssl: " << verify_proxy_host_ssl); + CTL_INF("Verify ssl: " << verify_ssl); + CTL_INF("Verify host ssl: " << verify_host_ssl); + + curl_easy_setopt(handle, CURLOPT_PROXY, proxy_url.c_str()); + curl_easy_setopt(handle, CURLOPT_SSL_VERIFYPEER, verify_ssl ? 1L : 0L); + curl_easy_setopt(handle, CURLOPT_SSL_VERIFYHOST, verify_host_ssl ? 2L : 0L); + + curl_easy_setopt(handle, CURLOPT_PROXY_SSL_VERIFYPEER, + verify_proxy_ssl ? 1L : 0L); + curl_easy_setopt(handle, CURLOPT_PROXY_SSL_VERIFYHOST, + verify_proxy_host_ssl ? 2L : 0L); + + auto proxy_auth = proxy_username + ":" + proxy_password; + curl_easy_setopt(handle, CURLOPT_PROXYUSERPWD, proxy_auth.c_str()); + + curl_easy_setopt(handle, CURLOPT_NOPROXY, no_proxy.c_str()); + } +} } // namespace inline std::optional> GetHeaders( @@ -51,6 +88,7 @@ inline cpp::result SimpleGet(const std::string& url, std::string readBuffer; + SetUpProxy(curl); curl_easy_setopt(curl, CURLOPT_URL, url.c_str()); curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, WriteCallback); curl_easy_setopt(curl, CURLOPT_WRITEDATA, &readBuffer); @@ -101,6 +139,7 @@ inline cpp::result SimpleRequest( } std::string readBuffer; + SetUpProxy(curl); curl_easy_setopt(curl, CURLOPT_HTTPHEADER, curl_headers); curl_easy_setopt(curl, CURLOPT_URL, url.c_str()); if (request_type == RequestType::PATCH) { @@ -111,8 +150,6 @@ inline cpp::result SimpleRequest( } else if (request_type == RequestType::DEL) { curl_easy_setopt(curl, CURLOPT_CUSTOMREQUEST, "DELETE"); } - // enable below line for debugging - // curl_easy_setopt(curl, CURLOPT_VERBOSE, 1L); curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L); curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, WriteCallback); curl_easy_setopt(curl, CURLOPT_WRITEDATA, &readBuffer); diff --git a/engine/utils/file_manager_utils.h b/engine/utils/file_manager_utils.h index a1403437b..73f92a10a 100644 --- a/engine/utils/file_manager_utils.h +++ b/engine/utils/file_manager_utils.h @@ -150,29 +150,18 @@ inline cpp::result UpdateCortexConfig( return DumpYamlConfig(config, config_path.string()); } -inline cpp::result CreateConfigFileIfNotExist() { +inline config_yaml_utils::CortexConfig GetDefaultConfig() { auto config_path = GetConfigurationPath(); - if (std::filesystem::exists(config_path)) { - // already exists, no need to create - return {}; - } - auto default_data_folder_name = GetDefaultDataFolderName(); + auto default_data_folder_path = + file_manager_utils::GetHomeDirectoryPath() / default_data_folder_name; - CLI_LOG("Config file not found. Creating one at " + config_path.string()); - auto defaultDataFolderPath = - cortex_data_folder_path.empty() - ? file_manager_utils::GetHomeDirectoryPath() / - default_data_folder_name - : std::filesystem::path(cortex_data_folder_path); - CLI_LOG("Default data folder path: " + defaultDataFolderPath.string()); - - auto config = config_yaml_utils::CortexConfig{ - .logFolderPath = defaultDataFolderPath.string(), + return config_yaml_utils::CortexConfig{ + .logFolderPath = default_data_folder_path.string(), .logLlamaCppPath = kLogsLlamacppBaseName, .logTensorrtLLMPath = kLogsTensorrtllmBaseName, .logOnnxPath = kLogsOnnxBaseName, - .dataFolderPath = defaultDataFolderPath.string(), + .dataFolderPath = default_data_folder_path.string(), .maxLogLines = config_yaml_utils::kDefaultMaxLines, .apiServerHost = config_yaml_utils::kDefaultHost, .apiServerPort = config_yaml_utils::kDefaultPort, @@ -182,7 +171,30 @@ inline cpp::result CreateConfigFileIfNotExist() { .latestRelease = config_yaml_utils::kDefaultLatestRelease, .latestLlamacppRelease = config_yaml_utils::kDefaultLatestLlamacppRelease, .enableCors = config_yaml_utils::kDefaultCorsEnabled, - .allowedOrigins = config_yaml_utils::kDefaultEnabledOrigins}; + .allowedOrigins = config_yaml_utils::kDefaultEnabledOrigins, + .proxyUrl = "", + .verifyProxySsl = true, + .verifyProxyHostSsl = true, + .proxyUsername = "", + .proxyPassword = "", + .noProxy = config_yaml_utils::kDefaultNoProxy, + .verifyPeerSsl = true, + .verifyHostSsl = true, + }; +} + +inline cpp::result CreateConfigFileIfNotExist() { + auto config_path = GetConfigurationPath(); + if (std::filesystem::exists(config_path)) { + // already exists, no need to create + return {}; + } + + auto default_data_folder_name = GetDefaultDataFolderName(); + + CLI_LOG("Config file not found. Creating one at " + config_path.string()); + auto config = GetDefaultConfig(); + CLI_LOG("Default data folder path: " + config.dataFolderPath); return DumpYamlConfig(config, config_path.string()); } @@ -191,23 +203,8 @@ inline config_yaml_utils::CortexConfig GetCortexConfig() { auto default_data_folder_name = GetDefaultDataFolderName(); auto default_data_folder_path = file_manager_utils::GetHomeDirectoryPath() / default_data_folder_name; - auto default_cfg = config_yaml_utils::CortexConfig{ - .logFolderPath = default_data_folder_path.string(), - .logLlamaCppPath = kLogsLlamacppBaseName, - .logTensorrtLLMPath = kLogsTensorrtllmBaseName, - .logOnnxPath = kLogsOnnxBaseName, - .dataFolderPath = default_data_folder_path.string(), - .maxLogLines = config_yaml_utils::kDefaultMaxLines, - .apiServerHost = config_yaml_utils::kDefaultHost, - .apiServerPort = config_yaml_utils::kDefaultPort, - .checkedForUpdateAt = config_yaml_utils::kDefaultCheckedForUpdateAt, - .checkedForLlamacppUpdateAt = - config_yaml_utils::kDefaultCheckedForLlamacppUpdateAt, - .latestRelease = config_yaml_utils::kDefaultLatestRelease, - .latestLlamacppRelease = config_yaml_utils::kDefaultLatestLlamacppRelease, - .enableCors = config_yaml_utils::kDefaultCorsEnabled, - .allowedOrigins = config_yaml_utils::kDefaultEnabledOrigins}; + auto default_cfg = GetDefaultConfig(); return config_yaml_utils::FromYaml(config_path.string(), default_cfg); } @@ -238,6 +235,7 @@ inline std::filesystem::path GetCortexDataPath() { inline std::filesystem::path GetCortexLogPath() { // TODO: We will need to support user to move the data folder to other place. // TODO: get the variant of cortex. As discussed, we will have: prod, beta, nightly + // currently we will store cortex data at ~/cortexcpp auto config = GetCortexConfig(); std::filesystem::path log_folder_path;