From 4431360d9625d5e86b6873e8e4809c44288a6d8d Mon Sep 17 00:00:00 2001 From: Arsen Ibragimov Date: Fri, 11 Oct 2024 21:56:53 +0200 Subject: [PATCH] add support for o1 models Added support for OpenAI o1-mini and o1-preview models by mapping them to the `o200k_base` encoding. This update ensures compatibility with the latest tiktoken changes as introduced in version 0.8.0 of the official library. Fixes compatibility issues with models not yet included in tiktoken-php. --- src/EncoderProvider.php | 1 + 1 file changed, 1 insertion(+) diff --git a/src/EncoderProvider.php b/src/EncoderProvider.php index 331dfdb..93776b5 100644 --- a/src/EncoderProvider.php +++ b/src/EncoderProvider.php @@ -47,6 +47,7 @@ final class EncoderProvider implements ResetInterface ], ]; private const MODEL_PREFIX_TO_ENCODING = [ + "o1-": "o200k_base", 'gpt-4o-' => 'o200k_base', 'gpt-4-' => 'cl100k_base', 'gpt-3.5-turbo-' => 'cl100k_base',