New Anthropic tokenizer.json URL

deepset-ai · Nov 11, 2024 · 8189be3 · 8189be3
1 parent 1bc3940
commit 8189be3
Showing 1 changed file with 4 additions and 1 deletion.
diff --git a/haystack/nodes/prompt/invocation_layer/anthropic_claude.py b/haystack/nodes/prompt/invocation_layer/anthropic_claude.py
@@ -22,8 +22,11 @@
 # Taken from:
 # https://github.com/anthropics/anthropic-sdk-python/blob/main/anthropic/tokenizer.py#L7
 # This is a JSON config to load the tokenizer used for Anthropic Claude.
+# Anthropic removed tokenizer.json from their repo (https://github.com/anthropics/anthropic-sdk-python/pull/726),
+# we need to use the commit from the latest version of the SDK that still
+# has it, i.e. 0.38.0 and commit hash 14afc93ffd809e60666a267763a57a328184c5e4.
 CLAUDE_TOKENIZER_REMOTE_FILE = (
-    "https://raw.githubusercontent.com/anthropics/anthropic-sdk-python/main/src/anthropic/tokenizer.json"
+    "https://raw.githubusercontent.com/anthropics/anthropic-sdk-python/14afc93ffd809e60666a267763a57a328184c5e4/src/anthropic/tokenizer.json"
 )