From 8189be31ecf196ed85b629c2b562f7ee41656847 Mon Sep 17 00:00:00 2001 From: Vladimir Blagojevic Date: Mon, 11 Nov 2024 11:15:29 +0100 Subject: [PATCH] New Anthropic tokenizer.json URL --- haystack/nodes/prompt/invocation_layer/anthropic_claude.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/haystack/nodes/prompt/invocation_layer/anthropic_claude.py b/haystack/nodes/prompt/invocation_layer/anthropic_claude.py index edd1c6c28c..caeb1db276 100644 --- a/haystack/nodes/prompt/invocation_layer/anthropic_claude.py +++ b/haystack/nodes/prompt/invocation_layer/anthropic_claude.py @@ -22,8 +22,11 @@ # Taken from: # https://github.com/anthropics/anthropic-sdk-python/blob/main/anthropic/tokenizer.py#L7 # This is a JSON config to load the tokenizer used for Anthropic Claude. +# Anthropic removed tokenizer.json from their repo (https://github.com/anthropics/anthropic-sdk-python/pull/726), +# we need to use the commit from the latest version of the SDK that still +# has it, i.e. 0.38.0 and commit hash 14afc93ffd809e60666a267763a57a328184c5e4. CLAUDE_TOKENIZER_REMOTE_FILE = ( - "https://raw.githubusercontent.com/anthropics/anthropic-sdk-python/main/src/anthropic/tokenizer.json" + "https://raw.githubusercontent.com/anthropics/anthropic-sdk-python/14afc93ffd809e60666a267763a57a328184c5e4/src/anthropic/tokenizer.json" )