Fix typo in tokenizers.js (#518)

huggingface · Jan 14, 2024 · b07336d · b07336d
1 parent 935274f
commit b07336d
Showing 1 changed file with 1 addition and 1 deletion.
diff --git a/src/tokenizers.js b/src/tokenizers.js
@@ -2463,7 +2463,7 @@ export class PreTrainedTokenizer extends Callable {
         this.sep_token = this.getToken('sep_token');
         this.sep_token_id = this.model.tokens_to_ids.get(this.sep_token);
 
-        this.unk_token = this.getToken(tokenizerConfig, 'unk_token');
+        this.unk_token = this.getToken('unk_token');
         this.unk_token_id = this.model.tokens_to_ids.get(this.unk_token);
 
         this.model_max_length = tokenizerConfig.model_max_length;