Update falcon tokenizer (#344)

* Update generate_tests.py * Do not add token types for `FalconTokenizer`
huggingface · Oct 3, 2023 · b6bd608 · b6bd608
1 parent 5b31129
commit b6bd608
Show file tree

Hide file tree

Showing 2 changed files with 2 additions and 7 deletions.
diff --git a/src/tokenizers.js b/src/tokenizers.js
@@ -2705,12 +2705,7 @@ export class CodeLlamaTokenizer extends PreTrainedTokenizer { }
 export class XLMRobertaTokenizer extends PreTrainedTokenizer { }
 export class MPNetTokenizer extends PreTrainedTokenizer { }
 
-export class FalconTokenizer extends PreTrainedTokenizer {
-    /** @type {add_token_types} */
-    prepare_model_inputs(inputs) {
-        return add_token_types(inputs);
-    }
-}
+export class FalconTokenizer extends PreTrainedTokenizer { }
 
 export class GPTNeoXTokenizer extends PreTrainedTokenizer { }
 

diff --git a/tests/generate_tests.py b/tests/generate_tests.py
@@ -10,7 +10,7 @@
 
 # List of tokenizers where the model isn't yet supported, but the tokenizer is
 ADDITIONAL_TOKENIZERS_TO_TEST = {
-    'RefinedWebModel': [
+    'falcon': [
         'tiiuae/falcon-7b',
     ],
     "llama": [