Merge branch 'main' into speecht5

huggingface · Oct 4, 2023 · 9dbe086 · 9dbe086
2 parents 93b697a + b6bd608
commit 9dbe086
Show file tree

Hide file tree

Showing 2 changed files with 2 additions and 7 deletions.
diff --git a/src/tokenizers.js b/src/tokenizers.js
@@ -2705,12 +2705,7 @@ export class CodeLlamaTokenizer extends PreTrainedTokenizer { }
 export class XLMRobertaTokenizer extends PreTrainedTokenizer { }
 export class MPNetTokenizer extends PreTrainedTokenizer { }
 
-export class FalconTokenizer extends PreTrainedTokenizer {
-    /** @type {add_token_types} */
-    prepare_model_inputs(inputs) {
-        return add_token_types(inputs);
-    }
-}
+export class FalconTokenizer extends PreTrainedTokenizer { }
 
 export class GPTNeoXTokenizer extends PreTrainedTokenizer { }
 

diff --git a/tests/generate_tests.py b/tests/generate_tests.py
@@ -10,7 +10,7 @@
 
 # List of tokenizers where the model isn't yet supported, but the tokenizer is
 ADDITIONAL_TOKENIZERS_TO_TEST = {
-    'RefinedWebModel': [
+    'falcon': [
         'tiiuae/falcon-7b',
     ],
     "llama": [