From f46251fec0368d53483da1ba1742b36ea119ac4a Mon Sep 17 00:00:00 2001 From: dimapihtar Date: Wed, 3 Jul 2024 08:29:48 -0700 Subject: [PATCH 01/12] unpin transformers Signed-off-by: dimapihtar --- requirements/requirements_lightning.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements/requirements_lightning.txt b/requirements/requirements_lightning.txt index c7e67d21a693..1b3397f69033 100644 --- a/requirements/requirements_lightning.txt +++ b/requirements/requirements_lightning.txt @@ -4,6 +4,6 @@ hydra-core>1.3,<=1.3.2 omegaconf<=2.3 pytorch-lightning>2.2.1 torchmetrics>=0.11.0 -transformers>=4.36.0,<=4.40.2 +transformers wandb webdataset>=0.2.86 From 60d710a30da5e4e666fbf88b812b9cd045808822 Mon Sep 17 00:00:00 2001 From: dimapihtar Date: Wed, 3 Jul 2024 08:38:23 -0700 Subject: [PATCH 02/12] guard deprecated imports Signed-off-by: dimapihtar --- .../common/huggingface/huggingface_utils.py | 45 ++++++++++++------- 1 file changed, 30 insertions(+), 15 deletions(-) diff --git a/nemo/collections/nlp/modules/common/huggingface/huggingface_utils.py b/nemo/collections/nlp/modules/common/huggingface/huggingface_utils.py index cf692e07749d..c93b51036163 100644 --- a/nemo/collections/nlp/modules/common/huggingface/huggingface_utils.py +++ b/nemo/collections/nlp/modules/common/huggingface/huggingface_utils.py @@ -15,21 +15,36 @@ import os from typing import List, Optional -from transformers import ( - ALBERT_PRETRAINED_MODEL_ARCHIVE_LIST, - BERT_PRETRAINED_MODEL_ARCHIVE_LIST, - CAMEMBERT_PRETRAINED_MODEL_ARCHIVE_LIST, - DISTILBERT_PRETRAINED_MODEL_ARCHIVE_LIST, - GPT2_PRETRAINED_MODEL_ARCHIVE_LIST, - ROBERTA_PRETRAINED_MODEL_ARCHIVE_LIST, - AlbertConfig, - AutoModel, - BertConfig, - CamembertConfig, - DistilBertConfig, - GPT2Config, - RobertaConfig, -) +from nemo.collections.nlp.modules.common.megatron.utils import ApexGuardDefaults + +try: + from transformers import ( + ALBERT_PRETRAINED_MODEL_ARCHIVE_LIST, + BERT_PRETRAINED_MODEL_ARCHIVE_LIST, + CAMEMBERT_PRETRAINED_MODEL_ARCHIVE_LIST, + DISTILBERT_PRETRAINED_MODEL_ARCHIVE_LIST, + GPT2_PRETRAINED_MODEL_ARCHIVE_LIST, + ROBERTA_PRETRAINED_MODEL_ARCHIVE_LIST, + AlbertConfig, + AutoModel, + BertConfig, + CamembertConfig, + DistilBertConfig, + GPT2Config, + RobertaConfig, + ) + +except (ImportError, ModuleNotFoundError): + + # Guard deprecated imports + # Temporary solution + # TODO(@dimapihtar): fix current file in respect to transformers >= 4.40 + ALBERT_PRETRAINED_MODEL_ARCHIVE_LIST = ApexGuardDefaults + BERT_PRETRAINED_MODEL_ARCHIVE_LIST = ApexGuardDefaults + CAMEMBERT_PRETRAINED_MODEL_ARCHIVE_LIST = ApexGuardDefaults + DISTILBERT_PRETRAINED_MODEL_ARCHIVE_LIST = ApexGuardDefaults + GPT2_PRETRAINED_MODEL_ARCHIVE_LIST = ApexGuardDefaults + ROBERTA_PRETRAINED_MODEL_ARCHIVE_LIST = ApexGuardDefaults from nemo.collections.nlp.modules.common.huggingface.albert import AlbertEncoder from nemo.collections.nlp.modules.common.huggingface.bert import BertEncoder From ec435e53af57fbd53111c6c11caf57d517c0f245 Mon Sep 17 00:00:00 2001 From: dimapihtar Date: Wed, 3 Jul 2024 15:40:14 +0000 Subject: [PATCH 03/12] Apply isort and black reformatting Signed-off-by: dimapihtar --- .../nlp/modules/common/huggingface/huggingface_utils.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/nemo/collections/nlp/modules/common/huggingface/huggingface_utils.py b/nemo/collections/nlp/modules/common/huggingface/huggingface_utils.py index c93b51036163..093fff335e44 100644 --- a/nemo/collections/nlp/modules/common/huggingface/huggingface_utils.py +++ b/nemo/collections/nlp/modules/common/huggingface/huggingface_utils.py @@ -109,7 +109,9 @@ def get_huggingface_lm_model( - pretrained_model_name: str, config_dict: Optional[dict] = None, config_file: Optional[str] = None, + pretrained_model_name: str, + config_dict: Optional[dict] = None, + config_file: Optional[str] = None, ): """ Returns lm model instantiated with Huggingface @@ -150,7 +152,9 @@ def get_huggingface_lm_model( raise ValueError(f"Use HuggingFace API directly in NeMo for {pretrained_model_name}") -def get_huggingface_pretrained_lm_models_list(include_external: bool = False,) -> List[str]: +def get_huggingface_pretrained_lm_models_list( + include_external: bool = False, +) -> List[str]: """ Returns the list of pretrained HuggingFace language models From 59ed7f544e949fd253ef71b53e20b411cd958c89 Mon Sep 17 00:00:00 2001 From: dimapihtar Date: Wed, 3 Jul 2024 11:36:28 -0700 Subject: [PATCH 04/12] fix import guards Signed-off-by: dimapihtar --- .../nlp/modules/common/huggingface/huggingface_utils.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/nemo/collections/nlp/modules/common/huggingface/huggingface_utils.py b/nemo/collections/nlp/modules/common/huggingface/huggingface_utils.py index 093fff335e44..16a8b83f23bb 100644 --- a/nemo/collections/nlp/modules/common/huggingface/huggingface_utils.py +++ b/nemo/collections/nlp/modules/common/huggingface/huggingface_utils.py @@ -45,6 +45,13 @@ DISTILBERT_PRETRAINED_MODEL_ARCHIVE_LIST = ApexGuardDefaults GPT2_PRETRAINED_MODEL_ARCHIVE_LIST = ApexGuardDefaults ROBERTA_PRETRAINED_MODEL_ARCHIVE_LIST = ApexGuardDefaults + AlbertConfig = ApexGuardDefaults + AutoModel = ApexGuardDefaults + BertConfig = ApexGuardDefaults + CamembertConfig = ApexGuardDefaults + DistilBertConfig = ApexGuardDefaults + GPT2Config = ApexGuardDefaults + RobertaConfig = ApexGuardDefaults from nemo.collections.nlp.modules.common.huggingface.albert import AlbertEncoder from nemo.collections.nlp.modules.common.huggingface.bert import BertEncoder From 6643ebb62bc6473526fbe65d1b6014c1b4150faf Mon Sep 17 00:00:00 2001 From: dimapihtar Date: Wed, 3 Jul 2024 11:38:07 -0700 Subject: [PATCH 05/12] fix import guards Signed-off-by: dimapihtar --- .../common/huggingface/huggingface_utils.py | 24 ++++++++----------- 1 file changed, 10 insertions(+), 14 deletions(-) diff --git a/nemo/collections/nlp/modules/common/huggingface/huggingface_utils.py b/nemo/collections/nlp/modules/common/huggingface/huggingface_utils.py index 16a8b83f23bb..9ad6a8056e58 100644 --- a/nemo/collections/nlp/modules/common/huggingface/huggingface_utils.py +++ b/nemo/collections/nlp/modules/common/huggingface/huggingface_utils.py @@ -17,6 +17,16 @@ from nemo.collections.nlp.modules.common.megatron.utils import ApexGuardDefaults +from transformers import ( + AlbertConfig, + AutoModel, + BertConfig, + CamembertConfig, + DistilBertConfig, + GPT2Config, + RobertaConfig, +) + try: from transformers import ( ALBERT_PRETRAINED_MODEL_ARCHIVE_LIST, @@ -25,13 +35,6 @@ DISTILBERT_PRETRAINED_MODEL_ARCHIVE_LIST, GPT2_PRETRAINED_MODEL_ARCHIVE_LIST, ROBERTA_PRETRAINED_MODEL_ARCHIVE_LIST, - AlbertConfig, - AutoModel, - BertConfig, - CamembertConfig, - DistilBertConfig, - GPT2Config, - RobertaConfig, ) except (ImportError, ModuleNotFoundError): @@ -45,13 +48,6 @@ DISTILBERT_PRETRAINED_MODEL_ARCHIVE_LIST = ApexGuardDefaults GPT2_PRETRAINED_MODEL_ARCHIVE_LIST = ApexGuardDefaults ROBERTA_PRETRAINED_MODEL_ARCHIVE_LIST = ApexGuardDefaults - AlbertConfig = ApexGuardDefaults - AutoModel = ApexGuardDefaults - BertConfig = ApexGuardDefaults - CamembertConfig = ApexGuardDefaults - DistilBertConfig = ApexGuardDefaults - GPT2Config = ApexGuardDefaults - RobertaConfig = ApexGuardDefaults from nemo.collections.nlp.modules.common.huggingface.albert import AlbertEncoder from nemo.collections.nlp.modules.common.huggingface.bert import BertEncoder From 5d8971142576ff916acae9169a4ba423715db9d1 Mon Sep 17 00:00:00 2001 From: dimapihtar Date: Wed, 3 Jul 2024 18:39:10 +0000 Subject: [PATCH 06/12] Apply isort and black reformatting Signed-off-by: dimapihtar --- .../nlp/modules/common/huggingface/huggingface_utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nemo/collections/nlp/modules/common/huggingface/huggingface_utils.py b/nemo/collections/nlp/modules/common/huggingface/huggingface_utils.py index 9ad6a8056e58..45dbf1051537 100644 --- a/nemo/collections/nlp/modules/common/huggingface/huggingface_utils.py +++ b/nemo/collections/nlp/modules/common/huggingface/huggingface_utils.py @@ -15,8 +15,6 @@ import os from typing import List, Optional -from nemo.collections.nlp.modules.common.megatron.utils import ApexGuardDefaults - from transformers import ( AlbertConfig, AutoModel, @@ -27,6 +25,8 @@ RobertaConfig, ) +from nemo.collections.nlp.modules.common.megatron.utils import ApexGuardDefaults + try: from transformers import ( ALBERT_PRETRAINED_MODEL_ARCHIVE_LIST, From edf3971022215ab65354d1ea68c7e3ea7c396653 Mon Sep 17 00:00:00 2001 From: Chen Cui Date: Fri, 5 Jul 2024 10:10:26 -0700 Subject: [PATCH 07/12] try fixing Signed-off-by: Chen Cui --- .../modules/common/huggingface/huggingface_utils.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/nemo/collections/nlp/modules/common/huggingface/huggingface_utils.py b/nemo/collections/nlp/modules/common/huggingface/huggingface_utils.py index 45dbf1051537..9f637c6e54b1 100644 --- a/nemo/collections/nlp/modules/common/huggingface/huggingface_utils.py +++ b/nemo/collections/nlp/modules/common/huggingface/huggingface_utils.py @@ -166,9 +166,9 @@ def get_huggingface_pretrained_lm_models_list( Returns the list of HuggingFace models """ - - huggingface_models = [] - for model in HUGGINGFACE_MODELS: - model_names = HUGGINGFACE_MODELS[model]["pretrained_model_list"] - huggingface_models.extend(model_names) - return huggingface_models + logging.warning( + "pretrained_model_list no longer exists in HF transformers v4.40 and later. " + "See https://github.com/huggingface/transformers/pull/29112 for details. " + "`get_huggingface_pretrained_lm_models_list()` will return an empty list." + ) + return [] From fc23a5784313f4cce051e014484eaeba0f303f87 Mon Sep 17 00:00:00 2001 From: Dmytro Pykhtar Date: Fri, 5 Jul 2024 12:29:25 -0700 Subject: [PATCH 08/12] disable HF tests Signed-off-by: Dmytro Pykhtar --- tests/collections/nlp/{test_huggingface.py => huggingface.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename tests/collections/nlp/{test_huggingface.py => huggingface.py} (100%) diff --git a/tests/collections/nlp/test_huggingface.py b/tests/collections/nlp/huggingface.py similarity index 100% rename from tests/collections/nlp/test_huggingface.py rename to tests/collections/nlp/huggingface.py From 0efb5d5a612276e4c341ef9aa35e56055ac25091 Mon Sep 17 00:00:00 2001 From: Chen Cui Date: Mon, 8 Jul 2024 15:50:04 -0700 Subject: [PATCH 09/12] try fixing Signed-off-by: Chen Cui --- .../common/huggingface/huggingface_utils.py | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/nemo/collections/nlp/modules/common/huggingface/huggingface_utils.py b/nemo/collections/nlp/modules/common/huggingface/huggingface_utils.py index 9f637c6e54b1..9b53162c7a9e 100644 --- a/nemo/collections/nlp/modules/common/huggingface/huggingface_utils.py +++ b/nemo/collections/nlp/modules/common/huggingface/huggingface_utils.py @@ -25,8 +25,6 @@ RobertaConfig, ) -from nemo.collections.nlp.modules.common.megatron.utils import ApexGuardDefaults - try: from transformers import ( ALBERT_PRETRAINED_MODEL_ARCHIVE_LIST, @@ -42,12 +40,12 @@ # Guard deprecated imports # Temporary solution # TODO(@dimapihtar): fix current file in respect to transformers >= 4.40 - ALBERT_PRETRAINED_MODEL_ARCHIVE_LIST = ApexGuardDefaults - BERT_PRETRAINED_MODEL_ARCHIVE_LIST = ApexGuardDefaults - CAMEMBERT_PRETRAINED_MODEL_ARCHIVE_LIST = ApexGuardDefaults - DISTILBERT_PRETRAINED_MODEL_ARCHIVE_LIST = ApexGuardDefaults - GPT2_PRETRAINED_MODEL_ARCHIVE_LIST = ApexGuardDefaults - ROBERTA_PRETRAINED_MODEL_ARCHIVE_LIST = ApexGuardDefaults + ALBERT_PRETRAINED_MODEL_ARCHIVE_LIST = None + BERT_PRETRAINED_MODEL_ARCHIVE_LIST = None + CAMEMBERT_PRETRAINED_MODEL_ARCHIVE_LIST = None + DISTILBERT_PRETRAINED_MODEL_ARCHIVE_LIST = None + GPT2_PRETRAINED_MODEL_ARCHIVE_LIST = None + ROBERTA_PRETRAINED_MODEL_ARCHIVE_LIST = None from nemo.collections.nlp.modules.common.huggingface.albert import AlbertEncoder from nemo.collections.nlp.modules.common.huggingface.bert import BertEncoder From a33a9477215dea84268de9f1d5ac219b5c9c2f38 Mon Sep 17 00:00:00 2001 From: Chen Cui Date: Tue, 9 Jul 2024 16:27:57 -0700 Subject: [PATCH 10/12] hard code model lists Signed-off-by: Chen Cui --- .../common/huggingface/huggingface_utils.py | 86 ++++++++++++++----- .../{huggingface.py => test_huggingface.py} | 0 2 files changed, 64 insertions(+), 22 deletions(-) rename tests/collections/nlp/{huggingface.py => test_huggingface.py} (100%) diff --git a/nemo/collections/nlp/modules/common/huggingface/huggingface_utils.py b/nemo/collections/nlp/modules/common/huggingface/huggingface_utils.py index 9b53162c7a9e..c178528dbdf0 100644 --- a/nemo/collections/nlp/modules/common/huggingface/huggingface_utils.py +++ b/nemo/collections/nlp/modules/common/huggingface/huggingface_utils.py @@ -25,28 +25,6 @@ RobertaConfig, ) -try: - from transformers import ( - ALBERT_PRETRAINED_MODEL_ARCHIVE_LIST, - BERT_PRETRAINED_MODEL_ARCHIVE_LIST, - CAMEMBERT_PRETRAINED_MODEL_ARCHIVE_LIST, - DISTILBERT_PRETRAINED_MODEL_ARCHIVE_LIST, - GPT2_PRETRAINED_MODEL_ARCHIVE_LIST, - ROBERTA_PRETRAINED_MODEL_ARCHIVE_LIST, - ) - -except (ImportError, ModuleNotFoundError): - - # Guard deprecated imports - # Temporary solution - # TODO(@dimapihtar): fix current file in respect to transformers >= 4.40 - ALBERT_PRETRAINED_MODEL_ARCHIVE_LIST = None - BERT_PRETRAINED_MODEL_ARCHIVE_LIST = None - CAMEMBERT_PRETRAINED_MODEL_ARCHIVE_LIST = None - DISTILBERT_PRETRAINED_MODEL_ARCHIVE_LIST = None - GPT2_PRETRAINED_MODEL_ARCHIVE_LIST = None - ROBERTA_PRETRAINED_MODEL_ARCHIVE_LIST = None - from nemo.collections.nlp.modules.common.huggingface.albert import AlbertEncoder from nemo.collections.nlp.modules.common.huggingface.bert import BertEncoder from nemo.collections.nlp.modules.common.huggingface.camembert import CamembertEncoder @@ -57,6 +35,70 @@ __all__ = ["get_huggingface_lm_model", "get_huggingface_pretrained_lm_models_list", "VOCAB_FILE_NAME"] +# Manually specify the model archive lists since these are now removed in HF +# https://github.com/huggingface/transformers/blob/v4.40-release/src/transformers/models/deprecated/_archive_maps.py +ALBERT_PRETRAINED_MODEL_ARCHIVE_LIST = [ + "albert/albert-base-v1", + "albert/albert-large-v1", + "albert/albert-xlarge-v1", + "albert/albert-xxlarge-v1", + "albert/albert-base-v2", + "albert/albert-large-v2", + "albert/albert-xlarge-v2", + "albert/albert-xxlarge-v2", + ] + +BERT_PRETRAINED_MODEL_ARCHIVE_LIST = [ + "google-bert/bert-base-uncased", + "google-bert/bert-large-uncased", + "google-bert/bert-base-cased", + "google-bert/bert-large-cased", + "google-bert/bert-base-multilingual-uncased", + "google-bert/bert-base-multilingual-cased", + "google-bert/bert-base-chinese", + "google-bert/bert-base-german-cased", + "google-bert/bert-large-uncased-whole-word-masking", + "google-bert/bert-large-cased-whole-word-masking", + "google-bert/bert-large-uncased-whole-word-masking-finetuned-squad", + "google-bert/bert-large-cased-whole-word-masking-finetuned-squad", + "google-bert/bert-base-cased-finetuned-mrpc", + "google-bert/bert-base-german-dbmdz-cased", + "google-bert/bert-base-german-dbmdz-uncased", + "cl-tohoku/bert-base-japanese", + "cl-tohoku/bert-base-japanese-whole-word-masking", + "cl-tohoku/bert-base-japanese-char", + "cl-tohoku/bert-base-japanese-char-whole-word-masking", + "TurkuNLP/bert-base-finnish-cased-v1", + "TurkuNLP/bert-base-finnish-uncased-v1", + "wietsedv/bert-base-dutch-cased", + ] +CAMEMBERT_PRETRAINED_MODEL_ARCHIVE_LIST = ["almanach/camembert-base", "Musixmatch/umberto-commoncrawl-cased-v1", "Musixmatch/umberto-wikipedia-uncased-v1"] + +DISTILBERT_PRETRAINED_MODEL_ARCHIVE_LIST = [ + "distilbert-base-uncased", + "distilbert-base-uncased-distilled-squad", + "distilbert-base-cased", + "distilbert-base-cased-distilled-squad", + "distilbert-base-german-cased", + "distilbert-base-multilingual-cased", + "distilbert-base-uncased-finetuned-sst-2-english", + ] +GPT2_PRETRAINED_MODEL_ARCHIVE_LIST = [ + "openai-community/gpt2", + "openai-community/gpt2-medium", + "openai-community/gpt2-large", + "openai-community/gpt2-xl", + "distilbert/distilgpt2", + ] +ROBERTA_PRETRAINED_MODEL_ARCHIVE_LIST = [ + "FacebookAI/roberta-base", + "FacebookAI/roberta-large", + "FacebookAI/roberta-large-mnli", + "distilbert/distilroberta-base", + "openai-community/roberta-base-openai-detector", + "openai-community/roberta-large-openai-detector", + ] + HUGGINGFACE_MODELS = { "BertModel": { diff --git a/tests/collections/nlp/huggingface.py b/tests/collections/nlp/test_huggingface.py similarity index 100% rename from tests/collections/nlp/huggingface.py rename to tests/collections/nlp/test_huggingface.py From cfa1041a0aec9ec81b1e7dbab0348de8b75b0978 Mon Sep 17 00:00:00 2001 From: cuichenx Date: Tue, 9 Jul 2024 23:29:50 +0000 Subject: [PATCH 11/12] Apply isort and black reformatting Signed-off-by: cuichenx --- .../common/huggingface/huggingface_utils.py | 112 +++++++++--------- 1 file changed, 58 insertions(+), 54 deletions(-) diff --git a/nemo/collections/nlp/modules/common/huggingface/huggingface_utils.py b/nemo/collections/nlp/modules/common/huggingface/huggingface_utils.py index c178528dbdf0..ef6e34268a66 100644 --- a/nemo/collections/nlp/modules/common/huggingface/huggingface_utils.py +++ b/nemo/collections/nlp/modules/common/huggingface/huggingface_utils.py @@ -38,66 +38,70 @@ # Manually specify the model archive lists since these are now removed in HF # https://github.com/huggingface/transformers/blob/v4.40-release/src/transformers/models/deprecated/_archive_maps.py ALBERT_PRETRAINED_MODEL_ARCHIVE_LIST = [ - "albert/albert-base-v1", - "albert/albert-large-v1", - "albert/albert-xlarge-v1", - "albert/albert-xxlarge-v1", - "albert/albert-base-v2", - "albert/albert-large-v2", - "albert/albert-xlarge-v2", - "albert/albert-xxlarge-v2", - ] + "albert/albert-base-v1", + "albert/albert-large-v1", + "albert/albert-xlarge-v1", + "albert/albert-xxlarge-v1", + "albert/albert-base-v2", + "albert/albert-large-v2", + "albert/albert-xlarge-v2", + "albert/albert-xxlarge-v2", +] BERT_PRETRAINED_MODEL_ARCHIVE_LIST = [ - "google-bert/bert-base-uncased", - "google-bert/bert-large-uncased", - "google-bert/bert-base-cased", - "google-bert/bert-large-cased", - "google-bert/bert-base-multilingual-uncased", - "google-bert/bert-base-multilingual-cased", - "google-bert/bert-base-chinese", - "google-bert/bert-base-german-cased", - "google-bert/bert-large-uncased-whole-word-masking", - "google-bert/bert-large-cased-whole-word-masking", - "google-bert/bert-large-uncased-whole-word-masking-finetuned-squad", - "google-bert/bert-large-cased-whole-word-masking-finetuned-squad", - "google-bert/bert-base-cased-finetuned-mrpc", - "google-bert/bert-base-german-dbmdz-cased", - "google-bert/bert-base-german-dbmdz-uncased", - "cl-tohoku/bert-base-japanese", - "cl-tohoku/bert-base-japanese-whole-word-masking", - "cl-tohoku/bert-base-japanese-char", - "cl-tohoku/bert-base-japanese-char-whole-word-masking", - "TurkuNLP/bert-base-finnish-cased-v1", - "TurkuNLP/bert-base-finnish-uncased-v1", - "wietsedv/bert-base-dutch-cased", - ] -CAMEMBERT_PRETRAINED_MODEL_ARCHIVE_LIST = ["almanach/camembert-base", "Musixmatch/umberto-commoncrawl-cased-v1", "Musixmatch/umberto-wikipedia-uncased-v1"] + "google-bert/bert-base-uncased", + "google-bert/bert-large-uncased", + "google-bert/bert-base-cased", + "google-bert/bert-large-cased", + "google-bert/bert-base-multilingual-uncased", + "google-bert/bert-base-multilingual-cased", + "google-bert/bert-base-chinese", + "google-bert/bert-base-german-cased", + "google-bert/bert-large-uncased-whole-word-masking", + "google-bert/bert-large-cased-whole-word-masking", + "google-bert/bert-large-uncased-whole-word-masking-finetuned-squad", + "google-bert/bert-large-cased-whole-word-masking-finetuned-squad", + "google-bert/bert-base-cased-finetuned-mrpc", + "google-bert/bert-base-german-dbmdz-cased", + "google-bert/bert-base-german-dbmdz-uncased", + "cl-tohoku/bert-base-japanese", + "cl-tohoku/bert-base-japanese-whole-word-masking", + "cl-tohoku/bert-base-japanese-char", + "cl-tohoku/bert-base-japanese-char-whole-word-masking", + "TurkuNLP/bert-base-finnish-cased-v1", + "TurkuNLP/bert-base-finnish-uncased-v1", + "wietsedv/bert-base-dutch-cased", +] +CAMEMBERT_PRETRAINED_MODEL_ARCHIVE_LIST = [ + "almanach/camembert-base", + "Musixmatch/umberto-commoncrawl-cased-v1", + "Musixmatch/umberto-wikipedia-uncased-v1", +] DISTILBERT_PRETRAINED_MODEL_ARCHIVE_LIST = [ - "distilbert-base-uncased", - "distilbert-base-uncased-distilled-squad", - "distilbert-base-cased", - "distilbert-base-cased-distilled-squad", - "distilbert-base-german-cased", - "distilbert-base-multilingual-cased", - "distilbert-base-uncased-finetuned-sst-2-english", - ] + "distilbert-base-uncased", + "distilbert-base-uncased-distilled-squad", + "distilbert-base-cased", + "distilbert-base-cased-distilled-squad", + "distilbert-base-german-cased", + "distilbert-base-multilingual-cased", + "distilbert-base-uncased-finetuned-sst-2-english", +] GPT2_PRETRAINED_MODEL_ARCHIVE_LIST = [ - "openai-community/gpt2", - "openai-community/gpt2-medium", - "openai-community/gpt2-large", - "openai-community/gpt2-xl", - "distilbert/distilgpt2", - ] + "openai-community/gpt2", + "openai-community/gpt2-medium", + "openai-community/gpt2-large", + "openai-community/gpt2-xl", + "distilbert/distilgpt2", +] ROBERTA_PRETRAINED_MODEL_ARCHIVE_LIST = [ - "FacebookAI/roberta-base", - "FacebookAI/roberta-large", - "FacebookAI/roberta-large-mnli", - "distilbert/distilroberta-base", - "openai-community/roberta-base-openai-detector", - "openai-community/roberta-large-openai-detector", - ] + "FacebookAI/roberta-base", + "FacebookAI/roberta-large", + "FacebookAI/roberta-large-mnli", + "distilbert/distilroberta-base", + "openai-community/roberta-base-openai-detector", + "openai-community/roberta-large-openai-detector", +] HUGGINGFACE_MODELS = { From dbea3c10b43d3a9a89cc254854cc5d3ab9dea419 Mon Sep 17 00:00:00 2001 From: Chen Cui Date: Tue, 9 Jul 2024 16:30:20 -0700 Subject: [PATCH 12/12] hard code model lists Signed-off-by: Chen Cui --- .../modules/common/huggingface/huggingface_utils.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/nemo/collections/nlp/modules/common/huggingface/huggingface_utils.py b/nemo/collections/nlp/modules/common/huggingface/huggingface_utils.py index ef6e34268a66..d8f6936f7126 100644 --- a/nemo/collections/nlp/modules/common/huggingface/huggingface_utils.py +++ b/nemo/collections/nlp/modules/common/huggingface/huggingface_utils.py @@ -210,9 +210,9 @@ def get_huggingface_pretrained_lm_models_list( Returns the list of HuggingFace models """ - logging.warning( - "pretrained_model_list no longer exists in HF transformers v4.40 and later. " - "See https://github.com/huggingface/transformers/pull/29112 for details. " - "`get_huggingface_pretrained_lm_models_list()` will return an empty list." - ) - return [] + + huggingface_models = [] + for model in HUGGINGFACE_MODELS: + model_names = HUGGINGFACE_MODELS[model]["pretrained_model_list"] + huggingface_models.extend(model_names) + return huggingface_models