Pin mcore to 0.3 (#7751)

* pin mcore to 0.3 Signed-off-by: eharper <[email protected]> * upper bound tensorstore Signed-off-by: eharper <[email protected]> * upper bound opencc Signed-off-by: eharper <[email protected]> --------- Signed-off-by: eharper <[email protected]>
NVIDIA · Oct 19, 2023 · be47a3d · be47a3d
1 parent f169e83
commit be47a3d
Show file tree

Hide file tree

Showing 2 changed files with 14 additions and 13 deletions.
diff --git a/Jenkinsfile b/Jenkinsfile
@@ -57,16 +57,17 @@ pipeline {
       }
     }
 
-    stage('Megatron Core installation') {
-      steps {
-        // pinned MCore https://github.com/NVIDIA/Megatron-LM/commit/ab0336a5c8eab77aa74ae604ba1e73decbf6d560
-        // ToT for 23.08 branch
-        sh 'git clone https://github.com/NVIDIA/Megatron-LM.git && \
-            cd Megatron-LM && \
-            git checkout ab0336a5c8eab77aa74ae604ba1e73decbf6d560 && \
-            pip install -e .'
-      }
-    }
+    // megatron-core 0.3 has been pinned in the requirements, this should not be needed on r1.21.0
+    // stage('Megatron Core installation') {
+    //   steps {
+    //     // pinned MCore https://github.com/NVIDIA/Megatron-LM/commit/ab0336a5c8eab77aa74ae604ba1e73decbf6d560
+    //     // ToT for 23.08 branch
+    //     sh 'git clone https://github.com/NVIDIA/Megatron-LM.git && \
+    //         cd Megatron-LM && \
+    //         git checkout ab0336a5c8eab77aa74ae604ba1e73decbf6d560 && \
+    //         pip install -e .'
+    //   }
+    // }
 
 
     stage('PyTorch Lightning version') {

diff --git a/requirements/requirements_nlp.txt b/requirements/requirements_nlp.txt
@@ -11,13 +11,13 @@ ijson
 jieba
 markdown2
 matplotlib>=3.3.2
-megatron_core==0.2.0
+megatron_core==0.3.0
 nltk>=3.6.5
-opencc
+opencc<1.1.7
 pangu
 rapidfuzz
 rouge_score
 sacrebleu  # manually install sacrebleu[ja] for Japanese support; MeCab is unsupported in Python 3.11+
 sentence_transformers
-tensorstore
+tensorstore<0.1.46
 zarr