Merge branch 'main' of github.com:NVIDIA/NeMo into ashors/nemo-ux-dro…

…p-optim-states-async

.github/workflows/_test_template.yml

@@ -51,7 +51,7 @@ jobs:
 
         - name: Docker pull image
           run: |
-            docker pull nemoci.azurecr.io/nemo_container_${{ github.run_id }}
+            docker pull nemoci.azurecr.io/nemo_container:${{ github.run_id }}
 
         - name: Start container
           run: |
@@ -60,7 +60,16 @@ jobs:
               ARG=("--runtime=nvidia --gpus all")
             fi
 
-            docker run --rm -d --name nemo_container_${{ github.run_id }} ${ARG[@]} --shm-size=64g --env TRANSFORMERS_OFFLINE=0 --env HYDRA_FULL_ERROR=1 --volume /mnt/datadrive/TestData:/home/TestData nemoci.azurecr.io/nemo_container_${{ github.run_id }} bash -c "sleep $(( ${{ inputs.TIMEOUT }} * 60 + 60 ))"
+            docker run \
+              --rm \
+              -d \
+              --name nemo_container_${{ github.run_id }} ${ARG[@]} \
+              --shm-size=64g \
+              --env TRANSFORMERS_OFFLINE=0 \
+              --env HYDRA_FULL_ERROR=1 \
+              --env HF_HOME=/home/TestData/HF_HOME \
+              --volume /mnt/datadrive/TestData:/home/TestData nemoci.azurecr.io/nemo_container:${{ github.run_id }} \
+              bash -c "sleep $(( ${{ inputs.TIMEOUT }} * 60 + 60 ))"
 
         - id: main
           name: Run main script
@@ -95,4 +104,4 @@ jobs:
           if: always()
           run: |
             docker container stop nemo_container_${{ github.run_id }} || true
-            docker container rm nemo_container_${{ github.run_id }} || true
+            docker container rm nemo_container_${{ github.run_id }} || true

.github/workflows/build-test-publish-wheel.yml

@@ -0,0 +1,45 @@
+# Copyright (c) 2020-2021, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+name: Build, test, and publish a PyPi wheel (to testpypi)
+
+on: 
+  push:
+    branches:
+      - main
+      - 'r**'
+
+defaults:
+  run:
+    shell: bash -x -e -u -o pipefail {0}
+
+jobs:
+  build-test-publish-wheel:
+    uses: NVIDIA/NeMo-FW-CI-templates/.github/workflows/[email protected]
+    with:
+      image-name: nemo_container
+      dockerfile: Dockerfile.ci
+      image-label: nemo-core
+      build-args: |
+        IMAGE_LABEL=nemo-core
+      prune-filter-timerange: 24h
+      dry-run: true
+      python-package: nemo
+      container-workdir: /workspace
+      environment: public
+    secrets:
+      TWINE_USERNAME: ${{ secrets.TWINE_USERNAME }}
+      TWINE_PASSWORD: ${{ secrets.TWINE_PASSWORD }}
+      SLACK_WEBHOOK_ADMIN: ${{ secrets.SLACK_WEBHOOK_ADMIN }}
+      SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }}

.github/workflows/cherry-pick-release-commit.yml

@@ -6,133 +6,9 @@ on:
       - main
 
 jobs:
-  main:
-    runs-on: ubuntu-latest
-    environment: 
-      name: main
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v3
-        with:
-          fetch-depth: 0
-          token: ${{ secrets.PAT }}
-
-
-      - name: Cherry pick
-        env:
-          GH_TOKEN: ${{ secrets.PAT }}
-        run: |
-          set -x  
-          set +e
-
-          git config --global user.email "[email protected]"
-          git config --global user.name "NeMo Bot"
-          
-          SHA=$(git rev-list --no-merges -n 1 HEAD)
-          MESSAGE=$(git log -n 1 --pretty=format:%s $SHA)
-          PR_ID=$(echo $MESSAGE | awk -F'#' '{print $2}' | awk -F')' '{print $1}' )
-          USERNAME=$(git log -n 1 --pretty=format:%ae $SHA | awk -F'@' '{print $1}')
-
-          PR=$(curl -L \
-            -H "Accept: application/vnd.github+json" \
-            -H "Authorization: Bearer $GH_TOKEN" \
-            -H "X-GitHub-Api-Version: 2022-11-28" \
-            https://api.github.com/repos/NVIDIA/NeMo/pulls/$PR_ID)
-          PR_TITLE=$(echo -E $PR | jq '.title' | tr -d '"')
-          
-          LABELS=$(echo -E $PR | jq '.labels | [.[].name] | join(",")' | tr -d '"')
-          AUTHOR=$(echo -E $PR | jq '.user.login' | tr -d '"')
-
-          TARGET_BRANCHES=$(echo "$LABELS" | grep -o 'r[^,]*')
-      
-          if [[ $TARGET_BRANCHES == '' ]]; then
-            echo Nothing to cherry-pick
-            exit 0
-          fi
-
-          echo $TARGET_BRANCHES | while read -r RELEASE_BRANCH ; do
-            TARGET_BRANCH_EXISTS_OK=$([[ "$(git ls-remote --heads origin refs/heads/$RELEASE_BRANCH)" != "" ]] && echo true || echo false)
-
-            if [[ "$TARGET_BRANCH_EXISTS_OK" == "false" ]]; then
-              echo Release branch does not yet exist, will not  cherry-pick
-              continue
-            fi
-
-            (
-              git fetch origin $RELEASE_BRANCH:$RELEASE_BRANCH
-              git switch --force-create cherry-pick-$PR_ID-$RELEASE_BRANCH $RELEASE_BRANCH
-              git cherry-pick $SHA
-              git push -u origin --force cherry-pick-$PR_ID-$RELEASE_BRANCH
-              git checkout ${CI_DEFAULT_BRANCH:-main}
-            )
-
-            CHERRYPICK_SUCCESSFUL=$?
-
-            if [[ $CHERRYPICK_SUCCESSFUL -eq 0 ]]; then
-              PR_URL="https://github.com/NVIDIA/NeMo/pull/$PR_ID"
-
-              PAYLOAD=$(jq \
-                -n \
-                -c \
-                --arg TITLE "Cherry pick \`$PR_TITLE ($PR_ID)\` into \`$RELEASE_BRANCH\`" \
-                --arg HEAD "cherry-pick-$PR_ID-$RELEASE_BRANCH" \
-                --arg RELEASE_BRANCH "$RELEASE_BRANCH" \
-                --arg BODY "[🤖]: Hi @$AUTHOR 👋,<br><br>we've cherry picked #$PR_ID into \`$RELEASE_BRANCH\` for you! 🚀<br><br>Please review and approve this cherry pick by your convenience\!" \
-                '{
-                  "title": $TITLE,
-                  "head": $HEAD,
-                  "base": $RELEASE_BRANCH,
-                  "body": $BODY
-                }'
-              )
-
-              NEW_PR=$(curl -L \
-                -X POST \
-                -H "Accept: application/vnd.github+json" \
-                -H "Authorization: Bearer $GH_TOKEN" \
-                -H "X-GitHub-Api-Version: 2022-11-28" \
-                https://api.github.com/repos/NVIDIA/NeMo/pulls \
-                -d "$PAYLOAD")
-
-              NEW_PR_ID=$(echo -E $NEW_PR | jq '.number')
-              curl -L \
-                -X POST \
-                -H "Accept: application/vnd.github+json" \
-                -H "Authorization: Bearer $GH_TOKEN" \
-                -H "X-GitHub-Api-Version: 2022-11-28" \
-                https://api.github.com/repos/NVIDIA/NeMo/pulls/$NEW_PR_ID/requested_reviewers \
-                -d '{"reviewers":["'$AUTHOR'"]}'
-
-              curl -L \
-                -X POST \
-                -H "Accept: application/vnd.github+json" \
-                -H "Authorization: Bearer $GH_TOKEN" \
-                -H "X-GitHub-Api-Version: 2022-11-28" \
-                https://api.github.com/repos/NVIDIA/NeMo/issues/$NEW_PR_ID/labels \
-                -d '{"labels":["Run CICD", "cherry-pick"]}'
-
-            else
-              URL="https://github.com/NVIDIA/NeMo/pull/$PR_ID"
-
-              MESSAGE='{
-                "blocks": [
-                  {
-                    "type": "section",
-                    "text": {
-                      "type": "mrkdwn",
-                      "text": ":alert: Cherrypick bot 🤖: Hey <@'$USERNAME'>: Cherry-pick of <'$URL'|#'$PR_ID'> failed (3-way merge impossible). Please resolve manually and create a PR.\n\ncc: <!subteam^${{ secrets.SLACK_WEBHOOK_ADMIN }}>"
-                    }
-                  }
-                ]
-              }'
-
-              curl -X POST -H "Content-type: application/json" --data "$MESSAGE" ${{ secrets.SLACK_WEBHOOK }}
-
-            fi
-
-          done
-
-
-
-env:
-  GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+  cherry-pick:
+    uses: NVIDIA/NeMo-FW-CI-templates/.github/workflows/[email protected]
+    secrets:
+      PAT: ${{ secrets.PAT }}
+      SLACK_WEBHOOK_ADMIN: ${{ secrets.SLACK_WEBHOOK_ADMIN }}
+      SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }}