Release 2.0.0rc1

.github/CODEOWNERS

@@ -0,0 +1,2 @@
+.github/ @pablo-garay @ko3n1g
+Dockerfile.ci @pablo-garay @ko3n1g

.github/workflows/_test_template.yml

@@ -39,18 +39,13 @@ jobs:
     outputs:
       conclusion: ${{ steps.main.conclusion }}
       log: ${{ steps.main.outputs.log }}
-    container:
-      image: nemoci.azurecr.io/nemo_container_${{ github.run_id }}
-      options: 
-        --device=/dev/nvidia0
-        --gpus all
-        --shm-size=8g
-        --env TRANSFORMERS_OFFLINE=0 
-        --env HYDRA_FULL_ERROR=1
-        --volume /mnt/datadrive/TestData:/home/TestData
+    permissions:
+      actions: write # Required for cancelling workflows 
     steps:
-        - name: Checkout repository
-          uses: actions/checkout@v4
+        - name: Docker system cleanup
+          run: |
+            docker system prune -a --filter "until=48h" --force
+
         - id: main
           name: Run main script
           timeout-minutes: ${{ inputs.TIMEOUT }}
@@ -59,7 +54,7 @@ jobs:
             (  
               set -e
 
-              ${{ inputs.SCRIPT }}
+              docker run --rm --device=/dev/nvidia0 --gpus all --shm-size=8g --env TRANSFORMERS_OFFLINE=0 --env HYDRA_FULL_ERROR=1 --volume /mnt/datadrive/TestData:/home/TestData nemoci.azurecr.io/nemo_container_${{ github.run_id }} bash -c '${{ inputs.SCRIPT }}'
             ) 2> >(tee err.log)
 
             EXIT_CODE=$?
@@ -70,6 +65,9 @@ jobs:
 
         - uses: "NVIDIA/NeMo/.github/actions/cancel-workflow@main"
           if: failure() && inputs.IS_OPTIONAL == false
+
         - name: after_script
           if: always() && inputs.AFTER_SCRIPT != ':'
-          run: ${{ inputs.AFTER_SCRIPT }}
+          run: |
+            docker run --rm --device=/dev/nvidia0 --gpus all --shm-size=8g --env TRANSFORMERS_OFFLINE=0 --env HYDRA_FULL_ERROR=1 --volume /mnt/datadrive/TestData:/home/TestData nemoci.azurecr.io/nemo_container_${{ github.run_id }} bash -c '${{ inputs.AFTER_SCRIPT }}'
+

.github/workflows/config/changelog-config.json

@@ -1,47 +1,47 @@
 {
     "categories": [
       {
-        "title": "## ASR \n\n<details><summary>Changelog</summary>\n\n</details>\n\n",
+        "title": "## ASR\n\n<details><summary>Changelog</summary>",
         "labels": ["asr"],
         "exclude_labels": ["cherry-pick"]
       },
       {
-        "title": "## TTS \n\n<details><summary>Changelog</summary>\n\n</details>\n\n",
+        "title": "</details>\n\n## TTS\n\n<details><summary>Changelog</summary>",
         "labels": ["tts"],
         "exclude_labels": ["cherry-pick"]
       },
       {
-        "title": "## NLP / NMT \n\n<details><summary>Changelog</summary>\n\n</details>\n\n",
+        "title": "</details>\n\n## NLP / NMT\n\n<details><summary>Changelog</summary>",
         "labels": ["nlp", "nmt", "megatron"],
         "exclude_labels": ["cherry-pick"]
       },
       {
-        "title": "## Text Normalization / Inverse Text Normalization \n\n<details><summary>Changelog</summary>\n\n</details>\n\n",
+        "title": "</details>\n\n## Text Normalization / Inverse Text Normalization\n\n<details><summary>Changelog</summary>",
         "labels": ["tn", "itn"],
         "exclude_labels": ["cherry-pick"]
       },
       {
-        "title": "## NeMo Tools \n\n<details><summary>Changelog</summary>\n\n</details>\n\n",
+        "title": "</details>\n\n## NeMo Tools\n\n<details><summary>Changelog</summary>",
         "labels": ["tools"],
         "exclude_labels": ["cherry-pick"]
       },
       {
-        "title": "## Export \n\n<details><summary>Changelog</summary>\n\n</details>\n\n",
+        "title": "</details>\n\n## Export\n\n<details><summary>Changelog</summary>",
         "labels": ["export"],
         "exclude_labels": ["cherry-pick"]
       },
       {
-        "title": "## Documentation \n\n<details><summary>Changelog</summary>\n\n</details>\n\n",
+        "title": "</details>\n\n## Documentation\n\n<details><summary>Changelog</summary>",
         "labels": ["docs"],
         "exclude_labels": ["cherry-pick"]
       },
       {
-        "title": "## Bugfixes \n\n<details><summary>Changelog</summary>\n\n</details>\n\n",
+        "title": "</details>\n\n## Bugfixes\n\n<details><summary>Changelog</summary>",
         "labels": ["bug"],
         "exclude_labels": ["cherry-pick"]
       },
       {
-        "title": "## Cherrypick \n\n<details><summary>Changelog</summary>\n\n</details>\n\n",
+        "title": "</details>\n\n## Cherrypick\n\n<details><summary>Changelog</summary>",
         "labels": ["cherry-pick"],
         "exclude_labels": ["cherry-pick"]
       }
@@ -50,7 +50,7 @@
       "ignore"
     ],
     "sort": "ASC",
-    "template": "\n${{CHANGELOG}}\nUncategorized:\n${{UNCATEGORIZED}}\n\n",
+    "template": "\n${{CHANGELOG}}</details>\n\n## Uncategorized:\n\n<details><summary>Changelog</summary>\n\n${{UNCATEGORIZED}}\n</details>\n",
     "pr_template": "- ${{TITLE}} by @${{AUTHOR}} :: PR: #${{NUMBER}}",
     "empty_template": "${{OWNER}}\n${{REPO}}\n${{FROM_TAG}}\n${{TO_TAG}}",
     "label_extractor": [

.github/workflows/import-test.yml

@@ -12,7 +12,7 @@ jobs:
   test-asr-imports:
     runs-on: ubuntu-latest
     container:
-      image: pytorch/pytorch:2.0.1-cuda11.7-cudnn8-runtime
+      image: pytorch/pytorch:2.4.0-cuda11.8-cudnn9-runtime
     steps:
     - name: Checkout repo
       uses: actions/checkout@v2
@@ -43,7 +43,7 @@ jobs:
   test-tts-imports:
     runs-on: ubuntu-latest
     container:
-      image: pytorch/pytorch:2.0.1-cuda11.7-cudnn8-runtime
+      image: pytorch/pytorch:2.4.0-cuda11.8-cudnn9-runtime
     steps:
     - name: Checkout repo
       uses: actions/checkout@v2
@@ -70,4 +70,4 @@ jobs:
         # Run import checks
         python tests/core_ptl/check_imports.py --domain "tts"
         # Uninstall NeMo
-        pip uninstall -y nemo_toolkit
+        pip uninstall -y nemo_toolkit

.github/workflows/mcore-tag-bump-bot.yml

@@ -0,0 +1,59 @@
+# Regularly updates the CI container
+name: MCore Tag Bump Bot
+on:
+  workflow_dispatch:
+  schedule:
+    - cron: 0 0 * * *
+
+jobs:
+  main:
+    runs-on: ubuntu-latest
+    environment: main
+    steps:
+      - name: Checkout NVIDIA/Megatron-LM
+        uses: actions/checkout@v4
+        with:
+          repository: NVIDIA/Megatron-LM
+          ref: main
+          path: ${{ github.run_id }}
+
+      - name: Get latest mcore commit
+        id: ref
+        run: |
+          cd ${{ github.run_id }}      
+          sha=$(git rev-parse origin/main)
+          echo "sha=${sha}" >> "$GITHUB_OUTPUT"
+          echo "short_sha=${sha:0:7}" >> "$GITHUB_OUTPUT"
+          echo "date=$(date +%F)" >> "$GITHUB_OUTPUT"
+
+      - name: Checkout ${{ github.repository }}
+        uses: actions/checkout@v4
+        with:
+          path: ${{ github.run_id }}
+          token: ${{ secrets.PAT }}
+
+      - name: Bump MCORE_TAG
+        run: |
+          cd ${{ github.run_id }}     
+          sed -i 's/^ARG MCORE_TAG=.*$/ARG MCORE_TAG=${{ steps.ref.outputs.sha }}/' Dockerfile.ci
+
+      - name: Create Bump PR
+        uses: peter-evans/create-pull-request@v6
+        id: create-pull-request
+        with:
+          path: ${{ github.run_id }}
+          branch: bump-ci-container-${{ steps.ref.outputs.date }}
+          base: main
+          title: 'Bump `Dockerfile.ci` (${{ steps.ref.outputs.date }})'
+          token: ${{ secrets.PAT }}
+          body: |
+            🚀 PR to Bump `Dockerfile.ci`.  
+
+            📝 Please remember the following to-do's before merge: 
+            - [ ] Verify the presubmit CI  
+
+            🙏 Please merge this PR only if the CI workflow completed successfully.
+          commit-message: "[🤠]: Howdy folks, let's bump `Dockerfile.ci` to ${{ steps.ref.outputs.short_sha }} !"
+          signoff: true
+          reviewers: 'pablo-garay'
+          labels: 'Run CICD'

.github/workflows/release-freeze.yml

@@ -0,0 +1,192 @@
+name: "NeMo Code freeze"
+
+on:
+  workflow_dispatch:
+    inputs:
+      next_version:
+        description: 'MAJOR.MINOR.PATCH[rcN] (Example: 2.0.0rc1, or 2.1.0)'
+        required: true
+        type: string
+      mcore_version:
+        description: 'Version of MCore to use (must be a valid git ref)'
+        required: true
+        type: string
+jobs:
+  create-release-branch:
+    runs-on: ubuntu-latest
+    if: contains(fromJSON('["ko3n1g"]'), github.actor)
+    environment: 
+      name: main
+    outputs:
+      version: ${{ steps.release-branch.outputs.version }}
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+        with:
+          path: ${{ github.run_id }}
+          fetch-depth: 0
+          fetch-tags: true
+          ref: main
+
+      - name: Get Previous tag
+        id: previous-tag
+        # git for-each-ref --sort=-creatordate --format '%(refname)' refs/tags ==> refs/tags/vX.Y.Z in descending order of date
+        # awk 'FNR == 2 {print substr($1, 11, length($1))}') ==> Selects the 2nd tag from the list, then strips the /refs/tags/ part of the tag
+        # set-output name=tag_name:: ==> Takes the clean tag vX.Y.Z and sets it to steps.previous_tag.outputs.tag_name
+        run: |
+          TAG=$(git for-each-ref --sort=-creatordate --format '%(refname)' refs/tags | awk 'FNR == 2 {print substr($1, 11, length($1))}')
+          echo "tag-name=$TAG" >> "$GITHUB_OUTPUT"
+
+      - name: Get release branch ref
+        id: release-branch
+        run: |
+          cd ${{ github.run_id }}
+
+          VERSION=$(python -c 'import nemo; print(nemo.__version__)')
+                    echo "Release version r$VERSION" > version
+          echo "version=$VERSION" >> "$GITHUB_OUTPUT"
+
+      - name: Pin branch name in Notebooks
+        run: |
+          cd ${{ github.run_id }}
+          find tutorials -type f -name "*.ipynb" -exec sed -i "s/BRANCH = 'main'/BRANCH = 'r${{ steps.release-branch.outputs.version }}'/g" {} +
+
+      - name: Pin MCore in Dockerfile
+        run: |
+          cd ${{ github.run_id }}
+          sed -i 's/^ARG MCORE_TAG=.*$/ARG MCORE_TAG=${{ inputs.mcore_version }}/' Dockerfile.ci
+
+      - name: Build Changelog
+        id: build-changelog
+        uses: mikepenz/[email protected]
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        with:
+          # Configuration file is setup with filters for domains
+          # owner:repo must point to current repo
+          # fromTag: Auto resolved from historical tag order (previous tag compared to current tag)
+          # toTag: Current tag reference
+          configuration: ".github/workflows/config/changelog-config.json"
+          owner: ${{ github.repository_owner }}
+          repo: ${{ github.event.repository.name }} 
+          ignorePreReleases: "false"
+          failOnError: "false"
+          fromTag: ${{ steps.previous-tag.outputs.tag-name }}
+          toTag: main
+
+      - name: Append Changelog
+        run: |
+          echo "${{ steps.build-changelog.outputs.changelog }}"
+
+      - name: Create Release PR
+        uses: peter-evans/create-pull-request@v6
+        id: create-pull-request
+        with:
+          path: ${{ github.run_id }}
+          branch: r${{ steps.release-branch.outputs.version }}
+          title: 'Release `${{ steps.release-branch.outputs.version }}`'
+          body: |
+            🚀 PR to release NeMo `${{ steps.release-branch.outputs.version }}`.
+
+            📝 Please remember the following to-do's before merge:
+            - [ ] Fill-in the comment `Highlights`
+            - [ ] Review the comment `Detailed Changelogs`
+
+            🚨 Please also keep in mind to _not_ delete the headings of the task commits. They are required by the post-merge automation.
+
+            🙏 Please merge this PR only if the CI workflow completed successfully.
+
+          commit-message: "[🤠]: Howdy folks, let's release NeMo `${{ steps.release-branch.outputs.version }}` !"
+          signoff: true
+          assignees: okoenig
+          labels: 'Run CICD'
+
+      - name: Add Summary comment
+        uses: peter-evans/create-or-update-comment@v4
+        with:
+          issue-number: ${{ steps.create-pull-request.outputs.pull-request-number }}
+          body: |
+            # Highlights
+            _<here-goes-the-summary...>_
+
+      - name: Add Changelog comment
+        uses: peter-evans/create-or-update-comment@v4
+        with:
+          issue-number: ${{ steps.create-pull-request.outputs.pull-request-number }}
+          body: |
+            # Detailed Changelogs
+            ${{ steps.build-changelog.outputs.changelog }}
+
+  bump-next-version:
+    runs-on: ubuntu-latest
+    needs: [create-release-branch]
+    environment: 
+      name: main
+    env:
+      VERSION_FILE: nemo/package_info.py
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+        with:
+          path: ${{ github.run_id }}
+          fetch-depth: 0
+          fetch-tags: true
+          ref: main
+          token: ${{ secrets.PAT }}
+
+      - name: Bump version
+        id: bump-version
+        run: |
+          cd ${{ github.run_id }}
+          FULL_VERSION_NUM=${{ inputs.next_version }}
+          VERSION=${FULL_VERSION_NUM%%rc*}
+          MAJOR=$(echo "$VERSION" | cut -d. -f1)
+          MINOR=$(echo "$VERSION" | cut -d. -f2)
+          PATCH=$(echo "$VERSION" | cut -d. -f3)
+          PRE_RELEASE=${FULL_VERSION_NUM#$VERSION}
+
+          sed -i 's/^MAJOR\s*=\s*[0-9]\+/MAJOR = '$MAJOR'/' $VERSION_FILE
+          sed -i 's/^MINOR\s*=\s*[0-9]\+/MINOR = '$MINOR'/' $VERSION_FILE
+          sed -i 's/^PATCH\s*=\s*[0-9]\+/PATCH = '$PATCH'/' $VERSION_FILE
+          sed -i 's/^PRE_RELEASE\s*=\s*'.*'/PRE_RELEASE = '\'$PRE_RELEASE\''/' $VERSION_FILE
+
+          cat $VERSION_FILE
+          PRE_RELEASE=$(echo $PRE_RELEASE | tr -d "'")
+          echo "version=$MAJOR.$MINOR.$PATCH$PRE_RELEASE" >> "$GITHUB_OUTPUT"
+
+      - name: Create Version Bump PR
+        uses: peter-evans/create-pull-request@v6
+        id: create-pull-request
+        with:
+          path: ${{ github.run_id }}
+          branch: bot/chore/version-bump-${{ inputs.next_version }}
+          title: 'Version bump to `${{ inputs.next_version }}`'
+          body: |
+            🚀 Version bump NeMo toolkit to `${{ inputs.next_version }}`
+
+          commit-message: "[🤠]: Howdy folks, let's bump NeMo `${{ inputs.next_version }}` !"
+          signoff: true
+          assignees: okoenig
+          labels: 'Run CICD'
+
+  notify:
+    runs-on: ubuntu-latest
+    needs: [create-release-branch, bump-next-version]
+    environment: 
+      name: main
+    steps:
+      - name: Main
+        run: |
+          MESSAGE='{
+            "blocks": [
+              {
+                "type": "section",
+                "text": {
+                  "type": "mrkdwn",
+                  "text": "Releasebot 🤖: NeMo Toolkit has been frozen 🎉 to branch `r${{ needs.create-release-branch.outputs.version }}`"
+                }
+              }
+            ]
+          }'
+
+          curl -X POST -H "Content-type: application/json" --data "$MESSAGE" ${{ secrets.SLACK_RELEASE_ENDPOINT }}