ci: Multi-tenancy for tests and garbage collection

Signed-off-by: Oliver Koenig <[email protected]>
NVIDIA · May 15, 2024 · 226245e · 226245e
1 parent a0e9ee3
commit 226245e
Showing 1 changed file with 7 additions and 2 deletions.
diff --git a/.github/workflows/cicd-main.yml b/.github/workflows/cicd-main.yml
@@ -317,10 +317,15 @@ jobs:
         - name: Checkout repository
           uses: actions/checkout@v4
         - run: |
+            mkdir -p /home/TestData/nlp/megatron_gpt/starcoder-ci-hf/${{ github.run_id }};
             python scripts/checkpoint_converters/convert_starcoder_hf_to_nemo.py \
             --input_name_or_path /home/TestData/nlp/megatron_gpt/starcoder-ci-hf \
-            --output_path /home/TestData/nlp/megatron_gpt/starcoder-ci-hf
-            rm -f /home/TestData/nlp/megatron_gpt/starcoder-ci-hf/megatron_starcoder_tp1_pp1.nemo
+            --output_path /home/TestData/nlp/megatron_gpt/starcoder-ci-hf/${{ github.run_id }}
+        - name: Cleanup
+          if: "always()"
+          run: |
+            rm -rf /home/TestData/nlp/megatron_gpt/starcoder-ci-hf/megatron_starcoder_tp1_pp1.nemo; 
+            rm -rf /home/TestData/nlp/megatron_gpt/starcoder-ci-hf/${{ github.run_id }}/
         - uses: "NVIDIA/NeMo/.github/actions/cancel-workflow@main"
           if: "failure()"