ci: Better caching and pruning strategy (#10735)

Signed-off-by: Oliver Koenig <[email protected]>
NVIDIA · Oct 4, 2024 · 33448ae · 33448ae
1 parent bf9e649
commit 33448ae
Show file tree

Hide file tree

Showing 2 changed files with 10 additions and 5 deletions.
diff --git a/.github/workflows/cicd-main.yml b/.github/workflows/cicd-main.yml
@@ -66,9 +66,8 @@ jobs:
     steps:
     - name: Clean server from old files
       run: |
-        docker container prune --filter "until=24h" --force
-        docker image prune -a --filter "until=24h" --force
-
+        docker system prune --filter "until=24h" --filter "label=nemo.library=nemo-core" --force
+        
   cicd-test-container-setup:
     needs: [cicd-cluster-clean, pre-flight]
     runs-on: self-hosted-azure-builder
@@ -90,17 +89,22 @@ jobs:
         driver: docker
 
     - name: Restore cache
-      run: docker pull nemoci.azurecr.io/nemo_container:latest
+      run: |
+        docker pull nemoci.azurecr.io/nemo_container:latest
+        docker pull nemoci.azurecr.io/nemo_container_${{ github.event.number || 'noop' }} || true
 
     - name: Build and push
       uses: docker/build-push-action@v5
       with:
         file: Dockerfile.ci
         push: true
-        cache-from: nemoci.azurecr.io/nemo_container:latest
+        cache-from: |
+          nemoci.azurecr.io/nemo_container:latest
+          nemoci.azurecr.io/nemo_container_${{ github.event.number || 'noop' }}
         cache-to: type=inline
         tags: |
           nemoci.azurecr.io/nemo_container_${{ github.run_id }}
+          nemoci.azurecr.io/nemo_container_${{ github.event.number || 'noop' }}
           nemoci.azurecr.io/nemo_container:latest
 
     - name: Run some checks

diff --git a/Dockerfile.ci b/Dockerfile.ci
@@ -17,6 +17,7 @@
 ARG BASE_IMAGE=nvcr.io/nvidia/pytorch:24.07-py3
 
 FROM ${BASE_IMAGE}
+LABEL "nemo.library"="nemo-core"
 
 ENV TRANSFORMERS_OFFLINE=0
 ENV HYDRA_FULL_ERROR=1