Add pytorch-notebook image variants with cuda 11 and 12 (x86_64 versions only)

.github/actions/load-image/action.yml

@@ -8,17 +8,20 @@ inputs:
   platform:
     description: Image platform
     required: true
+  variant:
+    description: Variant tag prefix
+    required: true
 
 runs:
   using: composite
   steps:
     - name: Download built image 📥
       uses: actions/download-artifact@v4
       with:
-        name: ${{ inputs.image }}-${{ inputs.platform }}
+        name: ${{ inputs.image }}-${{ inputs.platform }}-${{ inputs.variant }}
         path: /tmp/jupyter/images/
     - name: Load downloaded image to docker 📥
       run: |
-        zstd --uncompress --stdout --rm /tmp/jupyter/images/${{ inputs.image }}-${{ inputs.platform }}.tar.zst | docker load
+        zstd --uncompress --stdout --rm /tmp/jupyter/images/${{ inputs.image }}-${{ inputs.platform }}-${{ inputs.variant }}.tar.zst | docker load
         docker image ls --all
       shell: bash

.github/workflows/docker-build-test-upload-all.yml

@@ -0,0 +1,244 @@
+name: Build, test, and push Docker Images
+
+env:
+  REGISTRY: quay.io
+  OWNER: ${{ github.repository_owner }}
+
+on:
+  workflow_call:
+    inputs:
+      platform:
+        description: Image platform
+        required: true
+        type: string
+      runs-on-fast:
+        description: GitHub Actions Runner image (fast)
+        required: true
+        type: string
+      runs-on-slow:
+        description: GitHub Actions Runner image (slow)
+        required: true
+        type: string
+    secrets:
+      REGISTRY_USERNAME:
+        required: true
+      REGISTRY_TOKEN:
+        required: true
+
+jobs:
+  foundation:
+    uses: ./.github/workflows/docker-build-test-upload.yml
+    with:
+      parent-image: ""
+      parent-variant: default
+      image: docker-stacks-foundation
+      variant: default
+      platform: ${{ inputs.platform }}
+      runs-on: ${{ inputs.runs-on-fast }}
+
+  base:
+    uses: ./.github/workflows/docker-build-test-upload.yml
+    with:
+      parent-image: docker-stacks-foundation
+      parent-variant: default
+      image: base-notebook
+      variant: default
+      platform: ${{ inputs.platform }}
+      runs-on: ${{ inputs.runs-on-fast }}
+    needs: [foundation]
+
+  minimal:
+    uses: ./.github/workflows/docker-build-test-upload.yml
+    with:
+      parent-image: base-notebook
+      parent-variant: default
+      image: minimal-notebook
+      variant: default
+      platform: ${{ inputs.platform }}
+      runs-on: ${{ inputs.runs-on-fast }}
+    needs: [base]
+    if: ${{ !contains(github.event.pull_request.title, '[FAST_BUILD]') }}
+
+  scipy:
+    uses: ./.github/workflows/docker-build-test-upload.yml
+    with:
+      parent-image: minimal-notebook
+      parent-variant: default
+      image: scipy-notebook
+      variant: default
+      platform: ${{ inputs.platform }}
+      runs-on: ${{ inputs.runs-on-fast }}
+    needs: [minimal]
+    if: ${{ !contains(github.event.pull_request.title, '[FAST_BUILD]') }}
+
+  r:
+    uses: ./.github/workflows/docker-build-test-upload.yml
+    with:
+      parent-image: minimal-notebook
+      parent-variant: default
+      image: r-notebook
+      variant: default
+      platform: ${{ inputs.platform }}
+      runs-on: ${{ inputs.runs-on-slow }}
+    needs: [minimal]
+    if: ${{ !contains(github.event.pull_request.title, '[FAST_BUILD]') }}
+
+  julia:
+    uses: ./.github/workflows/docker-build-test-upload.yml
+    with:
+      parent-image: minimal-notebook
+      parent-variant: default
+      image: julia-notebook
+      variant: default
+      platform: ${{ inputs.platform }}
+      runs-on: ${{ inputs.runs-on-slow }}
+    needs: [minimal]
+    if: ${{ !contains(github.event.pull_request.title, '[FAST_BUILD]') }}
+
+  tensorflow:
+    uses: ./.github/workflows/docker-build-test-upload.yml
+    with:
+      parent-image: scipy-notebook
+      parent-variant: default
+      image: tensorflow-notebook
+      variant: default
+      platform: ${{ inputs.platform }}
+      runs-on: ${{ inputs.runs-on-slow }}
+    needs: [scipy]
+    if: ${{ !contains(github.event.pull_request.title, '[FAST_BUILD]') }}
+
+  pytorch:
+    uses: ./.github/workflows/docker-build-test-upload.yml
+    with:
+      parent-image: scipy-notebook
+      parent-variant: default
+      image: pytorch-notebook
+      variant: default
+      platform: ${{ inputs.platform }}
+      runs-on: ${{ inputs.runs-on-slow }}
+    needs: [scipy]
+    if: ${{ !contains(github.event.pull_request.title, '[FAST_BUILD]') }}
+
+  pytorch-cuda:
+    uses: ./.github/workflows/docker-build-test-upload.yml
+    with:
+      parent-image: scipy-notebook
+      parent-variant: default
+      image: pytorch-notebook
+      variant: cuda
+      platform: ${{ inputs.platform }}
+      runs-on: ${{ inputs.runs-on-slow }}
+    needs: [scipy]
+    if: ${{ !contains(github.event.pull_request.title, '[FAST_BUILD]') }}
+
+  pytorch-cuda11:
+    uses: ./.github/workflows/docker-build-test-upload.yml
+    with:
+      parent-image: scipy-notebook
+      parent-variant: default
+      image: pytorch-notebook
+      variant: cuda11
+      platform: ${{ inputs.platform }}
+      runs-on: ${{ inputs.runs-on-slow }}
+    needs: [scipy]
+    if: ${{ !contains(github.event.pull_request.title, '[FAST_BUILD]') }}
+
+  datascience:
+    uses: ./.github/workflows/docker-build-test-upload.yml
+    with:
+      parent-image: scipy-notebook
+      parent-variant: default
+      image: datascience-notebook
+      variant: default
+      platform: ${{ inputs.platform }}
+      runs-on: ${{ inputs.runs-on-slow }}
+    needs: [scipy]
+    if: ${{ !contains(github.event.pull_request.title, '[FAST_BUILD]') }}
+
+  pyspark:
+    uses: ./.github/workflows/docker-build-test-upload.yml
+    with:
+      parent-image: scipy-notebook
+      parent-variant: default
+      image: pyspark-notebook
+      variant: default
+      platform: ${{ inputs.platform }}
+      runs-on: ${{ inputs.runs-on-fast }}
+    needs: [scipy]
+    if: ${{ !contains(github.event.pull_request.title, '[FAST_BUILD]') }}
+
+  all-spark:
+    uses: ./.github/workflows/docker-build-test-upload.yml
+    with:
+      parent-image: pyspark-notebook
+      parent-variant: default
+      image: all-spark-notebook
+      variant: default
+      platform: ${{ inputs.platform }}
+      runs-on: ${{ inputs.runs-on-fast }}
+    needs: [pyspark]
+    if: ${{ !contains(github.event.pull_request.title, '[FAST_BUILD]') }}
+
+  tag-push:
+    uses: ./.github/workflows/docker-tag-push.yml
+    with:
+      platform: ${{ inputs.platform }}
+      image: ${{ matrix.image-variant.image }}
+      variant: ${{ matrix.image-variant.variant }}
+    secrets:
+      REGISTRY_USERNAME: ${{ secrets.REGISTRY_USERNAME }}
+      REGISTRY_TOKEN: ${{ secrets.REGISTRY_TOKEN }}
+    strategy:
+      matrix:
+        image-variant:
+          [
+            { image: docker-stacks-foundation, variant: default },
+            { image: base-notebook, variant: default },
+            { image: minimal-notebook, variant: default },
+            { image: scipy-notebook, variant: default },
+            { image: r-notebook, variant: default },
+            { image: julia-notebook, variant: default },
+            { image: tensorflow-notebook, variant: default },
+            { image: pytorch-notebook, variant: default },
+            { image: pytorch-notebook, variant: cuda },
+            { image: pytorch-notebook, variant: cuda11 },
+            { image: datascience-notebook, variant: default },
+            { image: pyspark-notebook, variant: default },
+            { image: all-spark-notebook, variant: default },
+          ]
+    needs:
+      [
+        foundation,
+        base,
+        minimal,
+        scipy,
+        r,
+        julia,
+        tensorflow,
+        pytorch,
+        pytorch-cuda,
+        pytorch-cuda11,
+        datascience,
+        pyspark,
+        all-spark,
+      ]
+    if: ${{ !contains(github.event.pull_request.title, '[FAST_BUILD]') }}
+
+  tag-push-fast:
+    uses: ./.github/workflows/docker-tag-push.yml
+    with:
+      platform: ${{ inputs.platform }}
+      image: ${{ matrix.image-variant.image }}
+      variant: ${{ matrix.image-variant.variant }}
+    secrets:
+      REGISTRY_USERNAME: ${{ secrets.REGISTRY_USERNAME }}
+      REGISTRY_TOKEN: ${{ secrets.REGISTRY_TOKEN }}
+    strategy:
+      matrix:
+        image-variant:
+          [
+            { image: docker-stacks-foundation, variant: default },
+            { image: base-notebook, variant: default },
+          ]
+    needs: [foundation, base]
+    if: ${{ contains(github.event.pull_request.title, '[FAST_BUILD]') }}

.github/workflows/docker-build-test-upload.yml

@@ -11,10 +11,18 @@ on:
         description: Parent image name
         required: true
         type: string
+      parent-variant:
+        description: Parent variant tag prefix
+        required: true
+        type: string
       image:
         description: Image name
         required: true
         type: string
+      variant:
+        description: Variant tag prefix
+        required: true
+        type: string
       platform:
         description: Image platform
         required: true
@@ -29,6 +37,19 @@ jobs:
     runs-on: ${{ inputs.runs-on }}
 
     steps:
+      # Image with CUDA needs extra disk space
+      - name: Free disk space 🧹
+        if: contains(inputs.variant, 'cuda') && inputs.platform == 'x86_64'
+        uses: jlumbroso/free-disk-space@54081f138730dfa15788a46383842cd2f914a1be
+        with:
+          tool-cache: false
+          android: true
+          dotnet: true
+          haskell: true
+          large-packages: false
+          docker-images: false
+          swap-storage: false
+
       - name: Checkout Repo ⚡️
         uses: actions/checkout@v4
       - name: Create dev environment 📦
@@ -52,59 +73,70 @@ jobs:
         with:
           image: ${{ inputs.parent-image }}
           platform: ${{ inputs.platform }}
+          variant: ${{ inputs.parent-variant }}
 
       - name: Pull ubuntu:22.04 image 📥
         if: inputs.parent-image == ''
         run: docker pull ubuntu:22.04
         shell: bash
 
-      - name: Build image 🛠
+      - name: Build image (default) 🛠
+        if: inputs.variant == 'default'
         run: docker build --rm --force-rm --tag ${{ env.REGISTRY }}/${{ env.OWNER }}/${{ inputs.image }} images/${{ inputs.image }}/ --build-arg REGISTRY=${{ env.REGISTRY }} --build-arg OWNER=${{ env.OWNER }}
         env:
           DOCKER_BUILDKIT: 1
           # Full logs for CI build
           BUILDKIT_PROGRESS: plain
         shell: bash
 
+      - name: Build image (variant) 🛠
+        if: inputs.variant != 'default'
+        run: docker build --rm --force-rm --tag ${{ env.REGISTRY }}/${{ env.OWNER }}/${{ inputs.image }} images/${{ inputs.image }}/${{ inputs.variant }}/ --build-arg REGISTRY=${{ env.REGISTRY }} --build-arg OWNER=${{ env.OWNER }}
+        env:
+          DOCKER_BUILDKIT: 1
+          # Full logs for CI build
+          BUILDKIT_PROGRESS: plain
+        shell: bash
+
       - name: Run tests ✅
         run: python3 -m tests.run_tests --short-image-name ${{ inputs.image }} --registry ${{ env.REGISTRY }} --owner ${{ env.OWNER }}
         shell: bash
 
       - name: Write tags file 🏷
         run: |
-          python3 -m tagging.write_tags_file --short-image-name ${{ inputs.image }} --tags-dir /tmp/jupyter/tags/ --registry ${{ env.REGISTRY }} --owner ${{ env.OWNER }}
+          python3 -m tagging.write_tags_file --short-image-name ${{ inputs.image }} --tags-dir /tmp/jupyter/tags/ --registry ${{ env.REGISTRY }} --owner ${{ env.OWNER }} --variant ${{ inputs.variant }}
         shell: bash
       - name: Upload tags file 💾
         uses: actions/upload-artifact@v4
         with:
-          name: ${{ inputs.image }}-${{ inputs.platform }}-tags
-          path: /tmp/jupyter/tags/${{ inputs.platform }}-${{ inputs.image }}.txt
+          name: ${{ inputs.image }}-${{ inputs.platform }}-${{ inputs.variant }}-tags
+          path: /tmp/jupyter/tags/${{ inputs.platform }}-${{ inputs.variant }}-${{ inputs.image }}.txt
           retention-days: 3
 
       - name: Write manifest and build history file 🏷
-        run: python3 -m tagging.write_manifest --short-image-name ${{ inputs.image }} --hist-lines-dir /tmp/jupyter/hist_lines/ --manifests-dir /tmp/jupyter/manifests/ --registry ${{ env.REGISTRY }} --owner ${{ env.OWNER }}
+        run: python3 -m tagging.write_manifest --short-image-name ${{ inputs.image }} --hist-lines-dir /tmp/jupyter/hist_lines/ --manifests-dir /tmp/jupyter/manifests/ --registry ${{ env.REGISTRY }} --owner ${{ env.OWNER }} --variant ${{ inputs.variant }}
         shell: bash
       - name: Upload manifest file 💾
         uses: actions/upload-artifact@v4
         with:
-          name: ${{ inputs.image }}-${{ inputs.platform }}-manifest
-          path: /tmp/jupyter/manifests/${{ inputs.platform }}-${{ inputs.image }}-*.md
+          name: ${{ inputs.image }}-${{ inputs.platform }}-${{ inputs.variant }}-manifest
+          path: /tmp/jupyter/manifests/${{ inputs.platform }}-${{ inputs.variant }}-${{ inputs.image }}-*.md
           retention-days: 3
       - name: Upload build history line 💾
         uses: actions/upload-artifact@v4
         with:
-          name: ${{ inputs.image }}-${{ inputs.platform }}-history_line
-          path: /tmp/jupyter/hist_lines/${{ inputs.platform }}-${{ inputs.image }}-*.txt
+          name: ${{ inputs.image }}-${{ inputs.platform }}-${{ inputs.variant }}-history_line
+          path: /tmp/jupyter/hist_lines/${{ inputs.platform }}-${{ inputs.variant }}-${{ inputs.image }}-*.txt
           retention-days: 3
 
       - name: Save image as a tar for later use 💾
         run: |
           mkdir -p /tmp/jupyter/images/
-          docker save ${{ env.REGISTRY }}/${{ env.OWNER }}/${{ inputs.image }} | zstd > /tmp/jupyter/images/${{ inputs.image }}-${{ inputs.platform }}.tar.zst
+          docker save ${{ env.REGISTRY }}/${{ env.OWNER }}/${{ inputs.image }} | zstd > /tmp/jupyter/images/${{ inputs.image }}-${{ inputs.platform }}-${{ inputs.variant }}.tar.zst
         shell: bash
       - name: Upload image as artifact 💾
         uses: actions/upload-artifact@v4
         with:
-          name: ${{ inputs.image }}-${{ inputs.platform }}
-          path: /tmp/jupyter/images/${{ inputs.image }}-${{ inputs.platform }}.tar.zst
+          name: ${{ inputs.image }}-${{ inputs.platform }}-${{ inputs.variant }}
+          path: /tmp/jupyter/images/${{ inputs.image }}-${{ inputs.platform }}-${{ inputs.variant }}.tar.zst
           retention-days: 3