Add support for multi-arch docker images (#399)

* Don't hardcode CPU architecture in dockerfile The version of mambaforge appropriate for the architecture that the docker build is running on is built this way. Ref #396 * Build conda lock files for amd64, aarch64 & ppcle64 * [condalock-command] autogenerated conda-lock files * Dynamically pick conda-lock file based on architecture * Generate multi-arch lock files in Makefile * tidy dockerfile and make, comment out arm-incompatible packages * Update pangeo-notebook/environment.yml Co-authored-by: Wei Ji <[email protected]> * Update .github/workflows/CondaLock.yml Co-authored-by: Wei Ji <[email protected]> * miniforge3 not mambaforge * multiarch cmds in makefile and github actions * Test multiarch * Apply suggestions from code review Co-authored-by: Wei Ji <[email protected]> --------- Co-authored-by: pangeo-bot <[email protected]> Co-authored-by: Scott Henderson <[email protected]> Co-authored-by: Scott Henderson <[email protected]> Co-authored-by: Wei Ji <[email protected]>
pangeo-data · Jun 28, 2024 · fc92d8b · fc92d8b
1 parent bccbb8c
commit fc92d8b
Show file tree

Hide file tree

Showing 5 changed files with 56 additions and 45 deletions.
diff --git a/.github/workflows/Build.yml b/.github/workflows/Build.yml
@@ -32,6 +32,9 @@ jobs:
         DOCKER_TAG=$SHA7
         echo "DOCKER_TAG=${DOCKER_TAG}" >> $GITHUB_ENV
 
+    - name: Set up QEMU
+      uses: docker/setup-qemu-action@v3
+
     - name: Set up Docker Buildx
       uses: docker/setup-buildx-action@v3
 
@@ -52,6 +55,7 @@ jobs:
       uses: docker/build-push-action@v5
       with:
         context: ${{ env.IMAGE }}
+        platforms: linux/amd64,linux/arm64
         push: true
         tags: |
           ${{env.DOCKER_ORG}}/${{ env.IMAGE }}:master

diff --git a/.github/workflows/CondaLock.yml b/.github/workflows/CondaLock.yml
@@ -37,39 +37,37 @@ jobs:
       # Could run as single step in parallel, but would complicate logs...
       - name: Run conda-lock ${{ matrix.IMAGE }}
         run: |
-          cd ${{ matrix.IMAGE }}
-          if [ ${{ matrix.IMAGE }} = "base-notebook" ]; then
-            conda-lock lock -f environment.yml -p linux-64
-          elif [ ${{ matrix.IMAGE }} = "pangeo-notebook" ]; then
-            conda-lock lock -f environment.yml -f ../base-notebook/environment.yml -p linux-64
-          else
-            conda-lock lock -f environment.yml -f ../pangeo-notebook/environment.yml -f ../base-notebook/environment.yml -p linux-64
-          fi
-          conda-lock render -k explicit
+          cd base-notebook
+          conda-lock lock -p linux-64 -p linux-aarch64 -p osx-64 -p osx-arm64
+          conda-lock render -k explicit -p linux-64
           ../generate-packages-list.py conda-linux-64.lock > packages.txt
 
-      - name: Upload lockfiles
-        uses: actions/upload-artifact@v4
-        with:
-          name: ${{ matrix.image }}
-          path: ${{ matrix.image }}
-
+      - name: Run conda-lock pangeo-notebook
+        timeout-minutes: 5
+        continue-on-error: true
+        run: |
+          cd pangeo-notebook
+          conda-lock lock -f environment.yml -f ../base-notebook/environment.yml -p linux-64 -p linux-aarch64 -p osx-64 -p osx-arm64
+          conda-lock render -k explicit -p linux-64
+          ../generate-packages-list.py conda-linux-64.lock > packages.txt
 
-  # Each job will commit files, so we know it succeeds based on commits
-  commit-lockfiles:
-    needs: condalock
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout Repository
-        uses: actions/checkout@v4
-        with:
-          token: ${{ secrets.PANGEOBOT_TOKEN }}
-          # These lines are critical, otherwise Pangeo-bot pushes changes directly to master from PRs!
-          repository: ${{ github.event.client_payload.pull_request.head.repo.full_name }}
-          ref: ${{ github.event.client_payload.pull_request.head.ref }}
+      - name: Run conda-lock ml-notebook
+        timeout-minutes: 5
+        continue-on-error: true
+        run: |
+          cd ml-notebook
+          conda-lock lock -f environment.yml -f ../pangeo-notebook/environment.yml -f ../base-notebook/environment.yml -p linux-64
+          conda-lock render -k explicit -p linux-64
+          ../generate-packages-list.py conda-linux-64.lock > packages.txt
 
-      # Download all artifacts from previous matrix job
-      - uses: actions/download-artifact@v4
+      - name: Run conda-lock pytorch-notebook
+        timeout-minutes: 5
+        continue-on-error: true
+        run: |
+          cd pytorch-notebook
+          conda-lock lock -f environment.yml -f ../pangeo-notebook/environment.yml -f ../base-notebook/environment.yml -p linux-64
+          conda-lock render -k explicit -p linux-64
+          ../generate-packages-list.py conda-linux-64.lock > packages.txt
 
       - name: Commit condalock files to PR
         run: |

diff --git a/.github/workflows/Test.yml b/.github/workflows/Test.yml
@@ -42,6 +42,9 @@ jobs:
         sudo rm -rf /usr/local/lib/android /usr/share/dotnet /opt/ghc
         df -h
 
+    - name: Set up QEMU
+      uses: docker/setup-qemu-action@v3
+
     - name: Set up Docker Buildx
       uses: docker/setup-buildx-action@v3
       with:
@@ -51,6 +54,7 @@ jobs:
       uses: docker/build-push-action@v5
       with:
         context: base-image
+        platforms: linux/amd64,linux/arm64
         tags: localhost:5000/pangeo/base-image:PR
         push: true
         cache-from: type=gha
@@ -60,6 +64,7 @@ jobs:
       uses: docker/build-push-action@v5
       with:
         context: ${{ matrix.IMAGE }}
+        platforms: linux/amd64,linux/arm64
         push: true
         tags: localhost:5000/${{env.DOCKER_ORG}}/${{ matrix.IMAGE }}:PR
         build-args: PANGEO_BASE_IMAGE_TAG=PR

diff --git a/Makefile b/Makefile
@@ -6,36 +6,40 @@ TESTDIR=/srv/test
 .PHONY: base-image
 base-image :
 	cd base-image ; \
-	docker build -t pangeo/base-image:master .
+	docker build -t pangeo/base-image:master --progress=plain --platform linux/amd64,linux/arm64 .
 
 .PHONY: base-notebook
 base-notebook : base-image
 	cd base-notebook ; \
-	conda-lock lock --mamba -k explicit -f environment.yml -p linux-64; \
+	conda-lock lock -p linux-64 -p linux-aarch64 -p osx-64 -p osx-arm64; \
+	conda-lock render -k explicit -p linux-64; \
 	../generate-packages-list.py conda-linux-64.lock > packages.txt; \
-	docker build -t pangeo/base-notebook:master . ; \
+	docker build -t pangeo/base-notebook:master . --progress=plain --platform linux/amd64,linux/arm64 ; \
 	docker run -w $(TESTDIR) -v $(PWD):$(TESTDIR) pangeo/base-notebook:master ./run_tests.sh base-notebook
 
 .PHONY: pangeo-notebook
 pangeo-notebook : base-image
 	cd pangeo-notebook ; \
-	conda-lock lock --mamba -k explicit -f environment.yml -f ../base-notebook/environment.yml -p linux-64; \
+	conda-lock lock -f environment.yml -f ../base-notebook/environment.yml -p linux-64 -p linux-aarch64 -p osx-64 -p osx-arm64; \
+	conda-lock render -k explicit -p linux-64; \
 	../generate-packages-list.py conda-linux-64.lock > packages.txt; \
-	docker build -t pangeo/pangeo-notebook:master . ; \
+	docker build -t pangeo/pangeo-notebook:master . --progress=plain --platform linux/amd64,linux/arm64 ; \
 	docker run -w $(TESTDIR) -v $(PWD):$(TESTDIR) pangeo/pangeo-notebook:master ./run_tests.sh pangeo-notebook
 
 .PHONY: ml-notebook
 ml-notebook : base-image
 	cd ml-notebook ; \
-	conda-lock lock --mamba -k explicit -f environment.yml -f ../pangeo-notebook/environment.yml -f ../base-notebook/environment.yml  -p linux-64; \
+	conda-lock lock -f environment.yml -f ../pangeo-notebook/environment.yml -f ../base-notebook/environment.yml -p linux-64; \
+	conda-lock render -k explicit -p linux-64; \
 	../generate-packages-list.py conda-linux-64.lock > packages.txt; \
 	docker build -t pangeo/ml-notebook:master . ; \
 	docker run -w $(TESTDIR) -v $(PWD):$(TESTDIR) pangeo/ml-notebook:master ./run_tests.sh ml-notebook
 
 .PHONY: pytorch-notebook
 pytorch-notebook : base-image
 	cd pytorch-notebook ; \
-	conda-lock lock --mamba -k explicit -f environment.yml -f ../pangeo-notebook/environment.yml -f ../base-notebook/environment.yml -p linux-64; \
+	conda-lock lock -f environment.yml -f ../pangeo-notebook/environment.yml -f ../base-notebook/environment.yml -p linux-64; \
+	conda-lock render -k explicit -p linux-64; \
 	../generate-packages-list.py conda-linux-64.lock > packages.txt; \
 	docker build -t pangeo/pytorch-notebook:master . ; \
 	docker run -w $(TESTDIR) -v $(PWD):$(TESTDIR) pangeo/pytorch-notebook:master ./run_tests.sh pytorch-notebook
diff --git a/base-image/Dockerfile b/base-image/Dockerfile
@@ -67,7 +67,7 @@ WORKDIR ${HOME}
 
 # Install latest mambaforge in ${CONDA_DIR}
 RUN echo "Installing Miniforge..." \
-    && URL="https://github.com/conda-forge/miniforge/releases/latest/download/Miniforge3-Linux-x86_64.sh" \
+    && URL="https://github.com/conda-forge/miniforge/releases/latest/download/Miniforge3-Linux-$(uname -m).sh" \
     && wget --quiet ${URL} -O installer.sh \
     && /bin/bash installer.sh -u -b -p ${CONDA_DIR} \
     && rm installer.sh \
@@ -144,8 +144,10 @@ ONBUILD USER ${NB_USER}
 # We want to keep our images as reproducible as possible. If a lock
 # file with exact versions of all required packages is present, we use
 # it to install packages. conda-lock (https://github.com/conda-incubator/conda-lock)
-# is used to generate this conda-linux-64.lock file from a given environment.yml
-# file - so we get the exact same versions each time the image is built. This
+# is used to generate this conda-lock.yml file from a given environment.yml
+# file - so we get the exact same versions each time the image is built. Note that
+# different packages may be used for different CPU architectures, but still,
+# the same dockerfile can be used to build different architecture images. This
 # also lets us see what packages have changed between two images by diffing
 # the contents of the lock file between those image versions.
 # If a lock file is not present, we use the environment.yml file. And
@@ -157,13 +159,11 @@ ONBUILD USER ${NB_USER}
 ONBUILD RUN echo "Checking for 'conda-lock.yml' 'conda-linux-64.lock' or 'environment.yml'..." \
         ; [ -d binder ] && cd binder \
         ; [ -d .binder ] && cd .binder \
-        ; if test -f "conda-lock.yml" ; then \
-        conda-lock install --name ${CONDA_ENV} conda-lock.yml \
-        ; elif test -f "conda-linux-64.lock" ; then \
-        mamba create --name ${CONDA_ENV} --file conda-linux-64.lock \
-        ; elif test -f "environment.yml" ; then \
+        ; if test -f "conda-lock.yml" ; then echo "Using conda-lock.yml" & \
+        conda-lock install --name ${CONDA_ENV} \
+        ; elif test -f "environment.yml" ; then echo "Using environment.yml" & \
         mamba env create --name ${CONDA_ENV} -f environment.yml  \
-        ; else echo "No conda-lock.yml, conda-linux-64.lock, or environment.yml! *creating default env*" ; \
+        ; else echo "No conda-lock.yml or environment.yml! *creating default env*" ; \
         mamba create --name ${CONDA_ENV} pangeo-notebook \
         ; fi \
         && mamba clean -yaf \