Merge pull request #315 from weiji14/pytorch-notebook

Create pytorch-notebook docker image
pangeo-data · Apr 27, 2022 · 1939869 · 1939869
2 parents b5c38d0 + 3acef32
commit 1939869
Show file tree

Hide file tree

Showing 21 changed files with 1,531 additions and 158 deletions.
diff --git a/.github/workflows/Build.yml b/.github/workflows/Build.yml
@@ -49,7 +49,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        IMAGE: [base-notebook, pangeo-notebook, ml-notebook]
+        IMAGE: [base-notebook, pangeo-notebook, ml-notebook, pytorch-notebook]
     name: ${{ matrix.IMAGE }}
     runs-on: ubuntu-latest
     steps:

diff --git a/.github/workflows/CondaLock.yml b/.github/workflows/CondaLock.yml
@@ -9,6 +9,10 @@ on:
 jobs:
   condalock:
     runs-on: ubuntu-latest
+    defaults:
+      run:
+        shell: bash -l {0}
+
     steps:
       - name: Checkout Repository
         uses: actions/checkout@v2
@@ -23,6 +27,7 @@ jobs:
           echo "---\nbase-notebook:\n---\n" && cat base-notebook/environment.yml
           echo "---\npangeo-notebook:\n---\n" && cat pangeo-notebook/environment.yml
           echo "---\nml-notebook:\n---\n" && cat ml-notebook/environment.yml
+          echo "---\npytorch-notebook:\n---\n" && cat pytorch-notebook/environment.yml
 
       - name: Cache conda
         uses: actions/cache@v2
@@ -46,29 +51,33 @@ jobs:
 
       # Could run as single step in parallel, but would complicate logs...
       - name: Run conda-lock base-notebook
-        shell: bash -l {0}
         run: |
           conda clean --all --yes --force-pkgs-dirs
           cd base-notebook
           conda-lock lock --mamba -f environment.yml -p linux-64
           ../generate-packages-list.py conda-linux-64.lock > packages.txt
 
       - name: Run conda-lock pangeo-notebook
-        shell: bash -l {0}
         run: |
           conda clean --all --yes --force-pkgs-dirs
           cd pangeo-notebook
           conda-lock lock --mamba -f environment.yml -p linux-64
           ../generate-packages-list.py conda-linux-64.lock > packages.txt
 
       - name: Run conda-lock ml-notebook
-        shell: bash -l {0}
         run: |
           conda clean --all --yes --force-pkgs-dirs
           cd ml-notebook
           conda-lock lock --mamba -f environment.yml -f ../pangeo-notebook/environment.yml -p linux-64
           ../generate-packages-list.py conda-linux-64.lock > packages.txt
 
+      - name: Run conda-lock pytorch-notebook
+        run: |
+          conda clean --all --yes --force-pkgs-dirs
+          cd pytorch-notebook
+          conda-lock lock --mamba -f environment.yml -f ../pangeo-notebook/environment.yml -p linux-64
+          ../generate-packages-list.py conda-linux-64.lock > packages.txt
+
       # Commit the change to the PR branch (currently fails if no modified files)
       - name: Commit condalock files to PR
         run: |

diff --git a/.github/workflows/Publish.yml b/.github/workflows/Publish.yml
@@ -16,7 +16,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        IMAGE: [base-image, base-notebook, pangeo-notebook, ml-notebook]
+        IMAGE: [base-image, base-notebook, pangeo-notebook, ml-notebook, pytorch-notebook]
     name: ${{ matrix.IMAGE }}
     runs-on: ubuntu-latest
     steps:

diff --git a/.github/workflows/Test.yml b/.github/workflows/Test.yml
@@ -37,7 +37,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        IMAGE: [base-notebook, pangeo-notebook, ml-notebook]
+        IMAGE: [base-notebook, pangeo-notebook, ml-notebook, pytorch-notebook]
     name: ${{ matrix.IMAGE }}
     runs-on: ubuntu-latest
     steps:

diff --git a/Makefile b/Makefile
@@ -1,6 +1,6 @@
 # Makefile for convenience, (doesn't look for command outputs)
 .PHONY: all
-all: base-image base-notebook pangeo-notebook ml-notebook
+all: base-image base-notebook pangeo-notebook ml-notebook pytorch-notebook
 TESTDIR=/srv/test
 
 .PHONY: base-image
@@ -31,3 +31,11 @@ ml-notebook : base-image
 	../generate-packages-list.py conda-linux-64.lock > packages.txt; \
 	docker build -t pangeo/ml-notebook:master . ; \
 	docker run -w $(TESTDIR) -v $(PWD):$(TESTDIR) pangeo/ml-notebook:master ./run_tests.sh ml-notebook
+
+.PHONY: pytorch-notebook
+pytorch-notebook : base-image
+	cd pytorch-notebook ; \
+	conda-lock lock --mamba -f environment.yml -f ../pangeo-notebook/environment.yml -p linux-64; \
+	../generate-packages-list.py conda-linux-64.lock > packages.txt; \
+	docker build -t pangeo/pytorch-notebook:master . ; \
+	docker run -w $(TESTDIR) -v $(PWD):$(TESTDIR) pangeo/pytorch-notebook:master ./run_tests.sh pytorch-notebook
diff --git a/README.md b/README.md
@@ -12,11 +12,20 @@ Image are hosted on DockerHub: https://hub.docker.com/u/pangeo
 |-----------------|-----------------------------------------------|--------------|-------------|
 | base-image      | Foundational Dockerfile for builds            | ![](https://img.shields.io/docker/image-size/pangeo/base-image?sort=date) | ![](https://img.shields.io/docker/pulls/pangeo/base-image?sort=date)
 | [base-notebook](base-notebook/packages.txt) | minimally functional image for pangeo hubs | ![](https://img.shields.io/docker/image-size/pangeo/base-notebook?sort=date) | ![](https://img.shields.io/docker/pulls/pangeo/base-notebook?sort=date)
-| [pangeo-notebook](pangeo-notebook/packages.txt) | above + core earth science analysis packages | ![](https://img.shields.io/docker/image-size/pangeo/pangeo-notebook?sort=date) | ![](https://img.shields.io/docker/pulls/pangeo/pangeo-notebook?sort=date)
-| [ml-notebook](ml-notebook/packages.txt) | above + GPU-enabled tensorflow2 | ![](https://img.shields.io/docker/image-size/pangeo/ml-notebook?sort=date) | ![](https://img.shields.io/docker/pulls/pangeo/ml-notebook?sort=date)
+| [pangeo-notebook](pangeo-notebook/packages.txt) | base-notebook + core earth science analysis packages | ![](https://img.shields.io/docker/image-size/pangeo/pangeo-notebook?sort=date) | ![](https://img.shields.io/docker/pulls/pangeo/pangeo-notebook?sort=date)
+| [pytorch-notebook](pytorch-notebook/packages.txt) | pangeo-notebook + GPU-enabled pytorch | ![](https://img.shields.io/docker/image-size/pangeo/pytorch-notebook?sort=date) | ![](https://img.shields.io/docker/pulls/pangeo/pytorch-notebook?sort=date)
+| [ml-notebook](ml-notebook/packages.txt) | pangeo-notebook + GPU-enabled tensorflow2 | ![](https://img.shields.io/docker/image-size/pangeo/ml-notebook?sort=date) | ![](https://img.shields.io/docker/pulls/pangeo/ml-notebook?sort=date)
 
 *Click on the image name in the table above for a current list of installed packages and versions*
 
+```mermaid
+graph TD;
+    base-image-->base-notebook;
+    base-notebook-->pangeo-notebook;
+    pangeo-notebook-->pytorch-notebook;
+    pangeo-notebook-->ml-notebook;
+```
+
 ### How to use the pangeo-notebook image with Binder
 A major use-case for these images is running an ephemeral server on the Cloud with BinderHub. Anyone can launch a server running the latest-and-greatest `pangeo-notebook` image with the following URL
 
@@ -27,7 +36,7 @@ Users who need the special features offered by [Pangeo binder](https://binder.pa
 * https://binder.pangeo.io/v2/gh/pangeo-data/pangeo-docker-images/HEAD?urlpath=lab
 * https://aws-uswest2-binder.pangeo.io/v2/gh/pangeo-data/pangeo-docker-images/HEAD?urlpath=lab
 
-NOTE: the links above resolve to the [`pangeo-notebook` image](https://github.com/pangeo-data/pangeo-docker-images/tree/master/pangeo-notebook) and not `base-notebook` or `ml-notebook` that are also defined in this repository. Currently BinderHubs map to a single image definition per repository.
+NOTE: the links above resolve to the [`pangeo-notebook` image](https://github.com/pangeo-data/pangeo-docker-images/tree/master/pangeo-notebook) and not `base-notebook`, `ml-notebook` or `pytorch-notebook` that are also defined in this repository. Currently BinderHubs map to a single image definition per repository.
 
 #### Use nbgitpuller to automatically load content
 

diff --git a/base-notebook/conda-linux-64.lock b/base-notebook/conda-linux-64.lock
@@ -5,14 +5,14 @@
 https://conda.anaconda.org/conda-forge/linux-64/_libgcc_mutex-0.1-conda_forge.tar.bz2#d7c89558ba9fa0495403155b64376d81
 https://conda.anaconda.org/conda-forge/linux-64/ca-certificates-2021.10.8-ha878542_0.tar.bz2#575611b8a84f45960e87722eeb51fa26
 https://conda.anaconda.org/conda-forge/linux-64/ld_impl_linux-64-2.36.1-hea4e1c9_2.tar.bz2#bd4f2e711b39af170e7ff15163fe87ee
-https://conda.anaconda.org/conda-forge/linux-64/libgfortran5-11.2.0-h5c6108e_15.tar.bz2#7c2c0cd937b707aa46f3369d693a1f29
-https://conda.anaconda.org/conda-forge/linux-64/libstdcxx-ng-11.2.0-he4da1e4_15.tar.bz2#4ffda9c4352880439042fbb8a9863e8c
+https://conda.anaconda.org/conda-forge/linux-64/libgfortran5-11.2.0-h5c6108e_16.tar.bz2#ff034874d96195a5c5be34200689b5b7
+https://conda.anaconda.org/conda-forge/linux-64/libstdcxx-ng-11.2.0-he4da1e4_16.tar.bz2#8cfd1cd3273ff187be91b868ddf9a636
 https://conda.anaconda.org/conda-forge/linux-64/pandoc-2.18-ha770c72_0.tar.bz2#518b07342786b362238d22f76789ed59
 https://conda.anaconda.org/conda-forge/noarch/tzdata-2022a-h191b570_0.tar.bz2#84be5301069417a2221187d2f435e0f7
-https://conda.anaconda.org/conda-forge/linux-64/libgfortran-ng-11.2.0-h69a702a_15.tar.bz2#23b9afaf5975e1db0f59d1251425f5d5
-https://conda.anaconda.org/conda-forge/linux-64/libgomp-11.2.0-h1d223b6_15.tar.bz2#5b7bff215ad9f30325dbe86dd983cb49
-https://conda.anaconda.org/conda-forge/linux-64/_openmp_mutex-4.5-1_gnu.tar.bz2#561e277319a41d4f24f5c05a9ef63c04
-https://conda.anaconda.org/conda-forge/linux-64/libgcc-ng-11.2.0-h1d223b6_15.tar.bz2#8041e476e66fcde71f0aa3d614e0efcf
+https://conda.anaconda.org/conda-forge/linux-64/libgfortran-ng-11.2.0-h69a702a_16.tar.bz2#27974aad841c189854df09426b1b9fac
+https://conda.anaconda.org/conda-forge/linux-64/libgomp-11.2.0-h1d223b6_16.tar.bz2#e935fb0c92c6ffb63c736a2012604d72
+https://conda.anaconda.org/conda-forge/linux-64/_openmp_mutex-4.5-2_gnu.tar.bz2#73aaf86a425cc6e73fcf236a5a46396d
+https://conda.anaconda.org/conda-forge/linux-64/libgcc-ng-11.2.0-h1d223b6_16.tar.bz2#71feb63a30085cbce51847d5ef1f769d
 https://conda.anaconda.org/conda-forge/linux-64/bzip2-1.0.8-h7f98852_4.tar.bz2#a1fd65c7ccbf10880423d82bca54eb54
 https://conda.anaconda.org/conda-forge/linux-64/c-ares-1.18.1-h7f98852_0.tar.bz2#f26ef8098fab1f719c91eb760d63381a
 https://conda.anaconda.org/conda-forge/linux-64/giflib-5.2.1-h36c2ea0_2.tar.bz2#626e68ae9cc5912d6adb79d318cf962d
@@ -55,7 +55,7 @@ https://conda.anaconda.org/conda-forge/linux-64/libtiff-4.3.0-h542a066_3.tar.bz2
 https://conda.anaconda.org/conda-forge/linux-64/sqlite-3.38.2-h4ff8645_0.tar.bz2#9d4b7a876d5531fdbb682ed526f8b847
 https://conda.anaconda.org/conda-forge/linux-64/freetype-2.10.4-h0708190_1.tar.bz2#4a06f2ac2e5bfae7b6b245171c3f07aa
 https://conda.anaconda.org/conda-forge/linux-64/lcms2-2.12-hddcbb42_0.tar.bz2#797117394a4aa588de6d741b06fad80f
-https://conda.anaconda.org/conda-forge/linux-64/libcurl-7.82.0-h7bff187_0.tar.bz2#fa26f833ca8796ad44f9561c5402013d
+https://conda.anaconda.org/conda-forge/linux-64/libcurl-7.83.0-h7bff187_0.tar.bz2#e2d939fa77fe69cd50f751961f17786a
 https://conda.anaconda.org/conda-forge/linux-64/libwebp-1.2.2-h3452ae3_0.tar.bz2#c363665b4aabe56aae4f8981cff5b153
 https://conda.anaconda.org/conda-forge/linux-64/openjpeg-2.4.0-hb52868f_1.tar.bz2#b7ad78ad2e9ee155f59e6428406ee824
 https://conda.anaconda.org/conda-forge/linux-64/python-3.9.12-h9a8a25e_1_cpython.tar.bz2#06dadf5df9d340439c2aa32e15099d31
@@ -78,7 +78,7 @@ https://conda.anaconda.org/conda-forge/noarch/idna-3.3-pyhd8ed1ab_0.tar.bz2#40b5
 https://conda.anaconda.org/conda-forge/noarch/ipython_genutils-0.2.0-py_1.tar.bz2#5071c982548b3a20caf70462f04f5287
 https://conda.anaconda.org/conda-forge/noarch/json5-0.9.5-pyh9f0ad1d_0.tar.bz2#10759827a94e6b14996e81fb002c0bda
 https://conda.anaconda.org/conda-forge/noarch/jupyterlab_widgets-1.1.0-pyhd8ed1ab_0.tar.bz2#e963a4a39cf442dbe5503f66edda083d
-https://conda.anaconda.org/conda-forge/noarch/locket-0.2.0-py_2.tar.bz2#709e8671651c7ec3d1ad07800339ff1d
+https://conda.anaconda.org/conda-forge/noarch/locket-1.0.0-pyhd8ed1ab_0.tar.bz2#91e27ef3d05cc772ce627e51cff111c4
 https://conda.anaconda.org/conda-forge/noarch/nest-asyncio-1.5.5-pyhd8ed1ab_0.tar.bz2#dc36c992aec485c0efff619ed2e63957
 https://conda.anaconda.org/conda-forge/noarch/pamela-1.0.0-py_0.tar.bz2#36f6f18d2f3ae0c93d77a9dbedad08c3
 https://conda.anaconda.org/conda-forge/noarch/pandocfilters-1.5.0-pyhd8ed1ab_0.tar.bz2#457c2c8c08e54905d6954e79cb5b5db9
@@ -127,7 +127,7 @@ https://conda.anaconda.org/conda-forge/noarch/matplotlib-inline-0.1.3-pyhd8ed1ab
 https://conda.anaconda.org/conda-forge/linux-64/mistune-0.8.4-py39h3811e60_1005.tar.bz2#95eb8cbf40bccdcb34888c9e56371570
 https://conda.anaconda.org/conda-forge/linux-64/msgpack-python-1.0.3-py39hf939315_1.tar.bz2#9d47ff7dffb54ed6b10bd4e5087af505
 https://conda.anaconda.org/conda-forge/linux-64/multidict-6.0.2-py39hb9d737c_1.tar.bz2#f13f8563cbabc9e3ed69037257124f5d
-https://conda.anaconda.org/conda-forge/linux-64/numpy-1.22.3-py39h18676bf_2.tar.bz2#53c86074bea729d8d33af4cbebadcff3
+https://conda.anaconda.org/conda-forge/linux-64/numpy-1.22.3-py39hc58783e_2.tar.bz2#e682ad4e85c7fda7dd0f0283d3b2ae8e
 https://conda.anaconda.org/conda-forge/noarch/packaging-21.3-pyhd8ed1ab_0.tar.bz2#71f1ab2de48613876becddd496371c85
 https://conda.anaconda.org/conda-forge/noarch/partd-1.2.0-pyhd8ed1ab_0.tar.bz2#0c32f563d7f22e3a34c95cad8cc95651
 https://conda.anaconda.org/conda-forge/noarch/pexpect-4.8.0-pyh9f0ad1d_2.tar.bz2#5909e7b978141dd80d28dbf9de627827
@@ -162,13 +162,13 @@ https://conda.anaconda.org/conda-forge/noarch/dask-core-2022.4.1-pyhd8ed1ab_0.ta
 https://conda.anaconda.org/conda-forge/noarch/importlib_metadata-4.11.3-hd8ed1ab_1.tar.bz2#bd6b6ae37c03e68061574d5e32fe5bd1
 https://conda.anaconda.org/conda-forge/noarch/jinja2-3.1.1-pyhd8ed1ab_0.tar.bz2#40b3f446c61729cdd21ed9d85627df6e
 https://conda.anaconda.org/conda-forge/noarch/jsonschema-4.4.0-pyhd8ed1ab_0.tar.bz2#17ec41acce882e5db4efdcc4c01ca7e0
-https://conda.anaconda.org/conda-forge/noarch/jupyter_client-7.2.2-pyhd8ed1ab_1.tar.bz2#9967e2ad57f060c15da2a7ba1bfb6997
+https://conda.anaconda.org/conda-forge/noarch/jupyter_client-7.3.0-pyhd8ed1ab_0.tar.bz2#3426dc8140fb663110aedfd1a3478db5
 https://conda.anaconda.org/conda-forge/noarch/mako-1.2.0-pyhd8ed1ab_1.tar.bz2#df0535cc0efcc9d418940fc689b1487c
 https://conda.anaconda.org/conda-forge/linux-64/pandas-1.4.2-py39h1832856_1.tar.bz2#264505bcd299b8d564195cfb3e6038f0
 https://conda.anaconda.org/conda-forge/noarch/pip-22.0.4-pyhd8ed1ab_0.tar.bz2#b1239ce8ef2a1eec485c398a683c5bff
-https://conda.anaconda.org/conda-forge/noarch/pygments-2.11.2-pyhd8ed1ab_0.tar.bz2#caef60540e2239e27bf62569a5015e3b
+https://conda.anaconda.org/conda-forge/noarch/pygments-2.12.0-pyhd8ed1ab_0.tar.bz2#cb27e2ded147e5bcc7eafc1c6d343cb3
 https://conda.anaconda.org/conda-forge/linux-64/ruamel.yaml-0.17.21-py39hb9d737c_1.tar.bz2#2b94cf785616198b112170b9838262a4
-https://conda.anaconda.org/conda-forge/linux-64/sqlalchemy-1.4.35-py39hb9d737c_0.tar.bz2#aba3f9a2c924f9b91dbc45ed727b1985
+https://conda.anaconda.org/conda-forge/linux-64/sqlalchemy-1.4.36-py39hb9d737c_0.tar.bz2#e432937ad2a6822f770c1a925a08200c
 https://conda.anaconda.org/conda-forge/noarch/stack_data-0.2.0-pyhd8ed1ab_0.tar.bz2#8c0ce3e6bf18a0c810125aef58a2a6f3
 https://conda.anaconda.org/conda-forge/linux-64/terminado-0.13.3-py39hf3d152e_1.tar.bz2#bebf6da1adb04ed902ff335e61438a6e
 https://conda.anaconda.org/conda-forge/linux-64/yarl-1.7.2-py39hb9d737c_2.tar.bz2#f2f2df8a6238bf5e68c3aaf0e86e08d0
@@ -192,7 +192,7 @@ https://conda.anaconda.org/conda-forge/noarch/kubernetes_asyncio-22.6.4-pyhd8ed1
 https://conda.anaconda.org/conda-forge/noarch/nbconvert-core-6.5.0-pyhd8ed1ab_0.tar.bz2#42f74c4b38a099025167e76a7437edf1
 https://conda.anaconda.org/conda-forge/noarch/requests-2.27.1-pyhd8ed1ab_0.tar.bz2#7c1c427246b057b8fa97200ecdb2ed62
 https://conda.anaconda.org/conda-forge/noarch/dask-2022.4.1-pyhd8ed1ab_0.tar.bz2#c7b70d267db42d97e1510d5906b6692c
-https://conda.anaconda.org/conda-forge/noarch/google-auth-2.6.5-pyh6c4a22f_0.tar.bz2#9744ee1831eb2f6cbe9d741115170fbd
+https://conda.anaconda.org/conda-forge/noarch/google-auth-2.6.6-pyh6c4a22f_0.tar.bz2#ad440163381987054135e5f397bcb95d
 https://conda.anaconda.org/conda-forge/linux-64/ipykernel-6.13.0-py39hef51801_0.tar.bz2#9fd2a497e68e4c715d9fdf3f36b44072
 https://conda.anaconda.org/conda-forge/noarch/jupyter_server-1.16.0-pyhd8ed1ab_1.tar.bz2#3b64b67485061d8296f55b1f463c324e
 https://conda.anaconda.org/conda-forge/noarch/jupyterhub-base-2.2.2-pyhd8ed1ab_1.tar.bz2#9f35d566b71bacc69429c8bad779ee50

diff --git a/base-notebook/packages.txt b/base-notebook/packages.txt
@@ -49,7 +49,7 @@ freetype==2.10.4
 frozenlist==1.3.0
 fsspec==2022.3.0
 giflib==5.2.1
-google-auth==2.6.5
+google-auth==2.6.6
 greenlet==1.1.2
 heapdict==1.0.1
 idna==3.3
@@ -67,7 +67,7 @@ jpeg==9e
 json5==0.9.5
 jsonschema==4.4.0
 jupyter-server-proxy==3.2.1
-jupyter_client==7.2.2
+jupyter_client==7.3.0
 jupyter_core==4.9.2
 jupyter_server==1.16.0
 jupyter_telemetry==0.1.0
@@ -85,7 +85,7 @@ ld_impl_linux-64==2.36.1
 lerc==3.0
 libblas==3.9.0
 libcblas==3.9.0
-libcurl==7.82.0
+libcurl==7.83.0
 libdeflate==1.10
 libedit==3.1.20191231
 libev==4.33
@@ -108,7 +108,7 @@ libwebp==1.2.2
 libwebp-base==1.2.2
 libxcb==1.13
 libzlib==1.2.11
-locket==0.2.0
+locket==1.0.0
 lz4==4.0.0
 lz4-c==1.9.3
 mako==1.2.0
@@ -155,7 +155,7 @@ pyasn1==0.4.8
 pyasn1-modules==0.2.7
 pycparser==2.21
 pycurl==7.45.1
-pygments==2.11.2
+pygments==2.12.0
 pyjwt==2.3.0
 pyopenssl==22.0.0
 pyparsing==3.0.8
@@ -184,7 +184,7 @@ six==1.16.0
 sniffio==1.2.0
 sortedcontainers==2.4.0
 soupsieve==2.3.1
-sqlalchemy==1.4.35
+sqlalchemy==1.4.36
 sqlite==3.38.2
 stack_data==0.2.0
 tblib==1.7.0