From 1910d770eda8e82925799aa8b261ea7a9fa43c2d Mon Sep 17 00:00:00 2001 From: YuviPanda Date: Sat, 3 Jul 2021 00:14:00 +0530 Subject: [PATCH 001/132] Add publichealth hub - Modeled after the ischool hub, but additional packages that were already installed for specific PH courses are installed - Admins for PH hub carried over from the R hub + https://github.com/berkeley-dsep-infra/datahub/pull/2441 --- Dockerfile | 71 +++++++++++++++++++ class-libs.R | 20 ++++++ infra-requirements.txt | 34 +++++++++ install-miniforge.bash | 42 +++++++++++ .../Untitled-checkpoint.ipynb | 6 ++ r-packages/ph-142.r | 28 ++++++++ r-packages/ph-290.r | 16 +++++ requirements.txt | 2 + 8 files changed, 219 insertions(+) create mode 100644 Dockerfile create mode 100644 class-libs.R create mode 100644 infra-requirements.txt create mode 100755 install-miniforge.bash create mode 100644 r-packages/.ipynb_checkpoints/Untitled-checkpoint.ipynb create mode 100755 r-packages/ph-142.r create mode 100755 r-packages/ph-290.r create mode 100644 requirements.txt diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..153ecf5 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,71 @@ +FROM rocker/geospatial:4.1.0 + +ENV NB_USER rstudio +ENV NB_UID 1000 +ENV CONDA_DIR /srv/conda + +# Set ENV for all programs... +ENV PATH ${CONDA_DIR}/bin:$PATH + +# And set ENV for R! It doesn't read from the environment... +RUN echo "PATH=${PATH}" >> /usr/local/lib/R/etc/Renviron.site + +# Add PATH to /etc/profile so it gets picked up by the terminal +RUN echo "PATH=${PATH}" >> /etc/profile +RUN echo "export PATH" >> /etc/profile + +ENV HOME /home/${NB_USER} + +WORKDIR ${HOME} + +# Install packages needed by notebook-as-pdf +# nodejs for installing notebook / jupyterhub from source +# libarchive-dev for https://github.com/berkeley-dsep-infra/datahub/issues/1997 +RUN apt-get update > /dev/null && \ + apt-get install --yes \ + libx11-xcb1 \ + libxtst6 \ + libxrandr2 \ + libasound2 \ + libpangocairo-1.0-0 \ + libatk1.0-0 \ + libatk-bridge2.0-0 \ + libgtk-3-0 \ + libnss3 \ + libxss1 \ + libssl1.1 \ + fonts-symbola \ + gdebi-core \ + nodejs npm > /dev/null && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* + +# 1.3.959 is latest version that works with jupyter-rsession-proxy +# See https://github.com/jupyterhub/jupyter-rsession-proxy/issues/93#issuecomment-725874693 +ENV RSTUDIO_URL https://download2.rstudio.org/server/bionic/amd64/rstudio-server-1.3.959-amd64.deb +RUN curl --silent --location --fail ${RSTUDIO_URL} > /tmp/rstudio.deb && \ + dpkg -i /tmp/rstudio.deb && \ + rm /tmp/rstudio.deb + +COPY install-miniforge.bash /tmp/install-miniforge.bash +RUN /tmp/install-miniforge.bash + +USER ${NB_USER} + +COPY infra-requirements.txt /tmp/infra-requirements.txt +RUN pip install --no-cache-dir -r /tmp/infra-requirements.txt + +COPY requirements.txt /tmp/requirements.txt +RUN pip install --no-cache-dir -r /tmp/requirements.txt + +# Install IRKernel +RUN R --quiet -e "install.packages('IRkernel', quiet = TRUE)" && \ + R --quiet -e "IRkernel::installspec(prefix='${CONDA_DIR}')" + +COPY class-libs.R /tmp/class-libs.R + +COPY r-packages/ph-290.r /tmp/r-packages/ +RUN r /tmp/r-packages/ph-290.r + +COPY r-packages/ph-142.r /tmp/r-packages/ +RUN r /tmp/r-packages/ph-142.r \ No newline at end of file diff --git a/class-libs.R b/class-libs.R new file mode 100644 index 0000000..1c93435 --- /dev/null +++ b/class-libs.R @@ -0,0 +1,20 @@ +#!/usr/bin/env Rscript + +class_libs_install_version <- function(class_name, class_libs) { + print(paste("Installing packages for", class_name)) + for (i in seq(1, length(class_libs), 2)) { + installed_packages <- rownames(installed.packages()) + package_name = class_libs[i] + version = class_libs[i+1] + # Only install packages if they haven't already been installed! + # devtools doesn't do that by default + if (!package_name %in% installed_packages) { + print(paste("Installing", package_name, version)) + devtools::install_version(package_name, version, quiet=TRUE) + } else { + # FIXME: This ignores version incompatibilities :'( + print(paste("Not installing", package_name, " as it is already installed")) + } + } + print(paste("Done installing packages for", class_name)) +} diff --git a/infra-requirements.txt b/infra-requirements.txt new file mode 100644 index 0000000..ffa4c2b --- /dev/null +++ b/infra-requirements.txt @@ -0,0 +1,34 @@ +# WARNING: Original source at scripts/infra-packages/requirements.txt +# PLEASE DO NOT EDIT ELSEWHERE +# After editing scripts/infra-packages/requirements.txt, please run +# scripts/infra-packages/sync.bash. + +# This file pins versions of notebook related python packages we want +# across all hubs. This makes sure we don't need to upgrade them +# everwhere one by one. + +# FIXME: Freeze this to get exact versions of all dependencies +notebook==6.4.0 +# from @chrispyles +# i see that nbformat was unpinned. since the pin, because of the new cell id +# parameter in notebook format v4 the notebook validation is failing whenever +# someone opens a notebook created w/ nbformat>=5.1.0. is it possible to pin +# below 5.1.0 while notebook patches this? +nbformat<5.1.0 +jupyterlab==3.0.16 +nbconvert==6.1.0 +retrolab==0.2.1 +nbgitpuller==0.10.0 +jupyter-resource-usage==0.5.1 +# Matches version in images/hub/Dockerfile +# https://github.com/jupyterhub/jupyterhub/pull/3381/ means we need to +# match jupyterhub versions on server and client again +git+https://github.com/jupyterhub/jupyterhub@11f00dbbe794191d328c6c2a44a9e85f7eb6ede5 +appmode==0.8.0 +ipywidgets==7.6.3 +otter-grader==2.2.0 +jupyter-tree-download==1.0.1 +git-credential-helpers==0.2 +# Enough people like this, let's load it in. +jupyter-contrib-nbextensions==0.5.1 +jupyter_nbextensions_configurator==0.4.1 \ No newline at end of file diff --git a/install-miniforge.bash b/install-miniforge.bash new file mode 100755 index 0000000..b893b0f --- /dev/null +++ b/install-miniforge.bash @@ -0,0 +1,42 @@ +#!/bin/bash +# This downloads and installs a pinned version of miniconda +set -ex + +cd $(dirname $0) +MINIFORGE_VERSION=4.10.2-0 + +URL="https://github.com/conda-forge/miniforge/releases/download/${MINIFORGE_VERSION}/Miniforge3-${MINIFORGE_VERSION}-Linux-x86_64.sh" +INSTALLER_PATH=/tmp/miniforge-installer.sh + +# make sure we don't do anything funky with user's $HOME +# since this is run as root +unset HOME + +wget --quiet $URL -O ${INSTALLER_PATH} +chmod +x ${INSTALLER_PATH} + +bash ${INSTALLER_PATH} -b -p ${CONDA_DIR} +export PATH="${CONDA_DIR}/bin:$PATH" + +# Do not attempt to auto update conda or dependencies +conda config --system --set auto_update_conda false +conda config --system --set show_channel_urls true + +# empty conda history file, +# which seems to result in some effective pinning of packages in the initial env, +# which we don't intend. +# this file must not be *removed*, however +echo '' > ${CONDA_DIR}/conda-meta/history + +# Clean things out! +conda clean --all -f -y + +# Remove the big installer so we don't increase docker image size too much +rm ${INSTALLER_PATH} + +# Remove the pip cache created as part of installing miniconda +rm -rf /root/.cache + +chown -R $NB_USER:$NB_USER ${CONDA_DIR} + +conda list -n root diff --git a/r-packages/.ipynb_checkpoints/Untitled-checkpoint.ipynb b/r-packages/.ipynb_checkpoints/Untitled-checkpoint.ipynb new file mode 100644 index 0000000..7fec515 --- /dev/null +++ b/r-packages/.ipynb_checkpoints/Untitled-checkpoint.ipynb @@ -0,0 +1,6 @@ +{ + "cells": [], + "metadata": {}, + "nbformat": 4, + "nbformat_minor": 4 +} diff --git a/r-packages/ph-142.r b/r-packages/ph-142.r new file mode 100755 index 0000000..e98df67 --- /dev/null +++ b/r-packages/ph-142.r @@ -0,0 +1,28 @@ +#!/usr/bin/env Rscript + # From https://github.com/berkeley-dsep-infra/datahub/issues/881 +print("Installing packages for PH142") + +source("/tmp/class-libs.R") + +class_name = "PH142" + +class_libs = c( + "fGarch", "3042.83.2", + "SASxport", "1.7.0", + "googlesheets", "0.3.0", + "googledrive", "1.0.1", + "ggrepel", "0.9.0", + "infer", "0.5.3", + "janitor", "2.1.0", + "latex2exp", "0.4.0", + "measurements", "1.4.0", + "dagitty", "0.3-0", + "cowplot", "1.1.1", + "patchwork", "1.1.1", + "tigris", "1.0", + "googlesheets4", "0.2.0" +) + +class_libs_install_version(class_name, class_libs) + +print("Done installing packages for PH142") diff --git a/r-packages/ph-290.r b/r-packages/ph-290.r new file mode 100755 index 0000000..8e49fc4 --- /dev/null +++ b/r-packages/ph-290.r @@ -0,0 +1,16 @@ +#!/usr/bin/env Rscript +# For https://github.com/berkeley-dsep-infra/datahub/issues/1921 +print("Installing packages for PH 290W") + +source("/tmp/class-libs.R") + +class_name = "PH 290W" + +class_libs = c( + "kableExtra", "1.3.1", + "plotly", "4.9.2.2", + "ggthemes", "4.2.0", + "formattable", "0.2.1" +) + +class_libs_install_version(class_name, class_libs) diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..06a835b --- /dev/null +++ b/requirements.txt @@ -0,0 +1,2 @@ +# RStudio support +jupyter-rsession-proxy==1.3 From 37cfb58078a95216f4189a50a0724beb2d1c7445 Mon Sep 17 00:00:00 2001 From: YuviPanda Date: Tue, 6 Jul 2021 12:06:47 +0530 Subject: [PATCH 002/132] publichealth: Install ottr Fixes https://github.com/berkeley-dsep-infra/datahub/issues/2483 --- r-packages/ph-142.r | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/r-packages/ph-142.r b/r-packages/ph-142.r index e98df67..3a5b69e 100755 --- a/r-packages/ph-142.r +++ b/r-packages/ph-142.r @@ -1,5 +1,5 @@ #!/usr/bin/env Rscript - # From https://github.com/berkeley-dsep-infra/datahub/issues/881 +# From https://github.com/berkeley-dsep-infra/datahub/issues/881 print("Installing packages for PH142") source("/tmp/class-libs.R") @@ -23,6 +23,10 @@ class_libs = c( "googlesheets4", "0.2.0" ) +# https://github.com/berkeley-dsep-infra/datahub/issues/2483 +print("Installing ottr...") +devtools::install_github('ucbds-infra/ottr', ref='0.1.0', upgrade_dependencies=FALSE, quiet=FALSE) + class_libs_install_version(class_name, class_libs) print("Done installing packages for PH142") From 30553a1c732438e0fa20afee8ee8f6240b5055ed Mon Sep 17 00:00:00 2001 From: YuviPanda Date: Wed, 7 Jul 2021 22:48:07 +0530 Subject: [PATCH 003/132] Pin R to 4.0.5 for publichealth & ischool Ref https://github.com/berkeley-dsep-infra/datahub/issues/2488 --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 153ecf5..1cc53b3 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM rocker/geospatial:4.1.0 +FROM rocker/geospatial:4.0.5 ENV NB_USER rstudio ENV NB_UID 1000 From cff592d47d0db1f896855ca47d61dc29286304df Mon Sep 17 00:00:00 2001 From: YuviPanda Date: Thu, 8 Jul 2021 23:06:58 +0530 Subject: [PATCH 004/132] Add popularity-contest to image https://github.com/yuvipanda/python-popcontest can be used to put metrics into prometheus about how often python packages are being used - this will help us clean up unused libraries. See https://discourse.jupyter.org/t/request-for-implementation-instrument-libraries-actively-used-by-users-on-a-jupyterhub/7994 for more information. More infrastructure is needed for this --- infra-requirements.txt | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/infra-requirements.txt b/infra-requirements.txt index ffa4c2b..6b6a263 100644 --- a/infra-requirements.txt +++ b/infra-requirements.txt @@ -31,4 +31,7 @@ jupyter-tree-download==1.0.1 git-credential-helpers==0.2 # Enough people like this, let's load it in. jupyter-contrib-nbextensions==0.5.1 -jupyter_nbextensions_configurator==0.4.1 \ No newline at end of file +jupyter_nbextensions_configurator==0.4.1 +# Measure popularity of different packages in our hubs +# https://discourse.jupyter.org/t/request-for-implementation-instrument-libraries-actively-used-by-users-on-a-jupyterhub/7994?u=yuvipanda +popularity-contest==0.1.1 \ No newline at end of file From 70a27a470fe98e4dbb83871bb5223490f8d682c7 Mon Sep 17 00:00:00 2001 From: YuviPanda Date: Fri, 9 Jul 2021 16:46:37 +0530 Subject: [PATCH 005/132] Bump version of popularity_contest --- infra-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/infra-requirements.txt b/infra-requirements.txt index 6b6a263..b005ef7 100644 --- a/infra-requirements.txt +++ b/infra-requirements.txt @@ -34,4 +34,4 @@ jupyter-contrib-nbextensions==0.5.1 jupyter_nbextensions_configurator==0.4.1 # Measure popularity of different packages in our hubs # https://discourse.jupyter.org/t/request-for-implementation-instrument-libraries-actively-used-by-users-on-a-jupyterhub/7994?u=yuvipanda -popularity-contest==0.1.1 \ No newline at end of file +popularity-contest==0.2 \ No newline at end of file From 342af184fcc060316ed78e068aaa75f6269dec60 Mon Sep 17 00:00:00 2001 From: YuviPanda Date: Sat, 10 Jul 2021 23:28:05 +0530 Subject: [PATCH 006/132] Bump version of popularity_contest --- infra-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/infra-requirements.txt b/infra-requirements.txt index b005ef7..c93efa0 100644 --- a/infra-requirements.txt +++ b/infra-requirements.txt @@ -34,4 +34,4 @@ jupyter-contrib-nbextensions==0.5.1 jupyter_nbextensions_configurator==0.4.1 # Measure popularity of different packages in our hubs # https://discourse.jupyter.org/t/request-for-implementation-instrument-libraries-actively-used-by-users-on-a-jupyterhub/7994?u=yuvipanda -popularity-contest==0.2 \ No newline at end of file +popularity-contest==0.3 \ No newline at end of file From 6d9812b5bffda9312e875e5908a27e7608f18d77 Mon Sep 17 00:00:00 2001 From: YuviPanda Date: Sun, 11 Jul 2021 23:39:03 +0530 Subject: [PATCH 007/132] Bump popularity contest version --- infra-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/infra-requirements.txt b/infra-requirements.txt index c93efa0..82c8f0c 100644 --- a/infra-requirements.txt +++ b/infra-requirements.txt @@ -34,4 +34,4 @@ jupyter-contrib-nbextensions==0.5.1 jupyter_nbextensions_configurator==0.4.1 # Measure popularity of different packages in our hubs # https://discourse.jupyter.org/t/request-for-implementation-instrument-libraries-actively-used-by-users-on-a-jupyterhub/7994?u=yuvipanda -popularity-contest==0.3 \ No newline at end of file +popularity-contest==0.4.1 \ No newline at end of file From 8436b05a80a14eda7bc56f7d07ae1c93b5902cf6 Mon Sep 17 00:00:00 2001 From: ryanlovett Date: Thu, 5 Aug 2021 17:11:51 -0700 Subject: [PATCH 008/132] Security update for CVE-2021-32797. --- infra-requirements.txt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/infra-requirements.txt b/infra-requirements.txt index 82c8f0c..0494136 100644 --- a/infra-requirements.txt +++ b/infra-requirements.txt @@ -8,14 +8,14 @@ # everwhere one by one. # FIXME: Freeze this to get exact versions of all dependencies -notebook==6.4.0 +notebook==6.4.1 # from @chrispyles # i see that nbformat was unpinned. since the pin, because of the new cell id # parameter in notebook format v4 the notebook validation is failing whenever # someone opens a notebook created w/ nbformat>=5.1.0. is it possible to pin # below 5.1.0 while notebook patches this? nbformat<5.1.0 -jupyterlab==3.0.16 +jupyterlab==3.0.17 nbconvert==6.1.0 retrolab==0.2.1 nbgitpuller==0.10.0 @@ -34,4 +34,4 @@ jupyter-contrib-nbextensions==0.5.1 jupyter_nbextensions_configurator==0.4.1 # Measure popularity of different packages in our hubs # https://discourse.jupyter.org/t/request-for-implementation-instrument-libraries-actively-used-by-users-on-a-jupyterhub/7994?u=yuvipanda -popularity-contest==0.4.1 \ No newline at end of file +popularity-contest==0.4.1 From 5d969299fbbfa78345ac0b5ab1ba3b909b9f7fe8 Mon Sep 17 00:00:00 2001 From: YuviPanda Date: Thu, 12 Aug 2021 10:54:34 +0530 Subject: [PATCH 009/132] Bump infra package versions We now unpin nbformat. Hopefully the notebooks have been fixed since --- infra-requirements.txt | 16 +++++----------- 1 file changed, 5 insertions(+), 11 deletions(-) diff --git a/infra-requirements.txt b/infra-requirements.txt index 0494136..4b5c604 100644 --- a/infra-requirements.txt +++ b/infra-requirements.txt @@ -8,16 +8,10 @@ # everwhere one by one. # FIXME: Freeze this to get exact versions of all dependencies -notebook==6.4.1 -# from @chrispyles -# i see that nbformat was unpinned. since the pin, because of the new cell id -# parameter in notebook format v4 the notebook validation is failing whenever -# someone opens a notebook created w/ nbformat>=5.1.0. is it possible to pin -# below 5.1.0 while notebook patches this? -nbformat<5.1.0 -jupyterlab==3.0.17 +notebook==6.4.3 +jupyterlab==3.1.4 nbconvert==6.1.0 -retrolab==0.2.1 +retrolab==0.2.2 nbgitpuller==0.10.0 jupyter-resource-usage==0.5.1 # Matches version in images/hub/Dockerfile @@ -26,7 +20,7 @@ jupyter-resource-usage==0.5.1 git+https://github.com/jupyterhub/jupyterhub@11f00dbbe794191d328c6c2a44a9e85f7eb6ede5 appmode==0.8.0 ipywidgets==7.6.3 -otter-grader==2.2.0 +otter-grader==2.2.5 jupyter-tree-download==1.0.1 git-credential-helpers==0.2 # Enough people like this, let's load it in. @@ -34,4 +28,4 @@ jupyter-contrib-nbextensions==0.5.1 jupyter_nbextensions_configurator==0.4.1 # Measure popularity of different packages in our hubs # https://discourse.jupyter.org/t/request-for-implementation-instrument-libraries-actively-used-by-users-on-a-jupyterhub/7994?u=yuvipanda -popularity-contest==0.4.1 +popularity-contest==0.4.1 \ No newline at end of file From 56162c6f96fac0991914d9150ae62b05bde5b125 Mon Sep 17 00:00:00 2001 From: YuviPanda Date: Thu, 12 Aug 2021 12:17:33 +0530 Subject: [PATCH 010/132] Bump otter-grader down It *might* be installing rpy2, which is not possible when R isn't installed. This causes image failures for the data8 and julia hubs, as they have no R --- infra-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/infra-requirements.txt b/infra-requirements.txt index 4b5c604..98f0992 100644 --- a/infra-requirements.txt +++ b/infra-requirements.txt @@ -20,7 +20,7 @@ jupyter-resource-usage==0.5.1 git+https://github.com/jupyterhub/jupyterhub@11f00dbbe794191d328c6c2a44a9e85f7eb6ede5 appmode==0.8.0 ipywidgets==7.6.3 -otter-grader==2.2.5 +otter-grader==2.2.0 jupyter-tree-download==1.0.1 git-credential-helpers==0.2 # Enough people like this, let's load it in. From adf3d66d6645c2ce8f5808a817a3c2467c7c1e55 Mon Sep 17 00:00:00 2001 From: YuviPanda Date: Thu, 12 Aug 2021 15:47:12 +0530 Subject: [PATCH 011/132] Add RISE to all hubs Fixes https://github.com/berkeley-dsep-infra/datahub/issues/2527 --- infra-requirements.txt | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/infra-requirements.txt b/infra-requirements.txt index 98f0992..42059fc 100644 --- a/infra-requirements.txt +++ b/infra-requirements.txt @@ -28,4 +28,6 @@ jupyter-contrib-nbextensions==0.5.1 jupyter_nbextensions_configurator==0.4.1 # Measure popularity of different packages in our hubs # https://discourse.jupyter.org/t/request-for-implementation-instrument-libraries-actively-used-by-users-on-a-jupyterhub/7994?u=yuvipanda -popularity-contest==0.4.1 \ No newline at end of file +popularity-contest==0.4.1 +# RISE is useful for presentations - see https://github.com/berkeley-dsep-infra/datahub/issues/2527 +RISE==5.7.1 \ No newline at end of file From e574a989538ee8d235fe2608b16e2a8956a6f2b3 Mon Sep 17 00:00:00 2001 From: YuviPanda Date: Tue, 17 Aug 2021 17:37:13 +0530 Subject: [PATCH 012/132] Bump python packages in publichealth hub --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 06a835b..3479f00 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,2 +1,2 @@ # RStudio support -jupyter-rsession-proxy==1.3 +jupyter-rsession-proxy==1.4 From 8fafc4c4fef4317834cdf7aa4d4315e200c6b920 Mon Sep 17 00:00:00 2001 From: YuviPanda Date: Tue, 17 Aug 2021 23:32:55 +0530 Subject: [PATCH 013/132] Test nbgitpuller master Brings in https://github.com/jupyterhub/nbgitpuller/pull/193, to test if it works ok --- infra-requirements.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/infra-requirements.txt b/infra-requirements.txt index 42059fc..8ec9a9d 100644 --- a/infra-requirements.txt +++ b/infra-requirements.txt @@ -12,7 +12,8 @@ notebook==6.4.3 jupyterlab==3.1.4 nbconvert==6.1.0 retrolab==0.2.2 -nbgitpuller==0.10.0 +# To test https://github.com/jupyterhub/nbgitpuller/pull/193 +git+https://github.com/jupyterhub/nbgitpuller@8401bd6bad8ce48cab89d69a348fe9ea42487730 jupyter-resource-usage==0.5.1 # Matches version in images/hub/Dockerfile # https://github.com/jupyterhub/jupyterhub/pull/3381/ means we need to From c2bb0c577ed1357c941fc2d56bc0348bca77654c Mon Sep 17 00:00:00 2001 From: YuviPanda Date: Wed, 18 Aug 2021 04:12:25 +0530 Subject: [PATCH 014/132] Install voila on all hubs --- infra-requirements.txt | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/infra-requirements.txt b/infra-requirements.txt index 8ec9a9d..35a49f5 100644 --- a/infra-requirements.txt +++ b/infra-requirements.txt @@ -31,4 +31,6 @@ jupyter_nbextensions_configurator==0.4.1 # https://discourse.jupyter.org/t/request-for-implementation-instrument-libraries-actively-used-by-users-on-a-jupyterhub/7994?u=yuvipanda popularity-contest==0.4.1 # RISE is useful for presentations - see https://github.com/berkeley-dsep-infra/datahub/issues/2527 -RISE==5.7.1 \ No newline at end of file +RISE==5.7.1 +# https://github.com/berkeley-dsep-infra/datahub/issues/784 +voila==0.2.10 \ No newline at end of file From 56a8b6d8d3b062aa49c21c6a9225e33bbb0caba6 Mon Sep 17 00:00:00 2001 From: YuviPanda Date: Wed, 18 Aug 2021 16:43:41 +0530 Subject: [PATCH 015/132] Revert "Test nbgitpuller master" Things mostly worked, but a couple small bugs exist: https://github.com/jupyterhub/nbgitpuller/pull/193#issuecomment-900690746 This reverts commit 8fafc4c4fef4317834cdf7aa4d4315e200c6b920. --- infra-requirements.txt | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/infra-requirements.txt b/infra-requirements.txt index 35a49f5..d750cba 100644 --- a/infra-requirements.txt +++ b/infra-requirements.txt @@ -12,8 +12,7 @@ notebook==6.4.3 jupyterlab==3.1.4 nbconvert==6.1.0 retrolab==0.2.2 -# To test https://github.com/jupyterhub/nbgitpuller/pull/193 -git+https://github.com/jupyterhub/nbgitpuller@8401bd6bad8ce48cab89d69a348fe9ea42487730 +nbgitpuller==0.10.0 jupyter-resource-usage==0.5.1 # Matches version in images/hub/Dockerfile # https://github.com/jupyterhub/jupyterhub/pull/3381/ means we need to From cfd6fa69ccb14e9ce537ea9ef8d8cb49232ef31a Mon Sep 17 00:00:00 2001 From: Jonathan Felder Date: Thu, 19 Aug 2021 10:16:58 -0700 Subject: [PATCH 016/132] Installing table one for PH252 issue #2556 --- Dockerfile | 5 ++++- r-packages/ph-252.r | 14 ++++++++++++++ 2 files changed, 18 insertions(+), 1 deletion(-) create mode 100755 r-packages/ph-252.r diff --git a/Dockerfile b/Dockerfile index 1cc53b3..0040148 100644 --- a/Dockerfile +++ b/Dockerfile @@ -68,4 +68,7 @@ COPY r-packages/ph-290.r /tmp/r-packages/ RUN r /tmp/r-packages/ph-290.r COPY r-packages/ph-142.r /tmp/r-packages/ -RUN r /tmp/r-packages/ph-142.r \ No newline at end of file +RUN r /tmp/r-packages/ph-142.r + +COPY r-packages/ph-252.r /tmp/r-packages/ +RUN r /tmp/r-packages/ph-252.r diff --git a/r-packages/ph-252.r b/r-packages/ph-252.r new file mode 100755 index 0000000..d0aeecc --- /dev/null +++ b/r-packages/ph-252.r @@ -0,0 +1,14 @@ +#!/usr/bin/env Rscript +# For https://github.com/berkeley-dsep-infra/datahub/issues/2556 +# Fall 2021 +print("Installing packages for PH 252") + +source("/tmp/class-libs.R") + +class_name = "PH 252" + +class_libs = c( + "tableone", "0.13.0" +) + +class_libs_install_version(class_name, class_libs) From 38c66f098726ca192172b476de47a6ea3e49cac4 Mon Sep 17 00:00:00 2001 From: Jonathan Felder Date: Thu, 19 Aug 2021 11:45:54 -0700 Subject: [PATCH 017/132] Grabbing 0.12.0 from github which is the latest tagged release, issue #2556 --- r-packages/ph-252.r | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/r-packages/ph-252.r b/r-packages/ph-252.r index d0aeecc..316d2a3 100755 --- a/r-packages/ph-252.r +++ b/r-packages/ph-252.r @@ -6,9 +6,4 @@ print("Installing packages for PH 252") source("/tmp/class-libs.R") class_name = "PH 252" - -class_libs = c( - "tableone", "0.13.0" -) - -class_libs_install_version(class_name, class_libs) +devtools::install_github('kaz-yos/tableone', ref='7cfcd71') From 3fd2c34dee966915111d9b0278271c58aabc553a Mon Sep 17 00:00:00 2001 From: YuviPanda Date: Fri, 20 Aug 2021 14:15:56 +0530 Subject: [PATCH 018/132] Switch back to 1.4.x of JupyterHub We were running on a fairly recent, unreleased version of JupyterHub to get access to https://github.com/jupyterhub/jupyterhub/pull/3488. It provides a seamless experience for users logging in to data8.datahub.berkeley.edu, as it uses datahub.berkeley.edu as the auth provider. However, it had also brought in a bunch of unreleased changes that broke the idle culler (https://github.com/berkeley-dsep-infra/datahub/issues/2452) - probably around pagination. https://github.com/jupyterhub/jupyterhub/pull/3579 backports just the change we want to the last released version of JupyterHub, and should hopefully fix culling. --- infra-requirements.txt | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/infra-requirements.txt b/infra-requirements.txt index d750cba..0180978 100644 --- a/infra-requirements.txt +++ b/infra-requirements.txt @@ -15,9 +15,7 @@ retrolab==0.2.2 nbgitpuller==0.10.0 jupyter-resource-usage==0.5.1 # Matches version in images/hub/Dockerfile -# https://github.com/jupyterhub/jupyterhub/pull/3381/ means we need to -# match jupyterhub versions on server and client again -git+https://github.com/jupyterhub/jupyterhub@11f00dbbe794191d328c6c2a44a9e85f7eb6ede5 +git+https://github.com/meeseeksmachine/jupyterhub@auto-backport-of-pr-3488-on-1.4.x appmode==0.8.0 ipywidgets==7.6.3 otter-grader==2.2.0 From 0a6ac4dcd9b95fad9ad91e6777b4912c1b1f7b2c Mon Sep 17 00:00:00 2001 From: YuviPanda Date: Sat, 21 Aug 2021 00:01:27 +0530 Subject: [PATCH 019/132] Pin jupyter-client<7.0 7.0, released today, seems to have a version conflict on nest-asyncio with jupyter-server-proxy: [W 2021-08-20 11:22:41.520 SingleUserNotebookApp notebookapp:2034] Error loading server extension jupyter_server_proxy Traceback (most recent call last): File "/opt/conda/lib/python3.9/site-packages/notebook/notebookapp.py", line 2030, in init_server_extensions func(self) File "/opt/conda/lib/python3.9/site-packages/jupyter_server_proxy/__init__.py", line 27, in _load_jupyter_server_extension server_processes += get_entrypoint_server_processes() File "/opt/conda/lib/python3.9/site-packages/jupyter_server_proxy/config.py", line 81, in get_entrypoint_server_processes make_server_process(entry_point.name, entry_point.load()()) File "/opt/conda/lib/python3.9/site-packages/pkg_resources/__init__.py", line 2471, in load self.require(*args, **kwargs) File "/opt/conda/lib/python3.9/site-packages/pkg_resources/__init__.py", line 2494, in require items = working_set.resolve(reqs, env, installer, extras=self.extras) File "/opt/conda/lib/python3.9/site-packages/pkg_resources/__init__.py", line 790, in resolve raise VersionConflict(dist, req).with_context(dependent_req) pkg_resources.ContextualVersionConflict: (nest-asyncio 1.2.0 (/opt/conda/lib/python3.9/site-packages), Requirement.parse('nest-asyncio>=1.5'), {'jupyter-client'}) This pin should help contain the damage while we investigate --- infra-requirements.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/infra-requirements.txt b/infra-requirements.txt index d750cba..d5d0bfb 100644 --- a/infra-requirements.txt +++ b/infra-requirements.txt @@ -9,6 +9,8 @@ # FIXME: Freeze this to get exact versions of all dependencies notebook==6.4.3 +# Seems to cause a version conflict with jupyter-server-proxy +jupyter-client<7.0 jupyterlab==3.1.4 nbconvert==6.1.0 retrolab==0.2.2 From 75ad26bf37a5f8fca70ec628ce56b2837df9f4e1 Mon Sep 17 00:00:00 2001 From: Jonathan Felder Date: Fri, 20 Aug 2021 16:20:26 -0700 Subject: [PATCH 020/132] Installing epi and foreign for issue #2524 --- r-packages/ph-252.r | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/r-packages/ph-252.r b/r-packages/ph-252.r index 316d2a3..5d37f3a 100755 --- a/r-packages/ph-252.r +++ b/r-packages/ph-252.r @@ -1,9 +1,20 @@ #!/usr/bin/env Rscript # For https://github.com/berkeley-dsep-infra/datahub/issues/2556 +# For https://github.com/berkeley-dsep-infra/datahub/issues/2524 # Fall 2021 print("Installing packages for PH 252") source("/tmp/class-libs.R") class_name = "PH 252" -devtools::install_github('kaz-yos/tableone', ref='7cfcd71') + +class_libs = c( + "epi", "2.44", + "foreign", "0.8-81" +) + +devtools::install_github('kaz-yos/tableone', ref='7cfcd71', upgrade_dependencies=FALSE, quiet=FALSE) + +class_libs_install_version(class_name, class_libs) + +print("Done installing packages for PH 252") From c5a0320d133b024d36e76f7149cca57f195721a7 Mon Sep 17 00:00:00 2001 From: Jonathan Felder Date: Fri, 20 Aug 2021 16:29:54 -0700 Subject: [PATCH 021/132] Trying epi from github for issue #2524 --- r-packages/ph-252.r | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/r-packages/ph-252.r b/r-packages/ph-252.r index 5d37f3a..1a528dd 100755 --- a/r-packages/ph-252.r +++ b/r-packages/ph-252.r @@ -9,10 +9,10 @@ source("/tmp/class-libs.R") class_name = "PH 252" class_libs = c( - "epi", "2.44", "foreign", "0.8-81" ) +devtools::install_github('cran/epi', ref='06efd3f', upgrade_dependencies=FALSE, quiet=FALSE) devtools::install_github('kaz-yos/tableone', ref='7cfcd71', upgrade_dependencies=FALSE, quiet=FALSE) class_libs_install_version(class_name, class_libs) From 21bfc8189166473fc4cd259ac74b37e2e978d08d Mon Sep 17 00:00:00 2001 From: Chris Pyles Date: Sun, 22 Aug 2021 12:09:41 -0700 Subject: [PATCH 022/132] upgrade to otter v3.0.1 --- infra-requirements.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/infra-requirements.txt b/infra-requirements.txt index 2cca334..23da4d4 100644 --- a/infra-requirements.txt +++ b/infra-requirements.txt @@ -20,7 +20,7 @@ jupyter-resource-usage==0.5.1 git+https://github.com/meeseeksmachine/jupyterhub@auto-backport-of-pr-3488-on-1.4.x appmode==0.8.0 ipywidgets==7.6.3 -otter-grader==2.2.0 +otter-grader==3.0.1 jupyter-tree-download==1.0.1 git-credential-helpers==0.2 # Enough people like this, let's load it in. @@ -32,4 +32,4 @@ popularity-contest==0.4.1 # RISE is useful for presentations - see https://github.com/berkeley-dsep-infra/datahub/issues/2527 RISE==5.7.1 # https://github.com/berkeley-dsep-infra/datahub/issues/784 -voila==0.2.10 \ No newline at end of file +voila==0.2.10 From d6eeed30fa597e6097a567d70a9c6d44f1f82950 Mon Sep 17 00:00:00 2001 From: YuviPanda Date: Tue, 24 Aug 2021 21:03:52 +0530 Subject: [PATCH 023/132] Upgrade infra requirement packages --- infra-requirements.txt | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/infra-requirements.txt b/infra-requirements.txt index 23da4d4..2de7cf2 100644 --- a/infra-requirements.txt +++ b/infra-requirements.txt @@ -11,16 +11,16 @@ notebook==6.4.3 # Seems to cause a version conflict with jupyter-server-proxy jupyter-client<7.0 -jupyterlab==3.1.4 +jupyterlab==3.1.7 nbconvert==6.1.0 retrolab==0.2.2 -nbgitpuller==0.10.0 -jupyter-resource-usage==0.5.1 +nbgitpuller==0.10.1 +jupyter-resource-usage==0.6.0 # Matches version in images/hub/Dockerfile git+https://github.com/meeseeksmachine/jupyterhub@auto-backport-of-pr-3488-on-1.4.x appmode==0.8.0 ipywidgets==7.6.3 -otter-grader==3.0.1 +otter-grader==3.0.2 jupyter-tree-download==1.0.1 git-credential-helpers==0.2 # Enough people like this, let's load it in. From 98fc04f565dd67ce953cde86e5fc28336623c363 Mon Sep 17 00:00:00 2001 From: YuviPanda Date: Wed, 25 Aug 2021 23:04:44 +0530 Subject: [PATCH 024/132] Bump nbgitpuller for security advisory Fixes https://github.com/jupyterhub/nbgitpuller/security/advisories/GHSA-mq5p-2mcr-m52j --- infra-requirements.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/infra-requirements.txt b/infra-requirements.txt index 2de7cf2..80c0f13 100644 --- a/infra-requirements.txt +++ b/infra-requirements.txt @@ -11,10 +11,10 @@ notebook==6.4.3 # Seems to cause a version conflict with jupyter-server-proxy jupyter-client<7.0 -jupyterlab==3.1.7 +jupyterlab==3.1.9 nbconvert==6.1.0 retrolab==0.2.2 -nbgitpuller==0.10.1 +nbgitpuller==0.10.2 jupyter-resource-usage==0.6.0 # Matches version in images/hub/Dockerfile git+https://github.com/meeseeksmachine/jupyterhub@auto-backport-of-pr-3488-on-1.4.x From 6844ad161e0d0fea5481de51ae5ab7720ae11e16 Mon Sep 17 00:00:00 2001 From: Chris Pyles Date: Sat, 28 Aug 2021 16:51:54 -0700 Subject: [PATCH 025/132] upgrade otter to v3.0.6 --- infra-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/infra-requirements.txt b/infra-requirements.txt index 23da4d4..60f1c65 100644 --- a/infra-requirements.txt +++ b/infra-requirements.txt @@ -20,7 +20,7 @@ jupyter-resource-usage==0.5.1 git+https://github.com/meeseeksmachine/jupyterhub@auto-backport-of-pr-3488-on-1.4.x appmode==0.8.0 ipywidgets==7.6.3 -otter-grader==3.0.1 +otter-grader==3.0.6 jupyter-tree-download==1.0.1 git-credential-helpers==0.2 # Enough people like this, let's load it in. From a25d50e5e7afaa2754d65ae361bd3cdc2cfd5c82 Mon Sep 17 00:00:00 2001 From: Chris Pyles Date: Sat, 28 Aug 2021 16:52:48 -0700 Subject: [PATCH 026/132] Revert "upgrade otter to v3.0.6" This reverts commit 6844ad161e0d0fea5481de51ae5ab7720ae11e16. --- infra-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/infra-requirements.txt b/infra-requirements.txt index 60f1c65..23da4d4 100644 --- a/infra-requirements.txt +++ b/infra-requirements.txt @@ -20,7 +20,7 @@ jupyter-resource-usage==0.5.1 git+https://github.com/meeseeksmachine/jupyterhub@auto-backport-of-pr-3488-on-1.4.x appmode==0.8.0 ipywidgets==7.6.3 -otter-grader==3.0.6 +otter-grader==3.0.1 jupyter-tree-download==1.0.1 git-credential-helpers==0.2 # Enough people like this, let's load it in. From d0270dbbbf42e9841e57bc45ed63eb87792a3694 Mon Sep 17 00:00:00 2001 From: Chris Pyles Date: Sat, 28 Aug 2021 16:53:39 -0700 Subject: [PATCH 027/132] upgrade otter to v3.0.6 --- infra-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/infra-requirements.txt b/infra-requirements.txt index 80c0f13..4ccd7eb 100644 --- a/infra-requirements.txt +++ b/infra-requirements.txt @@ -20,7 +20,7 @@ jupyter-resource-usage==0.6.0 git+https://github.com/meeseeksmachine/jupyterhub@auto-backport-of-pr-3488-on-1.4.x appmode==0.8.0 ipywidgets==7.6.3 -otter-grader==3.0.2 +otter-grader==3.0.6 jupyter-tree-download==1.0.1 git-credential-helpers==0.2 # Enough people like this, let's load it in. From 0ef477d78a08c95be9e77ab3ac1cd3d3f057e76f Mon Sep 17 00:00:00 2001 From: YuviPanda Date: Thu, 2 Sep 2021 01:03:11 +0530 Subject: [PATCH 028/132] Bump JupyterLab again Ref https://github.com/berkeley-dsep-infra/datahub/issues/2533#issuecomment-910469810 --- infra-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/infra-requirements.txt b/infra-requirements.txt index 4ccd7eb..df524b0 100644 --- a/infra-requirements.txt +++ b/infra-requirements.txt @@ -11,7 +11,7 @@ notebook==6.4.3 # Seems to cause a version conflict with jupyter-server-proxy jupyter-client<7.0 -jupyterlab==3.1.9 +jupyterlab==3.1.10 nbconvert==6.1.0 retrolab==0.2.2 nbgitpuller==0.10.2 From bcaeb3bc712e0e3f0dcc5163e1d77b5633080baf Mon Sep 17 00:00:00 2001 From: YuviPanda Date: Fri, 3 Sep 2021 16:39:23 +0530 Subject: [PATCH 029/132] Remove voila We suspect it caused the main thread to block, preventing all other server actions (like file saves) from working. Ref https://github.com/berkeley-dsep-infra/datahub/issues/2688 Undoes https://github.com/berkeley-dsep-infra/datahub/issues/784 --- infra-requirements.txt | 2 -- 1 file changed, 2 deletions(-) diff --git a/infra-requirements.txt b/infra-requirements.txt index df524b0..6cbb80b 100644 --- a/infra-requirements.txt +++ b/infra-requirements.txt @@ -31,5 +31,3 @@ jupyter_nbextensions_configurator==0.4.1 popularity-contest==0.4.1 # RISE is useful for presentations - see https://github.com/berkeley-dsep-infra/datahub/issues/2527 RISE==5.7.1 -# https://github.com/berkeley-dsep-infra/datahub/issues/784 -voila==0.2.10 From d60ea9b5c7fe844c0f7976db78ebef1f68bc4d88 Mon Sep 17 00:00:00 2001 From: Chris Pyles Date: Sat, 4 Sep 2021 10:56:30 -0700 Subject: [PATCH 030/132] upgrade to otter v3.1.0 --- infra-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/infra-requirements.txt b/infra-requirements.txt index 6cbb80b..3c146e6 100644 --- a/infra-requirements.txt +++ b/infra-requirements.txt @@ -20,7 +20,7 @@ jupyter-resource-usage==0.6.0 git+https://github.com/meeseeksmachine/jupyterhub@auto-backport-of-pr-3488-on-1.4.x appmode==0.8.0 ipywidgets==7.6.3 -otter-grader==3.0.6 +otter-grader==3.1.0 jupyter-tree-download==1.0.1 git-credential-helpers==0.2 # Enough people like this, let's load it in. From 00732508b31ba2750a295be1e4f592b49464920f Mon Sep 17 00:00:00 2001 From: YuviPanda Date: Mon, 6 Sep 2021 00:10:35 +0530 Subject: [PATCH 031/132] Bump ottr version Fixes https://github.com/berkeley-dsep-infra/datahub/issues/2700 --- r-packages/ph-142.r | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/r-packages/ph-142.r b/r-packages/ph-142.r index 3a5b69e..283af32 100755 --- a/r-packages/ph-142.r +++ b/r-packages/ph-142.r @@ -25,7 +25,7 @@ class_libs = c( # https://github.com/berkeley-dsep-infra/datahub/issues/2483 print("Installing ottr...") -devtools::install_github('ucbds-infra/ottr', ref='0.1.0', upgrade_dependencies=FALSE, quiet=FALSE) +devtools::install_github('ucbds-infra/ottr', ref='1.1.1', upgrade_dependencies=FALSE, quiet=FALSE) class_libs_install_version(class_name, class_libs) From 87f31a6680137d487af348e6593223b5b0f7f026 Mon Sep 17 00:00:00 2001 From: YuviPanda Date: Mon, 6 Sep 2021 15:13:02 +0530 Subject: [PATCH 032/132] Bump retrolab Our blockers have been fixed Ref https://github.com/berkeley-dsep-infra/datahub/issues/2422 --- infra-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/infra-requirements.txt b/infra-requirements.txt index 6cbb80b..0a8288b 100644 --- a/infra-requirements.txt +++ b/infra-requirements.txt @@ -13,7 +13,7 @@ notebook==6.4.3 jupyter-client<7.0 jupyterlab==3.1.10 nbconvert==6.1.0 -retrolab==0.2.2 +retrolab==0.3.4 nbgitpuller==0.10.2 jupyter-resource-usage==0.6.0 # Matches version in images/hub/Dockerfile From 86c25285f0230f658b7cfdef8c5b5e8799a2aea9 Mon Sep 17 00:00:00 2001 From: YuviPanda Date: Tue, 7 Sep 2021 18:18:05 +0530 Subject: [PATCH 033/132] Pin ottr to 0.1 on publichealth ChandlerB tells me that ph142 relies on features in 0.1 and those break when we move to 1.0. Let's pin this until a solution can be reached. --- r-packages/ph-142.r | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/r-packages/ph-142.r b/r-packages/ph-142.r index 283af32..1512444 100755 --- a/r-packages/ph-142.r +++ b/r-packages/ph-142.r @@ -23,9 +23,9 @@ class_libs = c( "googlesheets4", "0.2.0" ) -# https://github.com/berkeley-dsep-infra/datahub/issues/2483 print("Installing ottr...") -devtools::install_github('ucbds-infra/ottr', ref='1.1.1', upgrade_dependencies=FALSE, quiet=FALSE) +# Pinned to this version so we do not break assignments +devtools::install_github('ucbds-infra/ottr', ref='0.1.0', upgrade_dependencies=FALSE, quiet=FALSE) class_libs_install_version(class_name, class_libs) From fc4d638caced1444da82a4f358fb8126061c1886 Mon Sep 17 00:00:00 2001 From: YuviPanda Date: Thu, 9 Sep 2021 14:01:25 +0530 Subject: [PATCH 034/132] Bump retrolab version --- infra-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/infra-requirements.txt b/infra-requirements.txt index 0a8288b..b548f45 100644 --- a/infra-requirements.txt +++ b/infra-requirements.txt @@ -13,7 +13,7 @@ notebook==6.4.3 jupyter-client<7.0 jupyterlab==3.1.10 nbconvert==6.1.0 -retrolab==0.3.4 +retrolab==0.3.5 nbgitpuller==0.10.2 jupyter-resource-usage==0.6.0 # Matches version in images/hub/Dockerfile From eef6a0d27cc59221f71dee3a72f6e7da4190a48d Mon Sep 17 00:00:00 2001 From: Chris Pyles Date: Sun, 12 Sep 2021 11:40:51 -0700 Subject: [PATCH 035/132] update to Otter v3.1.3 --- infra-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/infra-requirements.txt b/infra-requirements.txt index 5d8b6c4..687bdcd 100644 --- a/infra-requirements.txt +++ b/infra-requirements.txt @@ -20,7 +20,7 @@ jupyter-resource-usage==0.6.0 git+https://github.com/meeseeksmachine/jupyterhub@auto-backport-of-pr-3488-on-1.4.x appmode==0.8.0 ipywidgets==7.6.3 -otter-grader==3.1.0 +otter-grader==3.1.3 jupyter-tree-download==1.0.1 git-credential-helpers==0.2 # Enough people like this, let's load it in. From 96d65caef1f9fa8607a70efdfb52681734f5d2f0 Mon Sep 17 00:00:00 2001 From: Jonathan Felder Date: Mon, 13 Sep 2021 14:31:46 -0700 Subject: [PATCH 036/132] Adding blm for issue #2748 --- r-packages/ph-252.r | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/r-packages/ph-252.r b/r-packages/ph-252.r index 1a528dd..d221b2e 100755 --- a/r-packages/ph-252.r +++ b/r-packages/ph-252.r @@ -1,6 +1,7 @@ #!/usr/bin/env Rscript # For https://github.com/berkeley-dsep-infra/datahub/issues/2556 # For https://github.com/berkeley-dsep-infra/datahub/issues/2524 +# For https://github.com/berkeley-dsep-infra/datahub/issues/2748 # Fall 2021 print("Installing packages for PH 252") @@ -9,7 +10,8 @@ source("/tmp/class-libs.R") class_name = "PH 252" class_libs = c( - "foreign", "0.8-81" + "foreign", "0.8-81", + "blm", "2013.2.4.4" ) devtools::install_github('cran/epi', ref='06efd3f', upgrade_dependencies=FALSE, quiet=FALSE) From d09fd5d1a0fb24761e7d8b5774b4c63ff2eca7fc Mon Sep 17 00:00:00 2001 From: Chris Pyles <40970945+chrispyles@users.noreply.github.com> Date: Mon, 13 Sep 2021 17:46:59 -0700 Subject: [PATCH 037/132] Pin public health hub to Otter v2.2.7 --- infra-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/infra-requirements.txt b/infra-requirements.txt index 687bdcd..f4382cd 100644 --- a/infra-requirements.txt +++ b/infra-requirements.txt @@ -20,7 +20,7 @@ jupyter-resource-usage==0.6.0 git+https://github.com/meeseeksmachine/jupyterhub@auto-backport-of-pr-3488-on-1.4.x appmode==0.8.0 ipywidgets==7.6.3 -otter-grader==3.1.3 +otter-grader==2.2.7 jupyter-tree-download==1.0.1 git-credential-helpers==0.2 # Enough people like this, let's load it in. From 358918ad76a797842cb0c6fe99f4bfeee1e4fff3 Mon Sep 17 00:00:00 2001 From: Chris Pyles <40970945+chrispyles@users.noreply.github.com> Date: Tue, 14 Sep 2021 11:58:56 -0700 Subject: [PATCH 038/132] Update requirements.txt --- requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements.txt b/requirements.txt index 3479f00..92072bc 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,2 +1,3 @@ # RStudio support jupyter-rsession-proxy==1.4 +otter-grader==2.2.7 From 83c5bcfbba739db50923bb3f13ea44dc85aea7c1 Mon Sep 17 00:00:00 2001 From: Chris Pyles Date: Fri, 17 Sep 2021 23:30:50 -0700 Subject: [PATCH 039/132] upgrade to otter 3.1.4 --- infra-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/infra-requirements.txt b/infra-requirements.txt index f4382cd..39b664a 100644 --- a/infra-requirements.txt +++ b/infra-requirements.txt @@ -20,7 +20,7 @@ jupyter-resource-usage==0.6.0 git+https://github.com/meeseeksmachine/jupyterhub@auto-backport-of-pr-3488-on-1.4.x appmode==0.8.0 ipywidgets==7.6.3 -otter-grader==2.2.7 +otter-grader==3.1.4 jupyter-tree-download==1.0.1 git-credential-helpers==0.2 # Enough people like this, let's load it in. From bb9bf04a0fad5a702fa275d569f0f7aa2e92021d Mon Sep 17 00:00:00 2001 From: Jonathan Felder Date: Wed, 22 Sep 2021 14:46:12 -0700 Subject: [PATCH 040/132] Installing geepack for issue #2788, fall 2021 ph252 --- r-packages/ph-252.r | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/r-packages/ph-252.r b/r-packages/ph-252.r index d221b2e..1bc923d 100755 --- a/r-packages/ph-252.r +++ b/r-packages/ph-252.r @@ -2,6 +2,7 @@ # For https://github.com/berkeley-dsep-infra/datahub/issues/2556 # For https://github.com/berkeley-dsep-infra/datahub/issues/2524 # For https://github.com/berkeley-dsep-infra/datahub/issues/2748 +# For https://github.com/berkeley-dsep-infra/datahub/issues/2788 # Fall 2021 print("Installing packages for PH 252") @@ -11,7 +12,8 @@ class_name = "PH 252" class_libs = c( "foreign", "0.8-81", - "blm", "2013.2.4.4" + "blm", "2013.2.4.4", + "geepack", "1.3-2" ) devtools::install_github('cran/epi', ref='06efd3f', upgrade_dependencies=FALSE, quiet=FALSE) From de7b52c9226df463c8514c7233d85ac33c3b17d9 Mon Sep 17 00:00:00 2001 From: Jonathan Felder Date: Tue, 28 Sep 2021 16:05:40 -0700 Subject: [PATCH 041/132] Installing epitools and multcomp for issues #2801 and #2802. --- r-packages/ph-252.r | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/r-packages/ph-252.r b/r-packages/ph-252.r index 1bc923d..8783856 100755 --- a/r-packages/ph-252.r +++ b/r-packages/ph-252.r @@ -13,7 +13,9 @@ class_name = "PH 252" class_libs = c( "foreign", "0.8-81", "blm", "2013.2.4.4", - "geepack", "1.3-2" + "geepack", "1.3-2", + "multcomp", "1.4-7", + "epitools", "0.5-10.1" ) devtools::install_github('cran/epi', ref='06efd3f', upgrade_dependencies=FALSE, quiet=FALSE) From b1d11c035169b2ce77f1b356c2c07c4190693344 Mon Sep 17 00:00:00 2001 From: Jonathan Felder Date: Tue, 28 Sep 2021 17:24:53 -0700 Subject: [PATCH 042/132] 17 not 7 for #2802. --- r-packages/ph-252.r | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/r-packages/ph-252.r b/r-packages/ph-252.r index 8783856..07ecf4c 100755 --- a/r-packages/ph-252.r +++ b/r-packages/ph-252.r @@ -14,7 +14,7 @@ class_libs = c( "foreign", "0.8-81", "blm", "2013.2.4.4", "geepack", "1.3-2", - "multcomp", "1.4-7", + "multcomp", "1.4-17", "epitools", "0.5-10.1" ) From de13a065091aff49aa5da2257a935be753ece031 Mon Sep 17 00:00:00 2001 From: Balaji Date: Mon, 4 Oct 2021 17:16:30 -0700 Subject: [PATCH 043/132] Adding R packages raised in the issue #2831 --- r-packages/ph-252.r | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/r-packages/ph-252.r b/r-packages/ph-252.r index 07ecf4c..4a92eae 100755 --- a/r-packages/ph-252.r +++ b/r-packages/ph-252.r @@ -16,6 +16,12 @@ class_libs = c( "geepack", "1.3-2", "multcomp", "1.4-17", "epitools", "0.5-10.1" + "rms", "6.2-0", + "lmtest","0.9-38", + "car","3.0-11", + "SurvRegCensCov","1.4", + "ProfileLikelihood","1.1" + "biostat3","0.1.5" ) devtools::install_github('cran/epi', ref='06efd3f', upgrade_dependencies=FALSE, quiet=FALSE) From e3fbdf826ca76da701568752561df761c97acba9 Mon Sep 17 00:00:00 2001 From: Yuvi Panda Date: Tue, 5 Oct 2021 11:37:34 +0530 Subject: [PATCH 044/132] Fix syntax errors --- r-packages/ph-252.r | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/r-packages/ph-252.r b/r-packages/ph-252.r index 4a92eae..960bf5f 100755 --- a/r-packages/ph-252.r +++ b/r-packages/ph-252.r @@ -15,12 +15,12 @@ class_libs = c( "blm", "2013.2.4.4", "geepack", "1.3-2", "multcomp", "1.4-17", - "epitools", "0.5-10.1" + "epitools", "0.5-10.1", "rms", "6.2-0", "lmtest","0.9-38", "car","3.0-11", "SurvRegCensCov","1.4", - "ProfileLikelihood","1.1" + "ProfileLikelihood","1.1", "biostat3","0.1.5" ) From ca7a9b545d759ae31a95e8cb401d60e65737cdb0 Mon Sep 17 00:00:00 2001 From: Balaji Date: Tue, 5 Oct 2021 18:48:48 -0700 Subject: [PATCH 045/132] Moving to 3.0-10 version for car package --- r-packages/ph-252.r | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/r-packages/ph-252.r b/r-packages/ph-252.r index 960bf5f..41e2665 100755 --- a/r-packages/ph-252.r +++ b/r-packages/ph-252.r @@ -18,7 +18,7 @@ class_libs = c( "epitools", "0.5-10.1", "rms", "6.2-0", "lmtest","0.9-38", - "car","3.0-11", + "car","3.0-10", "SurvRegCensCov","1.4", "ProfileLikelihood","1.1", "biostat3","0.1.5" From 18f629634e30d8d645007ee78683d9fa764ac590 Mon Sep 17 00:00:00 2001 From: YuviPanda Date: Wed, 6 Oct 2021 20:39:27 +0530 Subject: [PATCH 046/132] Bump JupyterLab Brings in https://github.com/jupyterlab/jupyterlab/pull/11205, to support us bringing in https://github.com/jupyterhub/jupyterhub/pull/3636 so we can have cleaner 503 error graphs. Ref https://github.com/berkeley-dsep-infra/datahub/issues/2693#issuecomment-926950119 --- infra-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/infra-requirements.txt b/infra-requirements.txt index 39b664a..d6cdaae 100644 --- a/infra-requirements.txt +++ b/infra-requirements.txt @@ -11,7 +11,7 @@ notebook==6.4.3 # Seems to cause a version conflict with jupyter-server-proxy jupyter-client<7.0 -jupyterlab==3.1.10 +jupyterlab==3.1.17 nbconvert==6.1.0 retrolab==0.3.5 nbgitpuller==0.10.2 From f105609c27da5659b5377858ea403f3208d76469 Mon Sep 17 00:00:00 2001 From: YuviPanda Date: Tue, 19 Oct 2021 23:40:03 +0530 Subject: [PATCH 047/132] Add tini to ischool & publichelath hubs We inherit from the geospatial rocker images, which ship with the s6 process manager. However, JupyterHub doesn't launch them using that. We consistently use tini across all our images instead. Ref https://github.com/berkeley-dsep-infra/datahub/issues/2878 Ref https://github.com/berkeley-dsep-infra/datahub/issues/2891 --- Dockerfile | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Dockerfile b/Dockerfile index 0040148..ac4c420 100644 --- a/Dockerfile +++ b/Dockerfile @@ -36,6 +36,7 @@ RUN apt-get update > /dev/null && \ libssl1.1 \ fonts-symbola \ gdebi-core \ + tini \ nodejs npm > /dev/null && \ apt-get clean && \ rm -rf /var/lib/apt/lists/* @@ -72,3 +73,5 @@ RUN r /tmp/r-packages/ph-142.r COPY r-packages/ph-252.r /tmp/r-packages/ RUN r /tmp/r-packages/ph-252.r + +ENTRYPOINT ["tini", "--"] \ No newline at end of file From 46a5a71e79102fc2a612fec4b08a26b7b61dc858 Mon Sep 17 00:00:00 2001 From: YuviPanda Date: Wed, 27 Oct 2021 13:05:46 +0530 Subject: [PATCH 048/132] Use mambaforge instead of miniforge everywhere Mamba is faster and provides error messages that are actually comprehendable on conflicts --- install-miniforge.bash | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/install-miniforge.bash b/install-miniforge.bash index b893b0f..13f4314 100755 --- a/install-miniforge.bash +++ b/install-miniforge.bash @@ -1,12 +1,12 @@ #!/bin/bash -# This downloads and installs a pinned version of miniconda +# This downloads and installs a pinned version of mambaforge set -ex cd $(dirname $0) -MINIFORGE_VERSION=4.10.2-0 +MAMBAFORGE_VERSION=4.10.3-7 -URL="https://github.com/conda-forge/miniforge/releases/download/${MINIFORGE_VERSION}/Miniforge3-${MINIFORGE_VERSION}-Linux-x86_64.sh" -INSTALLER_PATH=/tmp/miniforge-installer.sh +URL="https://github.com/conda-forge/miniforge/releases/download/${MAMBAFORGE_VERSION}/Mambaforge-${MAMBAFORGE_VERSION}-Linux-x86_64.sh" +INSTALLER_PATH=/tmp/mambaforge-installer.sh # make sure we don't do anything funky with user's $HOME # since this is run as root @@ -34,7 +34,7 @@ conda clean --all -f -y # Remove the big installer so we don't increase docker image size too much rm ${INSTALLER_PATH} -# Remove the pip cache created as part of installing miniconda +# Remove the pip cache created as part of installing mambaforge rm -rf /root/.cache chown -R $NB_USER:$NB_USER ${CONDA_DIR} From eeab5fe9f3d0ea45d9794b3e171c99806017b098 Mon Sep 17 00:00:00 2001 From: YuviPanda Date: Wed, 27 Oct 2021 13:58:29 +0530 Subject: [PATCH 049/132] Rename files to indicate we're installing mambaforge Not miniforge --- Dockerfile | 4 ++-- install-miniforge.bash => install-mambaforge.bash | 0 2 files changed, 2 insertions(+), 2 deletions(-) rename install-miniforge.bash => install-mambaforge.bash (100%) diff --git a/Dockerfile b/Dockerfile index ac4c420..fcc749b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -48,8 +48,8 @@ RUN curl --silent --location --fail ${RSTUDIO_URL} > /tmp/rstudio.deb && \ dpkg -i /tmp/rstudio.deb && \ rm /tmp/rstudio.deb -COPY install-miniforge.bash /tmp/install-miniforge.bash -RUN /tmp/install-miniforge.bash +COPY install-mambaforge.bash /tmp/install-mambaforge.bash +RUN /tmp/install-mambaforge.bash USER ${NB_USER} diff --git a/install-miniforge.bash b/install-mambaforge.bash similarity index 100% rename from install-miniforge.bash rename to install-mambaforge.bash From 7a4feb1db55b6d39e5731849d921b08425c1e001 Mon Sep 17 00:00:00 2001 From: Chandler Beon Date: Tue, 2 Nov 2021 10:31:44 -0700 Subject: [PATCH 050/132] Public Health 142 Library Installation Updates - Update `googlesheets4` to version 1.0.0 - Install `coxed` version 0.3.3 --- r-packages/ph-142.r | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/r-packages/ph-142.r b/r-packages/ph-142.r index 1512444..dd7606a 100755 --- a/r-packages/ph-142.r +++ b/r-packages/ph-142.r @@ -20,7 +20,8 @@ class_libs = c( "cowplot", "1.1.1", "patchwork", "1.1.1", "tigris", "1.0", - "googlesheets4", "0.2.0" + "googlesheets4", "1.0.0", + "coxed", "0.3.3" ) print("Installing ottr...") From 5eb90337f3482b0efeb5361aa2cbea6e621f2952 Mon Sep 17 00:00:00 2001 From: YuviPanda Date: Thu, 4 Nov 2021 23:02:43 +0530 Subject: [PATCH 051/132] Pin setuptools temporarily New builds are failing with this error message (https://app.circleci.com/pipelines/github/berkeley-dsep-infra/datahub/744/workflows/ee2f988b-b888-4e39-a8ce-fb8bca401fb2/jobs/5993) ``` self.add_defaults() File "/tmp/pip-build-env-rvc3nf2q/overlay/lib/python3.9/site-packages/setuptools/command/egg_info.py", line 578, in add_defaults sdist.add_defaults(self) File "/opt/conda/lib/python3.9/distutils/command/sdist.py", line 226, in add_defaults self._add_defaults_python() File "/tmp/pip-build-env-rvc3nf2q/overlay/lib/python3.9/site-packages/setuptools/command/sdist.py", line 113, in _add_defaults_python self._add_data_files(self._safe_data_files(build_py)) File "/tmp/pip-build-env-rvc3nf2q/overlay/lib/python3.9/site-packages/setuptools/command/egg_info.py", line 621, in _safe_data_files return build_py.get_data_files_without_manifest() File "/opt/conda/lib/python3.9/distutils/cmd.py", line 103, in __getattr__ raise AttributeError(attr) AttributeError: get_data_files_without_manifest ``` Googling leads to https://github.com/pypa/setuptools/issues/2849, which matches our problem. Let's pin setuptools to an older version until this gets fixed. --- infra-requirements.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/infra-requirements.txt b/infra-requirements.txt index d6cdaae..196d8d6 100644 --- a/infra-requirements.txt +++ b/infra-requirements.txt @@ -31,3 +31,5 @@ jupyter_nbextensions_configurator==0.4.1 popularity-contest==0.4.1 # RISE is useful for presentations - see https://github.com/berkeley-dsep-infra/datahub/issues/2527 RISE==5.7.1 +# Remove once https://github.com/pypa/setuptools/issues/2849 is fixed +setuptools<=58.4 \ No newline at end of file From f9c7cfd1975ecf93b3c39956c188e1f94015e329 Mon Sep 17 00:00:00 2001 From: YuviPanda Date: Thu, 4 Nov 2021 23:41:23 +0530 Subject: [PATCH 052/132] Bump to JupyterHub 1.5.0 Setuptools bug is triggered by installing from git, and JupyterHub was just released a few hours ago! Much easier than trying to figure out how to pin setuptools to be exactly right https://github.com/berkeley-dsep-infra/datahub/pull/2964 This reverts commit 5eb90337f3482b0efeb5361aa2cbea6e621f2952. --- infra-requirements.txt | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/infra-requirements.txt b/infra-requirements.txt index 196d8d6..872f5b4 100644 --- a/infra-requirements.txt +++ b/infra-requirements.txt @@ -17,7 +17,7 @@ retrolab==0.3.5 nbgitpuller==0.10.2 jupyter-resource-usage==0.6.0 # Matches version in images/hub/Dockerfile -git+https://github.com/meeseeksmachine/jupyterhub@auto-backport-of-pr-3488-on-1.4.x +jupyterhub==1.5.0 appmode==0.8.0 ipywidgets==7.6.3 otter-grader==3.1.4 @@ -31,5 +31,3 @@ jupyter_nbextensions_configurator==0.4.1 popularity-contest==0.4.1 # RISE is useful for presentations - see https://github.com/berkeley-dsep-infra/datahub/issues/2527 RISE==5.7.1 -# Remove once https://github.com/pypa/setuptools/issues/2849 is fixed -setuptools<=58.4 \ No newline at end of file From f30f8df2c75a7dcaf5d740bbd5fbb2c5d7aed6aa Mon Sep 17 00:00:00 2001 From: YuviPanda Date: Fri, 5 Nov 2021 14:02:22 +0530 Subject: [PATCH 053/132] Bump some packages in infra-requirements.txt Especially as retrolab is being tested out for https://github.com/berkeley-dsep-infra/datahub/issues/2422 --- infra-requirements.txt | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/infra-requirements.txt b/infra-requirements.txt index 872f5b4..94c0acd 100644 --- a/infra-requirements.txt +++ b/infra-requirements.txt @@ -8,13 +8,11 @@ # everwhere one by one. # FIXME: Freeze this to get exact versions of all dependencies -notebook==6.4.3 -# Seems to cause a version conflict with jupyter-server-proxy -jupyter-client<7.0 -jupyterlab==3.1.17 +notebook==6.4.5 +jupyterlab==3.2.2 nbconvert==6.1.0 -retrolab==0.3.5 -nbgitpuller==0.10.2 +retrolab==0.3.12 +nbgitpuller==1.0.2 jupyter-resource-usage==0.6.0 # Matches version in images/hub/Dockerfile jupyterhub==1.5.0 From 8699b737d4a857da19d02a901deaea65ac4e0711 Mon Sep 17 00:00:00 2001 From: YuviPanda Date: Fri, 5 Nov 2021 17:06:25 +0530 Subject: [PATCH 054/132] Pin jupyter-client version again Apparently https://github.com/berkeley-dsep-infra/datahub/pull/2038 has not been solved yet. --- infra-requirements.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/infra-requirements.txt b/infra-requirements.txt index 94c0acd..c0d2d1f 100644 --- a/infra-requirements.txt +++ b/infra-requirements.txt @@ -9,6 +9,8 @@ # FIXME: Freeze this to get exact versions of all dependencies notebook==6.4.5 +# FIXME: Conflicts with jupyter-server-proxy somehow +jupyter-client<7.0 jupyterlab==3.2.2 nbconvert==6.1.0 retrolab==0.3.12 From 3110d5cfc7dcd372abe95f492c2d3429c5f9b9e4 Mon Sep 17 00:00:00 2001 From: YuviPanda Date: Sun, 14 Nov 2021 22:00:39 +0530 Subject: [PATCH 055/132] Bump retrolab version Fixes https://github.com/berkeley-dsep-infra/datahub/issues/2989 --- infra-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/infra-requirements.txt b/infra-requirements.txt index c0d2d1f..decff90 100644 --- a/infra-requirements.txt +++ b/infra-requirements.txt @@ -13,7 +13,7 @@ notebook==6.4.5 jupyter-client<7.0 jupyterlab==3.2.2 nbconvert==6.1.0 -retrolab==0.3.12 +retrolab==0.3.13 nbgitpuller==1.0.2 jupyter-resource-usage==0.6.0 # Matches version in images/hub/Dockerfile From c4fc1b93d23f15f523122daafb4f9eff53d2a99f Mon Sep 17 00:00:00 2001 From: YuviPanda Date: Fri, 19 Nov 2021 01:14:38 +0530 Subject: [PATCH 056/132] Enable Real Time Collaboration on DataHub - Bump up JupyterLab to get latest bugfixes - Enable the --collaborative flag - Install jupyterlab-link-share, required to generate the link you can give others to collaborate with you. It adds a 'Share' menu item to JupyterLab, and also a 'Copy downloadable link' item to the context menu. More info at https://pypi.org/project/jupyterlab-link-share/ --- infra-requirements.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/infra-requirements.txt b/infra-requirements.txt index decff90..1653b80 100644 --- a/infra-requirements.txt +++ b/infra-requirements.txt @@ -11,7 +11,8 @@ notebook==6.4.5 # FIXME: Conflicts with jupyter-server-proxy somehow jupyter-client<7.0 -jupyterlab==3.2.2 +jupyterlab==3.2.4 +jupyterlab-link-share==0.2.1 nbconvert==6.1.0 retrolab==0.3.13 nbgitpuller==1.0.2 From 63cad4754801655016e7cf310a587820707de5ef Mon Sep 17 00:00:00 2001 From: YuviPanda Date: Thu, 16 Dec 2021 10:10:08 +0530 Subject: [PATCH 057/132] Upgrade publicheahlth hub to R 4.1 + newer RStudio To be merged the week of Dec 20th Ref https://github.com/berkeley-dsep-infra/datahub/issues/2945 --- Dockerfile | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/Dockerfile b/Dockerfile index fcc749b..227a90b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM rocker/geospatial:4.0.5 +FROM rocker/geospatial:4.1.2 ENV NB_USER rstudio ENV NB_UID 1000 @@ -41,13 +41,6 @@ RUN apt-get update > /dev/null && \ apt-get clean && \ rm -rf /var/lib/apt/lists/* -# 1.3.959 is latest version that works with jupyter-rsession-proxy -# See https://github.com/jupyterhub/jupyter-rsession-proxy/issues/93#issuecomment-725874693 -ENV RSTUDIO_URL https://download2.rstudio.org/server/bionic/amd64/rstudio-server-1.3.959-amd64.deb -RUN curl --silent --location --fail ${RSTUDIO_URL} > /tmp/rstudio.deb && \ - dpkg -i /tmp/rstudio.deb && \ - rm /tmp/rstudio.deb - COPY install-mambaforge.bash /tmp/install-mambaforge.bash RUN /tmp/install-mambaforge.bash @@ -59,6 +52,9 @@ RUN pip install --no-cache-dir -r /tmp/infra-requirements.txt COPY requirements.txt /tmp/requirements.txt RUN pip install --no-cache-dir -r /tmp/requirements.txt +# Support latest RStudio +RUN pip install --no-cache 'jupyter-rsession-proxy>=2.0' + # Install IRKernel RUN R --quiet -e "install.packages('IRkernel', quiet = TRUE)" && \ R --quiet -e "IRkernel::installspec(prefix='${CONDA_DIR}')" From 2ccb6868f8a0fed2a2c5c49067cb28314ce18a6b Mon Sep 17 00:00:00 2001 From: YuviPanda Date: Mon, 3 Jan 2022 20:58:03 +0530 Subject: [PATCH 058/132] Bump version of retrolab Ref https://github.com/berkeley-dsep-infra/datahub/issues/2422#issuecomment-1004129398 --- infra-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/infra-requirements.txt b/infra-requirements.txt index 1653b80..010c1b2 100644 --- a/infra-requirements.txt +++ b/infra-requirements.txt @@ -14,7 +14,7 @@ jupyter-client<7.0 jupyterlab==3.2.4 jupyterlab-link-share==0.2.1 nbconvert==6.1.0 -retrolab==0.3.13 +retrolab==0.3.15 nbgitpuller==1.0.2 jupyter-resource-usage==0.6.0 # Matches version in images/hub/Dockerfile From 15d7fef943c6122e3facc479ac7903c7fcc39727 Mon Sep 17 00:00:00 2001 From: YuviPanda Date: Tue, 4 Jan 2022 12:33:06 +0530 Subject: [PATCH 059/132] Move ph272A from main hub to publichealth hub --- Dockerfile | 4 ++++ r-packages/2021-spring-phw-272a.r | 11 +++++++++++ 2 files changed, 15 insertions(+) create mode 100644 r-packages/2021-spring-phw-272a.r diff --git a/Dockerfile b/Dockerfile index 227a90b..7e6ffba 100644 --- a/Dockerfile +++ b/Dockerfile @@ -70,4 +70,8 @@ RUN r /tmp/r-packages/ph-142.r COPY r-packages/ph-252.r /tmp/r-packages/ RUN r /tmp/r-packages/ph-252.r +COPY r-packages/2021-spring-phw-272a.r /tmp/r-packages/ +RUN r /tmp/r-packages/2021-spring-phw-272a.r + + ENTRYPOINT ["tini", "--"] \ No newline at end of file diff --git a/r-packages/2021-spring-phw-272a.r b/r-packages/2021-spring-phw-272a.r new file mode 100644 index 0000000..3344c47 --- /dev/null +++ b/r-packages/2021-spring-phw-272a.r @@ -0,0 +1,11 @@ +#!/usr/bin/env Rscript + +source("/tmp/class-libs.R") + +class_name = "PHW 272a Spring 2021" +class_libs = c( + "ggmap", "3.0.0", + "tmap", "3.3-2", + "rgdal", "1.5-28" +) +class_libs_install_version(class_name, class_libs) From c7a2f9ca6753fde061f67f373c3d4dd218c0e08c Mon Sep 17 00:00:00 2001 From: Fernando Perez Date: Wed, 19 Jan 2022 00:24:43 -0800 Subject: [PATCH 060/132] Push changes from global infra reqs file to all images. --- infra-requirements.txt | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/infra-requirements.txt b/infra-requirements.txt index 010c1b2..cad19fe 100644 --- a/infra-requirements.txt +++ b/infra-requirements.txt @@ -8,19 +8,19 @@ # everwhere one by one. # FIXME: Freeze this to get exact versions of all dependencies -notebook==6.4.5 +notebook==6.4.7 # FIXME: Conflicts with jupyter-server-proxy somehow jupyter-client<7.0 -jupyterlab==3.2.4 -jupyterlab-link-share==0.2.1 -nbconvert==6.1.0 -retrolab==0.3.15 +jupyterlab==3.2.8 +jupyterlab-link-share==0.2.4 +nbconvert==6.4.0 +retrolab==0.3.16 nbgitpuller==1.0.2 jupyter-resource-usage==0.6.0 # Matches version in images/hub/Dockerfile jupyterhub==1.5.0 appmode==0.8.0 -ipywidgets==7.6.3 +ipywidgets==7.6.5 otter-grader==3.1.4 jupyter-tree-download==1.0.1 git-credential-helpers==0.2 From 396fa7c748efe3b7fa1c0c5c6599f927db71e3f4 Mon Sep 17 00:00:00 2001 From: Fernando Perez Date: Wed, 19 Jan 2022 00:36:06 -0800 Subject: [PATCH 061/132] Propagate reqs to images --- infra-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/infra-requirements.txt b/infra-requirements.txt index cad19fe..5f6d3d3 100644 --- a/infra-requirements.txt +++ b/infra-requirements.txt @@ -16,7 +16,7 @@ jupyterlab-link-share==0.2.4 nbconvert==6.4.0 retrolab==0.3.16 nbgitpuller==1.0.2 -jupyter-resource-usage==0.6.0 +jupyter-resource-usage==0.6.1 # Matches version in images/hub/Dockerfile jupyterhub==1.5.0 appmode==0.8.0 From 4116382cbc82c02b3e285d02e19a67e9842960bf Mon Sep 17 00:00:00 2001 From: YuviPanda Date: Fri, 21 Jan 2022 10:31:50 +0530 Subject: [PATCH 062/132] Remove jupyter-client pin This was causing failures reported in https://github.com/berkeley-dsep-infra/datahub/issues/3166. I manually checked RStudio, JupyterLab and some Jupyter Notebooks --- infra-requirements.txt | 2 -- 1 file changed, 2 deletions(-) diff --git a/infra-requirements.txt b/infra-requirements.txt index 5f6d3d3..e32c25f 100644 --- a/infra-requirements.txt +++ b/infra-requirements.txt @@ -9,8 +9,6 @@ # FIXME: Freeze this to get exact versions of all dependencies notebook==6.4.7 -# FIXME: Conflicts with jupyter-server-proxy somehow -jupyter-client<7.0 jupyterlab==3.2.8 jupyterlab-link-share==0.2.4 nbconvert==6.4.0 From b61ce90a4fb055247c93306d3fcb5770ea866ac3 Mon Sep 17 00:00:00 2001 From: YuviPanda Date: Sun, 23 Jan 2022 17:20:15 +0530 Subject: [PATCH 063/132] Setup syncthing for dropbox-like functionality https://syncthing.net/ provides dropbox-like per-folder synchronization across machines securely, and is fully open source. Primarily useful for projects that might have different students working on it together in groups. --- Dockerfile | 2 ++ infra-requirements.txt | 2 ++ 2 files changed, 4 insertions(+) diff --git a/Dockerfile b/Dockerfile index 7e6ffba..f997096 100644 --- a/Dockerfile +++ b/Dockerfile @@ -52,6 +52,8 @@ RUN pip install --no-cache-dir -r /tmp/infra-requirements.txt COPY requirements.txt /tmp/requirements.txt RUN pip install --no-cache-dir -r /tmp/requirements.txt +RUN mamba install -c conda-forge syncthing==1.18.6 + # Support latest RStudio RUN pip install --no-cache 'jupyter-rsession-proxy>=2.0' diff --git a/infra-requirements.txt b/infra-requirements.txt index e32c25f..5f50301 100644 --- a/infra-requirements.txt +++ b/infra-requirements.txt @@ -30,3 +30,5 @@ jupyter_nbextensions_configurator==0.4.1 popularity-contest==0.4.1 # RISE is useful for presentations - see https://github.com/berkeley-dsep-infra/datahub/issues/2527 RISE==5.7.1 +# syncthing for dropbox-like functionality +jupyter-syncthing-proxy==1.0.1 \ No newline at end of file From e7e5d4654a46196ac2508d2a569c7f98aaae12d0 Mon Sep 17 00:00:00 2001 From: YuviPanda Date: Thu, 27 Jan 2022 15:12:26 +0530 Subject: [PATCH 064/132] Bump version of retrolab --- infra-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/infra-requirements.txt b/infra-requirements.txt index 5f50301..f946829 100644 --- a/infra-requirements.txt +++ b/infra-requirements.txt @@ -12,7 +12,7 @@ notebook==6.4.7 jupyterlab==3.2.8 jupyterlab-link-share==0.2.4 nbconvert==6.4.0 -retrolab==0.3.16 +retrolab==0.3.17 nbgitpuller==1.0.2 jupyter-resource-usage==0.6.1 # Matches version in images/hub/Dockerfile From 13010a5e7f5ead737138d05171279af50e8697e0 Mon Sep 17 00:00:00 2001 From: YuviPanda Date: Thu, 27 Jan 2022 17:46:01 +0530 Subject: [PATCH 065/132] Bump retrolab again --- infra-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/infra-requirements.txt b/infra-requirements.txt index f946829..e1c77c3 100644 --- a/infra-requirements.txt +++ b/infra-requirements.txt @@ -12,7 +12,7 @@ notebook==6.4.7 jupyterlab==3.2.8 jupyterlab-link-share==0.2.4 nbconvert==6.4.0 -retrolab==0.3.17 +retrolab==0.3.18 nbgitpuller==1.0.2 jupyter-resource-usage==0.6.1 # Matches version in images/hub/Dockerfile From e4f455ac15f41c8c3381ba56204f0a81cf0db08c Mon Sep 17 00:00:00 2001 From: Jonathan Felder Date: Tue, 15 Feb 2022 12:30:45 -0800 Subject: [PATCH 066/132] Moving ottr installation Issue #3263 --- Dockerfile | 6 +++++- install.R | 6 ++++++ r-packages/ph-142.r | 4 ---- 3 files changed, 11 insertions(+), 5 deletions(-) create mode 100755 install.R diff --git a/Dockerfile b/Dockerfile index f997096..1471502 100644 --- a/Dockerfile +++ b/Dockerfile @@ -62,6 +62,10 @@ RUN R --quiet -e "install.packages('IRkernel', quiet = TRUE)" && \ R --quiet -e "IRkernel::installspec(prefix='${CONDA_DIR}')" COPY class-libs.R /tmp/class-libs.R +RUN mkdir -p /tmp/r-packages + +COPY install.R /tmp/install.R +RUN /tmp/install.R && rm -rf /tmp/downloaded_packages COPY r-packages/ph-290.r /tmp/r-packages/ RUN r /tmp/r-packages/ph-290.r @@ -76,4 +80,4 @@ COPY r-packages/2021-spring-phw-272a.r /tmp/r-packages/ RUN r /tmp/r-packages/2021-spring-phw-272a.r -ENTRYPOINT ["tini", "--"] \ No newline at end of file +ENTRYPOINT ["tini", "--"] diff --git a/install.R b/install.R new file mode 100755 index 0000000..2f337bd --- /dev/null +++ b/install.R @@ -0,0 +1,6 @@ +#!/usr/bin/env r + +source("/tmp/class-libs.R") + +# install ottr, needs to go first issue #3216, #3263 +devtools::install_github('ucbds-infra/ottr', ref='0.1.0', upgrade_dependencies=FALSE, quiet=FALSE) diff --git a/r-packages/ph-142.r b/r-packages/ph-142.r index dd7606a..b83c95c 100755 --- a/r-packages/ph-142.r +++ b/r-packages/ph-142.r @@ -24,10 +24,6 @@ class_libs = c( "coxed", "0.3.3" ) -print("Installing ottr...") -# Pinned to this version so we do not break assignments -devtools::install_github('ucbds-infra/ottr', ref='0.1.0', upgrade_dependencies=FALSE, quiet=FALSE) - class_libs_install_version(class_name, class_libs) print("Done installing packages for PH142") From c893fe99c218fae5082629954f193f0a4fb6bd7f Mon Sep 17 00:00:00 2001 From: YuviPanda Date: Sun, 6 Mar 2022 07:50:19 -0800 Subject: [PATCH 067/132] Bump JupyterLab version to 3.3.0 --- infra-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/infra-requirements.txt b/infra-requirements.txt index e1c77c3..16f42b6 100644 --- a/infra-requirements.txt +++ b/infra-requirements.txt @@ -9,7 +9,7 @@ # FIXME: Freeze this to get exact versions of all dependencies notebook==6.4.7 -jupyterlab==3.2.8 +jupyterlab==3.3.0 jupyterlab-link-share==0.2.4 nbconvert==6.4.0 retrolab==0.3.18 From efee0041cadbc73485ca499fdb26b673a222ef32 Mon Sep 17 00:00:00 2001 From: Jonathan Felder Date: Mon, 11 Apr 2022 14:43:11 -0500 Subject: [PATCH 068/132] Telling tlmgr to use 2021 repo for texlive and installing packages to speed up knitting. Issue #3338 --- Dockerfile | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/Dockerfile b/Dockerfile index 1471502..665bf22 100644 --- a/Dockerfile +++ b/Dockerfile @@ -79,5 +79,37 @@ RUN r /tmp/r-packages/ph-252.r COPY r-packages/2021-spring-phw-272a.r /tmp/r-packages/ RUN r /tmp/r-packages/2021-spring-phw-272a.r +RUN tlmgr repository add https://ftp.math.utah.edu/pub/tex/historic/systems/texlive/2021/tlnet-final +RUN tlmgr option repository https://ftp.math.utah.edu/pub/tex/historic/systems/texlive/2021/tlnet-final +RUN tlmgr --verify-repo=none update --self && \ + tlmgr --verify-repo=none install \ + amsmath \ + latex-amsmath-dev \ + iftex \ + kvoptions \ + ltxcmds \ + kvsetkeys \ + etoolbox \ + xcolor \ + auxhook \ + bigintcalc \ + bitset \ + etexcmds \ + gettitlestring \ + hycolor \ + hyperref \ + intcalc \ + kvdefinekeys \ + letltxmacro \ + pdfescape \ + refcount \ + rerunfilecheck \ + stringenc \ + uniquecounter \ + zapfding \ + pdftexcmds \ + infwarerr \ + geometry \ + epstopdf-pkg ENTRYPOINT ["tini", "--"] From 8b8d15c3d3abc9b8cf88ab04b046d9eb22d090c2 Mon Sep 17 00:00:00 2001 From: Jonathan Felder Date: Mon, 11 Apr 2022 15:03:52 -0500 Subject: [PATCH 069/132] Ottr...always ottr. --- Dockerfile | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Dockerfile b/Dockerfile index 665bf22..bb0114a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -57,16 +57,16 @@ RUN mamba install -c conda-forge syncthing==1.18.6 # Support latest RStudio RUN pip install --no-cache 'jupyter-rsession-proxy>=2.0' -# Install IRKernel -RUN R --quiet -e "install.packages('IRkernel', quiet = TRUE)" && \ - R --quiet -e "IRkernel::installspec(prefix='${CONDA_DIR}')" - COPY class-libs.R /tmp/class-libs.R RUN mkdir -p /tmp/r-packages COPY install.R /tmp/install.R RUN /tmp/install.R && rm -rf /tmp/downloaded_packages +# Install IRKernel +RUN R --quiet -e "install.packages('IRkernel', quiet = TRUE)" && \ + R --quiet -e "IRkernel::installspec(prefix='${CONDA_DIR}')" + COPY r-packages/ph-290.r /tmp/r-packages/ RUN r /tmp/r-packages/ph-290.r From b747abfc9677fc2397e13ebd2d55d8ac6444193b Mon Sep 17 00:00:00 2001 From: Jonathan Felder Date: Mon, 11 Apr 2022 15:12:56 -0500 Subject: [PATCH 070/132] Moving ottr higher. --- Dockerfile | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/Dockerfile b/Dockerfile index bb0114a..90f13b3 100644 --- a/Dockerfile +++ b/Dockerfile @@ -46,6 +46,12 @@ RUN /tmp/install-mambaforge.bash USER ${NB_USER} +COPY class-libs.R /tmp/class-libs.R +RUN mkdir -p /tmp/r-packages + +COPY install.R /tmp/install.R +RUN /tmp/install.R && rm -rf /tmp/downloaded_packages + COPY infra-requirements.txt /tmp/infra-requirements.txt RUN pip install --no-cache-dir -r /tmp/infra-requirements.txt @@ -57,12 +63,6 @@ RUN mamba install -c conda-forge syncthing==1.18.6 # Support latest RStudio RUN pip install --no-cache 'jupyter-rsession-proxy>=2.0' -COPY class-libs.R /tmp/class-libs.R -RUN mkdir -p /tmp/r-packages - -COPY install.R /tmp/install.R -RUN /tmp/install.R && rm -rf /tmp/downloaded_packages - # Install IRKernel RUN R --quiet -e "install.packages('IRkernel', quiet = TRUE)" && \ R --quiet -e "IRkernel::installspec(prefix='${CONDA_DIR}')" From 46054fff6c75e7e2684fdc7d0706379914ac141e Mon Sep 17 00:00:00 2001 From: Jonathan Felder Date: Tue, 12 Apr 2022 17:48:04 -0500 Subject: [PATCH 071/132] Fixes ottr? --- Dockerfile | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/Dockerfile b/Dockerfile index 90f13b3..4fbf782 100644 --- a/Dockerfile +++ b/Dockerfile @@ -46,12 +46,6 @@ RUN /tmp/install-mambaforge.bash USER ${NB_USER} -COPY class-libs.R /tmp/class-libs.R -RUN mkdir -p /tmp/r-packages - -COPY install.R /tmp/install.R -RUN /tmp/install.R && rm -rf /tmp/downloaded_packages - COPY infra-requirements.txt /tmp/infra-requirements.txt RUN pip install --no-cache-dir -r /tmp/infra-requirements.txt @@ -67,6 +61,15 @@ RUN pip install --no-cache 'jupyter-rsession-proxy>=2.0' RUN R --quiet -e "install.packages('IRkernel', quiet = TRUE)" && \ R --quiet -e "IRkernel::installspec(prefix='${CONDA_DIR}')" +COPY class-libs.R /tmp/class-libs.R +RUN mkdir -p /tmp/r-packages + +# Workaround to install ottr, issue 3342 +RUN wget https://github.com/ucbds-infra/ottr/archive/refs/tags/0.1.0.tar.gz -O /tmp/ottr.tar.gz && R CMD INSTALL /tmp/ottr.tar.gz && rm /tmp/ottr.tar.gz + +#COPY install.R /tmp/install.R +#RUN /tmp/install.R && rm -rf /tmp/downloaded_packages + COPY r-packages/ph-290.r /tmp/r-packages/ RUN r /tmp/r-packages/ph-290.r From 673188dfff855f4a5243517a8c7c063b0a7becd7 Mon Sep 17 00:00:00 2001 From: Balaji Alwar Date: Tue, 3 May 2022 17:34:43 -0700 Subject: [PATCH 072/132] Bump jupyter-syncthing-proxy version from 1.0.1 to 1.0.2 --- infra-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/infra-requirements.txt b/infra-requirements.txt index 16f42b6..405c3b2 100644 --- a/infra-requirements.txt +++ b/infra-requirements.txt @@ -31,4 +31,4 @@ popularity-contest==0.4.1 # RISE is useful for presentations - see https://github.com/berkeley-dsep-infra/datahub/issues/2527 RISE==5.7.1 # syncthing for dropbox-like functionality -jupyter-syncthing-proxy==1.0.1 \ No newline at end of file +jupyter-syncthing-proxy==1.0.2 \ No newline at end of file From 1d002a231b2f61ccf08e70b955131ed2ad13a74a Mon Sep 17 00:00:00 2001 From: Balaji Alwar Date: Wed, 11 May 2022 17:10:06 -0700 Subject: [PATCH 073/132] Removing the storage policy related document from this commit --- infra-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/infra-requirements.txt b/infra-requirements.txt index 405c3b2..491fe50 100644 --- a/infra-requirements.txt +++ b/infra-requirements.txt @@ -31,4 +31,4 @@ popularity-contest==0.4.1 # RISE is useful for presentations - see https://github.com/berkeley-dsep-infra/datahub/issues/2527 RISE==5.7.1 # syncthing for dropbox-like functionality -jupyter-syncthing-proxy==1.0.2 \ No newline at end of file +jupyter-syncthing-proxy==1.0.3 \ No newline at end of file From d86a375a33725f34e18a5ffccc4447cc107a1081 Mon Sep 17 00:00:00 2001 From: Jonathan Felder Date: Thu, 26 May 2022 13:00:15 -0700 Subject: [PATCH 074/132] Upgrading ottr to 1.1.4 via CRAN Issue #3403 --- Dockerfile | 5 ++++- r-packages/ottr.r | 15 +++++++++++++++ 2 files changed, 19 insertions(+), 1 deletion(-) create mode 100755 r-packages/ottr.r diff --git a/Dockerfile b/Dockerfile index 4fbf782..b6b8ca3 100644 --- a/Dockerfile +++ b/Dockerfile @@ -65,11 +65,14 @@ COPY class-libs.R /tmp/class-libs.R RUN mkdir -p /tmp/r-packages # Workaround to install ottr, issue 3342 -RUN wget https://github.com/ucbds-infra/ottr/archive/refs/tags/0.1.0.tar.gz -O /tmp/ottr.tar.gz && R CMD INSTALL /tmp/ottr.tar.gz && rm /tmp/ottr.tar.gz +#RUN wget https://github.com/ucbds-infra/ottr/archive/refs/tags/0.1.0.tar.gz -O /tmp/ottr.tar.gz && R CMD INSTALL /tmp/ottr.tar.gz && rm /tmp/ottr.tar.gz #COPY install.R /tmp/install.R #RUN /tmp/install.R && rm -rf /tmp/downloaded_packages +COPY r-packages/ottr.r /tmp/r-packages/ +RUN r /tmp/r-packages/ottr.r + COPY r-packages/ph-290.r /tmp/r-packages/ RUN r /tmp/r-packages/ph-290.r diff --git a/r-packages/ottr.r b/r-packages/ottr.r new file mode 100755 index 0000000..8e3fbb9 --- /dev/null +++ b/r-packages/ottr.r @@ -0,0 +1,15 @@ +#!/usr/bin/env Rscript +# From https://github.com/berkeley-dsep-infra/datahub/issues/3403 +print("Installing packages for ottr") + +source("/tmp/class-libs.R") + +class_name = "ottr" + +class_libs = c( + "ottr", "1.1.4" +) + +class_libs_install_version(class_name, class_libs) + +print("Done installing packages for ottr") From dd2e8c96e902f709c260c69d87a8ba2d7a1b240b Mon Sep 17 00:00:00 2001 From: Jonathan Felder Date: Thu, 26 May 2022 13:26:54 -0700 Subject: [PATCH 075/132] Ottr round 2. --- r-packages/ottr.r | 13 ++----------- 1 file changed, 2 insertions(+), 11 deletions(-) diff --git a/r-packages/ottr.r b/r-packages/ottr.r index 8e3fbb9..2a5174f 100755 --- a/r-packages/ottr.r +++ b/r-packages/ottr.r @@ -1,15 +1,6 @@ #!/usr/bin/env Rscript # From https://github.com/berkeley-dsep-infra/datahub/issues/3403 -print("Installing packages for ottr") - -source("/tmp/class-libs.R") - -class_name = "ottr" - -class_libs = c( - "ottr", "1.1.4" -) - -class_libs_install_version(class_name, class_libs) +print("Installing packages for ottr") +devtools::install_version("ottr", version = "1.1.4", repos = "https://cran.r-project.org", upgrade = "never", quiet = FALSE) print("Done installing packages for ottr") From ac703626168d2bdf53b1fc74bb9e5d6da2b28881 Mon Sep 17 00:00:00 2001 From: YuviPanda Date: Tue, 31 May 2022 10:00:16 +0530 Subject: [PATCH 076/132] Bump z2jh to a more recent version - Bump z2jh version to latest master - a release is imminent. Latest version from https://jupyterhub.github.io/helm-chart/#development-releases-jupyterhub - Bump jupyterhub packages and jupyter packages in the images --- infra-requirements.txt | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/infra-requirements.txt b/infra-requirements.txt index 491fe50..479b073 100644 --- a/infra-requirements.txt +++ b/infra-requirements.txt @@ -8,17 +8,17 @@ # everwhere one by one. # FIXME: Freeze this to get exact versions of all dependencies -notebook==6.4.7 -jupyterlab==3.3.0 +notebook==6.4.11 +jupyterlab==3.4.2 jupyterlab-link-share==0.2.4 nbconvert==6.4.0 -retrolab==0.3.18 +retrolab==0.3.21 nbgitpuller==1.0.2 jupyter-resource-usage==0.6.1 # Matches version in images/hub/Dockerfile -jupyterhub==1.5.0 +jupyterhub==2.3.0 appmode==0.8.0 -ipywidgets==7.6.5 +ipywidgets==7.7.0 otter-grader==3.1.4 jupyter-tree-download==1.0.1 git-credential-helpers==0.2 From 098146dbc5009cdbcd41e2fae4360a48c6bbc2c5 Mon Sep 17 00:00:00 2001 From: YuviPanda Date: Tue, 31 May 2022 10:16:44 +0530 Subject: [PATCH 077/132] Unpin nbconvert jupyterlab will pick up a new one. --- infra-requirements.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/infra-requirements.txt b/infra-requirements.txt index 479b073..afe2deb 100644 --- a/infra-requirements.txt +++ b/infra-requirements.txt @@ -11,7 +11,6 @@ notebook==6.4.11 jupyterlab==3.4.2 jupyterlab-link-share==0.2.4 -nbconvert==6.4.0 retrolab==0.3.21 nbgitpuller==1.0.2 jupyter-resource-usage==0.6.1 From 4d443a70f2c00fd4d1ab929e8e9e77430abb1578 Mon Sep 17 00:00:00 2001 From: YuviPanda Date: Tue, 31 May 2022 23:47:47 +0530 Subject: [PATCH 078/132] Revert staging upgrade to z2jh master Reverts https://github.com/berkeley-dsep-infra/datahub/pull/3417 and https://github.com/berkeley-dsep-infra/datahub/pull/3420. They both work, and I think we're good to go - but I didn't want to deploy it just now as I'll be asleep soon. Ref https://github.com/berkeley-dsep-infra/datahub/issues/3418 --- infra-requirements.txt | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/infra-requirements.txt b/infra-requirements.txt index afe2deb..491fe50 100644 --- a/infra-requirements.txt +++ b/infra-requirements.txt @@ -8,16 +8,17 @@ # everwhere one by one. # FIXME: Freeze this to get exact versions of all dependencies -notebook==6.4.11 -jupyterlab==3.4.2 +notebook==6.4.7 +jupyterlab==3.3.0 jupyterlab-link-share==0.2.4 -retrolab==0.3.21 +nbconvert==6.4.0 +retrolab==0.3.18 nbgitpuller==1.0.2 jupyter-resource-usage==0.6.1 # Matches version in images/hub/Dockerfile -jupyterhub==2.3.0 +jupyterhub==1.5.0 appmode==0.8.0 -ipywidgets==7.7.0 +ipywidgets==7.6.5 otter-grader==3.1.4 jupyter-tree-download==1.0.1 git-credential-helpers==0.2 From 76c4366d604866b90939577d5ce56c3be0cd76ae Mon Sep 17 00:00:00 2001 From: YuviPanda Date: Wed, 1 Jun 2022 15:56:58 +0530 Subject: [PATCH 079/132] Revert "Merge pull request #3421 from yuvipanda/z2jh-rev" I'm back, and can deploy this properly now This reverts commit 197e492c9d1a12bd94088dd7f9f2c2c7f5a33926, reversing changes made to 3968cc5e8ec11e1589b4fea023731d057b8fa3d6. --- infra-requirements.txt | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/infra-requirements.txt b/infra-requirements.txt index 491fe50..afe2deb 100644 --- a/infra-requirements.txt +++ b/infra-requirements.txt @@ -8,17 +8,16 @@ # everwhere one by one. # FIXME: Freeze this to get exact versions of all dependencies -notebook==6.4.7 -jupyterlab==3.3.0 +notebook==6.4.11 +jupyterlab==3.4.2 jupyterlab-link-share==0.2.4 -nbconvert==6.4.0 -retrolab==0.3.18 +retrolab==0.3.21 nbgitpuller==1.0.2 jupyter-resource-usage==0.6.1 # Matches version in images/hub/Dockerfile -jupyterhub==1.5.0 +jupyterhub==2.3.0 appmode==0.8.0 -ipywidgets==7.6.5 +ipywidgets==7.7.0 otter-grader==3.1.4 jupyter-tree-download==1.0.1 git-credential-helpers==0.2 From 1cd1e1982011643141d80081682a4cc8673684d9 Mon Sep 17 00:00:00 2001 From: YuviPanda Date: Mon, 13 Jun 2022 13:24:37 +0300 Subject: [PATCH 080/132] Bump nbgitpuller version Brings in some bug fixes: https://github.com/jupyterhub/nbgitpuller/pull/260/files#diff-06572a96a58dc510037d5efa622f9bec8519bc1beab13c9f251e97e657a9d4edR11 --- infra-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/infra-requirements.txt b/infra-requirements.txt index afe2deb..2160833 100644 --- a/infra-requirements.txt +++ b/infra-requirements.txt @@ -12,7 +12,7 @@ notebook==6.4.11 jupyterlab==3.4.2 jupyterlab-link-share==0.2.4 retrolab==0.3.21 -nbgitpuller==1.0.2 +nbgitpuller==1.1.0 jupyter-resource-usage==0.6.1 # Matches version in images/hub/Dockerfile jupyterhub==2.3.0 From 2ae3d1b3fa869673dbafce8a1c2f0fb00421f7a0 Mon Sep 17 00:00:00 2001 From: YuviPanda Date: Thu, 21 Jul 2022 01:04:02 -0700 Subject: [PATCH 081/132] Remove link share from all images Just disabling it seems to break retrolab *only* on data8xv2 for some reason?! --- infra-requirements.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/infra-requirements.txt b/infra-requirements.txt index afe2deb..548fd37 100644 --- a/infra-requirements.txt +++ b/infra-requirements.txt @@ -10,7 +10,6 @@ # FIXME: Freeze this to get exact versions of all dependencies notebook==6.4.11 jupyterlab==3.4.2 -jupyterlab-link-share==0.2.4 retrolab==0.3.21 nbgitpuller==1.0.2 jupyter-resource-usage==0.6.1 From f952267f911c8e96d3b965d689a8648fe0520cbf Mon Sep 17 00:00:00 2001 From: YuviPanda Date: Mon, 22 Aug 2022 11:25:58 -0700 Subject: [PATCH 082/132] Bump some base packages Ref https://github.com/berkeley-dsep-infra/datahub/issues/3605 --- infra-requirements.txt | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/infra-requirements.txt b/infra-requirements.txt index 72b98b3..9b35d55 100644 --- a/infra-requirements.txt +++ b/infra-requirements.txt @@ -8,15 +8,15 @@ # everwhere one by one. # FIXME: Freeze this to get exact versions of all dependencies -notebook==6.4.11 -jupyterlab==3.4.2 +notebook==6.4.12 +jupyterlab==3.4.5 retrolab==0.3.21 nbgitpuller==1.1.0 jupyter-resource-usage==0.6.1 # Matches version in images/hub/Dockerfile -jupyterhub==2.3.0 +jupyterhub==2.3.1 appmode==0.8.0 -ipywidgets==7.7.0 +ipywidgets==8.0.1 otter-grader==3.1.4 jupyter-tree-download==1.0.1 git-credential-helpers==0.2 From 67abf2ed78c1616f25ddfcb4d659705eedc8dfd7 Mon Sep 17 00:00:00 2001 From: YuviPanda Date: Tue, 23 Aug 2022 21:52:49 -0700 Subject: [PATCH 083/132] Add qgrid back, stick to ipywidgets 7.x qgrid isn't compatible with 8.x, let's stick to 7.x for now. @ericvd-ucb says MCB32 uses it still. Should move away soon. --- infra-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/infra-requirements.txt b/infra-requirements.txt index 9b35d55..cf0e9c8 100644 --- a/infra-requirements.txt +++ b/infra-requirements.txt @@ -16,7 +16,7 @@ jupyter-resource-usage==0.6.1 # Matches version in images/hub/Dockerfile jupyterhub==2.3.1 appmode==0.8.0 -ipywidgets==8.0.1 +ipywidgets==7.7.2 otter-grader==3.1.4 jupyter-tree-download==1.0.1 git-credential-helpers==0.2 From 9d45464fac7d0fa8b87889cfe0d9f976501ffd18 Mon Sep 17 00:00:00 2001 From: Ryan Lovett Date: Tue, 27 Sep 2022 11:08:12 -0700 Subject: [PATCH 084/132] Add file-locks configuration to rocker images. Since a single process can seemingly trigger a node to issue a lot of ops, lets configure RStudio in the rocker images to behave the same as RStudio in our other images. --- Dockerfile | 3 +++ file-locks | 13 +++++++++++++ 2 files changed, 16 insertions(+) create mode 100644 file-locks diff --git a/Dockerfile b/Dockerfile index b6b8ca3..ba983cb 100644 --- a/Dockerfile +++ b/Dockerfile @@ -118,4 +118,7 @@ RUN tlmgr --verify-repo=none update --self && \ geometry \ epstopdf-pkg +# Use simpler locking strategy +COPY file-locks /etc/rstudio/file-locks + ENTRYPOINT ["tini", "--"] diff --git a/file-locks b/file-locks new file mode 100644 index 0000000..7b1a3fc --- /dev/null +++ b/file-locks @@ -0,0 +1,13 @@ +# https://docs.rstudio.com/ide/server-pro/load_balancing/configuration.html#file-locking + +# rocker sets this to advisory, but this might be causing NFS issues. +# lets set it to the default (default: linkbased) +lock-type=linkbased + +# we'll also reduce the frequency by 1/3 +refresh-rate=60 +timeout-interval=90 + +# log attempts +# enable-logging=1 +# log-file=/tmp/rstudio-locking.log From 8eb7361c010c0d594da60709b226d373115a108c Mon Sep 17 00:00:00 2001 From: ryanlovett Date: Mon, 19 Dec 2022 09:44:18 -0800 Subject: [PATCH 085/132] Bump jupyterhub to 3.1.0. Per slack with @yuvipanda. --- infra-requirements.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/infra-requirements.txt b/infra-requirements.txt index cf0e9c8..c557c7a 100644 --- a/infra-requirements.txt +++ b/infra-requirements.txt @@ -14,7 +14,7 @@ retrolab==0.3.21 nbgitpuller==1.1.0 jupyter-resource-usage==0.6.1 # Matches version in images/hub/Dockerfile -jupyterhub==2.3.1 +jupyterhub==3.1.0 appmode==0.8.0 ipywidgets==7.7.2 otter-grader==3.1.4 @@ -29,4 +29,4 @@ popularity-contest==0.4.1 # RISE is useful for presentations - see https://github.com/berkeley-dsep-infra/datahub/issues/2527 RISE==5.7.1 # syncthing for dropbox-like functionality -jupyter-syncthing-proxy==1.0.3 \ No newline at end of file +jupyter-syncthing-proxy==1.0.3 From 5449e75a859b85536a727844431e3200b9789fdc Mon Sep 17 00:00:00 2001 From: shane knapp Date: Tue, 28 Mar 2023 12:01:13 -0700 Subject: [PATCH 086/132] upgrade jupyterhub to 4.0.0b2, normalize python env files --- infra-requirements.txt | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/infra-requirements.txt b/infra-requirements.txt index c557c7a..4a6e199 100644 --- a/infra-requirements.txt +++ b/infra-requirements.txt @@ -14,10 +14,9 @@ retrolab==0.3.21 nbgitpuller==1.1.0 jupyter-resource-usage==0.6.1 # Matches version in images/hub/Dockerfile -jupyterhub==3.1.0 +jupyterhub==4.0.0b2 appmode==0.8.0 ipywidgets==7.7.2 -otter-grader==3.1.4 jupyter-tree-download==1.0.1 git-credential-helpers==0.2 # Enough people like this, let's load it in. From 10e1728658f4a24f9af3bcbbfbcaa568047a16c3 Mon Sep 17 00:00:00 2001 From: balajialg Date: Fri, 7 Jul 2023 14:43:23 -0700 Subject: [PATCH 087/132] Bump Public Health geospatial to v4.3.1 --- Dockerfile | 2 +- requirements.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index ba983cb..e1ef11d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM rocker/geospatial:4.1.2 +FROM rocker/geospatial:4.3.1 ENV NB_USER rstudio ENV NB_UID 1000 diff --git a/requirements.txt b/requirements.txt index 92072bc..9957350 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,3 @@ # RStudio support -jupyter-rsession-proxy==1.4 +jupyter-rsession-proxy==2.2.0 otter-grader==2.2.7 From 02374b96f5ee8d3a905589549768bc6abbd08076 Mon Sep 17 00:00:00 2001 From: balajialg Date: Fri, 7 Jul 2023 15:05:39 -0700 Subject: [PATCH 088/132] Move requirements.txt content to env.yml, mirror mamba-forge.bash from Stat 20 --- Dockerfile | 3 --- environment.yml | 7 +++++++ install-mambaforge.bash | 2 +- requirements.txt | 3 --- 4 files changed, 8 insertions(+), 7 deletions(-) create mode 100644 environment.yml delete mode 100644 requirements.txt diff --git a/Dockerfile b/Dockerfile index e1ef11d..180883c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -49,9 +49,6 @@ USER ${NB_USER} COPY infra-requirements.txt /tmp/infra-requirements.txt RUN pip install --no-cache-dir -r /tmp/infra-requirements.txt -COPY requirements.txt /tmp/requirements.txt -RUN pip install --no-cache-dir -r /tmp/requirements.txt - RUN mamba install -c conda-forge syncthing==1.18.6 # Support latest RStudio diff --git a/environment.yml b/environment.yml new file mode 100644 index 0000000..4182789 --- /dev/null +++ b/environment.yml @@ -0,0 +1,7 @@ +dependencies: +- pip: + - -r /tmp/infra-requirements.txt +# RStudio support + - jupyter-rsession-proxy==2.2.0 + - otter-grader==2.2.7 +~ diff --git a/install-mambaforge.bash b/install-mambaforge.bash index 13f4314..812319e 100755 --- a/install-mambaforge.bash +++ b/install-mambaforge.bash @@ -3,7 +3,7 @@ set -ex cd $(dirname $0) -MAMBAFORGE_VERSION=4.10.3-7 +MAMBAFORGE_VERSION=22.9.0-2 URL="https://github.com/conda-forge/miniforge/releases/download/${MAMBAFORGE_VERSION}/Mambaforge-${MAMBAFORGE_VERSION}-Linux-x86_64.sh" INSTALLER_PATH=/tmp/mambaforge-installer.sh diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index 9957350..0000000 --- a/requirements.txt +++ /dev/null @@ -1,3 +0,0 @@ -# RStudio support -jupyter-rsession-proxy==2.2.0 -otter-grader==2.2.7 From a95c404a9410a1026af5e8ce50fbec08332a1623 Mon Sep 17 00:00:00 2001 From: balajialg Date: Fri, 7 Jul 2023 15:28:10 -0700 Subject: [PATCH 089/132] Removing libssl as it is not supported in the latest ubuntu version --- Dockerfile | 1 - 1 file changed, 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 180883c..692fdb6 100644 --- a/Dockerfile +++ b/Dockerfile @@ -33,7 +33,6 @@ RUN apt-get update > /dev/null && \ libgtk-3-0 \ libnss3 \ libxss1 \ - libssl1.1 \ fonts-symbola \ gdebi-core \ tini \ From 5e4c0aca6f7fa6234a10b8d21187e472be5578a1 Mon Sep 17 00:00:00 2001 From: balajialg Date: Fri, 7 Jul 2023 17:11:49 -0700 Subject: [PATCH 090/132] Replace tlmgr with texlive in dockerfile --- Dockerfile | 48 ++++++++++++++++-------------------------------- 1 file changed, 16 insertions(+), 32 deletions(-) diff --git a/Dockerfile b/Dockerfile index 692fdb6..7dd9e1c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -81,38 +81,22 @@ RUN r /tmp/r-packages/ph-252.r COPY r-packages/2021-spring-phw-272a.r /tmp/r-packages/ RUN r /tmp/r-packages/2021-spring-phw-272a.r -RUN tlmgr repository add https://ftp.math.utah.edu/pub/tex/historic/systems/texlive/2021/tlnet-final -RUN tlmgr option repository https://ftp.math.utah.edu/pub/tex/historic/systems/texlive/2021/tlnet-final -RUN tlmgr --verify-repo=none update --self && \ - tlmgr --verify-repo=none install \ - amsmath \ - latex-amsmath-dev \ - iftex \ - kvoptions \ - ltxcmds \ - kvsetkeys \ - etoolbox \ - xcolor \ - auxhook \ - bigintcalc \ - bitset \ - etexcmds \ - gettitlestring \ - hycolor \ - hyperref \ - intcalc \ - kvdefinekeys \ - letltxmacro \ - pdfescape \ - refcount \ - rerunfilecheck \ - stringenc \ - uniquecounter \ - zapfding \ - pdftexcmds \ - infwarerr \ - geometry \ - epstopdf-pkg +# texlive-xetex pulls in texlive-latex-extra > texlive-latex-recommended +# We use Ubuntu's TeX because rocker's doesn't have most packages by default, +# and we don't want them to be downloaded on demand by students. +# tini is necessary because it is our ENTRYPOINT below. +RUN apt-get update && \ + apt-get -qq install \ + tini \ + fonts-symbola \ + pandoc \ + texlive-xetex \ + texlive-fonts-recommended \ + texlive-fonts-extra \ + texlive-plain-generic \ + > /dev/null && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* # Use simpler locking strategy COPY file-locks /etc/rstudio/file-locks From 8c42455e4b6d8983c948e3dd7795dbcc64edd8dc Mon Sep 17 00:00:00 2001 From: balajialg Date: Sat, 8 Jul 2023 16:01:51 -0700 Subject: [PATCH 091/132] Testing the placement of apt-get solves for root access --- Dockerfile | 34 +++++++++++++++++----------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/Dockerfile b/Dockerfile index 7dd9e1c..15ef320 100644 --- a/Dockerfile +++ b/Dockerfile @@ -40,6 +40,23 @@ RUN apt-get update > /dev/null && \ apt-get clean && \ rm -rf /var/lib/apt/lists/* +# texlive-xetex pulls in texlive-latex-extra > texlive-latex-recommended +# We use Ubuntu's TeX because rocker's doesn't have most packages by default, +# and we don't want them to be downloaded on demand by students. +# tini is necessary because it is our ENTRYPOINT below. +RUN apt-get update && \ + apt-get -qq install \ + tini \ + fonts-symbola \ + pandoc \ + texlive-xetex \ + texlive-fonts-recommended \ + texlive-fonts-extra \ + texlive-plain-generic \ + > /dev/null && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* + COPY install-mambaforge.bash /tmp/install-mambaforge.bash RUN /tmp/install-mambaforge.bash @@ -81,23 +98,6 @@ RUN r /tmp/r-packages/ph-252.r COPY r-packages/2021-spring-phw-272a.r /tmp/r-packages/ RUN r /tmp/r-packages/2021-spring-phw-272a.r -# texlive-xetex pulls in texlive-latex-extra > texlive-latex-recommended -# We use Ubuntu's TeX because rocker's doesn't have most packages by default, -# and we don't want them to be downloaded on demand by students. -# tini is necessary because it is our ENTRYPOINT below. -RUN apt-get update && \ - apt-get -qq install \ - tini \ - fonts-symbola \ - pandoc \ - texlive-xetex \ - texlive-fonts-recommended \ - texlive-fonts-extra \ - texlive-plain-generic \ - > /dev/null && \ - apt-get clean && \ - rm -rf /var/lib/apt/lists/* - # Use simpler locking strategy COPY file-locks /etc/rstudio/file-locks From bbfa9fb6d7caaffa7ec5fcd66892d00b1e3e6a8c Mon Sep 17 00:00:00 2001 From: balajialg Date: Sat, 8 Jul 2023 16:22:47 -0700 Subject: [PATCH 092/132] Optimizing get-apt command in dockerfile --- Dockerfile | 20 ++++++-------------- 1 file changed, 6 insertions(+), 14 deletions(-) diff --git a/Dockerfile b/Dockerfile index 15ef320..0c5f555 100644 --- a/Dockerfile +++ b/Dockerfile @@ -21,6 +21,10 @@ WORKDIR ${HOME} # Install packages needed by notebook-as-pdf # nodejs for installing notebook / jupyterhub from source # libarchive-dev for https://github.com/berkeley-dsep-infra/datahub/issues/1997 +# texlive-xetex pulls in texlive-latex-extra > texlive-latex-recommended +# We use Ubuntu's TeX because rocker's doesn't have most packages by default, +# and we don't want them to be downloaded on demand by students. +# tini is necessary because it is our ENTRYPOINT below. RUN apt-get update > /dev/null && \ apt-get install --yes \ libx11-xcb1 \ @@ -36,24 +40,12 @@ RUN apt-get update > /dev/null && \ fonts-symbola \ gdebi-core \ tini \ - nodejs npm > /dev/null && \ - apt-get clean && \ - rm -rf /var/lib/apt/lists/* - -# texlive-xetex pulls in texlive-latex-extra > texlive-latex-recommended -# We use Ubuntu's TeX because rocker's doesn't have most packages by default, -# and we don't want them to be downloaded on demand by students. -# tini is necessary because it is our ENTRYPOINT below. -RUN apt-get update && \ - apt-get -qq install \ - tini \ - fonts-symbola \ - pandoc \ + pandoc \ texlive-xetex \ texlive-fonts-recommended \ texlive-fonts-extra \ texlive-plain-generic \ - > /dev/null && \ + nodejs npm > /dev/null && \ apt-get clean && \ rm -rf /var/lib/apt/lists/* From e003174f9439e7423905f113b1b5671cdbd0e6f7 Mon Sep 17 00:00:00 2001 From: ryanlovett Date: Mon, 14 Aug 2023 21:04:02 -0700 Subject: [PATCH 093/132] Run infra-packages script. --- infra-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/infra-requirements.txt b/infra-requirements.txt index 4a6e199..afa0268 100644 --- a/infra-requirements.txt +++ b/infra-requirements.txt @@ -14,7 +14,7 @@ retrolab==0.3.21 nbgitpuller==1.1.0 jupyter-resource-usage==0.6.1 # Matches version in images/hub/Dockerfile -jupyterhub==4.0.0b2 +jupyterhub==4.0.2 appmode==0.8.0 ipywidgets==7.7.2 jupyter-tree-download==1.0.1 From d7877ea656b7b6bdc43b571c1399f215682e16ad Mon Sep 17 00:00:00 2001 From: shane knapp Date: Tue, 31 Oct 2023 11:58:04 -0700 Subject: [PATCH 094/132] update popcon to the hash of my PR --- infra-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/infra-requirements.txt b/infra-requirements.txt index afa0268..75e9ab7 100644 --- a/infra-requirements.txt +++ b/infra-requirements.txt @@ -24,7 +24,7 @@ jupyter-contrib-nbextensions==0.5.1 jupyter_nbextensions_configurator==0.4.1 # Measure popularity of different packages in our hubs # https://discourse.jupyter.org/t/request-for-implementation-instrument-libraries-actively-used-by-users-on-a-jupyterhub/7994?u=yuvipanda -popularity-contest==0.4.1 +popularity-contest==git+https://github.com/yuvipanda/python-popularity-contest@9e30ec414e45615e75a0e86424141b7093c8f686 # RISE is useful for presentations - see https://github.com/berkeley-dsep-infra/datahub/issues/2527 RISE==5.7.1 # syncthing for dropbox-like functionality From 8013344da9d14c0cf4e691908c7408e4dc7c2fad Mon Sep 17 00:00:00 2001 From: shane knapp Date: Tue, 31 Oct 2023 12:21:40 -0700 Subject: [PATCH 095/132] s/python-popularity==// --- infra-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/infra-requirements.txt b/infra-requirements.txt index 75e9ab7..611bfb5 100644 --- a/infra-requirements.txt +++ b/infra-requirements.txt @@ -24,7 +24,7 @@ jupyter-contrib-nbextensions==0.5.1 jupyter_nbextensions_configurator==0.4.1 # Measure popularity of different packages in our hubs # https://discourse.jupyter.org/t/request-for-implementation-instrument-libraries-actively-used-by-users-on-a-jupyterhub/7994?u=yuvipanda -popularity-contest==git+https://github.com/yuvipanda/python-popularity-contest@9e30ec414e45615e75a0e86424141b7093c8f686 +git+https://github.com/yuvipanda/python-popularity-contest@9e30ec414e45615e75a0e86424141b7093c8f686 # RISE is useful for presentations - see https://github.com/berkeley-dsep-infra/datahub/issues/2527 RISE==5.7.1 # syncthing for dropbox-like functionality From 67ad02d281e870f11fe335c12dfb58c5a5da4b32 Mon Sep 17 00:00:00 2001 From: shane knapp Date: Wed, 1 Nov 2023 14:36:10 -0700 Subject: [PATCH 096/132] fixing traitlets thing --- environment.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/environment.yml b/environment.yml index 4182789..5cd1e49 100644 --- a/environment.yml +++ b/environment.yml @@ -1,4 +1,11 @@ +name: publichealth + +channels: +- conda-forge + dependencies: +# bug w/notebook and traitlets: https://github.com/jupyter/notebook/issues/7048 +- traitlets=5.9.* - pip: - -r /tmp/infra-requirements.txt # RStudio support From 912dd2882b9da41002b9216454aeb0a75a61796e Mon Sep 17 00:00:00 2001 From: shane knapp Date: Wed, 1 Nov 2023 15:36:50 -0700 Subject: [PATCH 097/132] publichealth is still sad --- environment.yml | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/environment.yml b/environment.yml index 5cd1e49..07e7fe5 100644 --- a/environment.yml +++ b/environment.yml @@ -1,12 +1,7 @@ -name: publichealth - -channels: -- conda-forge - dependencies: -# bug w/notebook and traitlets: https://github.com/jupyter/notebook/issues/7048 -- traitlets=5.9.* - pip: + # bug w/notebook and traitlets: https://github.com/jupyter/notebook/issues/7048 + - traitlets==5.9.0 - -r /tmp/infra-requirements.txt # RStudio support - jupyter-rsession-proxy==2.2.0 From a0f7c874bc71a602548a1ee55b4a06c51c573160 Mon Sep 17 00:00:00 2001 From: shane knapp Date: Wed, 1 Nov 2023 17:13:39 -0700 Subject: [PATCH 098/132] duplicate install might be breaking things --- Dockerfile | 3 --- 1 file changed, 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index 0c5f555..d62739c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -59,9 +59,6 @@ RUN pip install --no-cache-dir -r /tmp/infra-requirements.txt RUN mamba install -c conda-forge syncthing==1.18.6 -# Support latest RStudio -RUN pip install --no-cache 'jupyter-rsession-proxy>=2.0' - # Install IRKernel RUN R --quiet -e "install.packages('IRkernel', quiet = TRUE)" && \ R --quiet -e "IRkernel::installspec(prefix='${CONDA_DIR}')" From f9fc209ead330e5f9ceae3b5993f974292d453b0 Mon Sep 17 00:00:00 2001 From: shane knapp Date: Wed, 1 Nov 2023 17:24:59 -0700 Subject: [PATCH 099/132] do it here and first? --- infra-requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/infra-requirements.txt b/infra-requirements.txt index 611bfb5..ced897d 100644 --- a/infra-requirements.txt +++ b/infra-requirements.txt @@ -8,6 +8,7 @@ # everwhere one by one. # FIXME: Freeze this to get exact versions of all dependencies +traitlets==5.9.0 notebook==6.4.12 jupyterlab==3.4.5 retrolab==0.3.21 From b386d0902f4d8dffa5f17c03a59c0ab555df5502 Mon Sep 17 00:00:00 2001 From: shane knapp Date: Thu, 2 Nov 2023 11:04:11 -0700 Subject: [PATCH 100/132] revert popcon, move conda install to env.yml --- Dockerfile | 2 -- environment.yml | 1 + infra-requirements.txt | 2 +- 3 files changed, 2 insertions(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index d62739c..ce65d8f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -57,8 +57,6 @@ USER ${NB_USER} COPY infra-requirements.txt /tmp/infra-requirements.txt RUN pip install --no-cache-dir -r /tmp/infra-requirements.txt -RUN mamba install -c conda-forge syncthing==1.18.6 - # Install IRKernel RUN R --quiet -e "install.packages('IRkernel', quiet = TRUE)" && \ R --quiet -e "IRkernel::installspec(prefix='${CONDA_DIR}')" diff --git a/environment.yml b/environment.yml index 07e7fe5..70d5b8f 100644 --- a/environment.yml +++ b/environment.yml @@ -1,4 +1,5 @@ dependencies: +- syncthing==1.18.6 - pip: # bug w/notebook and traitlets: https://github.com/jupyter/notebook/issues/7048 - traitlets==5.9.0 diff --git a/infra-requirements.txt b/infra-requirements.txt index ced897d..3e37945 100644 --- a/infra-requirements.txt +++ b/infra-requirements.txt @@ -25,7 +25,7 @@ jupyter-contrib-nbextensions==0.5.1 jupyter_nbextensions_configurator==0.4.1 # Measure popularity of different packages in our hubs # https://discourse.jupyter.org/t/request-for-implementation-instrument-libraries-actively-used-by-users-on-a-jupyterhub/7994?u=yuvipanda -git+https://github.com/yuvipanda/python-popularity-contest@9e30ec414e45615e75a0e86424141b7093c8f686 +popularity-contest==0.4.1 # RISE is useful for presentations - see https://github.com/berkeley-dsep-infra/datahub/issues/2527 RISE==5.7.1 # syncthing for dropbox-like functionality From 222b4d49c4de9e874cac8fc1c4126873f1bd58ab Mon Sep 17 00:00:00 2001 From: shane knapp Date: Thu, 2 Nov 2023 12:12:11 -0700 Subject: [PATCH 101/132] remove ~ and linespace at EOF, move rsession proxy to conda --- environment.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/environment.yml b/environment.yml index 70d5b8f..d5ef8e1 100644 --- a/environment.yml +++ b/environment.yml @@ -1,10 +1,9 @@ dependencies: - syncthing==1.18.6 +- jupyter-rsession-proxy==2.2.0 - pip: # bug w/notebook and traitlets: https://github.com/jupyter/notebook/issues/7048 - traitlets==5.9.0 - -r /tmp/infra-requirements.txt # RStudio support - - jupyter-rsession-proxy==2.2.0 - otter-grader==2.2.7 -~ From 210ac66edcea63e375abac1c1236ac8d49b33775 Mon Sep 17 00:00:00 2001 From: shane knapp Date: Thu, 2 Nov 2023 12:18:09 -0700 Subject: [PATCH 102/132] actually install what we want to have installed --- Dockerfile | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/Dockerfile b/Dockerfile index ce65d8f..d4420b7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -50,8 +50,13 @@ RUN apt-get update > /dev/null && \ rm -rf /var/lib/apt/lists/* COPY install-mambaforge.bash /tmp/install-mambaforge.bash +COPY environment.yml /tmp/environment.yml + RUN /tmp/install-mambaforge.bash +RUN mamba env update -p ${CONDA_DIR} -f /tmp/environment.yml && \ + mamba clean -afy + USER ${NB_USER} COPY infra-requirements.txt /tmp/infra-requirements.txt From d1b2c28491a29bc54e6e4c1775a2ab811f320e9f Mon Sep 17 00:00:00 2001 From: shane knapp Date: Thu, 2 Nov 2023 12:33:35 -0700 Subject: [PATCH 103/132] lots of borked things in here --- 1q | 96 +++++++++++++++++++++++++++++++++++++++++++++++++ Dockerfile | 4 +-- environment.yml | 3 +- 3 files changed, 100 insertions(+), 3 deletions(-) create mode 100644 1q diff --git a/1q b/1q new file mode 100644 index 0000000..45ca1bb --- /dev/null +++ b/1q @@ -0,0 +1,96 @@ +FROM rocker/geospatial:4.3.1 + +ENV NB_USER rstudio +ENV NB_UID 1000 +ENV CONDA_DIR /srv/conda + +# Set ENV for all programs... +ENV PATH ${CONDA_DIR}/bin:$PATH + +# And set ENV for R! It doesn't read from the environment... +RUN echo "PATH=${PATH}" >> /usr/local/lib/R/etc/Renviron.site + +# Add PATH to /etc/profile so it gets picked up by the terminal +RUN echo "PATH=${PATH}" >> /etc/profile +RUN echo "export PATH" >> /etc/profile + +ENV HOME /home/${NB_USER} + +WORKDIR ${HOME} + +# Install packages needed by notebook-as-pdf +# nodejs for installing notebook / jupyterhub from source +# libarchive-dev for https://github.com/berkeley-dsep-infra/datahub/issues/1997 +# texlive-xetex pulls in texlive-latex-extra > texlive-latex-recommended +# We use Ubuntu's TeX because rocker's doesn't have most packages by default, +# and we don't want them to be downloaded on demand by students. +# tini is necessary because it is our ENTRYPOINT below. +RUN apt-get update > /dev/null && \ + apt-get install --yes \ + libx11-xcb1 \ + libxtst6 \ + libxrandr2 \ + libasound2 \ + libpangocairo-1.0-0 \ + libatk1.0-0 \ + libatk-bridge2.0-0 \ + libgtk-3-0 \ + libnss3 \ + libxss1 \ + fonts-symbola \ + gdebi-core \ + tini \ + pandoc \ + texlive-xetex \ + texlive-fonts-recommended \ + texlive-fonts-extra \ + texlive-plain-generic \ + nodejs npm > /dev/null && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* + +USER ${NB_USER} + +COPY install-mambaforge.bash /tmp/install-mambaforge.bash +COPY environment.yml /tmp/environment.yml + +RUN /tmp/install-mambaforge.bash + +RUN mamba env update -p ${CONDA_DIR} -f /tmp/environment.yml && \ + mamba clean -afy + +COPY infra-requirements.txt /tmp/infra-requirements.txt +RUN pip install --no-cache-dir -r /tmp/infra-requirements.txt + +# Install IRKernel +RUN R --quiet -e "install.packages('IRkernel', quiet = TRUE)" && \ + R --quiet -e "IRkernel::installspec(prefix='${CONDA_DIR}')" + +COPY class-libs.R /tmp/class-libs.R +RUN mkdir -p /tmp/r-packages + +# Workaround to install ottr, issue 3342 +#RUN wget https://github.com/ucbds-infra/ottr/archive/refs/tags/0.1.0.tar.gz -O /tmp/ottr.tar.gz && R CMD INSTALL /tmp/ottr.tar.gz && rm /tmp/ottr.tar.gz + +#COPY install.R /tmp/install.R +#RUN /tmp/install.R && rm -rf /tmp/downloaded_packages + +COPY r-packages/ottr.r /tmp/r-packages/ +RUN r /tmp/r-packages/ottr.r + +COPY r-packages/ph-290.r /tmp/r-packages/ +RUN r /tmp/r-packages/ph-290.r + +COPY r-packages/ph-142.r /tmp/r-packages/ +RUN r /tmp/r-packages/ph-142.r + +COPY r-packages/ph-252.r /tmp/r-packages/ +RUN r /tmp/r-packages/ph-252.r + +COPY r-packages/2021-spring-phw-272a.r /tmp/r-packages/ +RUN r /tmp/r-packages/2021-spring-phw-272a.r + +# Use simpler locking strategy +COPY file-locks /etc/rstudio/file-locks + +ENTRYPOINT ["tini", "--"] diff --git a/Dockerfile b/Dockerfile index d4420b7..45ca1bb 100644 --- a/Dockerfile +++ b/Dockerfile @@ -49,6 +49,8 @@ RUN apt-get update > /dev/null && \ apt-get clean && \ rm -rf /var/lib/apt/lists/* +USER ${NB_USER} + COPY install-mambaforge.bash /tmp/install-mambaforge.bash COPY environment.yml /tmp/environment.yml @@ -57,8 +59,6 @@ RUN /tmp/install-mambaforge.bash RUN mamba env update -p ${CONDA_DIR} -f /tmp/environment.yml && \ mamba clean -afy -USER ${NB_USER} - COPY infra-requirements.txt /tmp/infra-requirements.txt RUN pip install --no-cache-dir -r /tmp/infra-requirements.txt diff --git a/environment.yml b/environment.yml index d5ef8e1..028a6ad 100644 --- a/environment.yml +++ b/environment.yml @@ -1,9 +1,10 @@ dependencies: +- pip - syncthing==1.18.6 - jupyter-rsession-proxy==2.2.0 - pip: # bug w/notebook and traitlets: https://github.com/jupyter/notebook/issues/7048 - traitlets==5.9.0 - - -r /tmp/infra-requirements.txt + # - -r /tmp/infra-requirements.txt # RStudio support - otter-grader==2.2.7 From d577b9d3927309badbc7716a266d8fc651e92a33 Mon Sep 17 00:00:00 2001 From: shane knapp Date: Thu, 2 Nov 2023 12:34:10 -0700 Subject: [PATCH 104/132] derp vim fat finger --- 1q | 96 -------------------------------------------------------------- 1 file changed, 96 deletions(-) delete mode 100644 1q diff --git a/1q b/1q deleted file mode 100644 index 45ca1bb..0000000 --- a/1q +++ /dev/null @@ -1,96 +0,0 @@ -FROM rocker/geospatial:4.3.1 - -ENV NB_USER rstudio -ENV NB_UID 1000 -ENV CONDA_DIR /srv/conda - -# Set ENV for all programs... -ENV PATH ${CONDA_DIR}/bin:$PATH - -# And set ENV for R! It doesn't read from the environment... -RUN echo "PATH=${PATH}" >> /usr/local/lib/R/etc/Renviron.site - -# Add PATH to /etc/profile so it gets picked up by the terminal -RUN echo "PATH=${PATH}" >> /etc/profile -RUN echo "export PATH" >> /etc/profile - -ENV HOME /home/${NB_USER} - -WORKDIR ${HOME} - -# Install packages needed by notebook-as-pdf -# nodejs for installing notebook / jupyterhub from source -# libarchive-dev for https://github.com/berkeley-dsep-infra/datahub/issues/1997 -# texlive-xetex pulls in texlive-latex-extra > texlive-latex-recommended -# We use Ubuntu's TeX because rocker's doesn't have most packages by default, -# and we don't want them to be downloaded on demand by students. -# tini is necessary because it is our ENTRYPOINT below. -RUN apt-get update > /dev/null && \ - apt-get install --yes \ - libx11-xcb1 \ - libxtst6 \ - libxrandr2 \ - libasound2 \ - libpangocairo-1.0-0 \ - libatk1.0-0 \ - libatk-bridge2.0-0 \ - libgtk-3-0 \ - libnss3 \ - libxss1 \ - fonts-symbola \ - gdebi-core \ - tini \ - pandoc \ - texlive-xetex \ - texlive-fonts-recommended \ - texlive-fonts-extra \ - texlive-plain-generic \ - nodejs npm > /dev/null && \ - apt-get clean && \ - rm -rf /var/lib/apt/lists/* - -USER ${NB_USER} - -COPY install-mambaforge.bash /tmp/install-mambaforge.bash -COPY environment.yml /tmp/environment.yml - -RUN /tmp/install-mambaforge.bash - -RUN mamba env update -p ${CONDA_DIR} -f /tmp/environment.yml && \ - mamba clean -afy - -COPY infra-requirements.txt /tmp/infra-requirements.txt -RUN pip install --no-cache-dir -r /tmp/infra-requirements.txt - -# Install IRKernel -RUN R --quiet -e "install.packages('IRkernel', quiet = TRUE)" && \ - R --quiet -e "IRkernel::installspec(prefix='${CONDA_DIR}')" - -COPY class-libs.R /tmp/class-libs.R -RUN mkdir -p /tmp/r-packages - -# Workaround to install ottr, issue 3342 -#RUN wget https://github.com/ucbds-infra/ottr/archive/refs/tags/0.1.0.tar.gz -O /tmp/ottr.tar.gz && R CMD INSTALL /tmp/ottr.tar.gz && rm /tmp/ottr.tar.gz - -#COPY install.R /tmp/install.R -#RUN /tmp/install.R && rm -rf /tmp/downloaded_packages - -COPY r-packages/ottr.r /tmp/r-packages/ -RUN r /tmp/r-packages/ottr.r - -COPY r-packages/ph-290.r /tmp/r-packages/ -RUN r /tmp/r-packages/ph-290.r - -COPY r-packages/ph-142.r /tmp/r-packages/ -RUN r /tmp/r-packages/ph-142.r - -COPY r-packages/ph-252.r /tmp/r-packages/ -RUN r /tmp/r-packages/ph-252.r - -COPY r-packages/2021-spring-phw-272a.r /tmp/r-packages/ -RUN r /tmp/r-packages/2021-spring-phw-272a.r - -# Use simpler locking strategy -COPY file-locks /etc/rstudio/file-locks - -ENTRYPOINT ["tini", "--"] From f3cef57c68cdf1b8b7fe4d47eda9fb8972e06844 Mon Sep 17 00:00:00 2001 From: shane knapp Date: Thu, 2 Nov 2023 12:47:05 -0700 Subject: [PATCH 105/132] positioning is key --- Dockerfile | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/Dockerfile b/Dockerfile index 45ca1bb..05ad03c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -49,13 +49,12 @@ RUN apt-get update > /dev/null && \ apt-get clean && \ rm -rf /var/lib/apt/lists/* -USER ${NB_USER} - COPY install-mambaforge.bash /tmp/install-mambaforge.bash -COPY environment.yml /tmp/environment.yml - RUN /tmp/install-mambaforge.bash +USER ${NB_USER} + +COPY environment.yml /tmp/environment.yml RUN mamba env update -p ${CONDA_DIR} -f /tmp/environment.yml && \ mamba clean -afy From 2f5e8b7546e6110566b88f1cec7d92e2b05a8508 Mon Sep 17 00:00:00 2001 From: shane knapp Date: Tue, 7 Nov 2023 11:48:22 -0800 Subject: [PATCH 106/132] bump popcon --- environment.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/environment.yml b/environment.yml index 028a6ad..6f90f9c 100644 --- a/environment.yml +++ b/environment.yml @@ -2,6 +2,7 @@ dependencies: - pip - syncthing==1.18.6 - jupyter-rsession-proxy==2.2.0 +- git+https://github.com/yuvipanda/python-popularity-contest@9e30ec414e45615e75a0e86424141b7093c8f686 - pip: # bug w/notebook and traitlets: https://github.com/jupyter/notebook/issues/7048 - traitlets==5.9.0 From fb590a372af906c7d26e05c1ee58e1dae83d77a2 Mon Sep 17 00:00:00 2001 From: shane knapp Date: Tue, 7 Nov 2023 11:58:02 -0800 Subject: [PATCH 107/132] pip not conda --- environment.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/environment.yml b/environment.yml index 6f90f9c..f60ff2a 100644 --- a/environment.yml +++ b/environment.yml @@ -2,10 +2,10 @@ dependencies: - pip - syncthing==1.18.6 - jupyter-rsession-proxy==2.2.0 -- git+https://github.com/yuvipanda/python-popularity-contest@9e30ec414e45615e75a0e86424141b7093c8f686 - pip: # bug w/notebook and traitlets: https://github.com/jupyter/notebook/issues/7048 - traitlets==5.9.0 # - -r /tmp/infra-requirements.txt # RStudio support - otter-grader==2.2.7 + - git+https://github.com/yuvipanda/python-popularity-contest@9e30ec414e45615e75a0e86424141b7093c8f686 From 7050437e6880174977a51c3925cd229a1bb8a57b Mon Sep 17 00:00:00 2001 From: shane knapp Date: Tue, 7 Nov 2023 13:47:47 -0800 Subject: [PATCH 108/132] update infra-reqs, all hubs etc to run latest popcon with better error handling --- environment.yml | 5 ++--- infra-requirements.txt | 3 +-- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/environment.yml b/environment.yml index f60ff2a..f6e6469 100644 --- a/environment.yml +++ b/environment.yml @@ -5,7 +5,6 @@ dependencies: - pip: # bug w/notebook and traitlets: https://github.com/jupyter/notebook/issues/7048 - traitlets==5.9.0 - # - -r /tmp/infra-requirements.txt -# RStudio support + # - -r /tmp/infra-requirements.txt + # RStudio support - otter-grader==2.2.7 - - git+https://github.com/yuvipanda/python-popularity-contest@9e30ec414e45615e75a0e86424141b7093c8f686 diff --git a/infra-requirements.txt b/infra-requirements.txt index 3e37945..ca42d16 100644 --- a/infra-requirements.txt +++ b/infra-requirements.txt @@ -8,7 +8,6 @@ # everwhere one by one. # FIXME: Freeze this to get exact versions of all dependencies -traitlets==5.9.0 notebook==6.4.12 jupyterlab==3.4.5 retrolab==0.3.21 @@ -25,7 +24,7 @@ jupyter-contrib-nbextensions==0.5.1 jupyter_nbextensions_configurator==0.4.1 # Measure popularity of different packages in our hubs # https://discourse.jupyter.org/t/request-for-implementation-instrument-libraries-actively-used-by-users-on-a-jupyterhub/7994?u=yuvipanda -popularity-contest==0.4.1 +git+https://github.com/shaneknapp/python-popularity-contest.git@add-error-handling # RISE is useful for presentations - see https://github.com/berkeley-dsep-infra/datahub/issues/2527 RISE==5.7.1 # syncthing for dropbox-like functionality From a74f5a396093c59b992261357618699d85876dce Mon Sep 17 00:00:00 2001 From: shane knapp Date: Thu, 16 Nov 2023 14:40:08 -0800 Subject: [PATCH 109/132] less means more (images built) --- Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Dockerfile b/Dockerfile index 05ad03c..1ef7954 100644 --- a/Dockerfile +++ b/Dockerfile @@ -27,6 +27,7 @@ WORKDIR ${HOME} # tini is necessary because it is our ENTRYPOINT below. RUN apt-get update > /dev/null && \ apt-get install --yes \ + less \ libx11-xcb1 \ libxtst6 \ libxrandr2 \ From aeec6e21bd8af3b57dd36b00253de550a80c2701 Mon Sep 17 00:00:00 2001 From: shane knapp Date: Thu, 11 Jan 2024 13:09:37 -0800 Subject: [PATCH 110/132] jupyterlab 4.0.10 redux --- infra-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/infra-requirements.txt b/infra-requirements.txt index ca42d16..6493f7d 100644 --- a/infra-requirements.txt +++ b/infra-requirements.txt @@ -9,7 +9,7 @@ # FIXME: Freeze this to get exact versions of all dependencies notebook==6.4.12 -jupyterlab==3.4.5 +jupyterlab==4.0.10 retrolab==0.3.21 nbgitpuller==1.1.0 jupyter-resource-usage==0.6.1 From 7e3ca36f0d593e48b40afe2b7da9920f360447d2 Mon Sep 17 00:00:00 2001 From: shane knapp Date: Thu, 11 Jan 2024 13:20:15 -0800 Subject: [PATCH 111/132] remove retrolab --- infra-requirements.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/infra-requirements.txt b/infra-requirements.txt index 6493f7d..664bf97 100644 --- a/infra-requirements.txt +++ b/infra-requirements.txt @@ -10,7 +10,6 @@ # FIXME: Freeze this to get exact versions of all dependencies notebook==6.4.12 jupyterlab==4.0.10 -retrolab==0.3.21 nbgitpuller==1.1.0 jupyter-resource-usage==0.6.1 # Matches version in images/hub/Dockerfile From 162986c878eef6a6b177e8ab83be8fca88ed4fb2 Mon Sep 17 00:00:00 2001 From: shane knapp Date: Thu, 11 Jan 2024 13:29:04 -0800 Subject: [PATCH 112/132] bump jupyter-resource-usage --- infra-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/infra-requirements.txt b/infra-requirements.txt index 664bf97..ffe5c10 100644 --- a/infra-requirements.txt +++ b/infra-requirements.txt @@ -11,7 +11,7 @@ notebook==6.4.12 jupyterlab==4.0.10 nbgitpuller==1.1.0 -jupyter-resource-usage==0.6.1 +jupyter-resource-usage==1.0.1 # Matches version in images/hub/Dockerfile jupyterhub==4.0.2 appmode==0.8.0 From f74ac692bf4917cc23013e5223f16411fb2ad3e0 Mon Sep 17 00:00:00 2001 From: shane knapp Date: Thu, 11 Jan 2024 13:41:37 -0800 Subject: [PATCH 113/132] adding jupyter-archive to all hubs --- infra-requirements.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/infra-requirements.txt b/infra-requirements.txt index ffe5c10..f36a59a 100644 --- a/infra-requirements.txt +++ b/infra-requirements.txt @@ -28,3 +28,5 @@ git+https://github.com/shaneknapp/python-popularity-contest.git@add-error-handli RISE==5.7.1 # syncthing for dropbox-like functionality jupyter-syncthing-proxy==1.0.3 +# jupyter archival tool for easy user downloads +jupyter-archive==3.4.0 From ca1a04ad7b1a62e39eeffff72fb994ff9367b453 Mon Sep 17 00:00:00 2001 From: shane knapp Date: Sun, 14 Jan 2024 09:48:12 -0800 Subject: [PATCH 114/132] fix datahub build --- infra-requirements.txt | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/infra-requirements.txt b/infra-requirements.txt index f36a59a..ef16a52 100644 --- a/infra-requirements.txt +++ b/infra-requirements.txt @@ -10,17 +10,17 @@ # FIXME: Freeze this to get exact versions of all dependencies notebook==6.4.12 jupyterlab==4.0.10 -nbgitpuller==1.1.0 +nbgitpuller==1.2.0 jupyter-resource-usage==1.0.1 # Matches version in images/hub/Dockerfile jupyterhub==4.0.2 appmode==0.8.0 -ipywidgets==7.7.2 +ipywidgets==8.0.7 jupyter-tree-download==1.0.1 git-credential-helpers==0.2 # Enough people like this, let's load it in. -jupyter-contrib-nbextensions==0.5.1 -jupyter_nbextensions_configurator==0.4.1 +jupyter-contrib-nbextensions==0.7.0 +jupyter_nbextensions_configurator==0.6.3 # Measure popularity of different packages in our hubs # https://discourse.jupyter.org/t/request-for-implementation-instrument-libraries-actively-used-by-users-on-a-jupyterhub/7994?u=yuvipanda git+https://github.com/shaneknapp/python-popularity-contest.git@add-error-handling From 8e2a63182aa372da1a45fc76a091267284b02dcb Mon Sep 17 00:00:00 2001 From: shane knapp Date: Sun, 14 Jan 2024 12:49:20 -0800 Subject: [PATCH 115/132] bump notebook to 7 --- infra-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/infra-requirements.txt b/infra-requirements.txt index ef16a52..e08dea3 100644 --- a/infra-requirements.txt +++ b/infra-requirements.txt @@ -8,7 +8,7 @@ # everwhere one by one. # FIXME: Freeze this to get exact versions of all dependencies -notebook==6.4.12 +notebook==7.0.6 jupyterlab==4.0.10 nbgitpuller==1.2.0 jupyter-resource-usage==1.0.1 From 0a32061f8c4e02684e5e1e521fe6fe5f0901dc2a Mon Sep 17 00:00:00 2001 From: shane knapp Date: Wed, 24 Jan 2024 11:01:43 -0800 Subject: [PATCH 116/132] bump rocker geospatial, pull from new repo --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 1ef7954..5a233b9 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM rocker/geospatial:4.3.1 +FROM ghcr.io/rocker-org/geospatial:4.3.2 ENV NB_USER rstudio ENV NB_UID 1000 From 6869528ffb4a133af43994f3462b5b4313bd555b Mon Sep 17 00:00:00 2001 From: shane knapp Date: Thu, 25 Jan 2024 14:02:34 -0800 Subject: [PATCH 117/132] publichealth --- environment.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/environment.yml b/environment.yml index f6e6469..87f4d16 100644 --- a/environment.yml +++ b/environment.yml @@ -7,4 +7,8 @@ dependencies: - traitlets==5.9.0 # - -r /tmp/infra-requirements.txt # RStudio support - - otter-grader==2.2.7 + - otter-grader==2.2.7 + # for notebook exporting + - nbconvert==7.6.0 + - nb2pdf==0.6.2 + - nbpdfexport==0.2.1 From 513962c39a75b5d25f0cf7d0448a9fb956c7293c Mon Sep 17 00:00:00 2001 From: shane knapp Date: Mon, 29 Jan 2024 11:30:56 -0800 Subject: [PATCH 118/132] remove deprecated nbconvert packages --- infra-requirements.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/infra-requirements.txt b/infra-requirements.txt index e08dea3..0b14278 100644 --- a/infra-requirements.txt +++ b/infra-requirements.txt @@ -19,8 +19,8 @@ ipywidgets==8.0.7 jupyter-tree-download==1.0.1 git-credential-helpers==0.2 # Enough people like this, let's load it in. -jupyter-contrib-nbextensions==0.7.0 -jupyter_nbextensions_configurator==0.6.3 +# jupyter-contrib-nbextensions==0.7.0 +# jupyter_nbextensions_configurator==0.6.3 # Measure popularity of different packages in our hubs # https://discourse.jupyter.org/t/request-for-implementation-instrument-libraries-actively-used-by-users-on-a-jupyterhub/7994?u=yuvipanda git+https://github.com/shaneknapp/python-popularity-contest.git@add-error-handling From d4a7e7506774e9e4d2c633db97b5706f31df0b72 Mon Sep 17 00:00:00 2001 From: shane knapp Date: Mon, 29 Jan 2024 11:34:57 -0800 Subject: [PATCH 119/132] remove rather than comment out --- infra-requirements.txt | 3 --- 1 file changed, 3 deletions(-) diff --git a/infra-requirements.txt b/infra-requirements.txt index 0b14278..d029ba5 100644 --- a/infra-requirements.txt +++ b/infra-requirements.txt @@ -18,9 +18,6 @@ appmode==0.8.0 ipywidgets==8.0.7 jupyter-tree-download==1.0.1 git-credential-helpers==0.2 -# Enough people like this, let's load it in. -# jupyter-contrib-nbextensions==0.7.0 -# jupyter_nbextensions_configurator==0.6.3 # Measure popularity of different packages in our hubs # https://discourse.jupyter.org/t/request-for-implementation-instrument-libraries-actively-used-by-users-on-a-jupyterhub/7994?u=yuvipanda git+https://github.com/shaneknapp/python-popularity-contest.git@add-error-handling From 454ef5176ed23804fa8daaf3dd08363a37e75095 Mon Sep 17 00:00:00 2001 From: shane knapp Date: Tue, 6 Feb 2024 10:05:17 -0800 Subject: [PATCH 120/132] notebook 7.0.7 --- infra-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/infra-requirements.txt b/infra-requirements.txt index d029ba5..52a0479 100644 --- a/infra-requirements.txt +++ b/infra-requirements.txt @@ -8,7 +8,7 @@ # everwhere one by one. # FIXME: Freeze this to get exact versions of all dependencies -notebook==7.0.6 +notebook==7.0.7 jupyterlab==4.0.10 nbgitpuller==1.2.0 jupyter-resource-usage==1.0.1 From ca887ce57e19c0994e8c2a8891b0390f72492442 Mon Sep 17 00:00:00 2001 From: shane knapp Date: Tue, 6 Feb 2024 10:09:05 -0800 Subject: [PATCH 121/132] jupyterlab via infra-reqs to 4.0.11 --- infra-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/infra-requirements.txt b/infra-requirements.txt index 52a0479..af781f4 100644 --- a/infra-requirements.txt +++ b/infra-requirements.txt @@ -9,7 +9,7 @@ # FIXME: Freeze this to get exact versions of all dependencies notebook==7.0.7 -jupyterlab==4.0.10 +jupyterlab==4.0.11 nbgitpuller==1.2.0 jupyter-resource-usage==1.0.1 # Matches version in images/hub/Dockerfile From 2656d0c48eb0805710ffdb1ca7a494a537d78202 Mon Sep 17 00:00:00 2001 From: ryanlovett Date: Wed, 13 Mar 2024 12:44:34 -0700 Subject: [PATCH 122/132] Bump jupyter-server-proxy to 4.1.1. Upgrade, and also explicitly add as dependency where this version wouldn't normally be pulled in. Also bump jupyter-rsession-proxy where relevant. --- environment.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/environment.yml b/environment.yml index 87f4d16..a8bff42 100644 --- a/environment.yml +++ b/environment.yml @@ -1,6 +1,7 @@ dependencies: - pip - syncthing==1.18.6 +- jupyter-server-proxy==4.1.1 - jupyter-rsession-proxy==2.2.0 - pip: # bug w/notebook and traitlets: https://github.com/jupyter/notebook/issues/7048 From c371f6d178f83ed007bcf9c2648782e25fc56b3b Mon Sep 17 00:00:00 2001 From: ryanlovett Date: Wed, 13 Mar 2024 16:56:04 -0700 Subject: [PATCH 123/132] Bump to jupyter-server-proxy 4.1.2. It was released earlier today after a regression was spotted in 4.1.1. --- environment.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/environment.yml b/environment.yml index a8bff42..d875077 100644 --- a/environment.yml +++ b/environment.yml @@ -1,7 +1,7 @@ dependencies: - pip - syncthing==1.18.6 -- jupyter-server-proxy==4.1.1 +- jupyter-server-proxy==4.1.2 - jupyter-rsession-proxy==2.2.0 - pip: # bug w/notebook and traitlets: https://github.com/jupyter/notebook/issues/7048 From 444a3ca8d5f57f15ec6fed2564145a36d2309fa1 Mon Sep 17 00:00:00 2001 From: shane knapp Date: Sat, 30 Mar 2024 16:18:22 -0700 Subject: [PATCH 124/132] bump jupyterhub --- infra-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/infra-requirements.txt b/infra-requirements.txt index af781f4..62504c7 100644 --- a/infra-requirements.txt +++ b/infra-requirements.txt @@ -13,7 +13,7 @@ jupyterlab==4.0.11 nbgitpuller==1.2.0 jupyter-resource-usage==1.0.1 # Matches version in images/hub/Dockerfile -jupyterhub==4.0.2 +jupyterhub==4.1.0 appmode==0.8.0 ipywidgets==8.0.7 jupyter-tree-download==1.0.1 From ace8cbe1928e8af8ba9e93015af6fb44e26fb6d4 Mon Sep 17 00:00:00 2001 From: shane knapp Date: Mon, 1 Apr 2024 14:28:47 -0700 Subject: [PATCH 125/132] more bumping --- infra-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/infra-requirements.txt b/infra-requirements.txt index 62504c7..d4feb19 100644 --- a/infra-requirements.txt +++ b/infra-requirements.txt @@ -13,7 +13,7 @@ jupyterlab==4.0.11 nbgitpuller==1.2.0 jupyter-resource-usage==1.0.1 # Matches version in images/hub/Dockerfile -jupyterhub==4.1.0 +jupyterhub==4.1.4 appmode==0.8.0 ipywidgets==8.0.7 jupyter-tree-download==1.0.1 From 92d91bfe643e1ab39bd17098278769daa965cff3 Mon Sep 17 00:00:00 2001 From: shane knapp Date: Thu, 4 Apr 2024 22:27:05 -0700 Subject: [PATCH 126/132] debump jh 4.1.4 --- infra-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/infra-requirements.txt b/infra-requirements.txt index d4feb19..96b9029 100644 --- a/infra-requirements.txt +++ b/infra-requirements.txt @@ -13,7 +13,7 @@ jupyterlab==4.0.11 nbgitpuller==1.2.0 jupyter-resource-usage==1.0.1 # Matches version in images/hub/Dockerfile -jupyterhub==4.1.4 +jupyterhub==4.0.1 appmode==0.8.0 ipywidgets==8.0.7 jupyter-tree-download==1.0.1 From b3f380bef05e9ce6410f2cb154985b6307c5a0bd Mon Sep 17 00:00:00 2001 From: shane knapp Date: Fri, 5 Apr 2024 13:52:42 -0700 Subject: [PATCH 127/132] upgrading is fun --- infra-requirements.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/infra-requirements.txt b/infra-requirements.txt index 96b9029..b01c9d8 100644 --- a/infra-requirements.txt +++ b/infra-requirements.txt @@ -10,10 +10,10 @@ # FIXME: Freeze this to get exact versions of all dependencies notebook==7.0.7 jupyterlab==4.0.11 -nbgitpuller==1.2.0 +nbgitpuller==1.2.1 jupyter-resource-usage==1.0.1 # Matches version in images/hub/Dockerfile -jupyterhub==4.0.1 +jupyterhub==4.1.5 appmode==0.8.0 ipywidgets==8.0.7 jupyter-tree-download==1.0.1 From 6a0ff94f5ef50577c12060e4533a13b8eeb1de58 Mon Sep 17 00:00:00 2001 From: ryanlovett Date: Mon, 17 Jun 2024 00:15:09 -0700 Subject: [PATCH 128/132] Bump jupyter-server-proxy for security update. https://github.com/jupyterhub/jupyter-server-proxy/security/advisories/GHSA-fvcq-4x64-hqxr --- environment.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/environment.yml b/environment.yml index d875077..6a1ca02 100644 --- a/environment.yml +++ b/environment.yml @@ -1,7 +1,7 @@ dependencies: - pip - syncthing==1.18.6 -- jupyter-server-proxy==4.1.2 +- jupyter-server-proxy==4.2.0 - jupyter-rsession-proxy==2.2.0 - pip: # bug w/notebook and traitlets: https://github.com/jupyter/notebook/issues/7048 From f29d41e94a96d75971f7241d9127c85a1ebedeb9 Mon Sep 17 00:00:00 2001 From: shane knapp Date: Mon, 15 Jul 2024 10:46:16 -0700 Subject: [PATCH 129/132] pin jupytext on hubs that need it --- environment.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/environment.yml b/environment.yml index 6a1ca02..63f3c75 100644 --- a/environment.yml +++ b/environment.yml @@ -13,3 +13,5 @@ dependencies: - nbconvert==7.6.0 - nb2pdf==0.6.2 - nbpdfexport==0.2.1 + # pulled in by ottr, if not pinned to 1.16.2, 1.16.3 causes DH-323 + - jupytext==1.16.2 From e78cd35aa2f73d65c5a865593a0387b702df05cd Mon Sep 17 00:00:00 2001 From: Jonathan Felder Date: Tue, 16 Jul 2024 16:10:54 -0700 Subject: [PATCH 130/132] [DH-331] make webpdf export work for publichealth --- Dockerfile | 5 +++++ environment.yml | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 5a233b9..ded0c4e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -37,6 +37,7 @@ RUN apt-get update > /dev/null && \ libatk-bridge2.0-0 \ libgtk-3-0 \ libnss3 \ + libnspr4 \ libxss1 \ fonts-symbola \ gdebi-core \ @@ -62,6 +63,10 @@ RUN mamba env update -p ${CONDA_DIR} -f /tmp/environment.yml && \ COPY infra-requirements.txt /tmp/infra-requirements.txt RUN pip install --no-cache-dir -r /tmp/infra-requirements.txt +# DH-331, very similar to what was done for datahub in DH-164 +ENV PLAYWRIGHT_BROWSERS_PATH ${CONDA_DIR} +RUN playwright install chromium + # Install IRKernel RUN R --quiet -e "install.packages('IRkernel', quiet = TRUE)" && \ R --quiet -e "IRkernel::installspec(prefix='${CONDA_DIR}')" diff --git a/environment.yml b/environment.yml index 63f3c75..a6a1385 100644 --- a/environment.yml +++ b/environment.yml @@ -10,7 +10,7 @@ dependencies: # RStudio support - otter-grader==2.2.7 # for notebook exporting - - nbconvert==7.6.0 + - nbconvert[webpdf]==7.16.4 - nb2pdf==0.6.2 - nbpdfexport==0.2.1 # pulled in by ottr, if not pinned to 1.16.2, 1.16.3 causes DH-323 From 8f98472690865ec53228602423946f7473cbe76c Mon Sep 17 00:00:00 2001 From: shane knapp Date: Mon, 12 Aug 2024 10:52:51 -0700 Subject: [PATCH 131/132] updating rocker/geospatial to latest --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index ded0c4e..802e5ba 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM ghcr.io/rocker-org/geospatial:4.3.2 +FROM ghcr.io/rocker-org/geospatial:4.4.1 ENV NB_USER rstudio ENV NB_UID 1000 From 3a32c1b8051326722acde2a7e5b2289734fcdbf7 Mon Sep 17 00:00:00 2001 From: shane knapp Date: Thu, 15 Aug 2024 10:17:07 -0700 Subject: [PATCH 132/132] bumping jupyterhub --- infra-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/infra-requirements.txt b/infra-requirements.txt index b01c9d8..0fb0bd9 100644 --- a/infra-requirements.txt +++ b/infra-requirements.txt @@ -13,7 +13,7 @@ jupyterlab==4.0.11 nbgitpuller==1.2.1 jupyter-resource-usage==1.0.1 # Matches version in images/hub/Dockerfile -jupyterhub==4.1.5 +jupyterhub==4.1.6 appmode==0.8.0 ipywidgets==8.0.7 jupyter-tree-download==1.0.1