Merge pull request #662 from rstudio/sj/python

add Python to Package Manager images
rstudio · Nov 20, 2023 · b11f202 · b11f202
2 parents 641b44c + f9f3b76
commit b11f202
Show file tree

Hide file tree

Showing 15 changed files with 68 additions and 154 deletions.
diff --git a/.github/workflows/build-prerelease.yaml b/.github/workflows/build-prerelease.yaml
@@ -30,7 +30,6 @@ jobs:
           - {product: "workbench", type: "preview", os: "ubuntu2204", r-primary: "4.2.3", r-alternate: "4.1.3", py-primary: "3.9.17", py-alternate: "3.8.17"}
           - {product: "connect", type: "daily", os: "ubuntu2204", r-primary: "4.2.3", r-alternate: "4.1.3", py-primary: "3.9.17", py-alternate: "3.8.17"}
           - {product: "connect-content-init", type: "daily", os: "ubuntu2204", r-primary: "4.2.3", r-alternate: "4.1.3", py-primary: "3.9.17", py-alternate: "3.8.17"}
-          - {product: "package-manager", type: "daily", os: "ubuntu1804", r-primary: "4.2.3", r-alternate: "4.1.3", py-primary: "3.9.17", py-alternate: "3.8.17"}
           - {product: "package-manager", type: "daily", os: "ubuntu2204", r-primary: "4.2.3", r-alternate: "4.1.3", py-primary: "3.9.17", py-alternate: "3.8.17"}
           - {product: "r-session-complete", type: "daily", os: "ubuntu2204", r-primary: "4.2.3", r-alternate: "4.1.3", py-primary: "3.9.17", py-alternate: "3.8.17"}
           - {product: "r-session-complete", type: "daily", os: 'centos7', r-primary: "4.2.3", r-alternate: "4.1.3", py-primary: "3.9.14", py-alternate: "3.8.15"}

diff --git a/.github/workflows/build-release.yaml b/.github/workflows/build-release.yaml
@@ -248,8 +248,7 @@ jobs:
           - {product: 'workbench', os: 'ubuntu2204', r-primary: "4.2.3", r-alternate: "4.1.3", py-primary: "3.9.14", py-alternate: "3.8.15"}
           - {product: 'connect', os: 'ubuntu2204', r-primary: "4.2.3", r-alternate: "4.1.3", py-primary: "3.9.17", py-alternate: "3.8.17"}
           - {product: 'connect-content-init', os: 'ubuntu2204', r-primary: "4.2.3", r-alternate: "4.1.3", py-primary: "3.9.17", py-alternate: "3.8.17"}
-          - {product: 'package-manager', os: 'ubuntu1804', r-primary: "4.2.0", r-alternate: "3.6.2", py-primary: "3.9.5", py-alternate: "3.8.10"}
-          - {product: 'package-manager', os: 'ubuntu2204', r-primary: "4.2.0", r-alternate: "3.6.2", py-primary: "3.9.5", py-alternate: "3.8.10"}
+          - {product: 'package-manager', os: 'ubuntu2204', r-primary: "4.2.3", r-alternate: "4.1.3", py-primary: "3.9.17", py-alternate: "3.8.17"}
           - {product: 'r-session-complete', os: 'centos7', r-primary: "4.2.3", r-alternate: "4.1.3", py-primary: "3.9.14", py-alternate: "3.8.15"}
           - {product: 'r-session-complete', os: 'ubuntu2204', r-primary: "4.2.3", r-alternate: "4.1.3", py-primary: "3.9.14", py-alternate: "3.8.15"}
           - {product: 'workbench-for-microsoft-azure-ml', os: 'ubuntu2204', r-primary: "4.2.3", r-alternate: "4.1.3", py-primary: "3.9.14", py-alternate: "3.8.15"}

diff --git a/.github/workflows/lint.yaml b/.github/workflows/lint.yaml
@@ -23,7 +23,6 @@ jobs:
           - {product: 'workbench', os: 'ubuntu2204'}
           - {product: 'connect', os: 'ubuntu2204'}
           - {product: 'connect-content-init', os: 'ubuntu2204'}
-          - {product: 'package-manager', os: 'ubuntu1804'}
           - {product: 'package-manager', os: 'ubuntu2204'}
           - {product: 'r-session-complete', os: 'centos7'}
           - {product: 'r-session-complete', os: 'ubuntu2204'}

diff --git a/ci.Justfile b/ci.Justfile
@@ -134,6 +134,12 @@ get-product-args $PRODUCT $OS $VERSION $USE_S3="false" $BRANCH=`git branch --sho
     else
       SRC_IMAGE_NAME="product-base-pro-dev"
     fi
+  elif [[ $PRODUCT == "package-manager" ]]; then
+    if [[ $BRANCH == "main" ]]; then
+      SRC_IMAGE_NAME="product-base"
+    else
+      SRC_IMAGE_NAME="product-base-dev"
+    fi
   fi
 
   if [[ "${OS}" == "centos7" ]]; then
@@ -217,6 +223,12 @@ get-prerelease-args $TYPE $PRODUCT $OS $VERSION $BRANCH=`git branch --show`:
     else
       SRC_IMAGE_NAME="product-base-pro-dev"
     fi
+  elif [[ $PRODUCT == "package-manager" ]]; then
+    if [[ $BRANCH == "main" ]]; then
+      SRC_IMAGE_NAME="product-base"
+    else
+      SRC_IMAGE_NAME="product-base-dev"
+    fi
   fi
 
   if [[ "${OS}" == "centos7" ]]; then

diff --git a/docker-compose.yml b/docker-compose.yml
@@ -41,7 +41,7 @@ services:
     container_name: rstudio-package-manager
     build:
       context: ./package-manager
-      dockerfile: "Dockerfile.${IMAGE_OS:-ubuntu1804}"
+      dockerfile: "Dockerfile.${IMAGE_OS:-ubuntu2204}"
       args:
         RSPM_VERSION: 2023.08.4-20
     image: rstudio/rstudio-package-manager:2023.08.4

diff --git a/hadolint.yaml b/hadolint.yaml
@@ -4,6 +4,8 @@ ignored:
   - DL3008
     # Suppresses warning for version pinning on yum installs. We currently do not practice this approach.
   - DL3033
+  # Suppresses warning for version pinning on pip installs. We currently do not practice this approach.
+  - DL3013
     # Suppresses info for too many consecutive RUN statements. Multiple consecutive RUN statements are often a conscious
     # design choice in our Dockerfiles, but we could consider condensing them in the future.
   - DL3059

diff --git a/matrix-preview.json b/matrix-preview.json
@@ -3,7 +3,7 @@
   {"product": "rstudio-workbench-preview", "dir": "workbench", "prefix": "RSW", "os": "bionic", "arch": "amd64", "version": "preview"},
   {"product": "rstudio-connect-preview", "dir": "connect", "prefix": "RSC", "os": "bionic", "arch": "amd64", "version": "daily"},
   {"product": "rstudio-connect-content-init-preview", "dir": "connect-content-init", "prefix": "RSC", "os": "bionic", "arch": "amd64", "version": "daily"},
-  {"product": "rstudio-package-manager-preview", "dir": "package-manager", "prefix": "RSPM", "os": "bionic", "arch": "amd64", "version": "daily"},
+  {"product": "rstudio-package-manager-preview", "dir": "package-manager", "prefix": "RSPM", "os": "jammy", "arch": "amd64", "version": "daily"},
   {"product": "r-session-complete-preview", "tag_prefix": "bionic-", "dir": "r-session-complete", "os": "bionic", "arch": "amd64", "prefix": "RSW", "version": "daily"},
   {"product": "r-session-complete-preview", "tag_prefix": "centos7-", "dir": "r-session-complete", "os": "centos7", "arch": "x86_64", "prefix": "RSW", "version": "daily"},
   {"product": "r-session-complete-preview", "tag_prefix": "bionic-", "dir": "r-session-complete", "os": "bionic", "arch": "amd64", "prefix": "RSW", "version": "preview"},

diff --git a/package-manager/.env b/package-manager/.env
@@ -1,3 +1,5 @@
 RSPM_VERSION=2023.08.4-20
 R_VERSION=4.2.3
 R_VERSION_ALT=4.1.3
+PYTHON_VERSION=3.9.17
+PYTHON_VERSION_ALT=3.8.17
diff --git a/package-manager/Dockerfile.ubuntu1804 b/package-manager/Dockerfile.ubuntu1804
diff --git a/package-manager/Dockerfile.ubuntu2204 b/package-manager/Dockerfile.ubuntu2204
@@ -1,70 +1,37 @@
 ARG R_VERSION=4.2.3
-FROM rstudio/r-base:${R_VERSION}-jammy
+ARG R_VERSION_ALT=4.1.3
+ARG PYTHON_VERSION=3.9.17
+ARG PYTHON_VERSION_ALT=3.8.17
+ARG SRC_IMAGE_NAME=product-base
+ARG REGISTRY=ghcr.io
+FROM ${REGISTRY}/rstudio/${SRC_IMAGE_NAME}:ubuntu2204-r${R_VERSION}_${R_VERSION_ALT}-py${PYTHON_VERSION}_${PYTHON_VERSION_ALT}
 LABEL maintainer="RStudio Docker <[email protected]>"
 
-ARG R_VERSION=4.2.3
+ARG PYTHON_VERSION=3.9.17
+ARG PYTHON_VERSION_ALT=3.8.17
 
 # Locale configuration --------------------------------------------------------#
-RUN localedef -i en_US -f UTF-8 en_US.UTF-8
-ENV LANG en_US.UTF-8
-ENV LANGUAGE en_US:en
-ENV LC_ALL en_US.UTF-8
 ENV STARTUP_DEBUG_MODE 0
-ARG DEBIAN_FRONTEND=noninteractive
-ARG TINI_VERSION=0.19.0
 
 ENV PATH /opt/rstudio-pm/bin:$PATH
 
-# Upgrade/install packages ----------------------------------------------------#
-RUN apt-get update --fix-missing \
-    && apt-get upgrade -yq \
-    && apt-get install -yq --no-install-recommends \
-      ca-certificates \
-      dirmngr \
-      gpg \
-      gpg-agent \
-    && rm -rf /var/lib/apt/lists/*
-
-# Runtime settings ------------------------------------------------------------#
-ADD https://cdn.rstudio.com/platform/tini/v${TINI_VERSION}/tini-amd64 /tini
-ADD https://cdn.rstudio.com/platform/tini/v${TINI_VERSION}/tini-amd64.asc /tini.asc
-RUN gpg --update-trustdb \
-    && gpg --batch --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys 595E85A6B1B4779EA4DAAEC70B588DFF0527A9B7 \
-    && gpg --batch --verify /tini.asc /tini \
-    && chmod +x /tini \
-    && ln -s /tini /usr/local/bin/tini
-
-# Add another R version -------------------------------------------------------#
-ARG R_VERSION_ALT=4.1.3
-RUN apt-get update -qq && \
-    curl -O https://cdn.rstudio.com/r/ubuntu-2204/pkgs/r-${R_VERSION_ALT}_1_amd64.deb && \
-    DEBIAN_FRONTEND=noninteractive apt-get install -y ./r-${R_VERSION_ALT}_1_amd64.deb && \
-    rm -f ./r-${R_VERSION_ALT}_1_amd64.deb \
-    && apt-get clean \
-    && rm -rf /var/lib/apt/lists/*
+# Required Python packages ----------------------------------------------------#
+RUN /opt/python/${PYTHON_VERSION}/bin/python3 -m pip install --no-cache-dir build virtualenv
+RUN /opt/python/${PYTHON_VERSION_ALT}/bin/python3 -m pip install --no-cache-dir build virtualenv
 
 # Download RStudio Package Manager ---------------------------------------------#
 ARG RSPM_VERSION=2023.08.4-20
 ARG RSPM_DOWNLOAD_URL=https://cdn.rstudio.com/package-manager/deb/amd64
-RUN apt-get update --fix-missing \
-    && apt-get install -y --no-install-recommends ca-certificates gdebi-core gpg dpkg-sig \
-    && curl -O ${RSPM_DOWNLOAD_URL}/rstudio-pm_${RSPM_VERSION}_amd64.deb \
+RUN curl -O ${RSPM_DOWNLOAD_URL}/rstudio-pm_${RSPM_VERSION}_amd64.deb \
     # Pre 7/25/23 packages
     && gpg --keyserver keyserver.ubuntu.com --recv-keys 3F32EE77E331692F \
     # Post 7/25 packages
     && gpg --keyserver keys.openpgp.org --recv-keys 51C0B5BB19F92D60 \
     && dpkg-sig --verify rstudio-pm_${RSPM_VERSION}_amd64.deb \
-    && RSTUDIO_INSTALL_NO_LICENSE_INITIALIZATION=1 gdebi -n rstudio-pm_${RSPM_VERSION}_amd64.deb \
+    && RSTUDIO_INSTALL_NO_LICENSE_INITIALIZATION=1 apt-get install -y ./rstudio-pm_${RSPM_VERSION}_amd64.deb \
     && rm rstudio-pm_${RSPM_VERSION}_amd64.deb \
-    && apt-get purge -y gdebi-core dpkg-sig gpg gpg-agent \
-    && apt-get autoremove -y \
-    && apt-get clean \
-    && rm -rf /var/lib/apt/lists/* \
     && (ln -s /opt/rstudio-pm/bin/rspm /usr/local/bin/rspm || echo "/usr/local/bin/rspm symlink already exists")
 
-# Set default R version symlink
-RUN ln -s /opt/R/${R_VERSION} /opt/R/default
-
 # Add run script and set permissions -------------------------------------------#
 COPY --chmod=0775 startup.sh /usr/local/bin/startup.sh
 RUN mkdir -p /var/run/rstudio-pm \

diff --git a/package-manager/Justfile b/package-manager/Justfile
@@ -48,6 +48,8 @@ build OS=IMAGE_OS VERSION=RSPM_VERSION *TAGS="":
     --build-arg RSPM_VERSION="{{ VERSION }}" \
     --build-arg R_VERSION="{{ R_VERSION }}" \
     --build-arg R_VERSION_ALT="{{ R_VERSION_ALT }}" \
+    --build-arg PYTHON_VERSION="{{ PYTHON_VERSION }}" \
+    --build-arg PYTHON_VERSION_ALT="{{ PYTHON_VERSION_ALT }}" \
     --file=./Dockerfile.$(just -f ../Justfile _parse-os {{OS}}) .
 
 # Test Package Manager image - just test rstudio/rstudio-package-manager:ubuntu2204-2022.07.2-11 2022.07.2-11
@@ -60,6 +62,8 @@ test TAG=`just _make-default-tag` VERSION=RSPM_VERSION CMD="":
   RSPM_LICENSE_SERVER="{{ RSPM_LICENSE_SERVER }}" \
   R_VERSION="{{ R_VERSION }}" \
   R_VERSION_ALT="{{ R_VERSION_ALT }}" \
+  PYTHON_VERSION="{{ PYTHON_VERSION }}" \
+  PYTHON_VERSION_ALT="{{ PYTHON_VERSION_ALT }}" \
   docker-compose -f ./docker-compose.test.yml run sut {{ CMD }}
 
 # Test Package Manager image interactively - just test-i rstudio/rstudio-package-manager:ubuntu2204-2022.07.2-11 2022.07.2-11

diff --git a/package-manager/NEWS.md b/package-manager/NEWS.md
@@ -1,3 +1,8 @@
+# 2023-11-20
+- Add Python
+- Update R to 4.2.3
+- Deprecate Ubuntu 18.04
+
 # 2023-11-15
 - Removed Launcher references. Package manager no longer uses the Job Launcher for building git packages.
 

diff --git a/package-manager/README.md b/package-manager/README.md
@@ -6,7 +6,6 @@
 
 # Supported tags and respective Dockerfile links
 
-* [`2023.08.4`, `bionic`, `ubuntu1804`, `bionic-2023.08.4`, `ubuntu1804-2023.08.4`](https://github.com/rstudio/rstudio-docker-products/blob/main/package-manager/Dockerfile.ubuntu1804)
 * [`jammy`, `ubuntu2204`, `jammy-2023.08.4`, `ubuntu2204-2023.08.4`](https://github.com/rstudio/rstudio-docker-products/blob/main/package-manager/Dockerfile.ubuntu2204)
 
 # What is RStudio Package Manager?
@@ -50,7 +49,7 @@ export RSPM_LICENSE=XXXX-XXXX-XXXX-XXXX-XXXX-XXXX-XXXX
 docker run -it \
     -p 4242:4242 \
     -e RSPM_LICENSE=$RSPM_LICENSE \
-    rstudio/rstudio-package-manager:ubuntu1804
+    rstudio/rstudio-package-manager:ubuntu2204
 ```
 Open [http://localhost:4242](http://localhost:4242) to access RStudio Package Manager UI.
 
@@ -60,8 +59,9 @@ Note that running the RStudio Package Manager Docker image requires a valid RStu
 
 This container includes:
 
-1. One version of R
-2. RStudio Package Manager
+1. Two versions of R
+2. Two versions of Python
+3. RStudio Package Manager
 
 > NOTE: Package Manager is currently not very particular about R version. Changing the R version is rarely necessary.
 
@@ -120,15 +120,15 @@ docker run -it \
     -p 4242:4242 \
     -v $PWD/package-manager/rstudio-pm.gcfg:/etc/rstudio-pm/rstudio-pm.gcfg \
     -e RSPM_LICENSE=$RSPM_LICENSE \
-    rstudio/rstudio-package-manager:ubuntu1804
+    rstudio/rstudio-package-manager:ubuntu2204
 
 # Run with persistent data and using an external configuration
 docker run -it \
     -p 4242:4242 \
     -v $PWD/data/rspm:/data \
     -v $PWD/package-manager/rstudio-pm.gcfg:/etc/rstudio-pm/rstudio-pm.gcfg \
     -e RSPM_LICENSE=$RSPM_LICENSE \
-    rstudio/rstudio-package-manager:ubuntu1804
+    rstudio/rstudio-package-manager:ubuntu2204
 ```
 
 Open [http://localhost:4242](http://localhost:4242) to access RStudio Package Manager UI.

diff --git a/package-manager/docker-compose.test.yml b/package-manager/docker-compose.test.yml
@@ -10,6 +10,8 @@ services:
       - RSPM_VERSION
       - R_VERSION
       - R_VERSION_ALT
+      - PYTHON_VERSION
+      - PYTHON_VERSION_ALT
       - RSPM_LICENSE
       - RSPM_LICENSE_SERVER
     volumes:

diff --git a/package-manager/test/goss.yaml b/package-manager/test/goss.yaml
@@ -32,6 +32,10 @@ file:
     exists: true
     contains:
     - "!Unable to initialize the configuration"
+  /opt/python/{{.Env.PYTHON_VERSION}}/bin/python:
+    exists: true
+  /opt/python/{{.Env.PYTHON_VERSION_ALT}}/bin/python:
+    exists: true
 # TODO: Ensure that this does _not_ exist in the future
 #  /var/log/rstudio-pm.log:
 #    exists: true
@@ -49,18 +53,31 @@ command:
   "pgrep rstudio-pm":
     title: rspm_running
     exit-status: 0
-
+  
 # Ensure correct R version
   "/opt/R/{{ .Env.R_VERSION }}/bin/R --version":
     title: r_version_match
     exit-status: 0
     stdout: [
       "{{ .Env.R_VERSION }}"
     ]
-
   "/opt/R/{{ .Env.R_VERSION_ALT }}/bin/R --version":
     title: r_version_alt_match
     exit-status: 0
     stdout: [
       "{{ .Env.R_VERSION_ALT }}"
     ]
+
+  # Ensure correct python version
+  "/opt/python/{{ .Env.PYTHON_VERSION }}/bin/python --version":
+    title: python_version_matches
+    exit-status: 0
+    stdout: [
+      "{{ .Env.PYTHON_VERSION }}"
+    ]
+  "/opt/python/{{ .Env.PYTHON_VERSION_ALT }}/bin/python --version":
+    title: python_version_alt_matches
+    exit-status: 0
+    stdout: [
+      "{{ .Env.PYTHON_VERSION_ALT }}"
+    ]