From 3734ea1d2daf77e605283fed462f828dcea419af Mon Sep 17 00:00:00 2001 From: YuviPanda Date: Fri, 6 Mar 2020 14:32:07 +0530 Subject: [PATCH 01/12] Add miniforge3 simple base image A slim miniforge3 image, equivalent to miniconda3 but with some slightly different choices. - Use ubuntu LTS as base instead of debian - Keep $HOME clean, so folks can mount volumes into it Ref https://github.com/conda-forge/miniforge/issues/20 --- miniforge3/ubuntu/Dockerfile | 43 ++++++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) create mode 100644 miniforge3/ubuntu/Dockerfile diff --git a/miniforge3/ubuntu/Dockerfile b/miniforge3/ubuntu/Dockerfile new file mode 100644 index 00000000..1591968c --- /dev/null +++ b/miniforge3/ubuntu/Dockerfile @@ -0,0 +1,43 @@ +FROM ubuntu:bionic + +ARG MINIFORGE_VERSION=4.8.2-1 +ARG TINI_VERSION=v0.18.0 + +ENV CONDA_DIR=/opt/conda +ENV LANG=C.UTF-8 LC_ALL=C.UTF-8 +ENV PATH=${CONDA_DIR}/bin:${PATH} + +# Install just enough for conda to work +RUN apt-get update > /dev/null && \ + apt-get install --no-install-recommends --yes \ + wget bzip2 ca-certificates \ + git > /dev/null && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* + +RUN wget --quiet https://github.com/krallin/tini/releases/download/${TINI_VERSION}/tini -O /usr/local/bin/tini && \ + chmod +x /usr/local/bin/tini + +# 1. Install miniforge from GitHub releases +# 2. Apply some cleanup tips from https://jcrist.github.io/conda-docker-tips.html +# Particularly, we remove pyc and a files. The default install has no js, we can skip that +RUN wget --quiet https://github.com/conda-forge/miniforge/releases/download/${MINIFORGE_VERSION}/Miniforge3-${MINIFORGE_VERSION}-Linux-x86_64.sh -O /tmp/miniforge.sh && \ + /bin/bash /tmp/miniforge.sh -b -p ${CONDA_DIR} && \ + rm /tmp/miniforge.sh && \ + conda clean -tipsy && \ + find ${CONDA_DIR} -follow -type f -name '*.a' -delete && \ + find ${CONDA_DIR} -follow -type f -name '*.pyc' -delete && \ + conda clean -afy + +# Set up default conda environment systemwide for all interactive shells. +# This allows users to mount volumes in $HOME and still have fully working conda +# FIXME: In bash, we have everything setup right *except* for PS1, which doesn't show (base) +# FIXME: In sh, we don't seem to have conda set up properly +RUN ln -s ${CONDA_DIR}/etc/profile.d/conda.sh /etc/profile.d/conda.sh && \ + echo "source ${CONDA_DIR}/etc/profile.d/conda.sh" >> /etc/bash.bashrc && \ + echo "conda activate base" >> /etc/bash.bashrc + +ENTRYPOINT ["tini", "--"] + +# Without the "-l", conda activate won't work out of the box, since profiles aren't loaded +CMD [ "/bin/bash", "-l" ] \ No newline at end of file From 36e2a35c26990cc75bb7f26a8792182848ca4684 Mon Sep 17 00:00:00 2001 From: YuviPanda Date: Fri, 6 Mar 2020 14:52:50 +0530 Subject: [PATCH 02/12] Add miniforge3 buid test step to CI --- .circleci/config.yml | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index efa5aa20..1f376861 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,7 +1,16 @@ version: 2 jobs: - build: + build-miniforge3: + steps: + - checkout + - setup_remote_docker + - run: + name: Build miniforge image + command: | + cd minforge3/ubuntu + docker build -t condaforge/miniforge3 . + build-anvil: environment: DOCKERIMAGE: linux-anvil-comp7 working_directory: ~/test @@ -25,4 +34,5 @@ workflows: version: 2 build_and_test: jobs: - - build + - build-anvil + - build-miniforge3 From 9df32eaeccaed5e46249f6f38a3915d76e40fec7 Mon Sep 17 00:00:00 2001 From: YuviPanda Date: Fri, 6 Mar 2020 14:56:21 +0530 Subject: [PATCH 03/12] Specify which image CircleCI should use We just need the docker commandline tool, but using python3 since we might want py.test later --- .circleci/config.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index 1f376861..590fc8f4 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -2,6 +2,8 @@ version: 2 jobs: build-miniforge3: + docker: + - image: circleci/python:3.8.2 steps: - checkout - setup_remote_docker From 898b4305f7f9639a8a2baba4f16d9a786df98399 Mon Sep 17 00:00:00 2001 From: YuviPanda Date: Fri, 6 Mar 2020 14:58:00 +0530 Subject: [PATCH 04/12] Fix typo --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 590fc8f4..dea74f82 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -10,7 +10,7 @@ jobs: - run: name: Build miniforge image command: | - cd minforge3/ubuntu + cd miniforge3/ubuntu docker build -t condaforge/miniforge3 . build-anvil: environment: From 91da7a78f444c754851386236494281301cc4a36 Mon Sep 17 00:00:00 2001 From: YuviPanda Date: Fri, 6 Mar 2020 15:19:36 +0530 Subject: [PATCH 05/12] Activate base by default in bash shells - Put activate commands in user's .bashrc, since this is where PS1 is set. If we put the activate commands in /etc/bash.bashrc, the environments are set up properly but PS1 is not. This is very confusing. - Put activate commands in /etc/skel as well, so any new non-root users created also get base activated by default - Stop using 'login' shells, as they are no longer needed. --- miniforge3/ubuntu/Dockerfile | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/miniforge3/ubuntu/Dockerfile b/miniforge3/ubuntu/Dockerfile index 1591968c..0498d8d3 100644 --- a/miniforge3/ubuntu/Dockerfile +++ b/miniforge3/ubuntu/Dockerfile @@ -29,15 +29,15 @@ RUN wget --quiet https://github.com/conda-forge/miniforge/releases/download/${MI find ${CONDA_DIR} -follow -type f -name '*.pyc' -delete && \ conda clean -afy -# Set up default conda environment systemwide for all interactive shells. -# This allows users to mount volumes in $HOME and still have fully working conda -# FIXME: In bash, we have everything setup right *except* for PS1, which doesn't show (base) -# FIXME: In sh, we don't seem to have conda set up properly -RUN ln -s ${CONDA_DIR}/etc/profile.d/conda.sh /etc/profile.d/conda.sh && \ - echo "source ${CONDA_DIR}/etc/profile.d/conda.sh" >> /etc/bash.bashrc && \ - echo "conda activate base" >> /etc/bash.bashrc +# Activate base by default when running as any *non-root* user as well +# Good security practice requires running most workloads as non-root +# This makes sure any non-root users created also have base activated +# for their interactive shells. +RUN echo ". ${CONDA_DIR}/etc/profile.d/conda.sh && conda activate base" >> /etc/skel/.bashrc -ENTRYPOINT ["tini", "--"] +# Activate base by default when running as root as well +# The root user is already created, so won't pick up changes to /etc/skel +RUN echo ". ${CONDA_DIR}/etc/profile.d/conda.sh && conda activate base" >> ~/.bashrc -# Without the "-l", conda activate won't work out of the box, since profiles aren't loaded -CMD [ "/bin/bash", "-l" ] \ No newline at end of file +ENTRYPOINT ["tini", "--"] +CMD [ "/bin/bash" ] \ No newline at end of file From c0aa9daa6af424231d9f449f7ef579435c300abf Mon Sep 17 00:00:00 2001 From: YuviPanda Date: Sun, 8 Mar 2020 12:38:17 +0530 Subject: [PATCH 06/12] Don't put .wget-hsts files in $HOME --- miniforge3/ubuntu/Dockerfile | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/miniforge3/ubuntu/Dockerfile b/miniforge3/ubuntu/Dockerfile index 0498d8d3..c77cee78 100644 --- a/miniforge3/ubuntu/Dockerfile +++ b/miniforge3/ubuntu/Dockerfile @@ -15,13 +15,14 @@ RUN apt-get update > /dev/null && \ apt-get clean && \ rm -rf /var/lib/apt/lists/* -RUN wget --quiet https://github.com/krallin/tini/releases/download/${TINI_VERSION}/tini -O /usr/local/bin/tini && \ +# Keep $HOME clean (no .wget-hsts file), since HSTS isn't useful in this context +RUN wget --no-hsts --quiet https://github.com/krallin/tini/releases/download/${TINI_VERSION}/tini -O /usr/local/bin/tini && \ chmod +x /usr/local/bin/tini # 1. Install miniforge from GitHub releases # 2. Apply some cleanup tips from https://jcrist.github.io/conda-docker-tips.html # Particularly, we remove pyc and a files. The default install has no js, we can skip that -RUN wget --quiet https://github.com/conda-forge/miniforge/releases/download/${MINIFORGE_VERSION}/Miniforge3-${MINIFORGE_VERSION}-Linux-x86_64.sh -O /tmp/miniforge.sh && \ +RUN wget --no-hsts --quiet https://github.com/conda-forge/miniforge/releases/download/${MINIFORGE_VERSION}/Miniforge3-${MINIFORGE_VERSION}-Linux-x86_64.sh -O /tmp/miniforge.sh && \ /bin/bash /tmp/miniforge.sh -b -p ${CONDA_DIR} && \ rm /tmp/miniforge.sh && \ conda clean -tipsy && \ From 6b38a80a742500573eb3b4ed82862c7b59d6f780 Mon Sep 17 00:00:00 2001 From: YuviPanda Date: Sun, 8 Mar 2020 12:40:06 +0530 Subject: [PATCH 07/12] Add minimal tests for miniforge3 --- .circleci/config.yml | 9 ++++++--- miniforge3/tests/Dockerfile | 8 ++++++++ miniforge3/tests/tests/clean-homedir.py | 5 +++++ miniforge3/tests/tests/ps1.py | 9 +++++++++ 4 files changed, 28 insertions(+), 3 deletions(-) create mode 100644 miniforge3/tests/Dockerfile create mode 100755 miniforge3/tests/tests/clean-homedir.py create mode 100755 miniforge3/tests/tests/ps1.py diff --git a/.circleci/config.yml b/.circleci/config.yml index dea74f82..3068728c 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -8,10 +8,13 @@ jobs: - checkout - setup_remote_docker - run: - name: Build miniforge image + name: Build miniforge3 image command: | - cd miniforge3/ubuntu - docker build -t condaforge/miniforge3 . + docker build -t condaforge/miniforge3 miniforge3/ubuntu + - run: + name: Run miniforge3 tests + command: | + docker build miniforge3/tests --arg IMAGE_TO_TEST=condaforge/miniforge3 build-anvil: environment: DOCKERIMAGE: linux-anvil-comp7 diff --git a/miniforge3/tests/Dockerfile b/miniforge3/tests/Dockerfile new file mode 100644 index 00000000..bd54b5fe --- /dev/null +++ b/miniforge3/tests/Dockerfile @@ -0,0 +1,8 @@ +ARG IMAGE_TO_TEST=miniforge3 +FROM ${IMAGE_TO_TEST} + +COPY tests /tmp/tests +RUN for f in /tmp/tests/*; do \ + echo "Executing $f"; \ + $f; \ + done \ No newline at end of file diff --git a/miniforge3/tests/tests/clean-homedir.py b/miniforge3/tests/tests/clean-homedir.py new file mode 100755 index 00000000..54d1952b --- /dev/null +++ b/miniforge3/tests/tests/clean-homedir.py @@ -0,0 +1,5 @@ +#!/usr/bin/env python3 +import os + +# Only .bashrc and .profile should be in $HOME +assert sorted(os.listdir(os.environ['HOME'])) == ['.bashrc', '.profile'] \ No newline at end of file diff --git a/miniforge3/tests/tests/ps1.py b/miniforge3/tests/tests/ps1.py new file mode 100755 index 00000000..dde2f026 --- /dev/null +++ b/miniforge3/tests/tests/ps1.py @@ -0,0 +1,9 @@ +#!/usr/bin/env python3 +import subprocess + +# Interactive mode in bash should have (base) in PS1 +# -i requires stdin to be a tty, but we don't care. stderr is silenced to keep output clean. +assert subprocess.check_output([ + '/bin/bash', '-i', '-c', + 'echo $PS1' +], stderr=subprocess.DEVNULL).decode().startswith("(base)") \ No newline at end of file From bc9ea94f57802b11e3e371d3c800b4918142a199 Mon Sep 17 00:00:00 2001 From: YuviPanda Date: Sun, 8 Mar 2020 13:19:54 +0530 Subject: [PATCH 08/12] Fix typo --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 3068728c..f936350d 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -14,7 +14,7 @@ jobs: - run: name: Run miniforge3 tests command: | - docker build miniforge3/tests --arg IMAGE_TO_TEST=condaforge/miniforge3 + docker build miniforge3/tests --build-arg IMAGE_TO_TEST=condaforge/miniforge3 build-anvil: environment: DOCKERIMAGE: linux-anvil-comp7 From 25716b0751ef3921bc8dc8d05b039a4b49bc7c1b Mon Sep 17 00:00:00 2001 From: YuviPanda Date: Sun, 8 Mar 2020 13:33:49 +0530 Subject: [PATCH 09/12] Add tests as non-root as well --- .circleci/config.yml | 12 ++++++++++-- miniforge3/tests/Dockerfile.non-root | 17 +++++++++++++++++ .../tests/{Dockerfile => Dockerfile.root} | 0 miniforge3/tests/tests/ps1.py | 1 + 4 files changed, 28 insertions(+), 2 deletions(-) create mode 100644 miniforge3/tests/Dockerfile.non-root rename miniforge3/tests/{Dockerfile => Dockerfile.root} (100%) diff --git a/.circleci/config.yml b/.circleci/config.yml index f936350d..4905a6dc 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -12,9 +12,17 @@ jobs: command: | docker build -t condaforge/miniforge3 miniforge3/ubuntu - run: - name: Run miniforge3 tests + name: Run miniforge3 tests as root command: | - docker build miniforge3/tests --build-arg IMAGE_TO_TEST=condaforge/miniforge3 + docker build miniforge3/tests \ + --build-arg IMAGE_TO_TEST=condaforge/miniforge3 \ + -f miniforge3/tests/Dockerfile.root + - run: + name: Run miniforge3 tests as non-root + command: | + docker build miniforge3/tests \ + --build-arg IMAGE_TO_TEST=condaforge/miniforge3 \ + -f miniforge3/tests/Dockerfile.non-root build-anvil: environment: DOCKERIMAGE: linux-anvil-comp7 diff --git a/miniforge3/tests/Dockerfile.non-root b/miniforge3/tests/Dockerfile.non-root new file mode 100644 index 00000000..44356918 --- /dev/null +++ b/miniforge3/tests/Dockerfile.non-root @@ -0,0 +1,17 @@ +ARG IMAGE_TO_TEST=miniforge3 +FROM ${IMAGE_TO_TEST} + +RUN useradd \ + --comment "Default user" \ + --create-home \ + --shell /bin/bash \ + --uid 1000 \ + conder + +USER conder + +COPY tests /tmp/tests +RUN for f in /tmp/tests/*; do \ + echo "Executing $f"; \ + $f; \ + done \ No newline at end of file diff --git a/miniforge3/tests/Dockerfile b/miniforge3/tests/Dockerfile.root similarity index 100% rename from miniforge3/tests/Dockerfile rename to miniforge3/tests/Dockerfile.root diff --git a/miniforge3/tests/tests/ps1.py b/miniforge3/tests/tests/ps1.py index dde2f026..3ed410b5 100755 --- a/miniforge3/tests/tests/ps1.py +++ b/miniforge3/tests/tests/ps1.py @@ -1,5 +1,6 @@ #!/usr/bin/env python3 import subprocess +import os # Interactive mode in bash should have (base) in PS1 # -i requires stdin to be a tty, but we don't care. stderr is silenced to keep output clean. From eb3e8a7c3e0e3663b933c60eb1e88f9dc70fcdda Mon Sep 17 00:00:00 2001 From: YuviPanda Date: Sun, 8 Mar 2020 13:40:47 +0530 Subject: [PATCH 10/12] Actually fail the build when test fails --- miniforge3/tests/Dockerfile.non-root | 3 ++- miniforge3/tests/Dockerfile.root | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/miniforge3/tests/Dockerfile.non-root b/miniforge3/tests/Dockerfile.non-root index 44356918..2a903803 100644 --- a/miniforge3/tests/Dockerfile.non-root +++ b/miniforge3/tests/Dockerfile.non-root @@ -11,7 +11,8 @@ RUN useradd \ USER conder COPY tests /tmp/tests -RUN for f in /tmp/tests/*; do \ +RUN set -e; \ + for f in /tmp/tests/*; do \ echo "Executing $f"; \ $f; \ done \ No newline at end of file diff --git a/miniforge3/tests/Dockerfile.root b/miniforge3/tests/Dockerfile.root index bd54b5fe..b0abfd83 100644 --- a/miniforge3/tests/Dockerfile.root +++ b/miniforge3/tests/Dockerfile.root @@ -2,7 +2,8 @@ ARG IMAGE_TO_TEST=miniforge3 FROM ${IMAGE_TO_TEST} COPY tests /tmp/tests -RUN for f in /tmp/tests/*; do \ +RUN set -e; \ + for f in /tmp/tests/*; do \ echo "Executing $f"; \ $f; \ done \ No newline at end of file From 46bc6dd7558cc60a1e4ac21fed87cf701638dee5 Mon Sep 17 00:00:00 2001 From: YuviPanda Date: Sun, 8 Mar 2020 13:40:58 +0530 Subject: [PATCH 11/12] Fix clean-home test for non-root users --- miniforge3/tests/tests/clean-homedir.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/miniforge3/tests/tests/clean-homedir.py b/miniforge3/tests/tests/clean-homedir.py index 54d1952b..3a0e4be9 100755 --- a/miniforge3/tests/tests/clean-homedir.py +++ b/miniforge3/tests/tests/clean-homedir.py @@ -1,5 +1,9 @@ #!/usr/bin/env python3 import os -# Only .bashrc and .profile should be in $HOME -assert sorted(os.listdir(os.environ['HOME'])) == ['.bashrc', '.profile'] \ No newline at end of file +# .bash_logout could optionally be in $HOME, otherwise only .bashrc and .profile +home_contents = sorted(os.listdir(os.environ['HOME'])) +if len(home_contents) == 3: + assert home_contents == ['.bash_logout', '.bashrc', '.profile'], home_contents +else: + assert home_contents == ['.bashrc', '.profile'], home_contents \ No newline at end of file From a0216d1fa820fffbf920f462145b80398b9ebf41 Mon Sep 17 00:00:00 2001 From: YuviPanda Date: Fri, 13 Mar 2020 12:35:32 +0530 Subject: [PATCH 12/12] Add newlines to end of all files --- miniforge3/tests/Dockerfile.non-root | 2 +- miniforge3/tests/Dockerfile.root | 2 +- miniforge3/tests/tests/clean-homedir.py | 2 +- miniforge3/tests/tests/ps1.py | 2 +- miniforge3/ubuntu/Dockerfile | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/miniforge3/tests/Dockerfile.non-root b/miniforge3/tests/Dockerfile.non-root index 2a903803..7b66c1d5 100644 --- a/miniforge3/tests/Dockerfile.non-root +++ b/miniforge3/tests/Dockerfile.non-root @@ -15,4 +15,4 @@ RUN set -e; \ for f in /tmp/tests/*; do \ echo "Executing $f"; \ $f; \ - done \ No newline at end of file + done diff --git a/miniforge3/tests/Dockerfile.root b/miniforge3/tests/Dockerfile.root index b0abfd83..0784506d 100644 --- a/miniforge3/tests/Dockerfile.root +++ b/miniforge3/tests/Dockerfile.root @@ -6,4 +6,4 @@ RUN set -e; \ for f in /tmp/tests/*; do \ echo "Executing $f"; \ $f; \ - done \ No newline at end of file + done diff --git a/miniforge3/tests/tests/clean-homedir.py b/miniforge3/tests/tests/clean-homedir.py index 3a0e4be9..2a616f56 100755 --- a/miniforge3/tests/tests/clean-homedir.py +++ b/miniforge3/tests/tests/clean-homedir.py @@ -6,4 +6,4 @@ if len(home_contents) == 3: assert home_contents == ['.bash_logout', '.bashrc', '.profile'], home_contents else: - assert home_contents == ['.bashrc', '.profile'], home_contents \ No newline at end of file + assert home_contents == ['.bashrc', '.profile'], home_contents diff --git a/miniforge3/tests/tests/ps1.py b/miniforge3/tests/tests/ps1.py index 3ed410b5..e96ecc26 100755 --- a/miniforge3/tests/tests/ps1.py +++ b/miniforge3/tests/tests/ps1.py @@ -7,4 +7,4 @@ assert subprocess.check_output([ '/bin/bash', '-i', '-c', 'echo $PS1' -], stderr=subprocess.DEVNULL).decode().startswith("(base)") \ No newline at end of file +], stderr=subprocess.DEVNULL).decode().startswith("(base)") diff --git a/miniforge3/ubuntu/Dockerfile b/miniforge3/ubuntu/Dockerfile index c77cee78..62fc0b6d 100644 --- a/miniforge3/ubuntu/Dockerfile +++ b/miniforge3/ubuntu/Dockerfile @@ -41,4 +41,4 @@ RUN echo ". ${CONDA_DIR}/etc/profile.d/conda.sh && conda activate base" >> /etc/ RUN echo ". ${CONDA_DIR}/etc/profile.d/conda.sh && conda activate base" >> ~/.bashrc ENTRYPOINT ["tini", "--"] -CMD [ "/bin/bash" ] \ No newline at end of file +CMD [ "/bin/bash" ]