From 3cc22d07178739eeca6862343f70f2a887f898a1 Mon Sep 17 00:00:00 2001 From: Thomas Li <47963215+lithomas1@users.noreply.github.com> Date: Wed, 6 Dec 2023 22:33:38 -0500 Subject: [PATCH 01/11] Refresh the old dockerfile --- gitpod/gitpod.Dockerfile | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/gitpod/gitpod.Dockerfile b/gitpod/gitpod.Dockerfile index ab3f25b231e67..aeeb3fda45b11 100644 --- a/gitpod/gitpod.Dockerfile +++ b/gitpod/gitpod.Dockerfile @@ -1,7 +1,7 @@ # Doing a local shallow clone - keeps the container secure # and much slimmer than using COPY directly or making a # remote clone -ARG BASE_CONTAINER="pandas/pandas-dev:latest" +#ARG BASE_CONTAINER="pandas/pandas-dev:latest" FROM gitpod/workspace-base:latest as clone # the clone should be deep enough for versioneer to work @@ -11,7 +11,7 @@ RUN git clone https://github.com/pandas-dev/pandas --depth 12 /tmp/pandas # Using the pandas-dev Docker image as a base # This way, we ensure we have all the needed compilers and dependencies # while reducing the build time -FROM ${BASE_CONTAINER} as build +#FROM ${BASE_CONTAINER} as build # ----------------------------------------------------------------------------- USER root @@ -34,8 +34,7 @@ WORKDIR ${WORKSPACE} # Build pandas to populate the cache used by ccache RUN git config --global --add safe.directory /workspace/pandas RUN conda activate ${CONDA_ENV} && \ - python -m pip install -e . --no-build-isolation && \ - python setup.py build_ext --inplace && \ + python -m pip install -ve . --no-build-isolation --config-settings=editable-verbose=true && \ ccache -s # Gitpod will load the repository into /workspace/pandas. We remove the From ccf1f0aa119dd01450f59971ecfc0f487b53d276 Mon Sep 17 00:00:00 2001 From: Thomas Li <47963215+lithomas1@users.noreply.github.com> Date: Wed, 6 Dec 2023 22:40:36 -0500 Subject: [PATCH 02/11] Update .gitpod.yml --- .gitpod.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitpod.yml b/.gitpod.yml index 9222639136a17..c61153a16381f 100644 --- a/.gitpod.yml +++ b/.gitpod.yml @@ -6,13 +6,14 @@ # images for gitpod pandas are in https://hub.docker.com/r/pandas/pandas-gitpod/tags # we're using the Dockerfile in the base of the repo image: - file: Dockerfile + file: gitpod/gitpod.Dockerfile tasks: - name: Prepare development environment init: | mkdir -p .vscode cp gitpod/settings.json .vscode/settings.json git fetch --tags + conda env update --file environment.yml --prune python -m pip install -ve . --no-build-isolation --config-settings editable-verbose=true pre-commit install command: | From 85f354f858ff8f0f0d59cf8694de20fcd54078d4 Mon Sep 17 00:00:00 2001 From: Thomas Li <47963215+lithomas1@users.noreply.github.com> Date: Wed, 6 Dec 2023 22:53:27 -0500 Subject: [PATCH 03/11] Update .gitpod.yml --- .gitpod.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.gitpod.yml b/.gitpod.yml index c61153a16381f..70cc2fa9ec487 100644 --- a/.gitpod.yml +++ b/.gitpod.yml @@ -5,8 +5,8 @@ # images for gitpod pandas are in https://hub.docker.com/r/pandas/pandas-gitpod/tags # we're using the Dockerfile in the base of the repo -image: - file: gitpod/gitpod.Dockerfile +# image: +# file: gitpod/gitpod.Dockerfile tasks: - name: Prepare development environment init: | @@ -14,6 +14,7 @@ tasks: cp gitpod/settings.json .vscode/settings.json git fetch --tags conda env update --file environment.yml --prune + conda activate pandas-dev python -m pip install -ve . --no-build-isolation --config-settings editable-verbose=true pre-commit install command: | From 4b6129d83335cf09eb60937f5fd9e8d3f6a1453e Mon Sep 17 00:00:00 2001 From: Thomas Li Date: Thu, 7 Dec 2023 04:07:20 +0000 Subject: [PATCH 04/11] update workspace files --- .gitpod.yml | 4 +- gitpod/gitpod.Dockerfile | 79 ++++++++++++++++++++++++---------------- 2 files changed, 50 insertions(+), 33 deletions(-) diff --git a/.gitpod.yml b/.gitpod.yml index 70cc2fa9ec487..c96383596e0c9 100644 --- a/.gitpod.yml +++ b/.gitpod.yml @@ -5,8 +5,8 @@ # images for gitpod pandas are in https://hub.docker.com/r/pandas/pandas-gitpod/tags # we're using the Dockerfile in the base of the repo -# image: -# file: gitpod/gitpod.Dockerfile +image: + file: gitpod/gitpod.Dockerfile tasks: - name: Prepare development environment init: | diff --git a/gitpod/gitpod.Dockerfile b/gitpod/gitpod.Dockerfile index aeeb3fda45b11..b5914b47a9a12 100644 --- a/gitpod/gitpod.Dockerfile +++ b/gitpod/gitpod.Dockerfile @@ -4,44 +4,61 @@ #ARG BASE_CONTAINER="pandas/pandas-dev:latest" FROM gitpod/workspace-base:latest as clone -# the clone should be deep enough for versioneer to work -RUN git clone https://github.com/pandas-dev/pandas --depth 12 /tmp/pandas +USER root +# Install base utilities +RUN apt-get update \ + && apt-get install -y build-essential \ + && apt-get install -y wget \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* -# ----------------------------------------------------------------------------- -# Using the pandas-dev Docker image as a base -# This way, we ensure we have all the needed compilers and dependencies -# while reducing the build time -#FROM ${BASE_CONTAINER} as build +# Install miniconda +ENV CONDA_DIR /opt/conda +RUN wget --quiet https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh -O ~/miniconda.sh && \ + /bin/bash ~/miniconda.sh -b -p /opt/conda -# ----------------------------------------------------------------------------- -USER root +# Put conda in path so we can use conda activate +ENV PATH=$CONDA_DIR/bin:$PATH + +# Use the speedy libmamba solver +RUN conda install -n base conda-libmamba-solver +RUN conda config --set solver libmamba + +# # ----------------------------------------------------------------------------- +# # Using the pandas-dev Docker image as a base +# # This way, we ensure we have all the needed compilers and dependencies +# # while reducing the build time +# #FROM ${BASE_CONTAINER} as build + +# # ----------------------------------------------------------------------------- +# USER root -# ----------------------------------------------------------------------------- -# ---- ENV variables ---- -# ---- Directories needed ---- -ENV WORKSPACE=/workspace/pandas/ \ - CONDA_ENV=pandas-dev +# # ----------------------------------------------------------------------------- +# # ---- ENV variables ---- +# # ---- Directories needed ---- +# ENV WORKSPACE=/workspace/pandas/ \ +# CONDA_ENV=pandas-dev -# Allows this micromamba.Dockerfile to activate conda environments -SHELL ["/bin/bash", "--login", "-o", "pipefail", "-c"] +# # Allows this micromamba.Dockerfile to activate conda environments +# SHELL ["/bin/bash", "--login", "-o", "pipefail", "-c"] -# Copy over the shallow clone -COPY --from=clone --chown=gitpod /tmp/pandas ${WORKSPACE} +# # Copy over the shallow clone +# COPY --from=clone --chown=gitpod /tmp/pandas ${WORKSPACE} -# Everything happens in the /workspace/pandas directory -WORKDIR ${WORKSPACE} +# # Everything happens in the /workspace/pandas directory +# WORKDIR ${WORKSPACE} -# Build pandas to populate the cache used by ccache -RUN git config --global --add safe.directory /workspace/pandas -RUN conda activate ${CONDA_ENV} && \ - python -m pip install -ve . --no-build-isolation --config-settings=editable-verbose=true && \ - ccache -s +# # Build pandas to populate the cache used by ccache +# RUN git config --global --add safe.directory /workspace/pandas +# RUN conda activate ${CONDA_ENV} && \ +# python -m pip install -ve . --no-build-isolation --config-settings=editable-verbose=true && \ +# ccache -s -# Gitpod will load the repository into /workspace/pandas. We remove the -# directory from the image to prevent conflicts -RUN rm -rf ${WORKSPACE} +# # Gitpod will load the repository into /workspace/pandas. We remove the +# # directory from the image to prevent conflicts +# RUN rm -rf ${WORKSPACE} -# ----------------------------------------------------------------------------- -# Always return to non privileged user -RUN chown -R gitpod:gitpod /home/gitpod/.cache/ +# # ----------------------------------------------------------------------------- +# # Always return to non privileged user +# RUN chown -R gitpod:gitpod /home/gitpod/.cache/ USER gitpod From c305380fb42cd660834801ecf10a299fec3353ae Mon Sep 17 00:00:00 2001 From: Thomas Li Date: Thu, 7 Dec 2023 04:40:16 +0000 Subject: [PATCH 05/11] more updates --- .gitpod.yml | 2 ++ gitpod/gitpod.Dockerfile | 41 ++++++---------------------------------- 2 files changed, 8 insertions(+), 35 deletions(-) diff --git a/.gitpod.yml b/.gitpod.yml index c96383596e0c9..7d14b9f753c5a 100644 --- a/.gitpod.yml +++ b/.gitpod.yml @@ -13,6 +13,8 @@ tasks: mkdir -p .vscode cp gitpod/settings.json .vscode/settings.json git fetch --tags + conda init + source ~/.bashrc conda env update --file environment.yml --prune conda activate pandas-dev python -m pip install -ve . --no-build-isolation --config-settings editable-verbose=true diff --git a/gitpod/gitpod.Dockerfile b/gitpod/gitpod.Dockerfile index b5914b47a9a12..49f5680fe3a7b 100644 --- a/gitpod/gitpod.Dockerfile +++ b/gitpod/gitpod.Dockerfile @@ -20,45 +20,16 @@ RUN wget --quiet https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86 # Put conda in path so we can use conda activate ENV PATH=$CONDA_DIR/bin:$PATH +# init conda +RUN conda init + # Use the speedy libmamba solver RUN conda install -n base conda-libmamba-solver RUN conda config --set solver libmamba -# # ----------------------------------------------------------------------------- -# # Using the pandas-dev Docker image as a base -# # This way, we ensure we have all the needed compilers and dependencies -# # while reducing the build time -# #FROM ${BASE_CONTAINER} as build - -# # ----------------------------------------------------------------------------- -# USER root - -# # ----------------------------------------------------------------------------- -# # ---- ENV variables ---- -# # ---- Directories needed ---- -# ENV WORKSPACE=/workspace/pandas/ \ -# CONDA_ENV=pandas-dev - -# # Allows this micromamba.Dockerfile to activate conda environments -# SHELL ["/bin/bash", "--login", "-o", "pipefail", "-c"] - -# # Copy over the shallow clone -# COPY --from=clone --chown=gitpod /tmp/pandas ${WORKSPACE} - -# # Everything happens in the /workspace/pandas directory -# WORKDIR ${WORKSPACE} - -# # Build pandas to populate the cache used by ccache -# RUN git config --global --add safe.directory /workspace/pandas -# RUN conda activate ${CONDA_ENV} && \ -# python -m pip install -ve . --no-build-isolation --config-settings=editable-verbose=true && \ -# ccache -s - -# # Gitpod will load the repository into /workspace/pandas. We remove the -# # directory from the image to prevent conflicts -# RUN rm -rf ${WORKSPACE} - # # ----------------------------------------------------------------------------- # # Always return to non privileged user -# RUN chown -R gitpod:gitpod /home/gitpod/.cache/ +RUN chown -R gitpod:gitpod /opt/conda/pkgs +RUN chown -R gitpod:gitpod /opt/conda/envs +RUN chown -R gitpod:gitpod /home/gitpod/.conda USER gitpod From f780db92a71d630cde009349fc33df4cd06688a5 Mon Sep 17 00:00:00 2001 From: Thomas Li Date: Thu, 7 Dec 2023 04:50:12 +0000 Subject: [PATCH 06/11] one more missing permissions --- gitpod/gitpod.Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gitpod/gitpod.Dockerfile b/gitpod/gitpod.Dockerfile index 49f5680fe3a7b..1f05875f1a8bd 100644 --- a/gitpod/gitpod.Dockerfile +++ b/gitpod/gitpod.Dockerfile @@ -1,7 +1,6 @@ # Doing a local shallow clone - keeps the container secure # and much slimmer than using COPY directly or making a # remote clone -#ARG BASE_CONTAINER="pandas/pandas-dev:latest" FROM gitpod/workspace-base:latest as clone USER root @@ -32,4 +31,5 @@ RUN conda config --set solver libmamba RUN chown -R gitpod:gitpod /opt/conda/pkgs RUN chown -R gitpod:gitpod /opt/conda/envs RUN chown -R gitpod:gitpod /home/gitpod/.conda +RUN chown -R gitpod:gitpod /home/gitpod/.cache USER gitpod From a3ff3cd4d405e1868e1e6bb8143504ad8a0552a7 Mon Sep 17 00:00:00 2001 From: Thomas Li Date: Thu, 7 Dec 2023 13:41:05 +0000 Subject: [PATCH 07/11] more fixes --- .gitpod.yml | 2 ++ gitpod/gitpod.Dockerfile | 8 ++++---- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/.gitpod.yml b/.gitpod.yml index 7d14b9f753c5a..1fb6fe5ee11a5 100644 --- a/.gitpod.yml +++ b/.gitpod.yml @@ -20,6 +20,8 @@ tasks: python -m pip install -ve . --no-build-isolation --config-settings editable-verbose=true pre-commit install command: | + source ~/.bashrc + conda env update --file environment.yml --prune python -m pip install -ve . --no-build-isolation --config-settings editable-verbose=true echo "✨ Pre-build complete! You can close this terminal ✨ " diff --git a/gitpod/gitpod.Dockerfile b/gitpod/gitpod.Dockerfile index 1f05875f1a8bd..2bb8c9be54109 100644 --- a/gitpod/gitpod.Dockerfile +++ b/gitpod/gitpod.Dockerfile @@ -1,7 +1,4 @@ -# Doing a local shallow clone - keeps the container secure -# and much slimmer than using COPY directly or making a -# remote clone -FROM gitpod/workspace-base:latest as clone +FROM gitpod/workspace-base:latest USER root # Install base utilities @@ -26,6 +23,9 @@ RUN conda init RUN conda install -n base conda-libmamba-solver RUN conda config --set solver libmamba +# Install dependencies +RUN conda env update --file https://raw.githubusercontent.com/pandas-dev/pandas/main/environment.yml --prune + # # ----------------------------------------------------------------------------- # # Always return to non privileged user RUN chown -R gitpod:gitpod /opt/conda/pkgs From 722df466919a96ef3c1b9a1bdcccb2706533ae3f Mon Sep 17 00:00:00 2001 From: Thomas Li Date: Thu, 7 Dec 2023 13:41:34 +0000 Subject: [PATCH 08/11] one more activate --- .gitpod.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitpod.yml b/.gitpod.yml index 1fb6fe5ee11a5..90448a7cd4c24 100644 --- a/.gitpod.yml +++ b/.gitpod.yml @@ -21,6 +21,7 @@ tasks: pre-commit install command: | source ~/.bashrc + conda activate pandas-dev conda env update --file environment.yml --prune python -m pip install -ve . --no-build-isolation --config-settings editable-verbose=true echo "✨ Pre-build complete! You can close this terminal ✨ " From 4b27d447ca54f1811eae251b41e10b883d08351c Mon Sep 17 00:00:00 2001 From: Thomas Li <47963215+lithomas1@users.noreply.github.com> Date: Thu, 7 Dec 2023 15:17:02 -0500 Subject: [PATCH 09/11] Update gitpod.Dockerfile --- gitpod/gitpod.Dockerfile | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/gitpod/gitpod.Dockerfile b/gitpod/gitpod.Dockerfile index 2bb8c9be54109..1c67179f29f60 100644 --- a/gitpod/gitpod.Dockerfile +++ b/gitpod/gitpod.Dockerfile @@ -1,4 +1,4 @@ -FROM gitpod/workspace-base:latest +FROM gitpod/workspace-base:2023-11-24-15-04-57 USER root # Install base utilities @@ -16,12 +16,8 @@ RUN wget --quiet https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86 # Put conda in path so we can use conda activate ENV PATH=$CONDA_DIR/bin:$PATH -# init conda -RUN conda init - -# Use the speedy libmamba solver -RUN conda install -n base conda-libmamba-solver -RUN conda config --set solver libmamba +# Init conda and use the speedy libmamba solver +RUN conda init && conda install -n base conda-libmamba-solver && conda config --set solver libmamba # Install dependencies RUN conda env update --file https://raw.githubusercontent.com/pandas-dev/pandas/main/environment.yml --prune From 2342fb63e3b58ce5ac45703d6f84017e6c1d6cce Mon Sep 17 00:00:00 2001 From: Thomas Li <47963215+lithomas1@users.noreply.github.com> Date: Thu, 7 Dec 2023 18:30:17 -0500 Subject: [PATCH 10/11] Update gitpod.Dockerfile --- gitpod/gitpod.Dockerfile | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/gitpod/gitpod.Dockerfile b/gitpod/gitpod.Dockerfile index 1c67179f29f60..e253b75a4fe0f 100644 --- a/gitpod/gitpod.Dockerfile +++ b/gitpod/gitpod.Dockerfile @@ -8,6 +8,10 @@ RUN apt-get update \ && apt-get clean \ && rm -rf /var/lib/apt/lists/* +# ----------------------------------------------------------------------------- +# Always return to non privileged user +USER gitpod + # Install miniconda ENV CONDA_DIR /opt/conda RUN wget --quiet https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh -O ~/miniconda.sh && \ @@ -22,10 +26,8 @@ RUN conda init && conda install -n base conda-libmamba-solver && conda config -- # Install dependencies RUN conda env update --file https://raw.githubusercontent.com/pandas-dev/pandas/main/environment.yml --prune -# # ----------------------------------------------------------------------------- -# # Always return to non privileged user -RUN chown -R gitpod:gitpod /opt/conda/pkgs -RUN chown -R gitpod:gitpod /opt/conda/envs -RUN chown -R gitpod:gitpod /home/gitpod/.conda -RUN chown -R gitpod:gitpod /home/gitpod/.cache -USER gitpod +#RUN chown -R gitpod:gitpod /opt/conda/pkgs +#RUN chown -R gitpod:gitpod /opt/conda/envs +#RUN chown -R gitpod:gitpod /home/gitpod/.conda +#RUN chown -R gitpod:gitpod /home/gitpod/.cache + From 1835adb0f89b309f0ee2ed5f9902c22f3b5bd193 Mon Sep 17 00:00:00 2001 From: Thomas Li Date: Thu, 7 Dec 2023 23:46:28 +0000 Subject: [PATCH 11/11] simplify greatly --- gitpod/gitpod.Dockerfile | 27 +-------------------------- 1 file changed, 1 insertion(+), 26 deletions(-) diff --git a/gitpod/gitpod.Dockerfile b/gitpod/gitpod.Dockerfile index e253b75a4fe0f..6132f7514908e 100644 --- a/gitpod/gitpod.Dockerfile +++ b/gitpod/gitpod.Dockerfile @@ -1,24 +1,4 @@ -FROM gitpod/workspace-base:2023-11-24-15-04-57 - -USER root -# Install base utilities -RUN apt-get update \ - && apt-get install -y build-essential \ - && apt-get install -y wget \ - && apt-get clean \ - && rm -rf /var/lib/apt/lists/* - -# ----------------------------------------------------------------------------- -# Always return to non privileged user -USER gitpod - -# Install miniconda -ENV CONDA_DIR /opt/conda -RUN wget --quiet https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh -O ~/miniconda.sh && \ - /bin/bash ~/miniconda.sh -b -p /opt/conda - -# Put conda in path so we can use conda activate -ENV PATH=$CONDA_DIR/bin:$PATH +FROM condaforge/miniforge3:23.3.1-1 # Init conda and use the speedy libmamba solver RUN conda init && conda install -n base conda-libmamba-solver && conda config --set solver libmamba @@ -26,8 +6,3 @@ RUN conda init && conda install -n base conda-libmamba-solver && conda config -- # Install dependencies RUN conda env update --file https://raw.githubusercontent.com/pandas-dev/pandas/main/environment.yml --prune -#RUN chown -R gitpod:gitpod /opt/conda/pkgs -#RUN chown -R gitpod:gitpod /opt/conda/envs -#RUN chown -R gitpod:gitpod /home/gitpod/.conda -#RUN chown -R gitpod:gitpod /home/gitpod/.cache -