Merge pull request #53 from jiminhsieh/align-version

Upgrade to correct version
techops-recsys-lateral-hiring · May 22, 2024 · e0f5ef2 · e0f5ef2
2 parents d20d831 + e0e7f7e
commit e0f5ef2
Show file tree

Hide file tree

Showing 8 changed files with 282 additions and 272 deletions.
diff --git a/.gitpod.Dockerfile b/.gitpod.Dockerfile
@@ -6,11 +6,11 @@ WORKDIR /opt
 RUN if [ "$(arch)" = "aarch64" ] ; then ARCHITECTURE="aarch64" ; else ARCHITECTURE="x64"; fi && \
     wget -O OpenJDK.tar.gz https://github.com/AdoptOpenJDK/openjdk11-binaries/releases/download/jdk-11.0.11%2B9/OpenJDK11U-jdk_${ARCHITECTURE}_linux_hotspot_11.0.11_9.tar.gz && \
     wget -O scala.tgz https://downloads.lightbend.com/scala/2.13.5/scala-2.13.5.tgz && \
-    wget -O spark-hadoop.tgz https://archive.apache.org/dist/spark/spark-3.2.1/spark-3.2.1-bin-hadoop3.2.tgz
+    wget -O spark-hadoop.tgz https://archive.apache.org/dist/spark/spark-3.5.1/spark-3.5.1-bin-hadoop3.tgz
 RUN tar xzf OpenJDK.tar.gz && \
     tar xvf scala.tgz && \
     tar xvf spark-hadoop.tgz
-ENV PATH="/opt/jdk-11.0.11+9/bin:/opt/scala-2.13.5/bin:/opt/spark-3.2.1-bin-hadoop3.2/bin:$PATH"
+ENV PATH="/opt/jdk-11.0.11+9/bin:/opt/scala-2.13.5/bin:/opt/spark-3.5.1-bin-hadoop3/bin:$PATH"
 
 
 #TODO : Change the user to non root user
@@ -19,4 +19,4 @@ WORKDIR /app
 
 COPY ./pyproject.toml /app/pyproject.toml
 
-RUN pyenv install 3.9.10 && pyenv global 3.9.10
+RUN pyenv install 3.11.4 && pyenv global 3.11.4
diff --git a/Dockerfile b/Dockerfile
@@ -5,11 +5,11 @@ WORKDIR /opt
 RUN if [ "$(arch)" = "aarch64" ] ; then ARCHITECTURE="aarch64" ; else ARCHITECTURE="x64"; fi && \
     wget -O OpenJDK.tar.gz https://github.com/AdoptOpenJDK/openjdk11-binaries/releases/download/jdk-11.0.11%2B9/OpenJDK11U-jdk_${ARCHITECTURE}_linux_hotspot_11.0.11_9.tar.gz && \
     wget -O scala.tgz https://downloads.lightbend.com/scala/2.13.5/scala-2.13.5.tgz && \
-    wget -O spark-hadoop.tgz https://archive.apache.org/dist/spark/spark-3.4.1/spark-3.4.1-bin-hadoop3-scala2.13.tgz
+    wget -O spark-hadoop.tgz https://archive.apache.org/dist/spark/spark-3.5.1/spark-3.5.1-bin-hadoop3-scala2.13.tgz
 RUN tar xzf OpenJDK.tar.gz && \
     tar xvf scala.tgz && \
     tar xvf spark-hadoop.tgz
-ENV PATH="/opt/jdk-11.0.11+9/bin:/opt/scala-2.13.5/bin:/opt/spark-3.2.1-bin-hadoop3.2/bin:$PATH"
+ENV PATH="/opt/jdk-11.0.11+9/bin:/opt/scala-2.13.5/bin:/opt/spark-3.5.1-bin-hadoop3/bin:$PATH"
 
 RUN curl -sSL https://install.python-poetry.org | python3 -
 ENV PATH="/root/.local/bin:${PATH}"

diff --git a/README-LOCAL.md b/README-LOCAL.md
@@ -7,7 +7,7 @@ These jobs are using _PySpark_ to process larger volumes of data and are suppose
 
 Please make sure you have the following installed and can run them
 
-- Python (3.11 or later), you can use for example [pyenv](https://github.com/pyenv/pyenv#installation) to manage your python versions locally
+- Python (3.11.x), you can use for example [pyenv](https://github.com/pyenv/pyenv#installation) to manage your python versions locally
 - [Poetry](https://python-poetry.org/docs/#installation)
 - Java (11)
 

diff --git a/README.md b/README.md
@@ -32,7 +32,7 @@ OR
 scripts\install_choco.ps1
 scripts\install.bat
 
-# For local laptop setup ensure that Java 11 with Spark 3.2.1 is available. More details in README-LOCAL.md
+# For local laptop setup ensure that Java 11 with Spark 3.5.1 is available. More details in README-LOCAL.md
 ```
 
 > **If you are using Colima, please ensure that you start Colima. For staring Colima, you could use following command:**

diff --git a/batect b/batect
@@ -8,8 +8,8 @@
     # You should commit this file to version control alongside the rest of your project. It should not be installed globally.
     # For more information, visit https://github.com/batect/batect.
 
-    VERSION="0.84.0"
-    CHECKSUM="${BATECT_DOWNLOAD_CHECKSUM:-e39f3e73f0772b3716a4a01624e26009d9da5f5a274464a598d5b265c1e52964}"
+    VERSION="0.85.0"
+    CHECKSUM="${BATECT_DOWNLOAD_CHECKSUM:-901ed73295be75d295cec1d06315f7026b36ccb1666660b8af432cfbbc7feae8}"
     DOWNLOAD_URL_ROOT=${BATECT_DOWNLOAD_URL_ROOT:-"https://updates.batect.dev/v1/files"}
     DOWNLOAD_URL=${BATECT_DOWNLOAD_URL:-"$DOWNLOAD_URL_ROOT/$VERSION/batect-$VERSION.jar"}
     QUIET_DOWNLOAD=${BATECT_QUIET_DOWNLOAD:-false}

diff --git a/batect.cmd b/batect.cmd
@@ -6,7 +6,7 @@ rem For more information, visit https://github.com/batect/batect.
 
 setlocal EnableDelayedExpansion
 
-set "version=0.84.0"
+set "version=0.85.0"
 
 if "%BATECT_CACHE_DIR%" == "" (
     set "BATECT_CACHE_DIR=%USERPROFILE%\.batect\cache"
@@ -22,7 +22,7 @@ $ErrorActionPreference = 'Stop'^
 
 ^
 
-$Version='0.84.0'^
+$Version='0.85.0'^
 
 ^
 
@@ -48,7 +48,7 @@ $UrlEncodedVersion = [Uri]::EscapeDataString($Version)^
 
 $DownloadUrl = getValueOrDefault $env:BATECT_DOWNLOAD_URL "$DownloadUrlRoot/$UrlEncodedVersion/batect-$UrlEncodedVersion.jar"^
 
-$ExpectedChecksum = getValueOrDefault $env:BATECT_DOWNLOAD_CHECKSUM 'e39f3e73f0772b3716a4a01624e26009d9da5f5a274464a598d5b265c1e52964'^
+$ExpectedChecksum = getValueOrDefault $env:BATECT_DOWNLOAD_CHECKSUM '901ed73295be75d295cec1d06315f7026b36ccb1666660b8af432cfbbc7feae8'^
 
 ^