upgrade to 3.4.1

dsaidgovsg · Nov 20, 2023 · bab5aad · bab5aad
1 parent 74e0752
commit bab5aad
Show file tree

Hide file tree

Showing 5 changed files with 72 additions and 6 deletions.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -165,6 +165,66 @@ jobs:
           hadoop:     "3.3.2"
           scala:      "2.13"
           java:       "11"
+        - jupyterhub: "1.5.0"
+          spark:      "3.4.1"
+          hadoop:     "3.3.4"
+          scala:      "2.12"
+          java:       "8"
+        - jupyterhub: "1.5.0"
+          spark:      "3.4.1"
+          hadoop:     "3.3.4"
+          scala:      "2.12"
+          java:       "11"
+        - jupyterhub: "1.5.0"
+          spark:      "3.4.1"
+          hadoop:     "3.3.4"
+          scala:      "2.13"
+          java:       "8"
+        - jupyterhub: "1.5.0"
+          spark:      "3.4.1"
+          hadoop:     "3.3.4"
+          scala:      "2.13"
+          java:       "11"
+        - jupyterhub: "2.2.2"
+          spark:      "3.4.1"
+          hadoop:     "3.3.4"
+          scala:      "2.12"
+          java:       "8"
+        - jupyterhub: "2.2.2"
+          spark:      "3.4.1"
+          hadoop:     "3.3.4"
+          scala:      "2.12"
+          java:       "11"
+        - jupyterhub: "2.2.2"
+          spark:      "3.4.1"
+          hadoop:     "3.3.4"
+          scala:      "2.13"
+          java:       "8"
+        - jupyterhub: "2.2.2"
+          spark:      "3.4.1"
+          hadoop:     "3.3.4"
+          scala:      "2.13"
+          java:       "11"
+        - jupyterhub: "2.3.1"
+          spark:      "3.4.1"
+          hadoop:     "3.3.4"
+          scala:      "2.12"
+          java:       "8"
+        - jupyterhub: "2.3.1"
+          spark:      "3.4.1"
+          hadoop:     "3.3.4"
+          scala:      "2.12"
+          java:       "11"
+        - jupyterhub: "2.3.1"
+          spark:      "3.4.1"
+          hadoop:     "3.3.4"
+          scala:      "2.13"
+          java:       "8"
+        - jupyterhub: "2.3.1"
+          spark:      "3.4.1"
+          hadoop:     "3.3.4"
+          scala:      "2.13"
+          java:       "11"
     runs-on: ubuntu-latest
     env:
       IMAGE_NAME: spark-jupyterhub

diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -3,7 +3,7 @@
 ## v2
 
 - Adds Java 8 and 11 differentiation.
-- Supports Spark "3.1.3", "3.2.2", "3.3.0".
+- Supports Spark "3.1.3", "3.2.2", "3.3.0", "3.4.1".
 - Supports JupyterHub "1.5.0", "2.2.2", "2.3.1".
 - Adds `jlab-enhanced/launcher` as `labextension` to improve launcher UI.
 - Change to Debian distro.

diff --git a/Dockerfile b/Dockerfile
@@ -1,8 +1,8 @@
 ARG BASE_VERSION=v5
-ARG SPARK_VERSION=3.3.0
-ARG HADOOP_VERSION=3.3.2
+ARG SPARK_VERSION=3.4.1
+ARG HADOOP_VERSION=3.3.4
 ARG SCALA_VERSION=2.13
-ARG JAVA_VERSION=11
+ARG JAVA_VERSION=8
 ARG PYTHON_VERSION=3.9
 
 FROM dsaidgovsg/spark-k8s-addons:${BASE_VERSION}_${SPARK_VERSION}_hadoop-${HADOOP_VERSION}_scala-${SCALA_VERSION}_java-${JAVA_VERSION}_python-${PYTHON_VERSION}

diff --git a/templates/apply-vars.sh b/templates/apply-vars.sh
@@ -1,6 +1,6 @@
 #!/usr/bin/env bash
 if [[ ! -v TERA ]] && ! command -v tera >/dev/null; then
-    printf "Run \"cargo install tera-cli\" to install tera first.\nLinux users may download from https://github.com/guangie88/tera-cli/releases instead."
+    printf "Run \"cargo install tera-cli --version=^0.4.0\" to install tera first.\nLinux users may download from https://github.com/guangie88/tera-cli/releases instead."
     return 1
 elif [[ ! -v TERA ]]; then
     TERA="$(command -v tera)"
@@ -9,4 +9,4 @@ fi
 DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
 
 ${TERA} -f "${DIR}/ci.yml.tmpl" --yaml "${DIR}/vars.yml" > "${DIR}/../.github/workflows/ci.yml" && \
-printf "Successfully applied template into ci.yml!\n"
+printf "Successfully applied template into ci.yml!\n"
diff --git a/templates/vars.yml b/templates/vars.yml
@@ -19,3 +19,9 @@ versions:
   hadoop:     ["3.3.2"]
   scala:      ["2.12", "2.13"]
   java:       ["8", "11"]
+
+- jupyterhub: ["1.5.0", "2.2.2", "2.3.1"]
+  spark:      ["3.4.1"]
+  hadoop:     ["3.3.4"]
+  scala:      ["2.12", "2.13"]
+  java:       ["8", "11"]