From edaff6980b3b12907b57ccf383ace5ca6cee947c Mon Sep 17 00:00:00 2001
From: VickyTheViking <serinamail313@gmail.com>
Date: Sat, 7 Dec 2024 23:26:13 +0100
Subject: [PATCH] spark-master to 127.0.0.1 because it is not resolvable from
 outside of docker network use python3 included containers, otherwise install
 python3, python3-pip and pyspark python package

---
 .../community/spark/app/docker-compose.yml    |  1 +
 .../scripts/updater/community/spark/update.sh | 22 +++++++++++++++++--
 2 files changed, 21 insertions(+), 2 deletions(-)

diff --git a/google/fingerprinters/web/scripts/updater/community/spark/app/docker-compose.yml b/google/fingerprinters/web/scripts/updater/community/spark/app/docker-compose.yml
index e8dd5a5c6..778f944bb 100644
--- a/google/fingerprinters/web/scripts/updater/community/spark/app/docker-compose.yml
+++ b/google/fingerprinters/web/scripts/updater/community/spark/app/docker-compose.yml
@@ -11,6 +11,7 @@ services:
     environment:
       - SPARK_MASTER_HOST=spark-master
       - SPARK_MASTER_PORT=7077
+      - SPARK_LOCAL_HOSTNAME=127.0.0.1
     command: /opt/spark/bin/spark-class org.apache.spark.deploy.master.Master
 
   spark-worker:
diff --git a/google/fingerprinters/web/scripts/updater/community/spark/update.sh b/google/fingerprinters/web/scripts/updater/community/spark/update.sh
index 102edf1d8..fbaecf092 100755
--- a/google/fingerprinters/web/scripts/updater/community/spark/update.sh
+++ b/google/fingerprinters/web/scripts/updater/community/spark/update.sh
@@ -40,7 +40,23 @@ mkdir -p "${FINGERPRINTS_PATH}"
 startSpark() {
   local version="$1"
   pushd "${SPARK_APP_PATH}" >/dev/null
-    SPARK_VERSION="${version}" docker compose up -d
+    # if version-python3 exists then we have a spark container with python3
+    # otherwise we must install python3
+    if SPARK_VERSION="${version}-python3" docker compose up -d | grep -q "manifest unknown"
+    then
+      echo "\nInstalling python3 into worker container" && \
+      SPARK_VERSION="${version}" docker compose up -d && \
+      installPython3InSpark "${version}"
+    fi
+  popd >/dev/null
+}
+
+installPython3InSpark() {
+  local version="$1"
+  pushd "${SPARK_APP_PATH}" >/dev/null
+    docker exec  -it -u 0 spark-master apt update >/dev/null
+    docker exec  -it -u 0 spark-master apt install python3 python3-pip -y >/dev/null
+    docker exec  -it -u 0 spark-master pip3 install pyspark=="${version}" >/dev/null
   popd >/dev/null
 }
 
@@ -48,6 +64,8 @@ stopSpark() {
   local version="$1"
   pushd "${SPARK_APP_PATH}" >/dev/null
     SPARK_VERSION="${version}" docker compose down --volumes --remove-orphans
+    # or stop the python3 contained version
+    SPARK_VERSION="${version}-python3" docker compose down --volumes --remove-orphans
   popd >/dev/null
 }
 
@@ -119,4 +137,4 @@ done
 convertFingerprint "${JSON_DATA}" "${BIN_DATA}"
 
 echo "Fingerprint updated for Spark. Please commit the following file:"
-echo "  ${BIN_DATA}"
\ No newline at end of file
+echo "  ${BIN_DATA}"