spark-master to 127.0.0.1 because it is not resolvable from outside o…

…f docker network

use python3 included containers, otherwise install python3, python3-pip and pyspark python package

google/fingerprinters/web/scripts/updater/community/spark/app/docker-compose.yml

@@ -11,6 +11,7 @@ services:
     environment:
       - SPARK_MASTER_HOST=spark-master
       - SPARK_MASTER_PORT=7077
+      - SPARK_LOCAL_HOSTNAME=127.0.0.1
     command: /opt/spark/bin/spark-class org.apache.spark.deploy.master.Master
 
   spark-worker:

google/fingerprinters/web/scripts/updater/community/spark/update.sh

@@ -40,14 +40,32 @@ mkdir -p "${FINGERPRINTS_PATH}"
 startSpark() {
   local version="$1"
   pushd "${SPARK_APP_PATH}" >/dev/null
-    SPARK_VERSION="${version}" docker compose up -d
+    # if version-python3 exists then we have a spark container with python3
+    # otherwise we must install python3
+    if SPARK_VERSION="${version}-python3" docker compose up -d | grep -q "manifest unknown"
+    then
+      echo "\nInstalling python3 into worker container" && \
+      SPARK_VERSION="${version}" docker compose up -d && \
+      installPython3InSpark "${version}"
+    fi
+  popd >/dev/null
+}
+
+installPython3InSpark() {
+  local version="$1"
+  pushd "${SPARK_APP_PATH}" >/dev/null
+    docker exec  -it -u 0 spark-master apt update >/dev/null
+    docker exec  -it -u 0 spark-master apt install python3 python3-pip -y >/dev/null
+    docker exec  -it -u 0 spark-master pip3 install pyspark=="${version}" >/dev/null
   popd >/dev/null
 }
 
 stopSpark() {
   local version="$1"
   pushd "${SPARK_APP_PATH}" >/dev/null
     SPARK_VERSION="${version}" docker compose down --volumes --remove-orphans
+    # or stop the python3 contained version
+    SPARK_VERSION="${version}-python3" docker compose down --volumes --remove-orphans
   popd >/dev/null
 }
 
@@ -119,4 +137,4 @@ done
 convertFingerprint "${JSON_DATA}" "${BIN_DATA}"
 
 echo "Fingerprint updated for Spark. Please commit the following file:"
-echo "  ${BIN_DATA}"
+echo "  ${BIN_DATA}"