From 1c0f23c94a3fc30a2c26d5bb4eecbbff7e560311 Mon Sep 17 00:00:00 2001
From: Dion <qdraw@users.noreply.github.com>
Date: Mon, 18 Nov 2024 14:41:25 +0100
Subject: [PATCH] #1833 mirror scripts copy from PR

---
 starsky-tools/build-tools/.gitignore          |   6 +
 .../build-tools/download-mirror-exiftool.sh   |  59 ++++++++
 .../build-tools/download-mirror-ffmpeg.sh     | 127 ++++++++++++++++++
 .../build-tools/download-mirror-geonames.sh   |  48 +++++++
 starsky-tools/build-tools/download-mirror.sh  |  21 +++
 5 files changed, 261 insertions(+)
 create mode 100644 starsky-tools/build-tools/download-mirror-exiftool.sh
 create mode 100644 starsky-tools/build-tools/download-mirror-ffmpeg.sh
 create mode 100644 starsky-tools/build-tools/download-mirror-geonames.sh
 create mode 100644 starsky-tools/build-tools/download-mirror.sh

diff --git a/starsky-tools/build-tools/.gitignore b/starsky-tools/build-tools/.gitignore
index e7bf2596f8..dccd478c7d 100644
--- a/starsky-tools/build-tools/.gitignore
+++ b/starsky-tools/build-tools/.gitignore
@@ -1 +1,7 @@
 create-docusaurus-tmp-folder/**
+
+mirror/ffmpeg/*.zip
+mirror/exiftool/*.zip
+mirror/exiftool/*.tar.gz
+mirror/geonames/**
+
diff --git a/starsky-tools/build-tools/download-mirror-exiftool.sh b/starsky-tools/build-tools/download-mirror-exiftool.sh
new file mode 100644
index 0000000000..1b341cd3d5
--- /dev/null
+++ b/starsky-tools/build-tools/download-mirror-exiftool.sh
@@ -0,0 +1,59 @@
+#!/bin/bash
+
+EXIFTOOL_DOMAIN="https://exiftool.org"
+EXIFTOOL_CHECKSUMS_API=$EXIFTOOL_DOMAIN"/checksums.txt"
+BINARY_FOLDERNAME="mirror/exiftool"
+INDEX_FILE="checksums.txt"
+
+LAST_CHAR_EXIFTOOL_DOMAIN=${EXIFTOOL_DOMAIN:length-1:1}
+[[ $LAST_CHAR_EXIFTOOL_DOMAIN != "/" ]] && EXIFTOOL_DOMAIN="$EXIFTOOL_DOMAIN/"; :
+USER_AGENT="Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.3"
+
+SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )"
+LAST_CHAR_SCRIPT_DIR=${SCRIPT_DIR:length-1:1}
+[[ $LAST_CHAR_SCRIPT_DIR != "/" ]] && SCRIPT_DIR="$SCRIPT_DIR/"; :
+
+LAST_CHAR_SCRIPT_DIR=${SCRIPT_DIR:length-1:1}
+[[ $LAST_CHAR_SCRIPT_DIR != "/" ]] && SCRIPT_DIR="$SCRIPT_DIR/"; :
+
+LAST_CHAR_BINARY_FOLDERNAME=${BINARY_FOLDERNAME:length-1:1}
+[[ $LAST_CHAR_BINARY_FOLDERNAME != "/" ]] && BINARY_FOLDERNAME="$BINARY_FOLDERNAME/"; :
+
+# Fetch the JSON data
+EXIFTOOL_JSON=$(curl -s $EXIFTOOL_CHECKSUMS_API)
+
+# Extract the latest version
+LATEST_EXIFTOOL_VERSION=$(echo "$EXIFTOOL_JSON" | grep -o '\b[0-9]\+\.[0-9]\+\b' | head -n 1)
+
+# Extract the Linux tar.gz filename using the dynamically fetched version
+LINUX_EXIFTOOL=$(echo "$EXIFTOOL_JSON" | grep -o "Image-ExifTool-${LATEST_EXIFTOOL_VERSION}.tar.gz" | head -n 1)
+WINDOWS_EXIFTOOL=$(echo "$EXIFTOOL_JSON" | grep -o "exiftool-${LATEST_EXIFTOOL_VERSION}_64.zip" | head -n 1)
+
+CHECK_FILES=($LINUX_EXIFTOOL $WINDOWS_EXIFTOOL)
+
+rm -rf $SCRIPT_DIR$BINARY_FOLDERNAME
+mkdir -p $SCRIPT_DIR$BINARY_FOLDERNAME
+cd $SCRIPT_DIR$BINARY_FOLDERNAME
+
+curl -L -A "$USER_AGENT" -O "$EXIFTOOL_DOMAIN$LINUX_EXIFTOOL"
+curl -L -A "$USER_AGENT" -O "$EXIFTOOL_DOMAIN$WINDOWS_EXIFTOOL"
+curl -L -A "$USER_AGENT" -O "$EXIFTOOL_CHECKSUMS_API"
+
+
+if [ ${#CHECK_FILES[@]} -ne 2 ]; then
+  echo "⛌ FAIL CHECK_FILES does not contain exactly two items. Exiting..."
+  exit 1
+fi
+
+for CHECK_FILE in "${CHECK_FILES[@]}"; do
+  if [ -f "$SCRIPT_DIR$BINARY_FOLDERNAME$CHECK_FILE" ] && [ "$(stat -c%s "$SCRIPT_DIR$BINARY_FOLDERNAME$CHECK_FILE" 2>/dev/null || stat -f%z "$SCRIPT_DIR$BINARY_FOLDERNAME$CHECK_FILE")" -gt 6300000 ]; then
+    echo "✅ $CHECK_FILE exists and is larger than 7 MB."
+  elif [ -f "$SCRIPT_DIR$BINARY_FOLDERNAME$CHECK_FILE" ]; then
+    echo "⛌ FAIL -> $CHECK_FILE exists but is 7 MB or smaller."
+    exit 1
+  else
+    echo "⛌ FAIL -> $CHECK_FILE does not exist."
+    echo "                 $SCRIPT_DIR$BINARY_FOLDERNAME$CHECK_FILE is missing."
+    exit 1
+  fi
+done
\ No newline at end of file
diff --git a/starsky-tools/build-tools/download-mirror-ffmpeg.sh b/starsky-tools/build-tools/download-mirror-ffmpeg.sh
new file mode 100644
index 0000000000..e3e06dc56e
--- /dev/null
+++ b/starsky-tools/build-tools/download-mirror-ffmpeg.sh
@@ -0,0 +1,127 @@
+#!/bin/bash
+FFBINARIES_API="https://ffbinaries.com/api/v1/version/6.1"
+OSX_ARM64_URL="https://www.osxexperts.net/ffmpeg71arm.zip"
+OSX_ARM64_NAME="osx-arm64"
+SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )"
+BINARY_FOLDERNAME="mirror/ffmpeg"
+INDEX_FILE="index.json"
+CHECK_FILES=("ffmpeg-linux-64.zip" "ffmpeg-linux-arm-64.zip" "ffmpeg-linux-armhf-32.zip" "ffmpeg-macos-64.zip" "ffmpeg-win-64.zip" "ffmpeg71arm.zip")
+
+LAST_CHAR_SCRIPT_DIR=${SCRIPT_DIR:length-1:1}
+[[ $LAST_CHAR_SCRIPT_DIR != "/" ]] && SCRIPT_DIR="$SCRIPT_DIR/"; :
+
+LAST_CHAR_BINARY_FOLDERNAME=${BINARY_FOLDERNAME:length-1:1}
+[[ $LAST_CHAR_BINARY_FOLDERNAME != "/" ]] && BINARY_FOLDERNAME="$BINARY_FOLDERNAME/"; :
+
+INDEX_FILE_PATH=$SCRIPT_DIR$BINARY_FOLDERNAME$INDEX_FILE
+
+
+# Fetch the JSON data
+FFBINARIES_JSON=$(curl -s $FFBINARIES_API)
+
+# Create a directory to store the binaries
+echo "Cleaning up previous binaries... $SCRIPT_DIR$BINARY_FOLDERNAME"
+rm -rf $SCRIPT_DIR$BINARY_FOLDERNAME
+mkdir -p $SCRIPT_DIR$BINARY_FOLDERNAME
+cd $SCRIPT_DIR$BINARY_FOLDERNAME
+
+# Initialize JSON output
+OUTPUT_JSON="{\"binaries\":["
+
+ARCHITECTURES=$(echo "$FFBINARIES_JSON" | grep -o '"[^"]\+":{"ffmpeg"' | sed 's/"\([^"]\+\)".*/\1/')
+BINARY_URLS=$(echo "$FFBINARIES_JSON" | grep -o '"ffmpeg":"[^"]*"' | sed 's/"ffmpeg":"//;s/"//')
+
+MAP_ARCHITECTURE_NAME () {
+  if [ "$1" == "windows-64" ]; then
+    echo "windows-x64"
+  elif [ "$1" == "linux-64" ]; then
+    echo "linux-x64"
+  elif [ "$1" == "linux-armhf" ]; then
+    echo "linux-arm"
+  elif [ "$1" == "osx-64" ]; then
+    echo "osx-x64"
+  else
+    echo $1
+  fi
+}
+
+populate_array_from_variable() {
+  local variable_content="$1"
+  local array_name="$2"
+
+  IFS=$'\n' read -rd '' -a temp_array <<< "$variable_content"
+  eval "$array_name=(\"\${temp_array[@]}\")"
+}
+
+# Initialize arrays
+ARCHITECTURES_ARRAY=()
+BINARY_URLS_ARRAY=()
+
+# Populate arrays using the function
+populate_array_from_variable "$ARCHITECTURES" ARCHITECTURES_ARRAY
+populate_array_from_variable "$BINARY_URLS" BINARY_URLS_ARRAY
+
+
+for i in "${!ARCHITECTURES_ARRAY[@]}"; do
+
+  ARCHITECTURE="${ARCHITECTURES_ARRAY[$i]}"
+  # skip if linux-armel or linux-32
+  if [ "$ARCHITECTURE" == "linux-32" ] || [ "$ARCHITECTURE" == "linux-armel" ]; then
+    continue
+  fi
+    
+  echo "Processing architecture: $ARCHITECTURE..."
+  
+  URL="${BINARY_URLS_ARRAY[$i]}"
+
+  # Extract architecture and URL
+  FILENAME=$(basename "$URL")
+  FILENAME_UPDATED=$(echo "$FILENAME" | sed -E "s/[-.]([0-9]+\.[0-9]+)[-.]/-/")
+
+  CURRENT_ARCHITECTURE="$(MAP_ARCHITECTURE_NAME $ARCHITECTURE)"
+
+  # Download the binary
+  echo "Downloading $URL for $CURRENT_ARCHITECTURE [$FILENAME] $FILENAME_UPDATED..."
+  curl -L -O "$URL"
+
+  FILE_HASH=$(openssl dgst -sha256 "$FILENAME" | awk '{print $2}')
+
+  mv "$FILENAME" "$FILENAME_UPDATED"
+
+  # Add to output JSON
+  OUTPUT_JSON="${OUTPUT_JSON}{\"architecture\":\"$CURRENT_ARCHITECTURE\",\"url\":\"$FILENAME_UPDATED\",\"sha256\":\"$FILE_HASH\"},"
+done
+
+# Add osx-arm64 explicitly
+echo "Adding custom architecture $OSX_ARM64_NAME with URL $OSX_ARM64_URL..."
+curl -L -O "$OSX_ARM64_URL"
+
+OSX_ARM64_FILENAME=$(basename "$OSX_ARM64_URL")
+OSX_ARM64_HASH=$(openssl dgst -sha256 "$OSX_ARM64_FILENAME" | awk '{print $2}')
+OUTPUT_JSON="${OUTPUT_JSON}{\"architecture\":\"$OSX_ARM64_NAME\",\"url\":\"$OSX_ARM64_FILENAME\",\"sha256\":\"$OSX_ARM64_HASH\"},"
+
+# Finalize JSON
+OUTPUT_JSON="${OUTPUT_JSON%,}]}" # Remove trailing comma and close the JSON array
+
+# Write JSON to file
+echo "$OUTPUT_JSON" > $INDEX_FILE_PATH
+
+echo "All ffmpeg binaries downloaded successfully."
+
+node -e "console.log(JSON.stringify(JSON.parse(require('fs') \
+      .readFileSync(process.argv[1])), null, 4));" $INDEX_FILE_PATH > $INDEX_FILE_PATH.bak
+mv $INDEX_FILE_PATH.bak $INDEX_FILE_PATH
+
+for CHECK_FILE in "${CHECK_FILES[@]}"; do
+  if [ -f "$SCRIPT_DIR$BINARY_FOLDERNAME$CHECK_FILE" ] && [ "$(stat -c%s "$SCRIPT_DIR$BINARY_FOLDERNAME$CHECK_FILE" 2>/dev/null || stat -f%z "$SCRIPT_DIR$BINARY_FOLDERNAME$CHECK_FILE")" -gt 17874368 ]; then
+    echo "✅ $CHECK_FILE exists and is larger than 17 MB."
+  elif [ -f "$SCRIPT_DIR$BINARY_FOLDERNAME$CHECK_FILE" ]; then
+    echo "⛌ FAIL -> $CHECK_FILE exists but is 17 MB or smaller."
+    exit 1
+  else
+    echo "⛌ FAIL -> $CHECK_FILE does not exist."
+    exit 1
+  fi
+done
+
+echo "URLs with architectures saved to $INDEX_FILE_PATH"
diff --git a/starsky-tools/build-tools/download-mirror-geonames.sh b/starsky-tools/build-tools/download-mirror-geonames.sh
new file mode 100644
index 0000000000..31bd336f17
--- /dev/null
+++ b/starsky-tools/build-tools/download-mirror-geonames.sh
@@ -0,0 +1,48 @@
+#!/bin/bash
+GEONAMES_DUMP="https://download.geonames.org/export/dump/"
+BINARY_FOLDERNAME="mirror/geonames"
+ADMIN1_CODES="admin1CodesASCII.txt"
+CITIES1000="cities1000.zip"
+
+CHECK_FILES=($ADMIN1_CODES $CITIES1000)
+
+SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )"
+LAST_CHAR_SCRIPT_DIR=${SCRIPT_DIR:length-1:1}
+[[ $LAST_CHAR_SCRIPT_DIR != "/" ]] && SCRIPT_DIR="$SCRIPT_DIR/"; :
+
+LAST_CHAR_GEONAMES_DUMP=${GEONAMES_DUMP:length-1:1}
+[[ $LAST_CHAR_GEONAMES_DUMP != "/" ]] && GEONAMES_DUMP="$GEONAMES_DUMP/"; :
+LAST_CHAR_BINARY_FOLDERNAME=${BINARY_FOLDERNAME:length-1:1}
+[[ $LAST_CHAR_BINARY_FOLDERNAME != "/" ]] && BINARY_FOLDERNAME="$BINARY_FOLDERNAME/"; :
+
+rm -rf $SCRIPT_DIR$BINARY_FOLDERNAME
+mkdir -p $SCRIPT_DIR$BINARY_FOLDERNAME
+cd $SCRIPT_DIR$BINARY_FOLDERNAME
+
+curl -L -O "$GEONAMES_DUMP$ADMIN1_CODES"
+curl -L -O "$GEONAMES_DUMP$CITIES1000"
+
+for CHECK_FILE in "${CHECK_FILES[@]}"; do
+  # Construct the full file path
+  FULL_PATH="${SCRIPT_DIR}${BINARY_FOLDERNAME}${CHECK_FILE}"
+  
+  # Check if the file exists
+  if [ ! -f "$FULL_PATH" ]; then
+    echo "$CHECK_FILE does not exist."
+    echo "                 $FULL_PATH is missing."
+    exit 1
+  fi
+  
+  # Get file size
+  FILE_SIZE=$(stat -c%s "$FULL_PATH" 2>/dev/null || stat -f%z "$FULL_PATH")
+
+  # Check conditions
+  if [ "$FILE_SIZE" -gt 147000 ] && [ "$CHECK_FILE" == "$ADMIN1_CODES" ]; then
+    echo "✅ $CHECK_FILE contains $ADMIN1_CODES and is at least 150 KB /0.14mb."
+  elif [ "$FILE_SIZE" -gt 9300000 ]; then
+    echo "✅ $CHECK_FILE exists and is larger than 8 MB."
+  else
+    echo "$CHECK_FILE exists but does not meet size requirements."
+    exit 1
+  fi
+done
diff --git a/starsky-tools/build-tools/download-mirror.sh b/starsky-tools/build-tools/download-mirror.sh
new file mode 100644
index 0000000000..b2383e9631
--- /dev/null
+++ b/starsky-tools/build-tools/download-mirror.sh
@@ -0,0 +1,21 @@
+#!/bin/bash
+
+GH_SCRIPT_DIR="https://raw.githubusercontent.com/qdraw/starsky/refs/heads/feature/202411_ffmpeg/starsky-tools/build-tools/"
+
+LAST_CHAR_GH_SCRIPT_DIR=${GH_SCRIPT_DIR:length-1:1}
+[[ $LAST_CHAR_GH_SCRIPT_DIR != "/" ]] && GH_SCRIPT_DIR="$GH_SCRIPT_DIR/"; :
+
+SCRIPT_FILES=("download-mirror-exiftool.sh" "download-mirror-ffmpeg.sh" "download-mirror-geonames.sh")
+
+# Loop through the array
+for SCRIPT_FILE in "${SCRIPT_FILES[@]}"; do
+  echo "Processing $item..."
+  # Example action: Check if the file exists
+  if [ -f "$SCRIPT_FILE" ]; then
+    echo "$SCRIPT_FILE exists."
+  else
+    echo "$SCRIPT_FILE does not exist."
+    curl -o "$SCRIPT_FILE" "$GH_SCRIPT_DIR$SCRIPT_FILE"
+  fi
+  bash $SCRIPT_FILE
+done