Auto-expire mastodon posts (#199)

README-Mastodon.md

@@ -44,8 +44,7 @@ Once you have an account, please do the following:
 ![image](https://user-images.githubusercontent.com/15090643/208438462-b40cc847-f36c-4db7-bacb-54a68fae2cff.png)
 ![image](https://user-images.githubusercontent.com/15090643/208438987-3e1fd9c2-5ce9-46c0-92e9-20bb78f55a8c.png)
 
-Note -- if you post lots of traffic to Mastodon, please consider adding an Automatic Post Deletion time of 1 week.
-This will help manage storage costs for the operator of the Mastodon server!
+Note -- the `MASTODON_RETENTION_TIME` parameter in `planefence.config` determines the retention time (in days) of any Toots you send to Mastodon. This will help manage storage costs for the operator of the Mastodon server! Note - the default expiration time if the parameter is omitted, is `7` (days). If you want your Toots to never expire, please set the parameter value to `off` or `0`. However, please be cognizant that your Mastodon Server operator is probably paying for disk storage out of their own pocket - so please leave this retention time as short as you can afford.
 
 ## Configuring Planefence to use Mastodon
 

rootfs/etc/s6-overlay/scripts/cleanup

@@ -1,15 +1,9 @@
 #!/command/with-contenv bash
-#shellcheck shell=bash
+#shellcheck shell=bash disable=SC1091,SC2154,SC2015
 
-# redirect stderr to stdout so it's picked up in the docker logs
-exec 2>&1
-# all errors will show a line number and the command used to produce the error
-SCRIPT_PATH="$( cd -- "$(dirname "$0")" >/dev/null 2>&1 ; pwd)/$(basename "$0")"
-trap 'echo -e "[ERROR] $SCRIPT_PATH in line $LINENO when executing: $BASH_COMMAND"' ERR
+source /scripts/common
 
-APPNAME="$(hostname)/cleanup"
-
-[[ "$LOGLEVEL" != "ERROR" ]] && echo "[$APPNAME][$(date)] Cleanup started as an s6 service" || true
+[[ "${LOGLEVEL,,}" != "error" ]] && "${s6wrap[@]}" echo "Cleanup started as an s6 service" || true
 
 # -----------------------------------------------------------------------------------
 # Copyright 2020-2024 Ramon F. Kolb - licensed under the terms and conditions
@@ -28,11 +22,16 @@ LOOPTIME="3h"
 # It specifically applies to files in the HTML directory.
 # If $PF_DELETEAFTER is set to "0" then we never delete.
 # Note - files in /tmp will get deleted if they are older than 2 days.
-PF_DELETEAFTER=$(grep "^PF_DELETEAFTER" /usr/share/planefence/persist/planefence.config 2>/dev/null| awk -F "=" '{ print $2 }')
-[[ "x$PF_DELETEAFTER" == "x" ]] && OLDERTHAN=14 || OLDERTHAN=$PF_DELETEAFTER
 
-PF_CHECKREMOTEDB=$(grep "^PF_CHECKREMOTEDB" /usr/share/planefence/persist/planefence.config 2>/dev/null| awk -F "=" '{ print $2 }')
 
+if [[ -f /usr/share/planefence/persist/planefence.config ]]; then
+  source /usr/share/planefence/persist/planefence.config
+else
+  "${s6wrap[@]}" echo "[ERROR] - Can't find /usr/share/planefence/persist/planefence.config which is needed for this app to run"
+  exit 1
+fi
+
+OLDERTHAN=${PF_DELETEAFTER:-14}
 #
 # MAXLOGLINES contains the max number of lines that we will keep in /tmp/planefence.log
 MAXLOGLINES=500
@@ -43,19 +42,19 @@ CLEANUP ()
 {
   if (( OLDERTHAN > 0 ))
   then
-    [[ "$LOGLEVEL" != "ERROR" ]] && echo "[$APPNAME][$(date)] Cleaning up web files older than $OLDERTHAN days" || true
-    [[ "$LOGLEVEL" != "ERROR" ]] && echo "[$APPNAME][$(date)] Cleaning up logs and tmp files older than 1 day" || true
-    [[ "$LOGLEVEL" != "ERROR" ]] && echo "[$APPNAME][$(date)] Reducing planefence and noise2capt logs to $MAXLOGLINES lines (if needed)" || true
+    [[ "${LOGLEVEL,,}" != "error" ]] && "${s6wrap[@]}" echo "Cleaning up web files older than $OLDERTHAN days" || true
+    [[ "${LOGLEVEL,,}" != "error" ]] && "${s6wrap[@]}" echo "Cleaning up logs and tmp files older than 1 day" || true
+    [[ "${LOGLEVEL,,}" != "error" ]] && "${s6wrap[@]}" echo "Reducing planefence and noise2capt logs to $MAXLOGLINES lines (if needed)" || true
 
     #these are temp actions to facilitate the transition to the new directory structure:
     mkdir -p /usr/share/planefence/persist/.internal
     mv -f /usr/share/planefence/persist/*.tmp /usr/share/planefence/persist/.internal 2>/dev/null
     mv -f /usr/share/planefence/persist/*.log /usr/share/planefence/persist/.internal 2>/dev/null
     mv -f /usr/share/planefence/persist/planeownerscache.txt /usr/share/planefence/persist/.internal 2>/dev/null
 
-    touch -t $(date -d "yesterday 00:00:00" +%Y%m%d%H%M) /tmp/timestamp
-    find /usr/share/planefence/html/plane*{.html,.js,.csv} -mtime +$OLDERTHAN -delete 2>/dev/null
-    find /usr/share/planefence/html/noise*.png -mtime +$OLDERTHAN -delete 2>/dev/null
+    touch -t "$(date -d "yesterday 00:00:00" +%Y%m%d%H%M)" /tmp/timestamp
+    find /usr/share/planefence/html/plane*{.html,.js,.csv} -mtime +"$OLDERTHAN" -delete 2>/dev/null
+    find /usr/share/planefence/html/noise*.png -mtime +"$OLDERTHAN" -delete 2>/dev/null
     rm -f /run/socket30003/*.log
     find /run/socket30003/*.txt -type f ! -newer /tmp/timestamp -delete 2>/dev/null
     find /usr/share/planefence/persist/.internal/*.tmp -type f ! -newer /tmp/timestamp -delete 2>/dev/null
@@ -70,7 +69,7 @@ CLEANUP ()
         fdate=${f: -10:6}   # get the date component from the file name
 
         # remove anything older than yesterday:
-        (( fdate < $(date -d yesterday +%y%m%d) )) && rm -v -f $f && continue
+        (( fdate < $(date -d yesterday +%y%m%d) )) && rm -v -f "$f" && continue
     done
 
 
@@ -99,7 +98,7 @@ CLEANUP ()
     sed -i '/^$/d' /usr/share/planefence/persist/planefence-ignore.txt 2>/dev/null
 
   else
-    [[ "$LOGLEVEL" != "ERROR" ]] && echo "[$APPNAME][$(date)] $LOOPTIME set to 0 - cleanup skipped" || true
+    [[ "${LOGLEVEL,,}" != "error" ]] && "${s6wrap[@]}" echo "$LOOPTIME set to 0 - cleanup skipped" || true
   fi
 }
 
@@ -109,38 +108,32 @@ GET_ICAO_DB ()
 {
     if (( $(date -r /run/planefence/icao2plane.txt +%s 2>/dev/null || echo 0) < $(date -d "next monday - 7 days" +%s) ))
     then
-        [[ "$LOGLEVEL" != "ERROR" ]] && echo "[$APPNAME][$(date)] Retrieving ICAO to TAIL database from https://www.mictronics.de ... " || true
+        [[ "${LOGLEVEL,,}" != "error" ]] && "${s6wrap[@]}" echo "Retrieving ICAO to TAIL database from https://www.mictronics.de ... " || true
         # note - the curl won't fail, even if the file is not found because the PHP page doesn't return a 400 code but a regular result webpage
-        curl -s -L -f -o /tmp/icao24plus.zip https://www.mictronics.de/aircraft-database/icao24plus.php
-        if [[ "$?" != "0" ]]
-        then
-            echo "[$APPNAME][$(date)] Retrieving ICAO to TAIL database from https://www.mictronics.de FAILED!"
+        if ! curl -s -L -f -o /tmp/icao24plus.zip https://www.mictronics.de/aircraft-database/icao24plus.php; then
+            "${s6wrap[@]}" echo "Retrieving ICAO to TAIL database from https://www.mictronics.de FAILED!"
             return 1
         fi
 
-        [[ "$LOGLEVEL" != "ERROR" ]] && echo "[$APPNAME][$(date)] unzipping ... " || true
+        [[ "${LOGLEVEL,,}" != "error" ]] && "${s6wrap[@]}" echo "unzipping ... " || true
         # unzipping *will* fail when the retrieved file is not a valid ZIP file
-        unzip -qq -d /tmp -o /tmp/icao24plus.zip 2>/dev/null
-        if [[ "$?" != "0" ]]
-        then
-            echo "[$APPNAME][$(date)] Unzipping ICAO to TAIL database from https://www.mictronics.de FAILED!"
+        if ! unzip -qq -d /tmp -o /tmp/icao24plus.zip 2>/dev/null; then
+            "${s6wrap[@]}" echo "Unzipping ICAO to TAIL database from https://www.mictronics.de FAILED!"
             return 1
         fi
 
         rm -f /tmp/icao24plus.zip
         # now convert the text based file into a CSV file
-        [[ "$LOGLEVEL" != "ERROR" ]] && echo "[$APPNAME][$(date)] converting ... " || true
-        sed -i 's|\([0-9A-F]\{6\}\)\s*\([A-Z0-9\-]*\)\s*\([A-Z0-9]*\)\s*\(.*\)|\1,\2,\3,\4|g' /tmp/icao24plus.txt
-        if [[ "$?" != "0" ]]
-        then
-            echo "[$APPNAME][$(date)] Converting ICAO to TAIL database from https://www.mictronics.de FAILED!"
+        [[ "${LOGLEVEL,,}" != "error" ]] && "${s6wrap[@]}" echo "converting ... " || true
+        if ! sed -i 's|\([0-9A-F]\{6\}\)\s*\([A-Z0-9\-]*\)\s*\([A-Z0-9]*\)\s*\(.*\)|\1,\2,\3,\4|g' /tmp/icao24plus.txt; then
+            "${s6wrap[@]}" echo "Converting ICAO to TAIL database from https://www.mictronics.de FAILED!"
             return 1
         fi
 
         mv -f /tmp/icao24plus.txt /run/planefence/icao2plane.txt
-        [[ "$LOGLEVEL" != "ERROR" ]] && echo "[$APPNAME][$(date)] done!" || true
+        [[ "${LOGLEVEL,,}" != "error" ]] && "${s6wrap[@]}" echo "done!" || true
     else
-        [[ "$LOGLEVEL" != "ERROR" ]] && echo "[$APPNAME][$(date)] ICAO to TAIL database is up to date" || true
+        [[ "${LOGLEVEL,,}" != "error" ]] && "${s6wrap[@]}" echo "ICAO to TAIL database is up to date" || true
     fi
 }
 #
@@ -152,14 +145,14 @@ GET_AIRLINE_DB ()
 {
     if [[ -f /usr/share/planefence/persist/airlinecodes.txt ]] && [[ $(find /usr/share/planefence/persist/airlinecodes.txt -mtime -7 | wc -l) == 1 ]]; then
         # file exists and is less than 7 days old
-        [[ "$LOGLEVEL" != "ERROR" ]] && echo "[$APPNAME][$(date)] Airlinecodes database is up to date" || true
+        [[ "${LOGLEVEL,,}" != "error" ]] && "${s6wrap[@]}" echo "Airlinecodes database is up to date" || true
         return 0
     fi
 
-    [[ "$LOGLEVEL" != "ERROR" ]] && echo "[$APPNAME][$(date)] Updating airline names database..." || true
+    [[ "${LOGLEVEL,,}" != "error" ]] && "${s6wrap[@]}" echo "Updating airline names database..." || true
     rm -f /tmp/airlinecodes.txt /tmp/airlines.csv /tmp/airlinecodes.txt.tmp
-    curl --compressed -s -L -f https://raw.githubusercontent.com/kx1t/planefence-airlinecodes/main/airlinecodes.txt -o /tmp/airlinecodes.txt && [[ "$LOGLEVEL" != "ERROR" ]] && echo "[$APPNAME][$(date)] Got kx1t/planefence-airlinecodes" || true
-    curl --compressed -s -L -f https://raw.githubusercontent.com/jbroutier/whatisflying-db/master/data/airlines.csv -o /tmp/airlines.csv && [[ "$LOGLEVEL" != "ERROR" ]] && echo "[$APPNAME][$(date)] Got jbroutier/whatisflying-db" || true
+    curl --compressed -s -L -f https://raw.githubusercontent.com/kx1t/planefence-airlinecodes/main/airlinecodes.txt -o /tmp/airlinecodes.txt && [[ "${LOGLEVEL,,}" != "error" ]] && "${s6wrap[@]}" echo "Got kx1t/planefence-airlinecodes" || true
+    curl --compressed -s -L -f https://raw.githubusercontent.com/jbroutier/whatisflying-db/master/data/airlines.csv -o /tmp/airlines.csv && [[ "${LOGLEVEL,,}" != "error" ]] && "${s6wrap[@]}" echo "Got jbroutier/whatisflying-db" || true
 
     #convert JBroutier's DB into our format:
     if [[ -f /tmp/airlines.csv ]]
@@ -173,29 +166,32 @@ GET_AIRLINE_DB ()
 	#echo "header has ${#header[@]} elements and contains:"
 	#echo ${header[@]}
 	#echo "icao=$icao name=$name country=$country callsign=$callsign"
-        if [[ ! -z "$icao" ]] && [[ ! -z "$name" ]] && [[ ! -z "$country" ]] && [[ ! -z "$callsign" ]]
+        if [[ -n "$icao" ]] && [[ -n "$name" ]] && [[ -n "$country" ]] && [[ -n "$callsign" ]]
         then
             tail -n +2 /tmp/airlines.csv | awk -F "," -v icao=$(( icao + 1 )) -v name=$(( name + 1 )) -v country=$(( country + 1 )) -v callsign=$(( callsign + 1 )) '{print $icao "," $name "," $callsign "," $country}' >> /tmp/airlinecodes.txt.tmp
             [[ -f /tmp/airlinecodes.txt ]] && cat /tmp/airlinecodes.txt /tmp/airlinecodes.txt.tmp > /tmp/airlinecodes.txt.tmp2 || cat /tmp/airlinecodes.txt.tmp > /tmp/airlinecodes.txt.tmp2
-	        [[ "$LOGLEVEL" != "ERROR" ]] && echo "[$APPNAME][$(date)] Removed $(awk -F',' 'seen[$1]++' /tmp/airlinecodes.txt.tmp2 |wc -l) overlapping entries, left $(awk -F',' '!seen[$1]++' /tmp/airlinecodes.txt.tmp2 |wc -l) entries in place" || true
+	        [[ "${LOGLEVEL,,}" != "error" ]] && "${s6wrap[@]}" echo "Removed $(awk -F',' 'seen[$1]++' /tmp/airlinecodes.txt.tmp2 |wc -l) overlapping entries, left $(awk -F',' '!seen[$1]++' /tmp/airlinecodes.txt.tmp2 |wc -l) entries in place" || true
             awk -F',' '!seen[$1]++' /tmp/airlinecodes.txt.tmp2 >/tmp/airlinecodes.txt
         fi
     fi
-    # make sure the file is minimally disrupte and programs who have still have it open don't have the data connected to the inode modified
+    # make sure the file is minimally disrupted and programs who have still have it open don't have the data connected to the inode modified
     mv -f /tmp/airlinecodes.txt /usr/share/planefence/persist/airlinecodes.txt.tmp
     rm -f /usr/share/planefence/persist/airlinecodes.txt
     mv -f /usr/share/planefence/persist/airlinecodes.txt.tmp /usr/share/planefence/persist/airlinecodes.txt
     rm -f /tmp/airlines.csv /tmp/airlinecodes.txt.tmp /tmp/airlinecodes.txt.tmp2
-    [[ "$LOGLEVEL" != "ERROR" ]] && echo "[$APPNAME][$(date)] Airlinecodes database update complete" || true
+    [[ "${LOGLEVEL,,}" != "error" ]] && "${s6wrap[@]}" echo "Airlinecodes database update complete" || true
 }
 
 #Now loop forevah:
 while true
 do
-  [[ "$LOGLEVEL" != "ERROR" ]] && echo "[$APPNAME][$(date)] Performing a cleanup run..." || true
+  [[ "${LOGLEVEL,,}" != "error" ]] && "${s6wrap[@]}" echo "Performing a cleanup run..." || true
   CLEANUP
   GET_ICAO_DB
   GET_AIRLINE_DB
-  [[ "$LOGLEVEL" != "ERROR" ]] && echo "[$APPNAME][$(date)] Cleanup done. Sleeping for $LOOPTIME" || true
+  if [[ -n "$MASTODON_SERVER$MASTODON_ACCESS_TOKEN" ]]; then 
+    /scripts/masto_expire.sh delete
+  fi
+  [[ "${LOGLEVEL,,}" != "error" ]] && "${s6wrap[@]}" echo "Cleanup done. Sleeping for $LOOPTIME" || true
   sleep $LOOPTIME
 done

rootfs/scripts/masto_expire.sh

@@ -1,5 +1,5 @@
 #!/command/with-contenv bash
-#shellcheck shell=bash disable=SC1091,SC2174
+#shellcheck shell=bash disable=SC1091,SC2174,SC2015,SC2154
 # -----------------------------------------------------------------------------------
 # Copyright 2024 Ramon F. Kolb - licensed under the terms and conditions
 # of GPLv3. The terms and conditions of this license are included with the Github
@@ -9,25 +9,42 @@
 # This package may incorporate other software and license terms.
 # -----------------------------------------------------------------------------------
 
-if [[ -f /usr/share/planefence/persist/planefence.config ]]; then source /usr/share/planefence/persist/planefence.config; fi
+if [[ -f /usr/share/planefence/persist/planefence.config ]]; then
+    source /usr/share/planefence/persist/planefence.config
+fi
+
+source /scripts/common
 
 ACCESS_TOKEN=$MASTODON_ACCESS_TOKEN
 INSTANCE_URL="https://$MASTODON_SERVER"
 
-RETENTION_DAYS="${MASTODON_RETENTION_TIME}"
+RETENTION_DAYS="${MASTODON_RETENTION_TIME:-7}"
 
 delete_toot() {
     local toot_id="$1"
     local result
-    if result="$(curl -s --fail -X DELETE -H "Authorization: Bearer $ACCESS_TOKEN" "$INSTANCE_URL/api/v1/statuses/$toot_id" 2>&1)"; then
-        echo "successfully deleted"
-    else
-        echo "error: $result"
+    if ! result="$(curl -s --fail -X DELETE -H "Authorization: Bearer $ACCESS_TOKEN" "$INSTANCE_URL/api/v1/statuses/$toot_id" 2>&1)"; then
+        echo ""
+        "${s6wrap[@]}" echo "error deleting $toot_id: $result"
     fi
 }
 
-if [[ -z "$RETENTION_DAYS" ]]; then
-    echo "RETENTION_DAYS not set. Exiting."
+if chk_disabled "$MASTODON_RETENTION_TIME"; then
+    "${s6wrap[@]}" echo "MASTODON_RETENTION_TIME is set to $MASTODON_RETENTION_TIME (disabled); nothing to do!"
+    exit 0
+fi
+
+if [[ -z "$MASTODON_RETENTION_TIME" ]]; then
+    "${s6wrap[@]}" echo "Warning: MASTODON_RETENTION_TIME not set. Defaulting to $RETENTION_DAYS days."
+fi
+
+if [[ -z "$MASTODON_ACCESS_TOKEN" ]]; then
+    "${s6wrap[@]}" echo "MASTODON_ACCESS_TOKEN not set. Exiting."
+    exit 1
+fi
+
+if [[ -z "$MASTODON_SERVER" ]]; then
+    "${s6wrap[@]}" echo "MASTODON_SERVER not set. Exiting."
     exit 1
 fi
 
@@ -37,37 +54,42 @@ declare -A toot_dates
 now="$(date +%s)"
 
 masto_id="$(curl -s -H "Authorization: Bearer $ACCESS_TOKEN" "$INSTANCE_URL/api/v1/accounts/verify_credentials" | jq -r '.id')"
-
+counter=0
 while : ; do
-    echo -n "Indexing Media IDs round $((++counter))"
+    expired=0
+    unexpired=0
+    oldest=33000000000
+    newest=0
+    output=("Indexing Toots")
     toots="$(curl -s -H "Authorization: Bearer $ACCESS_TOKEN" "$INSTANCE_URL/api/v1/accounts/$masto_id/statuses?limit=40${last_id:+&max_id=}${last_id}")"
     # shellcheck disable=SC2207
     toot_ids=($(jq -r '.[] | .id' <<< "$toots" 2>/dev/null))
     if (( ${#toot_ids[@]} == 0)); then
-        echo "No more toots, we are done!"
-        exit
+        "${s6wrap[@]}" echo "No more Toots; done!"
+        exit 0
     fi
     last_id="${toot_ids[-1]}"
-    echo " ${#toot_ids[@]} toots"
+
+    output+=("$((counter+1)) - $((counter+${#toot_ids[@]})) (${#toot_ids[@]} toots).")
+    (( counter+=${#toot_ids[@]} )) || true
     for t in "${toot_ids[@]}"; do
         if [[ -z "${toot_dates[$t]}" ]]; then
             toot_dates[$t]="$(date -d "$(jq -r 'map(select(.id == "'"$t"'"))[].created_at'  <<< "$toots")" +%s)"
-            echo -n "$t --> $(date -d @"${toot_dates[$t]}") "
+            if (( toot_dates[$t] < oldest )); then oldest="${toot_dates[$t]}"; fi
+            if (( toot_dates[$t]  > newest )); then newest="${toot_dates[$t]}"; fi
             if (( (now - toot_dates[$t])/(60*60*24) > RETENTION_DAYS )); then
-                echo -n " expired (age: $(( (now - toot_dates[$t])/(60*60*24) )) days): "
+                (( expired++ )) || true
                 if [[ "$1" == "delete" ]]; then
-                    echo -n "deleting... "
                     delete_toot "$t";
-                else
-                    echo "(not deleted)"
                 fi 
             else
-                echo " not expired (age: $(( (now
-                 - toot_dates[$t])/(60*60*24) )) days)"
+                (( unexpired++ )) || true
             fi
         else
-            echo "$t --> duplicate, we're done!"
+            "${s6wrap[@]}" echo "No more Toots; done!"
             exit
         fi
     done
+    output+=("($unexpired unexpired; $expired expired; oldest $(date -d "@$oldest") ($(( (now - oldest)/(60*60*24) )) days); newest $(date -d "@$newest") ($(( (now - newest)/(60*60*24) )) days))")
+    "${s6wrap[@]}" echo "${output[@]}"
 done