diff --git a/README-Mastodon.md b/README-Mastodon.md index 43390ec..9808a72 100644 --- a/README-Mastodon.md +++ b/README-Mastodon.md @@ -44,8 +44,7 @@ Once you have an account, please do the following: ![image](https://user-images.githubusercontent.com/15090643/208438462-b40cc847-f36c-4db7-bacb-54a68fae2cff.png) ![image](https://user-images.githubusercontent.com/15090643/208438987-3e1fd9c2-5ce9-46c0-92e9-20bb78f55a8c.png) -Note -- if you post lots of traffic to Mastodon, please consider adding an Automatic Post Deletion time of 1 week. -This will help manage storage costs for the operator of the Mastodon server! +Note -- the `MASTODON_RETENTION_TIME` parameter in `planefence.config` determines the retention time (in days) of any Toots you send to Mastodon. This will help manage storage costs for the operator of the Mastodon server! Note - the default expiration time if the parameter is omitted, is `7` (days). If you want your Toots to never expire, please set the parameter value to `off` or `0`. However, please be cognizant that your Mastodon Server operator is probably paying for disk storage out of their own pocket - so please leave this retention time as short as you can afford. ## Configuring Planefence to use Mastodon diff --git a/rootfs/etc/s6-overlay/scripts/cleanup b/rootfs/etc/s6-overlay/scripts/cleanup index 33055a6..8b069dc 100755 --- a/rootfs/etc/s6-overlay/scripts/cleanup +++ b/rootfs/etc/s6-overlay/scripts/cleanup @@ -1,15 +1,9 @@ #!/command/with-contenv bash -#shellcheck shell=bash +#shellcheck shell=bash disable=SC1091,SC2154,SC2015 -# redirect stderr to stdout so it's picked up in the docker logs -exec 2>&1 -# all errors will show a line number and the command used to produce the error -SCRIPT_PATH="$( cd -- "$(dirname "$0")" >/dev/null 2>&1 ; pwd)/$(basename "$0")" -trap 'echo -e "[ERROR] $SCRIPT_PATH in line $LINENO when executing: $BASH_COMMAND"' ERR +source /scripts/common -APPNAME="$(hostname)/cleanup" - -[[ "$LOGLEVEL" != "ERROR" ]] && echo "[$APPNAME][$(date)] Cleanup started as an s6 service" || true +[[ "${LOGLEVEL,,}" != "error" ]] && "${s6wrap[@]}" echo "Cleanup started as an s6 service" || true # ----------------------------------------------------------------------------------- # Copyright 2020-2024 Ramon F. Kolb - licensed under the terms and conditions @@ -28,11 +22,16 @@ LOOPTIME="3h" # It specifically applies to files in the HTML directory. # If $PF_DELETEAFTER is set to "0" then we never delete. # Note - files in /tmp will get deleted if they are older than 2 days. -PF_DELETEAFTER=$(grep "^PF_DELETEAFTER" /usr/share/planefence/persist/planefence.config 2>/dev/null| awk -F "=" '{ print $2 }') -[[ "x$PF_DELETEAFTER" == "x" ]] && OLDERTHAN=14 || OLDERTHAN=$PF_DELETEAFTER -PF_CHECKREMOTEDB=$(grep "^PF_CHECKREMOTEDB" /usr/share/planefence/persist/planefence.config 2>/dev/null| awk -F "=" '{ print $2 }') +if [[ -f /usr/share/planefence/persist/planefence.config ]]; then + source /usr/share/planefence/persist/planefence.config +else + "${s6wrap[@]}" echo "[ERROR] - Can't find /usr/share/planefence/persist/planefence.config which is needed for this app to run" + exit 1 +fi + +OLDERTHAN=${PF_DELETEAFTER:-14} # # MAXLOGLINES contains the max number of lines that we will keep in /tmp/planefence.log MAXLOGLINES=500 @@ -43,9 +42,9 @@ CLEANUP () { if (( OLDERTHAN > 0 )) then - [[ "$LOGLEVEL" != "ERROR" ]] && echo "[$APPNAME][$(date)] Cleaning up web files older than $OLDERTHAN days" || true - [[ "$LOGLEVEL" != "ERROR" ]] && echo "[$APPNAME][$(date)] Cleaning up logs and tmp files older than 1 day" || true - [[ "$LOGLEVEL" != "ERROR" ]] && echo "[$APPNAME][$(date)] Reducing planefence and noise2capt logs to $MAXLOGLINES lines (if needed)" || true + [[ "${LOGLEVEL,,}" != "error" ]] && "${s6wrap[@]}" echo "Cleaning up web files older than $OLDERTHAN days" || true + [[ "${LOGLEVEL,,}" != "error" ]] && "${s6wrap[@]}" echo "Cleaning up logs and tmp files older than 1 day" || true + [[ "${LOGLEVEL,,}" != "error" ]] && "${s6wrap[@]}" echo "Reducing planefence and noise2capt logs to $MAXLOGLINES lines (if needed)" || true #these are temp actions to facilitate the transition to the new directory structure: mkdir -p /usr/share/planefence/persist/.internal @@ -53,9 +52,9 @@ CLEANUP () mv -f /usr/share/planefence/persist/*.log /usr/share/planefence/persist/.internal 2>/dev/null mv -f /usr/share/planefence/persist/planeownerscache.txt /usr/share/planefence/persist/.internal 2>/dev/null - touch -t $(date -d "yesterday 00:00:00" +%Y%m%d%H%M) /tmp/timestamp - find /usr/share/planefence/html/plane*{.html,.js,.csv} -mtime +$OLDERTHAN -delete 2>/dev/null - find /usr/share/planefence/html/noise*.png -mtime +$OLDERTHAN -delete 2>/dev/null + touch -t "$(date -d "yesterday 00:00:00" +%Y%m%d%H%M)" /tmp/timestamp + find /usr/share/planefence/html/plane*{.html,.js,.csv} -mtime +"$OLDERTHAN" -delete 2>/dev/null + find /usr/share/planefence/html/noise*.png -mtime +"$OLDERTHAN" -delete 2>/dev/null rm -f /run/socket30003/*.log find /run/socket30003/*.txt -type f ! -newer /tmp/timestamp -delete 2>/dev/null find /usr/share/planefence/persist/.internal/*.tmp -type f ! -newer /tmp/timestamp -delete 2>/dev/null @@ -70,7 +69,7 @@ CLEANUP () fdate=${f: -10:6} # get the date component from the file name # remove anything older than yesterday: - (( fdate < $(date -d yesterday +%y%m%d) )) && rm -v -f $f && continue + (( fdate < $(date -d yesterday +%y%m%d) )) && rm -v -f "$f" && continue done @@ -99,7 +98,7 @@ CLEANUP () sed -i '/^$/d' /usr/share/planefence/persist/planefence-ignore.txt 2>/dev/null else - [[ "$LOGLEVEL" != "ERROR" ]] && echo "[$APPNAME][$(date)] $LOOPTIME set to 0 - cleanup skipped" || true + [[ "${LOGLEVEL,,}" != "error" ]] && "${s6wrap[@]}" echo "$LOOPTIME set to 0 - cleanup skipped" || true fi } @@ -109,38 +108,32 @@ GET_ICAO_DB () { if (( $(date -r /run/planefence/icao2plane.txt +%s 2>/dev/null || echo 0) < $(date -d "next monday - 7 days" +%s) )) then - [[ "$LOGLEVEL" != "ERROR" ]] && echo "[$APPNAME][$(date)] Retrieving ICAO to TAIL database from https://www.mictronics.de ... " || true + [[ "${LOGLEVEL,,}" != "error" ]] && "${s6wrap[@]}" echo "Retrieving ICAO to TAIL database from https://www.mictronics.de ... " || true # note - the curl won't fail, even if the file is not found because the PHP page doesn't return a 400 code but a regular result webpage - curl -s -L -f -o /tmp/icao24plus.zip https://www.mictronics.de/aircraft-database/icao24plus.php - if [[ "$?" != "0" ]] - then - echo "[$APPNAME][$(date)] Retrieving ICAO to TAIL database from https://www.mictronics.de FAILED!" + if ! curl -s -L -f -o /tmp/icao24plus.zip https://www.mictronics.de/aircraft-database/icao24plus.php; then + "${s6wrap[@]}" echo "Retrieving ICAO to TAIL database from https://www.mictronics.de FAILED!" return 1 fi - [[ "$LOGLEVEL" != "ERROR" ]] && echo "[$APPNAME][$(date)] unzipping ... " || true + [[ "${LOGLEVEL,,}" != "error" ]] && "${s6wrap[@]}" echo "unzipping ... " || true # unzipping *will* fail when the retrieved file is not a valid ZIP file - unzip -qq -d /tmp -o /tmp/icao24plus.zip 2>/dev/null - if [[ "$?" != "0" ]] - then - echo "[$APPNAME][$(date)] Unzipping ICAO to TAIL database from https://www.mictronics.de FAILED!" + if ! unzip -qq -d /tmp -o /tmp/icao24plus.zip 2>/dev/null; then + "${s6wrap[@]}" echo "Unzipping ICAO to TAIL database from https://www.mictronics.de FAILED!" return 1 fi rm -f /tmp/icao24plus.zip # now convert the text based file into a CSV file - [[ "$LOGLEVEL" != "ERROR" ]] && echo "[$APPNAME][$(date)] converting ... " || true - sed -i 's|\([0-9A-F]\{6\}\)\s*\([A-Z0-9\-]*\)\s*\([A-Z0-9]*\)\s*\(.*\)|\1,\2,\3,\4|g' /tmp/icao24plus.txt - if [[ "$?" != "0" ]] - then - echo "[$APPNAME][$(date)] Converting ICAO to TAIL database from https://www.mictronics.de FAILED!" + [[ "${LOGLEVEL,,}" != "error" ]] && "${s6wrap[@]}" echo "converting ... " || true + if ! sed -i 's|\([0-9A-F]\{6\}\)\s*\([A-Z0-9\-]*\)\s*\([A-Z0-9]*\)\s*\(.*\)|\1,\2,\3,\4|g' /tmp/icao24plus.txt; then + "${s6wrap[@]}" echo "Converting ICAO to TAIL database from https://www.mictronics.de FAILED!" return 1 fi mv -f /tmp/icao24plus.txt /run/planefence/icao2plane.txt - [[ "$LOGLEVEL" != "ERROR" ]] && echo "[$APPNAME][$(date)] done!" || true + [[ "${LOGLEVEL,,}" != "error" ]] && "${s6wrap[@]}" echo "done!" || true else - [[ "$LOGLEVEL" != "ERROR" ]] && echo "[$APPNAME][$(date)] ICAO to TAIL database is up to date" || true + [[ "${LOGLEVEL,,}" != "error" ]] && "${s6wrap[@]}" echo "ICAO to TAIL database is up to date" || true fi } # @@ -152,14 +145,14 @@ GET_AIRLINE_DB () { if [[ -f /usr/share/planefence/persist/airlinecodes.txt ]] && [[ $(find /usr/share/planefence/persist/airlinecodes.txt -mtime -7 | wc -l) == 1 ]]; then # file exists and is less than 7 days old - [[ "$LOGLEVEL" != "ERROR" ]] && echo "[$APPNAME][$(date)] Airlinecodes database is up to date" || true + [[ "${LOGLEVEL,,}" != "error" ]] && "${s6wrap[@]}" echo "Airlinecodes database is up to date" || true return 0 fi - [[ "$LOGLEVEL" != "ERROR" ]] && echo "[$APPNAME][$(date)] Updating airline names database..." || true + [[ "${LOGLEVEL,,}" != "error" ]] && "${s6wrap[@]}" echo "Updating airline names database..." || true rm -f /tmp/airlinecodes.txt /tmp/airlines.csv /tmp/airlinecodes.txt.tmp - curl --compressed -s -L -f https://raw.githubusercontent.com/kx1t/planefence-airlinecodes/main/airlinecodes.txt -o /tmp/airlinecodes.txt && [[ "$LOGLEVEL" != "ERROR" ]] && echo "[$APPNAME][$(date)] Got kx1t/planefence-airlinecodes" || true - curl --compressed -s -L -f https://raw.githubusercontent.com/jbroutier/whatisflying-db/master/data/airlines.csv -o /tmp/airlines.csv && [[ "$LOGLEVEL" != "ERROR" ]] && echo "[$APPNAME][$(date)] Got jbroutier/whatisflying-db" || true + curl --compressed -s -L -f https://raw.githubusercontent.com/kx1t/planefence-airlinecodes/main/airlinecodes.txt -o /tmp/airlinecodes.txt && [[ "${LOGLEVEL,,}" != "error" ]] && "${s6wrap[@]}" echo "Got kx1t/planefence-airlinecodes" || true + curl --compressed -s -L -f https://raw.githubusercontent.com/jbroutier/whatisflying-db/master/data/airlines.csv -o /tmp/airlines.csv && [[ "${LOGLEVEL,,}" != "error" ]] && "${s6wrap[@]}" echo "Got jbroutier/whatisflying-db" || true #convert JBroutier's DB into our format: if [[ -f /tmp/airlines.csv ]] @@ -173,29 +166,32 @@ GET_AIRLINE_DB () #echo "header has ${#header[@]} elements and contains:" #echo ${header[@]} #echo "icao=$icao name=$name country=$country callsign=$callsign" - if [[ ! -z "$icao" ]] && [[ ! -z "$name" ]] && [[ ! -z "$country" ]] && [[ ! -z "$callsign" ]] + if [[ -n "$icao" ]] && [[ -n "$name" ]] && [[ -n "$country" ]] && [[ -n "$callsign" ]] then tail -n +2 /tmp/airlines.csv | awk -F "," -v icao=$(( icao + 1 )) -v name=$(( name + 1 )) -v country=$(( country + 1 )) -v callsign=$(( callsign + 1 )) '{print $icao "," $name "," $callsign "," $country}' >> /tmp/airlinecodes.txt.tmp [[ -f /tmp/airlinecodes.txt ]] && cat /tmp/airlinecodes.txt /tmp/airlinecodes.txt.tmp > /tmp/airlinecodes.txt.tmp2 || cat /tmp/airlinecodes.txt.tmp > /tmp/airlinecodes.txt.tmp2 - [[ "$LOGLEVEL" != "ERROR" ]] && echo "[$APPNAME][$(date)] Removed $(awk -F',' 'seen[$1]++' /tmp/airlinecodes.txt.tmp2 |wc -l) overlapping entries, left $(awk -F',' '!seen[$1]++' /tmp/airlinecodes.txt.tmp2 |wc -l) entries in place" || true + [[ "${LOGLEVEL,,}" != "error" ]] && "${s6wrap[@]}" echo "Removed $(awk -F',' 'seen[$1]++' /tmp/airlinecodes.txt.tmp2 |wc -l) overlapping entries, left $(awk -F',' '!seen[$1]++' /tmp/airlinecodes.txt.tmp2 |wc -l) entries in place" || true awk -F',' '!seen[$1]++' /tmp/airlinecodes.txt.tmp2 >/tmp/airlinecodes.txt fi fi - # make sure the file is minimally disrupte and programs who have still have it open don't have the data connected to the inode modified + # make sure the file is minimally disrupted and programs who have still have it open don't have the data connected to the inode modified mv -f /tmp/airlinecodes.txt /usr/share/planefence/persist/airlinecodes.txt.tmp rm -f /usr/share/planefence/persist/airlinecodes.txt mv -f /usr/share/planefence/persist/airlinecodes.txt.tmp /usr/share/planefence/persist/airlinecodes.txt rm -f /tmp/airlines.csv /tmp/airlinecodes.txt.tmp /tmp/airlinecodes.txt.tmp2 - [[ "$LOGLEVEL" != "ERROR" ]] && echo "[$APPNAME][$(date)] Airlinecodes database update complete" || true + [[ "${LOGLEVEL,,}" != "error" ]] && "${s6wrap[@]}" echo "Airlinecodes database update complete" || true } #Now loop forevah: while true do - [[ "$LOGLEVEL" != "ERROR" ]] && echo "[$APPNAME][$(date)] Performing a cleanup run..." || true + [[ "${LOGLEVEL,,}" != "error" ]] && "${s6wrap[@]}" echo "Performing a cleanup run..." || true CLEANUP GET_ICAO_DB GET_AIRLINE_DB - [[ "$LOGLEVEL" != "ERROR" ]] && echo "[$APPNAME][$(date)] Cleanup done. Sleeping for $LOOPTIME" || true + if [[ -n "$MASTODON_SERVER$MASTODON_ACCESS_TOKEN" ]]; then + /scripts/masto_expire.sh delete + fi + [[ "${LOGLEVEL,,}" != "error" ]] && "${s6wrap[@]}" echo "Cleanup done. Sleeping for $LOOPTIME" || true sleep $LOOPTIME done diff --git a/rootfs/scripts/masto_expire.sh b/rootfs/scripts/masto_expire.sh index 79d2300..40df7e4 100755 --- a/rootfs/scripts/masto_expire.sh +++ b/rootfs/scripts/masto_expire.sh @@ -1,5 +1,5 @@ #!/command/with-contenv bash -#shellcheck shell=bash disable=SC1091,SC2174 +#shellcheck shell=bash disable=SC1091,SC2174,SC2015,SC2154 # ----------------------------------------------------------------------------------- # Copyright 2024 Ramon F. Kolb - licensed under the terms and conditions # of GPLv3. The terms and conditions of this license are included with the Github @@ -9,25 +9,42 @@ # This package may incorporate other software and license terms. # ----------------------------------------------------------------------------------- -if [[ -f /usr/share/planefence/persist/planefence.config ]]; then source /usr/share/planefence/persist/planefence.config; fi +if [[ -f /usr/share/planefence/persist/planefence.config ]]; then + source /usr/share/planefence/persist/planefence.config +fi + +source /scripts/common ACCESS_TOKEN=$MASTODON_ACCESS_TOKEN INSTANCE_URL="https://$MASTODON_SERVER" -RETENTION_DAYS="${MASTODON_RETENTION_TIME}" +RETENTION_DAYS="${MASTODON_RETENTION_TIME:-7}" delete_toot() { local toot_id="$1" local result - if result="$(curl -s --fail -X DELETE -H "Authorization: Bearer $ACCESS_TOKEN" "$INSTANCE_URL/api/v1/statuses/$toot_id" 2>&1)"; then - echo "successfully deleted" - else - echo "error: $result" + if ! result="$(curl -s --fail -X DELETE -H "Authorization: Bearer $ACCESS_TOKEN" "$INSTANCE_URL/api/v1/statuses/$toot_id" 2>&1)"; then + echo "" + "${s6wrap[@]}" echo "error deleting $toot_id: $result" fi } -if [[ -z "$RETENTION_DAYS" ]]; then - echo "RETENTION_DAYS not set. Exiting." +if chk_disabled "$MASTODON_RETENTION_TIME"; then + "${s6wrap[@]}" echo "MASTODON_RETENTION_TIME is set to $MASTODON_RETENTION_TIME (disabled); nothing to do!" + exit 0 +fi + +if [[ -z "$MASTODON_RETENTION_TIME" ]]; then + "${s6wrap[@]}" echo "Warning: MASTODON_RETENTION_TIME not set. Defaulting to $RETENTION_DAYS days." +fi + +if [[ -z "$MASTODON_ACCESS_TOKEN" ]]; then + "${s6wrap[@]}" echo "MASTODON_ACCESS_TOKEN not set. Exiting." + exit 1 +fi + +if [[ -z "$MASTODON_SERVER" ]]; then + "${s6wrap[@]}" echo "MASTODON_SERVER not set. Exiting." exit 1 fi @@ -37,37 +54,42 @@ declare -A toot_dates now="$(date +%s)" masto_id="$(curl -s -H "Authorization: Bearer $ACCESS_TOKEN" "$INSTANCE_URL/api/v1/accounts/verify_credentials" | jq -r '.id')" - +counter=0 while : ; do - echo -n "Indexing Media IDs round $((++counter))" + expired=0 + unexpired=0 + oldest=33000000000 + newest=0 + output=("Indexing Toots") toots="$(curl -s -H "Authorization: Bearer $ACCESS_TOKEN" "$INSTANCE_URL/api/v1/accounts/$masto_id/statuses?limit=40${last_id:+&max_id=}${last_id}")" # shellcheck disable=SC2207 toot_ids=($(jq -r '.[] | .id' <<< "$toots" 2>/dev/null)) if (( ${#toot_ids[@]} == 0)); then - echo "No more toots, we are done!" - exit + "${s6wrap[@]}" echo "No more Toots; done!" + exit 0 fi last_id="${toot_ids[-1]}" - echo " ${#toot_ids[@]} toots" + + output+=("$((counter+1)) - $((counter+${#toot_ids[@]})) (${#toot_ids[@]} toots).") + (( counter+=${#toot_ids[@]} )) || true for t in "${toot_ids[@]}"; do if [[ -z "${toot_dates[$t]}" ]]; then toot_dates[$t]="$(date -d "$(jq -r 'map(select(.id == "'"$t"'"))[].created_at' <<< "$toots")" +%s)" - echo -n "$t --> $(date -d @"${toot_dates[$t]}") " + if (( toot_dates[$t] < oldest )); then oldest="${toot_dates[$t]}"; fi + if (( toot_dates[$t] > newest )); then newest="${toot_dates[$t]}"; fi if (( (now - toot_dates[$t])/(60*60*24) > RETENTION_DAYS )); then - echo -n " expired (age: $(( (now - toot_dates[$t])/(60*60*24) )) days): " + (( expired++ )) || true if [[ "$1" == "delete" ]]; then - echo -n "deleting... " delete_toot "$t"; - else - echo "(not deleted)" fi else - echo " not expired (age: $(( (now - - toot_dates[$t])/(60*60*24) )) days)" + (( unexpired++ )) || true fi else - echo "$t --> duplicate, we're done!" + "${s6wrap[@]}" echo "No more Toots; done!" exit fi done + output+=("($unexpired unexpired; $expired expired; oldest $(date -d "@$oldest") ($(( (now - oldest)/(60*60*24) )) days); newest $(date -d "@$newest") ($(( (now - newest)/(60*60*24) )) days))") + "${s6wrap[@]}" echo "${output[@]}" done \ No newline at end of file