Sync data to remote hosts (#13)

* script to copy files

* sync instructions

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

---------

Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>

CMakeLists.txt

@@ -90,6 +90,11 @@ install(
     CODE "execute_process(COMMAND chmod +x ${CMAKE_INSTALL_PREFIX}/bin/feminos-viewer)"
 )
 
+install(FILES scripts/feminos-daq-sync.sh DESTINATION bin)
+install(
+    CODE "execute_process(COMMAND chmod +x ${CMAKE_INSTALL_PREFIX}/bin/feminos-daq-sync.sh)"
+)
+
 # Uninstall
 add_custom_target(uninstall COMMAND ${CMAKE_COMMAND} -P
                                     ${CMAKE_SOURCE_DIR}/cmake/uninstall.cmake)

README.md

@@ -206,3 +206,33 @@ This computation is done by the viewer program so it may take significant time t
 of counts. The computations are only performed as long as the `Observables` mode is selected (so it might be a good idea
 to leave this mode selected when performing a long acquisition run). The `Auto-Update` option will periodically refresh
 the observables so it's recommended to enable it when using the `Observables` mode.
+
+### Automatic syncing of output to remote server
+
+It is a frequent requirement that the data acquired by the `feminos-daq` program is stored in a remote server.
+
+This repository includes a script (`/scripts/feminos-daq-sync.sh`) that can be used to sync the data directory to a
+remote host using `rsync`.
+
+```bash
+feminos-daq-sync.sh local_data_directory/ remote_user@remote_host: /remote_data_directory/
+```
+
+It is recommended to setup a systemd service to run this script.
+The user running the service should have password-less ssh access to the remote host.
+
+`/etc/systemd/system/feminos-daq-sync.service`:
+
+```bash
+[Unit]
+Description=File Sync Service
+After=network.target
+
+[Service]
+ExecStart=/usr/local/bin/feminos-daq-sync.sh /home/user/data/ user@remote /remote/storage/data/
+Restart=always
+User=useriaxo
+
+[Install]
+WantedBy=multi-user.target
+```

examples/automated_acquisition.py

@@ -1,7 +1,5 @@
 #!/usr/bin/env python3
 
-import time
-import numpy as np
 import subprocess
 import sys
 import uproot
@@ -176,7 +174,9 @@
 
 # if "feminos-daq" is not globally accessible, add it to FEMINOS_DAQ_EXECUTABLE variable
 if "FEMINOS_DAQ_EXECUTABLE" not in os.environ:
-    feminos_daq_executable = subprocess.check_output("which feminos-daq", shell=True).strip().decode("utf-8")
+    feminos_daq_executable = (
+        subprocess.check_output("which feminos-daq", shell=True).strip().decode("utf-8")
+    )
     os.environ["FEMINOS_DAQ_EXECUTABLE"] = feminos_daq_executable
 else:
     feminos_daq_executable = os.environ["FEMINOS_DAQ_EXECUTABLE"]
@@ -196,9 +196,20 @@
 run_time = 30  # seconds
 
 result = subprocess.run(
-    [feminos_daq_executable, "-s", ip, "--time", str(run_time), "--skip-run-info", "--disable-aqs", "--input",
-     config_filename, "--output",
-     output_filename])
+    [
+        feminos_daq_executable,
+        "-s",
+        ip,
+        "--time",
+        str(run_time),
+        "--skip-run-info",
+        "--disable-aqs",
+        "--input",
+        config_filename,
+        "--output",
+        output_filename,
+    ]
+)
 
 # print return code
 print(f"Return code: {result.returncode}")

scripts/feminos-daq-sync.sh

@@ -0,0 +1,59 @@
+#!/bin/bash
+
+# input directory
+input_dir=$1 # e.g. /home/daq/data
+# remote host e.g. [email protected]
+remote_host=$2
+# output directory (in the remote host)
+output_dir=$3 # e.g. /storage/iaxo/iaxo-lab/iaxo-d1/rawData
+
+# show usage if bad arguments
+if [ $# -ne 3 ]; then
+    echo "Usage: $0 <input_dir> <remote_host> <output_dir_in_remote_host>"
+    exit 1
+fi
+
+echo "Input directory: $input_dir"
+echo "Remote host: $remote_host"
+echo "Output directory: $output_dir"
+
+# check if input directory exists
+if [ ! -d $input_dir ]; then
+    echo "Input directory does not exist"
+    exit 1
+fi
+
+# check remote host is accessible
+ssh -q $remote_host exit
+if [ $? -ne 0 ]; then
+    echo "Remote host is not accessible"
+    exit 1
+fi
+
+# create remote directory if it does not exist
+ssh -q $remote_host "[ -d $output_dir ] || mkdir -p $output_dir"
+
+# check if output directory exists
+ssh -q $remote_host "[ -d $output_dir ]"
+if [ $? -ne 0 ]; then
+    echo "Output directory does not exist"
+    exit 1
+fi
+
+# loop forever
+while true; do
+    # count files in local directory, output directory. Size of files in both and print
+    local_files=$(ls -1 $input_dir | wc -l)
+    remote_files=$(ssh -q $remote_host "ls -1 $output_dir | wc -l")
+    local_size=$(du -sh $input_dir | awk '{print $1}')
+    remote_size=$(ssh -q $remote_host "du -sh $output_dir" | awk '{print $1}')
+
+    echo "Number of files in $input_dir: $local_files, size on disk: $local_size"
+    echo "Number of remote files in $output_dir: $remote_files, size on disk: $remote_size"
+
+    rsync -av --human-readable --partial --progress --stats $input_dir/* $remote_host:$output_dir
+
+    time_sleep=60
+    echo "Sleeping for $time_sleep seconds"
+    sleep $time_sleep
+done