From ba176ffc0c1a4f0abf57c48ababfb52a819a8a0f Mon Sep 17 00:00:00 2001 From: Luis Antonio Obis Aparicio Date: Wed, 25 Sep 2024 13:04:10 +0200 Subject: [PATCH 1/4] script to copy files --- scripts/feminos-daq-sync.sh | 59 +++++++++++++++++++++++++++++++++++++ 1 file changed, 59 insertions(+) create mode 100644 scripts/feminos-daq-sync.sh diff --git a/scripts/feminos-daq-sync.sh b/scripts/feminos-daq-sync.sh new file mode 100644 index 0000000..78d6480 --- /dev/null +++ b/scripts/feminos-daq-sync.sh @@ -0,0 +1,59 @@ +#!/bin/bash + +# input directory +input_dir=$1 # e.g. /home/daq/data +# remote host e.g. lobis@sultan.unizar.es +remote_host=$2 +# output directory (in the remote host) +output_dir=$3 # e.g. /storage/iaxo/iaxo-lab/iaxo-d1/rawData + +# show usage if bad arguments +if [ $# -ne 3 ]; then + echo "Usage: $0 " + exit 1 +fi + +echo "Input directory: $input_dir" +echo "Remote host: $remote_host" +echo "Output directory: $output_dir" + +# check if input directory exists +if [ ! -d $input_dir ]; then + echo "Input directory does not exist" + exit 1 +fi + +# check remote host is accessible +ssh -q $remote_host exit +if [ $? -ne 0 ]; then + echo "Remote host is not accessible" + exit 1 +fi + +# create remote directory if it does not exist +ssh -q $remote_host "[ -d $output_dir ] || mkdir -p $output_dir" + +# check if output directory exists +ssh -q $remote_host "[ -d $output_dir ]" +if [ $? -ne 0 ]; then + echo "Output directory does not exist" + exit 1 +fi + +# loop forever +while true; do + # count files in local directory, output directory. Size of files in both and print + local_files=$(ls -1 $input_dir | wc -l) + remote_files=$(ssh -q $remote_host "ls -1 $output_dir | wc -l") + local_size=$(du -sh $input_dir | awk '{print $1}') + remote_size=$(ssh -q $remote_host "du -sh $output_dir" | awk '{print $1}') + + echo "Number of files in $input_dir: $local_files, size on disk: $local_size" + echo "Number of remote files in $output_dir: $remote_files, size on disk: $remote_size" + + rsync -av --human-readable --partial --progress --stats $input_dir/* $remote_host:$output_dir + + time_sleep=60 + echo "Sleeping for $time_sleep seconds" + sleep $time_sleep +done From 84ddbdff4f9862644c5db63d873653c635bcad35 Mon Sep 17 00:00:00 2001 From: Luis Antonio Obis Aparicio Date: Wed, 25 Sep 2024 13:10:07 +0200 Subject: [PATCH 2/4] sync instructions --- CMakeLists.txt | 8 ++++++++ README.md | 30 ++++++++++++++++++++++++++++++ 2 files changed, 38 insertions(+) diff --git a/CMakeLists.txt b/CMakeLists.txt index 4354198..0f758ab 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -90,6 +90,14 @@ install( CODE "execute_process(COMMAND chmod +x ${CMAKE_INSTALL_PREFIX}/bin/feminos-viewer)" ) +install( + FILES scripts/feminos-daq-sync.sh + DESTINATION bin +) +install( + CODE "execute_process(COMMAND chmod +x ${CMAKE_INSTALL_PREFIX}/bin/feminos-daq-sync.sh)" +) + # Uninstall add_custom_target(uninstall COMMAND ${CMAKE_COMMAND} -P ${CMAKE_SOURCE_DIR}/cmake/uninstall.cmake) diff --git a/README.md b/README.md index 52804ac..dfc8c33 100644 --- a/README.md +++ b/README.md @@ -206,3 +206,33 @@ This computation is done by the viewer program so it may take significant time t of counts. The computations are only performed as long as the `Observables` mode is selected (so it might be a good idea to leave this mode selected when performing a long acquisition run). The `Auto-Update` option will periodically refresh the observables so it's recommended to enable it when using the `Observables` mode. + +### Automatic syncing of output to remote server + +It is a frequent requirement that the data acquired by the `feminos-daq` program is stored in a remote server. + +This repository includes a script (`/scripts/feminos-daq-sync.sh`) that can be used to sync the data directory to a +remote host using `rsync`. + +```bash +feminos-daq-sync.sh local_data_directory/ remote_user@remote_host: /remote_data_directory/ +``` + +It is recommended to setup a systemd service to run this script. +The user running the service should have password-less ssh access to the remote host. + +`/etc/systemd/system/feminos-daq-sync.service`: + +```bash +[Unit] +Description=File Sync Service +After=network.target + +[Service] +ExecStart=/usr/local/bin/feminos-daq-sync.sh /home/user/data/ user@remote /remote/storage/data/ +Restart=always +User=useriaxo + +[Install] +WantedBy=multi-user.target +``` From 817b9aae4d322e90c60c441ffe0edca421ed5c52 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 25 Sep 2024 11:10:30 +0000 Subject: [PATCH 3/4] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- CMakeLists.txt | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 0f758ab..a04a41c 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -90,10 +90,7 @@ install( CODE "execute_process(COMMAND chmod +x ${CMAKE_INSTALL_PREFIX}/bin/feminos-viewer)" ) -install( - FILES scripts/feminos-daq-sync.sh - DESTINATION bin -) +install(FILES scripts/feminos-daq-sync.sh DESTINATION bin) install( CODE "execute_process(COMMAND chmod +x ${CMAKE_INSTALL_PREFIX}/bin/feminos-daq-sync.sh)" ) From 3116cddfa370c614daa8e7fced5d2960f8a4499d Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 25 Sep 2024 14:18:04 +0000 Subject: [PATCH 4/4] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- examples/automated_acquisition.py | 23 +++++++++++++++++------ 1 file changed, 17 insertions(+), 6 deletions(-) diff --git a/examples/automated_acquisition.py b/examples/automated_acquisition.py index 1b27d3d..7fb3337 100644 --- a/examples/automated_acquisition.py +++ b/examples/automated_acquisition.py @@ -1,7 +1,5 @@ #!/usr/bin/env python3 -import time -import numpy as np import subprocess import sys import uproot @@ -176,7 +174,9 @@ # if "feminos-daq" is not globally accessible, add it to FEMINOS_DAQ_EXECUTABLE variable if "FEMINOS_DAQ_EXECUTABLE" not in os.environ: - feminos_daq_executable = subprocess.check_output("which feminos-daq", shell=True).strip().decode("utf-8") + feminos_daq_executable = ( + subprocess.check_output("which feminos-daq", shell=True).strip().decode("utf-8") + ) os.environ["FEMINOS_DAQ_EXECUTABLE"] = feminos_daq_executable else: feminos_daq_executable = os.environ["FEMINOS_DAQ_EXECUTABLE"] @@ -196,9 +196,20 @@ run_time = 30 # seconds result = subprocess.run( - [feminos_daq_executable, "-s", ip, "--time", str(run_time), "--skip-run-info", "--disable-aqs", "--input", - config_filename, "--output", - output_filename]) + [ + feminos_daq_executable, + "-s", + ip, + "--time", + str(run_time), + "--skip-run-info", + "--disable-aqs", + "--input", + config_filename, + "--output", + output_filename, + ] +) # print return code print(f"Return code: {result.returncode}")