GitHub action unittest

.github/workflows/actions.yml

@@ -0,0 +1,48 @@
+name: CI
+
+on:
+  workflow_dispatch:
+  pull_request:
+  push:
+    branches:
+      - master
+
+jobs:
+  tests:
+    runs-on: self-hosted
+    defaults:
+      run:
+        shell: bash -l {0}
+    steps:
+    - uses: actions/checkout@v3
+    - name: Set up Miniconda
+      uses: conda-incubator/setup-miniconda@v2
+      with:
+        channels: conda-forge,defaults,mantid
+        auto-update-conda: true
+        miniforge-version: latest
+        python-version: "3.8"
+        environment-file: environment.yml
+        activate-environment: liquid-ref
+    - name: Restore cached conda environment
+      id: cache-load
+      uses: actions/cache/restore@v3
+      with:
+        path: /usr/share/miniconda/envs/liquid-ref
+        key: ${{ runner.os }}-conda-${{ hashFiles('environment.yml') }}
+    - name: Load Environment
+      if: steps.cache-load.outputs.cache-hit != 'true'
+      run: |
+        mamba env update --file environment.yml  --prune
+    - name: Start xvfb daemon
+      run: |
+        /sbin/start-stop-daemon --start --quiet --pidfile /tmp/custom_xvfb_99.pid --make-pidfile --background --exec /usr/bin/Xvfb -- :99 -ac -screen 0 1280x1024x16
+    - name: Test with pytest
+      run: |
+        git submodule add --force https://code.ornl.gov/sns-hfir-scse/infrastructure/test-data/liquidsreflectometer-data.git tests/data/liquidsreflectometer-data
+        git submodule update --init
+        python -m pytest --cov=reduction --cov=scripts --cov-report=xml --cov-report=term -m datarepo
+    - name: Upload coverage reports to Codecov
+      uses: codecov/codecov-action@v3
+      env:
+        CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}

.gitignore

@@ -50,6 +50,8 @@ coverage.xml
 *.py,cover
 .hypothesis/
 .pytest_cache/
+tests/results/*
+!tests/results/.gitkeep
 
 # Translations
 *.mo
@@ -109,6 +111,7 @@ venv/
 ENV/
 env.bak/
 venv.bak/
+.vscode/
 
 # Spyder project settings
 .spyderproject
@@ -127,3 +130,7 @@ dmypy.json
 
 # Pyre type checker
 .pyre/
+
+# project specific files
+reduction/_version.py
+conda.recipe/

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "tests/data/liquidsreflectometer-data"]
+	path = tests/data/liquidsreflectometer-data
+	url = https://code.ornl.gov/sns-hfir-scse/infrastructure/test-data/liquidsreflectometer-data.git

environment.yml

@@ -0,0 +1,18 @@
+name: liquid-ref
+channels:
+  - conda-forge
+  - default
+  - mantid
+dependencies:
+  - pre-commit
+  - pytest
+  - pytest-cov
+  - numpy
+  - python=3.8
+  - mantidworkbench>=6.7.0
+  - codecov
+  - conda-build
+  - python-build
+  - versioningit
+  - boa
+  - conda-verify

launcher/scripts/off_spec.py

@@ -22,7 +22,7 @@ class PixelData():
     det_distance = DEFAULT_4B_SAMPLE_DET_DISTANCE
 
     def __init__(self, run_number):
-        self.ws = api.LoadEventNexus("REF_L_%s" % run_number, OutputWorkspace='r%s' % run_number)
+        self.ws = api.LoadEventNexus("REF_L_%s.nxs.h5" % run_number, OutputWorkspace='r%s' % run_number)
         self.run_number = run_number
         self.get_parameters()
 

launcher/scripts/refracted_beam.py

@@ -401,7 +401,7 @@ def process_run(run_number, output_dir, material='Si'):
     """
         Process file and extract offset and pixel size
     """
-    ws_sc = api.LoadEventNexus("REF_L_%s" % run_number)
+    ws_sc = api.LoadEventNexus("REF_L_%s.nxs.h5" % run_number)
 
     refr = Refracted(ws_sc, material=material, offset=0.0, pixel_size=0.0007)
 

launcher/scripts/template_reduction.py

@@ -96,10 +96,10 @@ def reduce_30Hz(meas_run_30Hz, ref_run_30Hz, ref_data_60Hz, template_30Hz, scan_
     template_data = read_template(template_30Hz, scan_index)
 
     # Reduce the quartz at 30Hz
-    ref_ws_30Hz = api.LoadEventNexus("REF_L_%s" % ref_run_30Hz)
+    ref_ws_30Hz = api.LoadEventNexus("REF_L_%s.nxs.h5" % ref_run_30Hz)
 
     # Reduce the sample data at 30Hz
-    meas_ws_30Hz = api.LoadEventNexus("REF_L_%s" % meas_run_30Hz)
+    meas_ws_30Hz = api.LoadEventNexus("REF_L_%s.nxs.h5" % meas_run_30Hz)
 
     # Load the 60Hz reference data
     data_60Hz = np.loadtxt(ref_data_60Hz).T
@@ -209,7 +209,7 @@ def reduce_30Hz_from_ws(meas_ws_30Hz, ref_ws_30Hz, data_60Hz, template_data, sca
 def reduce_30Hz_slices(meas_run_30Hz, ref_run_30Hz, ref_data_60Hz, template_30Hz, 
                        time_interval, output_dir, scan_index=1, create_plot=True, template_reference=None):
 
-    meas_ws_30Hz = api.LoadEventNexus("REF_L_%s" % meas_run_30Hz)
+    meas_ws_30Hz = api.LoadEventNexus("REF_L_%s.nxs.h5" % meas_run_30Hz)
 
     return reduce_30Hz_slices_ws(meas_ws_30Hz, ref_run_30Hz, ref_data_60Hz, template_30Hz, 
                                  time_interval, output_dir, scan_index=scan_index, create_plot=create_plot,
@@ -218,7 +218,7 @@ def reduce_30Hz_slices(meas_run_30Hz, ref_run_30Hz, ref_data_60Hz, template_30Hz
 def reduce_60Hz_slices(meas_run, template_file,
                        time_interval, output_dir, scan_index=1, create_plot=True):
 
-    meas_ws = api.LoadEventNexus("REF_L_%s" % meas_run)
+    meas_ws = api.LoadEventNexus("REF_L_%s.nxs.h5" % meas_run)
 
     return reduce_60Hz_slices_ws(meas_ws, template_file,
                                  time_interval, output_dir, scan_index=scan_index, create_plot=create_plot)
@@ -244,7 +244,7 @@ def reduce_30Hz_slices_ws(meas_ws_30Hz, ref_run_30Hz, ref_data_60Hz, template_30
     if os.path.isfile(ref_run_30Hz):
         ref_ws_30Hz = api.LoadEventNexus(ref_run_30Hz)
     else:
-        ref_ws_30Hz = api.LoadEventNexus("REF_L_%s" % ref_run_30Hz)
+        ref_ws_30Hz = api.LoadEventNexus("REF_L_%s.nxs.h5" % ref_run_30Hz)
 
     # Reduce the sample data at 30Hz
     print("Reading sample data at 30Hz")

pyproject.toml

@@ -10,7 +10,7 @@ dependencies = [
 [tool.ruff]
 line-length = 140
 # https://beta.ruff.rs/docs/rules/
-select = ["A", "ARG", "E", "W", "F", "I", "PT", "RUF100"]
+select = ["A", "ARG", "E", "W", "F", "PT", "RUF100"]
 ignore = ["E402", # module level import not at top of file
           "E722", # bare except
           "E741", "E743", # ambiguous function name, generally 'l'
@@ -20,3 +20,21 @@ ignore = ["E402", # module level import not at top of file
 
 [tool.black]
 line-length = 140
+
+[tool.coverage.run]
+branch = true
+
+[tool.coverage.report]
+skip_empty = true
+
+[tool.pytest.ini_options]
+pythonpath = [
+  ".", "reduction", "scripts", "tests/data/liquidsreflectometer-data/nexus/"
+]
+
+testpaths = ["tests"]
+norecursedirs = [".git", "tmp*", "_tmp*", "__pycache__", "*dataset*", "*data_set*"]
+markers = [
+  "datarepo: mark a test as using LiquidsReflectometer-data repository",
+  "scripts: mark a test as a script that should be run manually",
+]

reduction/lr_reduction/template.py

@@ -158,7 +158,7 @@ def process_from_template_ws(ws_sc, template_data, q_summing=False,
     # Load normalization run
     normalize = normalize and template_data.apply_normalization
     if ws_db is None and normalize:
-        ws_db = api.LoadEventNexus("REF_L_%s" % template_data.norm_file)
+        ws_db = api.LoadEventNexus("REF_L_%s.nxs.h5" % template_data.norm_file)
 
     # If we run in theta-theta geometry, we'll need thi
     thi_value = ws_sc.getRun()['thi'].value[0]

reduction/lr_reduction/time_resolved.py

@@ -11,6 +11,7 @@
 from mantid.api import *
 from mantid.kernel import *
 from matplotlib import pyplot as plt
+from mantid.kernel import config
 
 mantid.kernel.config.setLogLevel(3)
 
@@ -31,10 +32,10 @@ def reduce_30Hz(meas_run_30Hz, ref_run_30Hz, ref_data_60Hz, template_30Hz, scan_
     template_data = template.read_template(template_30Hz, scan_index)
 
     # Reduce the quartz at 30Hz
-    ref_ws_30Hz = api.LoadEventNexus("REF_L_%s" % ref_run_30Hz)
+    ref_ws_30Hz = api.LoadEventNexus("REF_L_%s.nxs.h5" % ref_run_30Hz)
 
     # Reduce the sample data at 30Hz
-    meas_ws_30Hz = api.LoadEventNexus("REF_L_%s" % meas_run_30Hz)
+    meas_ws_30Hz = api.LoadEventNexus("REF_L_%s.nxs.h5" % meas_run_30Hz)
 
     # Load the 60Hz reference data
     data_60Hz = np.loadtxt(ref_data_60Hz).T
@@ -121,8 +122,10 @@ def reduce_30Hz_from_ws(meas_ws_30Hz, ref_ws_30Hz, data_60Hz, template_data, sca
 def reduce_30Hz_slices(meas_run_30Hz, ref_run_30Hz, ref_data_60Hz, template_30Hz,
                        time_interval, output_dir, scan_index=1, create_plot=True,
                        template_reference=None, q_summing=False): # noqa ARG001
-
-    meas_ws_30Hz = api.LoadEventNexus("REF_L_%s" % meas_run_30Hz)
+    print(f'time_resolved: {config.getDataSearchDirs()}')
+    import sys
+    print(f'syspath: {sys.path}')
+    meas_ws_30Hz = api.LoadEventNexus("REF_L_%s.nxs.h5" % meas_run_30Hz)
 
     return reduce_30Hz_slices_ws(meas_ws_30Hz, ref_run_30Hz, ref_data_60Hz, template_30Hz,
                                  time_interval, output_dir, scan_index=scan_index, create_plot=create_plot,
@@ -131,7 +134,7 @@ def reduce_30Hz_slices(meas_run_30Hz, ref_run_30Hz, ref_data_60Hz, template_30Hz
 def reduce_60Hz_slices(meas_run, template_file,
                        time_interval, output_dir, scan_index=1, create_plot=True):
 
-    meas_ws = api.LoadEventNexus("REF_L_%s" % meas_run)
+    meas_ws = api.LoadEventNexus("REF_L_%s.nxs.h5" % meas_run)
 
     return reduce_60Hz_slices_ws(meas_ws, template_file,
                                  time_interval, output_dir, scan_index=scan_index, create_plot=create_plot)
@@ -166,7 +169,7 @@ def reduce_30Hz_slices_ws(meas_ws_30Hz, ref_run_30Hz, ref_data_60Hz, template_30
     if os.path.isfile(ref_run_30Hz):
         ref_ws_30Hz = api.LoadEventNexus(ref_run_30Hz)
     else:
-        ref_ws_30Hz = api.LoadEventNexus("REF_L_%s" % ref_run_30Hz)
+        ref_ws_30Hz = api.LoadEventNexus("REF_L_%s.nxs.h5" % ref_run_30Hz)
 
     # Reduce the sample data at 30Hz
     print("Reading sample data at 30Hz")

reduction/lr_reduction/workflow.py

@@ -132,7 +132,7 @@ def reduce_fixed_two_theta(ws, template_file, output_dir, average_overlap=False,
     template_data = template.read_template(template_file, sequence_number)
 
     # Load normalization run
-    ws_db = mtd_api.LoadEventNexus("REF_L_%s" % template_data.norm_file)
+    ws_db = mtd_api.LoadEventNexus("REF_L_%s.nxs.h5" % template_data.norm_file)
     tthd_value = ws.getRun()['tthd'].value[0]
 
     # Look for parameters that might have been determined earlier for this measurement