From 23301b35c8e32ba06e5cb2df061fd8b178003c51 Mon Sep 17 00:00:00 2001 From: Jose Borreguero Date: Fri, 16 Feb 2024 11:18:10 -0500 Subject: [PATCH 01/18] add two_backgrounds option Signed-off-by: Jose Borreguero --- reduction/lr_reduction/reduction_template_reader.py | 1 + reduction/lr_reduction/template.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/reduction/lr_reduction/reduction_template_reader.py b/reduction/lr_reduction/reduction_template_reader.py index 25b8510..841f9a2 100644 --- a/reduction/lr_reduction/reduction_template_reader.py +++ b/reduction/lr_reduction/reduction_template_reader.py @@ -21,6 +21,7 @@ def __init__(self): # Signal selection self.data_peak_range = [140, 150] self.subtract_background = True + self.two_backgrounds: bool = False self.background_roi = [137, 153, 0, 0] self.tof_range = [9600., 21600.] self.select_tof_range = True diff --git a/reduction/lr_reduction/template.py b/reduction/lr_reduction/template.py index d77fab8..aa42e4f 100644 --- a/reduction/lr_reduction/template.py +++ b/reduction/lr_reduction/template.py @@ -202,7 +202,7 @@ def process_from_template_ws(ws_sc, template_data, q_summing=False, else: norm_low_res = None - # We are not subtrating background for the direct beam + # We are not subtracting background for the direct beam if template_data.subtract_norm_background: norm_bck = template_data.norm_background_roi else: From 3b1effc803516802125163fd1df0edea5f39dfef Mon Sep 17 00:00:00 2001 From: Jose Borreguero Date: Fri, 16 Feb 2024 11:28:13 -0500 Subject: [PATCH 02/18] check for two backgrounds Signed-off-by: Jose Borreguero --- reduction/lr_reduction/reduction_template_reader.py | 1 + reduction/lr_reduction/template.py | 4 ++++ 2 files changed, 5 insertions(+) diff --git a/reduction/lr_reduction/reduction_template_reader.py b/reduction/lr_reduction/reduction_template_reader.py index 841f9a2..726c468 100644 --- a/reduction/lr_reduction/reduction_template_reader.py +++ b/reduction/lr_reduction/reduction_template_reader.py @@ -33,6 +33,7 @@ def __init__(self): self.apply_normalization = True self.norm_peak_range = [140, 150] self.subtract_norm_background = True + self.two_norm_backgrounds: bool = False self.norm_background_roi = [137, 153] self.norm_x_range_flag = True self.norm_x_range = [115,210] diff --git a/reduction/lr_reduction/template.py b/reduction/lr_reduction/template.py index aa42e4f..6a477b1 100644 --- a/reduction/lr_reduction/template.py +++ b/reduction/lr_reduction/template.py @@ -185,6 +185,8 @@ def process_from_template_ws(ws_sc, template_data, q_summing=False, peak = template_data.data_peak_range if template_data.subtract_background: peak_bck = template_data.background_roi + if template_data.two_backgrounds is False: + peak_bck = peak_bck[0: 3] # retain only the first background else: peak_bck = None @@ -205,6 +207,8 @@ def process_from_template_ws(ws_sc, template_data, q_summing=False, # We are not subtracting background for the direct beam if template_data.subtract_norm_background: norm_bck = template_data.norm_background_roi + if template_data.two_norm_backgrounds is False: + norm_bck = norm_bck[0: 3] # retain only the first background else: norm_bck = None From ece500d8baab46fc88af4ff3515737293f9b3511 Mon Sep 17 00:00:00 2001 From: Jose Borreguero Date: Fri, 16 Feb 2024 12:16:24 -0500 Subject: [PATCH 03/18] remove the second background range for the direct-beam data Signed-off-by: Jose Borreguero --- reduction/lr_reduction/reduction_template_reader.py | 11 ++++++++--- reduction/lr_reduction/template.py | 2 -- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/reduction/lr_reduction/reduction_template_reader.py b/reduction/lr_reduction/reduction_template_reader.py index 726c468..639b38f 100644 --- a/reduction/lr_reduction/reduction_template_reader.py +++ b/reduction/lr_reduction/reduction_template_reader.py @@ -33,7 +33,6 @@ def __init__(self): self.apply_normalization = True self.norm_peak_range = [140, 150] self.subtract_norm_background = True - self.two_norm_backgrounds: bool = False self.norm_background_roi = [137, 153] self.norm_x_range_flag = True self.norm_x_range = [115,210] @@ -80,6 +79,7 @@ def to_xml(self): _xml += "%s\n" % str(self.data_peak_range[1]) _xml += "N/A\n" _xml += "%s\n" % str(self.subtract_background) + _xml += "%s\n" % str(self.two_backgrounds) _xml += "%s\n" % str(self.background_roi[0]) _xml += "%s\n" % str(self.background_roi[1]) _xml += "%s\n" % str(self.background_roi[2]) @@ -164,11 +164,15 @@ def from_xml_element(self, instrument_dom): self.norm_x_range = [getIntElement(instrument_dom, "norm_x_min"), getIntElement(instrument_dom, "norm_x_max")] - #background flag + # background flag self.subtract_background = getBoolElement(instrument_dom, "background_flag", default=self.subtract_background) - #background from/to pixels + # use two backgrounds flag + self.two_backgrounds = getBoolElement(instrument_dom, "two_backgrounds_flag", + default=self.two_backgrounds) + + # background from/to pixels self.background_roi = [getIntElement(instrument_dom, "back_roi1_from"), getIntElement(instrument_dom, "back_roi1_to"), getIntElement(instrument_dom, "back_roi2_from"), @@ -193,6 +197,7 @@ def from_xml_element(self, instrument_dom): # Background subtraction option self.subtract_norm_background = getBoolElement(instrument_dom, "norm_background_flag", default=self.subtract_norm_background) + self.norm_background_roi = [getIntElement(instrument_dom, "norm_from_back_pixels"), getIntElement(instrument_dom, "norm_to_back_pixels")] diff --git a/reduction/lr_reduction/template.py b/reduction/lr_reduction/template.py index 6a477b1..80a43ff 100644 --- a/reduction/lr_reduction/template.py +++ b/reduction/lr_reduction/template.py @@ -207,8 +207,6 @@ def process_from_template_ws(ws_sc, template_data, q_summing=False, # We are not subtracting background for the direct beam if template_data.subtract_norm_background: norm_bck = template_data.norm_background_roi - if template_data.two_norm_backgrounds is False: - norm_bck = norm_bck[0: 3] # retain only the first background else: norm_bck = None From fdc3d1f07f17e9927d55e6ac5597aafdb75ff3cf Mon Sep 17 00:00:00 2001 From: Jose Borreguero Date: Fri, 16 Feb 2024 14:16:05 -0500 Subject: [PATCH 04/18] add sanity check when importing data from a dictionary Signed-off-by: Jose Borreguero --- .../lr_reduction/reduction_template_reader.py | 13 +++++++++ .../test/test_reduction_template_reader.py | 29 +++++++++++++++++++ 2 files changed, 42 insertions(+) create mode 100644 reduction/test/test_reduction_template_reader.py diff --git a/reduction/lr_reduction/reduction_template_reader.py b/reduction/lr_reduction/reduction_template_reader.py index 639b38f..e2cb180 100644 --- a/reduction/lr_reduction/reduction_template_reader.py +++ b/reduction/lr_reduction/reduction_template_reader.py @@ -66,6 +66,19 @@ def __init__(self): self.incident_medium_index_selected = 0 def from_dict(self, data_dict): + r"""Update object's attributes with a dictionary with entries of the type attribute_name: attribute_value. + + Raises + ------ + ValueError + if one entry of the dictionary is not an attribute of this object + """ + + # check all keys are data_dict are attributes of object `self` + attribute_names = list(vars(self)) + if not all(key in attribute_names for key in data_dict): + raise ValueError("data_dir contains invalid entries") + # update attribute values for k, v in data_dict.items(): setattr(self, k, v) diff --git a/reduction/test/test_reduction_template_reader.py b/reduction/test/test_reduction_template_reader.py new file mode 100644 index 0000000..72b73b4 --- /dev/null +++ b/reduction/test/test_reduction_template_reader.py @@ -0,0 +1,29 @@ +# third-party imports +import pytest + +# lr_reduction imports +from lr_reduction.reduction_template_reader import ReductionParameters + + +class TestReductionParameters: + + def test_two_backgrounds(self): + r"""verify the xml dump writes what we want""" + redparms = ReductionParameters() + redparms.two_backgrounds = True + assert "True" in redparms.to_xml() + + def test_from_dict(self): + r"""verify method from_dict raises when passed some nonsense""" + + redparms = ReductionParameters() + # valid data dictionary + redparms.from_dict(dict(two_backgrounds=True)) + assert redparms.two_backgrounds + # invalid data dictionary + with pytest.raises(ValueError) as excinfo: + redparms.from_dict(dict(nonsense=True)) + assert "data_dir contains invalid entries" == str(excinfo.value) + +if __name__ == "__main__": + pytest.main(__file__) From 4d39d5e31072af4d90eb83665a4db175d947164e Mon Sep 17 00:00:00 2001 From: Jose Borreguero Date: Fri, 16 Feb 2024 14:17:52 -0500 Subject: [PATCH 05/18] add sanity check when importing data from a dictionary Signed-off-by: Jose Borreguero --- reduction/lr_reduction/reduction_template_reader.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/reduction/lr_reduction/reduction_template_reader.py b/reduction/lr_reduction/reduction_template_reader.py index e2cb180..804a8fe 100644 --- a/reduction/lr_reduction/reduction_template_reader.py +++ b/reduction/lr_reduction/reduction_template_reader.py @@ -66,7 +66,8 @@ def __init__(self): self.incident_medium_index_selected = 0 def from_dict(self, data_dict): - r"""Update object's attributes with a dictionary with entries of the type attribute_name: attribute_value. + r""" + Update object's attributes with a dictionary with entries of the type attribute_name: attribute_value. Raises ------ From a16e0cdb68ba13a0c7d60655e3b42605751eb0fb Mon Sep 17 00:00:00 2001 From: Jose Borreguero Date: Fri, 16 Feb 2024 17:18:59 -0500 Subject: [PATCH 06/18] session pytest setup Signed-off-by: Jose Borreguero --- .github/workflows/actions.yml | 2 -- reduction/test/conftest.py | 20 ++++++++++++++++++++ 2 files changed, 20 insertions(+), 2 deletions(-) create mode 100644 reduction/test/conftest.py diff --git a/.github/workflows/actions.yml b/.github/workflows/actions.yml index 3a42dcf..8049521 100644 --- a/.github/workflows/actions.yml +++ b/.github/workflows/actions.yml @@ -40,8 +40,6 @@ jobs: - name: Test with pytest working-directory: ./reduction run: | - echo datasearch.directories=/home/cloud/_work/LiquidsReflectometer/LiquidsReflectometer/reduction/tests/data/liquidsreflectometer-data/nexus/ >> ~/.mantid/Mantid.user.properties - cat ~/.mantid/Mantid.user.properties git submodule add --force https://code.ornl.gov/sns-hfir-scse/infrastructure/test-data/liquidsreflectometer-data.git tests/data/liquidsreflectometer-data git submodule update --init python -m pytest --cov=. --cov-report=xml --cov-report=term test diff --git a/reduction/test/conftest.py b/reduction/test/conftest.py new file mode 100644 index 0000000..e50487a --- /dev/null +++ b/reduction/test/conftest.py @@ -0,0 +1,20 @@ +# standard imports +import logging +from pathlib import Path +import os + +data_dir = Path(__file__).parent.parent / "data" + + +def pytest_sessionstart(session): + r"""invoked by pytest at the very beginning""" + logger = logging.getLogger() + # Insert data directory in Mantid config file + mantid_config_dir = Path(os.environ["HOME"]) / ".mantid" + os.makedirs(str(mantid_config_dir), exist_ok=True) # create directory if it doesn't exists + mantid_config_file = mantid_config_dir / "Mantid.user.properties" + write_mode = "a" if mantid_config_file.exists() else "w" # append or create-then-write + with open(mantid_config_file, write_mode) as file_handle: + data_path = data_dir / "liquidsreflectometer-data" / "nexus" + file_handle.write(f"datasearch.directories={str(data_path)}") + logger.info("Appending data directory to mantid config file") From 3eef8f9bae19baa80b1db4bef2f791f04258dc9c Mon Sep 17 00:00:00 2001 From: Jose Borreguero Date: Fri, 16 Feb 2024 17:31:33 -0500 Subject: [PATCH 07/18] simplify github actions Signed-off-by: Jose Borreguero --- .github/workflows/actions.yml | 20 ++------------------ environment.yml | 2 +- 2 files changed, 3 insertions(+), 19 deletions(-) diff --git a/.github/workflows/actions.yml b/.github/workflows/actions.yml index 8049521..fa0cfa8 100644 --- a/.github/workflows/actions.yml +++ b/.github/workflows/actions.yml @@ -9,34 +9,18 @@ on: jobs: tests: - runs-on: self-hosted + runs-on: ubuntu-22.04 defaults: run: shell: bash -l {0} steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v2 - name: Set up Miniconda uses: conda-incubator/setup-miniconda@v2 with: - channels: conda-forge,defaults,mantid auto-update-conda: true miniforge-version: latest - python-version: "3.8" environment-file: environment.yml - activate-environment: liquid-ref - - name: Restore cached conda environment - id: cache-load - uses: actions/cache/restore@v3 - with: - path: /usr/share/miniconda/envs/liquid-ref - key: ${{ runner.os }}-conda-${{ hashFiles('environment.yml') }} - - name: Load Environment - if: steps.cache-load.outputs.cache-hit != 'true' - run: | - mamba env update --file environment.yml --prune - - name: Start xvfb daemon - run: | - /sbin/start-stop-daemon --start --quiet --pidfile /tmp/custom_xvfb_99.pid --make-pidfile --background --exec /usr/bin/Xvfb -- :99 -ac -screen 0 1280x1024x16 - name: Test with pytest working-directory: ./reduction run: | diff --git a/environment.yml b/environment.yml index 879b7b3..6b957fb 100644 --- a/environment.yml +++ b/environment.yml @@ -4,12 +4,12 @@ channels: - default - mantid dependencies: + - python=3.8 - pre-commit - pytest - pytest-cov - numpy - lmfit - - python=3.8 - mantidworkbench>=6.7.0 - codecov - conda-build From 38b1baf2efab86332eac50d96e04eb6dd69cdd44 Mon Sep 17 00:00:00 2001 From: Jose Borreguero Date: Fri, 16 Feb 2024 17:33:31 -0500 Subject: [PATCH 08/18] more verbose messages from pytest Signed-off-by: Jose Borreguero --- .github/workflows/actions.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/actions.yml b/.github/workflows/actions.yml index fa0cfa8..915d89d 100644 --- a/.github/workflows/actions.yml +++ b/.github/workflows/actions.yml @@ -26,7 +26,7 @@ jobs: run: | git submodule add --force https://code.ornl.gov/sns-hfir-scse/infrastructure/test-data/liquidsreflectometer-data.git tests/data/liquidsreflectometer-data git submodule update --init - python -m pytest --cov=. --cov-report=xml --cov-report=term test + python -m pytest -vv --cov=. --cov-report=xml --cov-report=term test - name: Upload coverage reports to Codecov uses: codecov/codecov-action@v3 env: From 5b574c876437337d294095aa7a86bb6320d84370 Mon Sep 17 00:00:00 2001 From: Jose Borreguero Date: Fri, 16 Feb 2024 17:52:36 -0500 Subject: [PATCH 09/18] use github environment variables Signed-off-by: Jose Borreguero --- .github/workflows/actions.yml | 2 ++ reduction/test/conftest.py | 7 +++---- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/.github/workflows/actions.yml b/.github/workflows/actions.yml index 915d89d..cc1125e 100644 --- a/.github/workflows/actions.yml +++ b/.github/workflows/actions.yml @@ -24,6 +24,8 @@ jobs: - name: Test with pytest working-directory: ./reduction run: | + echo datasearch.directories=${{ github.workspace }}/reduction/tests/data/liquidsreflectometer-data/nexus/ >> ${{ runner.home }}/.mantid/Mantid.user.properties + cat ${{ runner.home }}/.mantid/Mantid.user.properties git submodule add --force https://code.ornl.gov/sns-hfir-scse/infrastructure/test-data/liquidsreflectometer-data.git tests/data/liquidsreflectometer-data git submodule update --init python -m pytest -vv --cov=. --cov-report=xml --cov-report=term test diff --git a/reduction/test/conftest.py b/reduction/test/conftest.py index e50487a..430382e 100644 --- a/reduction/test/conftest.py +++ b/reduction/test/conftest.py @@ -3,9 +3,7 @@ from pathlib import Path import os -data_dir = Path(__file__).parent.parent / "data" - - +''' def pytest_sessionstart(session): r"""invoked by pytest at the very beginning""" logger = logging.getLogger() @@ -15,6 +13,7 @@ def pytest_sessionstart(session): mantid_config_file = mantid_config_dir / "Mantid.user.properties" write_mode = "a" if mantid_config_file.exists() else "w" # append or create-then-write with open(mantid_config_file, write_mode) as file_handle: - data_path = data_dir / "liquidsreflectometer-data" / "nexus" + data_path = Path(__file__).parent.parent / "tests/data/liquidsreflectometer-data/nexus" file_handle.write(f"datasearch.directories={str(data_path)}") logger.info("Appending data directory to mantid config file") +''' \ No newline at end of file From 707485849c79ed859e90196af3258547bbbd24ae Mon Sep 17 00:00:00 2001 From: Jose Borreguero Date: Fri, 16 Feb 2024 17:57:47 -0500 Subject: [PATCH 10/18] print only mantid config file contents Signed-off-by: Jose Borreguero --- .github/workflows/actions.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/actions.yml b/.github/workflows/actions.yml index cc1125e..ccea8d0 100644 --- a/.github/workflows/actions.yml +++ b/.github/workflows/actions.yml @@ -25,10 +25,11 @@ jobs: working-directory: ./reduction run: | echo datasearch.directories=${{ github.workspace }}/reduction/tests/data/liquidsreflectometer-data/nexus/ >> ${{ runner.home }}/.mantid/Mantid.user.properties + echo ${{ runner.home }}/.mantid/Mantid.user.properties cat ${{ runner.home }}/.mantid/Mantid.user.properties - git submodule add --force https://code.ornl.gov/sns-hfir-scse/infrastructure/test-data/liquidsreflectometer-data.git tests/data/liquidsreflectometer-data - git submodule update --init - python -m pytest -vv --cov=. --cov-report=xml --cov-report=term test + # git submodule add --force https://code.ornl.gov/sns-hfir-scse/infrastructure/test-data/liquidsreflectometer-data.git tests/data/liquidsreflectometer-data + # git submodule update --init + # python -m pytest -vv --cov=. --cov-report=xml --cov-report=term test - name: Upload coverage reports to Codecov uses: codecov/codecov-action@v3 env: From f02003ad1b36bd991f65647aa277ef2d4bf4c052 Mon Sep 17 00:00:00 2001 From: Jose Borreguero Date: Fri, 16 Feb 2024 18:02:33 -0500 Subject: [PATCH 11/18] try with the pytest session setup Signed-off-by: Jose Borreguero --- .github/workflows/actions.yml | 12 ++++++------ reduction/test/conftest.py | 9 ++++----- 2 files changed, 10 insertions(+), 11 deletions(-) diff --git a/.github/workflows/actions.yml b/.github/workflows/actions.yml index ccea8d0..1712325 100644 --- a/.github/workflows/actions.yml +++ b/.github/workflows/actions.yml @@ -24,12 +24,12 @@ jobs: - name: Test with pytest working-directory: ./reduction run: | - echo datasearch.directories=${{ github.workspace }}/reduction/tests/data/liquidsreflectometer-data/nexus/ >> ${{ runner.home }}/.mantid/Mantid.user.properties - echo ${{ runner.home }}/.mantid/Mantid.user.properties - cat ${{ runner.home }}/.mantid/Mantid.user.properties - # git submodule add --force https://code.ornl.gov/sns-hfir-scse/infrastructure/test-data/liquidsreflectometer-data.git tests/data/liquidsreflectometer-data - # git submodule update --init - # python -m pytest -vv --cov=. --cov-report=xml --cov-report=term test + #echo datasearch.directories=${{ github.workspace }}/reduction/tests/data/liquidsreflectometer-data/nexus/ >> ${{ runner.home }}/.mantid/Mantid.user.properties + #echo ${{ runner.home }}/.mantid/Mantid.user.properties + #cat ${{ runner.home }}/.mantid/Mantid.user.properties + git submodule add --force https://code.ornl.gov/sns-hfir-scse/infrastructure/test-data/liquidsreflectometer-data.git tests/data/liquidsreflectometer-data + git submodule update --init + python -m pytest -vv --cov=. --cov-report=xml --cov-report=term test - name: Upload coverage reports to Codecov uses: codecov/codecov-action@v3 env: diff --git a/reduction/test/conftest.py b/reduction/test/conftest.py index 430382e..9058d88 100644 --- a/reduction/test/conftest.py +++ b/reduction/test/conftest.py @@ -3,7 +3,7 @@ from pathlib import Path import os -''' + def pytest_sessionstart(session): r"""invoked by pytest at the very beginning""" logger = logging.getLogger() @@ -13,7 +13,6 @@ def pytest_sessionstart(session): mantid_config_file = mantid_config_dir / "Mantid.user.properties" write_mode = "a" if mantid_config_file.exists() else "w" # append or create-then-write with open(mantid_config_file, write_mode) as file_handle: - data_path = Path(__file__).parent.parent / "tests/data/liquidsreflectometer-data/nexus" - file_handle.write(f"datasearch.directories={str(data_path)}") - logger.info("Appending data directory to mantid config file") -''' \ No newline at end of file + data_path = str(Path(__file__).parent.parent / "tests/data/liquidsreflectometer-data/nexus") + file_handle.write(f"datasearch.directories={data_path}") + logger.info(f"Appending data directory {data_path} to mantid config file {str(mantid_config_file)}") From 56f26f254ac6fb963ee24ccfa2861ecdc6b9ece7 Mon Sep 17 00:00:00 2001 From: Jose Borreguero Date: Fri, 16 Feb 2024 18:55:27 -0500 Subject: [PATCH 12/18] intorduce amend_config context manager Signed-off-by: Jose Borreguero --- .github/workflows/actions.yml | 3 -- reduction/lr_reduction/utils.py | 63 ++++++++++++++++++++++++++++ reduction/test/conftest.py | 21 ++++------ reduction/test/test_reduction.py | 58 +++++++++++++++---------- reduction/test/test_time_resolved.py | 18 +++++--- 5 files changed, 118 insertions(+), 45 deletions(-) create mode 100644 reduction/lr_reduction/utils.py diff --git a/.github/workflows/actions.yml b/.github/workflows/actions.yml index 1712325..915d89d 100644 --- a/.github/workflows/actions.yml +++ b/.github/workflows/actions.yml @@ -24,9 +24,6 @@ jobs: - name: Test with pytest working-directory: ./reduction run: | - #echo datasearch.directories=${{ github.workspace }}/reduction/tests/data/liquidsreflectometer-data/nexus/ >> ${{ runner.home }}/.mantid/Mantid.user.properties - #echo ${{ runner.home }}/.mantid/Mantid.user.properties - #cat ${{ runner.home }}/.mantid/Mantid.user.properties git submodule add --force https://code.ornl.gov/sns-hfir-scse/infrastructure/test-data/liquidsreflectometer-data.git tests/data/liquidsreflectometer-data git submodule update --init python -m pytest -vv --cov=. --cov-report=xml --cov-report=term test diff --git a/reduction/lr_reduction/utils.py b/reduction/lr_reduction/utils.py new file mode 100644 index 0000000..12155ba --- /dev/null +++ b/reduction/lr_reduction/utils.py @@ -0,0 +1,63 @@ +# standard imports +from contextlib import contextmanager +from copy import deepcopy +from typing import Union + +# third-party libraries +from mantid.kernel import ConfigService + + +@contextmanager +def amend_config( + new_config: dict = None, data_dir: Union[str, list] = None, data_dir_insert_mode: str = "prepend" +) -> None: + r""" + Context manager to safely modify Mantid Configuration Service while + the function is executed. + + Parameters + ---------- + new_config + (key, value) pairs to substitute in the configuration service + data_dir + prepend one (when passing a string) or more (when passing a list) + directories to the list of data search directories. Alternatively, replace instead of prepend. + data_dir_insert_mode + How to insert the data directories. Options are: "prepend" (default) and "replace". + """ + modified_keys = list() + backup = dict() + config = ConfigService.Instance() + if new_config is not None: + SEARCH_ARCHIVE = "datasearch.searcharchive" + if SEARCH_ARCHIVE not in new_config: + new_config[SEARCH_ARCHIVE] = "hfir, sns" + DEFAULT_FACILITY = "default.facility" + if DEFAULT_FACILITY not in new_config: + new_config[DEFAULT_FACILITY] = "SNS" + for key, val in new_config.items(): + backup[key] = config[key] + config[key] = val # config does not have an 'update' method + modified_keys.append(key) + if data_dir is not None: + data_dirs = ( + [ + data_dir, + ] + if isinstance(data_dir, str) + else data_dir + ) + key = "datasearch.directories" + backup[key] = deepcopy(config[key]) + # prepend or replace our custom data directories to the list of data search directories + if data_dir_insert_mode == "prepend": + config.setDataSearchDirs(data_dirs + list(config.getDataSearchDirs())) + elif data_dir_insert_mode == "replace": + config.setDataSearchDirs(data_dirs) + else: + raise ValueError(f"Invalid data_dir_insert_mode: {data_dir_insert_mode}") + try: + yield + finally: + for key in modified_keys: + config[key] = backup[key] diff --git a/reduction/test/conftest.py b/reduction/test/conftest.py index 9058d88..94a5b7e 100644 --- a/reduction/test/conftest.py +++ b/reduction/test/conftest.py @@ -1,18 +1,11 @@ # standard imports -import logging from pathlib import Path -import os +# third-party imports +import pytest -def pytest_sessionstart(session): - r"""invoked by pytest at the very beginning""" - logger = logging.getLogger() - # Insert data directory in Mantid config file - mantid_config_dir = Path(os.environ["HOME"]) / ".mantid" - os.makedirs(str(mantid_config_dir), exist_ok=True) # create directory if it doesn't exists - mantid_config_file = mantid_config_dir / "Mantid.user.properties" - write_mode = "a" if mantid_config_file.exists() else "w" # append or create-then-write - with open(mantid_config_file, write_mode) as file_handle: - data_path = str(Path(__file__).parent.parent / "tests/data/liquidsreflectometer-data/nexus") - file_handle.write(f"datasearch.directories={data_path}") - logger.info(f"Appending data directory {data_path} to mantid config file {str(mantid_config_file)}") + +@pytest.fixture(scope="session") +def nexus_dir() -> str: + r"""Absolute path to the event nexus files""" + return str(Path(__file__).parent.parent / "tests/data/liquidsreflectometer-data/nexus") \ No newline at end of file diff --git a/reduction/test/test_reduction.py b/reduction/test/test_reduction.py index 2ef8321..1e0b672 100644 --- a/reduction/test/test_reduction.py +++ b/reduction/test/test_reduction.py @@ -1,15 +1,20 @@ +# standard imports import os +# third-party imports import mantid import mantid.simpleapi as mtd_api import numpy as np + +# lr_reduction imports +from lr_reduction import event_reduction, template, workflow +from lr_reduction.utils import amend_config + + mtd_api.config["default.facility"] = "SNS" mtd_api.config["default.instrument"] = "REF_L" - mantid.kernel.config.setLogLevel(3) -from lr_reduction import event_reduction, template, workflow - def cleanup_partial_files(output_dir, runs): """ @@ -21,17 +26,18 @@ def cleanup_partial_files(output_dir, runs): os.remove(reduced_path) -def test_info(): +def test_info(nexus_dir): """ Test utility functions to get basic info """ - ws_sc = mtd_api.Load("REF_L_198409") + with amend_config(data_dir=nexus_dir): + ws_sc = mtd_api.Load("REF_L_198409") wl_min, wl_max = event_reduction.get_wl_range(ws_sc) assert(wl_min == 13.7) assert(wl_max == 16.3) -def test_full_reduction(): +def test_full_reduction(nexus_dir): """ Test the fill reduction chain """ @@ -41,7 +47,8 @@ def test_full_reduction(): d_refl_all = [] first_run = None for run_number in range(198409, 198417): - ws_sc = mtd_api.Load("REF_L_%s" % run_number) + with amend_config(data_dir=nexus_dir): + ws_sc = mtd_api.Load("REF_L_%s" % run_number) qz_mid, refl, d_refl = template.process_from_template_ws(ws_sc, template_path) if first_run is None: @@ -72,7 +79,7 @@ def test_full_reduction(): cleanup_partial_files(output_dir, range(198409, 198417)) -def test_reduce_workflow(): +def test_reduce_workflow(nexus_dir): template_path = 'data/template.xml' output_dir = 'data/' reduced_path = os.path.join(output_dir, 'REFL_198409_combined_data_auto.txt') @@ -80,7 +87,8 @@ def test_reduce_workflow(): os.remove(reduced_path) for i in range(198409, 198417): - ws = mtd_api.Load("REF_L_%s" % i) + with amend_config(data_dir=nexus_dir): + ws = mtd_api.Load("REF_L_%s" % i) workflow.reduce(ws, template_path, output_dir=output_dir, average_overlap=False) @@ -102,7 +110,7 @@ def test_reduce_workflow(): cleanup_partial_files(output_dir, range(198409, 198417)) -def test_reduce_functional_bck(): +def test_reduce_functional_bck(nexus_dir): template_path = 'data/template_fbck.xml' output_dir = 'data/' reduced_path = os.path.join(output_dir, 'REFL_198409_combined_data_auto.txt') @@ -110,7 +118,8 @@ def test_reduce_functional_bck(): os.remove(reduced_path) for i in range(198409, 198417): - ws = mtd_api.Load("REF_L_%s" % i) + with amend_config(data_dir=nexus_dir): + ws = mtd_api.Load("REF_L_%s" % i) workflow.reduce(ws, template_path, output_dir=output_dir, average_overlap=False, functional_background=True) @@ -136,7 +145,7 @@ def test_reduce_functional_bck(): cleanup_partial_files(output_dir, range(198409, 198417)) -def test_reduce_bck_option_mismatch(): +def test_reduce_bck_option_mismatch(nexus_dir): """ Ask for functional background but pass by a background range with only a single region. This will revert to simple averaging over the range. @@ -148,7 +157,8 @@ def test_reduce_bck_option_mismatch(): os.remove(reduced_path) for i in range(198409, 198417): - ws = mtd_api.Load("REF_L_%s" % i) + with amend_config(data_dir=nexus_dir): + ws = mtd_api.Load("REF_L_%s" % i) sequence_number = ws.getRun().getProperty("sequence_number").value[0] template_data = template.read_template(template_path, sequence_number) template_data.background_roi = template_data.background_roi[:2] @@ -174,7 +184,7 @@ def test_reduce_bck_option_mismatch(): cleanup_partial_files(output_dir, range(198409, 198417)) -def test_reduce_workflow_with_overlap_avg(): +def test_reduce_workflow_with_overlap_avg(nexus_dir): """ Test the complete working, but this time we average the point in the overlap regions. @@ -186,7 +196,8 @@ def test_reduce_workflow_with_overlap_avg(): os.remove(reduced_path) for i in range(198409, 198417): - ws = mtd_api.Load("REF_L_%s" % i) + with amend_config(data_dir=nexus_dir): + ws = mtd_api.Load("REF_L_%s" % i) workflow.reduce(ws, template_path, output_dir=output_dir, average_overlap=True) @@ -208,12 +219,13 @@ def test_reduce_workflow_with_overlap_avg(): cleanup_partial_files(output_dir, range(198409, 198417)) -def test_quick_reduce(): +def test_quick_reduce(nexus_dir): """ Test the quick reduction workflow """ - ws = mtd_api.Load("REF_L_201284") - ws_db = mtd_api.Load("REF_L_201045") + with amend_config(data_dir=nexus_dir): + ws = mtd_api.Load("REF_L_201284") + ws_db = mtd_api.Load("REF_L_201045") _refl = workflow.reduce_explorer(ws, ws_db, center_pixel=145, db_center_pixel=145) reference_path = 'data/reference_r201284_quick.txt' @@ -224,7 +236,7 @@ def test_quick_reduce(): assert(np.fabs(np.sum(_data[i] - _refl[i])) < 1e-10) -def test_reduce_workflow_201282(): +def test_reduce_workflow_201282(nexus_dir): """ Test to reproduce autoreduction output """ @@ -235,7 +247,8 @@ def test_reduce_workflow_201282(): os.remove(reduced_path) for i in range(201282, 201289): - ws = mtd_api.Load("REF_L_%s" % i) + with amend_config(data_dir=nexus_dir): + ws = mtd_api.Load("REF_L_%s" % i) workflow.reduce(ws, template_path, output_dir=output_dir, average_overlap=False) @@ -254,7 +267,7 @@ def test_reduce_workflow_201282(): assert(np.sum((_data[3]-_refl[3])/_refl[3])/len(_refl[3]) < 0.01) -def test_background_subtraction(): +def test_background_subtraction(nexus_dir): """ Test with background subtraction off for the data and on for the normalization """ @@ -265,7 +278,8 @@ def test_background_subtraction(): os.remove(reduced_path) for i in range(198388, 198390): - ws = mtd_api.Load("REF_L_%s" % i) + with amend_config(data_dir=nexus_dir): + ws = mtd_api.Load("REF_L_%s" % i) workflow.reduce(ws, template_path, output_dir=output_dir, average_overlap=False) diff --git a/reduction/test/test_time_resolved.py b/reduction/test/test_time_resolved.py index 34be588..675f47e 100644 --- a/reduction/test/test_time_resolved.py +++ b/reduction/test/test_time_resolved.py @@ -1,9 +1,15 @@ +# standard imports import os + +# third-party imports import numpy as np + +# lr_reduction imports from lr_reduction import time_resolved +from lr_reduction.utils import amend_config -def test_reduce_workflow(): +def test_reduce_workflow(nexus_dir): """ Test the time-resolved reduction that uses a measured reference. It is generally used at 30 Hz but it also works at 60 Hz. @@ -12,11 +18,11 @@ def test_reduce_workflow(): output_dir = 'data/' reduced_path = 'data/reference_rq_avg_overlap.txt' ref_data = np.loadtxt(reduced_path).T - - reduced = time_resolved.reduce_30Hz_slices(198413, 198413, ref_data_60Hz=reduced_path, - template_30Hz=template_path, - time_interval=300, output_dir=output_dir, - scan_index=5, create_plot=False) + with amend_config(data_dir=nexus_dir): + reduced = time_resolved.reduce_30Hz_slices(198413, 198413, ref_data_60Hz=reduced_path, + template_30Hz=template_path, + time_interval=300, output_dir=output_dir, + scan_index=5, create_plot=False) q_long = len(ref_data[0]) q_short = len(reduced[0][0]) From 83c38a4a6d9f70298dc9ce0116c7ad783c5f9d16 Mon Sep 17 00:00:00 2001 From: Jose Borreguero Date: Fri, 16 Feb 2024 19:08:50 -0500 Subject: [PATCH 13/18] check if abbreviation can be resolved Signed-off-by: Jose Borreguero --- .github/workflows/actions.yml | 2 +- reduction/test/test_reduction.py | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/actions.yml b/.github/workflows/actions.yml index 915d89d..0024217 100644 --- a/.github/workflows/actions.yml +++ b/.github/workflows/actions.yml @@ -26,7 +26,7 @@ jobs: run: | git submodule add --force https://code.ornl.gov/sns-hfir-scse/infrastructure/test-data/liquidsreflectometer-data.git tests/data/liquidsreflectometer-data git submodule update --init - python -m pytest -vv --cov=. --cov-report=xml --cov-report=term test + python -m pytest -vv --cov=. --cov-report=xml --cov-report=term test/test_reduction.py::test_info - name: Upload coverage reports to Codecov uses: codecov/codecov-action@v3 env: diff --git a/reduction/test/test_reduction.py b/reduction/test/test_reduction.py index 1e0b672..5e29804 100644 --- a/reduction/test/test_reduction.py +++ b/reduction/test/test_reduction.py @@ -31,6 +31,10 @@ def test_info(nexus_dir): Test utility functions to get basic info """ with amend_config(data_dir=nexus_dir): + from mantid.kernel import ConfigService + config = ConfigService.Instance() + print(f"\nMANTID DATA DIRS = {config['datasearch.directories']}\n") + ws_sc = mtd_api.Load(f"{nexus_dir}/REF_L_198409.nxs.h5") ws_sc = mtd_api.Load("REF_L_198409") wl_min, wl_max = event_reduction.get_wl_range(ws_sc) assert(wl_min == 13.7) From 591e050c1481e5956287e81b85d496a484412fc9 Mon Sep 17 00:00:00 2001 From: Jose Borreguero Date: Fri, 16 Feb 2024 19:46:26 -0500 Subject: [PATCH 14/18] reorder channel sequence Signed-off-by: Jose Borreguero --- environment.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/environment.yml b/environment.yml index 6b957fb..45eab11 100644 --- a/environment.yml +++ b/environment.yml @@ -1,10 +1,9 @@ name: liquid-ref channels: + - mantid/label/main - conda-forge - - default - - mantid + - defaults dependencies: - - python=3.8 - pre-commit - pytest - pytest-cov From 868a4e722c38d90b91a2b3023621873a5650c535 Mon Sep 17 00:00:00 2001 From: Jose Borreguero Date: Fri, 16 Feb 2024 19:51:45 -0500 Subject: [PATCH 15/18] run all tests Signed-off-by: Jose Borreguero --- .github/workflows/actions.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/actions.yml b/.github/workflows/actions.yml index 0024217..915d89d 100644 --- a/.github/workflows/actions.yml +++ b/.github/workflows/actions.yml @@ -26,7 +26,7 @@ jobs: run: | git submodule add --force https://code.ornl.gov/sns-hfir-scse/infrastructure/test-data/liquidsreflectometer-data.git tests/data/liquidsreflectometer-data git submodule update --init - python -m pytest -vv --cov=. --cov-report=xml --cov-report=term test/test_reduction.py::test_info + python -m pytest -vv --cov=. --cov-report=xml --cov-report=term test - name: Upload coverage reports to Codecov uses: codecov/codecov-action@v3 env: From f784f4a199bcc464abbe873fae94b9442ddaa11d Mon Sep 17 00:00:00 2001 From: Jose Borreguero Date: Fri, 16 Feb 2024 19:52:18 -0500 Subject: [PATCH 16/18] remove all the printout Signed-off-by: Jose Borreguero --- reduction/test/test_reduction.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/reduction/test/test_reduction.py b/reduction/test/test_reduction.py index 5e29804..1e0b672 100644 --- a/reduction/test/test_reduction.py +++ b/reduction/test/test_reduction.py @@ -31,10 +31,6 @@ def test_info(nexus_dir): Test utility functions to get basic info """ with amend_config(data_dir=nexus_dir): - from mantid.kernel import ConfigService - config = ConfigService.Instance() - print(f"\nMANTID DATA DIRS = {config['datasearch.directories']}\n") - ws_sc = mtd_api.Load(f"{nexus_dir}/REF_L_198409.nxs.h5") ws_sc = mtd_api.Load("REF_L_198409") wl_min, wl_max = event_reduction.get_wl_range(ws_sc) assert(wl_min == 13.7) From 6e8de64b287f4b328abf271c5db498a34fafee35 Mon Sep 17 00:00:00 2001 From: Jose Borreguero Date: Sat, 17 Feb 2024 16:59:12 -0500 Subject: [PATCH 17/18] fix test_reduce_functional_bck Signed-off-by: Jose Borreguero --- reduction/data/template_fbck.xml | 8 ++++++++ reduction/lr_reduction/reduction_template_reader.py | 2 +- reduction/lr_reduction/template.py | 2 +- reduction/test/conftest.py | 8 +++++++- reduction/test/test_reduction.py | 4 +++- 5 files changed, 20 insertions(+), 4 deletions(-) diff --git a/reduction/data/template_fbck.xml b/reduction/data/template_fbck.xml index 5cbe4bf..58f6bcd 100644 --- a/reduction/data/template_fbck.xml +++ b/reduction/data/template_fbck.xml @@ -14,6 +14,7 @@ 147 N/A True + True 133 150 0 @@ -61,6 +62,7 @@ 147 N/A True + True 133 150 0 @@ -108,6 +110,7 @@ 147 N/A True + True 133 150 0 @@ -155,6 +158,7 @@ 147 N/A True + True 133 150 0 @@ -202,6 +206,7 @@ 147 N/A True + True 133 150 0 @@ -249,6 +254,7 @@ 147 N/A True + True 126 131 153 @@ -296,6 +302,7 @@ 148 N/A True + True 131 151 0 @@ -343,6 +350,7 @@ 148 N/A True + True 131 151 0 diff --git a/reduction/lr_reduction/reduction_template_reader.py b/reduction/lr_reduction/reduction_template_reader.py index 804a8fe..e478a7b 100644 --- a/reduction/lr_reduction/reduction_template_reader.py +++ b/reduction/lr_reduction/reduction_template_reader.py @@ -333,7 +333,7 @@ def from_xml(xml_str): data_set.from_xml_element(item) data_sets.append(data_set) - if len(data_sets)==0: + if len(data_sets) == 0: data_sets = [ReductionParameters()] return data_sets diff --git a/reduction/lr_reduction/template.py b/reduction/lr_reduction/template.py index 80a43ff..87c7f31 100644 --- a/reduction/lr_reduction/template.py +++ b/reduction/lr_reduction/template.py @@ -186,7 +186,7 @@ def process_from_template_ws(ws_sc, template_data, q_summing=False, if template_data.subtract_background: peak_bck = template_data.background_roi if template_data.two_backgrounds is False: - peak_bck = peak_bck[0: 3] # retain only the first background + peak_bck = peak_bck[0: 2] # retain only the first background else: peak_bck = None diff --git a/reduction/test/conftest.py b/reduction/test/conftest.py index 94a5b7e..5d2683d 100644 --- a/reduction/test/conftest.py +++ b/reduction/test/conftest.py @@ -8,4 +8,10 @@ @pytest.fixture(scope="session") def nexus_dir() -> str: r"""Absolute path to the event nexus files""" - return str(Path(__file__).parent.parent / "tests/data/liquidsreflectometer-data/nexus") \ No newline at end of file + return str(Path(__file__).parent.parent / "tests/data/liquidsreflectometer-data/nexus") + + +@pytest.fixture(scope="session") +def template_dir() -> str: + r"""Absolute path to reduction/data/ directory""" + return str(Path(__file__).parent.parent / "data") diff --git a/reduction/test/test_reduction.py b/reduction/test/test_reduction.py index 1e0b672..0adce2b 100644 --- a/reduction/test/test_reduction.py +++ b/reduction/test/test_reduction.py @@ -1,4 +1,5 @@ # standard imports +from pathlib import Path import os # third-party imports @@ -110,7 +111,8 @@ def test_reduce_workflow(nexus_dir): cleanup_partial_files(output_dir, range(198409, 198417)) -def test_reduce_functional_bck(nexus_dir): +def test_reduce_functional_bck(nexus_dir, template_dir): + os.chdir(Path(template_dir).parent) template_path = 'data/template_fbck.xml' output_dir = 'data/' reduced_path = os.path.join(output_dir, 'REFL_198409_combined_data_auto.txt') From afacd4b9c7362102af9f1e92d2002e58caabdb92 Mon Sep 17 00:00:00 2001 From: Jose Borreguero Date: Mon, 19 Feb 2024 10:16:04 -0500 Subject: [PATCH 18/18] validate each data directory passed to ammend config Signed-off-by: Jose Borreguero --- reduction/lr_reduction/utils.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/reduction/lr_reduction/utils.py b/reduction/lr_reduction/utils.py index 12155ba..f2d03ae 100644 --- a/reduction/lr_reduction/utils.py +++ b/reduction/lr_reduction/utils.py @@ -1,6 +1,7 @@ # standard imports from contextlib import contextmanager from copy import deepcopy +from pathlib import Path from typing import Union # third-party libraries @@ -47,6 +48,10 @@ def amend_config( if isinstance(data_dir, str) else data_dir ) + # make sure the data_dirs exists and are directories + for path in data_dirs: + if Path(path).is_dir() is False: + raise ValueError(f"Data directory: {path} does not exist or is not a directory") key = "datasearch.directories" backup[key] = deepcopy(config[key]) # prepend or replace our custom data directories to the list of data search directories