From 9cbe8d599c0a830f2c7f00f366f562f2d32d9167 Mon Sep 17 00:00:00 2001 From: Nicholas Devenish Date: Wed, 2 Oct 2024 08:37:19 +0100 Subject: [PATCH 1/4] MNT: Pin old micromamba to avoid issue https://github.com/mamba-org/mamba/issues/3393 --- .azure-pipelines/bootstrap.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.azure-pipelines/bootstrap.py b/.azure-pipelines/bootstrap.py index 661e3b25d..a384e7973 100644 --- a/.azure-pipelines/bootstrap.py +++ b/.azure-pipelines/bootstrap.py @@ -43,13 +43,13 @@ def install_micromamba(python): """Download and install Micromamba""" if sys.platform.startswith("linux"): member = "bin/micromamba" - url = "https://micromamba.snakepit.net/api/micromamba/linux-64/latest" + url = "https://micro.mamba.pm/api/micromamba/linux-64/1.5.10" elif sys.platform == "darwin": member = "bin/micromamba" - url = "https://micromamba.snakepit.net/api/micromamba/osx-64/latest" + url = "https://micro.mamba.pm/api/micromamba/osx-64/1.5.10" elif os.name == "nt": member = "Library/bin/micromamba.exe" - url = "https://micromamba.snakepit.net/api/micromamba/win-64/latest" + url = "https://micro.mamba.pm/api/micromamba/win-64/1.5.10" else: raise NotImplementedError(f"Unsupported platform {os.name} / {sys.platform}") mamba_prefix = os.path.realpath("micromamba") From 2bde8b81548957c5d21457594a2b051eac9731e5 Mon Sep 17 00:00:00 2001 From: David McDonagh <60879630+toastisme@users.noreply.github.com> Date: Wed, 2 Oct 2024 23:11:44 +0100 Subject: [PATCH 2/4] Refactor panel positions of FormatISISSXD (#762) * Refactor panel positions of FormatISISSXD to account for differences depending on the date of data collection. --- newsfragments/762.bugfix | 1 + src/dxtbx/format/FormatISISSXD.py | 212 ++++++++++++++++++++++-------- 2 files changed, 159 insertions(+), 54 deletions(-) create mode 100644 newsfragments/762.bugfix diff --git a/newsfragments/762.bugfix b/newsfragments/762.bugfix new file mode 100644 index 000000000..f0030e44f --- /dev/null +++ b/newsfragments/762.bugfix @@ -0,0 +1 @@ +Refactor panel positions of FormatISISSXD to account for differences in panel positions depending on the date of data collection. diff --git a/src/dxtbx/format/FormatISISSXD.py b/src/dxtbx/format/FormatISISSXD.py index 549fe3064..e013e29d6 100644 --- a/src/dxtbx/format/FormatISISSXD.py +++ b/src/dxtbx/format/FormatISISSXD.py @@ -135,69 +135,163 @@ def _get_panel_trusted_range(self) -> Tuple[int, int]: def _get_panel_type(self) -> str: return "SENSOR_PAD" - def _panel_0_params(self): - if self._panel_0_flipped(): - return { - "slow_axis": (0.793, 0.0, 0.609), - "fast_axis": (0.0, -1.0, 0.0), - "origin": (60.81, 96.0, -236.946), - } - return { - "fast_axis": (-0.793, 0.0, -0.609), - "slow_axis": (0.0, 1.0, 0.0), - "origin": (213.099, -96.0, -120.041), - } - def _get_panel_origins(self) -> Tuple[Tuple[float, float, float]]: # (mm) - return ( - self._panel_0_params()["origin"], - (224.999, -96.0, 96.0), - (60.809, -96.0, 236.945), - (-214.172, -96.0, 118.198), - (-224.999, -96.0, -96.0), - (-60.809, -96.0, -236.945), - (127.534, -256.614, 96.0), - (-96.0, -256.614, 127.534), - (-123.036, -258.801, -96.0), - (96.0, -256.614, -127.534), - (96.0, -278.0, 96.0), - ) + + start_date = self.get_start_date() + assert start_date is not None + year, _, _ = start_date.split("-") + year = int(year) + if year < 2020: + return ( + (213.066, -96.0, -120.018), + (224.999, -96.0, 96.0), + (60.809, -96.0, 236.945), + (-214.172, -96.0, 118.198), + (-224.999, -96.0, -96.0), + (-60.809, -96.0, -236.945), + (127.534, -256.614, 96.0), + (-96.0, -256.614, 127.534), + (-123.036, -258.801, -96.0), + (96.0, -256.614, -127.534), + (96.0, -278.0, 96.0), + ) + + elif year < 2024: + return ( + (60.81, 96.0, -236.946), + (224.999, -96.0, 96.0), + (60.809, -96.0, 236.945), + (-214.172, -96.0, 118.198), + (-224.999, -96.0, -96.0), + (-60.809, -96.0, -236.945), + (127.534, -256.614, 96.0), + (-96.0, -256.614, 127.534), + (-123.036, -258.801, -96.0), + (96.0, -256.614, -127.534), + (96.0, -278.0, 96.0), + ) + else: + return ( + (60.81, 96.0, -236.946), + (224.999, 96.0, -96.0), + (213.065, 96.0, 120.017), + (-62.876000000000005, 96.0, 236.46999999999997), + (-224.999, 96.0, 96.0), + (-213.065, 96.0, -120.017), + (260.974, -118.56599999999997, -96.0), + (96.0, -118.56599999999997, 260.974), + (-258.78, -123.05699999999999, 96.0), + (-96.0, -118.56599999999997, -260.974), + (96.0, -278.0, 96.0), + ) def _panel_0_flipped(self) -> bool: - if self._nxs_file["raw_data_1"]["run_number"][0] > 30000: + start_date = self.get_start_date() + assert start_date is not None + year, _, _ = start_date.split("-") + if int(year) < 2024: return True return False def _get_panel_slow_axes(self) -> Tuple[Tuple[float, float, float]]: - return ( - self._panel_0_params()["slow_axis"], - (0.0, 1.0, 0.0), - (0.0, 1.0, 0.0), - (0.0, 1.0, 0.0), - (0.0, 1.0, 0.0), - (0.0, 1.0, 0.0), - (0.695, 0.719, -0.0), - (0.0, 0.719, 0.695), - (-0.707, 0.707, -0.0), - (0.0, 0.719, -0.695), - (-0.0, 0.0, -1.0), - ) + start_date = self.get_start_date() + assert start_date is not None + year, _, _ = start_date.split("-") + year = int(year) + if year < 2020: + return ( + (0.0, 1.0, 0.0), + (0.0, 1.0, 0.0), + (0.0, 1.0, 0.0), + (0.0, 1.0, 0.0), + (0.0, 1.0, 0.0), + (0.0, 1.0, 0.0), + (0.695, 0.719, -0.0), + (0.0, 0.719, 0.695), + (-0.707, 0.707, -0.0), + (0.0, 0.719, -0.695), + (-0.0, 0.0, -1.0), + ) + elif year < 2024: + return ( + (0.793, 0.0, 0.609), + (0.0, 1.0, 0.0), + (0.0, 1.0, 0.0), + (0.0, 1.0, 0.0), + (0.0, 1.0, 0.0), + (0.0, 1.0, 0.0), + (0.695, 0.719, -0.0), + (0.0, 0.719, 0.695), + (-0.707, 0.707, -0.0), + (0.0, 0.719, -0.695), + (-0.0, 0.0, -1.0), + ) + + else: + return ( + (0.0, -1.0, 0.0), + (0.0, -1.0, 0.0), + (0.0, -1.0, 0.0), + (0.0, -1.0, 0.0), + (0.0, -1.0, 0.0), + (0.0, -1.0, 0.0), + (-0.695, -0.719, 0.0), + (0.0, -0.719, -0.695), + (0.707, -0.707, 0.0), + (0.0, -0.719, 0.695), + (-0.0, 0.0, -1.0), + ) def _get_panel_fast_axes(self) -> Tuple[Tuple[float, float, float]]: - return ( - self._panel_0_params()["fast_axis"], - (-0.0, -0.0, -1.0), - (0.793, -0.0, -0.609), - (0.788, -0.0, 0.616), - (-0.0, -0.0, 1.0), - (-0.793, -0.0, 0.609), - (0.0, -0.0, -1.0), - (1.0, -0.0, -0.0), - (-0.0, -0.0, 1.0), - (-1.0, -0.0, -0.0), - (-1.0, -0.0, -0.0), - ) + start_date = self.get_start_date() + assert start_date is not None + year, _, _ = start_date.split("-") + year = int(year) + if year < 2020: + return ( + (-0.793, 0.0, -0.609), + (-0.0, -0.0, -1.0), + (0.793, -0.0, -0.609), + (0.788, -0.0, 0.616), + (-0.0, -0.0, 1.0), + (-0.793, -0.0, 0.609), + (0.0, -0.0, -1.0), + (1.0, -0.0, -0.0), + (-0.0, -0.0, 1.0), + (-1.0, -0.0, -0.0), + (-1.0, -0.0, -0.0), + ) + + elif year < 2024: + return ( + (0.0, -1.0, 0.0), + (-0.0, -0.0, -1.0), + (0.793, -0.0, -0.609), + (0.788, -0.0, 0.616), + (-0.0, -0.0, 1.0), + (-0.793, -0.0, 0.609), + (0.0, -0.0, -1.0), + (1.0, -0.0, -0.0), + (-0.0, -0.0, 1.0), + (-1.0, -0.0, -0.0), + (-1.0, -0.0, -0.0), + ) + + else: + return ( + (0.793, 0.0, 0.609), + (-0.0, -0.0, 1.0), + (-0.793, 0.0, 0.609), + (-0.788, 0.0, -0.616), + (0.0, 0.0, -1.0), + (0.793, 0.0, -0.609), + (0.0, 0.0, 1.0), + (-1.0, 0.0, 0.0), + (0.0, 0.0, -1.0), + (1.0, 0.0, 0.0), + (-1.0, 0.0, 0.0), + ) def _get_panel_projections_2d(self) -> dict: """ @@ -386,5 +480,15 @@ def get_flattened_pixel_data( tuple(self._nxs_file["raw_data_1/detector_1/counts"][0, idx, :].tolist()), ) - def get_proton_charge(self): + def get_proton_charge(self) -> float: return float(self._nxs_file["raw_data_1/proton_charge"][0]) + + def get_start_date(self) -> str: + """ + Date of experiment in YYYY-MM-DD format + """ + try: + raw_date = self._nxs_file["raw_data_1/start_time"][0].decode() + return raw_date.split("T")[0] + except IndexError: + return None From 898072e7530a53a1e5e778290e333cd9fc09f4fc Mon Sep 17 00:00:00 2001 From: David Waterman Date: Mon, 7 Oct 2024 16:56:30 +0100 Subject: [PATCH 3/4] Better error message, avoiding `UnicodeDecodeError` (#763) Catch UnicodeDecodeError and raise more appropriate InvalidExperimentListError Fixes https://github.com/dials/dials/issues/2453 --- newsfragments/763.bugfix | 1 + src/dxtbx/model/experiment_list.py | 11 ++++++++--- 2 files changed, 9 insertions(+), 3 deletions(-) create mode 100644 newsfragments/763.bugfix diff --git a/newsfragments/763.bugfix b/newsfragments/763.bugfix new file mode 100644 index 000000000..d66758c6f --- /dev/null +++ b/newsfragments/763.bugfix @@ -0,0 +1 @@ +Raise a more suitable error message when failing to load an experiment list. diff --git a/src/dxtbx/model/experiment_list.py b/src/dxtbx/model/experiment_list.py index 1720c62e2..9145bcf65 100644 --- a/src/dxtbx/model/experiment_list.py +++ b/src/dxtbx/model/experiment_list.py @@ -870,9 +870,14 @@ def from_json_file(filename, check_format=True): """Load an experiment list from a json file.""" filename = os.path.abspath(filename) directory = os.path.dirname(filename) - with open(filename) as infile: - return ExperimentListFactory.from_json( - infile.read(), check_format=check_format, directory=directory + try: + with open(filename) as infile: + return ExperimentListFactory.from_json( + infile.read(), check_format=check_format, directory=directory + ) + except UnicodeDecodeError: + raise InvalidExperimentListError( + f"Cannot interpret {filename} as an ExperimentList" ) @staticmethod From 590713d39e9ec55dda7a41dfd29a5bbe0c86bf8a Mon Sep 17 00:00:00 2001 From: David McDonagh <60879630+toastisme@users.noreply.github.com> Date: Tue, 8 Oct 2024 22:33:42 +0100 Subject: [PATCH 4/4] Add FormatESSNMX to read data from the NMX detector at ESS. (#764) * Add FormatESSNMX to read data from the NMX detector at ESS. --- newsfragments/764.feature | 1 + src/dxtbx/format/FormatESSNMX.py | 301 +++++++++++++++++++++++++++++++ 2 files changed, 302 insertions(+) create mode 100644 newsfragments/764.feature create mode 100644 src/dxtbx/format/FormatESSNMX.py diff --git a/newsfragments/764.feature b/newsfragments/764.feature new file mode 100644 index 000000000..fd656ad03 --- /dev/null +++ b/newsfragments/764.feature @@ -0,0 +1 @@ +Add format class to read data from the NMX ESS detector. diff --git a/src/dxtbx/format/FormatESSNMX.py b/src/dxtbx/format/FormatESSNMX.py new file mode 100644 index 000000000..3e6b5190a --- /dev/null +++ b/src/dxtbx/format/FormatESSNMX.py @@ -0,0 +1,301 @@ +from __future__ import annotations + +import logging +from typing import List, Tuple + +import h5py +import numpy as np + +import cctbx.array_family.flex as flex + +import dxtbx_flumpy as flumpy +from dxtbx import IncorrectFormatError +from dxtbx.format.FormatHDF5 import FormatHDF5 +from dxtbx.model import Detector +from dxtbx.model.beam import BeamFactory, PolychromaticBeam, Probe +from dxtbx.model.goniometer import Goniometer, GoniometerFactory +from dxtbx.model.scan import Scan, ScanFactory + +logger = logging.getLogger(__name__) + + +class FormatESSNMX(FormatHDF5): + """ + Class to read files from NMX + https://europeanspallationsource.se/instruments/nmx + preprocessed files in scipp to obtain binned data + """ + + def __init__(self, image_file, **kwargs) -> None: + if not FormatESSNMX.understand(image_file): + raise IncorrectFormatError(self, image_file) + self._nxs_file = h5py.File(image_file, "r") + self._raw_data = None + + @staticmethod + def understand(image_file: str) -> bool: + try: + return FormatESSNMX.is_nmx_file(image_file) + except (IOError, KeyError): + return False + + @staticmethod + def is_nmx_file(image_file: str) -> bool: + def get_name(image_file): + try: + with h5py.File(image_file, "r") as handle: + return handle["NMX_data"].attrs["name"] + except (IOError, KeyError, AttributeError): + return "" + + return get_name(image_file) == "NMX" + + def get_instrument_name(self) -> str: + return "NMX" + + def get_experiment_description(self) -> str: + return "Simulated data" + + def _load_raw_data(self) -> None: + raw_data = [] + image_size = self._get_image_size() + total_pixels = image_size[0] * image_size[1] + num_images = self.get_num_images() + for i in range(self._get_num_panels()): + spectra = self._nxs_file["NMX_data"]["detector_1"]["counts"][ + 0, total_pixels * i : total_pixels * (i + 1), : + ] + spectra = np.reshape(spectra, (image_size[0], image_size[1], num_images)) + raw_data.append(flumpy.from_numpy(spectra)) + + self._raw_data = tuple(raw_data) + + def get_raw_data( + self, index: int, use_loaded_data: bool = False + ) -> Tuple[flex.int]: + raw_data = [] + image_size = self._get_image_size() + total_pixels = image_size[0] * image_size[1] + + if use_loaded_data: + if self._raw_data is None: + self._load_raw_data() + for panel in self._raw_data: + data = panel[:, :, index : index + 1] + data.reshape(flex.grid(panel.all()[0], panel.all()[1])) + data.matrix_transpose_in_place() + raw_data.append(data) + + else: + for i in range(self._get_num_panels()): + spectra = self._nxs_file["NMX_data"]["detector_1"]["counts"][ + 0, total_pixels * i : total_pixels * (i + 1), index : index + 1 + ] + spectra = np.reshape(spectra, image_size) + raw_data.append(flumpy.from_numpy(np.ascontiguousarray(spectra))) + + return tuple(raw_data) + + def _get_time_channel_bins(self) -> List[float]: + # (usec) + return self._nxs_file["NMX_data"]["detector_1"]["t_bin"][:] * 10**6 + + def _get_time_of_flight(self) -> List[float]: + # (usec) + bins = self._get_time_channel_bins() + return [float((bins[i] + bins[i + 1]) * 0.5) for i in range(len(bins) - 1)] + + def get_num_images(self) -> int: + return len(self._get_time_of_flight()) + + def get_detector(self, index: int = None) -> Detector: + num_panels = self._get_num_panels() + panel_names = self._get_panel_names() + panel_type = self._get_panel_type() + image_size = self._get_image_size() + trusted_range = self._get_panel_trusted_range() + pixel_size = self._get_pixel_size() + fast_axes = self._get_panel_fast_axes() + slow_axes = self._get_panel_slow_axes() + panel_origins = self._get_panel_origins() + gain = self._get_panel_gain() + panel_projections = self._get_panel_projections_2d() + detector = Detector() + root = detector.hierarchy() + + for i in range(num_panels): + panel = root.add_panel() + panel.set_type(panel_type) + panel.set_name(panel_names[i]) + panel.set_image_size(image_size) + panel.set_trusted_range(trusted_range) + panel.set_pixel_size(pixel_size) + panel.set_local_frame(fast_axes[i], slow_axes[i], panel_origins[i]) + panel.set_gain(gain) + r, t = panel_projections[i] + r = tuple(map(int, r)) + t = tuple(map(int, t)) + panel.set_projection_2d(r, t) + + return detector + + def _get_num_panels(self) -> int: + return self._nxs_file["NMX_data/instrument"].attrs["nr_detector"] + + def _get_panel_names(self) -> List[str]: + return [ + "%02d" % (i + 1) + for i in range(self._nxs_file["NMX_data/instrument"].attrs["nr_detector"]) + ] + + def _get_panel_type(self) -> str: + return "SENSOR_PAD" + + def _get_image_size(self) -> Tuple[int, int]: + # (px) + return (1280, 1280) + + def _get_panel_trusted_range(self) -> Tuple[int, int]: + # 4 * 1280**2 plus buffer + return (-1, 7000000) + + def _get_pixel_size(self) -> Tuple[float, float]: + # (mm) + return (0.4, 0.4) + + def _get_panel_fast_axes(self) -> Tuple[Tuple[float, float, float]]: + return ((1.0, 0.0, 0.0), (0.0, 0.0, 1.0), (0.0, 0.0, -1.0)) + + def _get_panel_slow_axes(self) -> Tuple[Tuple[float, float, float]]: + return ((0.0, 1.0, 0.0), (0.0, 1.0, 0.0), (0.0, 1.0, 0.0)) + + def _get_panel_origins(self) -> Tuple[Tuple[float, float, float]]: + # (mm) + return ((-250, -250.0, -292.0), (290, -250.0, -250), (-290, -250.0, 250.0)) + + def _get_panel_projections_2d(self) -> dict[int : Tuple[Tuple, Tuple]]: + p_w, p_h = self._get_image_size() + p_w += 10 + p_h += 10 + panel_pos = { + 0: ((-1, 0, 0, -1), (p_h, 0)), + 1: ((-1, 0, 0, -1), (p_h, p_w)), + 2: ((-1, 0, 0, -1), (p_h, -p_w)), + } + + return panel_pos + + def get_beam(self, index: int = None) -> PolychromaticBeam: + direction = self._get_sample_to_source_direction() + distance = self._get_sample_to_source_distance() + wavelength_range = self._get_wavelength_range() + return BeamFactory.make_polychromatic_beam( + direction=direction, + sample_to_source_distance=distance, + probe=Probe.neutron, + wavelength_range=wavelength_range, + ) + + def _get_sample_to_source_direction(self) -> Tuple[float, float, float]: + return (0, 0, 1) + + def _get_wavelength_range(self) -> Tuple[float, float]: + # (A) + return (1.8, 2.55) + + def _get_sample_to_source_distance(self) -> float: + try: + dist = abs(self._nxs_file["NMX_data/NXsource/distance"][...]) * 1000 + return dist + except (KeyError, ValueError): + logger.warning("sample to moderator_distance not found, using dummy value") + return 157406 + + def _get_panel_gain(self) -> float: + return 1.0 + + def get_goniometer_phi_angle(self) -> float: + return self.get_goniometer_orientations()[0] + + def get_goniometer(self, index: int = None) -> Goniometer: + rotation_axis = (0.0, 1.0, 0.0) + fixed_rotation = (1.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 1.0) + goniometer = GoniometerFactory.make_goniometer(rotation_axis, fixed_rotation) + try: + angles = self.get_goniometer_orientations() + except KeyError: + return goniometer + axes = ((1, 0, 0), (0, 1, 0), (0, 0, 1)) + for idx, angle in enumerate(angles): + goniometer.rotate_around_origin(axes[idx], -angle) + return goniometer + + def get_goniometer_orientations(self) -> Tuple[float, float, float]: + # Angles in deg along x, y, z + return self._nxs_file["NMX_data/crystal_orientation"][...] + + def get_scan(self, index=None) -> Scan: + image_range = (1, self.get_num_images()) + properties = {"time_of_flight": tuple(self._get_time_of_flight())} + return ScanFactory.make_scan_from_properties( + image_range=image_range, properties=properties + ) + + def get_flattened_data( + self, image_range: None | Tuple = None, scale_data: bool = True + ) -> Tuple[flex.int]: + """ + Image data summed along the time-of-flight direction + """ + + panel_size = self._get_image_size() + total_pixels = panel_size[0] * panel_size[1] + max_val = None + num_tof_bins = len(self._get_time_channel_bins()) - 1 + raw_data = [] + for panel_idx in range(self._get_num_panels()): + panel_data = self._nxs_file["NMX_data"]["detector_1"]["counts"][ + 0, total_pixels * panel_idx : total_pixels * (panel_idx + 1), : + ] + panel_data = np.reshape( + panel_data, (panel_size[0], panel_size[1], num_tof_bins) + ) + if image_range is not None: + assert ( + len(image_range) == 2 + ), "expected image_range to be only two values" + assert ( + image_range[0] >= 0 and image_range[0] < image_range[1] + ), "image_range[0] out of range" + assert image_range[1] <= num_tof_bins, "image_range[1] out of range" + panel_data = np.sum( + panel_data[:, :, image_range[0] : image_range[1]], axis=2 + ).T + + else: + panel_data = np.sum(panel_data, axis=2).T + panel_max_val = np.max(panel_data) + if max_val is None or max_val < panel_max_val: + max_val = panel_max_val + raw_data.append(panel_data) + + if scale_data: + return tuple([(i / max_val).tolist() for i in raw_data]) + + return tuple([i.tolist() for i in raw_data]) + + def get_flattened_pixel_data( + self, panel_idx: int, x: int, y: int + ) -> Tuple[Tuple, Tuple]: + time_channels = self._get_time_of_flight() + panel_size = self._get_image_size() + height = panel_size[1] + total_pixels = panel_size[0] * panel_size[1] + idx = (panel_idx * total_pixels) + panel_idx + x * height + y + return ( + time_channels, + tuple(self._nxs_file["NMX_data/detector_1/counts"][0, idx, :].tolist()), + ) + + def get_proton_charge(self) -> float: + return self._nxs_file["NMX_data"]["proton_charge"][...]