From 7d4f95b0bc26d2ded26dcf3d0b08056828e8336d Mon Sep 17 00:00:00 2001 From: James Beilsten-Edmands <30625594+jbeilstenedmands@users.noreply.github.com> Date: Fri, 1 Sep 2023 13:29:03 +0100 Subject: [PATCH 1/5] Add format class to understand Jungfrau4M serial data from ID29, ESRF --- newsfragments/XXX.feature | 1 + src/dxtbx/format/FormatHDF5ESRFJungfrau4M.py | 163 +++++++++++++++++++ 2 files changed, 164 insertions(+) create mode 100644 newsfragments/XXX.feature create mode 100644 src/dxtbx/format/FormatHDF5ESRFJungfrau4M.py diff --git a/newsfragments/XXX.feature b/newsfragments/XXX.feature new file mode 100644 index 000000000..c374981ed --- /dev/null +++ b/newsfragments/XXX.feature @@ -0,0 +1 @@ +Add format class to read Jungfrau4M serial images from beamline ID29 at ESRF diff --git a/src/dxtbx/format/FormatHDF5ESRFJungfrau4M.py b/src/dxtbx/format/FormatHDF5ESRFJungfrau4M.py new file mode 100644 index 000000000..bd07f7aa5 --- /dev/null +++ b/src/dxtbx/format/FormatHDF5ESRFJungfrau4M.py @@ -0,0 +1,163 @@ +from __future__ import annotations + +import sys + +import h5py + +from scitbx.array_family import flex + +from dxtbx import flumpy +from dxtbx.format.FormatHDF5 import FormatHDF5 + + +class FormatHDF5ESRFJungfrau4M(FormatHDF5): + + # A class to understand still-shot images from ESRF collected on a Jungfrau 4M. + + _cached_mask = None + + @staticmethod + def understand(image_file): + with h5py.File(image_file, "r") as h5_handle: + if len(h5_handle) != 1: + return False + key = list(h5_handle.keys())[0] + if "instrument" not in h5_handle[key]: + return False + if "jungfrau4m_rr_smx" not in h5_handle[key]["instrument"]: + return False + return True + + def _start(self): + super()._start() + image_file = self.get_image_file() + self._h5_handle = h5py.File(image_file, "r") + self.key = list(self._h5_handle.keys())[0] + self.n_images = self._h5_handle[self.key]["instrument"]["jungfrau4m_rr_smx"][ + "data" + ].shape[0] + self.adus_per_photon = self._h5_handle[self.key]["instrument"][ + "jungfrau4m_rr_smx" + ]["detector_information"]["adus_per_photon"] + self.image_size = tuple( + self._h5_handle[self.key]["instrument"]["jungfrau4m_rr_smx"]["data"].shape[ + 1: + ] + ) + wavelength = self._h5_handle[self.key]["instrument"]["jungfrau4m_rr_smx"][ + "beam" + ]["incident_wavelength"][()] + x_pixel_size = ( + self._h5_handle[self.key]["instrument"]["jungfrau4m_rr_smx"][ + "detector_information" + ]["x_pixel_size"][()] + * 1000 + ) # convert m to mm + y_pixel_size = ( + self._h5_handle[self.key]["instrument"]["jungfrau4m_rr_smx"][ + "detector_information" + ]["y_pixel_size"][()] + * 1000 + ) # convert m to mm + distance = ( + self._h5_handle[self.key]["instrument"]["jungfrau4m_rr_smx"][ + "detector_information" + ]["detector_distance"][()] + * 1000 + ) # convert m to mm + beam_center_x = self._h5_handle[self.key]["instrument"]["jungfrau4m_rr_smx"][ + "detector_information" + ]["beam_center_x"][ + () + ] # in px + beam_center_y = self._h5_handle[self.key]["instrument"]["jungfrau4m_rr_smx"][ + "detector_information" + ]["beam_center_y"][ + () + ] # in px + + beam_center_x *= x_pixel_size + beam_center_y *= y_pixel_size + trusted_range = ( + self._h5_handle[self.key]["instrument"]["jungfrau4m_rr_smx"][ + "detector_information" + ]["underload_value"][()], + self._h5_handle[self.key]["instrument"]["jungfrau4m_rr_smx"][ + "detector_information" + ]["saturation_value"][()], + ) + exposure_time = self._h5_handle[self.key]["instrument"]["jungfrau4m_rr_smx"][ + "acquisition" + ]["exposure_time"][()] + + self._detector_model = self._detector_factory.simple( + sensor="UNKNOWN", + distance=distance, + beam_centre=( + beam_center_x, + beam_center_y, + ), + fast_direction="+x", + slow_direction="-y", + pixel_size=( + x_pixel_size, + y_pixel_size, + ), + image_size=(self.image_size[1], self.image_size[0]), + trusted_range=trusted_range, + mask=self.get_static_mask(), + ) + self._beam_model = self._beam_factory.simple(wavelength) + self._scan_model = self._scan_factory.make_scan( + image_range=(1, self.n_images), + exposure_times=exposure_time, + oscillation=(0.0, 0.0), + epochs=list(range(self.n_images)), + ) + self._goniometer_model = self._goniometer_factory.known_axis((0, 1, 0)) + + def get_raw_data(self, index=None): + if index is None: + index = 0 + + data = ( + self._h5_handle[self.key]["measurement"]["data"][index] + / self.adus_per_photon + ) + return flex.double(data.astype(float)) + + def get_num_images(self): + return self.n_images + + def get_beam(self, index=None): + return self._beam(index) + + def _beam(self, index=None): + return self._beam_model + + def get_detector(self, index=None): + return self._detector(index) + + def get_static_mask(self): + if FormatHDF5ESRFJungfrau4M._cached_mask is None: + mask = self._h5_handle[self.key]["instrument"]["jungfrau4m_rr_smx"][ + "detector_information" + ]["pixel_mask"] + mask = flumpy.from_numpy(mask[()]) + mask_array = mask == 0 + FormatHDF5ESRFJungfrau4M._cached_mask = mask_array + return FormatHDF5ESRFJungfrau4M._cached_mask + + def _detector(self, index=None): + return self._detector_model + + def _goniometer(self): + return self._goniometer_model + + def _scan(self): + return self._scan_model + + +if __name__ == "__main__": + for arg in sys.argv[1:]: + print(FormatHDF5ESRFJungfrau4M.understand(arg)) From e7dd338bebac88db968722983dec2df083f701a3 Mon Sep 17 00:00:00 2001 From: DiamondLightSource-build-server Date: Fri, 1 Sep 2023 12:31:01 +0000 Subject: [PATCH 2/5] Rename newsfragments/XXX.feature to newsfragments/659.feature --- newsfragments/{XXX.feature => 659.feature} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename newsfragments/{XXX.feature => 659.feature} (100%) diff --git a/newsfragments/XXX.feature b/newsfragments/659.feature similarity index 100% rename from newsfragments/XXX.feature rename to newsfragments/659.feature From 2f6edf33b23573b8e7cef5e374fad31d19e84879 Mon Sep 17 00:00:00 2001 From: James Beilsten-Edmands <30625594+jbeilstenedmands@users.noreply.github.com> Date: Fri, 23 Feb 2024 15:45:45 +0000 Subject: [PATCH 3/5] Updates to handle master file updates --- src/dxtbx/format/FormatHDF5ESRFJungfrau4M.py | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/src/dxtbx/format/FormatHDF5ESRFJungfrau4M.py b/src/dxtbx/format/FormatHDF5ESRFJungfrau4M.py index bd07f7aa5..01c6633b8 100644 --- a/src/dxtbx/format/FormatHDF5ESRFJungfrau4M.py +++ b/src/dxtbx/format/FormatHDF5ESRFJungfrau4M.py @@ -24,7 +24,10 @@ def understand(image_file): key = list(h5_handle.keys())[0] if "instrument" not in h5_handle[key]: return False - if "jungfrau4m_rr_smx" not in h5_handle[key]["instrument"]: + # instrument name is of form jungfrau4m_rr[X]_smx where X is empty, 4 or another number + if not h5_handle[key]["instrument"].startswith("jungfrau4m_rr"): + return False + if not h5_handle[key]["instrument"].endswith("smx"): return False return True @@ -119,10 +122,14 @@ def _start(self): def get_raw_data(self, index=None): if index is None: index = 0 - + # data can be int32 with adus_per_photon != 1.0 or float16 with adus_per_photon == 1.0 data = ( - self._h5_handle[self.key]["measurement"]["data"][index] - / self.adus_per_photon + ( + self._h5_handle[self.key]["measurement"]["data"][index] + / self.adus_per_photon + ) + if self.adus_per_photon != 1.0 + else self._h5_handle[self.key]["measurement"]["data"][index] ) return flex.double(data.astype(float)) From 00815067936ec28a1d1dfce0df088e1af4b9509e Mon Sep 17 00:00:00 2001 From: James Beilsten-Edmands <30625594+jbeilstenedmands@users.noreply.github.com> Date: Fri, 23 Feb 2024 16:12:35 +0000 Subject: [PATCH 4/5] Correctly use instrument name in format class --- src/dxtbx/format/FormatHDF5ESRFJungfrau4M.py | 66 ++++++-------------- 1 file changed, 18 insertions(+), 48 deletions(-) diff --git a/src/dxtbx/format/FormatHDF5ESRFJungfrau4M.py b/src/dxtbx/format/FormatHDF5ESRFJungfrau4M.py index 01c6633b8..3854fb40a 100644 --- a/src/dxtbx/format/FormatHDF5ESRFJungfrau4M.py +++ b/src/dxtbx/format/FormatHDF5ESRFJungfrau4M.py @@ -25,9 +25,10 @@ def understand(image_file): if "instrument" not in h5_handle[key]: return False # instrument name is of form jungfrau4m_rr[X]_smx where X is empty, 4 or another number - if not h5_handle[key]["instrument"].startswith("jungfrau4m_rr"): + instrument = list(h5_handle[key]["instrument"].keys())[0] + if not instrument.startswith("jungfrau4m_rr"): return False - if not h5_handle[key]["instrument"].endswith("smx"): + if not instrument.endswith("smx"): return False return True @@ -36,62 +37,31 @@ def _start(self): image_file = self.get_image_file() self._h5_handle = h5py.File(image_file, "r") self.key = list(self._h5_handle.keys())[0] - self.n_images = self._h5_handle[self.key]["instrument"]["jungfrau4m_rr_smx"][ - "data" - ].shape[0] - self.adus_per_photon = self._h5_handle[self.key]["instrument"][ - "jungfrau4m_rr_smx" - ]["detector_information"]["adus_per_photon"] - self.image_size = tuple( - self._h5_handle[self.key]["instrument"]["jungfrau4m_rr_smx"]["data"].shape[ - 1: - ] - ) - wavelength = self._h5_handle[self.key]["instrument"]["jungfrau4m_rr_smx"][ - "beam" - ]["incident_wavelength"][()] + self.instrument_name = list(self._h5_handle[self.key]["instrument"].keys())[0] + instrument = self._h5_handle[self.key]["instrument"][self.instrument_name] + self.n_images = instrument["data"].shape[0] + self.adus_per_photon = instrument["detector_information"]["adus_per_photon"] + self.image_size = tuple(instrument["data"].shape[1:]) + wavelength = instrument["beam"]["incident_wavelength"][()] x_pixel_size = ( - self._h5_handle[self.key]["instrument"]["jungfrau4m_rr_smx"][ - "detector_information" - ]["x_pixel_size"][()] - * 1000 + instrument["detector_information"]["x_pixel_size"][()] * 1000 ) # convert m to mm y_pixel_size = ( - self._h5_handle[self.key]["instrument"]["jungfrau4m_rr_smx"][ - "detector_information" - ]["y_pixel_size"][()] - * 1000 + instrument["detector_information"]["y_pixel_size"][()] * 1000 ) # convert m to mm distance = ( - self._h5_handle[self.key]["instrument"]["jungfrau4m_rr_smx"][ - "detector_information" - ]["detector_distance"][()] - * 1000 + instrument["detector_information"]["detector_distance"][()] * 1000 ) # convert m to mm - beam_center_x = self._h5_handle[self.key]["instrument"]["jungfrau4m_rr_smx"][ - "detector_information" - ]["beam_center_x"][ - () - ] # in px - beam_center_y = self._h5_handle[self.key]["instrument"]["jungfrau4m_rr_smx"][ - "detector_information" - ]["beam_center_y"][ - () - ] # in px + beam_center_x = instrument["detector_information"]["beam_center_x"][()] # in px + beam_center_y = instrument["detector_information"]["beam_center_y"][()] # in px beam_center_x *= x_pixel_size beam_center_y *= y_pixel_size trusted_range = ( - self._h5_handle[self.key]["instrument"]["jungfrau4m_rr_smx"][ - "detector_information" - ]["underload_value"][()], - self._h5_handle[self.key]["instrument"]["jungfrau4m_rr_smx"][ - "detector_information" - ]["saturation_value"][()], + instrument["detector_information"]["underload_value"][()], + instrument["detector_information"]["saturation_value"][()], ) - exposure_time = self._h5_handle[self.key]["instrument"]["jungfrau4m_rr_smx"][ - "acquisition" - ]["exposure_time"][()] + exposure_time = instrument["acquisition"]["exposure_time"][()] self._detector_model = self._detector_factory.simple( sensor="UNKNOWN", @@ -147,7 +117,7 @@ def get_detector(self, index=None): def get_static_mask(self): if FormatHDF5ESRFJungfrau4M._cached_mask is None: - mask = self._h5_handle[self.key]["instrument"]["jungfrau4m_rr_smx"][ + mask = self._h5_handle[self.key]["instrument"][self.instrument_name][ "detector_information" ]["pixel_mask"] mask = flumpy.from_numpy(mask[()]) From 557fbc634aacf6e9907903b25dfbb6f4a9fbf6df Mon Sep 17 00:00:00 2001 From: James Beilsten-Edmands <30625594+jbeilstenedmands@users.noreply.github.com> Date: Mon, 26 Feb 2024 08:55:48 +0000 Subject: [PATCH 5/5] Add comment --- src/dxtbx/format/FormatHDF5ESRFJungfrau4M.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/dxtbx/format/FormatHDF5ESRFJungfrau4M.py b/src/dxtbx/format/FormatHDF5ESRFJungfrau4M.py index 3854fb40a..64f41f5f5 100644 --- a/src/dxtbx/format/FormatHDF5ESRFJungfrau4M.py +++ b/src/dxtbx/format/FormatHDF5ESRFJungfrau4M.py @@ -87,6 +87,8 @@ def _start(self): oscillation=(0.0, 0.0), epochs=list(range(self.n_images)), ) + # Add a placeholder goniometer model, which has no practical effect on processing as the oscillation is 0. + # Some dxtbx format logic assumes both or neither scan + goniometer are None for still images self._goniometer_model = self._goniometer_factory.known_axis((0, 1, 0)) def get_raw_data(self, index=None):