From 1d45ceca84c86da8e9fc2502a295055fd4023955 Mon Sep 17 00:00:00 2001 From: James Beilsten-Edmands <30625594+jbeilstenedmands@users.noreply.github.com> Date: Wed, 13 Dec 2023 10:05:57 +0000 Subject: [PATCH] Update more programs and revert data reduction changes --- src/dials/command_line/anvil_correction.py | 11 +------- src/dials/command_line/cosym.py | 9 +------ src/dials/command_line/integrate.py | 3 ++- .../command_line/refine_bravais_settings.py | 9 +++---- src/dials/command_line/scale.py | 11 +++----- src/dials/command_line/shadow_plot.py | 6 ++--- .../command_line/stereographic_projection.py | 3 +++ src/dials/command_line/symmetry.py | 27 ++----------------- src/dials/command_line/two_theta_offset.py | 5 ++++ src/dials/command_line/two_theta_refine.py | 16 +++++------ src/dials/command_line/unit_cell_histogram.py | 3 ++- src/dials/util/multi_dataset_handling.py | 25 +++++++++++++---- 12 files changed, 54 insertions(+), 74 deletions(-) diff --git a/src/dials/command_line/anvil_correction.py b/src/dials/command_line/anvil_correction.py index e48bd8058f..20a0f46444 100644 --- a/src/dials/command_line/anvil_correction.py +++ b/src/dials/command_line/anvil_correction.py @@ -36,7 +36,6 @@ import dials.util.log from dials.array_family import flex -from dials.util.multi_dataset_handling import parse_multiple_datasets from dials.util.options import ArgumentParser, reflections_and_experiments_from_files Vector = Sequence[SupportsFloat] @@ -321,15 +320,8 @@ def run(args: List[str] = None, phil: libtbx.phil.scope = phil_scope) -> None: ) from dials.util.multi_dataset_handling import Expeditor - # Work around parse_multiple_datasets dropping unindexed reflections. - # unindexed = flex.reflection_table() - # for r_table in reflections_list: - # unindexed.extend(r_table.select(r_table["id"] == -1)) - # Get a single reflection table per experiment object. - reflections_list = parse_multiple_datasets(reflections_list) expeditor = Expeditor(experiments, reflections_list) experiments, reflections_list = expeditor.filter_experiments_with_crystals() - unindexed = expeditor.get_unindexed() # Record the density of diamond in g·cm⁻³ (for consistency with NIST tables, # https://doi.org/10.18434/T4D01F). @@ -356,8 +348,7 @@ def run(args: List[str] = None, phil: libtbx.phil.scope = phil_scope) -> None: experiments.as_file(params.output.experiments) logger.info("Writing the reflection table to %s", params.output.reflections) # Collate reflections into a single reflection table and save it to file. - unindexed.extend(reflections) - unindexed.as_file(params.output.reflections) + reflections.as_file(params.output.reflections) # Keep this minimal. Try to keep the command-line behaviour neatly encapsulated in run. diff --git a/src/dials/command_line/cosym.py b/src/dials/command_line/cosym.py index a308ad8334..ae3ccea57b 100644 --- a/src/dials/command_line/cosym.py +++ b/src/dials/command_line/cosym.py @@ -415,14 +415,7 @@ def run(args=None): if params.output.html or params.output.json: register_default_cosym_observers(cosym_instance) cosym_instance.run() - experiments, reflections = expeditor.combine_experiments_for_output( - cosym_instance._experiments, cosym_instance._reflections - ) - logger.info("Saving reindexed experiments to %s", params.output.experiments) - experiments.as_file(params.output.experiments) - logger.info("Saving reindexed reflections to %s", params.output.reflections) - reflections.as_file(params.output.reflections) - # cosym_instance.export() + cosym_instance.export() if __name__ == "__main__": diff --git a/src/dials/command_line/integrate.py b/src/dials/command_line/integrate.py index 781b8b315e..875a49a3e2 100644 --- a/src/dials/command_line/integrate.py +++ b/src/dials/command_line/integrate.py @@ -751,7 +751,8 @@ def run(args=None, phil=working_phil): expeditor = Expeditor(experiments, reference) experiments, reference = expeditor.filter_experiments_with_crystals() - reference = flex.reflection_table.concat(reference) + if reference: + reference = flex.reflection_table.concat(reference) try: experiments, reflections, report = run_integration( params, experiments, reference diff --git a/src/dials/command_line/refine_bravais_settings.py b/src/dials/command_line/refine_bravais_settings.py index 6b9db1b2be..83e8d0cd36 100644 --- a/src/dials/command_line/refine_bravais_settings.py +++ b/src/dials/command_line/refine_bravais_settings.py @@ -201,8 +201,9 @@ def run(args=None): if len(experiments) == 0: parser.print_help() return - expeditor = Expeditor(experiments, reflections) + expeditor = Expeditor(experiments, [reflections]) experiments, reflections = expeditor.filter_experiments_with_crystals() + reflections = flex.reflection_table.concat(reflections) if len(experiments.crystals()) > 1: if params.crystal_id is not None: @@ -241,10 +242,8 @@ def run(args=None): soln = int(subgroup.setting_number) bs_json = "%sbravais_setting_%i.expt" % (prefix, soln) logger.info("Saving solution %i as %s", soln, bs_json) - - expts = expeditor.generate_experiments_with_updated_crystal( - expts, params.crystal_id - ) + crystal_id = params.crystal_id if params.crystal_id else 0 + expts = expeditor.generate_experiments_with_updated_crystal(expts, crystal_id) expts.as_file(os.path.join(params.output.directory, bs_json)) diff --git a/src/dials/command_line/scale.py b/src/dials/command_line/scale.py index 61c8dfa2be..c50b162418 100644 --- a/src/dials/command_line/scale.py +++ b/src/dials/command_line/scale.py @@ -246,23 +246,20 @@ def run(args: List[str] = None, phil: phil.scope = phil_scope) -> None: else: # Note, cross validation mode does not produce scaled datafiles if scaled_experiments and joint_table: - experiments, reflections = expeditor.combine_experiments_for_output( - scaled_experiments, [joint_table] - ) logger.info( "Saving the scaled experiments to %s", params.output.experiments ) - experiments.as_file(params.output.experiments) + scaled_experiments.as_file(params.output.experiments) logger.info( "Saving the scaled reflections to %s", params.output.reflections ) - reflections.as_file(params.output.reflections) + joint_table.as_file(params.output.reflections) if params.output.unmerged_mtz: - _export_unmerged_mtz(params, experiments, reflections) + _export_unmerged_mtz(params, scaled_experiments, joint_table) if params.output.merged_mtz: - _export_merged_mtz(params, experiments, reflections) + _export_merged_mtz(params, scaled_experiments, joint_table) logger.info( "See dials.github.io/dials_scale_user_guide.html for more info on scaling options" diff --git a/src/dials/command_line/shadow_plot.py b/src/dials/command_line/shadow_plot.py index f83f84b92a..d4c0f62699 100644 --- a/src/dials/command_line/shadow_plot.py +++ b/src/dials/command_line/shadow_plot.py @@ -66,10 +66,8 @@ def run(args=None): parser.print_help() sys.exit(0) - assert len(experiments) == 1 - imagesets = experiments.imagesets() - - imageset = imagesets[0] + assert len(experiments.imagesets()) == 1 + imageset = experiments.imagesets()[0] goniometer = imageset.get_goniometer() detector = imageset.get_detector() scan = imageset.get_scan() diff --git a/src/dials/command_line/stereographic_projection.py b/src/dials/command_line/stereographic_projection.py index 8a3c72f208..a52c1ae423 100644 --- a/src/dials/command_line/stereographic_projection.py +++ b/src/dials/command_line/stereographic_projection.py @@ -177,7 +177,10 @@ def run(args=None): params, options = parser.parse_args(args=args, show_diff_phil=True) experiments = flatten_experiments(params.input.experiments) + from dials.util.multi_dataset_handling import Expeditor + expeditor = Expeditor(experiments) + experiments, _ = expeditor.filter_experiments_with_crystals() if not experiments: parser.print_help() return diff --git a/src/dials/command_line/symmetry.py b/src/dials/command_line/symmetry.py index 85b736fda9..334c25fd0c 100644 --- a/src/dials/command_line/symmetry.py +++ b/src/dials/command_line/symmetry.py @@ -34,7 +34,6 @@ get_selection_for_valid_image_ranges, ) from dials.util.filter_reflections import filtered_arrays_from_experiments_reflections -from dials.util.multi_dataset_handling import assign_unique_identifiers from dials.util.options import ArgumentParser, reflections_and_experiments_from_files from dials.util.version import dials_version @@ -510,7 +509,7 @@ def symmetry(experiments, reflection_tables, params=None): float(params.systematic_absences.significance_level), method=params.systematic_absences.method, ) - return experiments, reflection_tables + logger.info( "Saving reindexed experiments to %s in space group %s", params.output.experiments, @@ -618,32 +617,10 @@ def run(args=None): % (len(experiments), len(reflections)) ) try: - experiments, reflections = assign_unique_identifiers(experiments, reflections) - experiments, reflections = symmetry(experiments, reflections, params=params) + symmetry(experiments, reflections, params=params) except ValueError as e: sys.exit(e) - logger.info( - "Saving reindexed experiments to %s in space group %s", - params.output.experiments, - str(experiments[0].crystal.get_space_group().info()), - ) - experiments, reflections = expeditor.combine_experiments_for_output( - experiments, - reflections, - ) - experiments.as_file(params.output.experiments) - if params.output.reflections is not None: - logger.info( - "Saving %s reindexed reflections to %s", - len(reflections), - params.output.reflections, - ) - reflections.as_file(params.output.reflections) - - if params.output.html and params.systematic_absences.check: - ScrewAxisObserver().generate_html_report(params.output.html) - if __name__ == "__main__": run() diff --git a/src/dials/command_line/two_theta_offset.py b/src/dials/command_line/two_theta_offset.py index c19dcf2fc9..16bbdeea90 100644 --- a/src/dials/command_line/two_theta_offset.py +++ b/src/dials/command_line/two_theta_offset.py @@ -54,6 +54,11 @@ def run(self, args=None): # Check the number of experiments is at least 2 experiments = flatten_experiments(params.input.experiments) + from dials.util.multi_dataset_handling import Expeditor + + expeditor = Expeditor(experiments) + experiments, _ = expeditor.filter_experiments_with_crystals() + if len(experiments) < 2: self.parser.print_help() return diff --git a/src/dials/command_line/two_theta_refine.py b/src/dials/command_line/two_theta_refine.py index cac02391f7..710c2e51fc 100644 --- a/src/dials/command_line/two_theta_refine.py +++ b/src/dials/command_line/two_theta_refine.py @@ -32,7 +32,6 @@ from dials.array_family import flex from dials.util import log, tabulate from dials.util.filter_reflections import filter_reflection_table -from dials.util.multi_dataset_handling import parse_multiple_datasets from dials.util.options import ArgumentParser, reflections_and_experiments_from_files from dials.util.version import dials_version @@ -436,18 +435,17 @@ def run(self, args=None): # Parse the command line params, _ = self.parser.parse_args(args, show_diff_phil=False) - # set up global reflections list - reflections = flex.reflection_table() - - # loop through the input, building up the global lists reflections_list, input_experiments = reflections_and_experiments_from_files( params.input.reflections, params.input.experiments ) + # set up global reflections list + from dials.util.multi_dataset_handling import Expeditor + + expeditor = Expeditor(input_experiments, reflections_list) + input_experiments, reflections = expeditor.filter_experiments_with_crystals() + reflections = flex.reflection_table.concat(reflections) experiments = copy.deepcopy(input_experiments) - reflections_list = parse_multiple_datasets(reflections_list) - for refs in reflections_list: - reflections.extend(refs) # Try to load the models and data nexp = len(experiments) @@ -531,6 +529,8 @@ def run(self, args=None): logger.info(crystals[0]) logger.info(self.cell_param_table(crystals[0])) + experiments, _ = expeditor.combine_experiments_for_output(experiments) + # Save the refined experiments to file output_experiments_filename = params.output.experiments logger.info(f"Saving refined experiments to {output_experiments_filename}") diff --git a/src/dials/command_line/unit_cell_histogram.py b/src/dials/command_line/unit_cell_histogram.py index 70feb1b1b1..be75ce4261 100644 --- a/src/dials/command_line/unit_cell_histogram.py +++ b/src/dials/command_line/unit_cell_histogram.py @@ -6,6 +6,7 @@ import numpy as np import libtbx.phil +from dxtbx.model import ExperimentList from scitbx.array_family import flex from scitbx.math import five_number_summary @@ -77,7 +78,7 @@ def run(args=None): params, options = parser.parse_args(args, show_diff_phil=False) experiments = flatten_experiments(params.input.experiments) - + experiments = ExperimentList([e for e in experiments if e.crystal]) if len(experiments) == 0: parser.print_help() exit(0) diff --git a/src/dials/util/multi_dataset_handling.py b/src/dials/util/multi_dataset_handling.py index 1f65711363..0e908075cc 100644 --- a/src/dials/util/multi_dataset_handling.py +++ b/src/dials/util/multi_dataset_handling.py @@ -57,6 +57,11 @@ def __init__(self, experiments, reflection_tables=None): self.reflections_with_id_minus_1 = flex.reflection_table() if reflection_tables: if len(reflection_tables) > 1: + n_imagesets = 0 + for t in reflection_tables: + if "imageset_id" in t: + t["imageset_id"] += n_imagesets + n_imagesets += len(set(t["imageset_id"])) self.reflection_table = flex.reflection_table.concat(reflection_tables) else: self.reflection_table = reflection_tables[0] @@ -111,17 +116,26 @@ def filter_experiments_with_crystals(self): ] return expts_with_crystals, reflection_table.split_by_experiment_id() - def combine_experiments_for_output(self, experiments, reflection_tables=None): + def combine_experiments_for_output( + self, experiments, reflection_tables=None, include_unindexed=False + ): # When reflection tables exist, handle two types of input. # Either the program returns a list of reflections tables, one for each experiment, # or a combined table for all experiments. if reflection_tables: if len(experiments) != len(reflection_tables): if len(reflection_tables) == 1: - reflection_tables = reflection_tables[0].split_by_experiment_id() - assert len(reflection_tables) == len( + reflection_tables[0].assert_experiment_identifiers_are_consistent( experiments - ), f"{len(reflection_tables)} != {len(experiments)}" + ) + unassigned = None + new_unassigned_sel = reflection_tables[0]["id"] == -1 + if any(new_unassigned_sel): + logger.info(f"{new_unassigned_sel.count(True)} unassigned") + unassigned = reflection_tables[0].select(new_unassigned_sel) + reflection_tables = reflection_tables[0].split_by_experiment_id() + if unassigned and include_unindexed: + reflection_tables.append(unassigned) else: raise ValueError("Mismatch") imagesets = self.experiments.imagesets() @@ -132,6 +146,8 @@ def combine_experiments_for_output(self, experiments, reflection_tables=None): table["imageset_id"] = flex.int( table.size(), imagesets.index(expt.imageset) ) + if self.reflections_with_id_minus_1 and include_unindexed: + reflection_tables.append(self.reflections_with_id_minus_1) return experiments, flex.reflection_table.concat(reflection_tables) return experiments, None @@ -146,7 +162,6 @@ def combine_experiments_for_output(self, experiments, reflection_tables=None): # else: # reflection_table = reflection_tables[0] # tables = reflection_table.split_by_experiment_id() - for i, expt, table in zip(self.crystal_locs, experiments, reflection_tables): other_expts_sharing_scan = self.experiments.where(scan=expt.scan) for j in other_expts_sharing_scan: