Update more programs and revert data reduction changes

src/dials/command_line/anvil_correction.py

@@ -36,7 +36,6 @@
 
 import dials.util.log
 from dials.array_family import flex
-from dials.util.multi_dataset_handling import parse_multiple_datasets
 from dials.util.options import ArgumentParser, reflections_and_experiments_from_files
 
 Vector = Sequence[SupportsFloat]
@@ -321,15 +320,8 @@ def run(args: List[str] = None, phil: libtbx.phil.scope = phil_scope) -> None:
     )
     from dials.util.multi_dataset_handling import Expeditor
 
-    # Work around parse_multiple_datasets dropping unindexed reflections.
-    # unindexed = flex.reflection_table()
-    # for r_table in reflections_list:
-    #    unindexed.extend(r_table.select(r_table["id"] == -1))
-    # Get a single reflection table per experiment object.
-    reflections_list = parse_multiple_datasets(reflections_list)
     expeditor = Expeditor(experiments, reflections_list)
     experiments, reflections_list = expeditor.filter_experiments_with_crystals()
-    unindexed = expeditor.get_unindexed()
 
     # Record the density of diamond in g·cm⁻³ (for consistency with NIST tables,
     # https://doi.org/10.18434/T4D01F).
@@ -356,8 +348,7 @@ def run(args: List[str] = None, phil: libtbx.phil.scope = phil_scope) -> None:
         experiments.as_file(params.output.experiments)
     logger.info("Writing the reflection table to %s", params.output.reflections)
     # Collate reflections into a single reflection table and save it to file.
-    unindexed.extend(reflections)
-    unindexed.as_file(params.output.reflections)
+    reflections.as_file(params.output.reflections)
 
 
 # Keep this minimal.  Try to keep the command-line behaviour neatly encapsulated in run.

src/dials/command_line/cosym.py

@@ -415,14 +415,7 @@ def run(args=None):
     if params.output.html or params.output.json:
         register_default_cosym_observers(cosym_instance)
     cosym_instance.run()
-    experiments, reflections = expeditor.combine_experiments_for_output(
-        cosym_instance._experiments, cosym_instance._reflections
-    )
-    logger.info("Saving reindexed experiments to %s", params.output.experiments)
-    experiments.as_file(params.output.experiments)
-    logger.info("Saving reindexed reflections to %s", params.output.reflections)
-    reflections.as_file(params.output.reflections)
-    # cosym_instance.export()
+    cosym_instance.export()
 
 
 if __name__ == "__main__":

src/dials/command_line/integrate.py

@@ -751,7 +751,8 @@ def run(args=None, phil=working_phil):
 
     expeditor = Expeditor(experiments, reference)
     experiments, reference = expeditor.filter_experiments_with_crystals()
-    reference = flex.reflection_table.concat(reference)
+    if reference:
+        reference = flex.reflection_table.concat(reference)
     try:
         experiments, reflections, report = run_integration(
             params, experiments, reference

src/dials/command_line/refine_bravais_settings.py

@@ -201,8 +201,9 @@ def run(args=None):
     if len(experiments) == 0:
         parser.print_help()
         return
-    expeditor = Expeditor(experiments, reflections)
+    expeditor = Expeditor(experiments, [reflections])
     experiments, reflections = expeditor.filter_experiments_with_crystals()
+    reflections = flex.reflection_table.concat(reflections)
 
     if len(experiments.crystals()) > 1:
         if params.crystal_id is not None:
@@ -241,10 +242,8 @@ def run(args=None):
         soln = int(subgroup.setting_number)
         bs_json = "%sbravais_setting_%i.expt" % (prefix, soln)
         logger.info("Saving solution %i as %s", soln, bs_json)
-
-        expts = expeditor.generate_experiments_with_updated_crystal(
-            expts, params.crystal_id
-        )
+        crystal_id = params.crystal_id if params.crystal_id else 0
+        expts = expeditor.generate_experiments_with_updated_crystal(expts, crystal_id)
 
         expts.as_file(os.path.join(params.output.directory, bs_json))
 

src/dials/command_line/scale.py

@@ -246,23 +246,20 @@ def run(args: List[str] = None, phil: phil.scope = phil_scope) -> None:
     else:
         # Note, cross validation mode does not produce scaled datafiles
         if scaled_experiments and joint_table:
-            experiments, reflections = expeditor.combine_experiments_for_output(
-                scaled_experiments, [joint_table]
-            )
             logger.info(
                 "Saving the scaled experiments to %s", params.output.experiments
             )
-            experiments.as_file(params.output.experiments)
+            scaled_experiments.as_file(params.output.experiments)
             logger.info(
                 "Saving the scaled reflections to %s", params.output.reflections
             )
-            reflections.as_file(params.output.reflections)
+            joint_table.as_file(params.output.reflections)
 
             if params.output.unmerged_mtz:
-                _export_unmerged_mtz(params, experiments, reflections)
+                _export_unmerged_mtz(params, scaled_experiments, joint_table)
 
             if params.output.merged_mtz:
-                _export_merged_mtz(params, experiments, reflections)
+                _export_merged_mtz(params, scaled_experiments, joint_table)
 
     logger.info(
         "See dials.github.io/dials_scale_user_guide.html for more info on scaling options"

src/dials/command_line/shadow_plot.py

@@ -66,10 +66,8 @@ def run(args=None):
         parser.print_help()
         sys.exit(0)
 
-    assert len(experiments) == 1
-    imagesets = experiments.imagesets()
-
-    imageset = imagesets[0]
+    assert len(experiments.imagesets()) == 1
+    imageset = experiments.imagesets()[0]
     goniometer = imageset.get_goniometer()
     detector = imageset.get_detector()
     scan = imageset.get_scan()

src/dials/command_line/stereographic_projection.py

@@ -177,7 +177,10 @@ def run(args=None):
 
     params, options = parser.parse_args(args=args, show_diff_phil=True)
     experiments = flatten_experiments(params.input.experiments)
+    from dials.util.multi_dataset_handling import Expeditor
 
+    expeditor = Expeditor(experiments)
+    experiments, _ = expeditor.filter_experiments_with_crystals()
     if not experiments:
         parser.print_help()
         return

src/dials/command_line/symmetry.py

@@ -34,7 +34,6 @@
     get_selection_for_valid_image_ranges,
 )
 from dials.util.filter_reflections import filtered_arrays_from_experiments_reflections
-from dials.util.multi_dataset_handling import assign_unique_identifiers
 from dials.util.options import ArgumentParser, reflections_and_experiments_from_files
 from dials.util.version import dials_version
 
@@ -510,7 +509,7 @@ def symmetry(experiments, reflection_tables, params=None):
                 float(params.systematic_absences.significance_level),
                 method=params.systematic_absences.method,
             )
-    return experiments, reflection_tables
+
     logger.info(
         "Saving reindexed experiments to %s in space group %s",
         params.output.experiments,
@@ -618,32 +617,10 @@ def run(args=None):
             % (len(experiments), len(reflections))
         )
     try:
-        experiments, reflections = assign_unique_identifiers(experiments, reflections)
-        experiments, reflections = symmetry(experiments, reflections, params=params)
+        symmetry(experiments, reflections, params=params)
     except ValueError as e:
         sys.exit(e)
 
-    logger.info(
-        "Saving reindexed experiments to %s in space group %s",
-        params.output.experiments,
-        str(experiments[0].crystal.get_space_group().info()),
-    )
-    experiments, reflections = expeditor.combine_experiments_for_output(
-        experiments,
-        reflections,
-    )
-    experiments.as_file(params.output.experiments)
-    if params.output.reflections is not None:
-        logger.info(
-            "Saving %s reindexed reflections to %s",
-            len(reflections),
-            params.output.reflections,
-        )
-        reflections.as_file(params.output.reflections)
-
-    if params.output.html and params.systematic_absences.check:
-        ScrewAxisObserver().generate_html_report(params.output.html)
-
 
 if __name__ == "__main__":
     run()

src/dials/command_line/two_theta_offset.py

@@ -54,6 +54,11 @@ def run(self, args=None):
 
         # Check the number of experiments is at least 2
         experiments = flatten_experiments(params.input.experiments)
+        from dials.util.multi_dataset_handling import Expeditor
+
+        expeditor = Expeditor(experiments)
+        experiments, _ = expeditor.filter_experiments_with_crystals()
+
         if len(experiments) < 2:
             self.parser.print_help()
             return

src/dials/command_line/two_theta_refine.py

@@ -32,7 +32,6 @@
 from dials.array_family import flex
 from dials.util import log, tabulate
 from dials.util.filter_reflections import filter_reflection_table
-from dials.util.multi_dataset_handling import parse_multiple_datasets
 from dials.util.options import ArgumentParser, reflections_and_experiments_from_files
 from dials.util.version import dials_version
 
@@ -436,18 +435,17 @@ def run(self, args=None):
         # Parse the command line
         params, _ = self.parser.parse_args(args, show_diff_phil=False)
 
-        # set up global reflections list
-        reflections = flex.reflection_table()
-
-        # loop through the input, building up the global lists
         reflections_list, input_experiments = reflections_and_experiments_from_files(
             params.input.reflections, params.input.experiments
         )
+        # set up global reflections list
+        from dials.util.multi_dataset_handling import Expeditor
+
+        expeditor = Expeditor(input_experiments, reflections_list)
+        input_experiments, reflections = expeditor.filter_experiments_with_crystals()
+        reflections = flex.reflection_table.concat(reflections)
 
         experiments = copy.deepcopy(input_experiments)
-        reflections_list = parse_multiple_datasets(reflections_list)
-        for refs in reflections_list:
-            reflections.extend(refs)
 
         # Try to load the models and data
         nexp = len(experiments)
@@ -531,6 +529,8 @@ def run(self, args=None):
             logger.info(crystals[0])
             logger.info(self.cell_param_table(crystals[0]))
 
+        experiments, _ = expeditor.combine_experiments_for_output(experiments)
+
         # Save the refined experiments to file
         output_experiments_filename = params.output.experiments
         logger.info(f"Saving refined experiments to {output_experiments_filename}")

src/dials/command_line/unit_cell_histogram.py

@@ -6,6 +6,7 @@
 import numpy as np
 
 import libtbx.phil
+from dxtbx.model import ExperimentList
 from scitbx.array_family import flex
 from scitbx.math import five_number_summary
 
@@ -77,7 +78,7 @@ def run(args=None):
 
     params, options = parser.parse_args(args, show_diff_phil=False)
     experiments = flatten_experiments(params.input.experiments)
-
+    experiments = ExperimentList([e for e in experiments if e.crystal])
     if len(experiments) == 0:
         parser.print_help()
         exit(0)

src/dials/util/multi_dataset_handling.py

@@ -57,6 +57,11 @@ def __init__(self, experiments, reflection_tables=None):
         self.reflections_with_id_minus_1 = flex.reflection_table()
         if reflection_tables:
             if len(reflection_tables) > 1:
+                n_imagesets = 0
+                for t in reflection_tables:
+                    if "imageset_id" in t:
+                        t["imageset_id"] += n_imagesets
+                        n_imagesets += len(set(t["imageset_id"]))
                 self.reflection_table = flex.reflection_table.concat(reflection_tables)
             else:
                 self.reflection_table = reflection_tables[0]
@@ -111,17 +116,26 @@ def filter_experiments_with_crystals(self):
             ]
         return expts_with_crystals, reflection_table.split_by_experiment_id()
 
-    def combine_experiments_for_output(self, experiments, reflection_tables=None):
+    def combine_experiments_for_output(
+        self, experiments, reflection_tables=None, include_unindexed=False
+    ):
         # When reflection tables exist, handle two types of input.
         # Either the program returns a list of reflections tables, one for each experiment,
         # or a combined table for all experiments.
         if reflection_tables:
             if len(experiments) != len(reflection_tables):
                 if len(reflection_tables) == 1:
-                    reflection_tables = reflection_tables[0].split_by_experiment_id()
-                    assert len(reflection_tables) == len(
+                    reflection_tables[0].assert_experiment_identifiers_are_consistent(
                         experiments
-                    ), f"{len(reflection_tables)} != {len(experiments)}"
+                    )
+                    unassigned = None
+                    new_unassigned_sel = reflection_tables[0]["id"] == -1
+                    if any(new_unassigned_sel):
+                        logger.info(f"{new_unassigned_sel.count(True)} unassigned")
+                        unassigned = reflection_tables[0].select(new_unassigned_sel)
+                    reflection_tables = reflection_tables[0].split_by_experiment_id()
+                    if unassigned and include_unindexed:
+                        reflection_tables.append(unassigned)
                 else:
                     raise ValueError("Mismatch")
         imagesets = self.experiments.imagesets()
@@ -132,6 +146,8 @@ def combine_experiments_for_output(self, experiments, reflection_tables=None):
                         table["imageset_id"] = flex.int(
                             table.size(), imagesets.index(expt.imageset)
                         )
+                if self.reflections_with_id_minus_1 and include_unindexed:
+                    reflection_tables.append(self.reflections_with_id_minus_1)
                 return experiments, flex.reflection_table.concat(reflection_tables)
             return experiments, None
 
@@ -146,7 +162,6 @@ def combine_experiments_for_output(self, experiments, reflection_tables=None):
         # else:
         #    reflection_table = reflection_tables[0]
         # tables = reflection_table.split_by_experiment_id()
-
         for i, expt, table in zip(self.crystal_locs, experiments, reflection_tables):
             other_expts_sharing_scan = self.experiments.where(scan=expt.scan)
             for j in other_expts_sharing_scan: