Merge branch 'main' into update_docs2

src/python/tests/test_gather.py

@@ -669,8 +669,8 @@ def test_simple_with_manifest_loading(runtmp):
     sig63 = get_test_data('63.fa.sig.gz')
 
     make_file_list(against_list, [sig2, sig47, sig63])
-    query_manifest = get_test_data("query-manifest.csv")
-    against_manifest = get_test_data("against-manifest.csv")
+    query_manifest = runtmp.output("query-manifest.csv")
+    against_manifest = runtmp.output("against-manifest.csv")
 
     runtmp.sourmash("sig", "manifest", query, "-o", query_manifest)
     runtmp.sourmash("sig", "manifest", against_list, "-o", against_manifest)

src/utils.rs

@@ -364,25 +364,32 @@ fn collection_from_pathlist(
     })?;
     let reader = BufReader::new(file);
 
-    let mut n_failed = 0;
-    let records: Vec<Record> = reader
+    // load list of paths
+    let lines: Vec<_> = reader
         .lines()
-        .filter_map(|line| {
-            let path = line.ok()?;
-            match Signature::from_path(&path) {
-                Ok(signatures) => {
-                    let recs: Vec<Record> = signatures
-                        .into_iter()
-                        .flat_map(|v| Record::from_sig(&v, &path))
-                        .collect();
-                    Some(recs)
-                }
-                Err(err) => {
-                    eprintln!("Sketch loading error: {}", err);
-                    eprintln!("WARNING: could not load sketches from path '{}'", path);
-                    n_failed += 1;
-                    None
-                }
+        .filter_map(|line| match line {
+            Ok(path) => Some(path),
+            Err(_err) => None,
+        })
+        .collect();
+
+    // load sketches from paths in parallel.
+    let n_failed = AtomicUsize::new(0);
+    let records: Vec<Record> = lines
+        .par_iter()
+        .filter_map(|path| match Signature::from_path(&path) {
+            Ok(signatures) => {
+                let recs: Vec<Record> = signatures
+                    .into_iter()
+                    .flat_map(|v| Record::from_sig(&v, &path))
+                    .collect();
+                Some(recs)
+            }
+            Err(err) => {
+                eprintln!("Sketch loading error: {}", err);
+                eprintln!("WARNING: could not load sketches from path '{}'", path);
+                let _ = n_failed.fetch_add(1, atomic::Ordering::SeqCst);
+                None
             }
         })
         .flatten()
@@ -405,6 +412,8 @@ fn collection_from_pathlist(
                 .build(),
         ),
     );
+    let n_failed = n_failed.load(atomic::Ordering::SeqCst);
+
     Ok((collection, n_failed))
 }