Merge branch 'latest' into try-skipmers

.pre-commit-config.yaml

@@ -10,7 +10,7 @@ repos:
       - id: check-toml
       - id: debug-statements
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.8.2
+    rev: v0.8.3
     hooks:
       - id: ruff-format
       - id: ruff

doc/databases.md

@@ -37,7 +37,7 @@ genomes. Among other uses, they can be used to detect host
 contamination in microbial metagenomes.
 
 Each file includes sketches at k=21, k=31, and k=51, at a scaled of
-1000, and is about 110 MB.
+1000, and is under 50 MB.
 
 * Human (hg38) - [hg38.sig.zip](https://farm.cse.ucdavis.edu/~ctbrown/sourmash-db/host/hg38.sig.zip)
 * Cow (bosTau9) - [bosTau9.sig.zip](https://farm.cse.ucdavis.edu/~ctbrown/sourmash-db/host/bosTau9.sig.zip)
@@ -49,6 +49,18 @@ Each file includes sketches at k=21, k=31, and k=51, at a scaled of
 * Goat (oviAri4) - [oviAri4.sig.zip](https://farm.cse.ucdavis.edu/~ctbrown/sourmash-db/host/oviAri4.sig.zip)
 * Pig (susCr11) - [susScr11.sig.zip](https://farm.cse.ucdavis.edu/~ctbrown/sourmash-db/host/susScr11.sig.zip)
 
+## Sketches for plant genomes
+
+These sketches are for the plant genomes available in GenBank as of 2024-07.
+
+| K-mer size | Zipfile collection |
+| -------- | -------- |
+| k21 | [download (7G)](https://farm.cse.ucdavis.edu/\~ctbrown/sourmash-db/genbank-plant-2024-07/genbank-plants-2024-07.k21.zip) |
+| k31 | [download (8.8G)](https://farm.cse.ucdavis.edu/\~ctbrown/sourmash-db/genbank-plant-2024-07/genbank-plants-2024-07.k31.zip) |
+| k51 | [download (11G)](https://farm.cse.ucdavis.edu/\~ctbrown/sourmash-db/genbank-plant-2024-07/genbank-plants-2024-07.k51.zip) |
+
+Lineage spreadsheet for sourmash `tax` commands: [download](https://farm.cse.ucdavis.edu/\~ctbrown/sourmash-db/genbank-plant-2024-07/genbank-plants-2024-07.lineages.csv.gz)
+
 ## GTDB R08-RS214 - DNA databases
 
 [GTDB R08-RS214](https://forum.gtdb.ecogenomic.org/t/announcing-gtdb-r08-rs214/456) consists of 402,709 genomes organized into 85,205 species clusters.

src/core/Cargo.toml

@@ -53,9 +53,9 @@ piz = "0.5.0"
 primal-check = "0.3.4"
 rayon = { version = "1.10.0", optional = true }
 rkyv = { version = "0.7.44", optional = true }
-roaring = "0.10.8"
+roaring = "0.10.9"
 roots = "0.0.8"
-serde = { version = "1.0.215", features = ["derive"] }
+serde = { version = "1.0.216", features = ["derive"] }
 serde_json = "1.0.133"
 statrs = "0.18.0"
 streaming-stats = "0.2.3"
@@ -66,7 +66,7 @@ vec-collections = "0.4.3"
 
 [dev-dependencies]
 codspeed-criterion-compat = "2.7.2"
-proptest = { version = "1.5.0", default-features = false, features = ["std"]}
+proptest = { version = "1.6.0", default-features = false, features = ["std"]}
 rand = "0.8.2"
 tempfile = "3.14.0"
 

src/core/benches/minhash.rs

@@ -13,7 +13,7 @@ fn intersection(c: &mut Criterion) {
     filename.push("../../tests/test-data/gather-abund/genome-s10.fa.gz.sig");
     let file = File::open(filename).unwrap();
     let reader = BufReader::new(file);
-    let mut sigs: Vec<Signature> = serde_json::from_reader(reader).expect("Loading error");
+    let mut sigs = Signature::from_reader(reader).expect("Loading error");
     let mh = if let Sketch::MinHash(mh) = &sigs.swap_remove(0).sketches()[0] {
         mh.clone()
     } else {
@@ -24,7 +24,7 @@ fn intersection(c: &mut Criterion) {
     filename.push("../../tests/test-data/gather-abund/genome-s11.fa.gz.sig");
     let file = File::open(filename).unwrap();
     let reader = BufReader::new(file);
-    let mut sigs: Vec<Signature> = serde_json::from_reader(reader).expect("Loading error");
+    let mut sigs = Signature::from_reader(reader).expect("Loading error");
     let mh2 = if let Sketch::MinHash(mh) = &sigs.swap_remove(0).sketches()[0] {
         mh.clone()
     } else {

src/core/src/collection.rs

@@ -267,7 +267,7 @@ mod test {
         filename.push("../../tests/test-data/47+63-multisig.sig");
         let file = File::open(filename).unwrap();
         let reader = BufReader::new(file);
-        let sigs: Vec<Signature> = serde_json::from_reader(reader).expect("Loading error");
+        let sigs = Signature::from_reader(reader).expect("Loading error");
         // create Selection object
         let mut selection = Selection::default();
         selection.set_scaled(2000);
@@ -293,7 +293,7 @@ mod test {
         filename.push("../../tests/test-data/47+63-multisig.sig");
         let file = File::open(filename).unwrap();
         let reader = BufReader::new(file);
-        let sigs: Vec<Signature> = serde_json::from_reader(reader).expect("Loading error");
+        let sigs = Signature::from_reader(reader).expect("Loading error");
         // create Selection object
         let mut selection = Selection::default();
         selection.set_scaled(500);
@@ -314,7 +314,7 @@ mod test {
         filename.push("../../tests/test-data/genome-s11.fa.gz.sig");
         let file = File::open(filename).unwrap();
         let reader = BufReader::new(file);
-        let sigs: Vec<Signature> = serde_json::from_reader(reader).expect("Loading error");
+        let sigs = Signature::from_reader(reader).expect("Loading error");
         assert_eq!(sigs.len(), 4);
         // create Selection object
         let mut selection = Selection::default();
@@ -336,7 +336,7 @@ mod test {
         filename.push("../../tests/test-data/genome-s11.fa.gz.sig");
         let file = File::open(filename).unwrap();
         let reader = BufReader::new(file);
-        let sigs: Vec<Signature> = serde_json::from_reader(reader).expect("Loading error");
+        let sigs = Signature::from_reader(reader).expect("Loading error");
         let sigs_copy = sigs.clone();
         assert_eq!(sigs.len(), 4);
         // create Selection object
@@ -366,7 +366,7 @@ mod test {
         filename.push("../../tests/test-data/47+63-multisig.sig");
         let file = File::open(filename).unwrap();
         let reader = BufReader::new(file);
-        let sigs: Vec<Signature> = serde_json::from_reader(reader).expect("Loading error");
+        let sigs = Signature::from_reader(reader).expect("Loading error");
         assert_eq!(sigs.len(), 6);
         // create Selection object
         let mut selection = Selection::default();
@@ -388,7 +388,7 @@ mod test {
         filename.push("../../tests/test-data/genome-s11.fa.gz.sig");
         let file = File::open(filename).unwrap();
         let reader = BufReader::new(file);
-        let sigs: Vec<Signature> = serde_json::from_reader(reader).expect("Loading error");
+        let sigs = Signature::from_reader(reader).expect("Loading error");
         assert_eq!(sigs.len(), 4);
         // load sigs into collection + select compatible signatures
         let mut cl = Collection::from_sigs(sigs).unwrap();
@@ -413,7 +413,7 @@ mod test {
         filename.push("../../tests/test-data/47+63-multisig.sig");
         let file = File::open(filename).unwrap();
         let reader = BufReader::new(file);
-        let sigs: Vec<Signature> = serde_json::from_reader(reader).expect("Loading error");
+        let sigs = Signature::from_reader(reader).expect("Loading error");
         // create Selection object
         let mut selection = Selection::default();
         selection.set_scaled(2000);
@@ -480,7 +480,7 @@ mod test {
             .push("../../tests/test-data/prot/hp/GCA_001593925.1_ASM159392v1_protein.faa.gz.sig");
         let file = File::open(filename).unwrap();
         let reader = BufReader::new(file);
-        let sigs: Vec<Signature> = serde_json::from_reader(reader).expect("Loading error");
+        let sigs = Signature::from_reader(reader).expect("Loading error");
         // create Selection object
         let mut selection = Selection::default();
         selection.set_moltype(HashFunctions::Murmur64Hp);

src/core/src/ffi/signature.rs

@@ -13,6 +13,7 @@ use crate::sketch::Sketch;
 use crate::ffi::cmd::compute::SourmashComputeParameters;
 use crate::ffi::minhash::SourmashKmerMinHash;
 use crate::ffi::utils::{ForeignObject, SourmashStr};
+use crate::prelude::ToWriter;
 
 pub struct SourmashSignature;
 
@@ -193,8 +194,9 @@ unsafe fn signature_eq(ptr: *const SourmashSignature, other: *const SourmashSign
 ffi_fn! {
 unsafe fn signature_save_json(ptr: *const SourmashSignature) -> Result<SourmashStr> {
     let sig = SourmashSignature::as_rust(ptr);
-    let st = serde_json::to_string(sig)?;
-    Ok(SourmashStr::from_string(st))
+    let mut st: Vec<u8> = vec![];
+    sig.to_writer(&mut st)?;
+    Ok(SourmashStr::from_string(String::from_utf8_unchecked(st)))
 }
 }
 
@@ -248,7 +250,7 @@ unsafe fn signatures_save_buffer(ptr: *const *const SourmashSignature, size: usi
       } else {
           Box::new(&mut buffer)
       };
-      serde_json::to_writer(&mut writer, &rsigs)?;
+      rsigs.to_writer(&mut writer)?;
     }
 
     let b = buffer.into_boxed_slice();

src/core/src/index/revindex/disk_revindex.rs

@@ -351,7 +351,14 @@ impl RevIndexOps for RevIndex {
                         .collection
                         .record_for_dataset(dataset_id)
                         .expect("dataset not found");
-                    Some((row.name().into(), size))
+
+                    let name = [row.name(), row.filename(), row.md5()]
+                        .into_iter()
+                        .skip_while(|v| v.is_empty())
+                        .next()
+                        .unwrap(); // guaranteed to succeed because `md5` always exists
+
+                    Some((name.into(), size))
                 } else {
                     None
                 }

src/core/src/index/revindex/mod.rs

@@ -558,7 +558,7 @@ mod test {
         )?;
 
         assert_eq!(matches.len(), 1);
-        assert_eq!(matches[0].name(), "../genome-s10.fa.gz");
+        assert_eq!(matches[0].name(), ""); // signature name is empty
         assert_eq!(matches[0].f_match(), 1.0);
 
         Ok(())

src/core/src/manifest.rs

@@ -129,7 +129,7 @@ impl Record {
                 Self {
                     internal_location: path.into(),
                     moltype: moltype.to_string(),
-                    name: sig.name(),
+                    name: sig.name_str(),
                     ksize,
                     md5,
                     md5short,