MRG: provide --internal-storage and --no-internal-storage for `in…

…dex` (#390)

* add some tests for subdir location

* more changes...

* fix warning

* reminder to clean up

* make an actually breaking test

* upd

* yay a more complete set of breakages

* comment

* force error exit in mastiff multigather for failed gather

* upd sourmash core

* finally, sensical error messages :laugh:

* Use the internalize_storage method from revindex to make a self-contained rocksdb index

* make use of internal storage an option, default=True

* fix cargo fmt

* remove verbose error

---------

Co-authored-by: Luiz Irber <[email protected]>

src/index.rs

@@ -1,4 +1,5 @@
 use sourmash::index::revindex::RevIndex;
+use sourmash::index::revindex::RevIndexOps;
 use sourmash::prelude::*;
 use std::path::Path;
 
@@ -10,6 +11,7 @@ pub fn index<P: AsRef<Path>>(
     output: P,
     colors: bool,
     allow_failed_sigpaths: bool,
+    use_internal_storage: bool,
 ) -> Result<(), Box<dyn std::error::Error>> {
     println!("Loading siglist");
 
@@ -20,11 +22,15 @@ pub fn index<P: AsRef<Path>>(
         allow_failed_sigpaths,
     )?;
 
-    RevIndex::create(
+    let mut index = RevIndex::create(
         output.as_ref(),
         collection.select(selection)?.try_into()?,
         colors,
     )?;
 
+    if use_internal_storage {
+        index.internalize_storage()?;
+    }
+
     Ok(())
 }

src/lib.rs

@@ -179,10 +179,18 @@ fn do_index(
     moltype: String,
     output: String,
     colors: bool,
+    use_internal_storage: bool,
 ) -> anyhow::Result<u8> {
     let selection = build_selection(ksize, scaled, &moltype);
     let allow_failed_sigpaths = false;
-    match index::index(siglist, &selection, output, colors, allow_failed_sigpaths) {
+    match index::index(
+        siglist,
+        &selection,
+        output,
+        colors,
+        allow_failed_sigpaths,
+        use_internal_storage,
+    ) {
         Ok(_) => Ok(0),
         Err(e) => {
             eprintln!("Error: {e}");

src/python/sourmash_plugin_branchwater/__init__.py

@@ -190,6 +190,12 @@ def __init__(self, p):
                        help = 'molecule type (DNA, protein, dayhoff, or hp; default DNA)')
         p.add_argument('-c', '--cores', default=0, type=int,
                        help='number of cores to use (default is all available)')
+        p.add_argument('--internal-storage', default=True, action='store_true',
+                       help="build indexes that contain sketches and are relocatable (default: True)")
+        p.add_argument('--no-internal-storage', '--no-store-sketches',
+                       action='store_false',
+                       help="do not store sketches in the index; index may not be relocatable (default: False)",
+                       dest='internal_storage')
 
     def main(self, args):
         notify(f"ksize: {args.ksize} / scaled: {args.scaled} / moltype: {args.moltype} ")
@@ -205,7 +211,8 @@ def main(self, args):
                                                       args.scaled,
                                                       args.moltype,
                                                       args.output,
-                                                      False) # colors - currently must be false?
+                                                      False, # colors - currently must be false?
+                                                      args.internal_storage)
         if status == 0:
             notify(f"...index is done! results in '{args.output}'")
         return status
@@ -217,7 +224,7 @@ class Branchwater_Check(CommandLinePlugin):
     def __init__(self, p):
         super().__init__(p)
         p.add_argument('index',
-                       help='index file')
+                       help="RocksDB index file created with 'index'")
         p.add_argument('--quick', action='store_true')
 
     def main(self, args):

src/python/tests/conftest.py

@@ -6,3 +6,20 @@
 def runtmp():
     with TempDirectory() as location:
         yield RunnerContext(location)
+
+
+@pytest.fixture(params=["--internal-storage", "--no-internal-storage"])
+def toggle_internal_storage(request):
+    return request.param
+
+@pytest.fixture(params=[True, False])
+def zip_query(request):
+    return request.param
+
+@pytest.fixture(params=[True, False])
+def zip_against(request):
+    return request.param
+
+@pytest.fixture(params=[True, False])
+def indexed(request):
+    return request.param