Use renamed lantern access method

ci/scripts/build.sh

@@ -49,6 +49,7 @@ function setup_postgres() {
 
 function setup_lantern() {
    LANTERN_VERSION=main
+   LANTERN_VERSION=narek/pgvector-compat
     git clone --recursive https://github.com/lanterndata/lantern.git /tmp/lantern 
     pushd /tmp/lantern
       git checkout ${LANTERN_VERSION} && \

lantern_daemon/src/external_index_jobs.rs

@@ -135,6 +135,7 @@ async fn external_index_worker(
                         dims: 0,
                         out: index_path,
                         remote_database: true,
+                        pq: false,
                     }, progress_callback, Some(is_canceled_clone), Some(task_logger));
                     futures::executor::block_on(cancel_tx_clone.send(false))?;
                     result

lantern_external_index/Cargo.toml

@@ -13,7 +13,8 @@ clap = { version = "4.4.0", features = ["derive"] }
 cxx = "1.0.106"
 postgres = "0.19.7"
 postgres-types = { version = "0.2.6", features = ["derive"] }
-usearch = { git = "https://github.com/Ngalstyan4/usearch.git", branch = "main-lantern" }
+# usearch = { git = "https://github.com/Ngalstyan4/usearch.git", branch = "main-lantern" }
+usearch = {path = "../../third_party/usearch/"}
 lantern_logger = { path = "../lantern_logger" }
 lantern_utils = { path = "../lantern_utils" }
 rand = "0.8.5"

lantern_external_index/src/cli.rs

@@ -113,6 +113,10 @@ pub struct CreateIndexArgs {
     #[arg(short, long)]
     pub column: String,
 
+    /// Use already created codebook to create product-quantized binary index
+    #[arg(short, long, default_value_t = false)]
+    pub pq: bool,
+
     /// Number of neighbours for each vector
     #[arg(short, default_value_t = 16)]
     pub m: usize,

lantern_external_index/src/lib.rs

@@ -114,6 +114,7 @@ pub fn create_usearch_index(
     let full_table_name = get_full_table_name(&args.schema, &args.table);
 
     transaction.execute("SET lock_timeout='5s'", &[])?;
+    //todo:: ask-Varik: why is this necessary?
     transaction.execute(
         &format!("LOCK TABLE ONLY {full_table_name} IN SHARE MODE"),
         &[],
@@ -142,6 +143,60 @@ pub fn create_usearch_index(
         dimensions, args.m, args.ef, args.efc
     ));
 
+    let mut pq_codebook: *const f32 = std::ptr::null();
+    let mut num_centroids: usize = 0;
+    let mut num_subvectors: usize = 0;
+
+    if args.pq {
+        let rows_c = transaction.query(
+            &format!(
+                "SELECT count(*) FROM _lantern_internal._codebook_{table_name}_{column_name} WHERE subvector_id = 0;",
+                table_name = args.table,
+                column_name = args.column,
+            ),
+            &[],
+        )?;
+        let rows_sv = transaction.query(
+            &format!(
+                "SELECT count(*) FROM _lantern_internal._codebook_{table_name}_{column_name} WHERE centroid_id = 0;",
+                table_name = args.table,
+                column_name = args.column,
+            ),
+            &[],
+        )?;
+
+        if rows_c.len() == 0 || rows_sv.len() == 0 {
+            anyhow::bail!("Invalid codebook table");
+        }
+
+        num_centroids = rows_c.first().unwrap().get::<usize, i64>(0) as usize;
+        num_subvectors = rows_sv.first().unwrap().get::<usize, i64>(0) as usize;
+
+        let rows = transaction.query(
+            &format!(
+                "SELECT subvector_id, centroid_id, c FROM _lantern_internal._codebook_{table_name}_{column_name};",
+                table_name = args.table,
+                column_name = args.column,
+            ),
+            &[],
+        )?;
+        let mut v = vec![0.; num_centroids * dimensions];
+        pq_codebook = v.as_ptr();
+        logger.info(&format!(
+            "codebook has {} rows - {num_centroids} centroids and {num_subvectors} subvectors",
+            rows.len()
+        ));
+
+        for r in rows {
+            let subvector_id: i32 = r.get(0);
+            let centroid_id: i32 = r.get(1);
+            let subvector: Vec<f32> = r.get(2);
+            for i in 0..subvector.len() {
+                v[centroid_id as usize * dimensions + subvector_id as usize + i] = subvector[i];
+            }
+        }
+    }
+
     let options = IndexOptions {
         dimensions,
         metric: args.metric_kind.value(),
@@ -150,6 +205,20 @@ pub fn create_usearch_index(
         connectivity: args.m,
         expansion_add: args.efc,
         expansion_search: args.ef,
+
+        num_threads: 0, // automatic
+
+        // note: pq_construction and pq_output distinction is not yet implemented in usearch
+        // in the future, if pq_construction is false, we will use full vectors in memory (and
+        // require large memory for construction) but will output pq-quantized graph
+        //
+        // currently, regardless of pq_construction value, as long as pq_output is true,
+        // we construct a pq_quantized index using quantized values during construction
+        pq_construction: args.pq,
+        pq_output: args.pq,
+        num_centroids,
+        num_subvectors,
+        codebook: pq_codebook,
     };
     let index = Index::new(&options)?;
 
@@ -331,7 +400,7 @@ pub fn create_usearch_index(
             }
 
             transaction.execute(
-            &format!("CREATE INDEX {idx_name} ON {table_name} USING hnsw({column_name} {op_class}) WITH (_experimental_index_path='{index_path}', ef={ef}, dim={dim}, m={m}, ef_construction={ef_construction});", index_path=args.out, table_name=&get_full_table_name(&args.schema, &args.table),column_name=&quote_ident(&args.column), m=args.m, ef=args.ef, ef_construction=args.efc, dim=dimensions),
+            &format!("CREATE INDEX {idx_name} ON {table_name} USING lantern_hnsw({column_name} {op_class}) WITH (_experimental_index_path='{index_path}', ef={ef}, dim={dim}, m={m}, ef_construction={ef_construction});", index_path=args.out, table_name=&get_full_table_name(&args.schema, &args.table),column_name=&quote_ident(&args.column), m=args.m, ef=args.ef, ef_construction=args.efc, dim=dimensions),
             &[],
             )?;
 

lantern_external_index/src/postgres_large_objects.rs

@@ -57,7 +57,7 @@ impl<'a> LargeObject<'a> {
         }
 
         transaction.execute(
-            &format!("CREATE INDEX {idx_name} ON {table_name} USING hnsw({column_name} {op_class}) WITH (_experimental_index_path='{index_path}', ef={ef}, dim={dim}, m={m}, ef_construction={ef_construction});", index_path=self.index_path),
+            &format!("CREATE INDEX {idx_name} ON {table_name} USING lantern_hnsw({column_name} {op_class}) WITH (_experimental_index_path='{index_path}', ef={ef}, dim={dim}, m={m}, ef_construction={ef_construction});", index_path=self.index_path),
             &[],
         )?;
 

lantern_extras/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "lantern_extras"
-version = "0.1.2"
+version = "0.2.0"
 edition = "2021"
 
 [lib]

lantern_index_autotune/src/lib.rs

@@ -446,6 +446,7 @@ pub fn autotune_index(
                     dims: column_dims as usize,
                     index_name: Some(index_name.clone()),
                     remote_database: true,
+                    pq: false,
                 },
                 None,
                 Some(is_canceled.clone()),
@@ -531,6 +532,7 @@ pub fn autotune_index(
                 dims: column_dims as usize,
                 index_name: None,
                 remote_database: true,
+                pq: false,
             },
             None,
             Some(is_canceled.clone()),