From dcdfcd9c45f050e31f020b77aa54fd832d16cde8 Mon Sep 17 00:00:00 2001 From: Varik Matevosyan Date: Wed, 21 Feb 2024 13:08:31 +0400 Subject: [PATCH] Fix naming for uppercase table names, check if codebook table exists before processing --- ci/scripts/build.sh | 2 +- lantern_external_index/Cargo.toml | 2 +- lantern_external_index/src/lib.rs | 29 ++++++++++++++++++----------- lantern_extras/Cargo.toml | 2 +- 4 files changed, 21 insertions(+), 14 deletions(-) diff --git a/ci/scripts/build.sh b/ci/scripts/build.sh index 246c9d1..9283879 100755 --- a/ci/scripts/build.sh +++ b/ci/scripts/build.sh @@ -56,7 +56,7 @@ function setup_lantern() { git submodule update --recursive && \ mkdir build pushd build - cmake -DUSEARCH_NO_MARCH_NATIVE=ON .. && \ + cmake -DMARCH_NATIVE=OFF -DBUILD_FOR_DSTRIBUTING=1 .. && \ make install popd popd diff --git a/lantern_external_index/Cargo.toml b/lantern_external_index/Cargo.toml index 2908ac3..5a65aed 100644 --- a/lantern_external_index/Cargo.toml +++ b/lantern_external_index/Cargo.toml @@ -13,7 +13,7 @@ clap = { version = "4.4.0", features = ["derive"] } cxx = "1.0.106" postgres = "0.19.7" postgres-types = { version = "0.2.6", features = ["derive"] } -usearch = { git = "https://github.com/Ngalstyan4/usearch.git", branch = "main-lantern" } +usearch = { git = "https://github.com/Ngalstyan4/usearch.git", branch="main-lantern" } lantern_logger = { path = "../lantern_logger" } lantern_utils = { path = "../lantern_utils" } rand = "0.8.5" diff --git a/lantern_external_index/src/lib.rs b/lantern_external_index/src/lib.rs index 926fad0..e18c901 100644 --- a/lantern_external_index/src/lib.rs +++ b/lantern_external_index/src/lib.rs @@ -148,20 +148,26 @@ pub fn create_usearch_index( let mut num_subvectors: usize = 0; if args.pq { + let codebook_table_name = format!( + "_codebook_{table_name}_{column_name}", + table_name = &args.table, + column_name = &args.column + ); + let full_codebook_table_name = + get_full_table_name("_lantern_internal", &codebook_table_name); + + let rows_codebook_exists = transaction.query("SELECT true FROM information_schema.tables WHERE table_schema='_lantern_internal' AND table_name=$1;", &[&codebook_table_name])?; + + if rows_codebook_exists.len() == 0 { + anyhow::bail!("Codebook table {full_codebook_table_name} does not exist"); + } + let rows_c = transaction.query( - &format!( - "SELECT count(*) FROM _lantern_internal._codebook_{table_name}_{column_name} WHERE subvector_id = 0;", - table_name = args.table, - column_name = args.column, - ), + &format!("SELECT COUNT(*) FROM {full_codebook_table_name} WHERE subvector_id = 0;"), &[], )?; let rows_sv = transaction.query( - &format!( - "SELECT count(*) FROM _lantern_internal._codebook_{table_name}_{column_name} WHERE centroid_id = 0;", - table_name = args.table, - column_name = args.column, - ), + &format!("SELECT COUNT(*) FROM {full_codebook_table_name} WHERE centroid_id = 0;"), &[], )?; @@ -182,8 +188,9 @@ pub fn create_usearch_index( ), &[], )?; + logger.info(&format!( - "codebook has {} rows - {num_centroids} centroids and {num_subvectors} subvectors", + "Codebook has {} rows - {num_centroids} centroids and {num_subvectors} subvectors", rows.len() )); diff --git a/lantern_extras/Cargo.toml b/lantern_extras/Cargo.toml index f70ea44..e59e5d6 100644 --- a/lantern_extras/Cargo.toml +++ b/lantern_extras/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "lantern_extras" -version = "0.1.3" +version = "0.1.2" edition = "2021" [lib]