Skip to content

Commit

Permalink
fix(community): Escape libSQL vector store inserts by using object sy…
Browse files Browse the repository at this point in the history
…ntax with placeholders (#7041)

Co-authored-by: jacoblee93 <[email protected]>
  • Loading branch information
florivdg and jacoblee93 authored Oct 23, 2024
1 parent 4aee924 commit 7981aa9
Show file tree
Hide file tree
Showing 3 changed files with 61 additions and 35 deletions.
15 changes: 7 additions & 8 deletions docs/core_docs/docs/integrations/vectorstores/libsql.mdx
Original file line number Diff line number Diff line change
Expand Up @@ -10,9 +10,9 @@ This guide provides a quick overview for getting started with libSQL vector stor

## Integration details

| Class | Package | JS support | Package latest |
| Class | Package | PY support | Package latest |
| ------------------- | ---------------------- | ---------- | ----------------------------------------------------------------- |
| `LibSQLVectorStore` | `@langchain/community` | | ![npm version](https://img.shields.io/npm/v/@langchain/community) |
| `LibSQLVectorStore` | `@langchain/community` | | ![npm version](https://img.shields.io/npm/v/@langchain/community) |

## Setup

Expand Down Expand Up @@ -54,7 +54,7 @@ libsql://[database-name]-[your-username].turso.io

Execute the following SQL command to create a new table or add the embedding column to an existing table.

Make sure to mopdify the following parts of the SQL:
Make sure to modify the following parts of the SQL:

- `TABLE_NAME` is the name of the table you want to create.
- `content` is used to store the `Document.pageContent` values.
Expand All @@ -70,7 +70,7 @@ CREATE TABLE IF NOT EXISTS TABLE_NAME (
);
```

Now create an index on the `EMBEDDING_COLUMN` column:
Now create an index on the `EMBEDDING_COLUMN` column - the index name is important!:

```sql
CREATE INDEX IF NOT EXISTS idx_TABLE_NAME_EMBEDDING_COLUMN ON TABLE_NAME(libsql_vector_idx(EMBEDDING_COLUMN));
Expand Down Expand Up @@ -103,9 +103,8 @@ const libsqlClient = createClient({

const vectorStore = new LibSQLVectorStore(embeddings, {
db: libsqlClient,
tableName: "TABLE_NAME",
embeddingColumn: "EMBEDDING_COLUMN",
dimensions: 1536,
table: "TABLE_NAME",
column: "EMBEDDING_COLUMN",
});
```

Expand Down Expand Up @@ -154,7 +153,7 @@ const similaritySearchWithScoreResults =

for (const [doc, score] of similaritySearchWithScoreResults) {
console.log(
`${score.toFixed(3)} ${doc.pageContent} [${JSON.stringify(doc.metadata)}`
`${score.toFixed(3)} ${doc.pageContent} [${JSON.stringify(doc.metadata)}]`
);
}
```
Expand Down
36 changes: 9 additions & 27 deletions libs/langchain-community/src/vectorstores/libsql.ts
Original file line number Diff line number Diff line change
Expand Up @@ -82,10 +82,10 @@ export class LibSQLVectorStore extends VectorStore {

for (let i = 0; i < rows.length; i += batchSize) {
const chunk = rows.slice(i, i + batchSize);
const insertQueries = chunk.map(
(row) =>
`INSERT INTO ${this.table} (content, metadata, ${this.column}) VALUES (${row.content}, ${row.metadata}, vector(${row.embedding})) RETURNING id`
);
const insertQueries = chunk.map((row) => ({
sql: `INSERT INTO ${this.table} (content, metadata, ${this.column}) VALUES (?, ?, ?) RETURNING id`,
args: [row.content, row.metadata, row.embedding],
}));

const results = await this.db.batch(insertQueries);

Expand Down Expand Up @@ -124,17 +124,19 @@ export class LibSQLVectorStore extends VectorStore {
const queryVector = `[${query.join(",")}]`;

const sql = `
SELECT content, metadata, vector_distance_cos(${this.column}, vector(${queryVector})) AS distance
FROM vector_top_k('${this.table}_idx', vector(${queryVector}), ${k})
JOIN ${this.table} ON ${this.table}.rowid = id
SELECT ${this.table}.id, ${this.table}.content, ${this.table}.metadata, vector_distance_cos(${this.table}.${this.column}, vector('${queryVector}')) AS distance
FROM vector_top_k('idx_${this.table}_${this.column}', vector('${queryVector}'), ${k}) AS top_k
JOIN ${this.table} ON top_k.rowid = ${this.table}.id
`;

const results = await this.db.execute(sql);

// eslint-disable-next-line @typescript-eslint/no-explicit-any
return results.rows.map((row: any) => {
const metadata = JSON.parse(row.metadata);

const doc = new Document({
id: row.id,
metadata,
pageContent: row.content,
});
Expand All @@ -143,26 +145,6 @@ export class LibSQLVectorStore extends VectorStore {
});
}

/**
* Deletes vectors from the store.
* @param {Object} params - Delete parameters.
* @param {string[] | number[]} [params.ids] - The ids of the vectors to delete.
* @returns {Promise<void>}
*/
async delete(params: { ids?: string[] | number[] }): Promise<void> {
if (!params.ids) {
await this.db.execute(`DELETE FROM ${this.table}`);
return;
}

const idsToDelete = params.ids.join(", ");

await this.db.execute({
sql: `DELETE FROM ${this.table} WHERE id IN (?)`,
args: [idsToDelete],
});
}

/**
* Creates a new LibSQLVectorStore instance from texts.
* @param {string[]} texts - The texts to add to the store.
Expand Down
45 changes: 45 additions & 0 deletions libs/langchain-community/src/vectorstores/tests/libsql.int.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,45 @@
/* eslint-disable no-process-env */
/* eslint-disable @typescript-eslint/no-non-null-assertion */
import { expect, test } from "@jest/globals";
import { OpenAIEmbeddings } from "@langchain/openai";
import { Document } from "@langchain/core/documents";
import { createClient } from "@libsql/client";

import { LibSQLVectorStore } from "../libsql.js";

test("can create and query", async () => {
const client = createClient({
url: process.env.LIBSQL_URL!,
authToken: process.env.LIBSQL_AUTH_TOKEN,
});
const vectorStore = new LibSQLVectorStore(
new OpenAIEmbeddings({
model: "text-embedding-3-small",
dimensions: 1536,
}),
{
db: client,
table: "documents",
column: "embeddings",
}
);
const ids = await vectorStore.addDocuments([
new Document({
pageContent: "added first page",
}),
new Document({
pageContent: "added second page",
}),
new Document({
pageContent: "added third page",
}),
]);
const nextId = await vectorStore.addDocuments([
new Document({
pageContent: "added another first page",
}),
]);
ids.push(nextId[0]);
const results = await vectorStore.similaritySearchWithScore("added first", 4);
expect(results.length).toBe(4);
});

0 comments on commit 7981aa9

Please sign in to comment.