chunk_vector_store.py

from langchain.text_splitter import RecursiveCharacterTextSplitter
from langchain.vectorstores.utils import filter_complex_metadata
from langchain_community.document_loaders import PyPDFLoader
from langchain_community.vectorstores import pgvector
from langchain_community.embeddings import fastembed
from config import Config

class ChunkVectorStore:
    def split_into_chunks(self, file_path: str):
        doc = PyPDFLoader(file_path).load()
        text_splitter = RecursiveCharacterTextSplitter(chunk_size=1024, chunk_overlap=20)
        chunks = text_splitter.split_documents(doc)
        chunks = filter_complex_metadata(chunks)
        return chunks

    def store_to_vector_database(self, chunks):
        return pgvector.PGVector.from_documents(
            documents=chunks,
            embedding=fastembed.FastEmbedEmbeddings(),
            connection_string=Config.get_db_connection_string()
        )