diff --git a/conftest.py b/conftest.py index 56d4feb28..099bc3f8e 100644 --- a/conftest.py +++ b/conftest.py @@ -1,6 +1,7 @@ -import pytest from pathlib import Path +import pytest + pytest_plugins = ("jupyter_server.pytest_plugin",) @@ -11,7 +12,14 @@ def jp_server_config(jp_server_config): @pytest.fixture(scope="session") def static_test_files_dir() -> Path: - return Path(__file__).parent.resolve() / "packages" / "jupyter-ai" / "jupyter_ai" / "tests" / "static" + return ( + Path(__file__).parent.resolve() + / "packages" + / "jupyter-ai" + / "jupyter_ai" + / "tests" + / "static" + ) @pytest.fixture diff --git a/packages/jupyter-ai/jupyter_ai/document_loaders/directory.py b/packages/jupyter-ai/jupyter_ai/document_loaders/directory.py index cbb4197d5..2d50552a0 100644 --- a/packages/jupyter-ai/jupyter_ai/document_loaders/directory.py +++ b/packages/jupyter-ai/jupyter_ai/document_loaders/directory.py @@ -112,9 +112,11 @@ def flatten(*chunk_lists): """Selects eligible files, i.e., 1. Files not in excluded directories, and 2. Files that are in the valid file extensions list -Called from the `split` function. -Returns all the filepaths to eligible files. +Called from the `split` function. +Returns all the filepaths to eligible files. """ + + def collect_filepaths(path, all_files: bool): # Check if the path points to a single file if os.path.isfile(path): @@ -139,6 +141,8 @@ def collect_filepaths(path, all_files: bool): """Splits files into chunks for vector db in RAG""" + + def split(path, all_files: bool, splitter): chunks = [] filepaths = collect_filepaths(path, all_files) diff --git a/packages/jupyter-ai/jupyter_ai/tests/static/.hidden_file.txt b/packages/jupyter-ai/jupyter_ai/tests/static/.hidden_file.txt index 3a020444d..760cc808c 100644 --- a/packages/jupyter-ai/jupyter_ai/tests/static/.hidden_file.txt +++ b/packages/jupyter-ai/jupyter_ai/tests/static/.hidden_file.txt @@ -1,2 +1 @@ -Hidden temp text file. - +Hidden temp text file. diff --git a/packages/jupyter-ai/jupyter_ai/tests/static/file1.txt b/packages/jupyter-ai/jupyter_ai/tests/static/file1.txt index 2deb487fa..008cb927a 100644 --- a/packages/jupyter-ai/jupyter_ai/tests/static/file1.txt +++ b/packages/jupyter-ai/jupyter_ai/tests/static/file1.txt @@ -1,2 +1 @@ -This is a temp test text file. - +This is a temp test text file. diff --git a/packages/jupyter-ai/jupyter_ai/tests/static/file2.py b/packages/jupyter-ai/jupyter_ai/tests/static/file2.py index 6b4e9522e..746e17199 100644 --- a/packages/jupyter-ai/jupyter_ai/tests/static/file2.py +++ b/packages/jupyter-ai/jupyter_ai/tests/static/file2.py @@ -1,2 +1,3 @@ import os -print("Hello World") \ No newline at end of file + +print("Hello World") diff --git a/packages/jupyter-ai/jupyter_ai/tests/static/file3.csv b/packages/jupyter-ai/jupyter_ai/tests/static/file3.csv index 5bf7c55af..01fba4acc 100644 --- a/packages/jupyter-ai/jupyter_ai/tests/static/file3.csv +++ b/packages/jupyter-ai/jupyter_ai/tests/static/file3.csv @@ -1,3 +1,2 @@ Column1, Column2 Test1, test2 - diff --git a/packages/jupyter-ai/jupyter_ai/tests/test_directory.py b/packages/jupyter-ai/jupyter_ai/tests/test_directory.py index 8a7b33032..0695c2e10 100644 --- a/packages/jupyter-ai/jupyter_ai/tests/test_directory.py +++ b/packages/jupyter-ai/jupyter_ai/tests/test_directory.py @@ -6,11 +6,13 @@ import os import shutil -import pytest from pathlib import Path from typing import Tuple + +import pytest from jupyter_ai.document_loaders.directory import collect_filepaths + @pytest.fixture def staging_dir(static_test_files_dir, jp_ai_staging_dir) -> Path: file1_path = static_test_files_dir / ".hidden_file.pdf" @@ -42,6 +44,8 @@ def staging_dir(static_test_files_dir, jp_ai_staging_dir) -> Path: """Test that the number of valid files for `/learn` is correct""" + + def test_collect_filepaths(staging_dir): all_files = False staging_dir_filepath = staging_dir @@ -51,5 +55,5 @@ def test_collect_filepaths(staging_dir): assert len(result) == 3 # Test number of valid files filenames = [fp.name for fp in result] - assert "file0.html" in filenames # Check that valid file is included - assert "file3.xyz" not in filenames # Check that invalid file is excluded \ No newline at end of file + assert "file0.html" in filenames # Check that valid file is included + assert "file3.xyz" not in filenames # Check that invalid file is excluded