From f5fd86cf699636320c0802e22c2533e6ade0ef8f Mon Sep 17 00:00:00 2001 From: Jeffrey Martin Date: Fri, 19 Apr 2024 08:03:35 -0500 Subject: [PATCH] add tests for `ggml` generator Signed-off-by: Jeffrey Martin --- garak/generators/ggml.py | 5 ++- pyproject.toml | 9 ++++- requirements.txt | 5 ++- tests/generators/test_ggml.py | 74 +++++++++++++++++++++++++++++++++++ tests/test_reqs.py | 10 ++++- 5 files changed, 97 insertions(+), 6 deletions(-) create mode 100644 tests/generators/test_ggml.py diff --git a/garak/generators/ggml.py b/garak/generators/ggml.py index fc040c8f7..a956264d4 100644 --- a/garak/generators/ggml.py +++ b/garak/generators/ggml.py @@ -22,6 +22,7 @@ from garak.generators.base import Generator GGUF_MAGIC = bytes([0x47, 0x47, 0x55, 0x46]) +ENV_VAR = "GGML_MAIN_PATH" class GgmlGenerator(Generator): @@ -55,9 +56,9 @@ def command_params(self): } def __init__(self, name, generations=10): - self.path_to_ggml_main = os.getenv("GGML_MAIN_PATH") + self.path_to_ggml_main = os.getenv(ENV_VAR) if self.path_to_ggml_main is None: - raise RuntimeError("Executable not provided by environment GGML_MAIN_PATH") + raise RuntimeError(f"Executable not provided by environment {ENV_VAR}") if not os.path.isfile(self.path_to_ggml_main): raise FileNotFoundError( f"Path provided is not a file: {self.path_to_ggml_main}" diff --git a/pyproject.toml b/pyproject.toml index 841ea0a72..565c2ba43 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -38,7 +38,6 @@ dependencies = [ "cohere>=4.5.1,<5", "openai==1.12.0", "replicate>=0.8.3", - "pytest>=8.0", "google-api-python-client>=2.0", "backoff>=2.1.1", "rapidfuzz>=3.0.0", @@ -63,6 +62,14 @@ dependencies = [ "typing>=3.7,<3.8; python_version<'3.5'" ] +[project.optional-dependencies] +tests = [ + "pytest>=8.0", +] +lint = [ + "black>=22.3", +] + [project.urls] "Homepage" = "https://github.com/leondz/garak" "Bug Tracker" = "https://github.com/leondz/garak/issues" diff --git a/requirements.txt b/requirements.txt index 7d3dcd325..a6afbe26b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,7 +6,6 @@ tqdm>=4.64.0 cohere>=4.5.1,<5 openai==1.12.0 replicate>=0.8.3 -pytest>=8.0 google-api-python-client>=2.0 backoff>=2.1.1 rapidfuzz>=3.0.0 @@ -29,3 +28,7 @@ deepl==1.17.0 fschat>=0.2.36 litellm>=1.33.8 typing>=3.7,<3.8; python_version<'3.5' +# tests +pytest>=8.0 +# lint +black>=22.3 \ No newline at end of file diff --git a/tests/generators/test_ggml.py b/tests/generators/test_ggml.py new file mode 100644 index 000000000..1b69d6aea --- /dev/null +++ b/tests/generators/test_ggml.py @@ -0,0 +1,74 @@ +#!/usr/bin/env python3 + +import os +import pytest +import tempfile +import garak.generators.ggml + +DEFAULT_GENERATIONS_QTY = 10 + +stored_env = os.getenv(garak.generators.ggml.ENV_VAR) +model_name = None + + +@pytest.fixture(autouse=True) +def reset_env(request) -> None: + if stored_env is not None: + os.environ[garak.generators.ggml.ENV_VAR] = stored_env + model_name = tempfile.NamedTemporaryFile(suffix="_test_model.gguf") + + +@pytest.fixture(autouse=True) +def set_fake_env() -> None: + os.environ[garak.generators.ggml.ENV_VAR] = os.path.abspath(__file__) + + +def test_init_bad_app(): + with pytest.raises(RuntimeError) as exc_info: + if os.getenv(garak.generators.ggml.ENV_VAR) is not None: + del os.environ[garak.generators.ggml.ENV_VAR] + garak.generators.ggml.GgmlGenerator(model_name) + assert "not provided by environment" in str(exc_info.value) + + +def test_init_missing_model(): + model_name = tempfile.TemporaryFile().name + with pytest.raises(FileNotFoundError) as exc_info: + garak.generators.ggml.GgmlGenerator(model_name) + assert "File not found" in str(exc_info.value) + + +def test_init_bad_model(): + with tempfile.NamedTemporaryFile(mode="w", suffix="_test_model.gguf") as file: + file.write(file.name) + file.seek(0) + with pytest.raises(RuntimeError) as exc_info: + garak.generators.ggml.GgmlGenerator(file.name) + assert "not in GGUF" in str(exc_info.value) + + +def test_init_good_model(): + with tempfile.NamedTemporaryFile(suffix="_test_model.gguf") as file: + file.write(garak.generators.ggml.GGUF_MAGIC) + file.seek(0) + g = garak.generators.ggml.GgmlGenerator(file.name) + assert g is not None + + +# def test_subprocess_first_err(): +# with open(model_name, "wb") as file: +# file.write(garak.generators.ggml.GGUF_MAGIC) +# g = garak.generators.ggml.GgmlGenerator(model_name) +# with pytest.raises(subprocess.CalledProcessError) as exc_info: +# result = g.generate("mock prompt") +# assert exc_info is not None + + +# def test_subprocess_continue_err(): +# with open(model_name, "wb") as file: +# file.write(garak.generators.ggml.GGUF_MAGIC) +# g = garak.generators.ggml.GgmlGenerator(model_name) +# g.generate("mock prompt") +# with pytest.raises(subprocess.CalledProcessError) as exc_info: +# result = g.generate("mock prompt") +# assert exc_info is not None diff --git a/tests/test_reqs.py b/tests/test_reqs.py index b2216bbd0..6dd4c0120 100644 --- a/tests/test_reqs.py +++ b/tests/test_reqs.py @@ -17,10 +17,16 @@ def test_requirements_txt_pyproject_toml(): with open("requirements.txt", "r", encoding="utf-8") as req_file: reqtxt_reqs = req_file.readlines() - reqtxt_reqs = list(map(str.strip, reqtxt_reqs)) + reqtxt_reqs = list( + filter(lambda x: not x.startswith("#"), map(str.strip, reqtxt_reqs)) + ) reqtxt_reqs.sort() with open("pyproject.toml", "rb") as pyproject_file: - pyproject_reqs = tomllib.load(pyproject_file)["project"]["dependencies"] + pyproject_toml = tomllib.load(pyproject_file) + pyproject_reqs = pyproject_toml["project"]["dependencies"] + for test_deps in pyproject_toml["project"]["optional-dependencies"].values(): + for dep in test_deps: + pyproject_reqs.append(dep) pyproject_reqs.sort() # assert len(reqtxt_reqs) == len(pyproject_reqs) # same number of requirements assert (