diff --git a/garak/_config.py b/garak/_config.py index 10aa270fe..5036a86fa 100644 --- a/garak/_config.py +++ b/garak/_config.py @@ -15,6 +15,11 @@ import pathlib from typing import List import yaml +from xdg_base_dirs import ( + xdg_cache_home, + xdg_config_home, + xdg_data_home, +) DICT_CONFIG_AFTER_LOAD = False @@ -26,6 +31,7 @@ run_params = "seed deprefix eval_threshold generations probe_tags interactive".split() plugins_params = "model_type model_name extended_detectors".split() reporting_params = "taxonomy report_prefix".split() +project_dir = "garak" loaded = False @@ -53,9 +59,17 @@ class TransientConfig(GarakSubConfig): args = None # only access this when determining what was passed on CLI run_id = None basedir = pathlib.Path(__file__).parents[0] + config_dir = xdg_config_home() / project_dir + user_data_dir = xdg_data_home() / project_dir + cache_dir = xdg_cache_home() / project_dir starttime = None starttime_iso = None + # initialize the user home and cache paths if they do not exist + config_dir.mkdir(mode=0o740, parents=True, exist_ok=True) + user_data_dir.mkdir(mode=0o740, parents=True, exist_ok=True) + cache_dir.mkdir(mode=0o740, parents=True, exist_ok=True) + transient = TransientConfig() @@ -151,7 +165,7 @@ def load_config( settings_files = [str(transient.basedir / "resources" / "garak.core.yaml")] - fq_site_config_filename = str(transient.basedir / site_config_filename) + fq_site_config_filename = str(transient.config_dir / site_config_filename) if os.path.isfile(fq_site_config_filename): settings_files.append(fq_site_config_filename) else: diff --git a/garak/analyze/report_digest.py b/garak/analyze/report_digest.py index 347e3fd83..677930876 100644 --- a/garak/analyze/report_digest.py +++ b/garak/analyze/report_digest.py @@ -73,6 +73,7 @@ def compile_digest(report_path, taxonomy=_config.reporting.taxonomy): digest_content = header_template.render( { + # consider expecting a `Path` here or even putting the value in a path and calling `parent` "reportfile": report_path.split(os.sep)[-1], "garak_version": garak_version, "start_time": start_time, diff --git a/garak/command.py b/garak/command.py index 4a7007786..1a5fb6ef3 100644 --- a/garak/command.py +++ b/garak/command.py @@ -8,8 +8,10 @@ def start_logging(): + from garak import _config + logging.basicConfig( - filename="garak.log", + filename=_config.transient.config_dir / "garak.log", level=logging.DEBUG, format="%(asctime)s %(levelname)s %(message)s", ) @@ -32,6 +34,7 @@ def start_run(): import os import uuid + from pathlib import Path from garak import _config logging.info("started at %s", _config.transient.starttime_iso) @@ -41,19 +44,25 @@ def start_run(): "⚠️ The current/default config is optimised for speed rather than thoroughness. Try e.g. --config full for a stronger test, or specify some probes." ) _config.transient.run_id = str(uuid.uuid4()) # uuid1 is safe but leaks host info + # why is report prefix a condition of placing file in the report_dir? + report_path = Path(_config.reporting.report_dir) + if not report_path.is_absolute(): + logging.debug("relative report dir provided") + report_path = _config.transient.user_data_dir / _config.reporting.report_dir + if not os.path.isdir(report_path): + try: + report_path.mkdir(mode=0o740, parents=True, exist_ok=True) + except PermissionError as e: + raise PermissionError( + f"Can't create logging directory {report_path}, quitting" + ) from e + + filename = f"garak.{_config.transient.run_id}.report.jsonl" if not _config.reporting.report_prefix: - if not os.path.isdir(_config.reporting.report_dir): - try: - os.mkdir(_config.reporting.report_dir) - except PermissionError as e: - raise PermissionError( - f"Can't create logging directory {_config.reporting.report_dir}, quitting" - ) from e - _config.transient.report_filename = f"{_config.reporting.report_dir}/garak.{_config.transient.run_id}.report.jsonl" + filename = f"garak.{_config.transient.run_id}.report.jsonl" else: - _config.transient.report_filename = ( - _config.reporting.report_prefix + ".report.jsonl" - ) + filename = _config.reporting.report_prefix + ".report.jsonl" + _config.transient.report_filename = str(report_path / filename) _config.transient.reportfile = open( _config.transient.report_filename, "w", buffering=1, encoding="utf-8" ) diff --git a/garak/evaluators/base.py b/garak/evaluators/base.py index 6152e6951..15fea84e5 100644 --- a/garak/evaluators/base.py +++ b/garak/evaluators/base.py @@ -5,6 +5,7 @@ import json import logging +from pathlib import Path from typing import Iterable from colorama import Fore, Style @@ -72,12 +73,19 @@ def evaluate(self, attempts: Iterable[garak.attempt.Attempt]) -> None: hitlog_mode = ( "w" if _config.transient.hitlogfile is None else "a" ) + # why is report prefix a condition of placing file in the report_dir? + hitlog_path = Path(_config.transient.report_filename).parent if not _config.reporting.report_prefix: - hitlog_filename = f"{_config.reporting.report_dir}/garak.{_config.transient.run_id}.hitlog.jsonl" + hitlog_filename = ( + hitlog_path + / f"garak.{_config.transient.run_id}.hitlog.jsonl" + ) # else: hitlog_filename = ( - _config.reporting.report_prefix + ".hitlog.jsonl" + hitlog_path + / f"{_config.reporting.report_prefix}.hitlog.jsonl" ) + logging.info("hit log in %s", hitlog_filename) _config.transient.hitlogfile = open( hitlog_filename, diff --git a/garak/interactive.py b/garak/interactive.py index 309c3d569..3049491ff 100644 --- a/garak/interactive.py +++ b/garak/interactive.py @@ -233,7 +233,7 @@ def settings_ns_provider(self) -> argparse.Namespace: def banner(): """Display a random banner""" with open( - _config.transient.basedir / "resources/banners", "r", encoding="utf-8" + _config.transient.basedir / "resources" / "banners", "r", encoding="utf-8" ) as bannerfile: banners = bannerfile.read().lstrip().split("\n\n") print(Fore.LIGHTRED_EX + f"\n{random.choice(banners)}\n" + Fore.RESET) diff --git a/garak/probes/tap.py b/garak/probes/tap.py index 4d22f0d07..e64c04e0e 100644 --- a/garak/probes/tap.py +++ b/garak/probes/tap.py @@ -37,6 +37,7 @@ """ import logging +from pathlib import Path from typing import List import tqdm @@ -68,7 +69,7 @@ class TAPCached(Probe): def __init__( self, - prompts_location: str = _config.transient.basedir + prompts_location: Path = _config.transient.basedir / "resources" / "tap" / "data" diff --git a/garak/probes/visual_jailbreak.py b/garak/probes/visual_jailbreak.py index 4afafd2fe..24d683407 100644 --- a/garak/probes/visual_jailbreak.py +++ b/garak/probes/visual_jailbreak.py @@ -49,7 +49,7 @@ class FigStep(Probe): def _load_SafeBench(self): safebench_data_dir = ( - _config.transient.basedir / "resources" / "visual_jailbreak" / "SafeBench" + _config.transient.cache_dir / "resources" / "visual_jailbreak" / "SafeBench" ) if not os.path.exists(safebench_data_dir): # make the dir @@ -94,7 +94,7 @@ def probe(self, generator): self.prompts = [ { "text": prompt["text"], - "image": str(_config.transient.basedir / prompt["image"]), + "image": str(_config.transient.cache_dir / prompt["image"]), } for prompt in self.prompts ] @@ -125,7 +125,7 @@ def probe(self, generator): self.prompts = [ { "text": prompt["text"], - "image": str(_config.transient.basedir / prompt["image"]), + "image": str(_config.transient.cache_dir / prompt["image"]), } for prompt in self.prompts ] diff --git a/garak/resources/autodan/autodan.py b/garak/resources/autodan/autodan.py index a140bcc64..1e9c8d385 100644 --- a/garak/resources/autodan/autodan.py +++ b/garak/resources/autodan/autodan.py @@ -30,6 +30,9 @@ autodan_resource_data = ( garak._config.transient.basedir / "resources" / "autodan" / "data" ) +cached_autodan_resource_data = ( + garak._config.transient.cache_dir / "resources" / "autodan" / "data" +) autodan_parser = argparse.ArgumentParser(description="AutoDAN config") autodan_parser.add_argument( "--num_steps", type=int, default=100, help="Number of steps to run generation" @@ -87,9 +90,9 @@ def autodan_generate( mutation_generator_name: str = "gpt-3.5-turbo", mutation_generator_type: str = "openai", hierarchical: bool = False, - out_path: str = str(autodan_resource_data / "autodan_prompts.txt"), - init_prompt_path: str = str(autodan_resource_data / "autodan_init.txt"), - reference_path: str = str(autodan_resource_data / "prompt_group.pth"), + out_path: Path = cached_autodan_resource_data / "autodan_prompts.txt", + init_prompt_path: Path = autodan_resource_data / "autodan_init.txt", + reference_path: Path = autodan_resource_data / "prompt_group.pth", low_memory: bool = False, random_seed: int = None, ): @@ -108,9 +111,9 @@ def autodan_generate( mutation_generator_name (str): Name of model to use as the mutation generator mutation_generator_type (str): Type of model to use as the mutation generator hierarchical (bool): Whether ot use hierarchical GA - out_path (str): Path to write generated AutoDAN string - init_prompt_path (str): Path to initial prompts - reference_path (str): Path to reference prompt tensors + out_path (Path): Path to write generated AutoDAN string + init_prompt_path (Path): Path to initial prompts + reference_path (Path): Path to reference prompt tensors low_memory (bool): Whether to use low memory random_seed (int): Random seed, if used. diff --git a/garak/resources/autodan/genetic.py b/garak/resources/autodan/genetic.py index 2a22db7a7..9e3389eb3 100644 --- a/garak/resources/autodan/genetic.py +++ b/garak/resources/autodan/genetic.py @@ -2,6 +2,7 @@ # SPDX-License-Identifier: Apache-2.0 import gc +import nltk.downloader import numpy as np import torch import random @@ -11,6 +12,7 @@ import nltk from nltk.corpus import stopwords, wordnet from collections import defaultdict, OrderedDict +from pathlib import Path import sys import time from logging import getLogger @@ -20,15 +22,33 @@ logger = getLogger(__name__) + +def _nltk_data(): + from nltk.downloader import Downloader + + nltk_data_path = Path(Downloader.default_download_dir) + if not nltk_data_path.exists(): + # if path not found then place in the user cache + # get env var for NLTK_DATA, fallback to create in cachedir / nltk_data + nltk_data_path = os.getenv("NLTK_DATA", None) + if nltk_data_path is None: + from garak import _config + + nltk_data_path = _config.transient.cache_dir / "nltk_data" + os.environ["NLTK_DATA"] = nltk_data_path + return nltk_data_path + + # TODO: Refactor into setup.py try: _ = stopwords.words("english") _ = nltk.word_tokenize("This is a normal English sentence") _ = wordnet.synsets("word") except LookupError as e: - nltk.download("stopwords") - nltk.download("punkt") - nltk.download("wordnet") + nltk_data_path = _nltk_data() + nltk.download("stopwords", download_dir=nltk_data_path) + nltk.download("punkt", download_dir=nltk_data_path) + nltk.download("wordnet", download_dir=nltk_data_path) # TODO: Could probably clean up the inputs here by using imports. diff --git a/garak/resources/beast/beast_attack.py b/garak/resources/beast/beast_attack.py index 3b0402dbf..b04f50914 100644 --- a/garak/resources/beast/beast_attack.py +++ b/garak/resources/beast/beast_attack.py @@ -2,6 +2,7 @@ # SPDX-License-Identifier: Apache-2.0 import logging +from pathlib import Path from typing import Union, Optional, Tuple import torch @@ -17,7 +18,7 @@ "A chat between a curious user and an artificial intelligence assistant. " "The assistant gives helpful, detailed, and polite answers to the user's questions." ) -beast_resource_data = garak._config.transient.basedir / "resources" / "beast" / "data" +beast_resource_data = garak._config.transient.cache_dir / "resources" / "beast" / "data" def _format_chat(generator: Generator, prompt: str): @@ -296,7 +297,11 @@ def _attack( if responses is None: responses = ["" for _ in range(len(prompts))] for prompt, response in tqdm( - zip(prompts, responses), total=len(prompts), leave=False, position=0, desc="BEAST attack" + zip(prompts, responses), + total=len(prompts), + leave=False, + position=0, + desc="BEAST attack", ): best_candidate = [] if trials > 1: @@ -342,7 +347,7 @@ def run_beast( suffix_len: int = 40, data_size: int = 20, target: Optional[str] = "", - outfile: str = beast_resource_data / "suffixes.txt", + outfile: Path = beast_resource_data / "suffixes.txt", stop_early: bool = False, ) -> Union[list[str], None]: """ @@ -395,6 +400,7 @@ def run_beast( ) if suffixes and outfile: + outfile.parent.mkdir(mode=0o740, parents=True, exist_ok=True) with open(outfile, "a") as f: for suffix in suffixes: f.write(f"{suffix}\n") diff --git a/garak/resources/common.py b/garak/resources/common.py index 0f7d5cab7..8112b5413 100644 --- a/garak/resources/common.py +++ b/garak/resources/common.py @@ -1,4 +1,5 @@ import logging +import shutil import urllib.error from pathlib import Path import pandas as pd @@ -42,12 +43,21 @@ def load_advbench(size: int = 0) -> pd.DataFrame: - advbench_path = ( + advbench_base_path = ( garak._config.transient.basedir / "resources" / "advbench" / "harmful_behaviors.csv" ) + advbench_path = ( + garak._config.transient.cache_dir + / "resources" + / "advbench" + / "harmful_behaviors.csv" + ) + if advbench_base_path.is_file() and not advbench_path.is_file(): + shutil.copy2(advbench_base_path, advbench_path) + if not advbench_path.is_file(): try: hb = "https://raw.githubusercontent.com/llm-attacks/llm-attacks/main/data/advbench/harmful_behaviors.csv" diff --git a/garak/resources/gcg/generate_gcg.py b/garak/resources/gcg/generate_gcg.py index 8c2a3abe0..f004d265f 100644 --- a/garak/resources/gcg/generate_gcg.py +++ b/garak/resources/gcg/generate_gcg.py @@ -37,7 +37,8 @@ logger = getLogger(__name__) -gcg_resource_data = garak._config.transient.basedir / "resources" / "gcg" / "data" +resource_data = garak._config.transient.basedir / "resources" +gcg_resource_data = garak._config.transient.cache_dir / "resources" / "gcg" / "data" # GCG parser used by interactive mode gcg_parser = ArgumentParser() @@ -52,7 +53,7 @@ gcg_parser.add_argument( "--train_data", type=str, - default=gcg_resource_data / "advbench" / "harmful_behaviors.csv", + default=resource_data / "advbench" / "harmful_behaviors.csv", help="Path to training data", ) gcg_parser.add_argument( @@ -64,7 +65,7 @@ gcg_parser.add_argument( "--outfile", type=str, - default=gcg_resource_data / "gcg_prompts.txt", + default=resource_data / "gcg_prompts.txt", help="Location to write GCG attack output", ) gcg_parser.add_argument( @@ -92,10 +93,10 @@ def run_gcg( transfer: bool = False, progressive: bool = False, stop_success: bool = True, - train_data: Union[str,None] = None, + train_data: Union[str, None] = None, n_train: int = 50, n_test: int = 0, - outfile: str = gcg_resource_data / "gcg.txt", + outfile: Path = gcg_resource_data / "gcg.txt", control_init: str = CONTROL_INIT, deterministic: bool = True, n_steps: int = 500, @@ -124,7 +125,7 @@ def run_gcg( train_data (str): Path to training data n_train (int): Number of training examples to use n_test (int): Number of test examples to use - outfile (str): Where to write successful prompts + outfile (Path): Where to write successful prompts control_init (str): Initial adversarial suffix to modify deterministic (bool): Whether or not to use deterministic gbda n_steps (int): Number of training steps @@ -178,7 +179,7 @@ def run_gcg( logfile = gcg_resource_data / "logs" f"{timestamp}_{model_string}.json" # Create logfile directory - p = Path(logfile).parent + p = logfile.parent p.mkdir(parents=True, exist_ok=True) ( diff --git a/garak/resources/tap/tap_main.py b/garak/resources/tap/tap_main.py index ac6741845..61bdda3a0 100644 --- a/garak/resources/tap/tap_main.py +++ b/garak/resources/tap/tap_main.py @@ -4,6 +4,7 @@ import copy import re import torch.cuda +from pathlib import Path from tqdm import tqdm from logging import getLogger from typing import Union, Optional @@ -34,7 +35,7 @@ SAVE_RESULTS = True resources_tap_data_file = ( - garak._config.transient.basedir + garak._config.transient.cache_dir / "resources" / "tap" / "data" @@ -340,7 +341,7 @@ def run_tap( keep_last_n: int = 1, pruning: bool = True, save_results: bool = SAVE_RESULTS, - outfile: str = resources_tap_data_file, + outfile: Path = resources_tap_data_file, ): """ Function for generating attacks using TAP where a generator has already been instantiated. @@ -527,6 +528,7 @@ def run_tap( jailbreaks = list(set(jailbreaks)) msg = f"Found {len(jailbreaks)} jailbreak(s). Exiting." if save_results: + outfile.parent.mkdir(mode=0o740, parents=True, exist_ok=True) with open(outfile, "a", encoding="utf-8") as f: for jailbreak in jailbreaks: f.write(f"{jailbreak}\n") @@ -565,7 +567,7 @@ def generate_tap( n_streams: int = 1, keep_last_n: int = 1, save_results: bool = SAVE_RESULTS, - outfile: str = resources_tap_data_file, + outfile: Path = resources_tap_data_file, ): """ Function for generating attacks using TAP when a generator has not been instantiated. diff --git a/pyproject.toml b/pyproject.toml index b112d0a44..ebe1da6cf 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -67,7 +67,8 @@ dependencies = [ "fschat>=0.2.36", "litellm>=1.33.8", "jsonpath-ng>=1.6.1", - "lorem==0.1.1" + "lorem==0.1.1", + "xdg-base-dirs>=6.0.1", ] [project.optional-dependencies] diff --git a/requirements.txt b/requirements.txt index ac25f769e..192b642a2 100644 --- a/requirements.txt +++ b/requirements.txt @@ -29,6 +29,7 @@ fschat>=0.2.36 litellm>=1.33.8 jsonpath-ng>=1.6.1 lorem==0.1.1 +xdg-base-dirs>=6.0.1 # tests pytest>=8.0 requests-mock==1.12.1 diff --git a/tests/analyze/test_analyze.py b/tests/analyze/test_analyze.py index df93dd31d..9c8209e77 100644 --- a/tests/analyze/test_analyze.py +++ b/tests/analyze/test_analyze.py @@ -1,13 +1,12 @@ # SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -import os import subprocess import sys import pytest -from garak import cli +from garak import cli, _config TEMP_PREFIX = "_garak_internal_test_temp" @@ -23,7 +22,11 @@ def test_analyze_log_runs(): sys.executable, "-m", "garak.analyze.analyze_log", - TEMP_PREFIX + ".report.jsonl", + str( + _config.transient.user_data_dir + / _config.reporting.report_dir + / f"{TEMP_PREFIX}.report.jsonl" + ), ], check=True, ) @@ -36,7 +39,11 @@ def test_report_digest_runs(): sys.executable, "-m", "garak.analyze.report_digest", - TEMP_PREFIX + ".report.jsonl", + str( + _config.transient.user_data_dir + / _config.reporting.report_dir + / f"{TEMP_PREFIX}.report.jsonl" + ), ], check=True, ) diff --git a/tests/buffs/test_buff_config.py b/tests/buffs/test_buff_config.py index 5ab21c0f1..e7200caca 100644 --- a/tests/buffs/test_buff_config.py +++ b/tests/buffs/test_buff_config.py @@ -16,11 +16,13 @@ import pytest -import garak -import garak.cli +from garak import cli, _config PREFIX = "test_buff_single" + str(uuid.uuid4()) +_config.load_config() +REPORT_PATH = _config.transient.user_data_dir / _config.reporting.report_dir + def test_include_original_prompt(): # https://github.com/python/cpython/pull/97015 to ensure Windows compatibility @@ -34,13 +36,15 @@ def test_include_original_prompt(): ) ) tmp.close() - garak.cli.main( + cli.main( f"-m test -p test.Test -b lowercase.Lowercase --config {tmp.name} --report_prefix {PREFIX}".split() ) os.remove(tmp.name) prompts = [] - with open(f"{PREFIX}.report.jsonl", "r", encoding="utf-8") as reportfile: + with open( + REPORT_PATH / f"{PREFIX}.report.jsonl", "r", encoding="utf-8" + ) as reportfile: for line in reportfile: r = json.loads(line) if r["entry_type"] == "attempt" and r["status"] == 1: @@ -68,13 +72,15 @@ def test_exclude_original_prompt(): ) ) tmp.close() - garak.cli.main( + cli.main( f"-m test -p test.Test -b lowercase.Lowercase --config {tmp.name} --report_prefix {PREFIX}".split() ) os.remove(tmp.name) prompts = [] - with open(f"{PREFIX}.report.jsonl", "r", encoding="utf-8") as reportfile: + with open( + REPORT_PATH / f"{PREFIX}.report.jsonl", "r", encoding="utf-8" + ) as reportfile: for line in reportfile: r = json.loads(line) if r["entry_type"] == "attempt" and r["status"] == 1: @@ -89,9 +95,9 @@ def cleanup(request): def remove_buff_reports(): files = [ - f"{PREFIX}.report.jsonl", - f"{PREFIX}.report.html", - f"{PREFIX}.hitlog.jsonl", + REPORT_PATH / f"{PREFIX}.report.jsonl", + REPORT_PATH / f"{PREFIX}.report.html", + REPORT_PATH / f"{PREFIX}.hitlog.jsonl", ] for file in files: if os.path.exists(file): diff --git a/tests/test_attempt.py b/tests/test_attempt.py index dc4c50e5b..73edb7363 100644 --- a/tests/test_attempt.py +++ b/tests/test_attempt.py @@ -7,15 +7,19 @@ import pytest import garak.attempt -import garak.cli +from garak import cli, _config + +PREFIX = "_garak_test_attempt_sticky_params" def test_attempt_sticky_params(capsys): - garak.cli.main( - "-m test.Blank -g 1 -p atkgen,dan.Dan_6_0 --report_prefix _garak_test_attempt_sticky_params".split() + + cli.main( + f"-m test.Blank -g 1 -p atkgen,dan.Dan_6_0 --report_prefix {PREFIX}".split() ) + report_path = _config.transient.user_data_dir / _config.reporting.report_dir reportlines = ( - open("_garak_test_attempt_sticky_params.report.jsonl", "r", encoding="utf-8") + open(report_path / f"{PREFIX}.report.jsonl", "r", encoding="utf-8") .read() .split("\n") ) diff --git a/tests/test_config.py b/tests/test_config.py index b431c7c24..4190adf98 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -10,6 +10,7 @@ import pytest +from pathlib import Path from garak import _config import garak.cli @@ -95,16 +96,20 @@ def allow_site_config(request): site_cfg_moved = False try: - shutil.move("garak/garak.site.yaml", SITE_YAML_FILENAME) + shutil.move( + _config.transient.config_dir / "garak.site.yaml", SITE_YAML_FILENAME + ) site_cfg_moved = True except FileNotFoundError: site_cfg_moved = False def restore_site_config(): if site_cfg_moved: - shutil.move(SITE_YAML_FILENAME, "garak/garak.site.yaml") - elif os.path.exists("garak/garak.site.yaml"): - os.remove("garak/garak.site.yaml") + shutil.move( + SITE_YAML_FILENAME, _config.transient.config_dir / "garak.site.yaml" + ) + elif os.path.exists(_config.transient.config_dir / "garak.site.yaml"): + os.remove(_config.transient.config_dir / "garak.site.yaml") request.addfinalizer(restore_site_config) @@ -188,7 +193,9 @@ def test_yaml_param_settings(param): def test_site_yaml_overrides_core_yaml(): importlib.reload(_config) - with open("garak/garak.site.yaml", "w", encoding="utf-8") as f: + with open( + _config.transient.config_dir / "garak.site.yaml", "w", encoding="utf-8" + ) as f: f.write("---\nrun:\n eval_threshold: 0.777\n") f.flush() garak.cli.main(["--list_config"]) @@ -201,7 +208,9 @@ def test_site_yaml_overrides_core_yaml(): def test_run_yaml_overrides_site_yaml(): importlib.reload(_config) - with open("garak/garak.site.yaml", "w", encoding="utf-8") as f: + with open( + _config.transient.config_dir / "garak.site.yaml", "w", encoding="utf-8" + ) as f: file_data = [ "---", "run:", @@ -613,15 +622,46 @@ def test_tag_filter(): assert "probes.lmrc.SexualContent" in found +# when provided an absolute path as `reporting.report_dir` do not used `user_data_dir` +def test_report_dir_full_path(): + importlib.reload(_config) + + with tempfile.TemporaryDirectory() as tmpdir: + + report_path = Path(tmpdir).absolute() + with tempfile.NamedTemporaryFile(buffering=0, delete=False) as tmp: + tmp.write( + "\n".join( + [ + f"---", + f"reporting:", + f" report_dir: {report_path}", + ] + ).encode("utf-8") + ) + tmp.close() + garak.cli.main( + f"-m test.Blank --report_prefix abs_path_test -p test.Blank -d always.Fail --config {tmp.name}".split() + ) + os.remove(tmp.name) + assert os.path.isfile(report_path / "abs_path_test.report.jsonl") + assert os.path.isfile(report_path / "abs_path_test.report.html") + assert os.path.isfile(report_path / "abs_path_test.hitlog.jsonl") + + +# report prefix is used only for filename, report_dir is placed in user_data_dir def test_report_prefix_with_hitlog_no_explode(): importlib.reload(_config) garak.cli.main( "-m test.Blank --report_prefix kjsfhgkjahpsfdg -p test.Blank -d always.Fail".split() ) - assert os.path.isfile("kjsfhgkjahpsfdg.report.jsonl") - assert os.path.isfile("kjsfhgkjahpsfdg.report.html") - assert os.path.isfile("kjsfhgkjahpsfdg.hitlog.jsonl") + report_path = Path(_config.transient.report_filename).parent + assert _config.reporting.report_dir in str(report_path) + assert str(_config.transient.user_data_dir) in str(report_path) + assert os.path.isfile(report_path / "kjsfhgkjahpsfdg.report.jsonl") + assert os.path.isfile(report_path / "kjsfhgkjahpsfdg.report.html") + assert os.path.isfile(report_path / "kjsfhgkjahpsfdg.hitlog.jsonl") def test_nested():