diff --git a/tests/test_docs.py b/tests/test_docs.py index 7f7c0a9af..b352aabc6 100644 --- a/tests/test_docs.py +++ b/tests/test_docs.py @@ -1,8 +1,11 @@ import importlib import os from pathlib import Path +import yaml + import pytest + TOP_PATHS = ["probes", "detectors", "harnesses", "generators", "evaluators", "buffs"] DOC_SOURCE = os.path.join("docs", "source") @@ -165,3 +168,26 @@ def test_root_modules_docs(doc_index_source_text, root_module: str): assert ( f" {root_module.stem}\n" in doc_index_source_text ), f"root module doc page for {root_module.name} should be linked from doc root index" + + +def test_core_config_options_explained(): + import garak._config + + core_config_file_name = ( + garak._config.transient.package_dir / "resources" / "garak.core.yaml" + ) + doc_nodes_to_check = [] + with open(core_config_file_name, encoding="utf-8") as settings_file: + settings = yaml.safe_load(settings_file) + for top_level_setting in settings: + doc_nodes_to_check.append(top_level_setting) + for second_level_setting in settings[top_level_setting]: + doc_nodes_to_check.append(second_level_setting) + + configurable_rst = open( + Path("docs") / "source" / "configurable.rst", "r", encoding="utf-8" + ).read() + for doc_node_to_check in doc_nodes_to_check: + assert ( + f"``{doc_node_to_check}``" in configurable_rst + ), f"core config value '{doc_node_to_check}' must be documented in configurable.rst"