diff --git a/driver/tests.py b/driver/tests.py
index eaa74dcfa5..c594d87ebc 100644
--- a/driver/tests.py
+++ b/driver/tests.py
@@ -5,15 +5,19 @@
 """
 
 import os
+from pathlib import Path
 import subprocess
 import sys
+import traceback
 
 import pytest
 
 from .aliases import ALIASES, PORTFOLIOS
 from .arguments import EXAMPLES
+from .call import check_call
 from . import limits
 from . import returncodes
+from .run_components import get_executable, REL_SEARCH_PATH
 from .util import REPO_ROOT_DIR, find_domain_filename
 
 
@@ -59,6 +63,49 @@ def test_portfolios():
         run_driver(parameters)
 
 
+def _get_portfolio_configs(portfolio: Path):
+    content = portfolio.read_text()
+    attributes = {}
+    try:
+        exec(content, attributes)
+    except Exception:
+        traceback.print_exc()
+        raise SyntaxError(
+            f"The portfolio {portfolio} could not be loaded.")
+    if "CONFIGS" not in attributes:
+        raise ValueError("portfolios must define CONFIGS")
+    return [config for _, config in attributes["CONFIGS"]]
+
+
+def _convert_to_standalone_config(config):
+    replacements = [
+        ("H_COST_TRANSFORM", "no_transform()"),
+        ("S_COST_TYPE", "normal"),
+        ("BOUND", "infinity"),
+    ]
+    for index, part in enumerate(config):
+        for before, after in replacements:
+            part = part.replace(before, after)
+        config[index] = part
+    return config
+
+
+def _run_search(config):
+    check_call(
+        "search",
+        [get_executable("release", REL_SEARCH_PATH)] + list(config),
+        stdin="output.sas")
+
+
+def test_portfolio_configs():
+    all_configs = set()
+    for portfolio in PORTFOLIOS.values():
+        configs = _get_portfolio_configs(Path(portfolio))
+        all_configs |= set(tuple(_convert_to_standalone_config(config)) for config in configs)
+    for config in all_configs:
+        _run_search(config)
+
+
 @pytest.mark.skipif(not limits.can_set_time_limit(), reason="Cannot set time limits on this system")
 def test_hard_time_limit():
     def preexec_fn():