Workflow tests caching (#62)

* pass all parameters to cellfinder_run

* changed required, optional and internal fields. read_config passes. add method to config class (WIP)

* add signal and background data from local

* add missing data config fixtures

* mark GIN download data test as slow

* refactor config class methods and remove setup_workflow

* tests pass with default config

* remove unused fixtures

* update entry point

* change cellfinder-core to cellfinder in pyproject

* update cellfinder to cellfinder_core

* renaming cellfinder script workflow to cellfinder_core. fix entrypoint test.

* docstrings for class config

* fix entry point

* skip add input paths test

* split tests with and without CLI inputs

* all tests passing with default config

* all tests passing with default option

* make all paths input strings. move methods to class. make some fixtures dict. simplify other fixtures

* refactor setup_workflow

* cosmetic changes to fixtures

* remove spurious monkeypatched cwd from merge

* add skips that were removed in merge

* move fixtures to code where they are used

* bring back default_input_config_cellfinder fixture

* finalise adding local config as parameter in test_read_cellfinder_config

* add local config to remaining unit tests

* add local config and GIN config to test main

* monkeypatched home in integration tests (only works in main)

* skip tests that use subprocess for now

* monkeypatch pooch.retrieve when forcing download

* make config local fixture copy downloaded GIN data (rather than re-download again)

* fix output dir definition

* docstrings and remove merge artifact

* fill in docstrings

* add GIN default location as a fixture

* remove commented out GIN default location

* remove tests for running script (monkeypatched home function not passed to subprocess, and all subsequent steps already tested)

* replace entry point test for smoketest

* fixes from rebase

* fix ruff

brainglobe_workflows/cellfinder_core/cellfinder_core.py

@@ -10,9 +10,9 @@
 Example usage:
  - to pass a custom configuration, run (from the cellfinder_main.py
    parent directory):
-    python cellfinder_main.py --config path/to/input/config.json
+    python cellfinder_core.py --config path/to/input/config.json
  - to use the default configuration, run
-    python cellfinder_main.py
+    python cellfinder_core.py
 
 
 """

brainglobe_workflows/utils.py

@@ -70,14 +70,23 @@ def config_parser(
     # initialise argument parser
     parser = argparse.ArgumentParser(
         description=(
-            "To launch the workflow with "
-            "a specific set of input parameters, run: "
-            "`python brainglobe_workflows/cellfinder.py "
-            "--config path/to/config.json`"
-            "where path/to/input/config.json is the json file "
-            "containing the workflow parameters."
-        )
+            """
+            To launch the cellfinder workflow with default parameters, run:
+                `cellfinder-workflow`.
+            The default parameters are those specifed in brainglobe_workflows/
+            cellfinder_core/configs/cellfinder.json.
+
+
+            To launch the cellfinder workflow with a specific set of input
+            parameters, run:
+                `cellfinder-workflow --config path/to/config.json`,
+            where `path/to/input/config.json` is the json file with the
+            desired parameters.
+            """
+        ),
+        formatter_class=argparse.RawTextHelpFormatter,
     )
+
     # add arguments
     parser.add_argument(
         "-c",

pyproject.toml

@@ -24,7 +24,7 @@ classifiers = [
     "Topic :: Scientific/Engineering :: Image Recognition",
 ]
 
-# Below the dependenciess for brainmapper (the cellfinder CLI tool) only
+# Below the dependencies for brainmapper (the cellfinder CLI tool) only
 # (i.e., only what users will need for brainmapper)
 dependencies = [
     "brainglobe>=1.0.0",

tests/cellfinder_core/conftest.py

@@ -1,13 +1,47 @@
 """Pytest fixtures shared across unit and integration tests"""
 
+
+import json
 from pathlib import Path
 
+import pooch
 import pytest
 
 
+@pytest.fixture(autouse=True)
+def mock_home_directory(monkeypatch: pytest.MonkeyPatch):
+    """
+    Monkeypatch pathlib.Path.home()
+
+    Instead of returning the usual home path, the
+    monkeypatched version returns the path to
+    Path.home() / ".brainglobe-tests"
+
+    Parameters
+    ----------
+    monkeypatch : pytest.MonkeyPatch
+       a monkeypatch fixture
+    """
+    # define mock home path
+    home_path = Path.home()  # actual home path
+    mock_home_path = home_path / ".brainglobe-tests"
+
+    # create mock home directory if it doesn't exist
+    if not mock_home_path.exists():
+        mock_home_path.mkdir()
+
+    # monkeypatch Path.home() to point to the mock home
+    def mock_home():
+        return mock_home_path
+
+    monkeypatch.setattr(Path, "home", mock_home)
+
+
 @pytest.fixture()
 def default_input_config_cellfinder() -> Path:
-    """Return path to default input config for cellfinder workflow
+    """
+    Fixture for the path to the default input
+    configuration file for the cellfinder workflow
 
     Returns
     -------
@@ -20,18 +54,198 @@ def default_input_config_cellfinder() -> Path:
     return DEFAULT_JSON_CONFIG_PATH_CELLFINDER
 
 
-@pytest.fixture(autouse=True)
-def mock_home_directory(monkeypatch: pytest.MonkeyPatch):
-    # define mock home path
-    home_path = Path.home()  # actual home path
-    mock_home_path = home_path / ".brainglobe-tests"  # tmp_path  #
+@pytest.fixture(scope="session")
+def cellfinder_GIN_data() -> dict:
+    """
+    Fixture for the location of the test data in the GIN repository
 
-    # create dir if it doesn't exist
-    if not mock_home_path.exists():
-        mock_home_path.mkdir()
+    Returns
+    -------
+    dict
+        URL and hash of the GIN repository with the cellfinder test data
+    """
+    return {
+        "url": "https://gin.g-node.org/BrainGlobe/test-data/raw/master/cellfinder/cellfinder-test-data.zip",
+        "hash": "b0ef53b1530e4fa3128fcc0a752d0751909eab129d701f384fc0ea5f138c5914",  # noqa
+    }
 
-    # monkeypatch Path.home() to point to the mock home
-    def mock_home():
-        return mock_home_path
 
-    monkeypatch.setattr(Path, "home", mock_home)
+@pytest.fixture()
+def GIN_default_location() -> Path:
+    """A fixture returning a path to the default location
+    where GIN data is downloaded.
+
+    Returns
+    -------
+    Path
+        path to the default location where to download GIN data
+    """
+
+    return (
+        Path.home()
+        / ".brainglobe"
+        / "workflows"
+        / "cellfinder_core"
+        / "cellfinder_test_data"
+    )
+
+
+@pytest.fixture()
+def config_GIN_dict(
+    cellfinder_GIN_data: dict,
+    default_input_config_cellfinder: Path,
+    GIN_default_location: Path,
+) -> dict:
+    """
+    Fixture that returns a config as a dictionary, pointing to the location
+    where the GIN data would be by default, and download the data there.
+
+    If the file exists in the given path and the hash matches, pooch.retrieve()
+    will not download the file and instead its path is returned.
+
+    Parameters
+    ----------
+    cellfinder_GIN_data : dict
+        dictionary with the location of the test data in the GIN repository
+    default_input_config_cellfinder : Path
+        path to the default input configuration file for the cellfinder
+        workflow
+    GIN_default_location : Path
+        path to the default location where to download GIN data
+
+    Returns
+    -------
+    dict
+        dictionary with the config for a workflow that uses the downloaded
+        GIN data
+    """
+
+    # read default config as a dictionary
+    with open(default_input_config_cellfinder) as cfg:
+        config_dict = json.load(cfg)
+
+    # modify the default config:
+    # - add url
+    # - add data hash
+    # - remove input_data_dir if present
+    config_dict["data_url"] = cellfinder_GIN_data["url"]
+    config_dict["data_hash"] = cellfinder_GIN_data["hash"]
+    if "input_data_dir" in config_dict.keys():
+        del config_dict["input_data_dir"]
+
+    # download GIN data to default location for GIN
+    # if the file exists in the given path and the hash matches,
+    # it will not be downloaded and the absolute path to the file is returned.
+    pooch.retrieve(
+        url=cellfinder_GIN_data["url"],
+        known_hash=cellfinder_GIN_data["hash"],
+        path=GIN_default_location.parent,  # path to download zip to
+        progressbar=True,
+        processor=pooch.Unzip(extract_dir=GIN_default_location.stem),
+    )
+
+    return config_dict
+
+
+@pytest.fixture()
+def config_local_dict(
+    config_GIN_dict: dict,
+    GIN_default_location: Path,
+) -> dict:
+    """
+    Fixture that returns a config as a dictionary, pointing to a local dataset.
+
+    The data is copied to the local directory from the
+    default location used in the config_GIN_dict fixture.
+
+    Parameters
+    ----------
+    config_GIN_dict : dict
+        dictionary with the config for a workflow that uses the downloaded
+        GIN data
+    GIN_default_location : Path
+        path to the default location where to download GIN data\
+
+    Returns
+    -------
+    dict
+        dictionary with the config for a workflow that uses local data
+    """
+    import shutil
+
+    # copy GIN config as a dictionary
+    config_dict = config_GIN_dict.copy()
+
+    # modify the GIN config:
+    # - remove url
+    # - remove data hash
+    # - set input data directory to a local directory under home
+    config_dict["data_url"] = None
+    config_dict["data_hash"] = None
+    config_dict["input_data_dir"] = str(Path.home() / "local_cellfinder_data")
+
+    # copy data from default GIN location to the local location
+    shutil.copytree(
+        GIN_default_location,
+        config_dict["input_data_dir"],
+        dirs_exist_ok=True,
+    )
+
+    return config_dict
+
+
+@pytest.fixture()
+def config_GIN_json(config_GIN_dict: dict, tmp_path: Path) -> Path:
+    """
+    Fixture that returns a config as a JSON file path, that points to GIN
+    downloaded data as input
+
+    Parameters
+    ----------
+    config_GIN_dict : dict
+        dictionary with the config for a workflow that uses the downloaded
+        GIN data
+    tmp_path : Path
+        Pytest fixture providing a temporary path
+
+    Returns
+    -------
+    Path
+        path to a cellfinder config JSON file
+    """
+    # define location of input config file
+    config_file_path = tmp_path / "input_config.json"
+
+    # write config dict to that location
+    with open(config_file_path, "w") as js:
+        json.dump(config_GIN_dict, js)
+
+    return config_file_path
+
+
+@pytest.fixture()
+def config_local_json(config_local_dict: dict, tmp_path: Path) -> Path:
+    """
+    Fixture that returns config as a JSON file path, that points to local data
+    as input
+
+    Parameters
+    ----------
+    config_local_dict : dict
+        _description_
+    tmp_path : Path
+        Pytest fixture providing a temporary path
+
+    Returns
+    -------
+    Path
+        path to a cellfinder config JSON file
+    """
+    # define location of input config file
+    config_file_path = tmp_path / "input_config.json"
+
+    # write config dict to that location
+    with open(config_file_path, "w") as js:
+        json.dump(config_local_dict, js)
+
+    return config_file_path