Do not use Path objects as configuration dictionary keys to avoid e…

…rrors in `dask.config.merge` (#2578) Co-authored-by: Valeriu Predoi <[email protected]>
ESMValGroup · Nov 12, 2024 · a328578 · a328578
1 parent 7892678
commit a328578
Show file tree

Hide file tree

Showing 3 changed files with 9 additions and 3 deletions.
diff --git a/esmvalcore/config/_config_validators.py b/esmvalcore/config/_config_validators.py
@@ -227,8 +227,13 @@ def validate_rootpath(value):
             new_mapping[key] = validate_pathlist(paths)
         else:
             validate_dict(paths)
+
+            # dask.config.merge cannot handle pathlib.Path objects as dict keys
+            # -> we convert the validated Path back to a string and handle this
+            # downstream in local.py (see also
+            # https://github.com/ESMValGroup/ESMValCore/issues/2577)
             new_mapping[key] = {
-                validate_path(path): validate_string(drs)
+                str(validate_path(path)): validate_string(drs)
                 for path, drs in paths.items()
             }
 

diff --git a/esmvalcore/local.py b/esmvalcore/local.py
@@ -479,6 +479,7 @@ def _get_data_sources(project: str) -> list[DataSource]:
                 paths = {p: structure for p in paths}
             sources: list[DataSource] = []
             for path, structure in paths.items():
+                path = Path(path)
                 dir_templates = _select_drs("input_dir", project, structure)
                 file_templates = _select_drs("input_file", project, structure)
                 sources.extend(

diff --git a/tests/unit/config/test_config_validator.py b/tests/unit/config/test_config_validator.py
@@ -176,8 +176,8 @@ def generate_validator_testcases(valid):
                     },
                     {
                         "CMIP6": {
-                            Path("/a"): "DKRZ",
-                            Path("/b"): "ESGF",
+                            "/a": "DKRZ",
+                            "/b": "ESGF",
                         },
                     },
                 ),