From 4f70f1b3d226898848941e33725de9d9df4294fa Mon Sep 17 00:00:00 2001 From: BENR0 Date: Thu, 9 Nov 2023 21:10:54 +0100 Subject: [PATCH] Fix open_local returning list for pathlib.Path (#1418) --------- Co-authored-by: Martin Durant --- fsspec/core.py | 11 ++++++++-- fsspec/tests/test_core.py | 42 ++++++++++++++++++++++++++++++++++++++- 2 files changed, 50 insertions(+), 3 deletions(-) diff --git a/fsspec/core.py b/fsspec/core.py index c5bce58ab..a1e15b2eb 100644 --- a/fsspec/core.py +++ b/fsspec/core.py @@ -1,8 +1,11 @@ +from __future__ import annotations + import io import logging import os import re from glob import has_magic +from pathlib import Path # for backwards compat, we export cache things from here too from .caching import ( # noqa: F401 @@ -469,7 +472,11 @@ def open( return out[0] -def open_local(url, mode="rb", **storage_options): +def open_local( + url: str | list[str] | Path | list[Path], + mode: str = "rb", + **storage_options: dict, +) -> str | list[str]: """Open file(s) which can be resolved to local For files which either are local, or get downloaded upon open @@ -493,7 +500,7 @@ def open_local(url, mode="rb", **storage_options): ) with of as files: paths = [f.name for f in files] - if isinstance(url, str) and not has_magic(url): + if (isinstance(url, str) and not has_magic(url)) or isinstance(url, Path): return paths[0] return paths diff --git a/fsspec/tests/test_core.py b/fsspec/tests/test_core.py index 6e8e1c751..a14f68881 100644 --- a/fsspec/tests/test_core.py +++ b/fsspec/tests/test_core.py @@ -3,6 +3,7 @@ import tempfile import zipfile from contextlib import contextmanager +from pathlib import Path import pytest @@ -101,7 +102,7 @@ def test_openfile_open(m): assert m.size("somepath") == 5 -def test_open_local(): +def test_open_local_w_cache(): d1 = str(tempfile.mkdtemp()) f1 = os.path.join(d1, "f1") open(f1, "w").write("test1") @@ -112,6 +113,45 @@ def test_open_local(): assert d2 in fn +def test_open_local_w_magic(): + d1 = str(tempfile.mkdtemp()) + f1 = os.path.join(d1, "f1") + open(f1, "w").write("test1") + fn = open_local(os.path.join(d1, "f*")) + assert len(fn) == 1 + assert isinstance(fn, list) + + +def test_open_local_w_list_of_str(): + d1 = str(tempfile.mkdtemp()) + f1 = os.path.join(d1, "f1") + open(f1, "w").write("test1") + fn = open_local([f1, f1]) + assert len(fn) == 2 + assert isinstance(fn, list) + assert all(isinstance(elem, str) for elem in fn) + + +def test_open_local_w_path(): + d1 = str(tempfile.mkdtemp()) + f1 = os.path.join(d1, "f1") + open(f1, "w").write("test1") + p = Path(f1) + fn = open_local(p) + assert isinstance(fn, str) + + +def test_open_local_w_list_of_path(): + d1 = str(tempfile.mkdtemp()) + f1 = os.path.join(d1, "f1") + open(f1, "w").write("test1") + p = Path(f1) + fn = open_local([p, p]) + assert len(fn) == 2 + assert isinstance(fn, list) + assert all(isinstance(elem, str) for elem in fn) + + def test_xz_lzma_compressions(): pytest.importorskip("lzma") # Ensure that both 'xz' and 'lzma' compression names can be parsed