From 3b9104f40dc949ec1f5c83718f9238d55b1429e6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Benjamin=20R=C3=B6sner?= Date: Thu, 9 Nov 2023 09:00:41 +0100 Subject: [PATCH 1/7] test: add tests for pathlib.Path. --- fsspec/tests/test_utils.py | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/fsspec/tests/test_utils.py b/fsspec/tests/test_utils.py index b5732e9f1..c7bad93bc 100644 --- a/fsspec/tests/test_utils.py +++ b/fsspec/tests/test_utils.py @@ -1,5 +1,6 @@ import io import sys +from pathlib import Path from unittest.mock import Mock import pytest @@ -8,6 +9,7 @@ from fsspec.utils import ( can_be_local, common_prefix, + get_protocol, infer_storage_options, merge_offset_ranges, mirror_from, @@ -333,6 +335,24 @@ def test_log(): assert logger.level == logging.DEBUG +@pytest.mark.parametrize( + "par", + [ + ("afile", "file"), + ("file://afile", "file"), + ("noproto://afile", "noproto"), + ("noproto::stuff", "noproto"), + ("simplecache::stuff", "simplecache"), + ("simplecache://stuff", "simplecache"), + ("s3://afile", "s3"), + (Path("afile"), "file"), + ], +) +def test_get_protocol(par): + url, outcome = par + assert get_protocol(url) == outcome + + @pytest.mark.parametrize( "par", [ @@ -342,6 +362,7 @@ def test_log(): ("noproto::stuff", False), ("simplecache::stuff", True), ("simplecache://stuff", True), + (Path("afile"), True), ], ) def test_can_local(par): From c531a48f58cad47dc8c4137b8e4489913178fe90 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Benjamin=20R=C3=B6sner?= Date: Thu, 9 Nov 2023 09:01:41 +0100 Subject: [PATCH 2/7] fix: can_be_local for pathlib.Path. --- fsspec/utils.py | 1 + 1 file changed, 1 insertion(+) diff --git a/fsspec/utils.py b/fsspec/utils.py index 34f1ad821..543bd89d7 100644 --- a/fsspec/utils.py +++ b/fsspec/utils.py @@ -436,6 +436,7 @@ def isfilelike(f: Any) -> TypeGuard[IO[bytes]]: def get_protocol(url: str) -> str: + url = stringify_path(url) parts = re.split(r"(\:\:|\://)", url, 1) if len(parts) > 1: return parts[0] From 8207f2cd094a71858aadf8e649d00b55e02a70a0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Benjamin=20R=C3=B6sner?= Date: Thu, 9 Nov 2023 10:10:00 +0100 Subject: [PATCH 3/7] test: add tests for open_local (list of) path/str. --- fsspec/tests/test_core.py | 42 ++++++++++++++++++++++++++++++++++++++- 1 file changed, 41 insertions(+), 1 deletion(-) diff --git a/fsspec/tests/test_core.py b/fsspec/tests/test_core.py index 6e8e1c751..a14f68881 100644 --- a/fsspec/tests/test_core.py +++ b/fsspec/tests/test_core.py @@ -3,6 +3,7 @@ import tempfile import zipfile from contextlib import contextmanager +from pathlib import Path import pytest @@ -101,7 +102,7 @@ def test_openfile_open(m): assert m.size("somepath") == 5 -def test_open_local(): +def test_open_local_w_cache(): d1 = str(tempfile.mkdtemp()) f1 = os.path.join(d1, "f1") open(f1, "w").write("test1") @@ -112,6 +113,45 @@ def test_open_local(): assert d2 in fn +def test_open_local_w_magic(): + d1 = str(tempfile.mkdtemp()) + f1 = os.path.join(d1, "f1") + open(f1, "w").write("test1") + fn = open_local(os.path.join(d1, "f*")) + assert len(fn) == 1 + assert isinstance(fn, list) + + +def test_open_local_w_list_of_str(): + d1 = str(tempfile.mkdtemp()) + f1 = os.path.join(d1, "f1") + open(f1, "w").write("test1") + fn = open_local([f1, f1]) + assert len(fn) == 2 + assert isinstance(fn, list) + assert all(isinstance(elem, str) for elem in fn) + + +def test_open_local_w_path(): + d1 = str(tempfile.mkdtemp()) + f1 = os.path.join(d1, "f1") + open(f1, "w").write("test1") + p = Path(f1) + fn = open_local(p) + assert isinstance(fn, str) + + +def test_open_local_w_list_of_path(): + d1 = str(tempfile.mkdtemp()) + f1 = os.path.join(d1, "f1") + open(f1, "w").write("test1") + p = Path(f1) + fn = open_local([p, p]) + assert len(fn) == 2 + assert isinstance(fn, list) + assert all(isinstance(elem, str) for elem in fn) + + def test_xz_lzma_compressions(): pytest.importorskip("lzma") # Ensure that both 'xz' and 'lzma' compression names can be parsed From 7875821792b781c967e0dc8e8615e3e3e39cd814 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Benjamin=20R=C3=B6sner?= Date: Thu, 9 Nov 2023 10:26:19 +0100 Subject: [PATCH 4/7] fix: pathlib.Path returning list instead of string. --- fsspec/core.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/fsspec/core.py b/fsspec/core.py index c5bce58ab..dba8fa121 100644 --- a/fsspec/core.py +++ b/fsspec/core.py @@ -3,6 +3,8 @@ import os import re from glob import has_magic +from pathlib import Path +from typing import Any, Literal, Optional # for backwards compat, we export cache things from here too from .caching import ( # noqa: F401 @@ -469,7 +471,11 @@ def open( return out[0] -def open_local(url, mode="rb", **storage_options): +def open_local( + url: str | list[str] | Path | list[Path], + mode: Optional[Literal["rb"]] = "rb", + **storage_options: Optional[Any], +) -> str | list[str]: """Open file(s) which can be resolved to local For files which either are local, or get downloaded upon open @@ -493,7 +499,7 @@ def open_local(url, mode="rb", **storage_options): ) with of as files: paths = [f.name for f in files] - if isinstance(url, str) and not has_magic(url): + if (isinstance(url, str) and not has_magic(url)) or isinstance(url, Path): return paths[0] return paths From 5cce61850da2fc0a6716a5c0b0b0ffa97fa4b1b0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Benjamin=20R=C3=B6sner?= Date: Thu, 9 Nov 2023 10:43:20 +0100 Subject: [PATCH 5/7] fix: remove unrelated features. --- fsspec/tests/test_utils.py | 21 --------------------- fsspec/utils.py | 1 - 2 files changed, 22 deletions(-) diff --git a/fsspec/tests/test_utils.py b/fsspec/tests/test_utils.py index c7bad93bc..b5732e9f1 100644 --- a/fsspec/tests/test_utils.py +++ b/fsspec/tests/test_utils.py @@ -1,6 +1,5 @@ import io import sys -from pathlib import Path from unittest.mock import Mock import pytest @@ -9,7 +8,6 @@ from fsspec.utils import ( can_be_local, common_prefix, - get_protocol, infer_storage_options, merge_offset_ranges, mirror_from, @@ -335,24 +333,6 @@ def test_log(): assert logger.level == logging.DEBUG -@pytest.mark.parametrize( - "par", - [ - ("afile", "file"), - ("file://afile", "file"), - ("noproto://afile", "noproto"), - ("noproto::stuff", "noproto"), - ("simplecache::stuff", "simplecache"), - ("simplecache://stuff", "simplecache"), - ("s3://afile", "s3"), - (Path("afile"), "file"), - ], -) -def test_get_protocol(par): - url, outcome = par - assert get_protocol(url) == outcome - - @pytest.mark.parametrize( "par", [ @@ -362,7 +342,6 @@ def test_get_protocol(par): ("noproto::stuff", False), ("simplecache::stuff", True), ("simplecache://stuff", True), - (Path("afile"), True), ], ) def test_can_local(par): diff --git a/fsspec/utils.py b/fsspec/utils.py index 543bd89d7..34f1ad821 100644 --- a/fsspec/utils.py +++ b/fsspec/utils.py @@ -436,7 +436,6 @@ def isfilelike(f: Any) -> TypeGuard[IO[bytes]]: def get_protocol(url: str) -> str: - url = stringify_path(url) parts = re.split(r"(\:\:|\://)", url, 1) if len(parts) > 1: return parts[0] From a2258f38bbb44fb41dc228ba568fd49c5e23781a Mon Sep 17 00:00:00 2001 From: Martin Durant Date: Thu, 9 Nov 2023 14:42:22 -0500 Subject: [PATCH 6/7] git types --- fsspec/core.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/fsspec/core.py b/fsspec/core.py index dba8fa121..01dc14ab2 100644 --- a/fsspec/core.py +++ b/fsspec/core.py @@ -1,10 +1,11 @@ +from __future__ import annotations + import io import logging import os import re from glob import has_magic from pathlib import Path -from typing import Any, Literal, Optional # for backwards compat, we export cache things from here too from .caching import ( # noqa: F401 @@ -473,8 +474,8 @@ def open( def open_local( url: str | list[str] | Path | list[Path], - mode: Optional[Literal["rb"]] = "rb", - **storage_options: Optional[Any], + mode: str, + **storage_options: dict, ) -> str | list[str]: """Open file(s) which can be resolved to local From b9f6364a573eda20bb2fd9aac2d2e80369954a71 Mon Sep 17 00:00:00 2001 From: Martin Durant Date: Thu, 9 Nov 2023 14:59:56 -0500 Subject: [PATCH 7/7] readd removed default value --- fsspec/core.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fsspec/core.py b/fsspec/core.py index 01dc14ab2..a1e15b2eb 100644 --- a/fsspec/core.py +++ b/fsspec/core.py @@ -474,7 +474,7 @@ def open( def open_local( url: str | list[str] | Path | list[Path], - mode: str, + mode: str = "rb", **storage_options: dict, ) -> str | list[str]: """Open file(s) which can be resolved to local