From f818145f62ce15df03d8eb9e2f4a5b497e2487eb Mon Sep 17 00:00:00 2001 From: Tom Augspurger Date: Tue, 1 Jun 2021 10:01:35 -0500 Subject: [PATCH 1/6] POC for base tests This starts a suite of base tests to verify that downstream libraries correctly implement the spec. xref https://github.com/intake/filesystem_spec/issues/650 --- fsspec/tests/base/__init__.py | 3 ++ fsspec/tests/base/spec.py | 27 ++++++++++++ fsspec/tests/conftest.py | 19 +++++++++ fsspec/tests/spec/test_http.py | 77 ++++++++++++++++++++++++++++++++++ 4 files changed, 126 insertions(+) create mode 100644 fsspec/tests/base/__init__.py create mode 100644 fsspec/tests/base/spec.py create mode 100644 fsspec/tests/conftest.py create mode 100644 fsspec/tests/spec/test_http.py diff --git a/fsspec/tests/base/__init__.py b/fsspec/tests/base/__init__.py new file mode 100644 index 000000000..2762c8d6d --- /dev/null +++ b/fsspec/tests/base/__init__.py @@ -0,0 +1,3 @@ +from .spec import BaseFSTests, BaseReadTests + +__all__ = ["BaseFSTests", "BaseReadTests"] diff --git a/fsspec/tests/base/spec.py b/fsspec/tests/base/spec.py new file mode 100644 index 000000000..7ba9dfea1 --- /dev/null +++ b/fsspec/tests/base/spec.py @@ -0,0 +1,27 @@ +import pytest + + +class BaseSpecTests: + """Base class for all specification tests.""" + + +class BaseFSTests(BaseSpecTests): + """ + Tests that the fixture object provided meets expectations. Validate this first. + """ + + def test_files_exist(self, fs, prefix): + assert fs.exists(f"{prefix}/exists") + + +class BaseReadTests(BaseSpecTests): + """ + Tests that apply to read-only or read-write filesystems. + """ + + def test_ls_raises_filenotfound(self, fs, prefix): + with pytest.raises(FileNotFoundError): + fs.ls(f"{prefix}/not-a-key") + + with pytest.raises(FileNotFoundError): + fs.ls(f"{prefix}/not/a/key") diff --git a/fsspec/tests/conftest.py b/fsspec/tests/conftest.py new file mode 100644 index 000000000..c013f2862 --- /dev/null +++ b/fsspec/tests/conftest.py @@ -0,0 +1,19 @@ +import pytest + + +@pytest.fixture +def prefix(): + """The prefix to use as the root fo the filesystem.""" + raise NotImplementedError("Downstream implementations should define 'prefix'.") + + +@pytest.fixture +def fs(): + """ + An fsspec-compatible subclass of AbstractFileSystem with the following properties: + + **These files** + + * /root/a + """ + raise NotImplementedError("Downstream implementations should define 'fs'.") diff --git a/fsspec/tests/spec/test_http.py b/fsspec/tests/spec/test_http.py new file mode 100644 index 000000000..31a12d723 --- /dev/null +++ b/fsspec/tests/spec/test_http.py @@ -0,0 +1,77 @@ +import contextlib +import threading +from http.server import BaseHTTPRequestHandler, HTTPServer + +import pytest + +from fsspec.implementations.http import HTTPFileSystem +from fsspec.tests.base import BaseFSTests, BaseReadTests + +requests = pytest.importorskip("requests") +port = 9898 + + +class HTTPTestHandler(BaseHTTPRequestHandler): + _FILES = {"/exists"} + + def _respond(self, code=200, headers=None, data=b""): + headers = headers or {} + headers.update({"User-Agent": "test"}) + self.send_response(code) + for k, v in headers.items(): + self.send_header(k, str(v)) + self.end_headers() + if data: + self.wfile.write(data) + + def do_GET(self): + if self.path.rstrip("/") not in self._FILES: + self._respond(404) + return + + self._respond(200, data=b"data in /exists") + + def do_HEAD(self): + if "head_ok" not in self.headers: + self._respond(405) + return + self._respond(200) # OK response, but no useful info + + +@contextlib.contextmanager +def serve(): + server_address = ("", port) + httpd = HTTPServer(server_address, HTTPTestHandler) + th = threading.Thread(target=httpd.serve_forever) + th.daemon = True + th.start() + try: + yield "http://localhost:%i" % port + finally: + httpd.socket.close() + httpd.shutdown() + th.join() + + +@pytest.fixture(scope="module") +def server(): + with serve() as s: + yield s + + +@pytest.fixture +def prefix(): + return f"http://localhost:{port}" + + +@pytest.fixture +def fs(server): + return HTTPFileSystem() + + +class TestFS(BaseFSTests): + pass + + +class TestRead(BaseReadTests): + pass From f72b441acbb1788ed52bd828ccf286d144e9182d Mon Sep 17 00:00:00 2001 From: Tom Augspurger Date: Tue, 1 Jun 2021 10:08:24 -0500 Subject: [PATCH 2/6] failing memory --- fsspec/tests/base/spec.py | 1 + fsspec/tests/spec/test_memory.py | 26 ++++++++++++++++++++++++++ 2 files changed, 27 insertions(+) create mode 100644 fsspec/tests/spec/test_memory.py diff --git a/fsspec/tests/base/spec.py b/fsspec/tests/base/spec.py index 7ba9dfea1..5f960a8a0 100644 --- a/fsspec/tests/base/spec.py +++ b/fsspec/tests/base/spec.py @@ -12,6 +12,7 @@ class BaseFSTests(BaseSpecTests): def test_files_exist(self, fs, prefix): assert fs.exists(f"{prefix}/exists") + assert fs.cat(f"{prefix}/exists") == b"data from /exists" class BaseReadTests(BaseSpecTests): diff --git a/fsspec/tests/spec/test_memory.py b/fsspec/tests/spec/test_memory.py new file mode 100644 index 000000000..5f7633817 --- /dev/null +++ b/fsspec/tests/spec/test_memory.py @@ -0,0 +1,26 @@ +import pytest + +from fsspec.implementations.memory import MemoryFile, MemoryFileSystem +from fsspec.tests.base import BaseFSTests, BaseReadTests + + +@pytest.fixture +def prefix(): + return "/root/" + + +@pytest.fixture +def fs(prefix): + memfs = MemoryFileSystem() + memfs.store[f"{prefix}/exists"] = MemoryFile( + fs=memfs, path=f"{prefix}/exists", data=b"data from /exists" + ) + return memfs + + +class TestFS(BaseFSTests): + pass + + +class TestRead(BaseReadTests): + pass From e57adfba51347dfce85cdd212c2de8e271acaab9 Mon Sep 17 00:00:00 2001 From: Tom Augspurger Date: Wed, 2 Jun 2021 07:39:26 -0500 Subject: [PATCH 3/6] xfail ls --- fsspec/tests/spec/test_memory.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/fsspec/tests/spec/test_memory.py b/fsspec/tests/spec/test_memory.py index 5f7633817..a012a8e11 100644 --- a/fsspec/tests/spec/test_memory.py +++ b/fsspec/tests/spec/test_memory.py @@ -23,4 +23,6 @@ class TestFS(BaseFSTests): class TestRead(BaseReadTests): - pass + @pytest.mark.xfail(strict=True, reason="Bug in MemoryFileSystem") + def test_ls_raises_filenotfound(self, fs, prefix): + return super().test_ls_raises_filenotfound(fs, prefix) From abf65993a14db76de101abe129a9b6da91784858 Mon Sep 17 00:00:00 2001 From: Tom Augspurger Date: Wed, 2 Jun 2021 07:45:36 -0500 Subject: [PATCH 4/6] issue number --- fsspec/tests/spec/test_memory.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/fsspec/tests/spec/test_memory.py b/fsspec/tests/spec/test_memory.py index a012a8e11..f6a559622 100644 --- a/fsspec/tests/spec/test_memory.py +++ b/fsspec/tests/spec/test_memory.py @@ -23,6 +23,8 @@ class TestFS(BaseFSTests): class TestRead(BaseReadTests): - @pytest.mark.xfail(strict=True, reason="Bug in MemoryFileSystem") + @pytest.mark.xfail( + strict=True, reason="https://github.com/intake/filesystem_spec/issues/652" + ) def test_ls_raises_filenotfound(self, fs, prefix): return super().test_ls_raises_filenotfound(fs, prefix) From b926080c4f5dc8ac3e16d3404b0ec56f885d08f9 Mon Sep 17 00:00:00 2001 From: Tom Augspurger Date: Wed, 2 Jun 2021 07:47:54 -0500 Subject: [PATCH 5/6] fix content --- fsspec/tests/spec/test_http.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fsspec/tests/spec/test_http.py b/fsspec/tests/spec/test_http.py index 31a12d723..18cf179b7 100644 --- a/fsspec/tests/spec/test_http.py +++ b/fsspec/tests/spec/test_http.py @@ -29,7 +29,7 @@ def do_GET(self): self._respond(404) return - self._respond(200, data=b"data in /exists") + self._respond(200, data=b"data from /exists") def do_HEAD(self): if "head_ok" not in self.headers: From 9942d8dca89c9f8f0e9295190339b6cb3246f3a5 Mon Sep 17 00:00:00 2001 From: Tom Augspurger Date: Wed, 2 Jun 2021 07:49:08 -0500 Subject: [PATCH 6/6] fix docs --- fsspec/tests/conftest.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/fsspec/tests/conftest.py b/fsspec/tests/conftest.py index c013f2862..1f9a89772 100644 --- a/fsspec/tests/conftest.py +++ b/fsspec/tests/conftest.py @@ -12,8 +12,8 @@ def fs(): """ An fsspec-compatible subclass of AbstractFileSystem with the following properties: - **These files** + **These files exist with these contents** - * /root/a + * //exists: data from /exists """ raise NotImplementedError("Downstream implementations should define 'fs'.")