POC of base tests for fsspec behavior.

fsspec/tests/base/__init__.py

@@ -0,0 +1,3 @@
+from .spec import BaseFSTests, BaseReadTests
+
+__all__ = ["BaseFSTests", "BaseReadTests"]

fsspec/tests/base/spec.py

@@ -0,0 +1,28 @@
+import pytest
+
+
+class BaseSpecTests:
+    """Base class for all specification tests."""
+
+
+class BaseFSTests(BaseSpecTests):
+    """
+    Tests that the fixture object provided meets expectations. Validate this first.
+    """
+
+    def test_files_exist(self, fs, prefix):
+        assert fs.exists(f"{prefix}/exists")
+        assert fs.cat(f"{prefix}/exists") == b"data from /exists"
+
+
+class BaseReadTests(BaseSpecTests):
+    """
+    Tests that apply to read-only or read-write filesystems.
+    """
+
+    def test_ls_raises_filenotfound(self, fs, prefix):
+        with pytest.raises(FileNotFoundError):
+            fs.ls(f"{prefix}/not-a-key")
+
+        with pytest.raises(FileNotFoundError):
+            fs.ls(f"{prefix}/not/a/key")

fsspec/tests/conftest.py

@@ -0,0 +1,19 @@
+import pytest
+
+
+@pytest.fixture
+def prefix():
+    """The prefix to use as the root fo the filesystem."""
+    raise NotImplementedError("Downstream implementations should define 'prefix'.")
+
+
+@pytest.fixture
+def fs():
+    """
+    An fsspec-compatible subclass of AbstractFileSystem with the following properties:
+
+    **These files exist with these contents**
+
+    * /<prefix>/exists: data from /exists
+    """
+    raise NotImplementedError("Downstream implementations should define 'fs'.")

fsspec/tests/spec/test_http.py

@@ -0,0 +1,77 @@
+import contextlib
+import threading
+from http.server import BaseHTTPRequestHandler, HTTPServer
+
+import pytest
+
+from fsspec.implementations.http import HTTPFileSystem
+from fsspec.tests.base import BaseFSTests, BaseReadTests
+
+requests = pytest.importorskip("requests")
+port = 9898
+
+
+class HTTPTestHandler(BaseHTTPRequestHandler):
+    _FILES = {"/exists"}
+
+    def _respond(self, code=200, headers=None, data=b""):
+        headers = headers or {}
+        headers.update({"User-Agent": "test"})
+        self.send_response(code)
+        for k, v in headers.items():
+            self.send_header(k, str(v))
+        self.end_headers()
+        if data:
+            self.wfile.write(data)
+
+    def do_GET(self):
+        if self.path.rstrip("/") not in self._FILES:
+            self._respond(404)
+            return
+
+        self._respond(200, data=b"data from /exists")
+
+    def do_HEAD(self):
+        if "head_ok" not in self.headers:
+            self._respond(405)
+            return
+        self._respond(200)  # OK response, but no useful info
+
+
+@contextlib.contextmanager
+def serve():
+    server_address = ("", port)
+    httpd = HTTPServer(server_address, HTTPTestHandler)
+    th = threading.Thread(target=httpd.serve_forever)
+    th.daemon = True
+    th.start()
+    try:
+        yield "http://localhost:%i" % port
+    finally:
+        httpd.socket.close()
+        httpd.shutdown()
+        th.join()
+
+
+@pytest.fixture(scope="module")
+def server():
+    with serve() as s:
+        yield s
+
+
+@pytest.fixture
+def prefix():
+    return f"http://localhost:{port}"
+
+
+@pytest.fixture
+def fs(server):
+    return HTTPFileSystem()
+
+
+class TestFS(BaseFSTests):
+    pass
+
+
+class TestRead(BaseReadTests):
+    pass

fsspec/tests/spec/test_memory.py

@@ -0,0 +1,30 @@
+import pytest
+
+from fsspec.implementations.memory import MemoryFile, MemoryFileSystem
+from fsspec.tests.base import BaseFSTests, BaseReadTests
+
+
+@pytest.fixture
+def prefix():
+    return "/root/"
+
+
+@pytest.fixture
+def fs(prefix):
+    memfs = MemoryFileSystem()
+    memfs.store[f"{prefix}/exists"] = MemoryFile(
+        fs=memfs, path=f"{prefix}/exists", data=b"data from /exists"
+    )
+    return memfs
+
+
+class TestFS(BaseFSTests):
+    pass
+
+
+class TestRead(BaseReadTests):
+    @pytest.mark.xfail(
+        strict=True, reason="https://github.com/intake/filesystem_spec/issues/652"
+    )
+    def test_ls_raises_filenotfound(self, fs, prefix):
+        return super().test_ls_raises_filenotfound(fs, prefix)