diff --git a/docs/source/api.rst b/docs/source/api.rst index ae94de103..cb14fe7e1 100644 --- a/docs/source/api.rst +++ b/docs/source/api.rst @@ -162,9 +162,6 @@ Built-in Implementations .. autoclass:: fsspec.implementations.ftp.FTPFileSystem :members: __init__ -.. autoclass:: fsspec.implementations.ftp.FTPTLSFileSystem - :members: __init__ - .. autoclass:: fsspec.implementations.git.GitFileSystem :members: __init__ diff --git a/fsspec/implementations/ftp.py b/fsspec/implementations/ftp.py index 41b414d0b..f0fd91c8e 100644 --- a/fsspec/implementations/ftp.py +++ b/fsspec/implementations/ftp.py @@ -27,8 +27,7 @@ def __init__( tempdir=None, timeout=30, encoding="utf-8", - ssl=False, - prot_p=False, + tls=False, **kwargs, ): """ @@ -58,26 +57,28 @@ def __init__( Timeout of the ftp connection in seconds encoding: str Encoding to use for directories and filenames in FTP connection + tls: bool + Use FTP-TLS, by default False """ super().__init__(**kwargs) self.host = host self.port = port self.tempdir = tempdir or "/tmp" - self.cred = username, password, acct + self.cred = username or "", password or "", acct or "" + print(self.cred) self.timeout = timeout self.encoding = encoding if block_size is not None: self.blocksize = block_size else: self.blocksize = 2**16 - self.ssl = ssl - self.prot_p = prot_p + self.tls = tls self._connect() - if self.prot_p: + if self.tls: self.ftp.prot_p() def _connect(self): - if self.ssl: + if self.tls: ftp_cls = FTP_TLS else: ftp_cls = FTP diff --git a/fsspec/implementations/tests/test_ftp.py b/fsspec/implementations/tests/test_ftp.py index d443d865b..98561489b 100644 --- a/fsspec/implementations/tests/test_ftp.py +++ b/fsspec/implementations/tests/test_ftp.py @@ -2,6 +2,7 @@ import subprocess import sys import time +from ftplib import FTP, FTP_TLS import pytest @@ -29,6 +30,19 @@ def ftp(): P.wait() +@pytest.mark.parametrize( + "tls,exp_cls", + ( + (False, FTP), + (True, FTP_TLS), + ), +) +def test_tls(ftp, tls, exp_cls): + host, port = ftp + fs = FTPFileSystem(host, port, tls=tls) + assert isinstance(fs.ftp, exp_cls) + + def test_basic(ftp): host, port = ftp fs = FTPFileSystem(host, port) diff --git a/fsspec/implementations/tests/test_ftp_tls.py b/fsspec/implementations/tests/test_ftp_tls.py deleted file mode 100644 index 1ee7afd6d..000000000 --- a/fsspec/implementations/tests/test_ftp_tls.py +++ /dev/null @@ -1,189 +0,0 @@ -import os -import subprocess -import sys -import time - -import pytest - -import fsspec -from fsspec import open_files -from fsspec.implementations.ftp import FTPFileSystem - -ftplib = pytest.importorskip("ftplib") -here = os.path.dirname(os.path.abspath(__file__)) - - -@pytest.fixture() -def ftp(): - P = subprocess.Popen( - [sys.executable, os.path.join(os.path.dirname(__file__), "ftp_tls.py")], - stderr=subprocess.STDOUT, - stdout=subprocess.PIPE, - ) - try: - time.sleep(1) - yield "localhost", 2121, "user", "pass" - finally: - P.terminate() - P.wait() - - -def test_basic(ftp): - host, port, _, _ = ftp - fs = FTPFileSystem(host, port, timeout=1, ssl=True) - assert fs.ls("/", detail=False) == sorted(os.listdir(here)) - out = fs.cat(f"/{os.path.basename(__file__)}") - assert out == open(__file__, "rb").read() - - -def test_basic_prot_p(ftp): - host, port, _, _ = ftp - fs = FTPFileSystem(host, port, ssl=True, prot_p=True) - assert fs.ls("/", detail=False) == sorted(os.listdir(here)) - out = fs.cat(f"/{os.path.basename(__file__)}") - assert out == open(__file__, "rb").read() - - -def test_not_cached(ftp): - host, port, _, _ = ftp - fs = FTPFileSystem(host, port, ssl=True) - fs2 = FTPFileSystem(host, port, ssl=True) - assert fs is not fs2 - - -@pytest.mark.parametrize("cache_type", ["bytes", "mmap"]) -def test_complex(ftp, cache_type): - from fsspec.core import BytesCache - - host, port, user, pw = ftp - files = open_files( - "ftp:///ou*", - host=host, - port=port, - username=user, - password=pw, - block_size=10000, - cache_type=cache_type, - ssl=True, - ) - assert len(files) == 1 - with files[0] as fo: - assert fo.read(10) == b"hellohello" - if isinstance(fo.cache, BytesCache): - assert len(fo.cache.cache) == 10010 - assert fo.read(2) == b"he" - assert fo.tell() == 12 - - -def test_write_small(ftp): - host, port, user, pw = ftp - fs = FTPFileSystem(host, port, user, pw, ssl=True) - with fs.open("/out_tls2", "wb") as f: - f.write(b"oi") - assert fs.cat("/out_tls2") == b"oi" - - -def test_with_url(ftp): - host, port, user, pw = ftp - fo = fsspec.open(f"ftp://{user}:{pw}@{host}:{port}/out_tls", "wb") - with fo as f: - f.write(b"hello") - fo = fsspec.open(f"ftp://{user}:{pw}@{host}:{port}/out_tls", "rb") - with fo as f: - assert f.read() == b"hello" - - -@pytest.mark.parametrize("cache_type", ["bytes", "mmap"]) -def test_write_big(ftp, cache_type): - host, port, user, pw = ftp - fs = FTPFileSystem( - host, port, user, pw, block_size=1000, cache_type=cache_type, ssl=True - ) - fn = f"/bigger_tls_{cache_type}" - with fs.open(fn, "wb") as f: - f.write(b"o" * 500) - assert not fs.exists(fn) - f.write(b"o" * 1000) - fs.invalidate_cache() - assert fs.exists(fn) - f.write(b"o" * 200) - f.flush() - - assert fs.info(fn)["size"] == 1700 - assert fs.cat(fn) == b"o" * 1700 - fs.rm(fn) - - -def test_transaction(ftp): - host, port, user, pw = ftp - fs = FTPFileSystem(host, port, user, pw, ssl=True) - fs.mkdir("tmp_tls") - fn = "tr" - with fs.transaction: - with fs.open(fn, "wb") as f: - f.write(b"not") - assert not fs.exists(fn) - assert fs.exists(fn) - assert fs.cat(fn) == b"not" - - fs.rm(fn) - assert not fs.exists(fn) - - -def test_transaction_with_cache(ftp, tmpdir): - host, port, user, pw = ftp - fs = FTPFileSystem(host, port, user, pw, ssl=True) - fs.mkdirs("tmp_tls", exist_ok=True) - fs.mkdir("tmp_tls/dir") - assert "dir" in fs.ls("tmp_tls", detail=False) - - with fs.transaction: - fs.rmdir("tmp_tls/dir") - - assert "dir" not in fs.ls("tmp_tls", detail=False) - assert not fs.exists("tmp_tls/dir") - - -def test_cat_get(ftp, tmpdir): - host, port, user, pw = ftp - fs = FTPFileSystem(host, port, user, pw, block_size=500, ssl=True) - fs.mkdirs("tmp_tls", exist_ok=True) - data = b"hello" * 500 - fs.pipe("tmp_tls/myfile_tls", data) - assert fs.cat_file("tmp_tls/myfile_tls") == data - - fn = os.path.join(tmpdir, "lfile") - fs.get_file("tmp_tls/myfile_tls", fn) - assert open(fn, "rb").read() == data - - -def test_mkdir(ftp): - host, port, user, pw = ftp - fs = FTPFileSystem(host, port, user, pw, ssl=True) - with pytest.raises(ftplib.error_perm): - fs.mkdir("tmp_tls/not/exist_tls", create_parents=False) - fs.mkdir("tmp_tls/not/exist") - assert fs.exists("tmp_tls/not/exist") - fs.makedirs("tmp_tls/not/exist", exist_ok=True) - with pytest.raises(FileExistsError): - fs.makedirs("tmp_tls/not/exist", exist_ok=False) - fs.makedirs("tmp_tls/not/exist/inner/inner") - assert fs.isdir("tmp_tls/not/exist/inner/inner") - - -def test_rm_get_recursive(ftp, tmpdir): - tmpdir = str(tmpdir) - host, port, user, pw = ftp - fs = FTPFileSystem(host, port, user, pw, ssl=True) - fs.mkdir("tmp_tls/topdir") - fs.mkdir("tmp_tls/topdir/underdir") - fs.touch("tmp_tls/topdir/afile") - fs.touch("tmp_tls/topdir/underdir/afile") - - fs.get("tmp_tls/topdir", tmpdir, recursive=True) - - with pytest.raises(ftplib.error_perm): - fs.rmdir("tmp_tls/topdir") - - fs.rm("tmp_tls/topdir", recursive=True) - assert not fs.exists("tmp_tls/topdir")