From bc8cbb08e90da0618b2fa47e68a3893a6df1d36c Mon Sep 17 00:00:00 2001 From: mxmlnkn Date: Sat, 5 Oct 2024 22:19:54 +0200 Subject: [PATCH] fixup! [wip][feature] Add support for fsspec backends --- .github/workflows/tests.yml | 3 ++- core/ratarmountcore/factory.py | 8 +++++--- tests/.pylintrc | 2 +- tests/runtests.sh | 3 ++- tests/start-asyncssh-server.py | 1 + 5 files changed, 11 insertions(+), 6 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 2cee7697..add4eb7b 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -35,7 +35,8 @@ jobs: - name: Lint With Codespell run: | python3 -m pip install codespell - codespell --ignore-words-list fo,Nd,unx $( git ls-tree -r --name-only HEAD | 'grep' -E '[.](py|md|txt|sh|yml)' ) + # fsspec uses cachable instead of cacheable ... + codespell --ignore-words-list fo,Nd,unx,cachable $( git ls-tree -r --name-only HEAD | 'grep' -E '[.](py|md|txt|sh|yml)' ) - name: Lint With Flake8 run: | diff --git a/core/ratarmountcore/factory.py b/core/ratarmountcore/factory.py index 203c00c5..f11b36ef 100644 --- a/core/ratarmountcore/factory.py +++ b/core/ratarmountcore/factory.py @@ -34,7 +34,7 @@ try: from sshfs import SSHFileSystem except ImportError: - SSHFileSystem= None # type: ignore + SSHFileSystem = None # type: ignore def _openRarMountSource(fileOrPath: Union[str, IO[bytes]], **options) -> Optional[MountSource]: @@ -144,12 +144,12 @@ def openFsspec(url, options, printDebug: int) -> Optional[Union[MountSource, IO[ print("[Info] Try to open with fsspec") # Suppress warning about (default!) encoding not being support for Python<3.9 -.-. - if sys.version_info < (3,9) and protocol == 'ftp': + if sys.version_info < (3, 9) and protocol == 'ftp': with warnings.catch_warnings(): warnings.simplefilter("ignore") openFile = fsspec.open(url) elif protocol in FixedSSHFileSystem.protocols: - fs = FixedSSHFileSystem(**FixedSSHFileSystem._get_kwargs_from_urls(url)) + fs = FixedSSHFileSystem(**FixedSSHFileSystem._get_kwargs_from_urls(url)) # pytype: disable=attribute-error # Remove one leading / in order to add support for relative paths. E.g.: # ssh://127.0.0.1/relative/path @@ -185,10 +185,12 @@ def openFsspec(url, options, printDebug: int) -> Optional[Union[MountSource, IO[ # Avoid resource leaks, e.g., when the seek check fails. oldDel = getattr(result, '__del__', None) + def newDel(): if callable(oldDel): oldDel() result.close() + result.__del__ = newDel # Check that seeking works. May fail when, e.g., the HTTP server does not support range requests. diff --git a/tests/.pylintrc b/tests/.pylintrc index bebaa132..177eaf53 100644 --- a/tests/.pylintrc +++ b/tests/.pylintrc @@ -5,7 +5,7 @@ init-hook='import sys; sys.path.append("./core")' # run arbitrary code. extension-pkg-whitelist=indexed_gzip,indexed_bzip2,indexed_zstd,libarchive,libarchive.ffi,lzmaffi,rapidgzip,isal, PySquashfsImage,PySquashfsImage.compressor,zstandard,lz4,deflate,pyminizip,fast_zip_decryption, - asyncssh,sshfs + asyncssh,sshfs,fsspec # Specify a score threshold to be exceeded before program exits with error. fail-under=10.0 diff --git a/tests/runtests.sh b/tests/runtests.sh index 0acb88e5..45ecc348 100755 --- a/tests/runtests.sh +++ b/tests/runtests.sh @@ -2148,7 +2148,8 @@ if [[ -z "$CI" ]]; then while read -r file; do filesToSpellCheck+=( "$file" ) done < <( git ls-tree -r --name-only HEAD | 'grep' -E '[.](py|md|txt|sh|yml)' ) - codespell --ignore-words-list fo,Nd,unx "${filesToSpellCheck[@]}" + # fsspec uses cachable instead of cacheable ... + codespell --ignore-words-list fo,Nd,unx,cachable "${filesToSpellCheck[@]}" flake8 --config tests/.flake8 "${files[@]}" "${testFiles[@]}" || returnError "$LINENO" 'Flake8 failed!' diff --git a/tests/start-asyncssh-server.py b/tests/start-asyncssh-server.py index 52a98c3a..f3feeafd 100644 --- a/tests/start-asyncssh-server.py +++ b/tests/start-asyncssh-server.py @@ -6,6 +6,7 @@ # for pid in $( ps aux | grep start-asyncssh-server | grep -v grep | awk '{ print $2; }' ); do # kill "$pid"; sleep 0.1; kill -9 "$pid"; done + async def start_server(): await asyncssh.listen( "127.0.0.1",