Skip to content

Commit

Permalink
fixup! [wip][feature] Add support for fsspec backends
Browse files Browse the repository at this point in the history
  • Loading branch information
mxmlnkn committed Oct 5, 2024
1 parent 26edfda commit bc8cbb0
Show file tree
Hide file tree
Showing 5 changed files with 11 additions and 6 deletions.
3 changes: 2 additions & 1 deletion .github/workflows/tests.yml
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,8 @@ jobs:
- name: Lint With Codespell
run: |
python3 -m pip install codespell
codespell --ignore-words-list fo,Nd,unx $( git ls-tree -r --name-only HEAD | 'grep' -E '[.](py|md|txt|sh|yml)' )
# fsspec uses cachable instead of cacheable ...
codespell --ignore-words-list fo,Nd,unx,cachable $( git ls-tree -r --name-only HEAD | 'grep' -E '[.](py|md|txt|sh|yml)' )
- name: Lint With Flake8
run: |
Expand Down
8 changes: 5 additions & 3 deletions core/ratarmountcore/factory.py
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@
try:
from sshfs import SSHFileSystem
except ImportError:
SSHFileSystem= None # type: ignore
SSHFileSystem = None # type: ignore


def _openRarMountSource(fileOrPath: Union[str, IO[bytes]], **options) -> Optional[MountSource]:
Expand Down Expand Up @@ -144,12 +144,12 @@ def openFsspec(url, options, printDebug: int) -> Optional[Union[MountSource, IO[
print("[Info] Try to open with fsspec")

# Suppress warning about (default!) encoding not being support for Python<3.9 -.-.
if sys.version_info < (3,9) and protocol == 'ftp':
if sys.version_info < (3, 9) and protocol == 'ftp':
with warnings.catch_warnings():
warnings.simplefilter("ignore")
openFile = fsspec.open(url)
elif protocol in FixedSSHFileSystem.protocols:
fs = FixedSSHFileSystem(**FixedSSHFileSystem._get_kwargs_from_urls(url))
fs = FixedSSHFileSystem(**FixedSSHFileSystem._get_kwargs_from_urls(url)) # pytype: disable=attribute-error

# Remove one leading / in order to add support for relative paths. E.g.:
# ssh://127.0.0.1/relative/path
Expand Down Expand Up @@ -185,10 +185,12 @@ def openFsspec(url, options, printDebug: int) -> Optional[Union[MountSource, IO[

# Avoid resource leaks, e.g., when the seek check fails.
oldDel = getattr(result, '__del__', None)

def newDel():
if callable(oldDel):
oldDel()
result.close()

result.__del__ = newDel

# Check that seeking works. May fail when, e.g., the HTTP server does not support range requests.
Expand Down
2 changes: 1 addition & 1 deletion tests/.pylintrc
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ init-hook='import sys; sys.path.append("./core")'
# run arbitrary code.
extension-pkg-whitelist=indexed_gzip,indexed_bzip2,indexed_zstd,libarchive,libarchive.ffi,lzmaffi,rapidgzip,isal,
PySquashfsImage,PySquashfsImage.compressor,zstandard,lz4,deflate,pyminizip,fast_zip_decryption,
asyncssh,sshfs
asyncssh,sshfs,fsspec

# Specify a score threshold to be exceeded before program exits with error.
fail-under=10.0
Expand Down
3 changes: 2 additions & 1 deletion tests/runtests.sh
Original file line number Diff line number Diff line change
Expand Up @@ -2148,7 +2148,8 @@ if [[ -z "$CI" ]]; then
while read -r file; do
filesToSpellCheck+=( "$file" )
done < <( git ls-tree -r --name-only HEAD | 'grep' -E '[.](py|md|txt|sh|yml)' )
codespell --ignore-words-list fo,Nd,unx "${filesToSpellCheck[@]}"
# fsspec uses cachable instead of cacheable ...
codespell --ignore-words-list fo,Nd,unx,cachable "${filesToSpellCheck[@]}"

flake8 --config tests/.flake8 "${files[@]}" "${testFiles[@]}" || returnError "$LINENO" 'Flake8 failed!'

Expand Down
1 change: 1 addition & 0 deletions tests/start-asyncssh-server.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
# for pid in $( ps aux | grep start-asyncssh-server | grep -v grep | awk '{ print $2; }' ); do
# kill "$pid"; sleep 0.1; kill -9 "$pid"; done


async def start_server():
await asyncssh.listen(
"127.0.0.1",
Expand Down

0 comments on commit bc8cbb0

Please sign in to comment.