Add FTP support to jobstores #5142
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
|
There was a problem hiding this comment. This might be better placed in |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,214 @@ | ||
| # Copyright 2017 Oregon Health and Science University | ||
| # | ||
| # Copyright (C) 2015-2021 Regents of the University of California | ||
| # | ||
| # Licensed under the Apache License, Version 2.0 (the "License"); | ||
| # you may not use this file except in compliance with the License. | ||
| # You may obtain a copy of the License at | ||
| # | ||
| # http://www.apache.org/licenses/LICENSE-2.0 | ||
| # | ||
| # Unless required by applicable law or agreed to in writing, software | ||
| # distributed under the License is distributed on an "AS IS" BASIS, | ||
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| # See the License for the specific language governing permissions and | ||
| # limitations under the License. | ||
|
|
||
|
|
||
| import ftplib | ||
| import logging | ||
| import netrc | ||
| import os | ||
| from contextlib import closing | ||
| from typing import Optional, Any, cast, IO | ||
| from urllib.parse import urlparse | ||
| from urllib.request import urlopen | ||
|
|
||
| logger = logging.getLogger(__name__) | ||
|
|
||
|
|
||
| class FtpFsAccess: | ||
| """ | ||
| FTP access with upload. | ||
|
|
||
| Taken and modified from https://github.com/ohsu-comp-bio/cwl-tes/blob/03f0096f9fae8acd527687d3460a726e09190c3a/cwl_tes/ftp.py#L37-L251 | ||
| """ | ||
|
|
||
| def __init__( | ||
| self, cache: Optional[dict[Any, ftplib.FTP]] = None, insecure: bool = False | ||
| ): | ||
| """ | ||
| FTP object to handle FTP connections. By default, connect over FTP with TLS. | ||
|
|
||
| :param cache: cache of generated FTP objects | ||
| :param insecure: Whether to connect over FTP with TLS | ||
| """ | ||
| self.cache = cache or {} | ||
|
There was a problem hiding this comment. We probably want to be able to turn off using a connection cache at all, if we're going to make an FtpFsAccess that is never destroyed. |
||
| self.netrc = None | ||
| self.insecure = insecure | ||
| try: | ||
| if "HOME" in os.environ: | ||
| if os.path.exists(os.path.join(os.environ["HOME"], ".netrc")): | ||
| self.netrc = netrc.netrc(os.path.join(os.environ["HOME"], ".netrc")) | ||
| elif os.path.exists(os.path.join(os.curdir, ".netrc")): | ||
| self.netrc = netrc.netrc(os.path.join(os.curdir, ".netrc")) | ||
| except netrc.NetrcParseError as err: | ||
| logger.debug(err) | ||
|
|
||
| def exists(self, fn: str) -> bool: | ||
| """ | ||
| Check if a file/directory exists over an FTP server | ||
| :param fn: FTP url | ||
| :return: True or false depending on whether the object exists on the server | ||
| """ | ||
| return self.isfile(fn) or self.isdir(fn) | ||
|
|
||
| def isfile(self, fn: str) -> bool: | ||
| """ | ||
| Check if the FTP url points to a file | ||
| :param fn: FTP url | ||
| :return: True if url is file, else false | ||
| """ | ||
| ftp = self._connect(fn) | ||
| if ftp: | ||
| try: | ||
| if not self.size(fn) is None: | ||
| return True | ||
| else: | ||
| return False | ||
| except ftplib.all_errors: | ||
| return False | ||
| return False | ||
|
|
||
| def isdir(self, fn: str) -> bool: | ||
| """ | ||
| Check if the FTP url points to a directory | ||
| :param fn: FTP url | ||
| :return: True if url is directory, else false | ||
| """ | ||
| ftp = self._connect(fn) | ||
| if ftp: | ||
| try: | ||
| cwd = ftp.pwd() | ||
| ftp.cwd(urlparse(fn).path) | ||
| ftp.cwd(cwd) | ||
| return True | ||
| except ftplib.all_errors: | ||
| return False | ||
| return False | ||
|
|
||
| def open(self, fn: str, mode: str) -> IO[bytes]: | ||
| """ | ||
| Open an FTP url. | ||
|
|
||
| Only supports reading, no write support. | ||
| :param fn: FTP url | ||
| :param mode: Mode to open FTP url in | ||
| :return: | ||
| """ | ||
| if "r" in mode: | ||
| host, user, passwd, path = self._parse_url(fn) | ||
| handle = urlopen("ftp://{}:{}@{}/{}".format(user, passwd, host, path)) | ||
| return cast(IO[bytes], closing(handle)) | ||
| # TODO: support write mode | ||
| raise Exception("Write mode FTP not implemented") | ||
|
|
||
| def _parse_url( | ||
| self, url: str | ||
| ) -> tuple[Optional[str], Optional[str], Optional[str], str]: | ||
| """ | ||
| Parse an FTP url into hostname, username, password, and path | ||
| :param url: | ||
| :return: hostname, username, password, path | ||
| """ | ||
| parse = urlparse(url) | ||
| user = parse.username | ||
| passwd = parse.password | ||
| host = parse.hostname | ||
| path = parse.path | ||
| if parse.scheme == "ftp": | ||
| if not user and self.netrc: | ||
| if host is not None: | ||
| creds = self.netrc.authenticators(host) | ||
| if creds: | ||
| user, _, passwd = creds | ||
| if not user: | ||
| if host is not None: | ||
| user, passwd = self._recall_credentials(host) | ||
| if passwd is None: | ||
| passwd = "anonymous@" | ||
| if user is None: | ||
| user = "anonymous" | ||
|
|
||
| return host, user, passwd, path | ||
|
|
||
| def _connect(self, url: str) -> Optional[ftplib.FTP]: | ||
| """ | ||
| Connect to an FTP server. Handles authentication. | ||
| :param url: FTP url | ||
| :return: FTP object | ||
| """ | ||
| parse = urlparse(url) | ||
| if parse.scheme == "ftp": | ||
| host, user, passwd, _ = self._parse_url(url) | ||
| if host is None: | ||
| # there has to be a host | ||
| return None | ||
| if (host, user, passwd) in self.cache: | ||
| if self.cache[(host, user, passwd)].pwd(): | ||
| return self.cache[(host, user, passwd)] | ||
| ftp = ftplib.FTP_TLS() | ||
| ftp.set_debuglevel(1 if logger.isEnabledFor(logging.DEBUG) else 0) | ||
| ftp.connect(host) | ||
| env_user = os.getenv("TOIL_FTP_USER") | ||
| env_passwd = os.getenv("TOIL_FTP_PASSWORD") | ||
| if env_user: | ||
| user = env_user | ||
| if env_passwd: | ||
| passwd = env_passwd | ||
| ftp.login(user or "", passwd or "", secure=not self.insecure) | ||
|
There was a problem hiding this comment. OK, so according to the obviously reliable https://tedboy.github.io/python_stdlib/_modules/ftplib.html#FTP_TLS.login what So it looks like unless the user sets One server people might want to use is Another thing we don't do is call the So I think that by default we should allow unencrypted FTP connections without user intervention to accept them. That's the current behavior. If we want to change Toil to enforce secure FTP by default, then we also need to enforce security on the actual reads. And we probably would need a way to individually allow particular insecure servers? |
||
| self.cache[(host, user, passwd)] = ftp | ||
| return ftp | ||
| return None | ||
|
|
||
| def _recall_credentials( | ||
| self, desired_host: str | ||
| ) -> tuple[Optional[str], Optional[str]]: | ||
| """ | ||
| Grab the cached credentials | ||
| :param desired_host: FTP hostname | ||
| :return: username, password | ||
| """ | ||
| for host, user, passwd in self.cache: | ||
| if desired_host == host: | ||
| return user, passwd | ||
| return None, None | ||
|
|
||
| def size(self, fn: str) -> Optional[int]: | ||
| """ | ||
| Get the size of an FTP object | ||
| :param fn: FTP url | ||
| :return: Size of object | ||
| """ | ||
| ftp = self._connect(fn) | ||
| if ftp: | ||
| host, user, passwd, path = self._parse_url(fn) | ||
| try: | ||
| return ftp.size(path) | ||
| except ftplib.all_errors as e: | ||
| if str(e) == "550 SIZE not allowed in ASCII mode": | ||
| # some servers don't allow grabbing size in ascii mode | ||
| # https://stackoverflow.com/questions/22090001/get-folder-size-using-ftplib/22093848#22093848 | ||
| ftp.voidcmd("TYPE I") | ||
| return ftp.size(path) | ||
| if host is None: | ||
| # no host | ||
| return None | ||
| handle = urlopen("ftp://{}:{}@{}/{}".format(user, passwd, host, path)) | ||
| info = handle.info() | ||
| handle.close() | ||
| if "Content-length" in info: | ||
| return int(info["Content-length"]) | ||
| return None | ||
|
|
||
| return None | ||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
If we keep a class-level
FtpFsAccessinstance that is never destroyed, and theFtpFsAccessinstance caches open FTP connections to servers, when do we close our FTP connections? It might be never, and that's probably not what we want.