From b6ba7b51fd98bf61f3a3ff8df8fd488384b7527d Mon Sep 17 00:00:00 2001 From: Adrian Damian Date: Tue, 26 Mar 2024 16:58:52 -0700 Subject: [PATCH 1/5] Initial version --- README.rst | 11 +-- vos/vos/commands/vcp.py | 4 +- vos/vos/vos.py | 176 ++++++++++++++++------------------------ 3 files changed, 74 insertions(+), 117 deletions(-) diff --git a/README.rst b/README.rst index b0f3eab6..6e22edaf 100644 --- a/README.rst +++ b/README.rst @@ -30,7 +30,6 @@ Note: might need to escape chars in your shell :: cd vos && pip install -e .[test] - cd vofs && pip install -e .[test] Testing packages ---------------- @@ -43,14 +42,6 @@ Testing vos cd ./vos pytest vos -Testing vofs -~~~~~~~~~~~~ - -:: - - cd ./vofs - pytest vofs - Checkstyle @@ -60,7 +51,7 @@ not report errors :: - flake8 vos/vos vofs/vofs + flake8 vos/vos Testing with tox diff --git a/vos/vos/commands/vcp.py b/vos/vos/commands/vcp.py index 40b352dd..7408219f 100755 --- a/vos/vos/commands/vcp.py +++ b/vos/vos/commands/vcp.py @@ -140,7 +140,7 @@ class Nonlocal(): help="DEPRECATED") parser.add_argument( "--quick", action="store_true", - help="assuming CANFAR VOSpace, only comptible with CANFAR VOSpace.", + help="DEPRECATED", default=False) parser.add_argument( "-L", "--follow-links", @@ -169,7 +169,7 @@ class Nonlocal(): client = vos.Client( vospace_certfile=args.certfile, vospace_token=args.token, - transfer_shortcut=args.quick, insecure=args.insecure) + insecure=args.insecure) if not client.is_remote_file(dest): dest = os.path.abspath(dest) diff --git a/vos/vos/vos.py b/vos/vos/vos.py index 183ff349..4efd2ab8 100644 --- a/vos/vos/vos.py +++ b/vos/vos/vos.py @@ -1321,10 +1321,13 @@ def write(buf): class EndPoints(object): VOSPACE_WEBSERVICE = os.getenv('VOSPACE_WEBSERVICE', None) + VO_FILES_OLD = 'ivo://ivoa.net/std/VOSpace/v2.x#files' + VO_PROPERTIES_OLD = 'vos://cadc.nrc.ca~vospace/CADC/std/VOSpace#nodeprops' + # standard ids # TODO - VO_PROPERTIES has been replaced by VO_RECURSIVE_PROPS - VO_PROPERTIES = 'vos://cadc.nrc.ca~vospace/CADC/std/VOSpace#nodeprops' VO_NODES = 'ivo://ivoa.net/std/VOSpace/v2.0#nodes' + VO_FILES = 'ivo://ivoa.net/std/VOSpace#files-proto' VO_TRANSFER = 'ivo://ivoa.net/std/VOSpace#sync-2.1' VO_ASYNC_TRANSFER = 'ivo://ivoa.net/std/VOSpace/v2.0#transfers' VO_RECURSIVE_DEL = 'ivo://ivoa.net/std/VOSpace#recursive-delete-proto' @@ -1391,6 +1394,17 @@ def nodes(self): """ return self.conn.ws_client._get_url((self.VO_NODES, None)) + @property + def files(self): + """ + :return: The files service endpoint. + """ + try: + return self.conn.ws_client._get_url((self.VO_FILES, None)) + except KeyError: + # TODO Temporary + return self.conn.ws_client._get_url((self.VO_FILES_OLD, None)) + @property def recursive_del(self): """ @@ -1453,7 +1467,7 @@ class Client(object): def __init__(self, vospace_certfile=None, root_node=None, conn=None, - transfer_shortcut=False, http_debug=False, + transfer_shortcut=None, http_debug=False, secure_get=True, vospace_token=None, insecure=False): """This could/should be expanded to set various defaults :param vospace_certfile: x509 proxy certificate file location. The @@ -1469,9 +1483,7 @@ def __init__(self, vospace_certfile=None, :param root_node: the base of the VOSpace for uri references. :type root_node: unicode :param conn: DEPRECATED - :param transfer_shortcut: if True then just assumed data web service - urls - :type transfer_shortcut: bool + :param transfer_shortcut: DEPRECATED :param http_debug: turn on http debugging. :type http_debug: bool :param secure_get: Use HTTPS (by default): ie. transfer contents of @@ -1503,7 +1515,6 @@ def __init__(self, vospace_certfile=None, self.protocols = Client.VO_TRANSFER_PROTOCOLS self.rootNode = root_node # self.nodeCache = NodeCache() - self.transfer_shortcut = transfer_shortcut self.secure_get = secure_get self._endpoints = {} self.vospace_certfile = vospace_certfile is None and \ @@ -1819,11 +1830,15 @@ def copy(self, source, destination, send_md5=False, disposition=False, return dest_md5 return dest_size - get_urls = self.get_node_url(source, method='GET', cutout=cutout, - view=view) + files_url = self.get_node_url(source, method='GET', cutout=cutout, + view=view) + get_urls = [] + if files_url: + get_urls.append(files_url) + while not success: if len(get_urls) == 0: - if self.transfer_shortcut and not get_node_url_retried: + if not get_node_url_retried: get_urls = self.get_node_url(source, method='GET', cutout=cutout, view=view, full_negotiation=True) @@ -1832,7 +1847,13 @@ def copy(self, source, destination, send_md5=False, disposition=False, get_node_url_retried = True if len(get_urls) == 0: break - get_url = get_urls.pop(0) + try: + self.get_endpoints(source).recursive_props + get_url = self._add_soda_ops(get_urls.pop(0), view, cutout) + except KeyError: + # TODO - DELETE regression + get_url = self._add_soda_ops(get_urls.pop(0), view, cutout) + try: response = self.get_session(source).get( get_url, stream=True) @@ -1958,8 +1979,7 @@ def copy(self, source, destination, send_md5=False, disposition=False, md5_checksum=dest_md5) while not success: if len(put_urls) == 0: - if self.transfer_shortcut and not \ - get_node_url_retried: + if not get_node_url_retried: put_urls = self.get_node_url( destination, method='PUT', full_negotiation=True) @@ -2005,8 +2025,7 @@ def copy(self, source, destination, send_md5=False, disposition=False, put_urls = self.get_node_url(destination, 'PUT') while not success: if len(put_urls) == 0: - if self.transfer_shortcut and not \ - get_node_url_retried: + if not get_node_url_retried: put_urls = self.get_node_url( destination, method='PUT', full_negotiation=True) @@ -2262,17 +2281,10 @@ def get_node_url(self, uri, method='GET', view=None, limit=None, endpoints = self.get_endpoints(uri) - # full_negotiation is an override, so it can be used to force either - # shortcut (false) or full negotiation (true) - if full_negotiation is not None: - do_shortcut = not full_negotiation - else: - do_shortcut = self.transfer_shortcut + if not full_negotiation and method == 'GET' and view in ['data', 'cutout', 'header']: + return self._get(uri) - if not do_shortcut and method == 'GET' and view in ['data', 'cutout', 'header']: - return self._get(uri, view=view, cutout=cutout) - - if not do_shortcut and method == 'PUT': + if not full_negotiation and method == 'PUT': return self._put(uri, content_length=content_length, md5_checksum=md5_checksum) @@ -2309,64 +2321,9 @@ def get_node_url(self, uri, method='GET', view=None, limit=None, # parameters sent as arguments. direction = {'GET': 'pullFromVoSpace', 'PUT': 'pushToVoSpace'} - - # Future expansion: can use self.secure_get to filter the protocols - # sent to the server. - protocol = { - 'GET': {'https': ((self.secure_get and Client.VO_HTTPSGET_PROTOCOL) - or Client.VO_HTTPGET_PROTOCOL), - 'http': Client.VO_HTTPGET_PROTOCOL}, - 'PUT': {'https': Client.VO_HTTPSPUT_PROTOCOL, - 'http': Client.VO_HTTPPUT_PROTOCOL}} - - # build the url for that will request the url that provides access to - # the node. - - protocol_list = [] - for p in self.protocols: - protocol_list.append(protocol[method][p]) - - url = endpoints.transfer - args = { - 'TARGET': uri, - 'DIRECTION': direction[method], - 'PROTOCOL': protocol_list, - 'view': view} - - if cutout is not None: - args['cutout'] = cutout - headers = {"Content-type": "application/x-www-form-urlencoded", - "Accept": "text/plain"} - - response = self.get_session(uri).get( - endpoints.transfer, params=args, headers=headers, - allow_redirects=False) - logging.debug("Transfer Server said: {0}".format(response.content)) - - if response.status_code == 303: - # Normal case is a redirect - url = response.headers.get('Location', None) - elif response.status_code == 404: - # The file doesn't exist - raise OSError(errno.ENOENT, response.content, url) - elif response.status_code == 409: - raise OSError(errno.EREMOTE, response.content, url) - elif response.status_code == 413: - raise OSError(errno.E2BIG, response.content, url) - else: - logger.debug("Reverting to full negotiation") - return self.get_node_url(uri, - method=method, - view=view, - full_negotiation=True, - limit=limit, - next_uri=next_uri, - cutout=cutout, - sort=sort, - order=order) - - logger.debug("Sending short cut url: {0}".format(url)) - return [url] + urls = self.transfer(self.get_endpoints(uri).transfer, uri, direction[method], None, None) + logger.debug('Transfer URLs: ' + ', '.join(urls)) + return urls def link(self, src_uri, link_uri): """Make link_uri point to src_uri. @@ -2426,31 +2383,40 @@ def move(self, src_uri, destination_uri): headers={'Content-type': 'application/x-www-form-urlencoded'}) return self.get_transfer_error(job_url, src_uri) - def _get(self, uri, view=None, cutout=None): + def _get(self, uri): with nodeCache.volatile(uri): - urls = self.transfer(self.get_endpoints(uri).transfer, - uri, "pullFromVoSpace", None, None) - # assume that the returned urls point to SODA services - if view == 'header': - head_urls = [] - for url in urls: - head_urls.append('{}?META=true'.format(url)) - urls = head_urls - elif cutout: - cutout_urls = [] - for url in urls: - if cutout.strip().startswith('['): - # pixel cutout - cutout_urls.append('{}?SUB={}'.format(url, cutout)) - elif cutout.strip().startswith('CIRCLE'): - # circle cutout - cutout_urls.append('{}?{}'.format(url, cutout)) - else: - # TODO add support for other SODA cutouts SUB, POL etc - raise ValueError('Unknown cutout type: ' + cutout) - urls = cutout_urls + files_ep = self.get_endpoints(uri).files + if not files_ep: + return None + file_path = urlparse(uri).path + if not file_path: + return None + files_url = '{}{}'.format(files_ep, file_path) + try: + response = self.get_session(uri).get(files_url, allow_redirects=False) + response.raise_for_status() + except Exception: + return None + if response.status_code == 303: + return response.headers.get('Location', None) + return None - return urls + def _add_soda_ops(self, url, view=None, cutout=None): + # assume that the url points to a SODA service + result = url + if view == 'header': + result = '{}?META=true'.format(result) + elif cutout: + if cutout.strip().startswith('['): + # pixel cutout + result = '{}?SUB={}'.format(result, cutout) + elif cutout.strip().startswith('CIRCLE'): + # circle cutout + result = '{}?{}'.format(result, cutout) + else: + # TODO add support for other SODA cutouts SUB, POL etc + raise ValueError('Unknown cutout type: ' + cutout) + return result def _put(self, uri, content_length=None, md5_checksum=None): with nodeCache.volatile(uri): From 6131ffb669ecf657ef52f6086eb879b816a8d39f Mon Sep 17 00:00:00 2001 From: Adrian Damian Date: Wed, 27 Mar 2024 16:39:26 -0700 Subject: [PATCH 2/5] Added support for off line and legacy cutouts --- vos/test/scripts/vospace-link-atest.tcsh | 14 +--- vos/test/scripts/vospace-lock-atest.tcsh | 8 +-- vos/test/scripts/vospace-node-properties.tcsh | 29 ++++---- .../scripts_old/vospace-client-atest.tcsh | 13 ++-- vos/test/scripts_old/vospace-link-atest.tcsh | 12 ---- vos/test/scripts_old/vospace-lock-atest.tcsh | 1 + .../scripts_old/vospace-node-properties.tcsh | 1 + vos/vos/vos.py | 66 +++++++++++++------ 8 files changed, 72 insertions(+), 72 deletions(-) diff --git a/vos/test/scripts/vospace-link-atest.tcsh b/vos/test/scripts/vospace-link-atest.tcsh index b2021e89..7cebd96d 100755 --- a/vos/test/scripts/vospace-link-atest.tcsh +++ b/vos/test/scripts/vospace-link-atest.tcsh @@ -141,20 +141,8 @@ foreach resource ($resources) echo " [OK]" endif - echo -n "copy external link" - if ( ${?TESTING_CAVERN} ) then - echo " [SKIPPED, vos/issues/83]" - else - rm -f /tmp/e3link - $CPCMD $CERT -L $CONTAINER/e3link /tmp/ >& /dev/null || echo " [FAIL]" && exit -1 - grep -q google /tmp/e3link || echo " [FAIL]" && exit -1 - rm -f /tmp/e3link - echo " [OK]" - endif - - echo -n "Follow the invalid link and fail" - $CPCMD $CERT $CONTAINER/e3link/somefile /tmp >& /dev/null && echo " [FAIL]" && exit -1 + $CPCMD $CERT $CONTAINER/e3link/somefile /tmp >& /dev/null && echo " [FAIL]" && exit - echo " [OK]" echo -n "copy file to target through link" diff --git a/vos/test/scripts/vospace-lock-atest.tcsh b/vos/test/scripts/vospace-lock-atest.tcsh index 0f76fe1a..04f079cf 100755 --- a/vos/test/scripts/vospace-lock-atest.tcsh +++ b/vos/test/scripts/vospace-lock-atest.tcsh @@ -121,12 +121,10 @@ foreach resource ($resources) if ( ${?TESTING_CAVERN} ) then echo " [SKIPPED, vos/issues/82]" else - echo "$LOCKCMD $CERT $CONTAINER $VUNLOCK_ARGS" $LOCKCMD $CERT $CONTAINER $VUNLOCK_ARGS > /dev/null || echo " [FAIL]" && exit -1 echo " [OK]" echo -n "check unlocked container " - echo "$TAGCMD $CERT $CONTAINER $LIST_ARGS" - $TAGCMD $CERT $CONTAINER $LIST_ARGS | grep -q false && set SUCCESS = "true" + $TAGCMD $CERT $CONTAINER $LIST_ARGS | grep -q None && set SUCCESS = "true" if ( ${SUCCESS} == "true" ) then set SUCCESS = "false" @@ -175,7 +173,7 @@ foreach resource ($resources) $LOCKCMD $CERT $CONTAINER/target $VUNLOCK_ARGS> /dev/null || echo " [FAIL]" && exit -1 echo " [OK]" echo -n "check unlocked link " - $TAGCMD $CERT $CONTAINER/target $LIST_ARGS | grep -q false && set SUCCESS = "true" + $TAGCMD $CERT $CONTAINER/target $LIST_ARGS | grep -q None && set SUCCESS = "true" if ( ${SUCCESS} == "true" ) then set SUCCESS = "false" @@ -226,7 +224,7 @@ foreach resource ($resources) $LOCKCMD $CERT $CONTAINER/something.png $VUNLOCK_ARGS> /dev/null || echo " [FAIL]" && exit -1 echo " [OK]" echo -n "check unlocked node " - $TAGCMD $CERT $CONTAINER/something.png $LIST_ARGS | grep -q false && set SUCCESS = "true" + $TAGCMD $CERT $CONTAINER/something.png $LIST_ARGS | grep -q None && set SUCCESS = "true" if ( ${SUCCESS} == "true" ) then set SUCCESS = "false" echo " [OK]" diff --git a/vos/test/scripts/vospace-node-properties.tcsh b/vos/test/scripts/vospace-node-properties.tcsh index 1ff65a5d..533b8060 100755 --- a/vos/test/scripts/vospace-node-properties.tcsh +++ b/vos/test/scripts/vospace-node-properties.tcsh @@ -58,7 +58,7 @@ foreach resource ($resources) set TIMESTAMP=`date +%Y-%m-%dT%H-%M-%S` set CONTAINER = $BASE/$TIMESTAMP - echo -n "** checking base URI" +# echo -n "** checking base URI" # $RMCMD -R $CERT $BASE > /dev/null # echo -n ", creating base URI" # $MKDIRCMD $CERT $BASE || echo " [FAIL]" && exit -1 @@ -107,24 +107,23 @@ foreach resource ($resources) $LSCMD $CERT $CONTAINER | grep ccc | grep -q "drw-------" || echo " [FAIL]" && exit -1 echo " [OK]" -# echo -n "test vchmod with recursive option" -# -# $CHMODCMD $CERT -R g+r $CONTAINER $GROUP || echo " [FAIL]" && exit -1 -# echo -n " verify " -# $LSCMD $CERT $BASE | grep $TIMESTAMP | grep $GROUP | grep -q "drw-r-----" || echo " [FAIL]" && exit -1 -# $LSCMD $CERT $CONTAINER | grep aaa | grep $GROUP | grep -q "drw-r-----" || echo " [FAIL]" && exit -1 -# $LSCMD $CERT $CONTAINER | grep ccc | grep $GROUP | grep -q "drw-r-----" || echo " [FAIL]" && exit -1 -# $LSCMD $CERT $CONTAINER/aaa | grep bbb | grep $GROUP | grep -q "drw-r-----" || echo " [FAIL]" && exit -1 -# echo " [OK]" + echo -n "test vchmod with recursive option" + + $CHMODCMD $CERT -R g+r $CONTAINER $GROUP || echo " [FAIL]" && exit -1 + echo -n " verify " + $LSCMD $CERT $BASE | grep $TIMESTAMP | grep $GROUP | grep -q "drw-r-----" || echo " [FAIL]" && exit -1 + $LSCMD $CERT $CONTAINER | grep aaa | grep $GROUP | grep -q "drw-r-----" || echo " [FAIL]" && exit -1 + $LSCMD $CERT $CONTAINER | grep ccc | grep $GROUP | grep -q "drw-r-----" || echo " [FAIL]" && exit -1 + $LSCMD $CERT $CONTAINER/aaa | grep bbb | grep $GROUP | grep -q "drw-r-----" || echo " [FAIL]" && exit -1 + echo " [OK]" echo -n "test vchmod with multiple groups" set MULTIGROUP = "ABC $GROUP" - $CHMODCMD $CERT g+r $CONTAINER/aaa "$MULTIGROUP" || echo " [FAIL]" && exit -1 + $CHMODCMD $CERT g+r $CONTAINER/aaa "$MULTIGROUP" || echo " [FAIL1]" && exit -1 echo -n " verify " - $LSCMD $CERT $BASE | grep $TIMESTAMP | grep $GROUP | grep -q "drw-r-----" || echo " [FAIL]" && exit -1 - echo "" - $LSCMD $CERT $CONTAINER | grep aaa | grep "$MULTIGROUP" | grep -q "drw-r-----" || echo " [FAIL]" && exit -1 + $LSCMD $CERT $BASE | grep $TIMESTAMP | grep $GROUP | grep -q "drw-r-----" || echo " [FAIL2]" && exit -1 + $LSCMD $CERT $CONTAINER | grep aaa | grep "$MULTIGROUP" | grep -q "drw-r-----" || echo " [FAIL3]" && exit -1 echo " [OK]" echo -n "make a sub-container public" @@ -134,7 +133,7 @@ foreach resource ($resources) $LSCMD $CERT $BASE | grep $TIMESTAMP | grep $GROUP | grep -q "drw-r-----" || echo " [FAIL1]" && exit -1 $LSCMD $CERT $CONTAINER | grep aaa | grep "$MULTIGROUP" | grep -q "drw-r-----" || echo " [FAIL2]" && exit -1 $LSCMD $CERT $CONTAINER | grep ccc | grep $GROUP | grep -q "drw-r-----" || echo " [FAIL3]" && exit -1 - $LSCMD $CERT $CONTAINER/aaa | grep bbb | grep "$MULTIGROUP" | grep -q "drw-r--r--" || echo " [FAIL4]" && exit -1 + $LSCMD $CERT $CONTAINER/aaa | grep bbb | grep "$GROUP" | grep -q "drw-r--r--" || echo " [FAIL4]" && exit -1 echo " [OK]" # echo -n "recursively make all directories public" diff --git a/vos/test/scripts_old/vospace-client-atest.tcsh b/vos/test/scripts_old/vospace-client-atest.tcsh index 7902374c..b28eb70d 100755 --- a/vos/test/scripts_old/vospace-client-atest.tcsh +++ b/vos/test/scripts_old/vospace-client-atest.tcsh @@ -78,9 +78,9 @@ foreach resource ($resources) echo " [OK]" endif echo -n "** setting home and base to public, no groups" - $CHMODCMD $CERT o+r $VOHOME || echo " [FAIL]" && exit -1 + $CHMODCMD $CERT o+r $VOHOME || echo " [FAIL1]" && exit -1 echo -n " [OK]" - $CHMODCMD $CERT o+r $BASE || echo " [FAIL]" && exit -1 + $CHMODCMD $CERT o+r $BASE || echo " [FAIL2]" && exit -1 echo " [OK]" echo echo "*** starting test sequence ***" @@ -187,14 +187,15 @@ foreach resource ($resources) echo " [OK]" echo -n "copy data node to local filesystem " - $CPCMD $CERT $CONTAINER/something.png $THIS_DIR/something.png.2 || echo " [FAIL]" && exit -1 - cmp $THIS_DIR/something.png $THIS_DIR/something.png.2 || echo " [FAIL]" && exit -1 + + $CPCMD $CERT $CONTAINER/something.png $THIS_DIR/something.png.2 || echo " [FAIL1]" && exit -1 + cmp $THIS_DIR/something.png $THIS_DIR/something.png.2 || echo " [FAIL2]" && exit -1 \rm -f $THIS_DIR/something.png.2 echo " [OK]" echo -n "Check quick copy" - $CPCMD $CERT --quick $THIS_DIR/something.png $CONTAINER/something.png.3|| echo " [FAIL]" && exit -1 - $CPCMD $CERT --quick $CONTAINER/something.png.3 $THIS_DIR/something.png.3 || echo " [FAIL]" && exit -1 + $CPCMD $CERT --quick $THIS_DIR/something.png $CONTAINER/something.png.3|| echo " [FAIL1]" && exit -1 + $CPCMD $CERT --quick $CONTAINER/something.png.3 $THIS_DIR/something.png.3 || echo " [FAIL2]" && exit -1 cmp $THIS_DIR/something.png $THIS_DIR/something.png.3 || echo " [FAIL]" && exit -1 \rm -f $THIS_DIR/something.png.3 echo " [OK]" diff --git a/vos/test/scripts_old/vospace-link-atest.tcsh b/vos/test/scripts_old/vospace-link-atest.tcsh index f0f79385..6398dcc3 100755 --- a/vos/test/scripts_old/vospace-link-atest.tcsh +++ b/vos/test/scripts_old/vospace-link-atest.tcsh @@ -146,18 +146,6 @@ foreach resource ($resources) echo " [OK]" endif - echo -n "copy external link" - if ( ${?TESTING_CAVERN} ) then - echo " [SKIPPED, vos/issues/83]" - else - rm -f /tmp/e3link - $CPCMD $CERT -L $CONTAINER/e3link /tmp/ >& /dev/null || echo " [FAIL]" && exit -1 - grep -q google /tmp/e3link || echo " [FAIL]" && exit -1 - rm -f /tmp/e3link - echo " [OK]" - endif - - echo -n "Follow the invalid link and fail" $CPCMD $CERT $CONTAINER/e3link/somefile /tmp >& /dev/null && echo " [FAIL]" && exit -1 echo " [OK]" diff --git a/vos/test/scripts_old/vospace-lock-atest.tcsh b/vos/test/scripts_old/vospace-lock-atest.tcsh index 37c0318f..92145d9d 100755 --- a/vos/test/scripts_old/vospace-lock-atest.tcsh +++ b/vos/test/scripts_old/vospace-lock-atest.tcsh @@ -133,6 +133,7 @@ foreach resource ($resources) $LOCKCMD $CERT $CONTAINER $VUNLOCK_ARGS > /dev/null || echo " [FAIL]" && exit -1 echo " [OK]" echo -n "check unlocked container " + echo "$TAGCMD $CERT $CONTAINER $LIST_ARGS" $TAGCMD $CERT $CONTAINER $LIST_ARGS | grep -q None && set SUCCESS = "true" if ( ${SUCCESS} == "true" ) then diff --git a/vos/test/scripts_old/vospace-node-properties.tcsh b/vos/test/scripts_old/vospace-node-properties.tcsh index 0c57de03..09dd8b82 100755 --- a/vos/test/scripts_old/vospace-node-properties.tcsh +++ b/vos/test/scripts_old/vospace-node-properties.tcsh @@ -120,6 +120,7 @@ foreach resource ($resources) echo " [FAIL]" && exit -1 endif else + echo "$CHMODCMD $CERT -R g+r $CONTAINER $GROUP1" $CHMODCMD $CERT -R g+r $CONTAINER $GROUP1 || echo " [FAIL]" && exit -1 echo -n " verify " $LSCMD $CERT $BASE | grep $TIMESTAMP | grep $GROUP1 | grep -q "drw-r-----" || echo " [FAIL]" && exit -1 diff --git a/vos/vos/vos.py b/vos/vos/vos.py index 4efd2ab8..8383bb34 100644 --- a/vos/vos/vos.py +++ b/vos/vos/vos.py @@ -79,6 +79,7 @@ import fnmatch from enum import Enum import hashlib +from urllib.parse import quote try: from cStringIO import StringIO @@ -1417,7 +1418,11 @@ def recursive_props(self): """ :return: recusive property set endpoint """ - return self.conn.ws_client._get_url((self.VO_RECURSIVE_PROPS, None)) + try: + return self.conn.ws_client._get_url((self.VO_RECURSIVE_PROPS, None)) + except KeyError: + # TODO temporary for regression + return self.conn.ws_client._get_url((self.VO_PROPERTIES_OLD, None)) @property def session(self): @@ -1781,11 +1786,10 @@ def copy(self, source, destination, send_md5=False, disposition=False, if cutout_match.group('pix'): cutout = cutout_match.group('pix') elif cutout_match.group('wcs') is not None: - from urllib.parse import quote - cutout = 'CIRCLE=' + quote('{} {} {}'.format( + cutout = 'CIRCLE=' + '{} {} {}'.format( cutout_match.group('ra'), cutout_match.group('dec'), - cutout_match.group('rad'))) + cutout_match.group('rad')) else: raise ValueError("Bad source name: {}".format(source)) source = cutout_match.group('filename') @@ -1830,11 +1834,20 @@ def copy(self, source, destination, send_md5=False, disposition=False, return dest_md5 return dest_size - files_url = self.get_node_url(source, method='GET', cutout=cutout, - view=view) + # TODO - remove. This is temporary for regression + try: + self.get_endpoints(source).recursive_props + new_vos = True + except KeyError: + # TODO - to delete temporary regression code + new_vos = False + get_urls = [] - if files_url: - get_urls.append(files_url) + if new_vos: + files_url = self.get_node_url(source, method='GET', cutout=cutout, + view=view) + if files_url: + get_urls.append(files_url) while not success: if len(get_urls) == 0: @@ -1842,17 +1855,13 @@ def copy(self, source, destination, send_md5=False, disposition=False, get_urls = self.get_node_url(source, method='GET', cutout=cutout, view=view, full_negotiation=True) - # remove the first one as we already tried that one. - get_urls.pop(0) + # one of the ur get_node_url_retried = True if len(get_urls) == 0: break - try: - self.get_endpoints(source).recursive_props - get_url = self._add_soda_ops(get_urls.pop(0), view, cutout) - except KeyError: - # TODO - DELETE regression - get_url = self._add_soda_ops(get_urls.pop(0), view, cutout) + get_url = get_urls.pop(0) + if new_vos: + get_url = self._add_soda_ops(get_url, view, cutout) try: response = self.get_session(source).get( @@ -2321,9 +2330,21 @@ def get_node_url(self, uri, method='GET', view=None, limit=None, # parameters sent as arguments. direction = {'GET': 'pullFromVoSpace', 'PUT': 'pushToVoSpace'} - urls = self.transfer(self.get_endpoints(uri).transfer, uri, direction[method], None, None) - logger.debug('Transfer URLs: ' + ', '.join(urls)) - return urls + try: + self.get_endpoints(uri).recursive_del + urls = self.transfer(self.get_endpoints(uri).transfer, uri, direction[method], None, None) + logger.debug('Transfer URLs: ' + ', '.join(urls)) + return urls + except KeyError: + # TODO - delete temporary code for regression + old_cutout = cutout + if old_cutout: + old_cutout = old_cutout.replace('CIRCLE=', 'CIRCLE ICRS ') + old_cutout = old_cutout.replace('SUB=', 'CUTOUT ') + urls = self.transfer(self.get_endpoints(uri).transfer, uri, + direction[method], view=view, cutout=old_cutout) + logger.debug('Transfer URLs: ' + ', '.join(urls)) + return urls def link(self, src_uri, link_uri): """Make link_uri point to src_uri. @@ -2399,6 +2420,8 @@ def _get(self, uri): return None if response.status_code == 303: return response.headers.get('Location', None) + elif response.status_code == 200: + return files_url return None def _add_soda_ops(self, url, view=None, cutout=None): @@ -2412,7 +2435,7 @@ def _add_soda_ops(self, url, view=None, cutout=None): result = '{}?SUB={}'.format(result, cutout) elif cutout.strip().startswith('CIRCLE'): # circle cutout - result = '{}?{}'.format(result, cutout) + result = '{}?CIRCLE={}'.format(result, quote(cutout.replace('CIRCLE=', ''))) else: # TODO add support for other SODA cutouts SUB, POL etc raise ValueError('Unknown cutout type: ' + cutout) @@ -2677,12 +2700,13 @@ def update(self, node, recursive=False): try: property_url = endpoints.recursive_props except KeyError as ex: + property_url = endpoints.p logger.debug('recursive props endpoint does not exist: {0}'. format(str(ex))) raise Exception('Operation not supported') logger.debug("prop URL: {0}".format(property_url)) # quickly check target exists - session.get(endpoints.nodes + '/' + urlparse(node.uri).path) + session.get(endpoints.nodes + urlparse(node.uri).path) response = session.post(endpoints.recursive_props, data=str(node), allow_redirects=False, headers={'Content-type': 'text/xml'}) From ea8b0af28a01c19657da7d32603513f883e4ed50 Mon Sep 17 00:00:00 2001 From: Adrian Damian Date: Wed, 27 Mar 2024 19:18:53 -0700 Subject: [PATCH 3/5] Fixed unit test --- vos/vos/tests/test_vos.py | 52 ++++++++++++++++----------------------- vos/vos/vos.py | 23 +++++++++++------ 2 files changed, 37 insertions(+), 38 deletions(-) diff --git a/vos/vos/tests/test_vos.py b/vos/vos/tests/test_vos.py index f52a58f3..c7c8bf44 100644 --- a/vos/vos/tests/test_vos.py +++ b/vos/vos/tests/test_vos.py @@ -122,6 +122,7 @@ def test_get_node_url(): response = Mock(spec=requests.Response) response.status_code = 303 + resource_id = 'ivo://cadc.nrc.ca/vospace' session_mock = Mock(spec=requests.Session, get=Mock(return_value=response)) session_mock.headers = Mock() @@ -143,39 +144,27 @@ def test_get_node_url(): order='desc')).query assert ('order=desc' == unquote(equery)) - # test header view + # test files URL transfer_url = 'https://mystorage.org/minoc/files/abc:VOS/002' - client.transfer = Mock(return_value=[transfer_url]) - expected_url = transfer_url + '?META=true' - assert expected_url == \ + response.headers = {'Location': transfer_url} + mock_session = Mock(spec=requests.Session, get=Mock(return_value=response)) + client.get_session = Mock(return_value=mock_session) + assert transfer_url == \ client.get_node_url('vos://cadc.nrc.ca!vospace/auser', - view='header')[0] + view='header') + + # test fits header + assert transfer_url + "?META=true" == client._add_soda_ops(transfer_url, view='header') # test pixel cutouts - transfer_url1 = 'https://mystorage.org/minoc/files/abc:VOS/001' - transfer_url2 = 'https://myotherstorage.org/minoc/files/abc:VOS/001' - client.transfer = Mock(return_value=[transfer_url1, transfer_url2]) pcutout = '[1][100:125,100:175]' - expected_url1 = transfer_url1 + '?SUB=' + pcutout - expected_url2 = transfer_url2 + '?SUB=' + pcutout - assert expected_url1 == \ - client.get_node_url('vos://cadc.nrc.ca!vospace/auser', - cutout=pcutout, view='cutout')[0] - assert expected_url2 == \ - client.get_node_url('vos://cadc.nrc.ca!vospace/auser', - cutout=pcutout, view='cutout')[1] + assert transfer_url + "?SUB=" + pcutout == client._add_soda_ops(transfer_url, + cutout=pcutout) # test sky coordinates - client.transfer = Mock(return_value=[transfer_url1, transfer_url2]) - scutout = 'CIRCLE=' + urllib.parse.quote('(1.1 2.2 3.3') - expected_url1 = transfer_url1 + '?' + scutout - expected_url2 = transfer_url2 + '?' + scutout - assert expected_url1 == \ - client.get_node_url('vos://cadc.nrc.ca!vospace/auser', - cutout=scutout, view='cutout')[0] - assert expected_url2 == \ - client.get_node_url('vos://cadc.nrc.ca!vospace/auser', - cutout=scutout, view='cutout')[1] + scutout = '1.1,2.2,3.3' + assert (transfer_url + "?CIRCLE=" + urllib.parse.quote(scutout) == + client._add_soda_ops(transfer_url, cutout='CIRCLE='+scutout)) class TestClient(unittest.TestCase): @@ -444,6 +433,7 @@ def is_remote_file(uri): os.remove(osLocation) # copy from vospace test_client.is_remote_file = is_remote_file + test_client.get_endpoints = Mock() test_client.copy(vospaceLocation, osLocation) get_node_url_mock.assert_called_once_with(vospaceLocation, method='GET', @@ -603,8 +593,8 @@ def is_remote_file(uri): # test GET intermittent exceptions on both URLs props.get.side_effect = md5sum - get_node_url_mock = Mock( - return_value=['http://cadc1.ca/test', 'http://cadc2.ca/test']) + # first side effect corresponds to the files end point call, the second to full negotiation + get_node_url_mock = Mock(side_effect=['http://cadc1.ca/test', ['http://cadc2.ca/test']]) test_client.get_node_url = get_node_url_mock get_node_mock.reset_mock() response.iter_content.return_value = BytesIO(file_content) @@ -613,13 +603,13 @@ def is_remote_file(uri): session.get.side_effect = \ [exceptions.TransferException()] * 2 * vos.MAX_INTERMTTENT_RETRIES with pytest.raises(OSError): - test_client.copy(vospaceLocation, osLocation, head=True) + test_client.copy(vospaceLocation, osLocation, head=False) assert session.get.call_count == 2 * vos.MAX_INTERMTTENT_RETRIES # test GET Transfer error on one URL and a "permanent" one on the other props.get.side_effect = md5sum get_node_url_mock = Mock( - return_value=['http://cadc1.ca/test', 'http://cadc2.ca/test']) + side_effect=[None, ['http://cadc1.ca/test', 'http://cadc2.ca/test']]) test_client.get_node_url = get_node_url_mock get_node_mock.reset_mock() response.iter_content.return_value = BytesIO(file_content) @@ -636,7 +626,7 @@ def is_remote_file(uri): # test GET both "permanent" errors props.get.side_effect = md5sum get_node_url_mock = Mock( - return_value=['http://cadc1.ca/test', 'http://cadc2.ca/test']) + side_effect=['http://cadc1.ca/test', ['http://cadc2.ca/test']]) test_client.get_node_url = get_node_url_mock get_node_mock.reset_mock() response.iter_content.return_value = BytesIO(file_content) diff --git a/vos/vos/vos.py b/vos/vos/vos.py index 8383bb34..82483330 100644 --- a/vos/vos/vos.py +++ b/vos/vos/vos.py @@ -1836,18 +1836,19 @@ def copy(self, source, destination, send_md5=False, disposition=False, # TODO - remove. This is temporary for regression try: - self.get_endpoints(source).recursive_props + self.get_endpoints(source).recursive_del new_vos = True except KeyError: # TODO - to delete temporary regression code new_vos = False get_urls = [] + files_url = None if new_vos: files_url = self.get_node_url(source, method='GET', cutout=cutout, view=view) if files_url: - get_urls.append(files_url) + get_urls.append(self._add_soda_ops(files_url, view, cutout)) while not success: if len(get_urls) == 0: @@ -1855,13 +1856,16 @@ def copy(self, source, destination, send_md5=False, disposition=False, get_urls = self.get_node_url(source, method='GET', cutout=cutout, view=view, full_negotiation=True) - # one of the ur + # remove files_url that we've tried already + if new_vos: + get_urls = [self._add_soda_ops(url, view, cutout) + for url in get_urls if url != files_url] + else: + get_urls = [url for url in get_urls if url != files_url] get_node_url_retried = True if len(get_urls) == 0: break get_url = get_urls.pop(0) - if new_vos: - get_url = self._add_soda_ops(get_url, view, cutout) try: response = self.get_session(source).get( @@ -1994,7 +1998,8 @@ def copy(self, source, destination, send_md5=False, disposition=False, full_negotiation=True) # remove the first one as we already tried # that one. - put_urls.pop(0) + if put_urls: + put_urls.pop(0) get_node_url_retried = True if len(put_urls) == 0: break @@ -2040,7 +2045,8 @@ def copy(self, source, destination, send_md5=False, disposition=False, full_negotiation=True) # remove the first one as we already tried # that one. - put_urls.pop(0) + if put_urls: + put_urls.pop(0) get_node_url_retried = True if len(put_urls) == 0: break @@ -2421,6 +2427,9 @@ def _get(self, uri): if response.status_code == 303: return response.headers.get('Location', None) elif response.status_code == 200: + # TODO this happens for cavern. It is wasteful since the response + # already contains the bytes but there's no other way to find out the + # actual location of the bytes. return files_url return None From 2489f5b18aa48352b75f19bc014ebc0fdaaf8eb3 Mon Sep 17 00:00:00 2001 From: Adrian Damian Date: Wed, 27 Mar 2024 19:21:44 -0700 Subject: [PATCH 4/5] Fixed typo --- vos/vos/vos.py | 1 - 1 file changed, 1 deletion(-) diff --git a/vos/vos/vos.py b/vos/vos/vos.py index 82483330..499e84a1 100644 --- a/vos/vos/vos.py +++ b/vos/vos/vos.py @@ -2709,7 +2709,6 @@ def update(self, node, recursive=False): try: property_url = endpoints.recursive_props except KeyError as ex: - property_url = endpoints.p logger.debug('recursive props endpoint does not exist: {0}'. format(str(ex))) raise Exception('Operation not supported') From 34f45f9899529782198f08638d580599bed2c524 Mon Sep 17 00:00:00 2001 From: Adrian Damian Date: Thu, 28 Mar 2024 12:09:38 -0700 Subject: [PATCH 5/5] Streamlined cavern calls --- vos/vos/tests/test_vos.py | 11 +++++++++++ vos/vos/vos.py | 16 +++++++++++----- 2 files changed, 22 insertions(+), 5 deletions(-) diff --git a/vos/vos/tests/test_vos.py b/vos/vos/tests/test_vos.py index c7c8bf44..56d006ad 100644 --- a/vos/vos/tests/test_vos.py +++ b/vos/vos/tests/test_vos.py @@ -149,6 +149,7 @@ def test_get_node_url(): response.headers = {'Location': transfer_url} mock_session = Mock(spec=requests.Session, get=Mock(return_value=response)) client.get_session = Mock(return_value=mock_session) + client._fs_type = False assert transfer_url == \ client.get_node_url('vos://cadc.nrc.ca!vospace/auser', view='header') @@ -478,11 +479,21 @@ def is_remote_file(uri): response.iter_content.return_value = BytesIO(file_content) session.get.return_value = response test_client.get_session = Mock(return_value=session) + # client must be a vault client + test_client._fs_type = False test_client.copy('{}{}'.format(vospaceLocation, '[1][10:60]'), osLocation) get_node_url_mock.assert_called_once_with( vospaceLocation, method='GET', cutout='[1][10:60]', view='cutout') + # test cavern does not support SODA operations + test_client._fs_type = True + with pytest.raises(ValueError): + test_client.copy('{}{}'.format(vospaceLocation, '[1][10:60]'), osLocation) + with pytest.raises(ValueError): + test_client.copy(vospaceLocation, osLocation, head=True) + + test_client._fs_type = False # copy to vospace when md5 sums are the same -> only update occurs get_node_url_mock.reset_mock() computed_md5_mock.reset_mock() diff --git a/vos/vos/vos.py b/vos/vos/vos.py index 499e84a1..529e0869 100644 --- a/vos/vos/vos.py +++ b/vos/vos/vos.py @@ -1526,6 +1526,7 @@ def __init__(self, vospace_certfile=None, Client.VOSPACE_CERTFILE or vospace_certfile self.vospace_token = vospace_token self.insecure = insecure + self._fs_type = True # True - file system type (cavern), False - db type (vault) def glob(self, pathname): """Return a list of paths matching a pathname pattern. @@ -1690,6 +1691,10 @@ def get_endpoints(self, uri): else: raise OSError('No scheme in {}'.format(uri)) + # following is a CADC hack as others can deploy the services under different + # resource IDs + if 'vault' in resource_id: + self._fs_type = False if resource_id not in self._endpoints: try: self._endpoints[resource_id] = EndPoints( @@ -1844,6 +1849,8 @@ def copy(self, source, destination, send_md5=False, disposition=False, get_urls = [] files_url = None + if self._fs_type and (cutout or view == 'header'): + raise ValueError('cavern/arc service does not support cutouts or header operations') if new_vos: files_url = self.get_node_url(source, method='GET', cutout=cutout, view=view) @@ -2419,6 +2426,10 @@ def _get(self, uri): if not file_path: return None files_url = '{}{}'.format(files_ep, file_path) + if self._fs_type: + # files_url contains the bytes + return files_url + # remaining is for vault try: response = self.get_session(uri).get(files_url, allow_redirects=False) response.raise_for_status() @@ -2426,11 +2437,6 @@ def _get(self, uri): return None if response.status_code == 303: return response.headers.get('Location', None) - elif response.status_code == 200: - # TODO this happens for cavern. It is wasteful since the response - # already contains the bytes but there's no other way to find out the - # actual location of the bytes. - return files_url return None def _add_soda_ops(self, url, view=None, cutout=None):