From 6d80dc6c97f3f6854bd9f8bc52e8c18461dd58f2 Mon Sep 17 00:00:00 2001 From: andamian Date: Tue, 8 Jun 2021 16:42:12 -0700 Subject: [PATCH] Discover registry VOSpace services by their short name/scheme (#190) * Discover registry VOSpace services by their short name/scheme --- vos/setup.cfg | 2 +- vos/test/scripts/vospace-link-atest.tcsh | 18 +++- vos/vos/commands/tests/test_vls.py | 15 ++- vos/vos/commands/vcat.py | 8 +- vos/vos/commands/vcp.py | 32 ++++--- vos/vos/commands/vln.py | 12 +-- vos/vos/commands/vls.py | 12 ++- vos/vos/commands/vmv.py | 10 +- vos/vos/commands/vrm.py | 8 +- vos/vos/commands/vrmdir.py | 8 +- vos/vos/commands/vsync.py | 6 +- vos/vos/commonparser.py | 13 +-- vos/vos/tests/test_config.py | 5 - vos/vos/tests/test_vos.py | 36 +++++--- vos/vos/vos.py | 111 ++++++++++++----------- vos/vos/vosconfig.py | 5 +- 16 files changed, 166 insertions(+), 135 deletions(-) diff --git a/vos/setup.cfg b/vos/setup.cfg index 2a79ed168..963ff1d75 100644 --- a/vos/setup.cfg +++ b/vos/setup.cfg @@ -50,7 +50,7 @@ edit_on_github = False github_project = opencadc/vostools install_requires = html2text>=2016.5.29 cadcutils>=1.1.30 future aenum # version should be PEP440 compatible (http://www.python.org/dev/peps/pep-0440) -version = 3.3 +version = 3.3.1 [entry_points] diff --git a/vos/test/scripts/vospace-link-atest.tcsh b/vos/test/scripts/vospace-link-atest.tcsh index e9590bb72..77edacb99 100755 --- a/vos/test/scripts/vospace-link-atest.tcsh +++ b/vos/test/scripts/vospace-link-atest.tcsh @@ -65,7 +65,6 @@ foreach resource ($resources) endif echo -n "** setting home and base to public, no groups" $CHMODCMD $CERT o+r $VOHOME || echo " [FAIL]" && exit -1 - echo -n " [OK]" $CHMODCMD $CERT o+r $BASE || echo " [FAIL]" && exit -1 echo " [OK]" @@ -90,6 +89,17 @@ foreach resource ($resources) $LNCMD $CERT $CONTAINER/target $CONTAINER/clink >& /dev/null || echo " [FAIL]" && exit -1 echo " [OK]" + echo -n "list container" + # content displayed unless -l and no / at the end + # content displayed + $LSCMD $CERT $CONTAINER/clink | grep -q something || echo " [FAIL]" && exit -1 + $LSCMD $CERT $CONTAINER/clink/ | grep -q something || echo " [FAIL]" && exit -1 + $LSCMD $CERT -l $CONTAINER/clink/ | grep -q something || echo " [FAIL]" && exit -1 + # case where the content of the target is not displayed just the link + $LSCMD $CERT -l $CONTAINER/clink | grep 'clink ->' | grep -q target || echo " [FAIL]" && exit -1 + echo " [OK]" + + echo -n "Follow the link to get the file" $CPCMD $CERT $CONTAINER/clink/something.png /tmp || echo " [FAIL]" && exit -1 echo " [OK]" @@ -106,12 +116,12 @@ foreach resource ($resources) $CPCMD $CERT $CONTAINER/dlink /tmp || echo " [FAIL]" && exit -1 echo " [OK]" - echo -n "create link to unknown authority in URI" + echo -n "create link to external vos URI" if ( ${?TESTING_CAVERN} ) then echo " [SKIPPED, vos/issues/83]" else $RMCMD $CERT $CONTAINER/e1link >& /dev/null - $LNCMD $CERT vos://unknown.authority~vospace/unknown $CONTAINER/e1link >& /dev/null || echo " [FAIL]" && exit -1 + $LNCMD $CERT vos://cadc.nrc.ca~arc/unknown $CONTAINER/e1link >& /dev/null || echo " [FAIL]" && exit -1 echo " [OK]" endif @@ -120,7 +130,7 @@ foreach resource ($resources) echo " [OK]" echo -n "create link to unknown scheme in URI" - $LNCMD $CERT unknown://cadc.nrc.ca~vospace/CADCRegtest1 $CONTAINER/e2ink >& /dev/null && echo " [FAIL]" && exit -1 + $LNCMD $CERT unknown://cadc.nrc.ca~vault/CADCRegtest1 $CONTAINER/e2link >& /dev/null && echo " [FAIL]" && exit -1 echo " [OK]" echo -n "Follow the invalid link and fail" diff --git a/vos/vos/commands/tests/test_vls.py b/vos/vos/commands/tests/test_vls.py index c24863405..80522f53a 100644 --- a/vos/vos/commands/tests/test_vls.py +++ b/vos/vos/commands/tests/test_vls.py @@ -97,27 +97,38 @@ def test_vls(self, vos_client_mock): mock_node1.props = {'length': 100, 'date': 50000} mock_node1.isdir.return_value = False mock_node1.get_info.return_value = '' + mock_node1.islink.return_value = False mock_node2 = MagicMock(type='vos:DataNode') mock_node2.name = 'node2' mock_node2.props = {'length': 30, 'date': 70000} mock_node2.isdir.return_value = False mock_node2.get_info.return_value = '' + mock_node2.islink.return_value = False + + mock_node3_link = MagicMock(type='vos:DataNode') + mock_node3_link.name = 'node3_link' + mock_node3_link.props = {'length': 60, 'date': 20000} + mock_node3_link.isdir.return_value = False + mock_node3_link.get_info.return_value = '' + mock_node3_link.islink.return_value = True mock_node3 = MagicMock(type='vos:DataNode') mock_node3.name = 'node3' mock_node3.props = {'length': 60, 'date': 20000} mock_node3.isdir.return_value = False mock_node3.get_info.return_value = '' + mock_node3.islink.return_value = False vos_client_mock.return_value.glob.return_value = \ - ['target2', 'target3', 'target1'] + ['target2', 'target3_link', 'target1'] # vls command with sort == None (i.e. sort by node name), order == None out = 'node1\nnode2\nnode3\n' with patch('sys.stdout', new_callable=StringIO) as stdout_mock: vos_client_mock.return_value.get_node = \ - MagicMock(side_effect=[mock_node2, mock_node3, mock_node1]) + MagicMock(side_effect=[mock_node2, mock_node3_link, mock_node3, + mock_node1]) sys.argv = ['vls', 'vos:/CADCRegtest1'] cmd_attr = getattr(commands, 'vls') cmd_attr() diff --git a/vos/vos/commands/vcat.py b/vos/vos/commands/vcat.py index 98fb9a1e6..5eb0762a9 100755 --- a/vos/vos/commands/vcat.py +++ b/vos/vos/commands/vcat.py @@ -3,7 +3,7 @@ unicode_literals) import sys import logging -from ..vos import Client, is_remote_file +from ..vos import Client from ..commonparser import CommonParser, set_logging_level_from_args, \ exit_on_exception, URI_DESCRIPTION @@ -19,9 +19,9 @@ def _cat(uri, cert_filename=None, head=None): fh = None try: - if is_remote_file(uri): - view = head and 'header' or 'data' - fh = Client(vospace_certfile=cert_filename).open(uri, view=view) + view = head and 'header' or 'data' + fh = Client(vospace_certfile=cert_filename).open(uri, view=view) + if fh.is_remote_file(uri): sys.stdout.write(fh.read(return_response=True).text) sys.stdout.write('\n\n') else: diff --git a/vos/vos/commands/vcp.py b/vos/vos/commands/vcp.py index 1a3a77126..225db99c1 100755 --- a/vos/vos/commands/vcp.py +++ b/vos/vos/commands/vcp.py @@ -102,7 +102,11 @@ class Nonlocal(): if args.overwrite: warnings.warn("the --overwrite option is no longer supported") - if not vos.is_remote_file(dest): + client = vos.Client( + vospace_certfile=args.certfile, vospace_token=args.token, + transfer_shortcut=args.quick) + + if not client.is_remote_file(dest): dest = os.path.abspath(dest) cutout_pattern = re.compile( @@ -138,14 +142,14 @@ def get_node(filename, limit=None): def isdir(filename): logging.debug("Doing an isdir on %s" % filename) - if vos.is_remote_file(filename): + if client.is_remote_file(filename): return client.isdir(filename) else: return os.path.isdir(filename) def islink(filename): logging.debug("Doing an islink on %s" % filename) - if vos.is_remote_file(filename): + if client.is_remote_file(filename): try: return get_node(filename).islink() except exceptions.NotFoundException: @@ -161,7 +165,7 @@ def access(filename, mode): @return: True/False """ logging.debug("checking for access %s " % filename) - if vos.is_remote_file(filename): + if client.is_remote_file(filename): try: node = get_node(filename, limit=0) return node is not None @@ -176,27 +180,27 @@ def listdir(dirname): """Walk through the directory structure a al os.walk""" logging.debug("getting a dirlist %s " % dirname) - if vos.is_remote_file(dirname): + if client.is_remote_file(dirname): return client.listdir(dirname, force=True) else: return os.listdir(dirname) def mkdir(filename): logging.debug("Making directory %s " % filename) - if vos.is_remote_file(filename): + if client.is_remote_file(filename): return client.mkdir(filename) else: return os.mkdir(filename) def get_md5(filename): logging.debug("getting the MD5 for %s" % filename) - if vos.is_remote_file(filename): + if client.is_remote_file(filename): return get_node(filename).props.get('MD5', vos.ZERO_MD5) else: return md5_cache.MD5Cache.compute_md5(filename) def lglob(pathname): - if vos.is_remote_file(pathname): + if client.is_remote_file(pathname): return client.glob(pathname) else: return glob.glob(pathname) @@ -337,7 +341,7 @@ def copy(source_name, destination_name, exclude=None, include=None, try: for source_pattern in args.source: - if args.head and not vos.is_remote_file(source_pattern): + if args.head and not client.is_remote_file(source_pattern): logging.error("head only works for source files in vospace") continue @@ -345,10 +349,7 @@ def copy(source_name, destination_name, exclude=None, include=None, # strings off the end of the pattern before matching. This allows # cutouts on the vos service. The shell does pattern matching for # local files, so don't run glob on local files. - client = vos.Client( - vospace_certfile=args.certfile, vospace_token=args.token, - transfer_shortcut=args.quick) - if not vos.is_remote_file(source_pattern): + if not client.is_remote_file(source_pattern): sources = [source_pattern] else: cutout_match = cutout_pattern.search(source_pattern) @@ -366,7 +367,7 @@ def copy(source_name, destination_name, exclude=None, include=None, # stick back on the cutout pattern if there was one. sources = [s + cutout for s in sources] for source in sources: - if not vos.is_remote_file(source): + if not client.is_remote_file(source): source = os.path.abspath(source) # the source must exist, of course... if not access(source, os.R_OK): @@ -377,7 +378,8 @@ def copy(source_name, destination_name, exclude=None, include=None, continue # copying inside VOSpace not yet implemented - if vos.is_remote_file(source) and vos.is_remote_file(dest): + if client.is_remote_file(source) and \ + client.is_remote_file(dest): raise Exception( "Can not (yet) copy from VOSpace to VOSpace.") diff --git a/vos/vos/commands/vln.py b/vos/vos/commands/vln.py index 5af12f8a0..e8f835628 100755 --- a/vos/vos/commands/vln.py +++ b/vos/vos/commands/vln.py @@ -36,15 +36,15 @@ def vln(): try: opt = parser.parse_args() set_logging_level_from_args(opt) - - if not vos.is_remote_file(opt.source) or \ - not vos.is_remote_file(opt.target): - raise ArgumentError( - None, - "source must be vos node or http url, target must be vos node") client = vos.Client( vospace_certfile=opt.certfile, vospace_token=opt.token) + if not client.is_remote_file(opt.source) or \ + not client.is_remote_file(opt.target): + raise ArgumentError( + None, + "source must be vos node or http url, target must be vos node") + client.link(opt.source, opt.target) except ArgumentError as ex: parser.print_usage() diff --git a/vos/vos/commands/vls.py b/vos/vos/commands/vls.py index 15b5550e7..79da44341 100755 --- a/vos/vos/commands/vls.py +++ b/vos/vos/commands/vls.py @@ -131,18 +131,22 @@ def vls(): order = 'desc' if sort else 'asc' for node in opt.node: - if not vos.is_remote_file(file_name=node): - raise ArgumentError(opt.node, - "Invalid node name: {}".format(node)) - logging.debug("getting listing of: %s" % str(node)) client = vos.Client( vospace_certfile=opt.certfile, vospace_token=opt.token) + if not client.is_remote_file(file_name=node): + raise ArgumentError(opt.node, + "Invalid node name: {}".format(node)) + logging.debug("getting listing of: %s" % str(node)) + targets = client.glob(node) # segregate files from directories for target in targets: target_node = client.get_node(target) + if target.endswith('/') or not opt.long: + while target_node.islink(): + target_node = client.get_node(target_node.target) if target_node.isdir(): dirs.append((_get_sort_key(target_node, sort), target_node, target)) diff --git a/vos/vos/commands/vmv.py b/vos/vos/commands/vmv.py index bfd3ec7c2..fb0bfab08 100755 --- a/vos/vos/commands/vmv.py +++ b/vos/vos/commands/vmv.py @@ -30,16 +30,16 @@ def vmv(): try: source = args.source dest = args.destination - if not vos.is_remote_file(source): + client = vos.Client( + vospace_certfile=args.certfile, + vospace_token=args.token) + if not client.is_remote_file(source): raise ValueError('Source {} is not a remote node'.format(source)) - if not vos.is_remote_file(dest): + if not client.is_remote_file(dest): raise ValueError( 'Destination {} is not a remote node'.format(dest)) if urlparse(source).scheme != urlparse(dest).scheme: raise ValueError('Move between services not supported') - client = vos.Client( - vospace_certfile=args.certfile, - vospace_token=args.token) logging.info("{} -> {}".format(source, dest)) client.move(source, dest) except Exception as ex: diff --git a/vos/vos/commands/vrm.py b/vos/vos/commands/vrm.py index 306fe9e8c..51fe23bbb 100755 --- a/vos/vos/commands/vrm.py +++ b/vos/vos/commands/vrm.py @@ -23,12 +23,12 @@ def vrm(): try: for node in args.node: - if not vos.is_remote_file(node): + client = vos.Client( + vospace_certfile=args.certfile, + vospace_token=args.token) + if not client.is_remote_file(node): raise Exception( '{} is not a valid VOSpace handle'.format(node)) - client = vos.Client( - vospace_certfile=args.certfile, - vospace_token=args.token) if not node.endswith('/'): if client.get_node(node).islink(): logging.info('deleting link {}'.format(node)) diff --git a/vos/vos/commands/vrmdir.py b/vos/vos/commands/vrmdir.py index 2af0ca38a..ff003b7b2 100755 --- a/vos/vos/commands/vrmdir.py +++ b/vos/vos/commands/vrmdir.py @@ -24,12 +24,12 @@ def vrmdir(): try: for container_node in args.nodes: - if not vos.is_remote_file(container_node): + client = vos.Client( + vospace_certfile=args.certfile, + vospace_token=args.token) + if not client.is_remote_file(container_node): raise ValueError( "{} is not a valid VOSpace handle".format(container_node)) - client = vos.Client( - vospace_certfile=args.certfile, - vospace_token=args.token) if client.isdir(container_node): logging.info("deleting {}".format(container_node)) client.delete(container_node) diff --git a/vos/vos/commands/vsync.py b/vos/vos/commands/vsync.py index 5a78a905e..6dbfe462b 100755 --- a/vos/vos/commands/vsync.py +++ b/vos/vos/commands/vsync.py @@ -87,12 +87,12 @@ def signal_handler(h_stream, h_frame): global_md5_cache = md5_cache.MD5Cache(cache_db=opt.cache_filename) destination = opt.destination - if not vos.is_remote_file(destination): + client = vos.Client( + vospace_certfile=opt.certfile, vospace_token=opt.token) + if not client.is_remote_file(destination): parser.error("Only allows sync FROM local copy TO VOSpace") # Currently we don't create nodes in sync and we don't sync onto files logging.info("Connecting to VOSpace") - client = vos.Client( - vospace_certfile=opt.certfile, vospace_token=opt.token) logging.info("Confirming Destination is a directory") dest_is_dir = client.isdir(destination) diff --git a/vos/vos/commonparser.py b/vos/vos/commonparser.py index 60370384f..1fdef24f9 100644 --- a/vos/vos/commonparser.py +++ b/vos/vos/commonparser.py @@ -116,11 +116,8 @@ def __init__(self, *args, **kwargs): URI_DESCRIPTION = \ - 'Remote resources are identified either by their full ' \ - 'URIs (ivo://cadc.nrc.ca/vault) or by a user configured name in the ' \ - 'config file.\n' \ - 'Some names are reserved:\n' \ - ' vos for ivo:/cadc.nrc.ca/vault and\n' \ - ' arc for ivo:cadc.nc.ca/arbutus-cavern.\n' \ - 'Thus, arc:somepath is a shorter version of the full URI ' \ - 'vos://cadc.nrc.ca~arbutus-cavern/somepath' + 'Remote resources are identified either by their full URIs \n' \ + '(vos://cadc.nrc.ca~vault/) or by shorter equivalent URIs with \n' \ + 'the scheme representing the name of the service (vault:). \n' \ + 'Due to historical reasons, the `vos` scheme can be used to refer \n' \ + 'to the `vault` service, ie vos: and vault: are equivalent.\n' diff --git a/vos/vos/tests/test_config.py b/vos/vos/tests/test_config.py index 32e8d121a..eafe6e4fe 100644 --- a/vos/vos/tests/test_config.py +++ b/vos/vos/tests/test_config.py @@ -72,7 +72,6 @@ def test_get_resource_id(): # config file does not exist conf = vosconfig.VosConfig('somedir', 'somedir') assert conf.get_resource_id('vos') == 'ivo://cadc.nrc.ca/vault' - assert conf.get_resource_id('arc') == 'ivo://cadc.nrc.ca/arbutus-cavern' with pytest.raises(ValueError) as e: conf.get_resource_id(None) assert str(e.value) == 'resource name required' @@ -88,7 +87,6 @@ def test_get_resource_id(): open(config_file, 'w').write(config_content) vosconfig.VosConfig(config_file) assert conf.get_resource_id('vos') == 'ivo://cadc.nrc.ca/vault' - assert conf.get_resource_id('arc') == 'ivo://cadc.nrc.ca/arbutus-cavern' # default 1 resource without name should fail config_content = '[vos]\nresourceID = ivo://cadc.nrc.ca/vv\n' @@ -105,7 +103,6 @@ def test_get_resource_id(): open(config_file, 'w').write(config_content) conf = vosconfig.VosConfig(config_file) assert conf.get_resource_id('vos') == 'ivo://cadc.nrc.ca/vault' - assert conf.get_resource_id('arc') == 'ivo://cadc.nrc.ca/arbutus-cavern' assert conf.get_resource_id('cadcvos') == 'ivo://cadc.nrc.ca/vault' # extra resource to the config file with proper name @@ -114,7 +111,6 @@ def test_get_resource_id(): open(config_file, 'w').write(config_content) conf = vosconfig.VosConfig(config_file) assert conf.get_resource_id('vos') == 'ivo://cadc.nrc.ca/vault' - assert conf.get_resource_id('arc') == 'ivo://cadc.nrc.ca/arbutus-cavern' assert conf.get_resource_id('spvo') == 'ivo://some.provider/vo' assert conf.get_resource_id('sopvo') == 'ivo://some.other.provider/vo' @@ -123,7 +119,6 @@ def test_get_resource_id(): open(config_file, 'w').write(config_content) conf = vosconfig.VosConfig(config_file) assert conf.get_resource_id('vos') == 'ivo://cadc.nrc.ca/vault' - assert conf.get_resource_id('arc') == 'ivo://cadc.nrc.ca/arbutus-cavern' assert conf.get_resource_id('vault') == 'ivo://cadc.nrc.ca/vault' # attempt to assign a reserved name (vos or arc) diff --git a/vos/vos/tests/test_vos.py b/vos/vos/tests/test_vos.py index 6dd2d6836..50850d65e 100644 --- a/vos/vos/tests/test_vos.py +++ b/vos/vos/tests/test_vos.py @@ -117,21 +117,24 @@ def test_init_client(self): Client.VOSPACE_CERTFILE = "some-cert-file.pem" with patch('os.access'): client = Client(vospace_certfile=certfile) - client.get_session(uri='ivo://cadc.nrc.ca/vault') + client.is_remote_file = Mock() + client.get_session(uri='vos://cadc.nrc.ca~vault') conn = client._endpoints['ivo://cadc.nrc.ca/vault'].conn self.assertTrue(conn.subject.certificate) self.assertFalse(conn.vo_token) # Supplying an empty string for certfile implies anonymous / http client = Client(vospace_certfile='') - client.get_session(uri='ivo://cadc.nrc.ca/vault') + client.is_remote_file = Mock() + client.get_session(uri='vos://cadc.nrc.ca~vault') conn = client._endpoints['ivo://cadc.nrc.ca/vault'].conn self.assertTrue(conn.subject.anon) self.assertFalse(conn.vo_token) # Specifying a token implies authenticated / http client = Client(vospace_token='a_token_string') - client.get_session(uri='ivo://cadc.nrc.ca/vault') + client.is_remote_file = Mock() + client.get_session(uri='vos://cadc.nrc.ca~vault') conn = client._endpoints['ivo://cadc.nrc.ca/vault'].conn self.assertTrue(conn.subject.anon) self.assertTrue(conn.vo_token) @@ -140,14 +143,14 @@ def test_init_client(self): with patch('os.access'): client = Client(vospace_certfile=certfile, vospace_token='a_token_string') - client.get_session(uri='ivo://cadc.nrc.ca/vault') + client.get_session(uri='vos://cadc.nrc.ca~vault') conn = client._endpoints['ivo://cadc.nrc.ca/vault'].conn self.assertTrue(conn.subject.anon) self.assertTrue(conn.vo_token) # update auth for specific service with patch('os.access'): - client.set_auth(uri='ivo://cadc.nrc.ca/vault', + client.set_auth(uri='vos://cadc.nrc.ca~vault', vospace_certfile=certfile) conn = client._endpoints['ivo://cadc.nrc.ca/vault'].conn self.assertTrue(conn.subject.certificate) @@ -292,6 +295,7 @@ def test_glob(self): mock_base_node.name = 'vos:' mock_base_node.node_list = [mock_node1, mock_node2] client = Client() + client.is_remote_file = Mock() client.get_node = Mock( side_effect=[mock_base_node, mock_node1, mock_node2]) self.assertEqual(['vos:/bnode/sometests'], @@ -300,8 +304,11 @@ def test_glob(self): @patch('vos.vos.md5_cache.MD5Cache.compute_md5') @patch('__main__.open', MagicMock(), create=True) def test_copy(self, computed_md5_mock): - file_content = 'File content'.encode('utf-8') + def is_remote_file(uri): + return True if ':' in uri else False + + file_content = 'File content'.encode('utf-8') # the md5sum of the file being copied transfer_md5 = hashlib.md5() transfer_md5.update(file_content) @@ -339,11 +346,12 @@ def test_copy(self, computed_md5_mock): test_client.get_node = get_node_mock # time to test... - vospaceLocation = 'vos://test/foo' + vospaceLocation = 'vos://authority~test/foo' osLocation = '/tmp/foo' if os.path.isfile(osLocation): os.remove(osLocation) # copy from vospace + test_client.is_remote_file = is_remote_file test_client.copy(vospaceLocation, osLocation) get_node_url_mock.assert_called_once_with(vospaceLocation, method='GET', @@ -725,6 +733,7 @@ def test_getNode(self): mock_vofile = Mock() client = Client() client.open = Mock(return_value=mock_vofile) + client.is_remote_file = Mock() mock_vofile.read = Mock( return_value=NODE_XML.format(uri, '').encode('UTF-8')) @@ -750,22 +759,23 @@ def test_move(self): conn = Connection(resource_id='ivo://cadc.nrc.ca/vault') conn.session.post = Mock(return_value=mock_resp_403) client = Client(conn=conn) + client.is_remote_file = Mock() - uri1 = 'notvos://cadc.nrc.ca!vospace/nosuchfile1' - uri2 = 'notvos://cadc.nrc.ca!vospace/nosuchfile2' + uri1 = 'notvos://cadc.nrc.ca!vault/nosuchfile1' + uri2 = 'notvos://cadc.nrc.ca!vault/nosuchfile2' - with self.assertRaises(OSError): + with self.assertRaises(AttributeError): client.move(uri1, uri2) @patch('vos.vos.net.ws.WsCapabilities.get_access_url', - Mock(return_value='https://www.canfar.phys.uvic.ca/vospace/nodes')) + Mock(return_value='https://ws.canfar.net/vault/nodes')) def test_delete(self): certfile = '/tmp/SomeCert.pem' open(certfile, 'w+') with patch('os.access'): client = Client(vospace_certfile=certfile) - uri1 = 'vos://cadc.nrc.ca!vospace/nosuchfile1' - url = 'https://www.canfar.phys.uvic.ca/vospace/nodes/nosuchfile1' + uri1 = 'vos://cadc.nrc.ca!vault/nosuchfile1' + url = 'https://ws.canfar.net/vault/nodes/nosuchfile1' mock_session = Mock() client.get_session = Mock(return_value=mock_session) client.delete(uri1) diff --git a/vos/vos/vos.py b/vos/vos/vos.py index 9a65d82c2..d5c6496af 100644 --- a/vos/vos/vos.py +++ b/vos/vos/vos.py @@ -125,24 +125,6 @@ class SortNodeProperty(Enum): logging.getLogger('requests').setLevel(logging.ERROR) -def is_remote_file(file_name, resource_id=None): - if file_name.startswith(('http://', 'https://')): - # assume full uri - return True - file_scheme = urlparse(file_name).scheme - if file_scheme: - if resource_id: - # resource id already known - return True - if vos_config.get_resource_id(resource_name=file_scheme) is not None: - return True - else: - raise ValueError( - 'Unsupported resource name {}. Check the config file at ' - '~/.config/vos/vos-config'.format(file_scheme)) - return False - - def convert_vospace_time_to_seconds(str_date): """A convenience method that takes a string from a vospace time field (UTC) and converts it to seconds since epoch local time. @@ -1278,9 +1260,6 @@ def write(buf): class EndPoints(object): VOSPACE_WEBSERVICE = os.getenv('VOSPACE_WEBSERVICE', None) - VODataView = {'cadc.nrc.ca!vospace': 'ivo://cadc.nrc.ca/vospace', - 'cadc.nrc.ca~vospace': 'ivo://cadc.nrc.ca/vospace'} - # standard ids VO_PROPERTIES = 'vos://cadc.nrc.ca~vospace/CADC/std/VOSpace#nodeprops' VO_NODES = 'ivo://ivoa.net/std/VOSpace/v2.0#nodes' @@ -1556,19 +1535,20 @@ def set_auth(self, uri, vospace_certfile=None, vospace_token=None): self.get_endpoints(uri).set_auth(vospace_certfile=vospace_certfile, vospace_token=vospace_token) - def get_resource_id(self, uri): - uri_parts = urlparse(self.fix_uri(uri)) - if uri_parts.scheme.startswith('vos'): - if uri_parts.netloc is None: - resource_id = vos_config.get('vos', 'resourceID') - else: - resource_id = 'ivo://{0}'.format(uri_parts.netloc).replace( - "!", "/").replace("~", "/") - elif uri_parts.scheme.startswith('ivo'): - resource_id = uri - else: - raise OSError('Unsupported scheme in {}'.format(uri)) - return resource_id + def is_remote_file(self, file_name): + if file_name.startswith(('http://', 'https://')): + # assume full uri + return True + file_scheme = urlparse(file_name).scheme + if file_scheme: + try: + self.get_endpoints(file_name) + return True + except Exception as ex: + msg = 'No VOSpace service found for {}'.format(file_name) + logger.debug('{}, Reason: {}'.format(msg, ex)) + raise ValueError(msg) + return False def get_endpoints(self, uri): """ @@ -1582,17 +1562,46 @@ def get_endpoints(self, uri): :return: corresponding EndPoint object """ - uri_parts = urlparse(self.fix_uri(uri)) - if uri_parts.scheme is not None: - resource_id = self.get_resource_id(uri) + uri_parts = urlparse(uri) + if uri.startswith('ivo://'): + raise AttributeError( + 'BUG: VOSpace identifier expected (vos scheme), ' + 'received registry identifier {}'.format(uri)) + if uri.startswith('vos://'): + resource_id = 'ivo://{}'.format( + uri_parts.hostname.replace('!', '/').replace('~', '/')) else: - raise OSError('No scheme in {}'.format(uri)) + if uri_parts.scheme is not None: + # assume first that the file_scheme is the short name of the + # of the resource e.g. arc corresponds to ivo://cadc.nrc.ca/arc + # With a proper reg, this could be replaced by a TAP search + # into the registry + if uri_parts.scheme == 'vos': + # special shortcut + scheme = 'vault' + else: + scheme = uri_parts.scheme + resource_id = 'ivo://cadc.nrc.ca/{}'.format(scheme) + else: + raise OSError('No scheme in {}'.format(uri)) if resource_id not in self._endpoints: - self._endpoints[resource_id] = EndPoints( - resource_id, vospace_certfile=self.vospace_certfile, - vospace_token=self.vospace_token) - + try: + self._endpoints[resource_id] = EndPoints( + resource_id, vospace_certfile=self.vospace_certfile, + vospace_token=self.vospace_token) + except Exception: + # no services by that short name. Try a shortcut from + # config (only for backwards compatibility) + try: + resource_id = vos_config.get_resource_id(scheme) + self._endpoints[resource_id] = EndPoints( + resource_id, vospace_certfile=self.vospace_certfile, + vospace_token=self.vospace_token) + except Exception: + raise AttributeError( + 'No service with resource ID {} found in registry or ' + 'the config file'.format(resource_id)) return self._endpoints[resource_id] def get_session(self, uri): @@ -1652,7 +1661,7 @@ def copy(self, source, destination, send_md5=False, disposition=False, get_node_url_retried = False content_disposition = None - if is_remote_file(source): + if self.is_remote_file(source): # GET if destination is None: # Set the destination, initially, to the same directory as @@ -1942,7 +1951,7 @@ def fix_uri(self, uri): return uri parts = urlparse(uri) - if not is_remote_file(uri): + if not self.is_remote_file(uri): if self.rootNode is not None: uri = self.rootNode + uri else: @@ -1958,12 +1967,6 @@ def fix_uri(self, uri): if linkuri[0] is not None: # TODO This does not work with invalid links. Should it? return self.fix_uri(linkuri[0]) - if not vos_config.get_resource_id(parts.scheme): - # Just past this back, I don't know how to fix... - raise ValueError( - 'resource name {} is not configured. Update the vos config ' - 'file (~/.config/vos/vos-config) to associate it to a ' - 'resourceID'.format(parts.scheme)) # Check for filename values. path = FILENAME_PATTERN_MAGIC.match(os.path.normpath(parts.path)) @@ -1978,7 +1981,7 @@ def fix_uri(self, uri): host = parts.netloc if not host or host == '': # default host corresponds to the resource ID of the client - host = vos_config.get_resource_id(parts.scheme).\ + host = self.get_endpoints(uri).uri.\ replace('ivo://', '').replace('/', '!') path = os.path.normpath(filename).strip('/') @@ -2016,7 +2019,7 @@ def get_node(self, uri, limit=0, force=False): # using the uri directly, but if this a URL then the metadata # comes from the HTTP header. # TODO removed ad. Not sure it was used - if is_remote_file(uri): + if self.is_remote_file(uri): vo_fobj = self.open(uri, os.O_RDONLY, limit=limit) vo_xml_string = vo_fobj.read().decode('UTF-8') xml_file = StringIO(vo_xml_string) @@ -2710,7 +2713,7 @@ def _node_type(self, uri): node = self.get_node(uri, limit=0) while node.type == "vos:LinkNode": uri = node.target - if is_remote_file(uri): + if self.is_remote_file(uri): node = self.get_node(uri, limit=0) else: return "vos:DataNode" @@ -2720,7 +2723,7 @@ def size(self, uri): node = self.get_node(uri, limit=0) while node.type == "vos:LinkNode": uri = node.target - if is_remote_file(uri): + if self.is_remote_file(uri): node = self.get_node(uri, limit=0) else: return int(requests.head(uri).headers.get('Content-Length', 0)) diff --git a/vos/vos/vosconfig.py b/vos/vos/vosconfig.py index ad5b95243..e18c2af69 100644 --- a/vos/vos/vosconfig.py +++ b/vos/vos/vosconfig.py @@ -25,9 +25,8 @@ def __init__(self, config_path, default_config_path=None): super(VosConfig, self).__init__(config_path=config_path) else: super(VosConfig, self).__init__(config_path=default_config_path) - # default resources vos and arc - self.resource_id = {'vos': 'ivo://cadc.nrc.ca/vault', - 'arc': 'ivo://cadc.nrc.ca/arbutus-cavern'} + # default vos resource shortcut + self.resource_id = {'vos': 'ivo://cadc.nrc.ca/vault'} default_rname = set(self.resource_id.keys()) if self.get('vos', 'resourceID'): for resource in self.get('vos', 'resourceID').split('\n'):