Discover registry VOSpace services by their short name/scheme (#190)

* Discover registry VOSpace services by their short name/scheme

vos/setup.cfg

@@ -50,7 +50,7 @@ edit_on_github = False
 github_project = opencadc/vostools
 install_requires = html2text>=2016.5.29 cadcutils>=1.1.30 future aenum
 # version should be PEP440 compatible (http://www.python.org/dev/peps/pep-0440)
-version = 3.3
+version = 3.3.1
 
 
 [entry_points]

vos/test/scripts/vospace-link-atest.tcsh

@@ -65,7 +65,6 @@ foreach resource ($resources)
   endif
   echo -n "** setting home and base to public, no groups"
   $CHMODCMD $CERT o+r $VOHOME || echo " [FAIL]" && exit -1
-  echo -n " [OK]"
   $CHMODCMD $CERT o+r $BASE || echo " [FAIL]" && exit -1
   echo " [OK]"
 
@@ -90,6 +89,17 @@ foreach resource ($resources)
   $LNCMD $CERT $CONTAINER/target $CONTAINER/clink >& /dev/null || echo " [FAIL]" && exit -1
   echo " [OK]"
 
+  echo -n "list container"
+  # content displayed unless -l and no / at the end
+  # content displayed
+  $LSCMD $CERT $CONTAINER/clink | grep -q something || echo " [FAIL]" && exit -1
+  $LSCMD $CERT $CONTAINER/clink/ | grep -q something || echo " [FAIL]" && exit -1
+  $LSCMD $CERT -l $CONTAINER/clink/ | grep -q something || echo " [FAIL]" && exit -1
+  # case where the content of the target is not displayed just the link
+  $LSCMD $CERT -l $CONTAINER/clink | grep 'clink ->' | grep -q target || echo " [FAIL]" && exit -1
+  echo " [OK]"
+
+
   echo -n "Follow the link to get the file"
   $CPCMD $CERT $CONTAINER/clink/something.png /tmp || echo " [FAIL]" && exit -1
   echo " [OK]"
@@ -106,12 +116,12 @@ foreach resource ($resources)
   $CPCMD $CERT $CONTAINER/dlink /tmp || echo " [FAIL]" && exit -1
   echo " [OK]"
 
-  echo -n "create link to unknown authority in URI"
+  echo -n "create link to external vos URI"
   if ( ${?TESTING_CAVERN} ) then
       echo " [SKIPPED, vos/issues/83]"
   else
       $RMCMD $CERT $CONTAINER/e1link >& /dev/null
-      $LNCMD $CERT vos://unknown.authority~vospace/unknown $CONTAINER/e1link >& /dev/null || echo " [FAIL]" && exit -1
+      $LNCMD $CERT vos://cadc.nrc.ca~arc/unknown $CONTAINER/e1link >& /dev/null || echo " [FAIL]" && exit -1
       echo " [OK]"
   endif
 
@@ -120,7 +130,7 @@ foreach resource ($resources)
   echo " [OK]"
 
   echo -n "create link to unknown scheme in URI"
-  $LNCMD $CERT unknown://cadc.nrc.ca~vospace/CADCRegtest1 $CONTAINER/e2ink >& /dev/null && echo " [FAIL]" && exit -1
+  $LNCMD $CERT unknown://cadc.nrc.ca~vault/CADCRegtest1 $CONTAINER/e2link >& /dev/null && echo " [FAIL]" && exit -1
   echo " [OK]"
 
   echo -n "Follow the invalid link and fail"

vos/vos/commands/tests/test_vls.py

@@ -97,27 +97,38 @@ def test_vls(self, vos_client_mock):
         mock_node1.props = {'length': 100, 'date': 50000}
         mock_node1.isdir.return_value = False
         mock_node1.get_info.return_value = ''
+        mock_node1.islink.return_value = False
 
         mock_node2 = MagicMock(type='vos:DataNode')
         mock_node2.name = 'node2'
         mock_node2.props = {'length': 30, 'date': 70000}
         mock_node2.isdir.return_value = False
         mock_node2.get_info.return_value = ''
+        mock_node2.islink.return_value = False
+
+        mock_node3_link = MagicMock(type='vos:DataNode')
+        mock_node3_link.name = 'node3_link'
+        mock_node3_link.props = {'length': 60, 'date': 20000}
+        mock_node3_link.isdir.return_value = False
+        mock_node3_link.get_info.return_value = ''
+        mock_node3_link.islink.return_value = True
 
         mock_node3 = MagicMock(type='vos:DataNode')
         mock_node3.name = 'node3'
         mock_node3.props = {'length': 60, 'date': 20000}
         mock_node3.isdir.return_value = False
         mock_node3.get_info.return_value = ''
+        mock_node3.islink.return_value = False
 
         vos_client_mock.return_value.glob.return_value = \
-            ['target2', 'target3', 'target1']
+            ['target2', 'target3_link', 'target1']
 
         # vls command with sort == None (i.e. sort by node name), order == None
         out = 'node1\nnode2\nnode3\n'
         with patch('sys.stdout', new_callable=StringIO) as stdout_mock:
             vos_client_mock.return_value.get_node = \
-                MagicMock(side_effect=[mock_node2, mock_node3, mock_node1])
+                MagicMock(side_effect=[mock_node2, mock_node3_link, mock_node3,
+                                       mock_node1])
             sys.argv = ['vls', 'vos:/CADCRegtest1']
             cmd_attr = getattr(commands, 'vls')
             cmd_attr()

vos/vos/commands/vcat.py

@@ -3,7 +3,7 @@
                         unicode_literals)
 import sys
 import logging
-from ..vos import Client, is_remote_file
+from ..vos import Client
 from ..commonparser import CommonParser, set_logging_level_from_args, \
     exit_on_exception, URI_DESCRIPTION
 
@@ -19,9 +19,9 @@ def _cat(uri, cert_filename=None, head=None):
 
     fh = None
     try:
-        if is_remote_file(uri):
-            view = head and 'header' or 'data'
-            fh = Client(vospace_certfile=cert_filename).open(uri, view=view)
+        view = head and 'header' or 'data'
+        fh = Client(vospace_certfile=cert_filename).open(uri, view=view)
+        if fh.is_remote_file(uri):
             sys.stdout.write(fh.read(return_response=True).text)
             sys.stdout.write('\n\n')
         else:

vos/vos/commands/vcp.py

@@ -102,7 +102,11 @@ class Nonlocal():
     if args.overwrite:
         warnings.warn("the --overwrite option is no longer supported")
 
-    if not vos.is_remote_file(dest):
+    client = vos.Client(
+        vospace_certfile=args.certfile, vospace_token=args.token,
+        transfer_shortcut=args.quick)
+
+    if not client.is_remote_file(dest):
         dest = os.path.abspath(dest)
 
     cutout_pattern = re.compile(
@@ -138,14 +142,14 @@ def get_node(filename, limit=None):
 
     def isdir(filename):
         logging.debug("Doing an isdir on %s" % filename)
-        if vos.is_remote_file(filename):
+        if client.is_remote_file(filename):
             return client.isdir(filename)
         else:
             return os.path.isdir(filename)
 
     def islink(filename):
         logging.debug("Doing an islink on %s" % filename)
-        if vos.is_remote_file(filename):
+        if client.is_remote_file(filename):
             try:
                 return get_node(filename).islink()
             except exceptions.NotFoundException:
@@ -161,7 +165,7 @@ def access(filename, mode):
         @return: True/False
         """
         logging.debug("checking for access %s " % filename)
-        if vos.is_remote_file(filename):
+        if client.is_remote_file(filename):
             try:
                 node = get_node(filename, limit=0)
                 return node is not None
@@ -176,27 +180,27 @@ def listdir(dirname):
         """Walk through the directory structure a al os.walk"""
         logging.debug("getting a dirlist %s " % dirname)
 
-        if vos.is_remote_file(dirname):
+        if client.is_remote_file(dirname):
             return client.listdir(dirname, force=True)
         else:
             return os.listdir(dirname)
 
     def mkdir(filename):
         logging.debug("Making directory %s " % filename)
-        if vos.is_remote_file(filename):
+        if client.is_remote_file(filename):
             return client.mkdir(filename)
         else:
             return os.mkdir(filename)
 
     def get_md5(filename):
         logging.debug("getting the MD5 for %s" % filename)
-        if vos.is_remote_file(filename):
+        if client.is_remote_file(filename):
             return get_node(filename).props.get('MD5', vos.ZERO_MD5)
         else:
             return md5_cache.MD5Cache.compute_md5(filename)
 
     def lglob(pathname):
-        if vos.is_remote_file(pathname):
+        if client.is_remote_file(pathname):
             return client.glob(pathname)
         else:
             return glob.glob(pathname)
@@ -337,18 +341,15 @@ def copy(source_name, destination_name, exclude=None, include=None,
     try:
         for source_pattern in args.source:
 
-            if args.head and not vos.is_remote_file(source_pattern):
+            if args.head and not client.is_remote_file(source_pattern):
                 logging.error("head only works for source files in vospace")
                 continue
 
             # define this empty cutout string.  Then we strip possible cutout
             # strings off the end of the pattern before matching.  This allows
             # cutouts on the vos service. The shell does pattern matching for
             # local files, so don't run glob on local files.
-            client = vos.Client(
-                vospace_certfile=args.certfile, vospace_token=args.token,
-                transfer_shortcut=args.quick)
-            if not vos.is_remote_file(source_pattern):
+            if not client.is_remote_file(source_pattern):
                 sources = [source_pattern]
             else:
                 cutout_match = cutout_pattern.search(source_pattern)
@@ -366,7 +367,7 @@ def copy(source_name, destination_name, exclude=None, include=None,
                     # stick back on the cutout pattern if there was one.
                     sources = [s + cutout for s in sources]
             for source in sources:
-                if not vos.is_remote_file(source):
+                if not client.is_remote_file(source):
                     source = os.path.abspath(source)
                 # the source must exist, of course...
                 if not access(source, os.R_OK):
@@ -377,7 +378,8 @@ def copy(source_name, destination_name, exclude=None, include=None,
                     continue
 
                 # copying inside VOSpace not yet implemented
-                if vos.is_remote_file(source) and vos.is_remote_file(dest):
+                if client.is_remote_file(source) and \
+                   client.is_remote_file(dest):
                     raise Exception(
                         "Can not (yet) copy from VOSpace to VOSpace.")
 

vos/vos/commands/vln.py

@@ -36,15 +36,15 @@ def vln():
     try:
         opt = parser.parse_args()
         set_logging_level_from_args(opt)
-
-        if not vos.is_remote_file(opt.source) or \
-                not vos.is_remote_file(opt.target):
-            raise ArgumentError(
-                None,
-                "source must be vos node or http url, target must be vos node")
         client = vos.Client(
             vospace_certfile=opt.certfile,
             vospace_token=opt.token)
+        if not client.is_remote_file(opt.source) or \
+                not client.is_remote_file(opt.target):
+            raise ArgumentError(
+                None,
+                "source must be vos node or http url, target must be vos node")
+
         client.link(opt.source, opt.target)
     except ArgumentError as ex:
         parser.print_usage()

vos/vos/commands/vls.py

@@ -131,18 +131,22 @@ def vls():
             order = 'desc' if sort else 'asc'
 
         for node in opt.node:
-            if not vos.is_remote_file(file_name=node):
-                raise ArgumentError(opt.node,
-                                    "Invalid node name: {}".format(node))
-            logging.debug("getting listing of: %s" % str(node))
             client = vos.Client(
                 vospace_certfile=opt.certfile,
                 vospace_token=opt.token)
+            if not client.is_remote_file(file_name=node):
+                raise ArgumentError(opt.node,
+                                    "Invalid node name: {}".format(node))
+            logging.debug("getting listing of: %s" % str(node))
+
             targets = client.glob(node)
 
             # segregate files from directories
             for target in targets:
                 target_node = client.get_node(target)
+                if target.endswith('/') or not opt.long:
+                    while target_node.islink():
+                        target_node = client.get_node(target_node.target)
                 if target_node.isdir():
                     dirs.append((_get_sort_key(target_node, sort),
                                  target_node, target))

vos/vos/commands/vmv.py

@@ -30,16 +30,16 @@ def vmv():
     try:
         source = args.source
         dest = args.destination
-        if not vos.is_remote_file(source):
+        client = vos.Client(
+            vospace_certfile=args.certfile,
+            vospace_token=args.token)
+        if not client.is_remote_file(source):
             raise ValueError('Source {} is not a remote node'.format(source))
-        if not vos.is_remote_file(dest):
+        if not client.is_remote_file(dest):
             raise ValueError(
                 'Destination {} is not a remote node'.format(dest))
         if urlparse(source).scheme != urlparse(dest).scheme:
             raise ValueError('Move between services not supported')
-        client = vos.Client(
-            vospace_certfile=args.certfile,
-            vospace_token=args.token)
         logging.info("{} -> {}".format(source, dest))
         client.move(source, dest)
     except Exception as ex:

vos/vos/commands/vrm.py

@@ -23,12 +23,12 @@ def vrm():
 
     try:
         for node in args.node:
-            if not vos.is_remote_file(node):
+            client = vos.Client(
+                vospace_certfile=args.certfile,
+                vospace_token=args.token)
+            if not client.is_remote_file(node):
                 raise Exception(
                     '{} is not a valid VOSpace handle'.format(node))
-            client = vos.Client(
-                    vospace_certfile=args.certfile,
-                    vospace_token=args.token)
             if not node.endswith('/'):
                 if client.get_node(node).islink():
                     logging.info('deleting link {}'.format(node))

vos/vos/commands/vrmdir.py

@@ -24,12 +24,12 @@ def vrmdir():
 
     try:
         for container_node in args.nodes:
-            if not vos.is_remote_file(container_node):
+            client = vos.Client(
+                vospace_certfile=args.certfile,
+                vospace_token=args.token)
+            if not client.is_remote_file(container_node):
                 raise ValueError(
                     "{} is not a valid VOSpace handle".format(container_node))
-            client = vos.Client(
-                    vospace_certfile=args.certfile,
-                    vospace_token=args.token)
             if client.isdir(container_node):
                 logging.info("deleting {}".format(container_node))
                 client.delete(container_node)

vos/vos/commands/vsync.py

@@ -87,12 +87,12 @@ def signal_handler(h_stream, h_frame):
         global_md5_cache = md5_cache.MD5Cache(cache_db=opt.cache_filename)
 
     destination = opt.destination
-    if not vos.is_remote_file(destination):
+    client = vos.Client(
+        vospace_certfile=opt.certfile, vospace_token=opt.token)
+    if not client.is_remote_file(destination):
         parser.error("Only allows sync FROM local copy TO VOSpace")
     # Currently we don't create nodes in sync and we don't sync onto files
     logging.info("Connecting to VOSpace")
-    client = vos.Client(
-        vospace_certfile=opt.certfile, vospace_token=opt.token)
     logging.info("Confirming Destination is a directory")
     dest_is_dir = client.isdir(destination)
 

vos/vos/commonparser.py

@@ -116,11 +116,8 @@ def __init__(self, *args, **kwargs):
 
 
 URI_DESCRIPTION = \
-    'Remote resources are identified either by their full ' \
-    'URIs (ivo://cadc.nrc.ca/vault) or by a user configured name in the ' \
-    'config file.\n' \
-    'Some names are reserved:\n' \
-    '     vos for ivo:/cadc.nrc.ca/vault and\n' \
-    '     arc for ivo:cadc.nc.ca/arbutus-cavern.\n' \
-    'Thus, arc:somepath is a shorter version of the full URI ' \
-    'vos://cadc.nrc.ca~arbutus-cavern/somepath'
+    'Remote resources are identified either by their full URIs \n' \
+    '(vos://cadc.nrc.ca~vault/<path>) or by shorter equivalent URIs with \n' \
+    'the scheme representing the name of the service (vault:<path>). \n' \
+    'Due to historical reasons, the `vos` scheme can be used to refer \n' \
+    'to the `vault` service, ie vos:<path> and vault:<path> are equivalent.\n'