Fixed link to external sites and other small issues (#202)

opencadc · Jan 10, 2022 · 6644d61 · 6644d61
1 parent 7f755ec
commit 6644d61
Show file tree

Hide file tree

Showing 5 changed files with 53 additions and 55 deletions.
diff --git a/vos/setup.cfg b/vos/setup.cfg
@@ -50,7 +50,7 @@ edit_on_github = False
 github_project = opencadc/vostools
 install_requires = html2text>=2016.5.29 cadcutils>=1.2.6 future aenum
 # version should be PEP440 compatible (http://www.python.org/dev/peps/pep-0440)
-version = 3.3.5
+version = 3.3.6
 
 
 [entry_points]

diff --git a/vos/test/scripts/vospace-all.tcsh b/vos/test/scripts/vospace-all.tcsh
@@ -30,8 +30,8 @@ echo "vospace-node-properties.tcsh"
 $THIS_DIR/vospace-node-properties.tcsh $args || echo "FAIL vospace-node-properties.tcsh" && exit -1
 echo "vospace-lock-atest.tcsh"
 $THIS_DIR/vospace-lock-atest.tcsh $args || echo "FAIL vospace-lock-atest.tcsh" && exit -1
-echo "vospace-cookie-atest.tcsh"
-$THIS_DIR/vospace-token-atest.tcsh $args || echo "FAIL vospace-cookie-atest.tcsh" && exit -1
+#echo "vospace-cookie-atest.tcsh"
+#$THIS_DIR/vospace-token-atest.tcsh $args || echo "FAIL vospace-cookie-atest.tcsh" && exit -1
 
 echo
 echo "*** all test sequences passed ***"

diff --git a/vos/test/scripts/vospace-link-atest.tcsh b/vos/test/scripts/vospace-link-atest.tcsh
@@ -100,19 +100,19 @@ foreach resource ($resources)
   echo " [OK]"
 
 
-  echo -n "Follow the link to get the file"
+  echo -n "follow the link to get the file"
   $CPCMD $CERT $CONTAINER/clink/something.png /tmp || echo " [FAIL]" && exit -1
   echo " [OK]"
 
-  echo -n "Follow the link without read permission and fail"
+  echo -n "follow the link without read permission and fail"
   $CPCMD $CERT2 $CONTAINER/clink/something.png /tmp >& /dev/null && echo " [FAIL]" && exit -1
   echo " [OK]"
 
   echo -n "create link to target file"
   $LNCMD $CERT $CONTAINER/target/something.png $CONTAINER/dlink >& /dev/null || echo " [FAIL]" && exit -1
   echo " [OK]"
 
-  echo -n "Follow the link to get the file"
+  echo -n "follow the link to get the file"
   $CPCMD $CERT $CONTAINER/dlink /tmp || echo " [FAIL]" && exit -1
   echo " [OK]"
 
@@ -125,7 +125,7 @@ foreach resource ($resources)
       echo " [OK]"
   endif
 
-  echo -n "Follow the invalid link and fail"
+  echo -n "follow the invalid link and fail"
   $CPCMD $CERT $CONTAINER/e1link/somefile /tmp >& /dev/null && echo " [FAIL]" && exit -1
   echo " [OK]"
 
@@ -134,18 +134,30 @@ foreach resource ($resources)
   #$LNCMD $CERT unknown://cadc.nrc.ca~vault/CADCRegtest1 $CONTAINER/e2link >& /dev/null && echo " [FAIL]" && exit -1
   echo " [SKIPPED - TODO]"
 
-  echo -n "Follow the invalid link and fail"
+  echo -n "follow the invalid link and fail"
   $CPCMD $CERT $CONTAINER/e2link/somefile /tmp  >& /dev/null && echo " [FAIL]" && exit -1
   echo " [OK]"
 
   echo -n "create link to external http URI"
   if ( ${?TESTING_CAVERN} ) then
       echo " [SKIPPED, vos/issues/83]"
   else
-      $LNCMD $CERT http://www.google.ca $CONTAINER/e3link > /dev/null || echo " [FAIL]" && exit -1
+      $LNCMD $CERT https://www.google.ca $CONTAINER/e3link > /dev/null || echo " [FAIL]" && exit -1
       echo " [OK]"
   endif
 
+  echo -n "copy external link"
+  if ( ${?TESTING_CAVERN} ) then
+      echo " [SKIPPED, vos/issues/83]"
+  else
+      rm -f /tmp/e3link
+      $CPCMD $CERT -L $CONTAINER/e3link /tmp/ >& /dev/null || echo " [FAIL]" && exit -1
+      grep -q google /tmp/e3link || echo " [FAIL]" && exit -1
+      rm -f /tmp/e3link
+      echo " [OK]"
+  endif
+
+
   echo -n "Follow the invalid link and fail"
   $CPCMD $CERT $CONTAINER/e3link/somefile /tmp  >& /dev/null && echo " [FAIL]" && exit -1
   echo " [OK]"

diff --git a/vos/vos/commands/vsync.py b/vos/vos/commands/vsync.py
@@ -245,7 +245,6 @@ def build_file_list(paths, vos_root, recursive=False, include=None,
     :return: set of expanded (src, dest) pairs
     """
 
-    spinner = ['-', '\\', '|', '/', '-', '\\', '|', '/']
     count = 0
     results = []  # order is important to create the directories first
     vos_root = vos_root.strip('/')
@@ -284,8 +283,6 @@ def build_file_list(paths, vos_root, recursive=False, include=None,
                 if not validate(rel_name, include=include, exclude=exclude):
                     continue
                 count += 1
-                logging.info("Building list of files to transfer {}\r".format(
-                             spinner[count % len(spinner)]))
                 results.append((srcfilename, '{}/{}'.format(
                     vos_root, rel_name)))
             if not recursive:

diff --git a/vos/vos/vos.py b/vos/vos/vos.py
@@ -43,13 +43,6 @@
 from cadcutils import net, exceptions
 from . import md5_cache
 
-try:
-    from urllib import splittag
-except ImportError:
-    import six.moves.urllib.parse
-
-    splittag = six.moves.urllib.parse.splittag
-
 urlparse = six.moves.urllib.parse.urlparse
 parse_qs = six.moves.urllib.parse.parse_qs
 logger = logging.getLogger('vos')
@@ -482,39 +475,32 @@ def fix_prop(prop):
         property.
         :rtype unicode
         """
-        (url, tag) = splittag(prop)
-        if tag is None and url in ['title',
-                                   'creator',
-                                   'subject',
-                                   'description',
-                                   'publisher',
-                                   'contributer',
-                                   'date',
-                                   'type',
-                                   'format',
-                                   'identifier',
-                                   'source',
-                                   'language',
-                                   'relation',
-                                   'coverage',
-                                   'rights',
-                                   'availableSpace',
-                                   'groupread',
-                                   'groupwrite',
-                                   'publicread',
-                                   'quota',
-                                   'length',
-                                   'MD5',
-                                   'mtime',
-                                   'ctime',
-                                   'ispublic']:
-            tag = url
-            url = Node.IVOAURL
-            prop = url + "#" + tag
-
-        parts = urlparse(url)
-        if parts.path is None or tag is None:
-            raise ValueError("Invalid VOSpace property uri: {0}".format(prop))
+        if prop in ['title',
+                    'creator',
+                    'subject',
+                    'description',
+                    'publisher',
+                    'contributer',
+                    'date',
+                    'type',
+                    'format',
+                    'identifier',
+                    'source',
+                    'language',
+                    'relation',
+                    'coverage',
+                    'rights',
+                    'availableSpace',
+                    'groupread',
+                    'groupwrite',
+                    'publicread',
+                    'quota',
+                    'length',
+                    'MD5',
+                    'mtime',
+                    'ctime',
+                    'ispublic']:
+            prop = Node.IVOAURL + "#" + prop
 
         return prop
 
@@ -856,9 +842,10 @@ def get_prop_name(prop):
         :param prop: the uri of the property to get the name of.
 
         """
-        (url, prop_name) = splittag(prop)
-        if url == Node.IVOAURL:
-            return prop_name
+        parts = urlparse(prop)
+        if '{}://{}{}'.format(parts.scheme, parts.netloc, parts.path) == \
+                Node.IVOAURL:
+            return parts.fragment
         return prop
 
     @staticmethod
@@ -2760,6 +2747,8 @@ def _node_type(self, uri):
         while node.type == "vos:LinkNode":
             uri = node.target
             if self.is_remote_file(uri):
+                if uri.startswith("http"):
+                    return 'vos:DataNode'
                 node = self.get_node(uri, limit=0)
             else:
                 return "vos:DataNode"