Skip to content

Commit

Permalink
Changes to make vostools work with the new cavern and vault ser…
Browse files Browse the repository at this point in the history
…vices (CADC-12912) (#215)
  • Loading branch information
andamian authored Feb 13, 2024
1 parent d4edf89 commit 40c1767
Show file tree
Hide file tree
Showing 12 changed files with 301 additions and 162 deletions.
2 changes: 1 addition & 1 deletion .github/workflows/cibuild.yml
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@ jobs:
strategy:
fail-fast: true
matrix:
python-version: ["3.7", "3.8", "3.9", "3.10", "3.11"]
python-version: ["3.7", "3.8", "3.9", "3.10", "3.11", "3.12"]
package: [vos]
steps:
- name: Checkout code
Expand Down
6 changes: 4 additions & 2 deletions vos/setup.cfg
Original file line number Diff line number Diff line change
Expand Up @@ -21,9 +21,11 @@ omit = */tests/*
[tool:pytest]
minversion = 2.2
norecursedirs = build docs/_build
doctest_plus = enabled
testpaths = vos

[flake8]
max-line-length = 120

[pep8]
# E101 - mix of tabs and spaces
# W191 - use of tabs
Expand Down Expand Up @@ -54,7 +56,7 @@ install_requires =
aenum

# version should be PEP440 compatible (http://www.python.org/dev/peps/pep-0440)
version = 3.4
version = 3.5

[options.extras_require]
test =
Expand Down
3 changes: 1 addition & 2 deletions vos/setup.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,6 @@
import glob
import os
import sys
import imp
from setuptools.command.test import test as TestCommand
from setuptools import find_packages

Expand Down Expand Up @@ -128,7 +127,7 @@ def run_tests(self):
use_2to3=False,
setup_requires=['pytest-runner'],
entry_points=entry_points,
python_requires='!=3.0.*, !=3.1.*, !=3.2.*, !=3.3, <4',
python_requires='>=3.7, <4',
packages=find_packages(),
package_data={PACKAGENAME: ['data/*', 'tests/data/*', '*/data/*', '*/tests/data/*']},
classifiers=[
Expand Down
2 changes: 1 addition & 1 deletion vos/tox.ini
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ name = vos

[tox]
envlist =
py{37,38,39,310,311}
py{37,38,39,310,311,312}
requires =
pip >= 19.3.1

Expand Down
21 changes: 12 additions & 9 deletions vos/vos/commands/vchmod.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
# ****************** CANADIAN ASTRONOMY DATA CENTRE *******************
# ************* CENTRE CANADIEN DE DONNÉES ASTRONOMIQUES **************
#
# (c) 2022. (c) 2022.
# (c) 2024. (c) 2024.
# Government of Canada Gouvernement du Canada
# National Research Council Conseil national de recherches
# Ottawa, Canada, K1A 0R6 Ottawa, Canada, K1A 0R6
Expand Down Expand Up @@ -71,7 +71,7 @@
"""
from ..vos import Client
from ..vos import CADC_GMS_PREFIX
from ..commonparser import CommonParser, set_logging_level_from_args,\
from ..commonparser import CommonParser, set_logging_level_from_args, \
URI_DESCRIPTION
from ..commonparser import exit_on_exception
import logging
Expand Down Expand Up @@ -190,20 +190,23 @@ def vchmod():
vospace_token=opt.token,
insecure=opt.insecure)
node = client.get_node(opt.node)
if opt.recursive:
node.props.clear()
node.clear_properties()
# del node.node.findall(vos.Node.PROPERTIES)[0:]
node.props.clear()
node.clear_properties()
if 'readgroup' in props:
node.chrgrp(props['readgroup'])
if 'writegroup' in props:
node.chwgrp(props['writegroup'])
if 'ispublic' in props:
node.set_public(props['ispublic'])
logging.debug("Node: {0}".format(node))
status = client.update(node, opt.recursive)
if status:
sys.exit(status)
successes, failures = client.update(node, opt.recursive)
if opt.recursive:
if failures:
logging.error('WARN. updated count: {}, failed count: {}\n'.
format(successes, failures))
sys.exit(-1)
else:
logging.info('DONE. updated count: {}\n'.format(successes))
except Exception as ex:
exit_on_exception(ex)

Expand Down
17 changes: 2 additions & 15 deletions vos/vos/commands/vcp.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
# ****************** CANADIAN ASTRONOMY DATA CENTRE *******************
# ************* CENTRE CANADIEN DE DONNÉES ASTRONOMIQUES **************
#
# (c) 2022. (c) 2022.
# (c) 2024. (c) 2024.
# Government of Canada Gouvernement du Canada
# National Research Council Conseil national de recherches
# Ottawa, Canada, K1A 0R6 Ottawa, Canada, K1A 0R6
Expand Down Expand Up @@ -69,7 +69,7 @@
"""copy files vospace to local or local to VOSpace"""
from .. import md5_cache
from .. import vos
from ..commonparser import CommonParser, set_logging_level_from_args,\
from ..commonparser import CommonParser, set_logging_level_from_args, \
exit_on_exception, URI_DESCRIPTION

try:
Expand Down Expand Up @@ -324,19 +324,6 @@ def copy(source_name, destination_name, exclude=None, include=None,
if ans != 'y':
raise Exception("File exists")

if not access(os.path.dirname(destination_name), os.F_OK):
raise OSError(errno.EEXIST,
"vcp: ContainerNode %s does not exist" %
os.path.dirname(
destination_name))

if not isdir(os.path.dirname(destination_name)) and not islink(
os.path.dirname(destination_name)):
raise OSError(errno.ENOTDIR,
"vcp: %s is not a ContainerNode or LinkNode"
% os.path.dirname(
destination_name))

skip = False
if exclude is not None:
for thisIgnore in exclude.split(','):
Expand Down
40 changes: 28 additions & 12 deletions vos/vos/commands/vrm.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
# ****************** CANADIAN ASTRONOMY DATA CENTRE *******************
# ************* CENTRE CANADIEN DE DONNÉES ASTRONOMIQUES **************
#
# (c) 2022. (c) 2022.
# (c) 2024. (c) 2024.
# Government of Canada Gouvernement du Canada
# National Research Council Conseil national de recherches
# Ottawa, Canada, K1A 0R6 Ottawa, Canada, K1A 0R6
Expand Down Expand Up @@ -68,6 +68,8 @@

"""remove a vospace data node, fails if container node or node is locked."""
import logging
import sys

from ..commonparser import set_logging_level_from_args, exit_on_exception, \
CommonParser, URI_DESCRIPTION
from .. import vos
Expand All @@ -81,8 +83,12 @@

def vrm():
parser = CommonParser(description=DESCRIPTION)
parser.add_argument(
"-R", "--recursive", action="store_true",
help="Delete a file or directory even if it's not empty.",
default=False)
parser.add_argument('node',
help='dataNode or linkNode to delete from VOSpace',
help='file, link or possibly directory to delete from VOSpace',
nargs='+')

args = parser.parse_args()
Expand All @@ -97,17 +103,27 @@ def vrm():
if not client.is_remote_file(node):
raise Exception(
'{} is not a valid VOSpace handle'.format(node))
if not node.endswith('/'):
if client.get_node(node).islink():
logging.info('deleting link {}'.format(node))
client.delete(node)
elif client.isfile(node):
logging.info('deleting {}'.format(node))
client.delete(node)
elif client.isdir(node):
raise Exception('{} is a directory'.format(node))
if args.recursive:
successes, failures = client.recursive_delete(node)
if failures:
logging.error('WARN. deleted count: {}, failed count: '
'{}\n'.format(successes, failures))
sys.exit(-1)
else:
logging.info(
'DONE. deleted count: {}\n'.format(successes))
else:
raise Exception('{} is not a directory'.format(node))
if not node.endswith('/'):
if client.get_node(node).islink():
logging.info('deleting link {}'.format(node))
client.delete(node)
elif client.isfile(node):
logging.info('deleting {}'.format(node))
client.delete(node)
elif client.isdir(node):
raise Exception('{} is a directory'.format(node))
else:
raise Exception('{} is not a directory'.format(node))

except Exception as ex:
exit_on_exception(ex)
Expand Down
77 changes: 49 additions & 28 deletions vos/vos/commands/vtag.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
# ****************** CANADIAN ASTRONOMY DATA CENTRE *******************
# ************* CENTRE CANADIEN DE DONNÉES ASTRONOMIQUES **************
#
# (c) 2022. (c) 2022.
# (c) 2024. (c) 2024.
# Government of Canada Gouvernement du Canada
# National Research Council Conseil national de recherches
# Ottawa, Canada, K1A 0R6 Ottawa, Canada, K1A 0R6
Expand Down Expand Up @@ -71,7 +71,9 @@
The tag system is meant to allow tags, in addition to the standard node
properties. """
import logging
import pprint
import sys
from ..commonparser import CommonParser, set_logging_level_from_args, \
exit_on_exception, URI_DESCRIPTION
from .. import vos
Expand Down Expand Up @@ -107,14 +109,15 @@ def vtag():
nargs="*")
parser.add_option('--remove', action="store_true",
help='remove the listed property')
parser.add_option('-R', '--recursive', action="store_true",
help='perform the operation recursively on all the descendants')

opt = parser.parse_args()
args = opt
args = parser.parse_args()
set_logging_level_from_args(args)

# the node should be the first argument, the rest should contain
# the key/val pairs
node = args.node
node_arg = args.node

props = []
if args.remove:
Expand All @@ -128,35 +131,53 @@ def vtag():

try:
client = vos.Client(
vospace_certfile=opt.certfile,
vospace_token=opt.token,
insecure=opt.insecure)
node = client.get_node(node)
vospace_certfile=args.certfile,
vospace_token=args.token,
insecure=args.insecure)
node = client.get_node(node_arg)
if len(props) == 0:
# print all properties
pprint.pprint(node.props)

changed = False
for prop in props:
prop = prop.split('=')
if len(prop) == 1:
# get one property
pprint.pprint(node.props.get(prop[0], None))
elif len(prop) == 2:
# update properties
key, value = prop
if args.recursive:
node.props.clear()
node.clear_properties()
for prop in props:
key, value = prop.split('=')
if len(value) == 0:
value = None
if value != node.props.get(key, None):
node.props[key] = value
changed = True
else:
raise ValueError(
"Illegal keyword of value character ('=') used: %s" % (
'='.join(prop)))

if changed:
client.add_props(node)
node.props[key] = value
successes, failures = client.add_props(node, recursive=True)
if args.recursive:
if failures:
logging.error(
'WARN. updated count: {}, failed count: {}\n'.
format(successes, failures))
sys.exit(-1)
else:
logging.info(
'DONE. updated count: {}\n'.format(successes))
else:
changed = False
for prop in props:
prop = prop.split('=')
if len(prop) == 1:
# get one property
pprint.pprint(node.props.get(prop[0], None))
elif len(prop) == 2:
# update properties
key, value = prop
if len(value) == 0:
value = None
if value != node.props.get(key, None):
node.props[key] = value
changed = True
else:
raise ValueError(
"Illegal keyword of value character ('=') used: %s" % (
'='.join(prop)))

if changed:
client.add_props(node)
except Exception as ex:
exit_on_exception(ex)

Expand Down
19 changes: 10 additions & 9 deletions vos/vos/tests/test_config.py
Original file line number Diff line number Diff line change
Expand Up @@ -65,12 +65,12 @@
#
# ***********************************************************************
#

import unittest
# Test the vosconfig functionality
from unittest.mock import Mock, patch
import warnings

from vos import vosconfig, Connection
from vos import vosconfig
import pytest
import tempfile
import os
Expand All @@ -80,29 +80,30 @@
skipTests = False


# @unittest.skip('Functionality no longer used')
@patch('vos.vos.os.path.exists', Mock())
def test_update_config():
# Cause all warnings to always be triggered.
warnings.simplefilter("always")
with patch('vos.vos.open') as open_mock:
with patch('builtins.open') as open_mock:
old_content = 'blah'
new_config_mock = Mock()
open_mock.return_value.read.return_value = old_content
open_mock.return_value.write = new_config_mock
vosconfig._update_config()
assert new_config_mock.called_once_with(old_content)
new_config_mock.assert_called_once_with(old_content)

# test rewrite vospace resource in config file
new_config_mock.reset_mock()
# Cause all warnings to always be triggered.
warnings.simplefilter("always")
with patch('vos.vos.open') as open_mock:
with patch('builtins.open') as open_mock:
old_content = 'blah\nresourceID=ivo://cadc.nrc.ca/vospace\nfoo'
new_content = Mock()
open_mock.return_value.read.return_value = old_content
open_mock.return_value.write = new_content
vosconfig._update_config()
assert new_config_mock.called_once_with(old_content.replace(
new_content.assert_called_once_with(old_content.replace(
'vospace', 'vault'))

# test rewrite transfer protocol in config file
Expand All @@ -111,14 +112,14 @@ def test_update_config():
"# transfer protocol configuration is no longer supported\n"
# Cause all warnings to always be triggered.
warnings.simplefilter("always")
with patch('vos.vos.open') as open_mock:
with patch('builtins.open') as open_mock:
old_content = 'blah\nprotocol=http\nfoo'
new_content = Mock()
open_mock.return_value.read.return_value = old_content
open_mock.return_value.write = new_content
vosconfig._update_config()
assert new_config_mock.called_once_with(old_content.replace(
'protocol', '{}#protocol'.format(protocol_text)))
new_content.assert_called_once_with(old_content.replace(
'protocol', '{}# protocol'.format(protocol_text)))


@patch('vos.vosconfig.os.path.isfile', Mock())
Expand Down
Loading

0 comments on commit 40c1767

Please sign in to comment.