From 35fee0531bddecd0f94e1442f634d19ae94ea6c6 Mon Sep 17 00:00:00 2001 From: Adrian Damian Date: Wed, 6 Mar 2024 14:09:20 -0800 Subject: [PATCH] Initial version --- vault/README.md | 16 ++++++- vault/build.gradle | 4 +- vault/src/intTest/README.md | 30 +++++++++++++ .../opencadc/vault/NodePersistenceImpl.java | 43 +++++++++++++++++++ .../org/opencadc/vault/VaultInitAction.java | 24 +++++++++++ 5 files changed, 113 insertions(+), 4 deletions(-) create mode 100644 vault/src/intTest/README.md diff --git a/vault/README.md b/vault/README.md index 02e701ea..a2e43a88 100644 --- a/vault/README.md +++ b/vault/README.md @@ -1,7 +1,7 @@ # Storage Inventory VOSpace-2.1 service (vault) -The `vault` servcie is an implementation of the IVOA VOSpace -specification designed to co-exist with other storage-inventory components. It provides a heirarchical data +The `vault` service is an implementation of the IVOA VOSpace +specification designed to co-exist with other storage-inventory components. It provides a hierarchical data organization laye on top of the storage management of storage-inventory. The simplest configuration would be to deploy `vault` with `minoc` with a single metadata database and single @@ -80,6 +80,9 @@ A vault.properties file in /config is required to run this service. The followi # service identity org.opencadc.vault.resourceID = ivo://{authority}/{name} +# (optional) identify which container nodes are allocations +org.opencadc.vault.allocationParent = {top level node} + # consistency settings org.opencadc.vault.consistency.preventNotFound=true|false @@ -96,6 +99,15 @@ org.opencadc.vault.storage.namespace = {a storage inventory namespace to use} ``` The vault _resourceID_ is the resourceID of _this_ vault service. +The _allocationParent_ is a path to a container node (directory) which contains space allocations. An allocation +is owned by a user (usually different from the _rootOwner_ admin user) who is responsible for the allocation +and all content therein. The owner of an allocation is granted additional permissions within their +allocation (they can read/write/delete anything) so the owner cannot be blocked from access to any content +within their allocation. This probably only matters for multi-user projects. Multiple _allocationParent_(s) may +be configured to organise the top level of the content (e.g. /home and /projects). Paths configured to be +_allocationParent_(s) will be automatically created (if necessary), owned by the _rootOwner_, and will be +anonymously readable (public). Limitation: only top-level container nodes can be configured as _allocationParent_(s + The _preventNotFound_ key can be used to configure `vault` to prevent artifact-not-found errors that might result due to the eventual consistency nature of the storage system by directly checking for the artifact at _all known_ sites. It only makes sense to enable this when `vault` is running in a global inventory (along with diff --git a/vault/build.gradle b/vault/build.gradle index a1154999..9d358c4b 100644 --- a/vault/build.gradle +++ b/vault/build.gradle @@ -36,7 +36,7 @@ dependencies { compile 'org.opencadc:cadc-gms:[1.0.5,)' compile 'org.opencadc:cadc-rest:[1.3.16,)' compile 'org.opencadc:cadc-vos:[2.0.3,)' - compile 'org.opencadc:cadc-vos-server:[2.0.9,)' + compile 'org.opencadc:cadc-vos-server:[2.0.11,)' compile 'org.opencadc:cadc-vosi:[1.3.2,)' compile 'org.opencadc:cadc-uws:[1.0,)' compile 'org.opencadc:cadc-uws-server:[1.2.19,)' @@ -54,7 +54,7 @@ dependencies { runtime 'org.opencadc:cadc-gms:[1.0.5,)' intTestCompile 'org.opencadc:cadc-test-vosi:[1.0.11,)' - intTestCompile 'org.opencadc:cadc-test-vos:[2.1.6,)' + intTestCompile 'org.opencadc:cadc-test-vos:[2.1.8,)' } configurations { diff --git a/vault/src/intTest/README.md b/vault/src/intTest/README.md new file mode 100644 index 00000000..a1ee9176 --- /dev/null +++ b/vault/src/intTest/README.md @@ -0,0 +1,30 @@ +# Storage Inventory VOSpace-2.1 service (vault) + +The simplest configuration that deploys `vault` with `minoc` with a single metadata database and single +back end storage system is sufficient to run the `vault` integration tests. The tests also relly on the +presence of the root owner X509 certificate in `build/classes/java/intTest/vault-test.pem`. +Some tests (primarily permission tests) will be skipped unless the certificate of a second user is present +in `build/classes/java/intTest/vault-auth-test.pem`. This user has to be member of the `ivo://cadc.nrc.ca/gms?opencadc-vospace-test` +group. The names of these certificates and groups are hardcoded in the `vault` int tests classes. + +The int tests suite also relies on a specific configuration of the `vault` service: +### vault.properties +``` +# service identity +org.opencadc.vault.resourceID = ivo://opencadc.org/vault + +# (optional) identify which container nodes are allocations +org.opencadc.vault.allocationParent = / + +# consistency settings +org.opencadc.vault.consistency.preventNotFound=true + +# vault database settings +org.opencadc.vault.inventory.schema = inventory +org.opencadc.vault.vospace.schema = vault +org.opencadc.vault.singlePool = true + +# root container nodes +org.opencadc.vault.root.owner = {owner of root node} + +``` \ No newline at end of file diff --git a/vault/src/main/java/org/opencadc/vault/NodePersistenceImpl.java b/vault/src/main/java/org/opencadc/vault/NodePersistenceImpl.java index 0afa377a..480094eb 100644 --- a/vault/src/main/java/org/opencadc/vault/NodePersistenceImpl.java +++ b/vault/src/main/java/org/opencadc/vault/NodePersistenceImpl.java @@ -112,6 +112,7 @@ import org.opencadc.vospace.io.NodeWriter; import org.opencadc.vospace.server.LocalServiceURI; import org.opencadc.vospace.server.NodePersistence; +import org.opencadc.vospace.server.Utils; import org.opencadc.vospace.server.Views; import org.opencadc.vospace.server.transfers.TransferGenerator; @@ -160,6 +161,7 @@ public class NodePersistenceImpl implements NodePersistence { private final boolean singlePool; private final ContainerNode root; + private final Set allocationParents = new TreeSet<>(); private final Namespace storageNamespace; private final boolean localGroupsOnly; @@ -204,6 +206,33 @@ public NodePersistenceImpl(URI resourceID) { } else { throw new IllegalStateException("invalid config: missing/invalid preventNotFound configuration"); } + for (String ap : VaultInitAction.getAllocParentsConfig(config)) { + if (ap.isEmpty()) { + // allocations are in root + allocationParents.add(root); + log.info("allocationParent: /"); + } else { + try { + + // simple top-level names only + ContainerNode cn = (ContainerNode) get(root, ap); + String str = ""; + if (cn == null) { + cn = new ContainerNode(ap); + cn.parent = root; + str = "created/"; + } + cn.isPublic = true; + cn.owner = root.owner; + cn.inheritPermissions = false; + put(cn); + allocationParents.add(cn); + log.info(str + "loaded allocationParent: /" + cn.getName()); + } catch (NodeNotSupportedException bug) { + throw new RuntimeException("BUG: failed to update isPublic=true on allocationParent " + ap, bug); + } + } + } } private Subject getRootOwner(MultiValuedProperties mvp, IdentityManager im) { @@ -265,6 +294,20 @@ public ContainerNode getRootNode() { return root; } + @Override + public boolean isAllocation(ContainerNode cn) { + if (cn.parent == null) { + return false; // root is never an allocation + } + for (ContainerNode ap : allocationParents) { + if (Utils.getPath(ap).equals(Utils.getPath(cn.parent))) { + // same node + return true; + } + } + return false; + } + @Override public Set getAdminProps() { return Collections.unmodifiableSet(ADMIN_PROPS); diff --git a/vault/src/main/java/org/opencadc/vault/VaultInitAction.java b/vault/src/main/java/org/opencadc/vault/VaultInitAction.java index 6e62a021..9c9b1c68 100644 --- a/vault/src/main/java/org/opencadc/vault/VaultInitAction.java +++ b/vault/src/main/java/org/opencadc/vault/VaultInitAction.java @@ -69,6 +69,7 @@ import ca.nrc.cadc.db.DBUtil; import ca.nrc.cadc.rest.InitAction; +import ca.nrc.cadc.util.InvalidConfigException; import ca.nrc.cadc.util.MultiValuedProperties; import ca.nrc.cadc.util.PropertiesReader; import ca.nrc.cadc.util.RsaSignatureGenerator; @@ -76,6 +77,8 @@ import java.net.URI; import java.net.URISyntaxException; import java.security.KeyPair; +import java.util.ArrayList; +import java.util.List; import java.util.Map; import java.util.TreeMap; import javax.naming.Context; @@ -117,6 +120,7 @@ public class VaultInitAction extends InitAction { static final String SINGLE_POOL_KEY = VAULT_KEY + ".singlePool"; static final String ROOT_OWNER = VAULT_KEY + ".root.owner"; // numeric? + public static final String ALLOCATION_PARENT = VAULT_KEY + ".allocationParent"; static final String STORAGE_NAMESPACE_KEY = VAULT_KEY + ".storage.namespace"; @@ -261,6 +265,26 @@ static Map getInvConfig(MultiValuedProperties props) { ret.put("genSchema", props.getFirstPropertyValue(INVENTORY_SCHEMA_KEY)); // for complete init return ret; } + + static List getAllocParentsConfig(MultiValuedProperties mvp) { + List allocParents = new ArrayList<>(); + for (String sap : mvp.getProperty(ALLOCATION_PARENT)) { + String ap = sap; + if (ap.charAt(0) == '/') { + ap = ap.substring(1); + } + if (ap.length() > 0 && ap.charAt(ap.length() - 1) == '/') { + ap = ap.substring(0, ap.length() - 1); + } + if (ap.indexOf('/') >= 0) { + throw new InvalidConfigException("invalid " + ALLOCATION_PARENT + ": " + sap + + " reason: must be a top-level container node name"); + } + // empty string means root, otherwise child of root + allocParents.add(ap); + } + return allocParents; + } static Map getKeyPairConfig(MultiValuedProperties props) { return getDaoConfig(props);