Support for recursive node deletes (CADC-12558)

vault/build.gradle

@@ -30,18 +30,18 @@ def git_url = 'https://github.com/opencadc/vos'
 dependencies {
     compile 'javax.servlet:javax.servlet-api:[3.1,4.0)'
 
-    compile 'org.opencadc:cadc-util:[1.9.5,2.0)'
+    compile 'org.opencadc:cadc-util:[1.9.10,2.0)'
     compile 'org.opencadc:cadc-log:[1.1.6,2.0)'
     compile 'org.opencadc:cadc-gms:[1.0.5,)'
     compile 'org.opencadc:cadc-rest:[1.3.16,)'
     compile 'org.opencadc:cadc-vos:[2.0,)'
     compile 'org.opencadc:cadc-vos-server-alt:[2.0,)'
     compile 'org.opencadc:cadc-vosi:[1.3.2,)'
     compile 'org.opencadc:cadc-uws:[1.0,)'
-    compile 'org.opencadc:cadc-uws-server:[1.2.12,)'
+    compile 'org.opencadc:cadc-uws-server:[1.2.19,)'
     compile 'org.opencadc:cadc-access-control:[1.1.1,2.0)'
     compile 'org.opencadc:cadc-cdp:[1.2.3,)'
-    compile 'org.opencadc:cadc-registry:[1.5.15,)'
+    compile 'org.opencadc:cadc-registry:[1.7.4,)'
     compile 'org.opencadc:cadc-inventory:[0.9.4,1.0)'
     compile 'org.opencadc:cadc-inventory-db:[0.15.0,1.0)'
 

vault/src/intTest/java/org/opencadc/vault/NodesTest.java

@@ -67,6 +67,7 @@
 
 package org.opencadc.vault;
 
+import ca.nrc.cadc.util.FileUtil;
 import ca.nrc.cadc.util.Log4jInit;
 import java.net.URI;
 import org.apache.log4j.Level;
@@ -87,6 +88,10 @@ public class NodesTest extends org.opencadc.conformance.vos.NodesTest {
     }
 
     public NodesTest() {
-        super(URI.create("ivo://opencadc.org/vault"), "vault-test.pem", new GroupURI(URI.create("ivo://cadc.nrc.ca/gms?CADC_TEST_GROUP2")), "vault-test-auth.pem");
+        super(URI.create("ivo://opencadc.org/vault"), "vault-test.pem");
+        enablePermissionTests(new GroupURI(URI.create("ivo://cadc.nrc.ca/gms?opencadc-vospace-test")),
+                FileUtil.getFileFromResource("vault-auth-test.pem", NodesTest.class));
+        // vault does not check the actual groups in the permission props tests, hence they can be made up.
+        enablePermissionPropsTest(new GroupURI(URI.create("ivo://myauth/gms?gr1")), new GroupURI(URI.create("ivo://myauth/gms?gr2")));
     }
 }

vault/src/main/java/org/opencadc/vault/NodePersistenceImpl.java

@@ -99,13 +99,14 @@
 import org.opencadc.inventory.db.SQLGenerator;
 import org.opencadc.vospace.ContainerNode;
 import org.opencadc.vospace.DataNode;
-import org.opencadc.vospace.LinkNode;
 import org.opencadc.vospace.Node;
 import org.opencadc.vospace.NodeNotSupportedException;
 import org.opencadc.vospace.NodeProperty;
 import org.opencadc.vospace.VOS;
 import org.opencadc.vospace.db.NodeDAO;
 import org.opencadc.vospace.server.NodePersistence;
+import org.opencadc.vospace.server.Views;
+import org.opencadc.vospace.server.transfers.TransferGenerator;
 
 /**
  *
@@ -243,6 +244,16 @@ public Set<URI> getImmutableProps() {
         return immutableProps;
     }
 
+    @Override
+    public Views getViews() {
+        throw new UnsupportedOperationException("TODO");
+    }
+
+    @Override
+    public TransferGenerator getTransferGenerator() {
+        throw new UnsupportedOperationException("TODO");
+    }
+
     /**
      * Get a node by name. Concept: The caller uses this to navigate the path 
      * from the root node to the target, checking permissions and deciding what 

vault/src/main/java/org/opencadc/vault/RecursiveDeleteNodeJobManager.java

@@ -0,0 +1,113 @@
+/*
+************************************************************************
+*******************  CANADIAN ASTRONOMY DATA CENTRE  *******************
+**************  CENTRE CANADIEN DE DONNÉES ASTRONOMIQUES  **************
+*
+*  (c) 2023.                            (c) 2023.
+*  Government of Canada                 Gouvernement du Canada
+*  National Research Council            Conseil national de recherches
+*  Ottawa, Canada, K1A 0R6              Ottawa, Canada, K1A 0R6
+*  All rights reserved                  Tous droits réservés
+*
+*  NRC disclaims any warranties,        Le CNRC dénie toute garantie
+*  expressed, implied, or               énoncée, implicite ou légale,
+*  statutory, of any kind with          de quelque nature que ce
+*  respect to the software,             soit, concernant le logiciel,
+*  including without limitation         y compris sans restriction
+*  any warranty of merchantability      toute garantie de valeur
+*  or fitness for a particular          marchande ou de pertinence
+*  purpose. NRC shall not be            pour un usage particulier.
+*  liable in any event for any          Le CNRC ne pourra en aucun cas
+*  damages, whether direct or           être tenu responsable de tout
+*  indirect, special or general,        dommage, direct ou indirect,
+*  consequential or incidental,         particulier ou général,
+*  arising from the use of the          accessoire ou fortuit, résultant
+*  software.  Neither the name          de l'utilisation du logiciel. Ni
+*  of the National Research             le nom du Conseil National de
+*  Council of Canada nor the            Recherches du Canada ni les noms
+*  names of its contributors may        de ses  participants ne peuvent
+*  be used to endorse or promote        être utilisés pour approuver ou
+*  products derived from this           promouvoir les produits dérivés
+*  software without specific prior      de ce logiciel sans autorisation
+*  written permission.                  préalable et particulière
+*                                       par écrit.
+*
+*  This file is part of the             Ce fichier fait partie du projet
+*  OpenCADC project.                    OpenCADC.
+*
+*  OpenCADC is free software:           OpenCADC est un logiciel libre ;
+*  you can redistribute it and/or       vous pouvez le redistribuer ou le
+*  modify it under the terms of         modifier suivant les termes de
+*  the GNU Affero General Public        la “GNU Affero General Public
+*  License as published by the          License” telle que publiée
+*  Free Software Foundation,            par la Free Software Foundation
+*  either version 3 of the              : soit la version 3 de cette
+*  License, or (at your option)         licence, soit (à votre gré)
+*  any later version.                   toute version ultérieure.
+*
+*  OpenCADC is distributed in the       OpenCADC est distribué
+*  hope that it will be useful,         dans l’espoir qu’il vous
+*  but WITHOUT ANY WARRANTY;            sera utile, mais SANS AUCUNE
+*  without even the implied             GARANTIE : sans même la garantie
+*  warranty of MERCHANTABILITY          implicite de COMMERCIALISABILITÉ
+*  or FITNESS FOR A PARTICULAR          ni d’ADÉQUATION À UN OBJECTIF
+*  PURPOSE.  See the GNU Affero         PARTICULIER. Consultez la Licence
+*  General Public License for           Générale Publique GNU Affero
+*  more details.                        pour plus de détails.
+*
+*  You should have received             Vous devriez avoir reçu une
+*  a copy of the GNU Affero             copie de la Licence Générale
+*  General Public License along         Publique GNU Affero avec
+*  with OpenCADC.  If not, see          OpenCADC ; si ce n’est
+*  <http://www.gnu.org/licenses/>.      pas le cas, consultez :
+*                                       <http://www.gnu.org/licenses/>.
+*
+*  $Revision: 4 $
+*
+************************************************************************
+*/
+
+package org.opencadc.vault;
+
+import ca.nrc.cadc.auth.AuthenticationUtil;
+import ca.nrc.cadc.uws.server.JobExecutor;
+import ca.nrc.cadc.uws.server.JobPersistence;
+import ca.nrc.cadc.uws.server.JobUpdater;
+import ca.nrc.cadc.uws.server.SimpleJobManager;
+import ca.nrc.cadc.uws.server.ThreadPoolExecutor;
+import ca.nrc.cadc.uws.server.impl.PostgresJobPersistence;
+import org.apache.log4j.Logger;
+import org.opencadc.vospace.server.async.RecursiveDeleteNodeRunner;
+
+/**
+ *
+ * @author pdowler, majorb, yeunga, adriand
+ */
+public class RecursiveDeleteNodeJobManager extends SimpleJobManager {
+    private static final Logger log = Logger.getLogger(RecursiveDeleteNodeJobManager.class);
+
+    private static final Long MAX_EXEC_DURATION = Long.valueOf(12 * 7200L); // 24 hours?
+    private static final Long MAX_DESTRUCTION = Long.valueOf(7 * 24 * 3600L); // 1 week
+    private static final Long MAX_QUOTE = Long.valueOf(12 * 7200L); // same as exec
+
+    protected static JobPersistence jp;
+
+    static {
+        log.info("Creating shared (postgres) job manager");
+        jp = new PostgresJobPersistence(AuthenticationUtil.getIdentityManager());
+    }
+
+    public RecursiveDeleteNodeJobManager() {
+        super();
+        // jp is instantiated in parent org.opencadc.cavern.JobManager
+        JobUpdater ju = jp;
+        super.setJobPersistence(jp);
+
+        JobExecutor jobExec = new ThreadPoolExecutor(ju, RecursiveDeleteNodeRunner.class, 3);
+        super.setJobExecutor(jobExec);
+
+        super.setMaxExecDuration(MAX_EXEC_DURATION);
+        super.setMaxDestruction(MAX_DESTRUCTION);
+        super.setMaxQuote(MAX_QUOTE);
+    }
+}

vault/src/main/java/org/opencadc/vault/VaultInitAction.java

@@ -71,6 +71,7 @@
 import ca.nrc.cadc.rest.InitAction;
 import ca.nrc.cadc.util.MultiValuedProperties;
 import ca.nrc.cadc.util.PropertiesReader;
+import ca.nrc.cadc.uws.server.impl.InitDatabaseUWS;
 import java.net.URI;
 import java.net.URISyntaxException;
 import java.util.Map;
@@ -93,6 +94,7 @@ public class VaultInitAction extends InitAction {
     private static final Logger log = Logger.getLogger(VaultInitAction.class);
 
     static final String JNDI_DATASOURCE = "jdbc/nodes"; // context.xml
+    static final String JNDI_UWS_DATASOURCE = "jdbc/uws"; // context.xml
 
     // config keys
     private static final String VAULT_KEY = "org.opencadc.vault";
@@ -119,6 +121,7 @@ public VaultInitAction() {
     public void doInit() {
         initConfig();
         initDatabase();
+        initUWSDatabase();
         initNodePersistence();
     }
 
@@ -215,8 +218,22 @@ private void initDatabase() {
         }
     }
 
+    private void initUWSDatabase() {
+        log.info("initUWSDatabase: START");
+        try {
+            // Init UWS database
+            DataSource uws = DBUtil.findJNDIDataSource(JNDI_UWS_DATASOURCE);
+            InitDatabaseUWS uwsi = new InitDatabaseUWS(uws, null, "uws");
+            uwsi.doInit();
+            log.info("initDatabase: " + JNDI_UWS_DATASOURCE + " uws OK");
+
+        } catch (Exception ex) {
+            throw new RuntimeException("check/init uws database failed", ex);
+        }
+    }
+
     protected void initNodePersistence() {
-        jndiNodePersistence = componentID + ".nodePersistence";
+        jndiNodePersistence = appName + "-" + NodePersistence.class.getName();
         try {
             Context ctx = new InitialContext();
             try {

vault/src/main/resources/VOSpacePlugins.properties

@@ -0,0 +1,12 @@
+#
+# Configuration information for the vospace storage interface
+#
+
+# Define the class that implements the interface TransferGenerator
+# This class will be loaded at runtime to handle transfer requests.
+# ca.nrc.cadc.vos.transfers.TransferGenerator = <full class name of implementing interface>
+ca.nrc.cadc.vos.server.transfers.TransferGenerator = ca.nrc.cadc.vospace.transfers.ADTransferGenerator
+
+# Define an optional class that implements the interface NodePersistence
+# This class will be loaded at runtime to node persistence operations.
+ca.nrc.cadc.vos.server.NodePersistence = ca.nrc.cadc.vospace.VOSpaceNodePersistence

vault/src/main/resources/Views.properties

@@ -0,0 +1,72 @@
+###############################################################################
+#
+#  Views.properties
+#
+#  Defines the views available in this instance of VOSpace.  These are loaded
+#  upon the loading of class ca.nrc.cadc.vos.ViewFactory into the VM.
+# 
+#  The names of the views to be defined must be listed under the 'views' key
+#  and separated by a space.  For example:
+#      views = <viewName1> <viewName2> ... <viewNameN>
+#
+#  For each view name listed, three settings must be defined
+#      uri = The URI of the view
+#	   alias = The alias (or shortcut) name of the view
+#      class = The implementing class of the view
+#
+#  Additionally, two optional settings may be defined:
+#      accepts = true/false, true if this services accepts the view
+#      provides = true/false, true if this service provides the view
+#
+#  For example:
+#      <viewName1>.uri = ivo://cadc.nrc.ca/vospace/core#dataview
+#      <viewName1>.alias = data
+#      <viewName1>.class = ca.nrc.cadc.vos.DataView
+#      <viewName1>.accepts = false
+#      <viewName1>.provides = true
+#
+#  Notes:
+#      - View classes must extend class ca.nrc.cadc.vos.AbstractView
+#      - There cannot be a duplicate of any aliases or URIs in any of the
+#        view definitions
+#
+###############################################################################
+
+#views = data rss manifest cutout header
+views = data
+
+# data view definition
+data.uri = ivo://cadc.nrc.ca/vospace/view#data
+data.alias = data
+data.class = org.opencadc.vospace.server.DataView
+data.accepts = false
+data.provides = true
+
+# rss view definition
+rss.uri = ivo://cadc.nrc.ca/vospace/view#rss
+rss.alias = rss
+rss.class = ca.nrc.cadc.vos.server.RssView
+rss.accepts = false
+rss.provides = true
+
+# manifest view definition
+manifest.uri = ivo://cadc.nrc.ca/vospace/view#manifest
+manifest.alias = manifest
+manifest.class = ca.nrc.cadc.vos.server.ManifestView
+manifest.accepts = false
+manifest.provides = true
+
+# cutout view definition
+cutout.uri = ivo://cadc.nrc.ca/vospace/view#cutout
+cutout.alias = cutout
+cutout.class = ca.nrc.cadc.vospace.CutoutView
+cutout.accepts = false
+cutout.provides = true
+
+# header view definition
+header.uri = ivo://cadc.nrc.ca/vospace/view#header
+header.alias = header
+header.class = ca.nrc.cadc.vospace.HeaderView
+header.accepts = false
+header.provides = true
+

vault/src/main/webapp/META-INF/context.xml

@@ -15,4 +15,17 @@
         removeAbandoned="false"
         testOnBorrow="true" validationQuery="select 123" />
 
+
+    <Resource name="jdbc/uws"
+        auth="Container"
+        type="javax.sql.DataSource"
+        factory="org.apache.tomcat.jdbc.pool.DataSourceFactory" closeMethod="close"
+        minEvictableIdleTimeMillis="60000" timeBetweenEvictionRunsMillis="30000"
+        maxWait="20000"
+        initialSize="0" minIdle="0" maxIdle="${org.opencadc.vault.uws.maxActive}" maxActive="${org.opencadc.vault.uws.maxActive}"
+        username="${org.opencadc.vault.uws.username}" password="${org.opencadc.vault.uws.password}"
+        driverClassName="org.postgresql.Driver" url="${org.opencadc.vault.uws.url}"
+        removeAbandoned="false"
+        testOnBorrow="true" validationQuery="select 123" />
+
 </Context>

vault/src/main/webapp/WEB-INF/web.xml

@@ -23,6 +23,7 @@
           ca.nrc.cadc.rest
           ca.nrc.cadc.util
           ca.nrc.cadc.vosi
+          ca.nrc.cadc.uws
       </param-value>
     </init-param>
     <init-param>
@@ -62,6 +63,30 @@
     <load-on-startup>2</load-on-startup>
   </servlet>
 
+  <!-- async recursive node property setting servlet-->
+  <servlet>
+    <servlet-name>RecursiveDeleteNodeServlet</servlet-name>
+    <servlet-class>ca.nrc.cadc.uws.server.JobServlet</servlet-class>
+    <init-param>
+      <param-name>get</param-name>
+      <param-value>ca.nrc.cadc.uws.web.GetAction</param-value>
+    </init-param>
+    <init-param>
+      <param-name>post</param-name>
+      <param-value>ca.nrc.cadc.uws.web.PostAction</param-value>
+    </init-param>
+    <init-param>
+      <param-name>delete</param-name>
+      <param-value>ca.nrc.cadc.uws.web.DeleteAction</param-value>
+    </init-param>
+    <init-param>
+      <param-name>ca.nrc.cadc.uws.server.JobManager</param-name>
+      <param-value>org.opencadc.vault.RecursiveDeleteNodeJobManager</param-value>
+    </init-param>
+    <load-on-startup>3</load-on-startup>
+  </servlet>
+
+
   <!-- VOSI capabilities -->
   <servlet>
     <servlet-name>CapabilitiesServlet</servlet-name>
@@ -107,6 +132,11 @@
     <url-pattern>/nodes/*</url-pattern>
   </servlet-mapping>
 
+  <servlet-mapping>
+    <servlet-name>RecursiveDeleteNodeServlet</servlet-name>
+    <url-pattern>/recursiveDelete/*</url-pattern>
+  </servlet-mapping>
+
   <!-- Availability servlet endpoint -->
   <servlet-mapping>
     <servlet-name>AvailabilityServlet</servlet-name>