From 2556e702c3aac476c1c4b5df16748c67bbdb02b8 Mon Sep 17 00:00:00 2001 From: Patrick Dowler Date: Tue, 28 Mar 2023 17:51:45 -0700 Subject: [PATCH 01/52] incomplete exploratory dev --- cadc-inventory-db/build.gradle | 1 + .../org/opencadc/inventory/db/TestUtil.java | 16 +- .../org/opencadc/vospace/db/NodeDAOTest.java | 199 ++++++++++++++++++ .../opencadc/inventory/db/AbstractDAO.java | 8 +- .../opencadc/inventory/db/SQLGenerator.java | 54 ++++- .../opencadc/vospace/db/InitDatabaseVOS.java | 113 ++++++++++ .../java/org/opencadc/vospace/db/NodeDAO.java | 100 +++++++++ .../main/resources/vos.DeletedNodeEvent.sql | 17 ++ .../src/main/resources/vos.ModelVersion.sql | 9 + .../src/main/resources/vos.Node.sql | 31 +++ 10 files changed, 538 insertions(+), 10 deletions(-) create mode 100644 cadc-inventory-db/src/intTest/java/org/opencadc/vospace/db/NodeDAOTest.java create mode 100644 cadc-inventory-db/src/main/java/org/opencadc/vospace/db/InitDatabaseVOS.java create mode 100644 cadc-inventory-db/src/main/java/org/opencadc/vospace/db/NodeDAO.java create mode 100644 cadc-inventory-db/src/main/resources/vos.DeletedNodeEvent.sql create mode 100644 cadc-inventory-db/src/main/resources/vos.ModelVersion.sql create mode 100644 cadc-inventory-db/src/main/resources/vos.Node.sql diff --git a/cadc-inventory-db/build.gradle b/cadc-inventory-db/build.gradle index ff0e2012b..c1352fd70 100644 --- a/cadc-inventory-db/build.gradle +++ b/cadc-inventory-db/build.gradle @@ -27,6 +27,7 @@ mainClassName = 'org.opencadc.inventory.db.version.Main' dependencies { compile 'org.opencadc:cadc-util:[1.6.2,2.0)' compile 'org.opencadc:cadc-inventory:[0.9,)' + compile 'org.opencadc:cadc-vos:[2.0,3.0)' testCompile 'junit:junit:[4.0,)' diff --git a/cadc-inventory-db/src/intTest/java/org/opencadc/inventory/db/TestUtil.java b/cadc-inventory-db/src/intTest/java/org/opencadc/inventory/db/TestUtil.java index 92ba24bcc..8ab42bf71 100644 --- a/cadc-inventory-db/src/intTest/java/org/opencadc/inventory/db/TestUtil.java +++ b/cadc-inventory-db/src/intTest/java/org/opencadc/inventory/db/TestUtil.java @@ -79,10 +79,11 @@ public class TestUtil { private static final Logger log = Logger.getLogger(TestUtil.class); - static String SERVER = "INVENTORY_TEST"; - static String DATABASE = "cadctest"; - static String SCHEMA = "inventory"; - static String TABLE_PREFIX = null; + public static String SERVER = "INVENTORY_TEST"; + public static String DATABASE = "cadctest"; + public static String SCHEMA = "inventory"; + public static String VOS_SCHEMA = "vospace"; + public static String TABLE_PREFIX = null; static { try { @@ -102,12 +103,17 @@ public class TestUtil { if (s != null) { SCHEMA = s.trim(); } + s = props.getProperty("vos_schema"); + if (s != null) { + VOS_SCHEMA = s.trim(); + } s = props.getProperty("tablePrefix"); if (s != null) { TABLE_PREFIX = s.trim(); } } - log.info("intTest database config: " + SERVER + " " + DATABASE + " " + SCHEMA + " " + TABLE_PREFIX); + log.info("intTest database config: " + SERVER + " " + DATABASE + " " + SCHEMA + " " + VOS_SCHEMA + + " tablePrefix=" + TABLE_PREFIX); } catch (Exception oops) { log.debug("failed to load/read optional db config", oops); } diff --git a/cadc-inventory-db/src/intTest/java/org/opencadc/vospace/db/NodeDAOTest.java b/cadc-inventory-db/src/intTest/java/org/opencadc/vospace/db/NodeDAOTest.java new file mode 100644 index 000000000..ccf475b6d --- /dev/null +++ b/cadc-inventory-db/src/intTest/java/org/opencadc/vospace/db/NodeDAOTest.java @@ -0,0 +1,199 @@ +/* +************************************************************************ +******************* CANADIAN ASTRONOMY DATA CENTRE ******************* +************** CENTRE CANADIEN DE DONNÉES ASTRONOMIQUES ************** +* +* (c) 2022. (c) 2022. +* Government of Canada Gouvernement du Canada +* National Research Council Conseil national de recherches +* Ottawa, Canada, K1A 0R6 Ottawa, Canada, K1A 0R6 +* All rights reserved Tous droits réservés +* +* NRC disclaims any warranties, Le CNRC dénie toute garantie +* expressed, implied, or énoncée, implicite ou légale, +* statutory, of any kind with de quelque nature que ce +* respect to the software, soit, concernant le logiciel, +* including without limitation y compris sans restriction +* any warranty of merchantability toute garantie de valeur +* or fitness for a particular marchande ou de pertinence +* purpose. NRC shall not be pour un usage particulier. +* liable in any event for any Le CNRC ne pourra en aucun cas +* damages, whether direct or être tenu responsable de tout +* indirect, special or general, dommage, direct ou indirect, +* consequential or incidental, particulier ou général, +* arising from the use of the accessoire ou fortuit, résultant +* software. Neither the name de l'utilisation du logiciel. Ni +* of the National Research le nom du Conseil National de +* Council of Canada nor the Recherches du Canada ni les noms +* names of its contributors may de ses participants ne peuvent +* be used to endorse or promote être utilisés pour approuver ou +* products derived from this promouvoir les produits dérivés +* software without specific prior de ce logiciel sans autorisation +* written permission. préalable et particulière +* par écrit. +* +* This file is part of the Ce fichier fait partie du projet +* OpenCADC project. OpenCADC. +* +* OpenCADC is free software: OpenCADC est un logiciel libre ; +* you can redistribute it and/or vous pouvez le redistribuer ou le +* modify it under the terms of modifier suivant les termes de +* the GNU Affero General Public la “GNU Affero General Public +* License as published by the License” telle que publiée +* Free Software Foundation, par la Free Software Foundation +* either version 3 of the : soit la version 3 de cette +* License, or (at your option) licence, soit (à votre gré) +* any later version. toute version ultérieure. +* +* OpenCADC is distributed in the OpenCADC est distribué +* hope that it will be useful, dans l’espoir qu’il vous +* but WITHOUT ANY WARRANTY; sera utile, mais SANS AUCUNE +* without even the implied GARANTIE : sans même la garantie +* warranty of MERCHANTABILITY implicite de COMMERCIALISABILITÉ +* or FITNESS FOR A PARTICULAR ni d’ADÉQUATION À UN OBJECTIF +* PURPOSE. See the GNU Affero PARTICULIER. Consultez la Licence +* General Public License for Générale Publique GNU Affero +* more details. pour plus de détails. +* +* You should have received Vous devriez avoir reçu une +* a copy of the GNU Affero copie de la Licence Générale +* General Public License along Publique GNU Affero avec +* with OpenCADC. If not, see OpenCADC ; si ce n’est +* . pas le cas, consultez : +* . +* +************************************************************************ +*/ + +package org.opencadc.vospace.db; + +import ca.nrc.cadc.db.ConnectionConfig; +import ca.nrc.cadc.db.DBConfig; +import ca.nrc.cadc.db.DBUtil; +import ca.nrc.cadc.util.Log4jInit; +import java.net.URI; +import java.util.Map; +import java.util.TreeMap; +import java.util.UUID; +import javax.sql.DataSource; +import org.apache.log4j.Level; +import org.apache.log4j.Logger; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.opencadc.inventory.db.SQLGenerator; +import org.opencadc.inventory.db.TestUtil; +import org.opencadc.vospace.db.InitDatabaseVOS; +import org.opencadc.vospace.ContainerNode; +import org.opencadc.vospace.DataNode; +import org.opencadc.vospace.LinkNode; +import org.opencadc.vospace.Node; + +/** + * + * @author pdowler + */ +public class NodeDAOTest { + private static final Logger log = Logger.getLogger(NodeDAOTest.class); + + static { + Log4jInit.setLevel("org.opencadc.inventory", Level.INFO); + Log4jInit.setLevel("org.opencadc.inventory.db", Level.INFO); + Log4jInit.setLevel("ca.nrc.cadc.db", Level.INFO); + Log4jInit.setLevel("org.opencadc.vospace", Level.INFO); + } + + NodeDAO nodeDAO; + + public NodeDAOTest() throws Exception { + try { + DBConfig dbrc = new DBConfig(); + ConnectionConfig cc = dbrc.getConnectionConfig(TestUtil.SERVER, TestUtil.DATABASE); + DBUtil.createJNDIDataSource("jdbc/ArtifactDAOTest", cc); + + Map config = new TreeMap(); + config.put(SQLGenerator.class.getName(), SQLGenerator.class); + config.put("jndiDataSourceName", "jdbc/ArtifactDAOTest"); + config.put("database", TestUtil.DATABASE); + config.put("schema", TestUtil.SCHEMA); + config.put("vosSchema", TestUtil.VOS_SCHEMA); + + this.nodeDAO = new NodeDAO(); + nodeDAO.setConfig(config); + + } catch (Exception ex) { + log.error("setup failed", ex); + throw ex; + } + } + + @Before + public void init_cleanup() throws Exception { + log.info("init database..."); + InitDatabaseVOS init = new InitDatabaseVOS(nodeDAO.getDataSource(), TestUtil.DATABASE, TestUtil.VOS_SCHEMA); + init.doInit(); + log.info("init database... OK"); + + log.info("clearing old content..."); + SQLGenerator gen = nodeDAO.getSQLGenerator(); + DataSource ds = nodeDAO.getDataSource(); + String sql = "delete from " + gen.getTable(Node.class); + log.info("pre-test cleanup: " + sql); + ds.getConnection().createStatement().execute(sql); + log.info("clearing old content... OK"); + } + + @Test + public void testGetByID() { + UUID id = UUID.randomUUID(); + Node a = nodeDAO.get(id); + Assert.assertNull(a); + } + + //@Test + public void testPutGetDeleteContainerNode() { + ContainerNode root = new ContainerNode("root", false); + + ContainerNode n = new ContainerNode("container-test", false); + n.parent = root; + nodeDAO.put(n); + + Node a = nodeDAO.get(n.getID()); + Assert.assertNotNull(a); + Assert.assertTrue(a instanceof ContainerNode); + ContainerNode c = (ContainerNode) a; + Assert.assertEquals(n.getName(), a.getName()); + // these are set in put + Assert.assertEquals(n.getMetaChecksum(), a.getMetaChecksum()); + Assert.assertEquals(n.getLastModified(), a.getLastModified()); + ContainerNode ac = nodeDAO.getChildren(c); + Assert.assertNotNull(ac); + Assert.assertTrue(ac.nodes.isEmpty()); + + // add children + ContainerNode cont = new ContainerNode("container1", false); + cont.parent = c; + DataNode data = new DataNode("data1"); + data.parent = c; + LinkNode link = new LinkNode("link1", URI.create("cadc:ARCHIVE/data")); + link.parent = c; + nodeDAO.put(cont); + nodeDAO.put(data); + nodeDAO.put(link); + + ac = nodeDAO.getChildren(c); + Assert.assertNotNull(ac); + Assert.assertFalse(ac.nodes.isEmpty()); + Assert.assertEquals(3, ac.nodes.size()); + } + + @Test + public void testPutGetDeleteDataNode() { + log.info("TODO"); + } + + @Test + public void testPutGetDeleteLinkNode() { + log.info("TODO"); + } +} diff --git a/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/AbstractDAO.java b/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/AbstractDAO.java index 7b823e304..fe8533e75 100644 --- a/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/AbstractDAO.java +++ b/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/AbstractDAO.java @@ -168,7 +168,7 @@ public DataSource getDataSource() { return dataSource; } - SQLGenerator getSQLGenerator() { + public SQLGenerator getSQLGenerator() { checkInit(); return gen; } @@ -191,6 +191,7 @@ public Map getParams() { ret.put("jndiDataSourceName", String.class); ret.put("database", String.class); ret.put("schema", String.class); + ret.put("vosSchema", String.class); // optional ret.put(SQLGenerator.class.getName(), Class.class); return ret; } @@ -224,9 +225,10 @@ public void setConfig(Map config) { String database = (String) config.get("database"); String schema = (String) config.get("schema"); + String vosSchema = (String) config.get("vosSchema"); try { - Constructor ctor = genClass.getConstructor(String.class, String.class); - this.gen = (SQLGenerator) ctor.newInstance(database, schema); + Constructor ctor = genClass.getConstructor(String.class, String.class, String.class); + this.gen = (SQLGenerator) ctor.newInstance(database, schema, vosSchema); } catch (Exception ex) { throw new RuntimeException("failed to instantiate SQLGenerator: " + genClass.getName(), ex); } diff --git a/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/SQLGenerator.java b/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/SQLGenerator.java index 15a34d7b7..172472704 100644 --- a/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/SQLGenerator.java +++ b/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/SQLGenerator.java @@ -98,6 +98,8 @@ import org.opencadc.inventory.StorageLocation; import org.opencadc.inventory.StorageLocationEvent; import org.opencadc.inventory.StorageSite; +import org.opencadc.vospace.DeletedNodeEvent; +import org.opencadc.vospace.Node; import org.springframework.dao.DataAccessException; import org.springframework.jdbc.core.JdbcTemplate; import org.springframework.jdbc.core.ResultSetExtractor; @@ -115,6 +117,7 @@ public class SQLGenerator { protected final String database; // currently not used in SQL protected final String schema; // may be null + protected final String vosSchema; /** * Constructor. The database name is currently not used in any generated SQL; code assumes @@ -128,8 +131,13 @@ public class SQLGenerator { * @param schema schema name (may be null) */ public SQLGenerator(String database, String schema) { + this(database, schema, null); + } + + public SQLGenerator(String database, String schema, String vosSchema) { this.database = database; this.schema = schema; + this.vosSchema = vosSchema; init(); } @@ -202,6 +210,42 @@ protected void init() { "id" // last column is always PK }; this.columnMap.put(HarvestState.class, cols); + + // optional vospace + log.warn("vosSchema: " + vosSchema); + if (vosSchema != null) { + pref = vosSchema + "."; + tableMap.put(Node.class, pref + Node.class.getSimpleName()); + tableMap.put(DeletedNodeEvent.class, pref + DeletedNodeEvent.class.getSimpleName()); + + cols = new String[] { + "parentID", + "name", + "nodeType", + "ownerID", + "isPublic", + "isLocked", + "readOnlyGroups", + "readWriteGroups", + "properties", + "busyState", + "storageID", + "target", + "lastModified", + "metaChecksum", + "id" // last column is always PK + }; + this.columnMap.put(Node.class, cols); + + cols = new String[] { + "nodeType", + "storageID", + "lastModified", + "metaChecksum", + "id" // last column is always PK + }; + this.columnMap.put(DeletedNodeEvent.class, cols); + } } private static class ClassComp implements Comparator { @@ -217,8 +261,7 @@ public String getCurrentTimeSQL() { return "SELECT now()"; } - // test usage - String getTable(Class c) { + public String getTable(Class c) { return tableMap.get(c); } @@ -253,6 +296,13 @@ public EntityGet getEntityGet(Class c, boolean forUpdate) { if (HarvestState.class.equals(c)) { return new HarvestStateGet(); } + + if (Node.class.equals(c)) { + + } + if (DeletedNodeEvent.class.equals(c)) { + + } throw new UnsupportedOperationException("entity-get: " + c.getName()); } diff --git a/cadc-inventory-db/src/main/java/org/opencadc/vospace/db/InitDatabaseVOS.java b/cadc-inventory-db/src/main/java/org/opencadc/vospace/db/InitDatabaseVOS.java new file mode 100644 index 000000000..093618755 --- /dev/null +++ b/cadc-inventory-db/src/main/java/org/opencadc/vospace/db/InitDatabaseVOS.java @@ -0,0 +1,113 @@ +/* +************************************************************************ +******************* CANADIAN ASTRONOMY DATA CENTRE ******************* +************** CENTRE CANADIEN DE DONNÉES ASTRONOMIQUES ************** +* +* (c) 2023. (c) 2023. +* Government of Canada Gouvernement du Canada +* National Research Council Conseil national de recherches +* Ottawa, Canada, K1A 0R6 Ottawa, Canada, K1A 0R6 +* All rights reserved Tous droits réservés +* +* NRC disclaims any warranties, Le CNRC dénie toute garantie +* expressed, implied, or énoncée, implicite ou légale, +* statutory, of any kind with de quelque nature que ce +* respect to the software, soit, concernant le logiciel, +* including without limitation y compris sans restriction +* any warranty of merchantability toute garantie de valeur +* or fitness for a particular marchande ou de pertinence +* purpose. NRC shall not be pour un usage particulier. +* liable in any event for any Le CNRC ne pourra en aucun cas +* damages, whether direct or être tenu responsable de tout +* indirect, special or general, dommage, direct ou indirect, +* consequential or incidental, particulier ou général, +* arising from the use of the accessoire ou fortuit, résultant +* software. Neither the name de l'utilisation du logiciel. Ni +* of the National Research le nom du Conseil National de +* Council of Canada nor the Recherches du Canada ni les noms +* names of its contributors may de ses participants ne peuvent +* be used to endorse or promote être utilisés pour approuver ou +* products derived from this promouvoir les produits dérivés +* software without specific prior de ce logiciel sans autorisation +* written permission. préalable et particulière +* par écrit. +* +* This file is part of the Ce fichier fait partie du projet +* OpenCADC project. OpenCADC. +* +* OpenCADC is free software: OpenCADC est un logiciel libre ; +* you can redistribute it and/or vous pouvez le redistribuer ou le +* modify it under the terms of modifier suivant les termes de +* the GNU Affero General Public la “GNU Affero General Public +* License as published by the License” telle que publiée +* Free Software Foundation, par la Free Software Foundation +* either version 3 of the : soit la version 3 de cette +* License, or (at your option) licence, soit (à votre gré) +* any later version. toute version ultérieure. +* +* OpenCADC is distributed in the OpenCADC est distribué +* hope that it will be useful, dans l’espoir qu’il vous +* but WITHOUT ANY WARRANTY; sera utile, mais SANS AUCUNE +* without even the implied GARANTIE : sans même la garantie +* warranty of MERCHANTABILITY implicite de COMMERCIALISABILITÉ +* or FITNESS FOR A PARTICULAR ni d’ADÉQUATION À UN OBJECTIF +* PURPOSE. See the GNU Affero PARTICULIER. Consultez la Licence +* General Public License for Générale Publique GNU Affero +* more details. pour plus de détails. +* +* You should have received Vous devriez avoir reçu une +* a copy of the GNU Affero copie de la Licence Générale +* General Public License along Publique GNU Affero avec +* with OpenCADC. If not, see OpenCADC ; si ce n’est +* . pas le cas, consultez : +* . +* +************************************************************************ +*/ + +package org.opencadc.vospace.db; + +import java.net.URL; +import javax.sql.DataSource; +import org.apache.log4j.Logger; +import org.opencadc.inventory.db.version.InitDatabase; + +/** + * + * @author pdowler + */ +public class InitDatabaseVOS extends ca.nrc.cadc.db.version.InitDatabase { + private static final Logger log = Logger.getLogger(InitDatabaseVOS.class); + + public static final String MODEL_NAME = "vospace-inventory"; + public static final String MODEL_VERSION = "0.1"; + public static final String PREV_MODEL_VERSION = "n/a"; + //public static final String PREV_MODEL_VERSION = "DO-NOT_UPGRADE-BY-ACCIDENT"; + + static String[] CREATE_SQL = new String[] { + "vos.ModelVersion.sql", + "vos.Node.sql", + "vos.DeletedNodeEvent.sql", + "inventory.permissions.sql" + }; + + static String[] UPGRADE_SQL = new String[] { + "inventory.permissions.sql" + }; + + public InitDatabaseVOS(DataSource ds, String database, String schema) { + super(ds, database, schema, MODEL_NAME, MODEL_VERSION, PREV_MODEL_VERSION); + for (String s : CREATE_SQL) { + createSQL.add(s); + } + for (String s : UPGRADE_SQL) { + upgradeSQL.add(s); + } + } + + @Override + protected URL findSQL(String fname) { + // SQL files are stored inside the jar file + return InitDatabase.class.getClassLoader().getResource(fname); + } +} diff --git a/cadc-inventory-db/src/main/java/org/opencadc/vospace/db/NodeDAO.java b/cadc-inventory-db/src/main/java/org/opencadc/vospace/db/NodeDAO.java new file mode 100644 index 000000000..1d16fa40a --- /dev/null +++ b/cadc-inventory-db/src/main/java/org/opencadc/vospace/db/NodeDAO.java @@ -0,0 +1,100 @@ +/* +************************************************************************ +******************* CANADIAN ASTRONOMY DATA CENTRE ******************* +************** CENTRE CANADIEN DE DONNÉES ASTRONOMIQUES ************** +* +* (c) 2022. (c) 2022. +* Government of Canada Gouvernement du Canada +* National Research Council Conseil national de recherches +* Ottawa, Canada, K1A 0R6 Ottawa, Canada, K1A 0R6 +* All rights reserved Tous droits réservés +* +* NRC disclaims any warranties, Le CNRC dénie toute garantie +* expressed, implied, or énoncée, implicite ou légale, +* statutory, of any kind with de quelque nature que ce +* respect to the software, soit, concernant le logiciel, +* including without limitation y compris sans restriction +* any warranty of merchantability toute garantie de valeur +* or fitness for a particular marchande ou de pertinence +* purpose. NRC shall not be pour un usage particulier. +* liable in any event for any Le CNRC ne pourra en aucun cas +* damages, whether direct or être tenu responsable de tout +* indirect, special or general, dommage, direct ou indirect, +* consequential or incidental, particulier ou général, +* arising from the use of the accessoire ou fortuit, résultant +* software. Neither the name de l'utilisation du logiciel. Ni +* of the National Research le nom du Conseil National de +* Council of Canada nor the Recherches du Canada ni les noms +* names of its contributors may de ses participants ne peuvent +* be used to endorse or promote être utilisés pour approuver ou +* products derived from this promouvoir les produits dérivés +* software without specific prior de ce logiciel sans autorisation +* written permission. préalable et particulière +* par écrit. +* +* This file is part of the Ce fichier fait partie du projet +* OpenCADC project. OpenCADC. +* +* OpenCADC is free software: OpenCADC est un logiciel libre ; +* you can redistribute it and/or vous pouvez le redistribuer ou le +* modify it under the terms of modifier suivant les termes de +* the GNU Affero General Public la “GNU Affero General Public +* License as published by the License” telle que publiée +* Free Software Foundation, par la Free Software Foundation +* either version 3 of the : soit la version 3 de cette +* License, or (at your option) licence, soit (à votre gré) +* any later version. toute version ultérieure. +* +* OpenCADC is distributed in the OpenCADC est distribué +* hope that it will be useful, dans l’espoir qu’il vous +* but WITHOUT ANY WARRANTY; sera utile, mais SANS AUCUNE +* without even the implied GARANTIE : sans même la garantie +* warranty of MERCHANTABILITY implicite de COMMERCIALISABILITÉ +* or FITNESS FOR A PARTICULAR ni d’ADÉQUATION À UN OBJECTIF +* PURPOSE. See the GNU Affero PARTICULIER. Consultez la Licence +* General Public License for Générale Publique GNU Affero +* more details. pour plus de détails. +* +* You should have received Vous devriez avoir reçu une +* a copy of the GNU Affero copie de la Licence Générale +* General Public License along Publique GNU Affero avec +* with OpenCADC. If not, see OpenCADC ; si ce n’est +* . pas le cas, consultez : +* . +* +************************************************************************ +*/ + +package org.opencadc.vospace.db; + +import java.util.UUID; +import org.apache.log4j.Logger; +import org.opencadc.inventory.Entity; +import org.opencadc.inventory.db.AbstractDAO; +import org.opencadc.vospace.ContainerNode; +import org.opencadc.vospace.Node; + +/** + * + * @author pdowler + */ +public class NodeDAO extends AbstractDAO { + private static final Logger log = Logger.getLogger(NodeDAO.class); + + public NodeDAO() { + super(true); + } + + public Node get(UUID id) { + //return super.get(Node.class, id); + throw new UnsupportedOperationException(); + } + + public void put(Node n) { + throw new UnsupportedOperationException(); + } + + public ContainerNode getChildren(ContainerNode cn) { + throw new UnsupportedOperationException(); + } +} diff --git a/cadc-inventory-db/src/main/resources/vos.DeletedNodeEvent.sql b/cadc-inventory-db/src/main/resources/vos.DeletedNodeEvent.sql new file mode 100644 index 000000000..341a670e5 --- /dev/null +++ b/cadc-inventory-db/src/main/resources/vos.DeletedNodeEvent.sql @@ -0,0 +1,17 @@ + +create table .DeletedNodeEvent ( + -- type is immutable + nodeType char(1) not null, + + -- support cleanup of obsolete artifacts + storageID varchar(512), + + lastModified timestamp not null, + metaChecksum varchar(136) not null, + id uuid not null primary key +); + + + +create index dne_lastmodified on .DeletedNodeEvent(lastModified); + diff --git a/cadc-inventory-db/src/main/resources/vos.ModelVersion.sql b/cadc-inventory-db/src/main/resources/vos.ModelVersion.sql new file mode 100644 index 000000000..ca9126697 --- /dev/null +++ b/cadc-inventory-db/src/main/resources/vos.ModelVersion.sql @@ -0,0 +1,9 @@ + +create table .ModelVersion +( + model varchar(32) not null primary key, + version varchar(32) not null, + lastModified timestamp not null +) +; + diff --git a/cadc-inventory-db/src/main/resources/vos.Node.sql b/cadc-inventory-db/src/main/resources/vos.Node.sql new file mode 100644 index 000000000..7039b300a --- /dev/null +++ b/cadc-inventory-db/src/main/resources/vos.Node.sql @@ -0,0 +1,31 @@ + +create table .Node ( + -- require a special root ID value but prevent bugs + parentID uuid not null, + name varchar(512) not null, + nodeType char(1) not null, + + ownerID varchar(256) not null, + isPublic boolean, + isLocked boolean, + readOnlyGroups text, + readWriteGroups text, + + -- store all props in a 2D array + properties text[][], + + -- DataNode + busyState boolean, + storageID varchar(512), + + -- linkNode + target text, + + lastModified timestamp not null, + metaChecksum varchar(136) not null, + id uuid not null primary key +); + +create unique index node_parent_child on .Node(parentID,name); + +create index node_lastmodified on .Node(lastModified); From ce8a3719b77477d887a8cf148af627484e493343 Mon Sep 17 00:00:00 2001 From: Patrick Dowler Date: Tue, 28 Mar 2023 17:56:56 -0700 Subject: [PATCH 02/52] comment out old row lock code --- .../java/org/opencadc/inventory/db/SQLGenerator.java | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/SQLGenerator.java b/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/SQLGenerator.java index 172472704..767f0bb88 100644 --- a/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/SQLGenerator.java +++ b/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/SQLGenerator.java @@ -320,12 +320,14 @@ public EntityList getEntityList(Class c) { throw new UnsupportedOperationException("entity-list: " + c.getName()); } - public EntityLock getEntityLock(Class c) { + /* + private EntityLock getEntityLock(Class c) { if (Artifact.class.equals(c)) { return new EntityLockImpl(c); } throw new UnsupportedOperationException("entity-list: " + c.getName()); } + */ public EntityGet getSkeletonEntityGet(Class c) { EntityGet ret = new SkeletonGet(c); @@ -361,6 +363,8 @@ public EntityDelete getEntityDelete(Class c) { return new EntityDeleteImpl(c); } + /* + // replaced by select-for-update private class EntityLockImpl implements EntityLock { private final Calendar utc = Calendar.getInstance(DateUtil.UTC); private final Class entityClass; @@ -395,7 +399,8 @@ public PreparedStatement createPreparedStatement(Connection conn) throws SQLExce return prep; } } - + */ + private class SkeletonGet implements EntityGet { private UUID id; private final Class entityClass; From b2d8b2c77386b7d93ba8758d48bb90eaa972a131 Mon Sep 17 00:00:00 2001 From: Patrick Dowler Date: Wed, 5 Apr 2023 07:49:41 -0700 Subject: [PATCH 03/52] NodeDAO work --- .../org/opencadc/vospace/db/NodeDAOTest.java | 4 +- .../opencadc/inventory/db/AbstractDAO.java | 2 +- .../org/opencadc/inventory/db/EntityGet.java | 2 +- .../org/opencadc/inventory/db/EntityLock.java | 2 +- .../org/opencadc/inventory/db/EntityPut.java | 2 +- .../opencadc/inventory/db/SQLGenerator.java | 165 ++++++++++++------ .../java/org/opencadc/vospace/db/NodeDAO.java | 12 +- 7 files changed, 125 insertions(+), 64 deletions(-) diff --git a/cadc-inventory-db/src/intTest/java/org/opencadc/vospace/db/NodeDAOTest.java b/cadc-inventory-db/src/intTest/java/org/opencadc/vospace/db/NodeDAOTest.java index ccf475b6d..a1890cc69 100644 --- a/cadc-inventory-db/src/intTest/java/org/opencadc/vospace/db/NodeDAOTest.java +++ b/cadc-inventory-db/src/intTest/java/org/opencadc/vospace/db/NodeDAOTest.java @@ -83,7 +83,6 @@ import org.junit.Test; import org.opencadc.inventory.db.SQLGenerator; import org.opencadc.inventory.db.TestUtil; -import org.opencadc.vospace.db.InitDatabaseVOS; import org.opencadc.vospace.ContainerNode; import org.opencadc.vospace.DataNode; import org.opencadc.vospace.LinkNode; @@ -98,9 +97,10 @@ public class NodeDAOTest { static { Log4jInit.setLevel("org.opencadc.inventory", Level.INFO); - Log4jInit.setLevel("org.opencadc.inventory.db", Level.INFO); + Log4jInit.setLevel("org.opencadc.inventory.db", Level.DEBUG); Log4jInit.setLevel("ca.nrc.cadc.db", Level.INFO); Log4jInit.setLevel("org.opencadc.vospace", Level.INFO); + Log4jInit.setLevel("org.opencadc.vospace.db", Level.DEBUG); } NodeDAO nodeDAO; diff --git a/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/AbstractDAO.java b/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/AbstractDAO.java index fe8533e75..237668ba9 100644 --- a/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/AbstractDAO.java +++ b/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/AbstractDAO.java @@ -86,8 +86,8 @@ import javax.naming.NamingException; import javax.sql.DataSource; import org.apache.log4j.Logger; -import org.opencadc.inventory.Entity; import org.opencadc.inventory.InventoryUtil; +import org.opencadc.persist.Entity; import org.springframework.jdbc.BadSqlGrammarException; import org.springframework.jdbc.core.JdbcTemplate; import org.springframework.jdbc.core.RowMapper; diff --git a/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/EntityGet.java b/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/EntityGet.java index c065b58a6..029ca037b 100644 --- a/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/EntityGet.java +++ b/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/EntityGet.java @@ -68,7 +68,7 @@ package org.opencadc.inventory.db; import java.util.UUID; -import org.opencadc.inventory.Entity; +import org.opencadc.persist.Entity; import org.springframework.jdbc.core.JdbcTemplate; import org.springframework.jdbc.core.PreparedStatementCreator; diff --git a/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/EntityLock.java b/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/EntityLock.java index 238518d78..d99e41742 100644 --- a/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/EntityLock.java +++ b/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/EntityLock.java @@ -68,7 +68,7 @@ package org.opencadc.inventory.db; import java.util.UUID; -import org.opencadc.inventory.Entity; +import org.opencadc.persist.Entity; import org.springframework.jdbc.core.JdbcTemplate; import org.springframework.jdbc.core.PreparedStatementCreator; diff --git a/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/EntityPut.java b/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/EntityPut.java index 383347eba..e236c7a2b 100644 --- a/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/EntityPut.java +++ b/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/EntityPut.java @@ -69,7 +69,7 @@ package org.opencadc.inventory.db; -import org.opencadc.inventory.Entity; +import org.opencadc.persist.Entity; import org.springframework.jdbc.core.JdbcTemplate; import org.springframework.jdbc.core.PreparedStatementCreator; diff --git a/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/SQLGenerator.java b/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/SQLGenerator.java index 424f1d51f..7a83254c1 100644 --- a/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/SQLGenerator.java +++ b/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/SQLGenerator.java @@ -3,7 +3,7 @@ ******************* CANADIAN ASTRONOMY DATA CENTRE ******************* ************** CENTRE CANADIEN DE DONNÉES ASTRONOMIQUES ************** * -* (c) 2022. (c) 2022. +* (c) 2023. (c) 2023. * Government of Canada Gouvernement du Canada * National Research Council Conseil national de recherches * Ottawa, Canada, K1A 0R6 Ottawa, Canada, K1A 0R6 @@ -92,13 +92,13 @@ import org.opencadc.inventory.Artifact; import org.opencadc.inventory.DeletedArtifactEvent; import org.opencadc.inventory.DeletedStorageLocationEvent; -import org.opencadc.inventory.Entity; import org.opencadc.inventory.InventoryUtil; import org.opencadc.inventory.ObsoleteStorageLocation; import org.opencadc.inventory.SiteLocation; import org.opencadc.inventory.StorageLocation; import org.opencadc.inventory.StorageLocationEvent; import org.opencadc.inventory.StorageSite; +import org.opencadc.persist.Entity; import org.opencadc.vospace.DeletedNodeEvent; import org.opencadc.vospace.Node; import org.springframework.dao.DataAccessException; @@ -113,8 +113,8 @@ public class SQLGenerator { private static final Logger log = Logger.getLogger(SQLGenerator.class); - private final Map tableMap = new TreeMap(new ClassComp()); - private final Map columnMap = new TreeMap(new ClassComp()); + private final Map tableMap = new TreeMap<>(new ClassComp()); + private final Map columnMap = new TreeMap<>(new ClassComp()); protected final String database; // currently not used in SQL protected final String schema; // may be null @@ -278,6 +278,13 @@ public EntityGet getEntityGet(Class c, boolean forUpdate) { return new StorageSiteGet(forUpdate); } + if (Node.class.equals(c)) { + return new NodeGet(forUpdate); + } + if (DeletedNodeEvent.class.equals(c)) { + //return new DeletedNodeGet(); + } + if (forUpdate) { throw new UnsupportedOperationException("entity-get + forUpdate: " + c.getSimpleName()); } @@ -298,12 +305,6 @@ public EntityGet getEntityGet(Class c, boolean forUpdate) { return new HarvestStateGet(); } - if (Node.class.equals(c)) { - - } - if (DeletedNodeEvent.class.equals(c)) { - - } throw new UnsupportedOperationException("entity-get: " + c.getName()); } @@ -357,6 +358,12 @@ public EntityPut getEntityPut(Class c, boolean update) { if (HarvestState.class.equals(c)) { return new HarvestStatePut(update); } + if (Node.class.equals(c)) { + return new NodePut(update); + } + if (DeletedNodeEvent.class.equals(c)) { + //return new DeletedNodePut(update); + } throw new UnsupportedOperationException("entity-put: " + c.getName()); } @@ -364,44 +371,6 @@ public EntityDelete getEntityDelete(Class c) { return new EntityDeleteImpl(c); } - /* - // replaced by select-for-update - private class EntityLockImpl implements EntityLock { - private final Calendar utc = Calendar.getInstance(DateUtil.UTC); - private final Class entityClass; - private UUID id; - - EntityLockImpl(Class entityClass) { - this.entityClass = entityClass; - } - - @Override - public void setID(UUID id) { - this.id = id; - } - - @Override - public void execute(JdbcTemplate jdbc) throws EntityNotFoundException { - int n = jdbc.update(this); - if (n == 0) { - throw new EntityNotFoundException("not found: " + id); - } - } - - @Override - public PreparedStatement createPreparedStatement(Connection conn) throws SQLException { - String sql = getLockSQL(entityClass); - log.debug("EntityLockImpl: " + sql); - PreparedStatement prep = conn.prepareStatement(sql); - int col = 1; - prep.setObject(col++, id); - prep.setObject(col++, id); - - return prep; - } - } - */ - private class SkeletonGet implements EntityGet { private UUID id; private final Class entityClass; @@ -759,7 +728,6 @@ public ResourceIterator query(DataSource ds) { private class StorageSiteGet implements EntityGet { private UUID id; - private URI uri; private final boolean forUpdate; public StorageSiteGet(boolean forUpdate) { @@ -786,6 +754,9 @@ public PreparedStatement createPreparedStatement(Connection conn) throws SQLExce } else { throw new IllegalStateException("primary key is null"); } + if (forUpdate) { + sb.append(" FOR UPDATE"); + } String sql = sb.toString(); log.debug("StorageSiteGet: " + sql); PreparedStatement prep = conn.prepareStatement(sql); @@ -814,6 +785,45 @@ public PreparedStatement createPreparedStatement(Connection conn) throws SQLExce } } + private class NodeGet implements EntityGet { + private UUID id; + private final boolean forUpdate; + + public NodeGet(boolean forUpdate) { + this.forUpdate = forUpdate; + } + + @Override + public void setID(UUID id) { + this.id = id; + } + + @Override + public Node execute(JdbcTemplate jdbc) { + return (Node) jdbc.query(this, new NodeExtractor()); + } + + @Override + public PreparedStatement createPreparedStatement(Connection conn) throws SQLException { + StringBuilder sb = getSelectFromSQL(Node.class, false); + sb.append(" WHERE "); + if (id != null) { + String col = getKeyColumn(Node.class, true); + sb.append(col).append(" = ?"); + } else { + throw new IllegalStateException("primary key is null"); + } + if (forUpdate) { + sb.append(" FOR UPDATE"); + } + String sql = sb.toString(); + log.debug("Node: " + sql); + PreparedStatement prep = conn.prepareStatement(sql); + prep.setObject(1, id); + return prep; + } + } + private void safeSetString(PreparedStatement prep, int col, String value) throws SQLException { log.debug("safeSetString: " + col + " " + value); if (value != null) { @@ -1069,6 +1079,48 @@ public PreparedStatement createPreparedStatement(Connection conn) throws SQLExce } + private class NodePut implements EntityPut { + private final Calendar utc = Calendar.getInstance(DateUtil.UTC); + private final boolean update; + private Node value; + + NodePut(boolean update) { + this.update = update; + } + + @Override + public void setValue(Node value) { + this.value = value; + } + + @Override + public void execute(JdbcTemplate jdbc) { + jdbc.update(this); + } + + @Override + public PreparedStatement createPreparedStatement(Connection conn) throws SQLException { + String sql = null; + if (update) { + sql = getUpdateSQL(Node.class); + + } else { + sql = getInsertSQL(Node.class); + } + log.debug("StorageSitePut: " + sql); + PreparedStatement prep = conn.prepareStatement(sql); + int col = 1; + + throw new UnsupportedOperationException("TODO"); + + //prep.setTimestamp(col++, new Timestamp(value.getLastModified().getTime()), utc); + //prep.setString(col++, value.getMetaChecksum().toASCIIString()); + //prep.setObject(col++, value.getID()); + + //return prep; + } + } + private class EntityEventPut implements EntityPut { private final Calendar utc = Calendar.getInstance(DateUtil.UTC); private final boolean update; @@ -1520,4 +1572,19 @@ public StorageLocationEvent extractData(ResultSet rs) throws SQLException, DataA return ret; } } + + private class NodeExtractor implements ResultSetExtractor { + + final Calendar utc = Calendar.getInstance(DateUtil.UTC); + + @Override + public Node extractData(ResultSet rs) throws SQLException, DataAccessException { + if (!rs.next()) { + return null; + } + int col = 1; + throw new UnsupportedOperationException("TODO"); + } + + } } diff --git a/cadc-inventory-db/src/main/java/org/opencadc/vospace/db/NodeDAO.java b/cadc-inventory-db/src/main/java/org/opencadc/vospace/db/NodeDAO.java index 1d16fa40a..67c8fcee5 100644 --- a/cadc-inventory-db/src/main/java/org/opencadc/vospace/db/NodeDAO.java +++ b/cadc-inventory-db/src/main/java/org/opencadc/vospace/db/NodeDAO.java @@ -3,7 +3,7 @@ ******************* CANADIAN ASTRONOMY DATA CENTRE ******************* ************** CENTRE CANADIEN DE DONNÉES ASTRONOMIQUES ************** * -* (c) 2022. (c) 2022. +* (c) 2023. (c) 2023. * Government of Canada Gouvernement du Canada * National Research Council Conseil national de recherches * Ottawa, Canada, K1A 0R6 Ottawa, Canada, K1A 0R6 @@ -69,7 +69,6 @@ import java.util.UUID; import org.apache.log4j.Logger; -import org.opencadc.inventory.Entity; import org.opencadc.inventory.db.AbstractDAO; import org.opencadc.vospace.ContainerNode; import org.opencadc.vospace.Node; @@ -78,7 +77,7 @@ * * @author pdowler */ -public class NodeDAO extends AbstractDAO { +public class NodeDAO extends AbstractDAO { private static final Logger log = Logger.getLogger(NodeDAO.class); public NodeDAO() { @@ -86,12 +85,7 @@ public NodeDAO() { } public Node get(UUID id) { - //return super.get(Node.class, id); - throw new UnsupportedOperationException(); - } - - public void put(Node n) { - throw new UnsupportedOperationException(); + return super.get(Node.class, id); } public ContainerNode getChildren(ContainerNode cn) { From d244f50dd00db46b7026993e533b3c95fb9698ff Mon Sep 17 00:00:00 2001 From: Patrick Dowler Date: Thu, 6 Apr 2023 15:18:05 -0700 Subject: [PATCH 04/52] basic function of NodeDAO: put, get, update, delete --- .../org/opencadc/vospace/db/NodeDAOTest.java | 400 ++++++++++++++++-- .../opencadc/inventory/db/SQLGenerator.java | 258 +++++++++-- .../java/org/opencadc/inventory/db/Util.java | 148 +++++-- .../java/org/opencadc/vospace/db/NodeDAO.java | 17 +- .../src/main/resources/vos.Node.sql | 7 +- 5 files changed, 724 insertions(+), 106 deletions(-) diff --git a/cadc-inventory-db/src/intTest/java/org/opencadc/vospace/db/NodeDAOTest.java b/cadc-inventory-db/src/intTest/java/org/opencadc/vospace/db/NodeDAOTest.java index a1890cc69..71ec77702 100644 --- a/cadc-inventory-db/src/intTest/java/org/opencadc/vospace/db/NodeDAOTest.java +++ b/cadc-inventory-db/src/intTest/java/org/opencadc/vospace/db/NodeDAOTest.java @@ -70,8 +70,11 @@ import ca.nrc.cadc.db.ConnectionConfig; import ca.nrc.cadc.db.DBConfig; import ca.nrc.cadc.db.DBUtil; +import ca.nrc.cadc.io.ResourceIterator; import ca.nrc.cadc.util.Log4jInit; +import java.io.IOException; import java.net.URI; +import java.util.Date; import java.util.Map; import java.util.TreeMap; import java.util.UUID; @@ -87,6 +90,8 @@ import org.opencadc.vospace.DataNode; import org.opencadc.vospace.LinkNode; import org.opencadc.vospace.Node; +import org.opencadc.vospace.NodeProperty; +import org.opencadc.vospace.VOS; /** * @@ -137,7 +142,7 @@ public void init_cleanup() throws Exception { log.info("clearing old content..."); SQLGenerator gen = nodeDAO.getSQLGenerator(); DataSource ds = nodeDAO.getDataSource(); - String sql = "delete from " + gen.getTable(Node.class); + String sql = "delete from " + gen.getTable(ContainerNode.class); log.info("pre-test cleanup: " + sql); ds.getConnection().createStatement().execute(sql); log.info("clearing old content... OK"); @@ -150,50 +155,393 @@ public void testGetByID() { Assert.assertNull(a); } - //@Test - public void testPutGetDeleteContainerNode() { - ContainerNode root = new ContainerNode("root", false); + @Test + public void testPutGetUpdateDeleteContainerNode() throws InterruptedException { + UUID rootID = new UUID(0L, 0L); + ContainerNode root = new ContainerNode(rootID, "root", false); + + // put + ContainerNode orig = new ContainerNode("container-test", false); + orig.parent = root; + orig.ownerID = "the-owner"; + nodeDAO.put(orig); + + // get + Node a = nodeDAO.get(orig.getID()); + Assert.assertNotNull(a); + log.info("found: " + a.getID() + " aka " + a); + Assert.assertEquals(orig.getID(), a.getID()); + Assert.assertEquals(orig.getName(), a.getName()); + + Assert.assertNull(a.parent); // get-node-by-id: comes pack without parent + Assert.assertEquals(orig.getName(), a.getName()); + Assert.assertEquals(orig.ownerID, a.ownerID); + Assert.assertEquals(orig.isPublic, a.isPublic); + Assert.assertEquals(orig.isLocked, a.isLocked); + Assert.assertEquals(orig.readOnlyGroup, a.readOnlyGroup); + Assert.assertEquals(orig.readWriteGroup, a.readWriteGroup); + Assert.assertEquals(orig.properties, a.properties); + + Assert.assertTrue(a instanceof ContainerNode); + ContainerNode c = (ContainerNode) a; + Assert.assertEquals(orig.inheritPermissions, c.inheritPermissions); + + // these are set in put + Assert.assertEquals(orig.getMetaChecksum(), a.getMetaChecksum()); + Assert.assertEquals(orig.getLastModified(), a.getLastModified()); + + // update + Thread.sleep(10L); + orig.readOnlyGroup.add(URI.create("ivo://opencadc.org/gms?g1")); + orig.readWriteGroup.add(URI.create("ivo://opencadc.org/gms?g3")); + orig.properties.add(new NodeProperty(VOS.PROPERTY_URI_CONTENTLENGTH, "123")); + orig.isPublic = true; + nodeDAO.put(orig); + Node updated = nodeDAO.get(orig.getID()); + Assert.assertNotNull(updated); + Assert.assertEquals(orig.getID(), updated.getID()); + Assert.assertEquals(orig.getName(), updated.getName()); + Assert.assertTrue(a.getLastModified().before(updated.getLastModified())); + Assert.assertNotEquals(a.getMetaChecksum(), updated.getMetaChecksum()); + + Assert.assertNull(updated.parent); // get-node-by-id: comes pack without parent + Assert.assertEquals(orig.getName(), updated.getName()); + Assert.assertEquals(orig.ownerID, updated.ownerID); + Assert.assertEquals(orig.isPublic, updated.isPublic); + Assert.assertEquals(orig.isLocked, updated.isLocked); + Assert.assertEquals(orig.readOnlyGroup, updated.readOnlyGroup); + Assert.assertEquals(orig.readWriteGroup, updated.readWriteGroup); + Assert.assertEquals(orig.properties, updated.properties); + + Assert.assertTrue(updated instanceof ContainerNode); + ContainerNode uc = (ContainerNode) updated; + Assert.assertEquals(orig.inheritPermissions, uc.inheritPermissions); + + + nodeDAO.delete(orig.getID()); + Node gone = nodeDAO.get(orig.getID()); + Assert.assertNull(gone); + } + + @Test + public void testPutGetUpdateDeleteContainerNodeMax() throws InterruptedException { + UUID rootID = new UUID(0L, 0L); + ContainerNode root = new ContainerNode(rootID, "root", false); + + // TODO: use get-by-path to find and remove the test node + + ContainerNode orig = new ContainerNode("container-test", false); + orig.parent = root; + orig.ownerID = "the-owner"; + orig.isPublic = true; + orig.isLocked = false; + orig.inheritPermissions = false; + orig.readOnlyGroup.add(URI.create("ivo://opencadc.org/gms?g1")); + orig.readOnlyGroup.add(URI.create("ivo://opencadc.org/gms?g2")); + orig.readWriteGroup.add(URI.create("ivo://opencadc.org/gms?g3")); + orig.readWriteGroup.add(URI.create("ivo://opencadc.org/gms?g4,g5")); + orig.readWriteGroup.add(URI.create("ivo://opencadc.org/gms?g6-g7")); + orig.readWriteGroup.add(URI.create("ivo://opencadc.org/gms?g6.g7")); + orig.readWriteGroup.add(URI.create("ivo://opencadc.org/gms?g6_g7")); + orig.readWriteGroup.add(URI.create("ivo://opencadc.org/gms?g6~g7")); + + orig.properties.add(new NodeProperty(VOS.PROPERTY_URI_CONTENTLENGTH, "123")); + orig.properties.add(new NodeProperty(URI.create("custom:prop"), "spaces in value")); + orig.properties.add(new NodeProperty(URI.create("sketchy:a,b"), "comma in uri")); + orig.properties.add(new NodeProperty(URI.create("sketchy:funny"), "value-with-{delims}")); + nodeDAO.put(orig); + + Node a = nodeDAO.get(orig.getID()); + Assert.assertNotNull(a); + log.info("found: " + a.getID() + " aka " + a); + Assert.assertEquals(orig.getID(), a.getID()); + Assert.assertEquals(orig.getName(), a.getName()); + + Assert.assertNull(a.parent); // get-node-by-id: comes pack without parent + Assert.assertEquals(orig.getName(), a.getName()); + Assert.assertEquals(orig.ownerID, a.ownerID); + Assert.assertEquals(orig.isPublic, a.isPublic); + Assert.assertEquals(orig.isLocked, a.isLocked); + Assert.assertEquals(orig.readOnlyGroup, a.readOnlyGroup); + Assert.assertEquals(orig.readWriteGroup, a.readWriteGroup); + Assert.assertEquals(orig.properties, a.properties); + + Assert.assertTrue(a instanceof ContainerNode); + ContainerNode c = (ContainerNode) a; + Assert.assertEquals(orig.inheritPermissions, c.inheritPermissions); + + // these are set in put + Assert.assertEquals(orig.getMetaChecksum(), a.getMetaChecksum()); + Assert.assertEquals(orig.getLastModified(), a.getLastModified()); + + // update + Thread.sleep(10L); + orig.isPublic = false; + orig.isLocked = true; + orig.readOnlyGroup.clear(); + orig.readOnlyGroup.add(URI.create("ivo://opencadc.org/gms?g1")); + orig.readWriteGroup.clear(); + orig.readWriteGroup.add(URI.create("ivo://opencadc.org/gms?g3")); + orig.properties.clear(); + orig.properties.add(new NodeProperty(VOS.PROPERTY_URI_CONTENTLENGTH, "123")); + orig.inheritPermissions = true; + nodeDAO.put(orig); + Node updated = nodeDAO.get(orig.getID()); + Assert.assertNotNull(updated); + Assert.assertEquals(orig.getID(), updated.getID()); + Assert.assertEquals(orig.getName(), updated.getName()); + Assert.assertTrue(a.getLastModified().before(updated.getLastModified())); + Assert.assertNotEquals(a.getMetaChecksum(), updated.getMetaChecksum()); + + Assert.assertNull(updated.parent); // get-node-by-id: comes pack without parent + Assert.assertEquals(orig.getName(), updated.getName()); + Assert.assertEquals(orig.ownerID, updated.ownerID); + Assert.assertEquals(orig.isPublic, updated.isPublic); + Assert.assertEquals(orig.isLocked, updated.isLocked); + Assert.assertEquals(orig.readOnlyGroup, updated.readOnlyGroup); + Assert.assertEquals(orig.readWriteGroup, updated.readWriteGroup); + Assert.assertEquals(orig.properties, updated.properties); + + Assert.assertTrue(updated instanceof ContainerNode); + ContainerNode uc = (ContainerNode) updated; + Assert.assertEquals(orig.inheritPermissions, uc.inheritPermissions); + + nodeDAO.delete(orig.getID()); + Node gone = nodeDAO.get(orig.getID()); + Assert.assertNull(gone); + } + + @Test + public void testPutGetUpdateDeleteDataNode() throws InterruptedException { + UUID rootID = new UUID(0L, 0L); + ContainerNode root = new ContainerNode(rootID, "root", false); + + DataNode orig = new DataNode("data-test", URI.create("cadc:vault/" + UUID.randomUUID())); + orig.parent = root; + orig.ownerID = "the-owner"; + orig.isPublic = true; + orig.isLocked = false; + orig.properties.add(new NodeProperty(VOS.PROPERTY_URI_TYPE, "text/plain")); + orig.properties.add(new NodeProperty(VOS.PROPERTY_URI_DESCRIPTION, "this is the good stuff(tm)")); + nodeDAO.put(orig); + + Node a = nodeDAO.get(orig.getID()); + Assert.assertNotNull(a); + log.info("found: " + a.getID() + " aka " + a); + Assert.assertEquals(orig.getID(), a.getID()); + Assert.assertEquals(orig.getName(), a.getName()); + + Assert.assertNull(a.parent); // get-node-by-id: comes pack without parent + Assert.assertEquals(orig.getName(), a.getName()); + Assert.assertEquals(orig.ownerID, a.ownerID); + Assert.assertEquals(orig.isPublic, a.isPublic); + Assert.assertEquals(orig.isLocked, a.isLocked); + Assert.assertEquals(orig.readOnlyGroup, a.readOnlyGroup); + Assert.assertEquals(orig.readWriteGroup, a.readWriteGroup); + Assert.assertEquals(orig.properties, a.properties); - ContainerNode n = new ContainerNode("container-test", false); - n.parent = root; - nodeDAO.put(n); + Assert.assertTrue(a instanceof DataNode); + DataNode dn = (DataNode) a; + Assert.assertEquals(orig.getStorageID(), dn.getStorageID()); - Node a = nodeDAO.get(n.getID()); + // these are set in put + Assert.assertEquals(orig.getMetaChecksum(), a.getMetaChecksum()); + Assert.assertEquals(orig.getLastModified(), a.getLastModified()); + + // update + Thread.sleep(10L); + orig.isPublic = false; + orig.isLocked = true; + orig.readOnlyGroup.clear(); + orig.readOnlyGroup.add(URI.create("ivo://opencadc.org/gms?g1")); + orig.readWriteGroup.clear(); + orig.readWriteGroup.add(URI.create("ivo://opencadc.org/gms?g3")); + orig.properties.clear(); + orig.properties.add(new NodeProperty(VOS.PROPERTY_URI_CONTENTLENGTH, "123")); + // don't change storageID + nodeDAO.put(orig); + Node updated = nodeDAO.get(orig.getID()); + Assert.assertNotNull(updated); + Assert.assertEquals(orig.getID(), updated.getID()); + Assert.assertEquals(orig.getName(), updated.getName()); + Assert.assertTrue(a.getLastModified().before(updated.getLastModified())); + Assert.assertNotEquals(a.getMetaChecksum(), updated.getMetaChecksum()); + + Assert.assertNull(updated.parent); // get-node-by-id: comes pack without parent + Assert.assertEquals(orig.getName(), updated.getName()); + Assert.assertEquals(orig.ownerID, updated.ownerID); + Assert.assertEquals(orig.isPublic, updated.isPublic); + Assert.assertEquals(orig.isLocked, updated.isLocked); + Assert.assertEquals(orig.readOnlyGroup, updated.readOnlyGroup); + Assert.assertEquals(orig.readWriteGroup, updated.readWriteGroup); + Assert.assertEquals(orig.properties, updated.properties); + + + Assert.assertTrue(a instanceof DataNode); + DataNode udn = (DataNode) updated; + Assert.assertEquals(orig.getStorageID(), udn.getStorageID()); + + nodeDAO.delete(orig.getID()); + Node gone = nodeDAO.get(orig.getID()); + Assert.assertNull(gone); + } + + @Test + public void testPutGetUpdateDeleteLinkNode() throws InterruptedException { + UUID rootID = new UUID(0L, 0L); + ContainerNode root = new ContainerNode(rootID, "root", false); + + // TODO: use get-by-path to find and remove the test node + + LinkNode orig = new LinkNode("data-test", URI.create("vos://opencadc.org~srv/path/to/something")); + orig.parent = root; + orig.ownerID = "the-owner"; + orig.isPublic = true; + orig.isLocked = false; + orig.properties.add(new NodeProperty(VOS.PROPERTY_URI_DESCRIPTION, "link to the good stuff(tm)")); + nodeDAO.put(orig); + + Node a = nodeDAO.get(orig.getID()); Assert.assertNotNull(a); + log.info("found: " + a.getID() + " aka " + a); + Assert.assertEquals(orig.getID(), a.getID()); + Assert.assertEquals(orig.getName(), a.getName()); + + Assert.assertNull(a.parent); // get-node-by-id: comes pack without parent + Assert.assertEquals(orig.getName(), a.getName()); + Assert.assertEquals(orig.ownerID, a.ownerID); + Assert.assertEquals(orig.isPublic, a.isPublic); + Assert.assertEquals(orig.isLocked, a.isLocked); + Assert.assertEquals(orig.readOnlyGroup, a.readOnlyGroup); + Assert.assertEquals(orig.readWriteGroup, a.readWriteGroup); + Assert.assertEquals(orig.properties, a.properties); + + Assert.assertTrue(a instanceof LinkNode); + LinkNode link = (LinkNode) a; + Assert.assertEquals(orig.getTarget(), link.getTarget()); + + // these are set in put + Assert.assertEquals(orig.getMetaChecksum(), a.getMetaChecksum()); + Assert.assertEquals(orig.getLastModified(), a.getLastModified()); + + // update + Thread.sleep(10L); + orig.isPublic = false; + orig.isLocked = true; + orig.readOnlyGroup.clear(); + orig.readOnlyGroup.add(URI.create("ivo://opencadc.org/gms?g1")); + orig.readWriteGroup.clear(); + orig.readWriteGroup.add(URI.create("ivo://opencadc.org/gms?g3")); + orig.properties.clear(); + orig.properties.add(new NodeProperty(VOS.PROPERTY_URI_CONTENTLENGTH, "123")); + // don't change target + nodeDAO.put(orig); + Node updated = nodeDAO.get(orig.getID()); + Assert.assertNotNull(updated); + Assert.assertEquals(orig.getID(), updated.getID()); + Assert.assertEquals(orig.getName(), updated.getName()); + Assert.assertTrue(a.getLastModified().before(updated.getLastModified())); + Assert.assertNotEquals(a.getMetaChecksum(), updated.getMetaChecksum()); + + Assert.assertNull(updated.parent); // get-node-by-id: comes pack without parent + Assert.assertEquals(orig.getName(), updated.getName()); + Assert.assertEquals(orig.ownerID, updated.ownerID); + Assert.assertEquals(orig.isPublic, updated.isPublic); + Assert.assertEquals(orig.isLocked, updated.isLocked); + Assert.assertEquals(orig.readOnlyGroup, updated.readOnlyGroup); + Assert.assertEquals(orig.readWriteGroup, updated.readWriteGroup); + Assert.assertEquals(orig.properties, updated.properties); + + Assert.assertTrue(updated instanceof LinkNode); + LinkNode ulink = (LinkNode) updated; + Assert.assertEquals(orig.getTarget(), ulink.getTarget()); + + nodeDAO.delete(orig.getID()); + Node gone = nodeDAO.get(orig.getID()); + Assert.assertNull(gone); + } + + @Test + public void testPutGetDeleteContainerNodeChildren() throws IOException { + UUID rootID = new UUID(0L, 0L); + ContainerNode root = new ContainerNode(rootID, "root", false); + + ContainerNode orig = new ContainerNode("container-test", false); + orig.parent = root; + orig.ownerID = "the-owner"; + nodeDAO.put(orig); + + Node a = nodeDAO.get(orig.getID()); + Assert.assertNotNull(a); + log.info("found: " + a.getID() + " aka " + a); + Assert.assertEquals(orig.getID(), a.getID()); + Assert.assertEquals(orig.getName(), a.getName()); + Assert.assertTrue(a instanceof ContainerNode); ContainerNode c = (ContainerNode) a; - Assert.assertEquals(n.getName(), a.getName()); + // these are set in put - Assert.assertEquals(n.getMetaChecksum(), a.getMetaChecksum()); - Assert.assertEquals(n.getLastModified(), a.getLastModified()); - ContainerNode ac = nodeDAO.getChildren(c); - Assert.assertNotNull(ac); - Assert.assertTrue(ac.nodes.isEmpty()); + Assert.assertEquals(orig.getMetaChecksum(), a.getMetaChecksum()); + Assert.assertEquals(orig.getLastModified(), a.getLastModified()); + //ResourceIterator emptyIter = nodeDAO.childIterator(orig); + //Assert.assertNotNull(emptyIter); + //Assert.assertFalse(emptyIter.hasNext()); + //emptyIter.close(); // add children ContainerNode cont = new ContainerNode("container1", false); cont.parent = c; - DataNode data = new DataNode("data1"); + cont.ownerID = c.ownerID; + DataNode data = new DataNode("data1", URI.create("cadc:vault/" + UUID.randomUUID())); data.parent = c; + data.ownerID = c.ownerID; LinkNode link = new LinkNode("link1", URI.create("cadc:ARCHIVE/data")); link.parent = c; + link.ownerID = c.ownerID; + log.info("put child: " + cont + " of " + cont.parent); nodeDAO.put(cont); + log.info("put child: " + data + " of " + data.parent); nodeDAO.put(data); + log.info("put child: " + link + " of " + link.parent); nodeDAO.put(link); - ac = nodeDAO.getChildren(c); - Assert.assertNotNull(ac); - Assert.assertFalse(ac.nodes.isEmpty()); - Assert.assertEquals(3, ac.nodes.size()); - } - - @Test - public void testPutGetDeleteDataNode() { - log.info("TODO"); + ResourceIterator iter = nodeDAO.childIterator(orig); + Assert.assertNotNull(iter); + Assert.assertTrue(iter.hasNext()); + Node c1 = iter.next(); + Assert.assertTrue(iter.hasNext()); + Node c2 = iter.next(); + Assert.assertTrue(iter.hasNext()); + Node c3 = iter.next(); + + // default order: alpha + Assert.assertEquals(cont.getID(), c1.getID()); + Assert.assertEquals(cont.getName(), c1.getName()); + + Assert.assertEquals(data.getID(), c2.getID()); + Assert.assertEquals(data.getName(), c2.getName()); + + Assert.assertEquals(link.getID(), c3.getID()); + Assert.assertEquals(link.getName(), c3.getName()); + + // depth first delete required? + try { + nodeDAO.delete(orig.getID()); + Assert.fail("expected IllegalStateException but successfully deleted non-empty container"); + } catch (IllegalStateException expected) { + log.info("caught expected: " + expected); + } + + nodeDAO.delete(cont.getID()); + nodeDAO.delete(data.getID()); + nodeDAO.delete(link.getID()); + nodeDAO.delete(orig.getID()); + Node gone = nodeDAO.get(orig.getID()); + Assert.assertNull(gone); } - @Test - public void testPutGetDeleteLinkNode() { + //@Test + public void testPutGetDeleteNodeProperties() { log.info("TODO"); } } diff --git a/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/SQLGenerator.java b/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/SQLGenerator.java index 7a83254c1..3216ad3bd 100644 --- a/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/SQLGenerator.java +++ b/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/SQLGenerator.java @@ -72,6 +72,8 @@ import ca.nrc.cadc.util.StringUtil; import java.io.IOException; import java.net.URI; +import java.net.URISyntaxException; +import java.sql.Array; import java.sql.Connection; import java.sql.PreparedStatement; import java.sql.ResultSet; @@ -84,6 +86,7 @@ import java.util.List; import java.util.Map; import java.util.Set; +import java.util.StringTokenizer; import java.util.TreeMap; import java.util.TreeSet; import java.util.UUID; @@ -99,8 +102,13 @@ import org.opencadc.inventory.StorageLocationEvent; import org.opencadc.inventory.StorageSite; import org.opencadc.persist.Entity; +import org.opencadc.vospace.ContainerNode; +import org.opencadc.vospace.DataNode; import org.opencadc.vospace.DeletedNodeEvent; +import org.opencadc.vospace.LinkNode; import org.opencadc.vospace.Node; +import org.opencadc.vospace.NodeProperty; +import org.opencadc.vospace.VOS; import org.springframework.dao.DataAccessException; import org.springframework.jdbc.core.JdbcTemplate; import org.springframework.jdbc.core.ResultSetExtractor; @@ -229,7 +237,8 @@ protected void init() { "readOnlyGroups", "readWriteGroups", "properties", - "busyState", + "inheritPermissions", + "busy", "storageID", "target", "lastModified", @@ -263,7 +272,27 @@ public String getCurrentTimeSQL() { } public String getTable(Class c) { - return tableMap.get(c); + Class targetClass = c; + String ret = tableMap.get(targetClass); + if (ret == null) { + // enable finding a common table that stores subclass instances + targetClass = targetClass.getSuperclass(); + ret = tableMap.get(targetClass); + } + log.debug("table: " + c.getSimpleName() + " -> " + targetClass.getSimpleName() + " -> " + ret); + return ret; + } + + private String[] getColumns(Class c) { + Class targetClass = c; + String[] ret = columnMap.get(targetClass); + if (ret == null) { + // enable finding a common table that stores subclass instances + targetClass = targetClass.getSuperclass(); + ret = columnMap.get(targetClass); + } + log.debug("columns: " + c.getSimpleName() + " -> " + targetClass.getSimpleName() + " -> " + (ret == null ? null : ret.length)); + return ret; } public EntityGet getEntityGet(Class c) { @@ -278,7 +307,7 @@ public EntityGet getEntityGet(Class c, boolean forUpdate) { return new StorageSiteGet(forUpdate); } - if (Node.class.equals(c)) { + if (Node.class.equals(c) || Node.class.isInstance(c)) { return new NodeGet(forUpdate); } if (DeletedNodeEvent.class.equals(c)) { @@ -322,15 +351,6 @@ public EntityList getEntityList(Class c) { throw new UnsupportedOperationException("entity-list: " + c.getName()); } - /* - private EntityLock getEntityLock(Class c) { - if (Artifact.class.equals(c)) { - return new EntityLockImpl(c); - } - throw new UnsupportedOperationException("entity-list: " + c.getName()); - } - */ - public EntityGet getSkeletonEntityGet(Class c) { EntityGet ret = new SkeletonGet(c); return ret; @@ -358,7 +378,7 @@ public EntityPut getEntityPut(Class c, boolean update) { if (HarvestState.class.equals(c)) { return new HarvestStatePut(update); } - if (Node.class.equals(c)) { + if (Node.class.isAssignableFrom(c)) { return new NodePut(update); } if (DeletedNodeEvent.class.equals(c)) { @@ -467,7 +487,7 @@ public PreparedStatement createPreparedStatement(Connection conn) throws SQLExce String col = getKeyColumn(ObsoleteStorageLocation.class, true); sb.append(col).append(" = ?"); } else { - String[] cols = columnMap.get(ObsoleteStorageLocation.class); + String[] cols = getColumns(ObsoleteStorageLocation.class); sb.append(cols[0]).append(" = ?"); sb.append(" AND "); if (loc.storageBucket != null) { @@ -524,7 +544,7 @@ public PreparedStatement createPreparedStatement(Connection conn) throws SQLExce String col = getKeyColumn(HarvestState.class, true); sb.append(col).append(" = ?"); } else { - String[] cols = columnMap.get(HarvestState.class); + String[] cols = getColumns(HarvestState.class); sb.append(cols[0]).append(" = ?"); sb.append(" AND "); sb.append(cols[1]).append(" = ?"); @@ -824,6 +844,23 @@ public PreparedStatement createPreparedStatement(Connection conn) throws SQLExce } } + private void safeSetBoolean(PreparedStatement prep, int col, Boolean value) throws SQLException { + log.debug("safeSetBoolean: " + col + " " + value); + if (value != null) { + prep.setBoolean(col, value); + } else { + prep.setNull(col, Types.BOOLEAN); + } + } + + private void safeSetString(PreparedStatement prep, int col, URI value) throws SQLException { + String v = null; + if (value != null) { + v = value.toASCIIString(); + } + safeSetString(prep, col, v); + } + private void safeSetString(PreparedStatement prep, int col, String value) throws SQLException { log.debug("safeSetString: " + col + " " + value); if (value != null) { @@ -851,6 +888,24 @@ private void safeSetTimestamp(PreparedStatement prep, int col, Timestamp value, } } + private void safeSetArray(PreparedStatement prep, int col, Set values) throws SQLException { + + if (values != null && !values.isEmpty()) { + log.debug("safeSetArray: " + col + " " + values.size()); + String[] array1d = new String[values.size()]; + int i = 0; + for (URI u : values) { + array1d[i] = u.toASCIIString(); + i++; + } + java.sql.Array arr = prep.getConnection().createArrayOf("text", array1d); + prep.setObject(col, arr); + } else { + log.debug("safeSetArray: " + col + " null"); + prep.setNull(col, Types.ARRAY); + } + } + private void safeSetArray(PreparedStatement prep, int col, UUID[] value) throws SQLException { if (value != null) { @@ -863,6 +918,25 @@ private void safeSetArray(PreparedStatement prep, int col, UUID[] value) throws } } + private void safeSetProps(PreparedStatement prep, int col, Set values) throws SQLException { + + if (values != null && !values.isEmpty()) { + log.debug("safeSetProps: " + col + " " + values.size()); + String[][] array2d = new String[values.size()][2]; // TODO: w-h or h-w?? + int i = 0; + for (NodeProperty np : values) { + array2d[i][0] = np.getKey().toASCIIString(); + array2d[i][1] = np.getValue(); + i++; + } + java.sql.Array arr = prep.getConnection().createArrayOf("text", array2d); + prep.setObject(col, arr); + } else { + log.debug("safeSetProps: " + col + " = null"); + prep.setNull(col, Types.ARRAY); + } + } + private class ArtifactPut implements EntityPut { private final Calendar utc = Calendar.getInstance(DateUtil.UTC); private final boolean update; @@ -917,8 +991,8 @@ public PreparedStatement createPreparedStatement(Connection conn) throws SQLExce safeSetString(prep, col++, value.storageLocation.getStorageID().toASCIIString()); safeSetString(prep, col++, value.storageLocation.storageBucket); } else { - safeSetString(prep, col++, null); // storageLocation.storageID - safeSetString(prep, col++, null); // storageLocation.storageBucket + safeSetString(prep, col++, (URI) null); // storageLocation.storageID + safeSetString(prep, col++, (URI) null); // storageLocation.storageBucket } safeSetTimestamp(prep, col++, new Timestamp(value.getLastModified().getTime()), utc); @@ -1013,7 +1087,7 @@ public PreparedStatement createPreparedStatement(Connection conn) throws SQLExce if (value.getLocation().storageBucket != null) { safeSetString(prep, col++, value.getLocation().storageBucket); } else { - safeSetString(prep, col++, null); + safeSetString(prep, col++, (String) null); } prep.setTimestamp(col++, new Timestamp(value.getLastModified().getTime()), utc); @@ -1107,17 +1181,54 @@ public PreparedStatement createPreparedStatement(Connection conn) throws SQLExce } else { sql = getInsertSQL(Node.class); } - log.debug("StorageSitePut: " + sql); + log.debug("NodePut: " + sql); PreparedStatement prep = conn.prepareStatement(sql); int col = 1; - throw new UnsupportedOperationException("TODO"); + if (value.parent == null) { + throw new RuntimeException("BUG: cannot put Node without a parent: " + value); + } + prep.setObject(col++, value.parent.getID()); + prep.setString(col++, value.getName()); + prep.setString(col++, value.getClass().getSimpleName().substring(0, 1)); // HACK + if (value.ownerID == null) { + throw new RuntimeException("BUG: cannot put Node without an ownerID: " + value); + } + prep.setString(col++, value.ownerID.toString()); + safeSetBoolean(prep, col++, value.isPublic); + safeSetBoolean(prep, col++, value.isLocked); + safeSetArray(prep, col++, value.readOnlyGroup); + safeSetArray(prep, col++, value.readWriteGroup); + safeSetProps(prep, col++, value.properties); + if (value instanceof ContainerNode) { + ContainerNode cn = (ContainerNode) value; + safeSetBoolean(prep, col++, cn.inheritPermissions); + } else { + safeSetBoolean(prep, col++, null); + } + if (value instanceof DataNode) { + DataNode dn = (DataNode) value; + if (dn.getStorageID() == null) { + throw new RuntimeException("BUG: cannot put DataNode without a storageID: " + value); + } + safeSetBoolean(prep, col++, dn.busy); + safeSetString(prep, col++, dn.getStorageID()); + } else { + safeSetBoolean(prep, col++, null); + safeSetString(prep, col++, (URI) null); + } + if (value instanceof LinkNode) { + LinkNode ln = (LinkNode) value; + prep.setString(col++, ln.getTarget().toASCIIString()); + } else { + safeSetString(prep, col++, (URI) null); + } - //prep.setTimestamp(col++, new Timestamp(value.getLastModified().getTime()), utc); - //prep.setString(col++, value.getMetaChecksum().toASCIIString()); - //prep.setObject(col++, value.getID()); + prep.setTimestamp(col++, new Timestamp(value.getLastModified().getTime()), utc); + prep.setString(col++, value.getMetaChecksum().toASCIIString()); + prep.setObject(col++, value.getID()); - //return prep; + return prep; } } @@ -1190,11 +1301,13 @@ public PreparedStatement createPreparedStatement(Connection conn) throws SQLExce } private StringBuilder getSelectFromSQL(Class c, boolean entityCols) { - String tab = tableMap.get(c); - String[] cols = columnMap.get(c); + String tab = getTable(c); + Class targetClass = c; if (entityCols) { - cols = columnMap.get(Entity.class); + targetClass = Entity.class; } + String[] cols = getColumns(targetClass); + if (tab == null || cols == null) { throw new IllegalArgumentException("BUG: no table/columns for class " + c.getName()); } @@ -1213,21 +1326,13 @@ private StringBuilder getSelectFromSQL(Class c, boolean entityCols) { return sb; } - private String getLockSQL(Class c) { - StringBuilder sb = new StringBuilder(); - String pk = getKeyColumn(c, true); - sb.append("UPDATE "); - sb.append(tableMap.get(c)); - sb.append(" SET ").append(pk).append(" = ? WHERE ").append(pk).append(" = ?"); - return sb.toString(); - } - private String getUpdateSQL(Class c) { StringBuilder sb = new StringBuilder(); + String tab = getTable(c); sb.append("UPDATE "); - sb.append(tableMap.get(c)); + sb.append(tab); sb.append(" SET "); - String[] cols = columnMap.get(c); + String[] cols = getColumns(c); for (int i = 0; i < cols.length - 1; i++) { // PK is last if (i > 0) { sb.append(","); @@ -1244,10 +1349,11 @@ private String getUpdateSQL(Class c) { private String getInsertSQL(Class c) { StringBuilder sb = new StringBuilder(); + String tab = getTable(c); sb.append("INSERT INTO "); - sb.append(tableMap.get(c)); + sb.append(tab); sb.append(" ("); - String[] cols = columnMap.get(c); + String[] cols = getColumns(c); for (int i = 0; i < cols.length; i++) { if (i > 0) { sb.append(","); @@ -1268,14 +1374,15 @@ private String getInsertSQL(Class c) { private String getDeleteSQL(Class c) { StringBuilder sb = new StringBuilder(); + String tab = getTable(c); sb.append("DELETE FROM "); - sb.append(tableMap.get(c)); + sb.append(tab); sb.append(" WHERE id = ?"); return sb.toString(); } private String getKeyColumn(Class c, boolean pk) { - String[] cols = columnMap.get(c); + String[] cols = getColumns(c); if (cols == null) { throw new IllegalArgumentException("BUG: no table/columns for class " + c.getName()); } @@ -1583,8 +1690,73 @@ public Node extractData(ResultSet rs) throws SQLException, DataAccessException { return null; } int col = 1; - throw new UnsupportedOperationException("TODO"); + /* + "parentID", + "name", + "nodeType", + "ownerID", + "isPublic", + "isLocked", + "readOnlyGroups", + "readWriteGroups", + "properties", + + "inheritPermissions", + "busy", + "storageID", + "target", + + "lastModified", + "metaChecksum", + "id" // last column is always PK + */ + col++; //UUID unused = Util.getUUID(rs, col++); + final String name = rs.getString(col++); + final String nodeType = rs.getString(col++); + final String ownerID = rs.getString(col++); + final Boolean isPublic = Util.getBoolean(rs, col++); + final Boolean isLocked = Util.getBoolean(rs, col++); + final String rawROG = rs.getString(col++); + final String rawRWG = rs.getString(col++); + final String rawProps = rs.getString(col++); + final Boolean inheritPermissions = Util.getBoolean(rs, col++); + final Boolean busy = Util.getBoolean(rs, col++); + final URI storageID = Util.getURI(rs, col++); + final URI linkTarget = Util.getURI(rs, col++); + final Date lastModified = Util.getDate(rs, col++, utc); + final URI metaChecksum = Util.getURI(rs, col++); + final UUID id = Util.getUUID(rs, col++); + + Node ret; + if (nodeType.equals("C")) { + ret = new ContainerNode(id, name, inheritPermissions); + } else if (nodeType.equals("D")) { + ret = new DataNode(id, name, storageID); + } else if (nodeType.equals("L")) { + ret = new LinkNode(id, name, linkTarget); + } else { + throw new RuntimeException("BUG: unexpected node type code: " + nodeType); + } + ret.ownerID = ownerID; + ret.isPublic = isPublic; + ret.isLocked = isLocked; + + if (rawROG != null) { + Util.parseArrayURI(rawROG, ret.readOnlyGroup); + } + if (rawRWG != null) { + Util.parseArrayURI(rawRWG, ret.readWriteGroup); + } + if (rawProps != null) { + Util.parseArrayProps(rawProps, ret.properties); + } + + InventoryUtil.assignLastModified(ret, lastModified); + InventoryUtil.assignMetaChecksum(ret, metaChecksum); + + return ret; } + } } diff --git a/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/Util.java b/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/Util.java index 404841ff8..cd20020bf 100644 --- a/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/Util.java +++ b/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/Util.java @@ -75,10 +75,15 @@ import java.sql.Array; import java.sql.ResultSet; import java.sql.SQLException; +import java.util.ArrayList; import java.util.Calendar; import java.util.Date; +import java.util.List; +import java.util.Set; +import java.util.StringTokenizer; import java.util.UUID; import org.apache.log4j.Logger; +import org.opencadc.vospace.NodeProperty; /** * @@ -378,39 +383,114 @@ public static byte[] getByteArray(ResultSet rs, int col) throw new UnsupportedOperationException("converting " + o.getClass().getName() + " " + o + " to byte[]"); } - /* - * public static int[] getIntArray(ResultSet rs, int col) - * throws SQLException - * { - * Object o = rs.getObject(col); - * return toIntArray(o); - * } - * - * static int[] toIntArray(Object o) - * throws SQLException - * { - * if (o == null) - * return null; - * if (o instanceof Array) - * { - * Array a = (Array) o; - * o = a.getArray(); - * } - * if (o instanceof int[]) - * return (int[]) o; - * if (o instanceof byte[]) - * return CaomUtil.decodeIntArray((byte[]) o); - * if (o instanceof Integer[]) - * { - * Integer[] ia = (Integer[]) o; - * int[] ret = new int[ia.length]; - * for (int i=0; i dest) { + // postgresql 1D array: {a,"b,c"} + if (val == null || val.isEmpty()) { + return; + } + // GroupURI names can contain alphanumeric,comma,dash,dot,underscore,~ + // PG quotes them if comma is present (eg in the group name) + char delim = '"'; + int i = 0; + int j = val.indexOf(delim); + while (j != -1) { + String token = val.substring(i, j); + //log.warn("token: " + i + "," + j + " " + token); + i = j + 1; + j = val.indexOf(delim, i); + + handleToken(token, dest); + } + String token = val.substring(i); + //log.warn("token: " + i + " " + token); + handleToken(token, dest); + } + + private static void handleToken(String token, Set dest) { + if (token.startsWith("ivo://")) { + dest.add(URI.create(token)); + } else { + StringTokenizer st = new StringTokenizer(token, "{,}"); + while (st.hasMoreTokens()) { + String s = st.nextToken(); + dest.add(URI.create(s)); + } + } + } + + public static void parseArrayProps(String val, Set dest) { + // postgresql 2D array: {{a,b},{c,d}} + if (val == null || val.isEmpty()) { + return; + } + char open = '{'; + char close = '}'; + char quote = '"'; + int i = val.indexOf(open); + int j = val.lastIndexOf(close); + if (j > i) { + val = val.substring(i + 1, j); + } + i = val.indexOf(open); + j = val.indexOf(close, i + 1); + int k = 0; + while (i != -1 && j != -1 && k++ < 20) { + String t1 = val.substring(i + 1, j); + //log.warn("\tt1: " + i + "," + j + " " + t1); + handleProp(t1, dest); + + if (i != -1 && j > 0) { + i = val.indexOf(open, j); + j = val.indexOf(close, i + 1); + // look ahead for quotes + int q = val.indexOf(quote, i); + //log.warn("i=" + i + " j=" + j + " q=" + q); + if (q != -1 && q < j) { + int cq = val.indexOf(quote, q + 1); + j = val.indexOf(close, cq); + //log.warn("\tcq=" + cq + " j=" + j); + } + } + } + } + private static void handleProp(String token, Set dest) { + int q = token.indexOf('"'); + int cq = -1; + if (q == -1) { + q = Integer.MAX_VALUE; + } else { + cq = token.indexOf('"', q + 1); + } + int c = token.indexOf(','); + + String key; + int split = c; + if (c < q) { + // key + key = token.substring(0, c); + } else { + // "key" + key = token.substring(q + 1, cq); + split = cq + 1; + } + //log.warn("\tkey: " + key); + + q = token.indexOf('"', split + 1); + cq = -1; + if (q == -1) { + q = Integer.MAX_VALUE; + } else { + cq = token.indexOf('"', q + 1); + } + String val; + if (token.length() < q) { + val = token.substring(split + 1); + } else { + val = token.substring(q + 1, cq); + } + //log.warn("\tval: " + val); + + dest.add(new NodeProperty(URI.create(key), val)); + } } diff --git a/cadc-inventory-db/src/main/java/org/opencadc/vospace/db/NodeDAO.java b/cadc-inventory-db/src/main/java/org/opencadc/vospace/db/NodeDAO.java index 67c8fcee5..7c1155be7 100644 --- a/cadc-inventory-db/src/main/java/org/opencadc/vospace/db/NodeDAO.java +++ b/cadc-inventory-db/src/main/java/org/opencadc/vospace/db/NodeDAO.java @@ -67,11 +67,13 @@ package org.opencadc.vospace.db; +import ca.nrc.cadc.io.ResourceIterator; import java.util.UUID; import org.apache.log4j.Logger; import org.opencadc.inventory.db.AbstractDAO; import org.opencadc.vospace.ContainerNode; import org.opencadc.vospace.Node; +import org.opencadc.vospace.VOSURI; /** * @@ -84,11 +86,24 @@ public NodeDAO() { super(true); } + @Override + public void put(Node val) { + super.put(val); + } + public Node get(UUID id) { return super.get(Node.class, id); } - public ContainerNode getChildren(ContainerNode cn) { + public void delete(UUID id) { + super.delete(Node.class, id); + } + + public ResourceIterator childIterator(ContainerNode parent) { + return childIterator(parent, null, null); + } + + public ResourceIterator childIterator(ContainerNode parent, VOSURI start, Integer limit) { throw new UnsupportedOperationException(); } } diff --git a/cadc-inventory-db/src/main/resources/vos.Node.sql b/cadc-inventory-db/src/main/resources/vos.Node.sql index 7039b300a..e52a5ed8c 100644 --- a/cadc-inventory-db/src/main/resources/vos.Node.sql +++ b/cadc-inventory-db/src/main/resources/vos.Node.sql @@ -14,11 +14,14 @@ create table .Node ( -- store all props in a 2D array properties text[][], + -- ContainerNode + inheritPermissions boolean, + -- DataNode - busyState boolean, + busy boolean, storageID varchar(512), - -- linkNode + -- LinkNode target text, lastModified timestamp not null, From 7156929f4aaa6e672cb7bf35d0719460071e4906 Mon Sep 17 00:00:00 2001 From: Patrick Dowler Date: Thu, 6 Apr 2023 15:19:24 -0700 Subject: [PATCH 05/52] unit test code for postgresql array parsing --- .../org/opencadc/inventory/db/UtilTest.java | 160 ++++++++++++++++++ 1 file changed, 160 insertions(+) create mode 100644 cadc-inventory-db/src/test/java/org/opencadc/inventory/db/UtilTest.java diff --git a/cadc-inventory-db/src/test/java/org/opencadc/inventory/db/UtilTest.java b/cadc-inventory-db/src/test/java/org/opencadc/inventory/db/UtilTest.java new file mode 100644 index 000000000..adf255402 --- /dev/null +++ b/cadc-inventory-db/src/test/java/org/opencadc/inventory/db/UtilTest.java @@ -0,0 +1,160 @@ +/* +************************************************************************ +******************* CANADIAN ASTRONOMY DATA CENTRE ******************* +************** CENTRE CANADIEN DE DONNÉES ASTRONOMIQUES ************** +* +* (c) 2023. (c) 2023. +* Government of Canada Gouvernement du Canada +* National Research Council Conseil national de recherches +* Ottawa, Canada, K1A 0R6 Ottawa, Canada, K1A 0R6 +* All rights reserved Tous droits réservés +* +* NRC disclaims any warranties, Le CNRC dénie toute garantie +* expressed, implied, or énoncée, implicite ou légale, +* statutory, of any kind with de quelque nature que ce +* respect to the software, soit, concernant le logiciel, +* including without limitation y compris sans restriction +* any warranty of merchantability toute garantie de valeur +* or fitness for a particular marchande ou de pertinence +* purpose. NRC shall not be pour un usage particulier. +* liable in any event for any Le CNRC ne pourra en aucun cas +* damages, whether direct or être tenu responsable de tout +* indirect, special or general, dommage, direct ou indirect, +* consequential or incidental, particulier ou général, +* arising from the use of the accessoire ou fortuit, résultant +* software. Neither the name de l'utilisation du logiciel. Ni +* of the National Research le nom du Conseil National de +* Council of Canada nor the Recherches du Canada ni les noms +* names of its contributors may de ses participants ne peuvent +* be used to endorse or promote être utilisés pour approuver ou +* products derived from this promouvoir les produits dérivés +* software without specific prior de ce logiciel sans autorisation +* written permission. préalable et particulière +* par écrit. +* +* This file is part of the Ce fichier fait partie du projet +* OpenCADC project. OpenCADC. +* +* OpenCADC is free software: OpenCADC est un logiciel libre ; +* you can redistribute it and/or vous pouvez le redistribuer ou le +* modify it under the terms of modifier suivant les termes de +* the GNU Affero General Public la “GNU Affero General Public +* License as published by the License” telle que publiée +* Free Software Foundation, par la Free Software Foundation +* either version 3 of the : soit la version 3 de cette +* License, or (at your option) licence, soit (à votre gré) +* any later version. toute version ultérieure. +* +* OpenCADC is distributed in the OpenCADC est distribué +* hope that it will be useful, dans l’espoir qu’il vous +* but WITHOUT ANY WARRANTY; sera utile, mais SANS AUCUNE +* without even the implied GARANTIE : sans même la garantie +* warranty of MERCHANTABILITY implicite de COMMERCIALISABILITÉ +* or FITNESS FOR A PARTICULAR ni d’ADÉQUATION À UN OBJECTIF +* PURPOSE. See the GNU Affero PARTICULIER. Consultez la Licence +* General Public License for Générale Publique GNU Affero +* more details. pour plus de détails. +* +* You should have received Vous devriez avoir reçu une +* a copy of the GNU Affero copie de la Licence Générale +* General Public License along Publique GNU Affero avec +* with OpenCADC. If not, see OpenCADC ; si ce n’est +* . pas le cas, consultez : +* . +* +************************************************************************ +*/ + +package org.opencadc.inventory.db; + +import ca.nrc.cadc.util.Log4jInit; +import java.net.URI; +import java.util.Set; +import java.util.TreeSet; +import org.apache.log4j.Level; +import org.apache.log4j.Logger; +import org.junit.Test; +import org.opencadc.vospace.NodeProperty; + +/** + * + * @author pdowler + */ +public class UtilTest { + private static final Logger log = Logger.getLogger(UtilTest.class); + + static { + Log4jInit.setLevel("org.opencadc.inventory.db", Level.INFO); + } + + public UtilTest() { + } + + @Test + public void testParseArrayURI() throws Exception { + + String str = "{ivo://opencadc.org/gms?g3," + + "\"ivo://opencadc.org/gms?g4,g5\"," + + "ivo://opencadc.org/gms?g6-g7," + + "ivo://opencadc.org/gms?g6.g7," + + "ivo://opencadc.org/gms?g6_g7," + + "ivo://opencadc.org/gms?g6~g7}"; + + Set dest = new TreeSet<>(); + Util.parseArrayURI(str, dest); + for (URI u : dest) { + log.info("uri: " + u); + } + } + + @Test + public void testParseArrayNodeProperty() throws Exception { + + String str = ""; + Set dest = new TreeSet<>(); + + log.info("raw:\n" + str + "\n"); + Util.parseArrayProps(str, dest); + for (NodeProperty p : dest) { + log.info("prop: " + p.getKey() + " = " + p.getValue()); + } + + str = "{{ivo://ivoa.net/vospace/core#description,stuff}}"; + dest.clear(); + log.info("raw:\n" + str + "\n"); + Util.parseArrayProps(str, dest); + for (NodeProperty p : dest) { + log.info("prop: " + p.getKey() + " = " + p.getValue()); + } + + str = "{{ivo://ivoa.net/vospace/core#description,\"this is the good stuff(tm)\"}}"; + dest.clear(); + log.info("raw:\n" + str + "\n"); + Util.parseArrayProps(str, dest); + for (NodeProperty p : dest) { + log.info("prop: " + p.getKey() + " = " + p.getValue()); + } + + str = "{{ivo://ivoa.net/vospace/core#description,\"this is the good stuff(tm)\"}," + + "{ivo://ivoa.net/vospace/core#type,text/plain}}"; + dest.clear(); + log.info("raw:\n" + str + "\n"); + Util.parseArrayProps(str, dest); + for (NodeProperty p : dest) { + log.info("prop: " + p.getKey() + " = " + p.getValue()); + } + + str = "{{custom:prop,\"spaces in value\"}," + + "{ivo://ivoa.net/vospace/core#length,123}," + + "{ivo://ivoa.net/vospace/core#type,text/plain}," + + "{\"sketchy:a,b\",comma-in-uri}," + + "{sketchy:funny,\"value,with,{delims}\"}}"; + + dest.clear(); + log.info("raw:\n" + str + "\n"); + Util.parseArrayProps(str, dest); + for (NodeProperty p : dest) { + log.info("prop: " + p.getKey() + " = " + p.getValue()); + } + } +} From 2cdd2de31511dd82a9caad1dcb870f2c4ba6f427 Mon Sep 17 00:00:00 2001 From: Patrick Dowler Date: Thu, 13 Apr 2023 13:45:05 -0700 Subject: [PATCH 06/52] incomplete node iterator --- .../org/opencadc/vospace/db/NodeDAOTest.java | 86 ++++-- .../inventory/db/EntityIteratorQuery.java | 4 +- .../opencadc/inventory/db/SQLGenerator.java | 281 +++++++++++++----- .../java/org/opencadc/vospace/db/NodeDAO.java | 51 +++- 4 files changed, 325 insertions(+), 97 deletions(-) diff --git a/cadc-inventory-db/src/intTest/java/org/opencadc/vospace/db/NodeDAOTest.java b/cadc-inventory-db/src/intTest/java/org/opencadc/vospace/db/NodeDAOTest.java index 71ec77702..0f4a6a5fe 100644 --- a/cadc-inventory-db/src/intTest/java/org/opencadc/vospace/db/NodeDAOTest.java +++ b/cadc-inventory-db/src/intTest/java/org/opencadc/vospace/db/NodeDAOTest.java @@ -74,7 +74,6 @@ import ca.nrc.cadc.util.Log4jInit; import java.io.IOException; import java.net.URI; -import java.util.Date; import java.util.Map; import java.util.TreeMap; import java.util.UUID; @@ -116,7 +115,7 @@ public NodeDAOTest() throws Exception { ConnectionConfig cc = dbrc.getConnectionConfig(TestUtil.SERVER, TestUtil.DATABASE); DBUtil.createJNDIDataSource("jdbc/ArtifactDAOTest", cc); - Map config = new TreeMap(); + Map config = new TreeMap<>(); config.put(SQLGenerator.class.getName(), SQLGenerator.class); config.put("jndiDataSourceName", "jdbc/ArtifactDAOTest"); config.put("database", TestUtil.DATABASE); @@ -149,12 +148,24 @@ public void init_cleanup() throws Exception { } @Test - public void testGetByID() { + public void testGetByID_NotFound() { UUID id = UUID.randomUUID(); Node a = nodeDAO.get(id); Assert.assertNull(a); } + @Test + public void testGetByPath_NotFound() { + ContainerNode parent = new ContainerNode("not-found", false); + Node a = nodeDAO.get(parent, "not-found"); + Assert.assertNull(a); + + UUID rootID = new UUID(0L, 0L); + ContainerNode root = new ContainerNode(rootID, "root", false); + a = nodeDAO.get(root, "not-found"); + Assert.assertNull(a); + } + @Test public void testPutGetUpdateDeleteContainerNode() throws InterruptedException { UUID rootID = new UUID(0L, 0L); @@ -166,13 +177,22 @@ public void testPutGetUpdateDeleteContainerNode() throws InterruptedException { orig.ownerID = "the-owner"; nodeDAO.put(orig); - // get + // get-by-id Node a = nodeDAO.get(orig.getID()); Assert.assertNotNull(a); log.info("found: " + a.getID() + " aka " + a); Assert.assertEquals(orig.getID(), a.getID()); Assert.assertEquals(orig.getName(), a.getName()); + // get-by-path + Node aa = nodeDAO.get(root, orig.getName()); + Assert.assertNotNull(aa); + log.info("found: " + aa.getID() + " aka " + aa); + Assert.assertEquals(orig.getID(), aa.getID()); + Assert.assertEquals(orig.getName(), aa.getName()); + Assert.assertNotNull(aa.parent); + Assert.assertEquals(root.getID(), aa.parent.getID()); + Assert.assertNull(a.parent); // get-node-by-id: comes pack without parent Assert.assertEquals(orig.getName(), a.getName()); Assert.assertEquals(orig.ownerID, a.ownerID); @@ -251,12 +271,22 @@ public void testPutGetUpdateDeleteContainerNodeMax() throws InterruptedException orig.properties.add(new NodeProperty(URI.create("sketchy:funny"), "value-with-{delims}")); nodeDAO.put(orig); + // get-by-id Node a = nodeDAO.get(orig.getID()); Assert.assertNotNull(a); log.info("found: " + a.getID() + " aka " + a); Assert.assertEquals(orig.getID(), a.getID()); Assert.assertEquals(orig.getName(), a.getName()); + // get-by-path + Node aa = nodeDAO.get(root, orig.getName()); + Assert.assertNotNull(aa); + log.info("found: " + aa.getID() + " aka " + aa); + Assert.assertEquals(orig.getID(), aa.getID()); + Assert.assertEquals(orig.getName(), aa.getName()); + Assert.assertNotNull(aa.parent); + Assert.assertEquals(root.getID(), aa.parent.getID()); + Assert.assertNull(a.parent); // get-node-by-id: comes pack without parent Assert.assertEquals(orig.getName(), a.getName()); Assert.assertEquals(orig.ownerID, a.ownerID); @@ -325,12 +355,22 @@ public void testPutGetUpdateDeleteDataNode() throws InterruptedException { orig.properties.add(new NodeProperty(VOS.PROPERTY_URI_DESCRIPTION, "this is the good stuff(tm)")); nodeDAO.put(orig); + // get-by-id Node a = nodeDAO.get(orig.getID()); Assert.assertNotNull(a); log.info("found: " + a.getID() + " aka " + a); Assert.assertEquals(orig.getID(), a.getID()); Assert.assertEquals(orig.getName(), a.getName()); + // get-by-path + Node aa = nodeDAO.get(root, orig.getName()); + Assert.assertNotNull(aa); + log.info("found: " + aa.getID() + " aka " + aa); + Assert.assertEquals(orig.getID(), aa.getID()); + Assert.assertEquals(orig.getName(), aa.getName()); + Assert.assertNotNull(aa.parent); + Assert.assertEquals(root.getID(), aa.parent.getID()); + Assert.assertNull(a.parent); // get-node-by-id: comes pack without parent Assert.assertEquals(orig.getName(), a.getName()); Assert.assertEquals(orig.ownerID, a.ownerID); @@ -401,12 +441,22 @@ public void testPutGetUpdateDeleteLinkNode() throws InterruptedException { orig.properties.add(new NodeProperty(VOS.PROPERTY_URI_DESCRIPTION, "link to the good stuff(tm)")); nodeDAO.put(orig); + // get-by-id Node a = nodeDAO.get(orig.getID()); Assert.assertNotNull(a); log.info("found: " + a.getID() + " aka " + a); Assert.assertEquals(orig.getID(), a.getID()); Assert.assertEquals(orig.getName(), a.getName()); + // get-by-path + Node aa = nodeDAO.get(root, orig.getName()); + Assert.assertNotNull(aa); + log.info("found: " + aa.getID() + " aka " + aa); + Assert.assertEquals(orig.getID(), aa.getID()); + Assert.assertEquals(orig.getName(), aa.getName()); + Assert.assertNotNull(aa.parent); + Assert.assertEquals(root.getID(), aa.parent.getID()); + Assert.assertNull(a.parent); // get-node-by-id: comes pack without parent Assert.assertEquals(orig.getName(), a.getName()); Assert.assertEquals(orig.ownerID, a.ownerID); @@ -478,26 +528,25 @@ public void testPutGetDeleteContainerNodeChildren() throws IOException { Assert.assertEquals(orig.getName(), a.getName()); Assert.assertTrue(a instanceof ContainerNode); - ContainerNode c = (ContainerNode) a; // these are set in put Assert.assertEquals(orig.getMetaChecksum(), a.getMetaChecksum()); Assert.assertEquals(orig.getLastModified(), a.getLastModified()); - //ResourceIterator emptyIter = nodeDAO.childIterator(orig); - //Assert.assertNotNull(emptyIter); - //Assert.assertFalse(emptyIter.hasNext()); - //emptyIter.close(); + ResourceIterator emptyIter = nodeDAO.iterator(orig); + Assert.assertNotNull(emptyIter); + Assert.assertFalse(emptyIter.hasNext()); + emptyIter.close(); // add children ContainerNode cont = new ContainerNode("container1", false); - cont.parent = c; - cont.ownerID = c.ownerID; + cont.parent = orig; + cont.ownerID = orig.ownerID; DataNode data = new DataNode("data1", URI.create("cadc:vault/" + UUID.randomUUID())); - data.parent = c; - data.ownerID = c.ownerID; + data.parent = orig; + data.ownerID = orig.ownerID; LinkNode link = new LinkNode("link1", URI.create("cadc:ARCHIVE/data")); - link.parent = c; - link.ownerID = c.ownerID; + link.parent = orig; + link.ownerID = orig.ownerID; log.info("put child: " + cont + " of " + cont.parent); nodeDAO.put(cont); log.info("put child: " + data + " of " + data.parent); @@ -505,7 +554,7 @@ public void testPutGetDeleteContainerNodeChildren() throws IOException { log.info("put child: " + link + " of " + link.parent); nodeDAO.put(link); - ResourceIterator iter = nodeDAO.childIterator(orig); + ResourceIterator iter = nodeDAO.iterator(orig); Assert.assertNotNull(iter); Assert.assertTrue(iter.hasNext()); Node c1 = iter.next(); @@ -539,9 +588,4 @@ public void testPutGetDeleteContainerNodeChildren() throws IOException { Node gone = nodeDAO.get(orig.getID()); Assert.assertNull(gone); } - - //@Test - public void testPutGetDeleteNodeProperties() { - log.info("TODO"); - } } diff --git a/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/EntityIteratorQuery.java b/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/EntityIteratorQuery.java index 782de3129..04e065bb8 100644 --- a/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/EntityIteratorQuery.java +++ b/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/EntityIteratorQuery.java @@ -67,7 +67,7 @@ package org.opencadc.inventory.db; -import java.util.Iterator; +import ca.nrc.cadc.io.ResourceIterator; import javax.sql.DataSource; /** @@ -76,5 +76,5 @@ * @param entity subclass */ public interface EntityIteratorQuery { - Iterator query(DataSource ds); + ResourceIterator query(DataSource ds); } diff --git a/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/SQLGenerator.java b/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/SQLGenerator.java index 3216ad3bd..d9c45bc93 100644 --- a/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/SQLGenerator.java +++ b/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/SQLGenerator.java @@ -83,6 +83,7 @@ import java.util.Calendar; import java.util.Comparator; import java.util.Date; +import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Set; @@ -341,7 +342,10 @@ public EntityIteratorQuery getEntityIteratorQuery(Class c) { if (Artifact.class.equals(c)) { return new ArtifactIteratorQuery(); } - throw new UnsupportedOperationException("entity-list: " + c.getName()); + if (Node.class.equals(c)) { + return new NodeIteratorQuery(); + } + throw new UnsupportedOperationException("entity-iterator: " + c.getName()); } public EntityList getEntityList(Class c) { @@ -805,8 +809,10 @@ public PreparedStatement createPreparedStatement(Connection conn) throws SQLExce } } - private class NodeGet implements EntityGet { + public class NodeGet implements EntityGet { private UUID id; + private ContainerNode parent; + private String name; private final boolean forUpdate; public NodeGet(boolean forUpdate) { @@ -818,9 +824,14 @@ public void setID(UUID id) { this.id = id; } + public void setPath(ContainerNode parent, String name) { + this.parent = parent; + this.name = name; + } + @Override public Node execute(JdbcTemplate jdbc) { - return (Node) jdbc.query(this, new NodeExtractor()); + return (Node) jdbc.query(this, new NodeExtractor(parent)); } @Override @@ -830,6 +841,11 @@ public PreparedStatement createPreparedStatement(Connection conn) throws SQLExce if (id != null) { String col = getKeyColumn(Node.class, true); sb.append(col).append(" = ?"); + } else if (parent != null && name != null) { + String pidCol = "parentID"; + String nameCol = "name"; + // TODO: better way to get column names? + sb.append(pidCol).append(" = ? and ").append(nameCol).append(" = ?"); } else { throw new IllegalStateException("primary key is null"); } @@ -839,11 +855,83 @@ public PreparedStatement createPreparedStatement(Connection conn) throws SQLExce String sql = sb.toString(); log.debug("Node: " + sql); PreparedStatement prep = conn.prepareStatement(sql); - prep.setObject(1, id); + if (id != null) { + prep.setObject(1, id); + } else { + prep.setObject(1, parent.getID()); + prep.setObject(2, name); + } + return prep; } } + public class NodeIteratorQuery implements EntityIteratorQuery { + private ContainerNode parent; + private String start; + private Integer limit; + + public NodeIteratorQuery() { + } + + public void setParent(ContainerNode parent) { + this.parent = parent; + } + + public void setStart(String start) { + this.start = start; + } + + public void setLimit(Integer limit) { + this.limit = limit; + } + + @Override + public ResourceIterator query(DataSource ds) { + if (parent == null) { + throw new RuntimeException("BUG: cannot query for children with parent==null"); + } + + StringBuilder sb = getSelectFromSQL(Node.class, false); + sb.append(" WHERE parentID = ?"); + if (start != null) { + sb.append(" AND ? <= name"); + } + sb.append(" ORDER BY name ASC"); + if (limit != null) { + sb.append(" LIMIT ?"); + } + + String sql = sb.toString(); + log.debug("sql: " + sql); + + try { + Connection con = ds.getConnection(); + log.debug("NodeIteratorQuery: setAutoCommit(false)"); + con.setAutoCommit(false); + // defaults for options: ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY + PreparedStatement ps = con.prepareStatement(sql); + ps.setFetchSize(1000); + ps.setFetchDirection(ResultSet.FETCH_FORWARD); + int col = 1; + + ps.setObject(1, parent.getID()); + if (start != null) { + ps.setString(col++, start); + } + if (limit != null) { + ps.setInt(col++, limit); + } + ResultSet rs = ps.executeQuery(); + + return new NodeResultSetIterator(con, rs, parent); + } catch (SQLException ex) { + throw new RuntimeException("BUG: artifact iterator query failed", ex); + } + + } + } + private void safeSetBoolean(PreparedStatement prep, int col, Boolean value) throws SQLException { log.debug("safeSetBoolean: " + col + " " + value); if (value != null) { @@ -1495,6 +1583,63 @@ public Artifact next() { } } + private class NodeResultSetIterator implements ResourceIterator { + final Calendar utc = Calendar.getInstance(DateUtil.UTC); + private final Connection con; + private final ResultSet rs; + boolean hasRow; + + ContainerNode parent; + + public NodeResultSetIterator(Connection con, ResultSet rs, ContainerNode parent) { + this.con = con; + this.rs = rs; + this.parent = parent; + } + + @Override + public void close() throws IOException { + if (hasRow) { + log.debug("NodeResultSetIterator: " + super.toString() + " ctor - setAutoCommit(true)"); + try { + con.setAutoCommit(true); + hasRow = false; + } catch (SQLException ex) { + throw new RuntimeException("BUG: node list query failed during close()", ex); + } + } + } + + @Override + public boolean hasNext() { + return hasRow; + } + + @Override + public Node next() { + try { + Node ret = mapRowToNode(rs, utc, parent); + hasRow = rs.next(); + if (!hasRow) { + log.debug("NodeResultSetIterator: " + super.toString() + " DONE - setAutoCommit(true)"); + con.setAutoCommit(true); + } + return ret; + } catch (Exception ex) { + if (hasRow) { + log.debug("NodeResultSetIterator: " + super.toString() + " ResultSet.next() FAILED - setAutoCommit(true)"); + try { + close(); + hasRow = false; + } catch (IOException unexpected) { + log.debug("BUG: unexpected IOException from close", unexpected); + } + } + throw new RuntimeException("BUG: node list query failed while iterating", ex); + } + } + } + private Artifact mapRowToArtifact(ResultSet rs, Calendar utc) throws SQLException { int col = 1; final URI uri = Util.getURI(rs, col++); @@ -1528,6 +1673,62 @@ private Artifact mapRowToArtifact(ResultSet rs, Calendar utc) throws SQLExceptio return a; } + private Node mapRowToNode(ResultSet rs, Calendar utc, ContainerNode parent) throws SQLException { + int col = 1; + final UUID parentID = Util.getUUID(rs, col++); + final String name = rs.getString(col++); + final String nodeType = rs.getString(col++); + final String ownerID = rs.getString(col++); + final Boolean isPublic = Util.getBoolean(rs, col++); + final Boolean isLocked = Util.getBoolean(rs, col++); + final String rawROG = rs.getString(col++); + final String rawRWG = rs.getString(col++); + final String rawProps = rs.getString(col++); + final Boolean inheritPermissions = Util.getBoolean(rs, col++); + final Boolean busy = Util.getBoolean(rs, col++); + final URI storageID = Util.getURI(rs, col++); + final URI linkTarget = Util.getURI(rs, col++); + final Date lastModified = Util.getDate(rs, col++, utc); + final URI metaChecksum = Util.getURI(rs, col++); + final UUID id = Util.getUUID(rs, col++); + + Node ret; + if (nodeType.equals("C")) { + ret = new ContainerNode(id, name, inheritPermissions); + } else if (nodeType.equals("D")) { + ret = new DataNode(id, name, storageID); + } else if (nodeType.equals("L")) { + ret = new LinkNode(id, name, linkTarget); + } else { + throw new RuntimeException("BUG: unexpected node type code: " + nodeType); + } + ret.ownerID = ownerID; + ret.isPublic = isPublic; + ret.isLocked = isLocked; + + if (rawROG != null) { + Util.parseArrayURI(rawROG, ret.readOnlyGroup); + } + if (rawRWG != null) { + Util.parseArrayURI(rawRWG, ret.readWriteGroup); + } + if (rawProps != null) { + Util.parseArrayProps(rawProps, ret.properties); + } + + InventoryUtil.assignLastModified(ret, lastModified); + InventoryUtil.assignMetaChecksum(ret, metaChecksum); + + if (parent != null) { + if (!parent.getID().equals(parentID)) { + throw new RuntimeException("BUG: expected parentID=" + parent.getID() + " but got: " + parentID); + } + ret.parent = parent; + } + + return ret; + } + private class ObsoleteStorageLocationExtractor implements ResultSetExtractor { final Calendar utc = Calendar.getInstance(DateUtil.UTC); @@ -1681,80 +1882,20 @@ public StorageLocationEvent extractData(ResultSet rs) throws SQLException, DataA } private class NodeExtractor implements ResultSetExtractor { - + private ContainerNode parent; final Calendar utc = Calendar.getInstance(DateUtil.UTC); + + public NodeExtractor(ContainerNode parent) { + this.parent = parent; // optional + } @Override public Node extractData(ResultSet rs) throws SQLException, DataAccessException { if (!rs.next()) { return null; } - int col = 1; - /* - "parentID", - "name", - "nodeType", - "ownerID", - "isPublic", - "isLocked", - "readOnlyGroups", - "readWriteGroups", - "properties", - - "inheritPermissions", - "busy", - "storageID", - "target", - - "lastModified", - "metaChecksum", - "id" // last column is always PK - */ - col++; //UUID unused = Util.getUUID(rs, col++); - final String name = rs.getString(col++); - final String nodeType = rs.getString(col++); - final String ownerID = rs.getString(col++); - final Boolean isPublic = Util.getBoolean(rs, col++); - final Boolean isLocked = Util.getBoolean(rs, col++); - final String rawROG = rs.getString(col++); - final String rawRWG = rs.getString(col++); - final String rawProps = rs.getString(col++); - final Boolean inheritPermissions = Util.getBoolean(rs, col++); - final Boolean busy = Util.getBoolean(rs, col++); - final URI storageID = Util.getURI(rs, col++); - final URI linkTarget = Util.getURI(rs, col++); - final Date lastModified = Util.getDate(rs, col++, utc); - final URI metaChecksum = Util.getURI(rs, col++); - final UUID id = Util.getUUID(rs, col++); - - Node ret; - if (nodeType.equals("C")) { - ret = new ContainerNode(id, name, inheritPermissions); - } else if (nodeType.equals("D")) { - ret = new DataNode(id, name, storageID); - } else if (nodeType.equals("L")) { - ret = new LinkNode(id, name, linkTarget); - } else { - throw new RuntimeException("BUG: unexpected node type code: " + nodeType); - } - ret.ownerID = ownerID; - ret.isPublic = isPublic; - ret.isLocked = isLocked; - - if (rawROG != null) { - Util.parseArrayURI(rawROG, ret.readOnlyGroup); - } - if (rawRWG != null) { - Util.parseArrayURI(rawRWG, ret.readWriteGroup); - } - if (rawProps != null) { - Util.parseArrayProps(rawProps, ret.properties); - } - InventoryUtil.assignLastModified(ret, lastModified); - InventoryUtil.assignMetaChecksum(ret, metaChecksum); - - return ret; + return mapRowToNode(rs, utc, parent); } diff --git a/cadc-inventory-db/src/main/java/org/opencadc/vospace/db/NodeDAO.java b/cadc-inventory-db/src/main/java/org/opencadc/vospace/db/NodeDAO.java index 7c1155be7..756651843 100644 --- a/cadc-inventory-db/src/main/java/org/opencadc/vospace/db/NodeDAO.java +++ b/cadc-inventory-db/src/main/java/org/opencadc/vospace/db/NodeDAO.java @@ -70,10 +70,14 @@ import ca.nrc.cadc.io.ResourceIterator; import java.util.UUID; import org.apache.log4j.Logger; +import org.opencadc.inventory.Artifact; import org.opencadc.inventory.db.AbstractDAO; +import org.opencadc.inventory.db.SQLGenerator; import org.opencadc.vospace.ContainerNode; import org.opencadc.vospace.Node; import org.opencadc.vospace.VOSURI; +import org.springframework.jdbc.BadSqlGrammarException; +import org.springframework.jdbc.core.JdbcTemplate; /** * @@ -95,15 +99,54 @@ public Node get(UUID id) { return super.get(Node.class, id); } + public Node get(ContainerNode parent, String name) { + checkInit(); + log.debug("GET: " + parent.getID() + " + " + name); + long t = System.currentTimeMillis(); + + try { + JdbcTemplate jdbc = new JdbcTemplate(dataSource); + SQLGenerator.NodeGet get = (SQLGenerator.NodeGet) gen.getEntityGet(Node.class); + get.setPath(parent, name); + return get.execute(jdbc); + } catch (BadSqlGrammarException ex) { + handleInternalFail(ex); + } finally { + long dt = System.currentTimeMillis() - t; + log.debug("GET: " + parent.getID() + " + " + name + " " + dt + "ms"); + } + throw new RuntimeException("BUG: handleInternalFail did not throw"); + } + public void delete(UUID id) { super.delete(Node.class, id); } - public ResourceIterator childIterator(ContainerNode parent) { - return childIterator(parent, null, null); + public ResourceIterator iterator(ContainerNode parent) { + return NodeDAO.this.iterator(parent, null, null); } - public ResourceIterator childIterator(ContainerNode parent, VOSURI start, Integer limit) { - throw new UnsupportedOperationException(); + public ResourceIterator iterator(ContainerNode parent, String start, Integer limit) { + if (parent == null) { + throw new IllegalArgumentException("childIterator: parent cannot be null"); + } + log.debug("iterator: " + parent.getID()); + + checkInit(); + long t = System.currentTimeMillis(); + + try { + SQLGenerator.NodeIteratorQuery iter = (SQLGenerator.NodeIteratorQuery) gen.getEntityIteratorQuery(Node.class); + iter.setParent(parent); + iter.setStart(start); + iter.setLimit(limit); + return iter.query(dataSource); + } catch (BadSqlGrammarException ex) { + handleInternalFail(ex); + } finally { + long dt = System.currentTimeMillis() - t; + log.debug("iterator: " + parent.getID() + " " + dt + "ms"); + } + throw new RuntimeException("BUG: should be unreachable"); } } From 7905cc0c88068a63709c48200479881be4d07a0e Mon Sep 17 00:00:00 2001 From: Patrick Dowler Date: Fri, 14 Apr 2023 13:05:05 -0700 Subject: [PATCH 07/52] working node child iterator --- .../org/opencadc/vospace/db/NodeDAOTest.java | 48 +++++++++++-------- .../opencadc/inventory/db/SQLGenerator.java | 11 ++++- .../java/org/opencadc/vospace/db/NodeDAO.java | 1 + 3 files changed, 38 insertions(+), 22 deletions(-) diff --git a/cadc-inventory-db/src/intTest/java/org/opencadc/vospace/db/NodeDAOTest.java b/cadc-inventory-db/src/intTest/java/org/opencadc/vospace/db/NodeDAOTest.java index 0f4a6a5fe..62374a3f8 100644 --- a/cadc-inventory-db/src/intTest/java/org/opencadc/vospace/db/NodeDAOTest.java +++ b/cadc-inventory-db/src/intTest/java/org/opencadc/vospace/db/NodeDAOTest.java @@ -512,7 +512,7 @@ public void testPutGetUpdateDeleteLinkNode() throws InterruptedException { } @Test - public void testPutGetDeleteContainerNodeChildren() throws IOException { + public void testContainerNodeIterator() throws IOException { UUID rootID = new UUID(0L, 0L); ContainerNode root = new ContainerNode(rootID, "root", false); @@ -532,10 +532,19 @@ public void testPutGetDeleteContainerNodeChildren() throws IOException { // these are set in put Assert.assertEquals(orig.getMetaChecksum(), a.getMetaChecksum()); Assert.assertEquals(orig.getLastModified(), a.getLastModified()); - ResourceIterator emptyIter = nodeDAO.iterator(orig); - Assert.assertNotNull(emptyIter); - Assert.assertFalse(emptyIter.hasNext()); - emptyIter.close(); + try (ResourceIterator emptyIter = nodeDAO.iterator(orig)) { + Assert.assertNotNull(emptyIter); + Assert.assertFalse(emptyIter.hasNext()); + } // auto-close + + Node top = null; + try (ResourceIterator rootIter = nodeDAO.iterator(root)) { + if (rootIter.hasNext()) { + top = rootIter.next(); + } + } + Assert.assertNotNull(top); + Assert.assertEquals(orig.getID(), top.getID()); // add children ContainerNode cont = new ContainerNode("container1", false); @@ -554,14 +563,18 @@ public void testPutGetDeleteContainerNodeChildren() throws IOException { log.info("put child: " + link + " of " + link.parent); nodeDAO.put(link); - ResourceIterator iter = nodeDAO.iterator(orig); - Assert.assertNotNull(iter); - Assert.assertTrue(iter.hasNext()); - Node c1 = iter.next(); - Assert.assertTrue(iter.hasNext()); - Node c2 = iter.next(); - Assert.assertTrue(iter.hasNext()); - Node c3 = iter.next(); + Node c1; + Node c2; + Node c3; + try (ResourceIterator iter = nodeDAO.iterator(orig)) { + Assert.assertNotNull(iter); + Assert.assertTrue(iter.hasNext()); + c1 = iter.next(); + Assert.assertTrue(iter.hasNext()); + c2 = iter.next(); + Assert.assertTrue(iter.hasNext()); + c3 = iter.next(); + } // default order: alpha Assert.assertEquals(cont.getID(), c1.getID()); @@ -573,14 +586,7 @@ public void testPutGetDeleteContainerNodeChildren() throws IOException { Assert.assertEquals(link.getID(), c3.getID()); Assert.assertEquals(link.getName(), c3.getName()); - // depth first delete required? - try { - nodeDAO.delete(orig.getID()); - Assert.fail("expected IllegalStateException but successfully deleted non-empty container"); - } catch (IllegalStateException expected) { - log.info("caught expected: " + expected); - } - + // depth first delete required but not enforced by DAO nodeDAO.delete(cont.getID()); nodeDAO.delete(data.getID()); nodeDAO.delete(link.getID()); diff --git a/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/SQLGenerator.java b/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/SQLGenerator.java index d9c45bc93..d8074515a 100644 --- a/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/SQLGenerator.java +++ b/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/SQLGenerator.java @@ -916,11 +916,14 @@ public ResourceIterator query(DataSource ds) { int col = 1; ps.setObject(1, parent.getID()); + log.debug("parentID = " + parent.getID()); if (start != null) { ps.setString(col++, start); + log.debug("start = " + start); } if (limit != null) { ps.setInt(col++, limit); + log.debug("limit = " + limit); } ResultSet rs = ps.executeQuery(); @@ -1591,10 +1594,16 @@ private class NodeResultSetIterator implements ResourceIterator { ContainerNode parent; - public NodeResultSetIterator(Connection con, ResultSet rs, ContainerNode parent) { + public NodeResultSetIterator(Connection con, ResultSet rs, ContainerNode parent) throws SQLException { this.con = con; this.rs = rs; this.parent = parent; + hasRow = rs.next(); + log.debug("NodeResultSetIterator: " + super.toString() + " ctor " + hasRow); + if (!hasRow) { + log.debug("NodeResultSetIterator: " + super.toString() + " ctor - setAutoCommit(true)"); + con.setAutoCommit(true); + } } @Override diff --git a/cadc-inventory-db/src/main/java/org/opencadc/vospace/db/NodeDAO.java b/cadc-inventory-db/src/main/java/org/opencadc/vospace/db/NodeDAO.java index 756651843..b4d36d562 100644 --- a/cadc-inventory-db/src/main/java/org/opencadc/vospace/db/NodeDAO.java +++ b/cadc-inventory-db/src/main/java/org/opencadc/vospace/db/NodeDAO.java @@ -68,6 +68,7 @@ package org.opencadc.vospace.db; import ca.nrc.cadc.io.ResourceIterator; +import java.io.IOException; import java.util.UUID; import org.apache.log4j.Logger; import org.opencadc.inventory.Artifact; From d47e89653b71f9821446b1880d1d0420cbcceb01 Mon Sep 17 00:00:00 2001 From: Patrick Dowler Date: Wed, 19 Apr 2023 12:56:21 -0700 Subject: [PATCH 08/52] fix and test use of Node.parentID add checksum verification to NodeDAO tests --- cadc-inventory-db/build.gradle | 1 + .../org/opencadc/vospace/db/NodeDAOTest.java | 53 ++++++++++++++----- .../opencadc/inventory/db/SQLGenerator.java | 19 +++---- .../java/org/opencadc/vospace/db/NodeDAO.java | 5 ++ 4 files changed, 56 insertions(+), 22 deletions(-) diff --git a/cadc-inventory-db/build.gradle b/cadc-inventory-db/build.gradle index 798804dc9..940b17d71 100644 --- a/cadc-inventory-db/build.gradle +++ b/cadc-inventory-db/build.gradle @@ -27,6 +27,7 @@ mainClassName = 'org.opencadc.inventory.db.version.Main' dependencies { compile 'org.opencadc:cadc-util:[1.9.5,2.0)' compile 'org.opencadc:cadc-inventory:[0.9.4,)' + compile 'org.opencadc:cadc-vos:[2.0,3.0)' testCompile 'junit:junit:[4.0,)' diff --git a/cadc-inventory-db/src/intTest/java/org/opencadc/vospace/db/NodeDAOTest.java b/cadc-inventory-db/src/intTest/java/org/opencadc/vospace/db/NodeDAOTest.java index 62374a3f8..cbab0c4ba 100644 --- a/cadc-inventory-db/src/intTest/java/org/opencadc/vospace/db/NodeDAOTest.java +++ b/cadc-inventory-db/src/intTest/java/org/opencadc/vospace/db/NodeDAOTest.java @@ -3,7 +3,7 @@ ******************* CANADIAN ASTRONOMY DATA CENTRE ******************* ************** CENTRE CANADIEN DE DONNÉES ASTRONOMIQUES ************** * -* (c) 2022. (c) 2022. +* (c) 2023. (c) 2023. * Government of Canada Gouvernement du Canada * National Research Council Conseil national de recherches * Ottawa, Canada, K1A 0R6 Ottawa, Canada, K1A 0R6 @@ -74,6 +74,8 @@ import ca.nrc.cadc.util.Log4jInit; import java.io.IOException; import java.net.URI; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; import java.util.Map; import java.util.TreeMap; import java.util.UUID; @@ -167,7 +169,8 @@ public void testGetByPath_NotFound() { } @Test - public void testPutGetUpdateDeleteContainerNode() throws InterruptedException { + public void testPutGetUpdateDeleteContainerNode() throws InterruptedException, + NoSuchAlgorithmException { UUID rootID = new UUID(0L, 0L); ContainerNode root = new ContainerNode(rootID, "root", false); @@ -180,16 +183,18 @@ public void testPutGetUpdateDeleteContainerNode() throws InterruptedException { // get-by-id Node a = nodeDAO.get(orig.getID()); Assert.assertNotNull(a); - log.info("found: " + a.getID() + " aka " + a); + log.info("found by id: " + a.getID() + " aka " + a); Assert.assertEquals(orig.getID(), a.getID()); Assert.assertEquals(orig.getName(), a.getName()); + Assert.assertEquals(root.getID(), a.parentID); // get-by-path Node aa = nodeDAO.get(root, orig.getName()); Assert.assertNotNull(aa); - log.info("found: " + aa.getID() + " aka " + aa); + log.info("found by path: " + aa.getID() + " aka " + aa); Assert.assertEquals(orig.getID(), aa.getID()); Assert.assertEquals(orig.getName(), aa.getName()); + Assert.assertEquals(root.getID(), a.parentID); Assert.assertNotNull(aa.parent); Assert.assertEquals(root.getID(), aa.parent.getID()); @@ -210,6 +215,9 @@ public void testPutGetUpdateDeleteContainerNode() throws InterruptedException { Assert.assertEquals(orig.getMetaChecksum(), a.getMetaChecksum()); Assert.assertEquals(orig.getLastModified(), a.getLastModified()); + URI mcs = a.computeMetaChecksum(MessageDigest.getInstance("MD5")); + Assert.assertEquals("metaChecksum", a.getMetaChecksum(), mcs); + // update Thread.sleep(10L); orig.readOnlyGroup.add(URI.create("ivo://opencadc.org/gms?g1")); @@ -244,7 +252,8 @@ public void testPutGetUpdateDeleteContainerNode() throws InterruptedException { } @Test - public void testPutGetUpdateDeleteContainerNodeMax() throws InterruptedException { + public void testPutGetUpdateDeleteContainerNodeMax() throws InterruptedException, + NoSuchAlgorithmException { UUID rootID = new UUID(0L, 0L); ContainerNode root = new ContainerNode(rootID, "root", false); @@ -274,18 +283,20 @@ public void testPutGetUpdateDeleteContainerNodeMax() throws InterruptedException // get-by-id Node a = nodeDAO.get(orig.getID()); Assert.assertNotNull(a); - log.info("found: " + a.getID() + " aka " + a); + log.info("found by id: " + a.getID() + " aka " + a); Assert.assertEquals(orig.getID(), a.getID()); Assert.assertEquals(orig.getName(), a.getName()); + Assert.assertEquals(root.getID(), a.parentID); // get-by-path Node aa = nodeDAO.get(root, orig.getName()); Assert.assertNotNull(aa); - log.info("found: " + aa.getID() + " aka " + aa); + log.info("found by path: " + aa.getID() + " aka " + aa); Assert.assertEquals(orig.getID(), aa.getID()); Assert.assertEquals(orig.getName(), aa.getName()); Assert.assertNotNull(aa.parent); Assert.assertEquals(root.getID(), aa.parent.getID()); + Assert.assertEquals(root.getID(), aa.parentID); Assert.assertNull(a.parent); // get-node-by-id: comes pack without parent Assert.assertEquals(orig.getName(), a.getName()); @@ -304,6 +315,9 @@ public void testPutGetUpdateDeleteContainerNodeMax() throws InterruptedException Assert.assertEquals(orig.getMetaChecksum(), a.getMetaChecksum()); Assert.assertEquals(orig.getLastModified(), a.getLastModified()); + URI mcs = a.computeMetaChecksum(MessageDigest.getInstance("MD5")); + Assert.assertEquals("metaChecksum", a.getMetaChecksum(), mcs); + // update Thread.sleep(10L); orig.isPublic = false; @@ -342,11 +356,12 @@ public void testPutGetUpdateDeleteContainerNodeMax() throws InterruptedException } @Test - public void testPutGetUpdateDeleteDataNode() throws InterruptedException { + public void testPutGetUpdateDeleteDataNode() throws InterruptedException, + NoSuchAlgorithmException { UUID rootID = new UUID(0L, 0L); ContainerNode root = new ContainerNode(rootID, "root", false); - DataNode orig = new DataNode("data-test", URI.create("cadc:vault/" + UUID.randomUUID())); + DataNode orig = new DataNode(UUID.randomUUID(), "data-test", URI.create("cadc:vault/" + UUID.randomUUID())); orig.parent = root; orig.ownerID = "the-owner"; orig.isPublic = true; @@ -361,6 +376,8 @@ public void testPutGetUpdateDeleteDataNode() throws InterruptedException { log.info("found: " + a.getID() + " aka " + a); Assert.assertEquals(orig.getID(), a.getID()); Assert.assertEquals(orig.getName(), a.getName()); + Assert.assertEquals(root.getID(), a.parentID); + Assert.assertEquals(root.getID(), a.parentID); // get-by-path Node aa = nodeDAO.get(root, orig.getName()); @@ -370,6 +387,7 @@ public void testPutGetUpdateDeleteDataNode() throws InterruptedException { Assert.assertEquals(orig.getName(), aa.getName()); Assert.assertNotNull(aa.parent); Assert.assertEquals(root.getID(), aa.parent.getID()); + Assert.assertEquals(root.getID(), aa.parentID); Assert.assertNull(a.parent); // get-node-by-id: comes pack without parent Assert.assertEquals(orig.getName(), a.getName()); @@ -382,12 +400,15 @@ public void testPutGetUpdateDeleteDataNode() throws InterruptedException { Assert.assertTrue(a instanceof DataNode); DataNode dn = (DataNode) a; - Assert.assertEquals(orig.getStorageID(), dn.getStorageID()); + Assert.assertEquals(orig.storageID, dn.storageID); // these are set in put Assert.assertEquals(orig.getMetaChecksum(), a.getMetaChecksum()); Assert.assertEquals(orig.getLastModified(), a.getLastModified()); + URI mcs = a.computeMetaChecksum(MessageDigest.getInstance("MD5")); + Assert.assertEquals("metaChecksum", a.getMetaChecksum(), mcs); + // update Thread.sleep(10L); orig.isPublic = false; @@ -419,7 +440,7 @@ public void testPutGetUpdateDeleteDataNode() throws InterruptedException { Assert.assertTrue(a instanceof DataNode); DataNode udn = (DataNode) updated; - Assert.assertEquals(orig.getStorageID(), udn.getStorageID()); + Assert.assertEquals(orig.storageID, udn.storageID); nodeDAO.delete(orig.getID()); Node gone = nodeDAO.get(orig.getID()); @@ -427,7 +448,8 @@ public void testPutGetUpdateDeleteDataNode() throws InterruptedException { } @Test - public void testPutGetUpdateDeleteLinkNode() throws InterruptedException { + public void testPutGetUpdateDeleteLinkNode() throws InterruptedException, + NoSuchAlgorithmException { UUID rootID = new UUID(0L, 0L); ContainerNode root = new ContainerNode(rootID, "root", false); @@ -447,6 +469,7 @@ public void testPutGetUpdateDeleteLinkNode() throws InterruptedException { log.info("found: " + a.getID() + " aka " + a); Assert.assertEquals(orig.getID(), a.getID()); Assert.assertEquals(orig.getName(), a.getName()); + Assert.assertEquals(root.getID(), a.parentID); // get-by-path Node aa = nodeDAO.get(root, orig.getName()); @@ -456,6 +479,7 @@ public void testPutGetUpdateDeleteLinkNode() throws InterruptedException { Assert.assertEquals(orig.getName(), aa.getName()); Assert.assertNotNull(aa.parent); Assert.assertEquals(root.getID(), aa.parent.getID()); + Assert.assertEquals(root.getID(), aa.parentID); Assert.assertNull(a.parent); // get-node-by-id: comes pack without parent Assert.assertEquals(orig.getName(), a.getName()); @@ -474,6 +498,9 @@ public void testPutGetUpdateDeleteLinkNode() throws InterruptedException { Assert.assertEquals(orig.getMetaChecksum(), a.getMetaChecksum()); Assert.assertEquals(orig.getLastModified(), a.getLastModified()); + URI mcs = a.computeMetaChecksum(MessageDigest.getInstance("MD5")); + Assert.assertEquals("metaChecksum", a.getMetaChecksum(), mcs); + // update Thread.sleep(10L); orig.isPublic = false; @@ -550,7 +577,7 @@ public void testContainerNodeIterator() throws IOException { ContainerNode cont = new ContainerNode("container1", false); cont.parent = orig; cont.ownerID = orig.ownerID; - DataNode data = new DataNode("data1", URI.create("cadc:vault/" + UUID.randomUUID())); + DataNode data = new DataNode(UUID.randomUUID(), "data1", URI.create("cadc:vault/" + UUID.randomUUID())); data.parent = orig; data.ownerID = orig.ownerID; LinkNode link = new LinkNode("link1", URI.create("cadc:ARCHIVE/data")); diff --git a/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/SQLGenerator.java b/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/SQLGenerator.java index d8074515a..7aa14c010 100644 --- a/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/SQLGenerator.java +++ b/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/SQLGenerator.java @@ -476,7 +476,7 @@ public void setLocation(StorageLocation loc) { @Override public ObsoleteStorageLocation execute(JdbcTemplate jdbc) { - return (ObsoleteStorageLocation) jdbc.query(this, new ObsoleteStorageLocationExtractor()); + return jdbc.query(this, new ObsoleteStorageLocationExtractor()); } @Override @@ -1276,10 +1276,10 @@ public PreparedStatement createPreparedStatement(Connection conn) throws SQLExce PreparedStatement prep = conn.prepareStatement(sql); int col = 1; - if (value.parent == null) { - throw new RuntimeException("BUG: cannot put Node without a parent: " + value); + if (value.parentID == null) { + throw new RuntimeException("BUG: cannot put Node without a parentID: " + value); } - prep.setObject(col++, value.parent.getID()); + prep.setObject(col++, value.parentID); prep.setString(col++, value.getName()); prep.setString(col++, value.getClass().getSimpleName().substring(0, 1)); // HACK if (value.ownerID == null) { @@ -1299,11 +1299,11 @@ public PreparedStatement createPreparedStatement(Connection conn) throws SQLExce } if (value instanceof DataNode) { DataNode dn = (DataNode) value; - if (dn.getStorageID() == null) { + if (dn.storageID == null) { throw new RuntimeException("BUG: cannot put DataNode without a storageID: " + value); } safeSetBoolean(prep, col++, dn.busy); - safeSetString(prep, col++, dn.getStorageID()); + safeSetString(prep, col++, dn.storageID); } else { safeSetBoolean(prep, col++, null); safeSetString(prep, col++, (URI) null); @@ -1711,6 +1711,7 @@ private Node mapRowToNode(ResultSet rs, Calendar utc, ContainerNode parent) thro } else { throw new RuntimeException("BUG: unexpected node type code: " + nodeType); } + ret.parentID = parentID; ret.ownerID = ownerID; ret.isPublic = isPublic; ret.isLocked = isLocked; @@ -1738,12 +1739,12 @@ private Node mapRowToNode(ResultSet rs, Calendar utc, ContainerNode parent) thro return ret; } - private class ObsoleteStorageLocationExtractor implements ResultSetExtractor { + private class ObsoleteStorageLocationExtractor implements ResultSetExtractor { final Calendar utc = Calendar.getInstance(DateUtil.UTC); @Override - public Object extractData(ResultSet rs) throws SQLException, DataAccessException { + public ObsoleteStorageLocation extractData(ResultSet rs) throws SQLException, DataAccessException { if (!rs.next()) { return null; } @@ -1756,7 +1757,7 @@ public Object extractData(ResultSet rs) throws SQLException, DataAccessException StorageLocation s = new StorageLocation(storLoc); s.storageBucket = storBucket; - Entity ret = new ObsoleteStorageLocation(id, s); + ObsoleteStorageLocation ret = new ObsoleteStorageLocation(id, s); InventoryUtil.assignLastModified(ret, lastModified); InventoryUtil.assignMetaChecksum(ret, metaChecksum); return ret; diff --git a/cadc-inventory-db/src/main/java/org/opencadc/vospace/db/NodeDAO.java b/cadc-inventory-db/src/main/java/org/opencadc/vospace/db/NodeDAO.java index b4d36d562..1da5df287 100644 --- a/cadc-inventory-db/src/main/java/org/opencadc/vospace/db/NodeDAO.java +++ b/cadc-inventory-db/src/main/java/org/opencadc/vospace/db/NodeDAO.java @@ -93,6 +93,11 @@ public NodeDAO() { @Override public void put(Node val) { + // TBD: caller can assign parent or parentID before put, but here we need + // parentID and it must be assigned before metaChecksum compute in super.put() + if (val.parentID == null && val.parent != null) { + val.parentID = val.parent.getID(); + } super.put(val); } From 59e86b18f61e62b1dafbd789fd5b1e1270ba4add Mon Sep 17 00:00:00 2001 From: Patrick Dowler Date: Fri, 21 Apr 2023 12:26:03 -0700 Subject: [PATCH 09/52] vault: add init database, update README with additional config --- vault/README.md | 50 ++++- vault/build.gradle | 9 + .../opencadc/vault/NodePersistenceImpl.java | 182 ++++++++++++++++++ .../org/opencadc/vault/VaultInitAction.java | 49 +++-- 4 files changed, 271 insertions(+), 19 deletions(-) create mode 100644 vault/src/main/java/org/opencadc/vault/NodePersistenceImpl.java diff --git a/vault/README.md b/vault/README.md index 3dd77529a..20dad5cfa 100644 --- a/vault/README.md +++ b/vault/README.md @@ -1,8 +1,20 @@ -# Storage Inventory storage management service (vault) +# Storage Inventory VOSpace-2.1 service (vault) + +The `vault` servcie is an implementation of the IVOA VOSpace +specification designed to co-exist with other storage-inventory components. It provides a heirarchical data +organization laye on top of the storage management of storage-inventory. + +The simplest configuration would be to deploy `vault` with `minoc` with a single metadata database and single +back end storage system. Details: TBD. + +The other option would be to deploy `vault` with `raven` and `luskan` in a global inventory database and make +use of one or more of the network of known storage sites to store files. Details: TBD. ## configuration See the [cadc-tomcat](https://github.com/opencadc/docker-base/tree/master/cadc-tomcat) image docs -for expected deployment and general config requirements. The `vault` war file can be renamed +for expected deployment and general config requirements. + +The `vault` war file can be renamed at deployment time in order to support an alternate service name, including introducing additional path elements (see war-rename.conf). @@ -18,7 +30,7 @@ org.opencadc.vault.nodes.username={username for vospace pool} org.opencadc.vault.nodes.password={password for vospace pool} org.opencadc.vault.nodes.url=jdbc:postgresql://{server}/{database} ``` -The `nodes` account owns and manages (create, alter, drop) vault database objects and manages +The `nodes` account owns and manages (create, alter, drop) vospace database objects and manages all the content (insert, update, delete). The database is specified in the JDBC URL and the schema name is specified in the vault.properties (below). Failure to connect or initialize the database will show up in logs and in the VOSI-availability output. @@ -27,25 +39,47 @@ VOSI-availability output. A vault.properties file in /config is required to run this service. The following keys are required: ``` # service identity -org.opencadc.vault.resourceID=ivo://{authority}/{name} +org.opencadc.vault.resourceID = ivo://{authority}/{name} # vault database settings -org.opencadc.vault.nodes.schema={schema name} +org.opencadc.vault.inventory.schema = {inventory schema name} +org.opencadc.vault.vospace.schema = {vospace schema name} + +# root container nodes +org.opencadc.vault.root.owner = {owner of root node} + +# storage namespace +org.opencadc.vault.storage.namespace = {a storage inventory namespace to use} ``` The vault _resourceID_ is the resourceID of _this_ vault service. -The nodes _schema_ name is the name of the database schema used for all created database objects (tables, indices, etc). +The _inventory.schema_ name is the name of the database schema that contains the inventory database objects. The account nominally requires read-only (select) permission on those objects. + +The _vospace.schema_ name is the name of the database schema used for all created database objects (tables, indices, etc). Note that with a single connection pool, the two schemas must be in the same database and some operations may join tables +in the two schemas (probably just vospace.node join inventory.artifact). + +The root node owner has full read and write permission in the root container, so it can create and delete container +nodes at the root and to assign container node properties that are normally read-only to normal users: owner, quota, +etc. This is probably an X509 distingushed name of the user (to start). TBD. + +The _namespace_ configures `vault` to use the specified namespace in storage-inventory to store files. This only +applies to new data nodes that are created and will not effect previously created nodes and artifacts. Probably don't +want to change this... prevent change? ### vault-availability.properties (optional) ``` -The vault-availability.properties file specifies which users have the authority to change the availability state of the vault service. Each entry consists of a key=value pair. The key is always "users". The value is the x500 canonical user name. +The vault-availability.properties file specifies which users have the authority to change the availability state of +the vault service. Each entry consists of a key=value pair. The key is always "users". The value is the x500 canonical +user name. ``` Example: ``` users = {user identity} ``` -`users` specifies the user(s) who are authorized to make calls to the service. The value is a list of user identities (X500 distingushed name), one line per user. Optional: if the `vault-availability.properties` is not found or does not list any `users`, the service will function in the default mode (ReadWrite) and the state will not be changeable. +`users` specifies the user(s) who are authorized to make calls to the service. The value is a list of user identities +(X500 distingushed name), one line per user. Optional: if the `vault-availability.properties` is not found or does not +list any `users`, the service will function in the default mode (ReadWrite) and the state will not be changeable. ## building it ``` diff --git a/vault/build.gradle b/vault/build.gradle index dc09ec290..94f622694 100644 --- a/vault/build.gradle +++ b/vault/build.gradle @@ -26,14 +26,23 @@ war { } dependencies { + compile 'org.opencadc:cadc-util:[1.9,2.0)' compile 'org.opencadc:cadc-log:[1.1.6,2.0)' compile 'org.opencadc:cadc-vosi:[1.4.3,2.0)' + compile 'org.opencadc:cadc-vos:[2.0,3.0)' + //compile 'org.opencadc:cadc-vos-server:[2.0,3.0)' + compile 'org.opencadc:cadc-inventory:[0.9.4,1.0)' + compile 'org.opencadc:cadc-inventory-db:[0.15.0,1.0)' testCompile 'junit:junit:[4.0,)' + runtime 'org.opencadc:cadc-access-control-identity:[1.2.0,)' + runtime 'org.opencadc:cadc-gms:[1.0.4,)' + intTestCompile 'org.opencadc:cadc-test-vosi:[1.0.11,)' } configurations { + compile.exclude group: 'org.restlet.jee' runtime.exclude group: 'org.postgresql:postgresql' } diff --git a/vault/src/main/java/org/opencadc/vault/NodePersistenceImpl.java b/vault/src/main/java/org/opencadc/vault/NodePersistenceImpl.java new file mode 100644 index 000000000..3b770b479 --- /dev/null +++ b/vault/src/main/java/org/opencadc/vault/NodePersistenceImpl.java @@ -0,0 +1,182 @@ +/* +************************************************************************ +******************* CANADIAN ASTRONOMY DATA CENTRE ******************* +************** CENTRE CANADIEN DE DONNÉES ASTRONOMIQUES ************** +* +* (c) 2023. (c) 2023. +* Government of Canada Gouvernement du Canada +* National Research Council Conseil national de recherches +* Ottawa, Canada, K1A 0R6 Ottawa, Canada, K1A 0R6 +* All rights reserved Tous droits réservés +* +* NRC disclaims any warranties, Le CNRC dénie toute garantie +* expressed, implied, or énoncée, implicite ou légale, +* statutory, of any kind with de quelque nature que ce +* respect to the software, soit, concernant le logiciel, +* including without limitation y compris sans restriction +* any warranty of merchantability toute garantie de valeur +* or fitness for a particular marchande ou de pertinence +* purpose. NRC shall not be pour un usage particulier. +* liable in any event for any Le CNRC ne pourra en aucun cas +* damages, whether direct or être tenu responsable de tout +* indirect, special or general, dommage, direct ou indirect, +* consequential or incidental, particulier ou général, +* arising from the use of the accessoire ou fortuit, résultant +* software. Neither the name de l'utilisation du logiciel. Ni +* of the National Research le nom du Conseil National de +* Council of Canada nor the Recherches du Canada ni les noms +* names of its contributors may de ses participants ne peuvent +* be used to endorse or promote être utilisés pour approuver ou +* products derived from this promouvoir les produits dérivés +* software without specific prior de ce logiciel sans autorisation +* written permission. préalable et particulière +* par écrit. +* +* This file is part of the Ce fichier fait partie du projet +* OpenCADC project. OpenCADC. +* +* OpenCADC is free software: OpenCADC est un logiciel libre ; +* you can redistribute it and/or vous pouvez le redistribuer ou le +* modify it under the terms of modifier suivant les termes de +* the GNU Affero General Public la “GNU Affero General Public +* License as published by the License” telle que publiée +* Free Software Foundation, par la Free Software Foundation +* either version 3 of the : soit la version 3 de cette +* License, or (at your option) licence, soit (à votre gré) +* any later version. toute version ultérieure. +* +* OpenCADC is distributed in the OpenCADC est distribué +* hope that it will be useful, dans l’espoir qu’il vous +* but WITHOUT ANY WARRANTY; sera utile, mais SANS AUCUNE +* without even the implied GARANTIE : sans même la garantie +* warranty of MERCHANTABILITY implicite de COMMERCIALISABILITÉ +* or FITNESS FOR A PARTICULAR ni d’ADÉQUATION À UN OBJECTIF +* PURPOSE. See the GNU Affero PARTICULIER. Consultez la Licence +* General Public License for Générale Publique GNU Affero +* more details. pour plus de détails. +* +* You should have received Vous devriez avoir reçu une +* a copy of the GNU Affero copie de la Licence Générale +* General Public License along Publique GNU Affero avec +* with OpenCADC. If not, see OpenCADC ; si ce n’est +* . pas le cas, consultez : +* . +* +************************************************************************ +*/ + +package org.opencadc.vault; + +import ca.nrc.cadc.io.ResourceIterator; +import ca.nrc.cadc.net.TransientException; +import java.util.List; +import java.util.Map; +import java.util.TreeMap; +import org.apache.log4j.Logger; +import org.opencadc.inventory.db.SQLGenerator; +import org.opencadc.vospace.ContainerNode; +import org.opencadc.vospace.Node; +import org.opencadc.vospace.NodeNotFoundException; +import org.opencadc.vospace.NodeNotSupportedException; +import org.opencadc.vospace.NodeProperty; +import org.opencadc.vospace.db.NodeDAO; + +/** + * + * @author pdowler + */ +public class NodePersistenceImpl { //implements NodePersistence { + private static final Logger log = Logger.getLogger(NodePersistenceImpl.class); + + private final Map nodeDaoConfig = new TreeMap<>(); + + public NodePersistenceImpl(String dataSourceName, String inventorySchema, String vospaceSchema) { + nodeDaoConfig.put(SQLGenerator.class.getName(), SQLGenerator.class); + nodeDaoConfig.put("jndiDataSourceName", dataSourceName); + nodeDaoConfig.put("schema", inventorySchema); + nodeDaoConfig.put("vosSchema", vospaceSchema); + + } + + private NodeDAO getDAO() { + NodeDAO instance = new NodeDAO(); + instance.setConfig(nodeDaoConfig); + return instance; + } + + /** + * Get a node by name. + * @param parent parent node, may be special root node but not null + * @param name relative name of the child node + * @return the child node or null if it does not exist + * @throws TransientException + */ + public Node get(ContainerNode parent, String name) throws TransientException { + throw new UnsupportedOperationException(); + } + + /** + * Get an iterator over the children of a node. The output can optionally be + * limited to a specific number of children and can optionally start at a + * specific child (usually the last one from a previous "batch") to resume + * listing at a known position. + * + * @param parent the container to iterate + * @param limit max number of nodes to return, may be null + * @param start first node in order to consider, may be null + * @return iterator of matching child nodes, may be empty + */ + public ResourceIterator iterator(ContainerNode parent, Integer limit, String start) { + throw new UnsupportedOperationException(); + } + + /** + * Load additional node properties for the specified node. Note: this may not be + * necessary and may be removed. TBD. + * + * @param node + * @throws TransientException + */ + public void getProperties(Node node) throws TransientException { + throw new UnsupportedOperationException(); + } + + /** + * Put the specified node. This can be an insert or update; to update, the argument + * node must have been retrieved from persistence so it has the right Entity.id + * value. This method may modify the Entity.metaChecksum and the Entity.lastModified + * values. + * + * @param node the node to insert or update + * @return the possibly modified node + * @throws NodeNotSupportedException + * @throws TransientException + */ + public Node put(Node node) throws NodeNotSupportedException, TransientException { + // TODO: assign node.parentID here and remove from NodeDAO? + // TODO: assign DataNode.storageID here -- only when null aka new DataNode? + throw new UnsupportedOperationException(); + } + + /** + * This can be done via put(Node) so probably obsolete. TBD. + * @param node + * @param list + * @return + * @throws TransientException + */ + public Node updateProperties(Node node, List list) throws TransientException { + throw new UnsupportedOperationException(); + } + + /** + * Delete the specified node. + * + * @param node the node to delete + * @throws TransientException + */ + public void delete(Node node) throws TransientException { + throw new UnsupportedOperationException(); + } + +} diff --git a/vault/src/main/java/org/opencadc/vault/VaultInitAction.java b/vault/src/main/java/org/opencadc/vault/VaultInitAction.java index 2f6a34f71..e606d2ad0 100644 --- a/vault/src/main/java/org/opencadc/vault/VaultInitAction.java +++ b/vault/src/main/java/org/opencadc/vault/VaultInitAction.java @@ -77,6 +77,8 @@ import java.util.TreeMap; import javax.sql.DataSource; import org.apache.log4j.Logger; +import org.opencadc.inventory.Namespace; +import org.opencadc.vospace.db.InitDatabaseVOS; /** * @@ -90,10 +92,16 @@ public class VaultInitAction extends InitAction { // config keys private static final String VAULT_KEY = "org.opencadc.vault"; static final String RESOURCE_ID_KEY = VAULT_KEY + ".resourceID"; - static final String SCHEMA_KEY = VAULT_KEY + ".nodes.schema"; + static final String INVENTORY_SCHEMA_KEY = VAULT_KEY + ".inventory.schema"; + static final String VOSPACE_SCHEMA_KEY = VAULT_KEY + ".vospace.schema"; + + static final String ROOT_OWNER = VAULT_KEY + ".root.owner"; // numeric? + + static final String STORAGE_NAMESPACE_KEY = VAULT_KEY + ".storage.namespace"; MultiValuedProperties props; private URI resourceID; + private Namespace storageNamespace; private Map daoConfig; public VaultInitAction() { @@ -129,15 +137,33 @@ static MultiValuedProperties getConfig() { sb.append("OK"); } - String schema = mvp.getFirstPropertyValue(SCHEMA_KEY); - sb.append("\n\t").append(SCHEMA_KEY).append(": "); - if (schema == null) { + String invSchema = mvp.getFirstPropertyValue(INVENTORY_SCHEMA_KEY); + sb.append("\n\t").append(INVENTORY_SCHEMA_KEY).append(": "); + if (invSchema == null) { sb.append("MISSING"); ok = false; } else { sb.append("OK"); } - + + String vosSchema = mvp.getFirstPropertyValue(VOSPACE_SCHEMA_KEY); + sb.append("\n\t").append(VOSPACE_SCHEMA_KEY).append(": "); + if (vosSchema == null) { + sb.append("MISSING"); + ok = false; + } else { + sb.append("OK"); + } + + String ns = mvp.getFirstPropertyValue(STORAGE_NAMESPACE_KEY); + sb.append("\n\t").append(STORAGE_NAMESPACE_KEY).append(": "); + if (ns == null) { + sb.append("MISSING"); + ok = false; + } else { + sb.append("OK"); + } + if (!ok) { throw new IllegalStateException(sb.toString()); } @@ -148,7 +174,8 @@ static MultiValuedProperties getConfig() { static Map getDaoConfig(MultiValuedProperties props) { Map ret = new TreeMap<>(); ret.put("jndiDataSourceName", org.opencadc.vault.VaultInitAction.JNDI_DATASOURCE); - ret.put("schema", props.getFirstPropertyValue(org.opencadc.vault.VaultInitAction.SCHEMA_KEY)); + ret.put("schema", props.getFirstPropertyValue(org.opencadc.vault.VaultInitAction.INVENTORY_SCHEMA_KEY)); + ret.put("vosSchema", props.getFirstPropertyValue(org.opencadc.vault.VaultInitAction.VOSPACE_SCHEMA_KEY)); return ret; } @@ -156,9 +183,10 @@ private void initConfig() { log.info("initConfig: START"); this.props = getConfig(); String rid = props.getFirstPropertyValue(RESOURCE_ID_KEY); - + String ns = props.getFirstPropertyValue(STORAGE_NAMESPACE_KEY); try { this.resourceID = new URI(rid); + this.storageNamespace = new Namespace(ns); this.daoConfig = getDaoConfig(props); log.info("initConfig: OK"); } catch (URISyntaxException ex) { @@ -170,10 +198,9 @@ private void initDatabase() { log.info("initDatabase: START"); try { DataSource ds = DBUtil.findJNDIDataSource(JNDI_DATASOURCE); - String database = (String) daoConfig.get("database"); - String schema = (String) daoConfig.get("schema"); - //VaultInitDatabase init = new VaultInitDatabase(ds, database, schema); - //init.doInit(); + String schema = (String) daoConfig.get("vosSchema"); + InitDatabaseVOS init = new InitDatabaseVOS(ds, null, schema); + init.doInit(); log.info("initDatabase: " + JNDI_DATASOURCE + " " + schema + " OK"); } catch (Exception ex) { throw new IllegalStateException("check/init database failed", ex); From 34bfcd852f0d632e48c1826ecd1f3871e71cae70 Mon Sep 17 00:00:00 2001 From: Patrick Dowler Date: Fri, 21 Apr 2023 13:56:53 -0700 Subject: [PATCH 10/52] vault: implement more NodePersistenceImpl methods add TODO --- vault/TODO | 6 + .../opencadc/vault/NodePersistenceImpl.java | 209 ++++++++++++++++-- 2 files changed, 200 insertions(+), 15 deletions(-) create mode 100644 vault/TODO diff --git a/vault/TODO b/vault/TODO new file mode 100644 index 000000000..003ad444f --- /dev/null +++ b/vault/TODO @@ -0,0 +1,6 @@ + +NodePersistenceImpl: review for necessary transactions and locks + +NodePersistenceImpl: reconcile with NodePersistence API + + diff --git a/vault/src/main/java/org/opencadc/vault/NodePersistenceImpl.java b/vault/src/main/java/org/opencadc/vault/NodePersistenceImpl.java index 3b770b479..afa32fc85 100644 --- a/vault/src/main/java/org/opencadc/vault/NodePersistenceImpl.java +++ b/vault/src/main/java/org/opencadc/vault/NodePersistenceImpl.java @@ -69,16 +69,24 @@ import ca.nrc.cadc.io.ResourceIterator; import ca.nrc.cadc.net.TransientException; +import ca.nrc.cadc.util.MultiValuedProperties; +import java.io.IOException; +import java.net.URI; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.TreeMap; +import java.util.TreeSet; +import java.util.UUID; import org.apache.log4j.Logger; +import org.opencadc.inventory.Namespace; import org.opencadc.inventory.db.SQLGenerator; import org.opencadc.vospace.ContainerNode; +import org.opencadc.vospace.DataNode; import org.opencadc.vospace.Node; -import org.opencadc.vospace.NodeNotFoundException; import org.opencadc.vospace.NodeNotSupportedException; import org.opencadc.vospace.NodeProperty; +import org.opencadc.vospace.VOS; import org.opencadc.vospace.db.NodeDAO; /** @@ -89,13 +97,66 @@ public class NodePersistenceImpl { //implements NodePersistence { private static final Logger log = Logger.getLogger(NodePersistenceImpl.class); private final Map nodeDaoConfig = new TreeMap<>(); + private final ContainerNode root; + private final ContainerNode trash; + private final Namespace storageNamespace; + private final Set nonWritableProps = new TreeSet<>(); + private final Set rootAdminProps = new TreeSet<>(); + private final Set directNodeProps = new TreeSet<>(); - public NodePersistenceImpl(String dataSourceName, String inventorySchema, String vospaceSchema) { + public NodePersistenceImpl() { + MultiValuedProperties config = VaultInitAction.getConfig(); + String dataSourceName = VaultInitAction.JNDI_DATASOURCE; + String inventorySchema = config.getFirstPropertyValue(VaultInitAction.INVENTORY_SCHEMA_KEY); + String vospaceSchema = config.getFirstPropertyValue(VaultInitAction.VOSPACE_SCHEMA_KEY); nodeDaoConfig.put(SQLGenerator.class.getName(), SQLGenerator.class); nodeDaoConfig.put("jndiDataSourceName", dataSourceName); nodeDaoConfig.put("schema", inventorySchema); nodeDaoConfig.put("vosSchema", vospaceSchema); + + UUID rootID = new UUID(0L, 0L); + this.root = new ContainerNode(rootID, "", false); + String owner = config.getFirstPropertyValue(VaultInitAction.ROOT_OWNER); + //root.owner = ?? // subject needed to authorize admin requests + //root.ownerID not needed + + // TODO: do this setup in a txn with a lock on something + NodeDAO dao = getDAO(); + ContainerNode tn = (ContainerNode) dao.get(root, ".trash"); + if (tn == null) { + tn = new ContainerNode(".trash", false); + tn.ownerID = root.ownerID; + tn.owner = root.owner; + tn.isPublic = false; + dao.put(tn); + } + this.trash = tn; + + String ns = config.getFirstPropertyValue(VaultInitAction.STORAGE_NAMESPACE_KEY); + this.storageNamespace = new Namespace(ns); + + // node properties that match immutable Artifact fields + nonWritableProps.add(VOS.PROPERTY_URI_CONTENTLENGTH); // data nodes + nonWritableProps.add(VOS.PROPERTY_URI_CONTENTMD5); // data nodes + nonWritableProps.add(VOS.PROPERTY_URI_CREATION_DATE); // no touch + + // computed properties + nonWritableProps.add(VOS.PROPERTY_URI_WRITABLE); // prediction for current caller + + // props only the root admin can modify + rootAdminProps.add(VOS.PROPERTY_URI_AVAILABLESPACE); + rootAdminProps.add(VOS.PROPERTY_URI_CONTENTLENGTH); // container nodes + rootAdminProps.add(VOS.PROPERTY_URI_QUOTA); + rootAdminProps.add(VOS.PROPERTY_URI_CREATOR); // owner + + // props that are stored as Node fields + directNodeProps.add(VOS.PROPERTY_URI_GROUPREAD); + directNodeProps.add(VOS.PROPERTY_URI_GROUPWRITE); + directNodeProps.add(VOS.PROPERTY_URI_INHERIT_PERMISSIONS); + directNodeProps.add(VOS.PROPERTY_URI_ISLOCKED); + directNodeProps.add(VOS.PROPERTY_URI_ISPUBLIC); + } private NodeDAO getDAO() { @@ -103,16 +164,41 @@ private NodeDAO getDAO() { instance.setConfig(nodeDaoConfig); return instance; } + + private URI generateStorageID() { + UUID id = UUID.randomUUID(); + URI ret = URI.create(storageNamespace.getNamespace() + id.toString()); + return ret; + } /** - * Get a node by name. + * Get the container node that represents the root of all other nodes. + * This container node is used to navigate a path (from the root) using + * get(ContainerNode parent, String name). + * + * @return the root container node + */ + public ContainerNode getRootNode() { + return root; + } + + /** + * Get a node by name. Concept: The caller uses this to navigate the path from the root + * node to the target, checking permissions and deciding what to do about + * LinkNode(s) along the way. + * * @param parent parent node, may be special root node but not null * @param name relative name of the child node * @return the child node or null if it does not exist * @throws TransientException */ public Node get(ContainerNode parent, String name) throws TransientException { - throw new UnsupportedOperationException(); + if (parent == null || name == null) { + throw new IllegalArgumentException("args cannot be null: parent, name"); + } + NodeDAO dao = getDAO(); + Node ret = dao.get(parent, name); + return ret; } /** @@ -127,7 +213,12 @@ public Node get(ContainerNode parent, String name) throws TransientException { * @return iterator of matching child nodes, may be empty */ public ResourceIterator iterator(ContainerNode parent, Integer limit, String start) { - throw new UnsupportedOperationException(); + if (parent == null) { + throw new IllegalArgumentException("arg cannot be null: parent"); + } + NodeDAO dao = getDAO(); + ResourceIterator ret = dao.iterator(parent, limit, start); + return ret; } /** @@ -153,20 +244,86 @@ public void getProperties(Node node) throws TransientException { * @throws TransientException */ public Node put(Node node) throws NodeNotSupportedException, TransientException { - // TODO: assign node.parentID here and remove from NodeDAO? - // TODO: assign DataNode.storageID here -- only when null aka new DataNode? - throw new UnsupportedOperationException(); + if (node == null) { + throw new IllegalArgumentException("arg cannot be null: node"); + } + if (node.parentID == null) { + if (node.parent == null) { + throw new RuntimeException("BUG: cannot persist node without parent: " + node); + } + node.parentID = node.parent.getID(); + } + if (node instanceof DataNode) { + DataNode dn = (DataNode) node; + if (dn.storageID == null) { + // new data node? if lastModified is assigned, this looks sketchy + if (dn.getLastModified() != null) { + throw new RuntimeException( + "BUG: attempt to put a previously stored DataNode without persistent storageID: " + + dn.getID() + " aka " + dn); + } + // concept: use a persistent storageID in the node that resolves to a a file + // once someone puts the file to minoc, so Node.storageID == Artifact.uri + // but the artifact may or may not exist + dn.storageID = generateStorageID(); + } + } + NodeDAO dao = getDAO(); + dao.put(node); + return node; } /** - * This can be done via put(Node) so probably obsolete. TBD. - * @param node - * @param list - * @return + * Update properties of a node. This method is responsible for accepting/rejecting changes + * based on whether a node property is read-only or writable by the caller (implementation + * specific). + * + * @param node the node to update + * @param props new property values to set + * @return the modified node * @throws TransientException */ - public Node updateProperties(Node node, List list) throws TransientException { - throw new UnsupportedOperationException(); + public Node updateProperties(Node node, List props) throws TransientException { + if (node == null || props == null) { + throw new IllegalArgumentException("args cannot be null: node, props"); + } + + // merge props -> node and/or node.properties + for (NodeProperty np : props) { + if (nonWritableProps.contains(np.getKey())) { + log.debug("updateProperties: skip " + np.getKey()); + } else { + // some props only root admin can modify + if (rootAdminProps.contains(np.getKey())) { + throw new UnsupportedOperationException("TODO: allow admin to modify prop: " + np.getKey()); + } + + // some props are directly in the node + if (VOS.PROPERTY_URI_FORMAT.equals(np.getKey()) + || VOS.PROPERTY_URI_CONTENTENCODING.equals(np.getKey())) { + throw new UnsupportedOperationException("TODO: set mutable artifact prop: " + np.getKey()); + } + + // some props are directly in the artifact + if (directNodeProps.contains(np.getKey())) { + throw new UnsupportedOperationException("TODO: set mutable node prop: " + np.getKey()); + } + + // generic key-value props + if (node.properties.contains(np)) { + log.debug("updateProperties: remove previous " + np.getKey()); + node.properties.remove(np); + } + if (!np.isMarkedForDeletion()) { + node.properties.add(np); + } + } + } + + NodeDAO dao = getDAO(); + dao.put(node); + return node; + } /** @@ -176,7 +333,29 @@ public Node updateProperties(Node node, List list) throws Transien * @throws TransientException */ public void delete(Node node) throws TransientException { - throw new UnsupportedOperationException(); + if (node == null) { + throw new IllegalArgumentException("arg cannot be null: node"); + } + + NodeDAO dao = getDAO(); + boolean moveToTrash = false; + if (node instanceof ContainerNode) { + ContainerNode cn = (ContainerNode) node; + try (ResourceIterator iter = dao.iterator(cn, 1, null)) { + moveToTrash = !iter.hasNext(); + } catch (IOException ex) { + throw new TransientException("database IO failure", ex); + } + } + + // TODO: DeletedNodeEvent + if (moveToTrash) { + node.parentID = trash.getID(); + dao.put(node); + } else { + dao.delete(node.getID()); + } + } } From 8082aab99651c47181977b1fc86c9e6d2e1c9891 Mon Sep 17 00:00:00 2001 From: Patrick Dowler Date: Fri, 21 Apr 2023 13:58:12 -0700 Subject: [PATCH 11/52] NodeDAO: remove assign parentID in put --- .../java/org/opencadc/vospace/db/NodeDAOTest.java | 6 +++--- .../src/main/java/org/opencadc/vospace/db/NodeDAO.java | 9 +-------- 2 files changed, 4 insertions(+), 11 deletions(-) diff --git a/cadc-inventory-db/src/intTest/java/org/opencadc/vospace/db/NodeDAOTest.java b/cadc-inventory-db/src/intTest/java/org/opencadc/vospace/db/NodeDAOTest.java index cbab0c4ba..145db59a7 100644 --- a/cadc-inventory-db/src/intTest/java/org/opencadc/vospace/db/NodeDAOTest.java +++ b/cadc-inventory-db/src/intTest/java/org/opencadc/vospace/db/NodeDAOTest.java @@ -559,13 +559,13 @@ public void testContainerNodeIterator() throws IOException { // these are set in put Assert.assertEquals(orig.getMetaChecksum(), a.getMetaChecksum()); Assert.assertEquals(orig.getLastModified(), a.getLastModified()); - try (ResourceIterator emptyIter = nodeDAO.iterator(orig)) { + try (ResourceIterator emptyIter = nodeDAO.iterator(orig, null, null)) { Assert.assertNotNull(emptyIter); Assert.assertFalse(emptyIter.hasNext()); } // auto-close Node top = null; - try (ResourceIterator rootIter = nodeDAO.iterator(root)) { + try (ResourceIterator rootIter = nodeDAO.iterator(root, null, null)) { if (rootIter.hasNext()) { top = rootIter.next(); } @@ -593,7 +593,7 @@ public void testContainerNodeIterator() throws IOException { Node c1; Node c2; Node c3; - try (ResourceIterator iter = nodeDAO.iterator(orig)) { + try (ResourceIterator iter = nodeDAO.iterator(orig, null, null)) { Assert.assertNotNull(iter); Assert.assertTrue(iter.hasNext()); c1 = iter.next(); diff --git a/cadc-inventory-db/src/main/java/org/opencadc/vospace/db/NodeDAO.java b/cadc-inventory-db/src/main/java/org/opencadc/vospace/db/NodeDAO.java index 1da5df287..6fc157862 100644 --- a/cadc-inventory-db/src/main/java/org/opencadc/vospace/db/NodeDAO.java +++ b/cadc-inventory-db/src/main/java/org/opencadc/vospace/db/NodeDAO.java @@ -68,15 +68,12 @@ package org.opencadc.vospace.db; import ca.nrc.cadc.io.ResourceIterator; -import java.io.IOException; import java.util.UUID; import org.apache.log4j.Logger; -import org.opencadc.inventory.Artifact; import org.opencadc.inventory.db.AbstractDAO; import org.opencadc.inventory.db.SQLGenerator; import org.opencadc.vospace.ContainerNode; import org.opencadc.vospace.Node; -import org.opencadc.vospace.VOSURI; import org.springframework.jdbc.BadSqlGrammarException; import org.springframework.jdbc.core.JdbcTemplate; @@ -128,11 +125,7 @@ public void delete(UUID id) { super.delete(Node.class, id); } - public ResourceIterator iterator(ContainerNode parent) { - return NodeDAO.this.iterator(parent, null, null); - } - - public ResourceIterator iterator(ContainerNode parent, String start, Integer limit) { + public ResourceIterator iterator(ContainerNode parent, Integer limit, String start) { if (parent == null) { throw new IllegalArgumentException("childIterator: parent cannot be null"); } From 87c68bbb34b80ff6f07375a0839d612b9044f8a3 Mon Sep 17 00:00:00 2001 From: Patrick Dowler Date: Fri, 21 Apr 2023 13:59:15 -0700 Subject: [PATCH 12/52] cadc-inventory-db: nominally assign version as 0.15 --- cadc-inventory-db/build.gradle | 2 +- fenwick/build.gradle | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/cadc-inventory-db/build.gradle b/cadc-inventory-db/build.gradle index 940b17d71..2f1566ca2 100644 --- a/cadc-inventory-db/build.gradle +++ b/cadc-inventory-db/build.gradle @@ -17,7 +17,7 @@ sourceCompatibility = 1.8 group = 'org.opencadc' -version = '0.14.5' +version = '0.15.0' description = 'OpenCADC Storage Inventory database library' def git_url = 'https://github.com/opencadc/storage-inventory' diff --git a/fenwick/build.gradle b/fenwick/build.gradle index 15e17f012..6d9c2cdb8 100644 --- a/fenwick/build.gradle +++ b/fenwick/build.gradle @@ -16,8 +16,8 @@ group = 'org.opencadc' dependencies { compile 'org.opencadc:cadc-util:[1.9.6,2.0)' - compile 'org.opencadc:cadc-inventory:[0.9.4,2.0)' - compile 'org.opencadc:cadc-inventory-db:[0.14.5,2.0)' + compile 'org.opencadc:cadc-inventory:[0.9.4,1.0)' + compile 'org.opencadc:cadc-inventory-db:[0.14.5,1.0)' compile 'org.opencadc:cadc-inventory-util:[0.1.8,1.0)' compile 'org.opencadc:cadc-registry:[1.5,2.0)' compile 'org.opencadc:cadc-tap:[1.1.14,1.2)' // 1.2 upper bound is correct #reasons From 41dc8dcdee3d5a206670b815939fc5183826be02 Mon Sep 17 00:00:00 2001 From: Patrick Dowler Date: Fri, 21 Apr 2023 14:12:48 -0700 Subject: [PATCH 13/52] update TODO --- vault/TODO | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/vault/TODO b/vault/TODO index 003ad444f..e26e4199d 100644 --- a/vault/TODO +++ b/vault/TODO @@ -1,6 +1,15 @@ -NodePersistenceImpl: review for necessary transactions and locks +* NodePersistenceImpl: review for necessary transactions and locks -NodePersistenceImpl: reconcile with NodePersistence API +* NodePersistenceImpl: reconcile with NodePersistence API +* files endpoint: +- if coexist with minoc: generate pre-auth URL to it and redirect +- if coexist with raven: need raven ProtocolsGenerator + +* transfer negotiation: +- review cadc-vos-server and cavern implementations +- probably a complete TransferRunner; maybe separate sync and async runners +- if co-exist with minoc: generate pre-auth URL to it +- if co-exist with raven: need raven ProtocolsGenerator From 2512aa35c2686dd002449d9cc11bfb861c593e5e Mon Sep 17 00:00:00 2001 From: Patrick Dowler Date: Fri, 28 Apr 2023 09:09:34 -0700 Subject: [PATCH 14/52] vault: TODO updates --- vault/TODO | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/vault/TODO b/vault/TODO index e26e4199d..1481088d1 100644 --- a/vault/TODO +++ b/vault/TODO @@ -1,7 +1,10 @@ * NodePersistenceImpl: review for necessary transactions and locks -* NodePersistenceImpl: reconcile with NodePersistence API +* NodePersistenceImpl: reconcile with NodePersistence API and assign responsibilities +- property update checking +- permission checking +- link node resolution * files endpoint: - if coexist with minoc: generate pre-auth URL to it and redirect @@ -12,4 +15,8 @@ - probably a complete TransferRunner; maybe separate sync and async runners - if co-exist with minoc: generate pre-auth URL to it - if co-exist with raven: need raven ProtocolsGenerator +- figure out if/how vault can have it's own uws tables in db or share with inventory (luskan) +* pre-auth URL keys -- what to support? recommend? +- vault has it's own key pair && minoc(s) have multiple pub keys? +- vault and raven share private key? From 39f51eaf398aa0237af45d549c88a0eb305f7a94 Mon Sep 17 00:00:00 2001 From: Patrick Dowler Date: Tue, 2 May 2023 10:29:44 -0700 Subject: [PATCH 15/52] vault: update README --- vault/README.md | 41 ++++++++++++++++++++++++++--------------- 1 file changed, 26 insertions(+), 15 deletions(-) diff --git a/vault/README.md b/vault/README.md index 20dad5cfa..67d202847 100644 --- a/vault/README.md +++ b/vault/README.md @@ -10,19 +10,30 @@ back end storage system. Details: TBD. The other option would be to deploy `vault` with `raven` and `luskan` in a global inventory database and make use of one or more of the network of known storage sites to store files. Details: TBD. -## configuration -See the [cadc-tomcat](https://github.com/opencadc/docker-base/tree/master/cadc-tomcat) image docs -for expected deployment and general config requirements. +## deployment -The `vault` war file can be renamed -at deployment time in order to support an alternate service name, including introducing -additional path elements (see war-rename.conf). +The `vault` war file can be renamed at deployment time in order to support an alternate service name, +including introducing additional path elements. See +cadc-tomcat (war-rename.conf). -Runtime configuration must be made available via the `/config` directory. +## configuration +The following runtime configuration must be made available via the `/config` directory. ### catalina.properties -When running vault.war in tomcat, parameters of the connection pool in META-INF/context.xml need -to be configured in catalina.properties: +This file contains java system properties to configure the tomcat server and some of the java libraries used in the service. + +See cadc-tomcat +for system properties related to the deployment environment. + +See cadc-util +for common system properties. + +`minoc` includes multiple IdentityManager implementations to support authenticated access: +- See cadc-access-control-identity for CADC access-control system support. + +- See cadc-gms for OIDC token support. + +`vault` requires a connection pool to the local database: ``` # database connection pools org.opencadc.vault.nodes.maxActive={max connections for vospace pool} @@ -53,25 +64,25 @@ org.opencadc.vault.storage.namespace = {a storage inventory namespace to use} ``` The vault _resourceID_ is the resourceID of _this_ vault service. -The _inventory.schema_ name is the name of the database schema that contains the inventory database objects. The account nominally requires read-only (select) permission on those objects. +The _inventory.schema_ name is the name of the database schema that contains the inventory database objects. The account nominally requires read-only (select) permission on those objects. This currently must be "inventory" due to configuration +limitations in luskan. The _vospace.schema_ name is the name of the database schema used for all created database objects (tables, indices, etc). Note that with a single connection pool, the two schemas must be in the same database and some operations may join tables in the two schemas (probably just vospace.node join inventory.artifact). The root node owner has full read and write permission in the root container, so it can create and delete container -nodes at the root and to assign container node properties that are normally read-only to normal users: owner, quota, -etc. This is probably an X509 distingushed name of the user (to start). TBD. +nodes at the root and assign container node properties that are normally read-only to normal users: owner, quota, +etc. This is probably an X509 distingushed name of the user (to start). **not fully implemented** TBD. The _namespace_ configures `vault` to use the specified namespace in storage-inventory to store files. This only applies to new data nodes that are created and will not effect previously created nodes and artifacts. Probably don't -want to change this... prevent change? +want to change this... prevent change? TBD. ### vault-availability.properties (optional) -``` + The vault-availability.properties file specifies which users have the authority to change the availability state of the vault service. Each entry consists of a key=value pair. The key is always "users". The value is the x500 canonical user name. -``` Example: ``` From 83381d9192504d7b31688aa2a99d82c89ca2c633 Mon Sep 17 00:00:00 2001 From: Patrick Dowler Date: Tue, 2 May 2023 10:32:57 -0700 Subject: [PATCH 16/52] add vault to repo README --- README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/README.md b/README.md index 2838d9b05..1b2fbf4da 100644 --- a/README.md +++ b/README.md @@ -46,6 +46,10 @@ local artifact selection policy. This is used if the new fenwick policy excludes This is an implementation of the **file-validate** process that compares the inventory database against the back end storage at a storage site. +## vault +UNDER DEVELOPMENT: This is an implementation of an IVOA VOSpace +service that uses storage-inventory as the back end storage mechanism. + ## cadc-* These are libraries used in multiple services and applications. From c542400a1604c27ed42664a596a9f7983eaeb2df Mon Sep 17 00:00:00 2001 From: Patrick Dowler Date: Wed, 31 May 2023 14:05:00 -0700 Subject: [PATCH 17/52] update for Node API changes --- .../org/opencadc/vospace/db/NodeDAOTest.java | 122 +++++++++--------- .../opencadc/inventory/db/SQLGenerator.java | 12 +- .../java/org/opencadc/inventory/db/Util.java | 4 +- 3 files changed, 70 insertions(+), 68 deletions(-) diff --git a/cadc-inventory-db/src/intTest/java/org/opencadc/vospace/db/NodeDAOTest.java b/cadc-inventory-db/src/intTest/java/org/opencadc/vospace/db/NodeDAOTest.java index 145db59a7..b3aa5d1ef 100644 --- a/cadc-inventory-db/src/intTest/java/org/opencadc/vospace/db/NodeDAOTest.java +++ b/cadc-inventory-db/src/intTest/java/org/opencadc/vospace/db/NodeDAOTest.java @@ -203,9 +203,9 @@ public void testPutGetUpdateDeleteContainerNode() throws InterruptedException, Assert.assertEquals(orig.ownerID, a.ownerID); Assert.assertEquals(orig.isPublic, a.isPublic); Assert.assertEquals(orig.isLocked, a.isLocked); - Assert.assertEquals(orig.readOnlyGroup, a.readOnlyGroup); - Assert.assertEquals(orig.readWriteGroup, a.readWriteGroup); - Assert.assertEquals(orig.properties, a.properties); + Assert.assertEquals(orig.getReadOnlyGroup(), a.getReadOnlyGroup()); + Assert.assertEquals(orig.getReadWriteGroup(), a.getReadWriteGroup()); + Assert.assertEquals(orig.getProperties(), a.getProperties()); Assert.assertTrue(a instanceof ContainerNode); ContainerNode c = (ContainerNode) a; @@ -220,9 +220,9 @@ public void testPutGetUpdateDeleteContainerNode() throws InterruptedException, // update Thread.sleep(10L); - orig.readOnlyGroup.add(URI.create("ivo://opencadc.org/gms?g1")); - orig.readWriteGroup.add(URI.create("ivo://opencadc.org/gms?g3")); - orig.properties.add(new NodeProperty(VOS.PROPERTY_URI_CONTENTLENGTH, "123")); + orig.getReadOnlyGroup().add(URI.create("ivo://opencadc.org/gms?g1")); + orig.getReadWriteGroup().add(URI.create("ivo://opencadc.org/gms?g3")); + orig.getProperties().add(new NodeProperty(VOS.PROPERTY_URI_CONTENTLENGTH, "123")); orig.isPublic = true; nodeDAO.put(orig); Node updated = nodeDAO.get(orig.getID()); @@ -237,9 +237,9 @@ public void testPutGetUpdateDeleteContainerNode() throws InterruptedException, Assert.assertEquals(orig.ownerID, updated.ownerID); Assert.assertEquals(orig.isPublic, updated.isPublic); Assert.assertEquals(orig.isLocked, updated.isLocked); - Assert.assertEquals(orig.readOnlyGroup, updated.readOnlyGroup); - Assert.assertEquals(orig.readWriteGroup, updated.readWriteGroup); - Assert.assertEquals(orig.properties, updated.properties); + Assert.assertEquals(orig.getReadOnlyGroup(), updated.getReadOnlyGroup()); + Assert.assertEquals(orig.getReadWriteGroup(), updated.getReadWriteGroup()); + Assert.assertEquals(orig.getProperties(), updated.getProperties()); Assert.assertTrue(updated instanceof ContainerNode); ContainerNode uc = (ContainerNode) updated; @@ -265,19 +265,19 @@ public void testPutGetUpdateDeleteContainerNodeMax() throws InterruptedException orig.isPublic = true; orig.isLocked = false; orig.inheritPermissions = false; - orig.readOnlyGroup.add(URI.create("ivo://opencadc.org/gms?g1")); - orig.readOnlyGroup.add(URI.create("ivo://opencadc.org/gms?g2")); - orig.readWriteGroup.add(URI.create("ivo://opencadc.org/gms?g3")); - orig.readWriteGroup.add(URI.create("ivo://opencadc.org/gms?g4,g5")); - orig.readWriteGroup.add(URI.create("ivo://opencadc.org/gms?g6-g7")); - orig.readWriteGroup.add(URI.create("ivo://opencadc.org/gms?g6.g7")); - orig.readWriteGroup.add(URI.create("ivo://opencadc.org/gms?g6_g7")); - orig.readWriteGroup.add(URI.create("ivo://opencadc.org/gms?g6~g7")); - - orig.properties.add(new NodeProperty(VOS.PROPERTY_URI_CONTENTLENGTH, "123")); - orig.properties.add(new NodeProperty(URI.create("custom:prop"), "spaces in value")); - orig.properties.add(new NodeProperty(URI.create("sketchy:a,b"), "comma in uri")); - orig.properties.add(new NodeProperty(URI.create("sketchy:funny"), "value-with-{delims}")); + orig.getReadOnlyGroup().add(URI.create("ivo://opencadc.org/gms?g1")); + orig.getReadOnlyGroup().add(URI.create("ivo://opencadc.org/gms?g2")); + orig.getReadWriteGroup().add(URI.create("ivo://opencadc.org/gms?g3")); + orig.getReadWriteGroup().add(URI.create("ivo://opencadc.org/gms?g4,g5")); + orig.getReadWriteGroup().add(URI.create("ivo://opencadc.org/gms?g6-g7")); + orig.getReadWriteGroup().add(URI.create("ivo://opencadc.org/gms?g6.g7")); + orig.getReadWriteGroup().add(URI.create("ivo://opencadc.org/gms?g6_g7")); + orig.getReadWriteGroup().add(URI.create("ivo://opencadc.org/gms?g6~g7")); + + orig.getProperties().add(new NodeProperty(VOS.PROPERTY_URI_CONTENTLENGTH, "123")); + orig.getProperties().add(new NodeProperty(URI.create("custom:prop"), "spaces in value")); + orig.getProperties().add(new NodeProperty(URI.create("sketchy:a,b"), "comma in uri")); + orig.getProperties().add(new NodeProperty(URI.create("sketchy:funny"), "value-with-{delims}")); nodeDAO.put(orig); // get-by-id @@ -303,9 +303,9 @@ public void testPutGetUpdateDeleteContainerNodeMax() throws InterruptedException Assert.assertEquals(orig.ownerID, a.ownerID); Assert.assertEquals(orig.isPublic, a.isPublic); Assert.assertEquals(orig.isLocked, a.isLocked); - Assert.assertEquals(orig.readOnlyGroup, a.readOnlyGroup); - Assert.assertEquals(orig.readWriteGroup, a.readWriteGroup); - Assert.assertEquals(orig.properties, a.properties); + Assert.assertEquals(orig.getReadOnlyGroup(), a.getReadOnlyGroup()); + Assert.assertEquals(orig.getReadWriteGroup(), a.getReadWriteGroup()); + Assert.assertEquals(orig.getProperties(), a.getProperties()); Assert.assertTrue(a instanceof ContainerNode); ContainerNode c = (ContainerNode) a; @@ -322,12 +322,12 @@ public void testPutGetUpdateDeleteContainerNodeMax() throws InterruptedException Thread.sleep(10L); orig.isPublic = false; orig.isLocked = true; - orig.readOnlyGroup.clear(); - orig.readOnlyGroup.add(URI.create("ivo://opencadc.org/gms?g1")); - orig.readWriteGroup.clear(); - orig.readWriteGroup.add(URI.create("ivo://opencadc.org/gms?g3")); - orig.properties.clear(); - orig.properties.add(new NodeProperty(VOS.PROPERTY_URI_CONTENTLENGTH, "123")); + orig.getReadOnlyGroup().clear(); + orig.getReadOnlyGroup().add(URI.create("ivo://opencadc.org/gms?g1")); + orig.getReadWriteGroup().clear(); + orig.getReadWriteGroup().add(URI.create("ivo://opencadc.org/gms?g3")); + orig.getProperties().clear(); + orig.getProperties().add(new NodeProperty(VOS.PROPERTY_URI_CONTENTLENGTH, "123")); orig.inheritPermissions = true; nodeDAO.put(orig); Node updated = nodeDAO.get(orig.getID()); @@ -342,9 +342,9 @@ public void testPutGetUpdateDeleteContainerNodeMax() throws InterruptedException Assert.assertEquals(orig.ownerID, updated.ownerID); Assert.assertEquals(orig.isPublic, updated.isPublic); Assert.assertEquals(orig.isLocked, updated.isLocked); - Assert.assertEquals(orig.readOnlyGroup, updated.readOnlyGroup); - Assert.assertEquals(orig.readWriteGroup, updated.readWriteGroup); - Assert.assertEquals(orig.properties, updated.properties); + Assert.assertEquals(orig.getReadOnlyGroup(), updated.getReadOnlyGroup()); + Assert.assertEquals(orig.getReadWriteGroup(), updated.getReadWriteGroup()); + Assert.assertEquals(orig.getProperties(), updated.getProperties()); Assert.assertTrue(updated instanceof ContainerNode); ContainerNode uc = (ContainerNode) updated; @@ -366,8 +366,8 @@ public void testPutGetUpdateDeleteDataNode() throws InterruptedException, orig.ownerID = "the-owner"; orig.isPublic = true; orig.isLocked = false; - orig.properties.add(new NodeProperty(VOS.PROPERTY_URI_TYPE, "text/plain")); - orig.properties.add(new NodeProperty(VOS.PROPERTY_URI_DESCRIPTION, "this is the good stuff(tm)")); + orig.getProperties().add(new NodeProperty(VOS.PROPERTY_URI_TYPE, "text/plain")); + orig.getProperties().add(new NodeProperty(VOS.PROPERTY_URI_DESCRIPTION, "this is the good stuff(tm)")); nodeDAO.put(orig); // get-by-id @@ -394,9 +394,9 @@ public void testPutGetUpdateDeleteDataNode() throws InterruptedException, Assert.assertEquals(orig.ownerID, a.ownerID); Assert.assertEquals(orig.isPublic, a.isPublic); Assert.assertEquals(orig.isLocked, a.isLocked); - Assert.assertEquals(orig.readOnlyGroup, a.readOnlyGroup); - Assert.assertEquals(orig.readWriteGroup, a.readWriteGroup); - Assert.assertEquals(orig.properties, a.properties); + Assert.assertEquals(orig.getReadOnlyGroup(), a.getReadOnlyGroup()); + Assert.assertEquals(orig.getReadWriteGroup(), a.getReadWriteGroup()); + Assert.assertEquals(orig.getProperties(), a.getProperties()); Assert.assertTrue(a instanceof DataNode); DataNode dn = (DataNode) a; @@ -413,12 +413,12 @@ public void testPutGetUpdateDeleteDataNode() throws InterruptedException, Thread.sleep(10L); orig.isPublic = false; orig.isLocked = true; - orig.readOnlyGroup.clear(); - orig.readOnlyGroup.add(URI.create("ivo://opencadc.org/gms?g1")); - orig.readWriteGroup.clear(); - orig.readWriteGroup.add(URI.create("ivo://opencadc.org/gms?g3")); - orig.properties.clear(); - orig.properties.add(new NodeProperty(VOS.PROPERTY_URI_CONTENTLENGTH, "123")); + orig.getReadOnlyGroup().clear(); + orig.getReadOnlyGroup().add(URI.create("ivo://opencadc.org/gms?g1")); + orig.getReadWriteGroup().clear(); + orig.getReadWriteGroup().add(URI.create("ivo://opencadc.org/gms?g3")); + orig.getProperties().clear(); + orig.getProperties().add(new NodeProperty(VOS.PROPERTY_URI_CONTENTLENGTH, "123")); // don't change storageID nodeDAO.put(orig); Node updated = nodeDAO.get(orig.getID()); @@ -433,9 +433,9 @@ public void testPutGetUpdateDeleteDataNode() throws InterruptedException, Assert.assertEquals(orig.ownerID, updated.ownerID); Assert.assertEquals(orig.isPublic, updated.isPublic); Assert.assertEquals(orig.isLocked, updated.isLocked); - Assert.assertEquals(orig.readOnlyGroup, updated.readOnlyGroup); - Assert.assertEquals(orig.readWriteGroup, updated.readWriteGroup); - Assert.assertEquals(orig.properties, updated.properties); + Assert.assertEquals(orig.getReadOnlyGroup(), updated.getReadOnlyGroup()); + Assert.assertEquals(orig.getReadWriteGroup(), updated.getReadWriteGroup()); + Assert.assertEquals(orig.getProperties(), updated.getProperties()); Assert.assertTrue(a instanceof DataNode); @@ -460,7 +460,7 @@ public void testPutGetUpdateDeleteLinkNode() throws InterruptedException, orig.ownerID = "the-owner"; orig.isPublic = true; orig.isLocked = false; - orig.properties.add(new NodeProperty(VOS.PROPERTY_URI_DESCRIPTION, "link to the good stuff(tm)")); + orig.getProperties().add(new NodeProperty(VOS.PROPERTY_URI_DESCRIPTION, "link to the good stuff(tm)")); nodeDAO.put(orig); // get-by-id @@ -486,9 +486,9 @@ public void testPutGetUpdateDeleteLinkNode() throws InterruptedException, Assert.assertEquals(orig.ownerID, a.ownerID); Assert.assertEquals(orig.isPublic, a.isPublic); Assert.assertEquals(orig.isLocked, a.isLocked); - Assert.assertEquals(orig.readOnlyGroup, a.readOnlyGroup); - Assert.assertEquals(orig.readWriteGroup, a.readWriteGroup); - Assert.assertEquals(orig.properties, a.properties); + Assert.assertEquals(orig.getReadOnlyGroup(), a.getReadOnlyGroup()); + Assert.assertEquals(orig.getReadWriteGroup(), a.getReadWriteGroup()); + Assert.assertEquals(orig.getProperties(), a.getProperties()); Assert.assertTrue(a instanceof LinkNode); LinkNode link = (LinkNode) a; @@ -505,12 +505,12 @@ public void testPutGetUpdateDeleteLinkNode() throws InterruptedException, Thread.sleep(10L); orig.isPublic = false; orig.isLocked = true; - orig.readOnlyGroup.clear(); - orig.readOnlyGroup.add(URI.create("ivo://opencadc.org/gms?g1")); - orig.readWriteGroup.clear(); - orig.readWriteGroup.add(URI.create("ivo://opencadc.org/gms?g3")); - orig.properties.clear(); - orig.properties.add(new NodeProperty(VOS.PROPERTY_URI_CONTENTLENGTH, "123")); + orig.getReadOnlyGroup().clear(); + orig.getReadOnlyGroup().add(URI.create("ivo://opencadc.org/gms?g1")); + orig.getReadWriteGroup().clear(); + orig.getReadWriteGroup().add(URI.create("ivo://opencadc.org/gms?g3")); + orig.getProperties().clear(); + orig.getProperties().add(new NodeProperty(VOS.PROPERTY_URI_CONTENTLENGTH, "123")); // don't change target nodeDAO.put(orig); Node updated = nodeDAO.get(orig.getID()); @@ -525,9 +525,9 @@ public void testPutGetUpdateDeleteLinkNode() throws InterruptedException, Assert.assertEquals(orig.ownerID, updated.ownerID); Assert.assertEquals(orig.isPublic, updated.isPublic); Assert.assertEquals(orig.isLocked, updated.isLocked); - Assert.assertEquals(orig.readOnlyGroup, updated.readOnlyGroup); - Assert.assertEquals(orig.readWriteGroup, updated.readWriteGroup); - Assert.assertEquals(orig.properties, updated.properties); + Assert.assertEquals(orig.getReadOnlyGroup(), updated.getReadOnlyGroup()); + Assert.assertEquals(orig.getReadWriteGroup(), updated.getReadWriteGroup()); + Assert.assertEquals(orig.getProperties(), updated.getProperties()); Assert.assertTrue(updated instanceof LinkNode); LinkNode ulink = (LinkNode) updated; diff --git a/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/SQLGenerator.java b/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/SQLGenerator.java index 7aa14c010..97d27e985 100644 --- a/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/SQLGenerator.java +++ b/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/SQLGenerator.java @@ -1288,9 +1288,9 @@ public PreparedStatement createPreparedStatement(Connection conn) throws SQLExce prep.setString(col++, value.ownerID.toString()); safeSetBoolean(prep, col++, value.isPublic); safeSetBoolean(prep, col++, value.isLocked); - safeSetArray(prep, col++, value.readOnlyGroup); - safeSetArray(prep, col++, value.readWriteGroup); - safeSetProps(prep, col++, value.properties); + safeSetArray(prep, col++, value.getReadOnlyGroup()); + safeSetArray(prep, col++, value.getReadWriteGroup()); + safeSetProps(prep, col++, value.getProperties()); if (value instanceof ContainerNode) { ContainerNode cn = (ContainerNode) value; safeSetBoolean(prep, col++, cn.inheritPermissions); @@ -1717,13 +1717,13 @@ private Node mapRowToNode(ResultSet rs, Calendar utc, ContainerNode parent) thro ret.isLocked = isLocked; if (rawROG != null) { - Util.parseArrayURI(rawROG, ret.readOnlyGroup); + Util.parseArrayURI(rawROG, ret.getReadOnlyGroup()); } if (rawRWG != null) { - Util.parseArrayURI(rawRWG, ret.readWriteGroup); + Util.parseArrayURI(rawRWG, ret.getReadWriteGroup()); } if (rawProps != null) { - Util.parseArrayProps(rawProps, ret.properties); + Util.parseArrayProps(rawProps, ret.getProperties()); } InventoryUtil.assignLastModified(ret, lastModified); diff --git a/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/Util.java b/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/Util.java index cd20020bf..2e7e197b8 100644 --- a/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/Util.java +++ b/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/Util.java @@ -383,6 +383,7 @@ public static byte[] getByteArray(ResultSet rs, int col) throw new UnsupportedOperationException("converting " + o.getClass().getName() + " " + o + " to byte[]"); } + // fills the dest set public static void parseArrayURI(String val, Set dest) { // postgresql 1D array: {a,"b,c"} if (val == null || val.isEmpty()) { @@ -417,7 +418,8 @@ private static void handleToken(String token, Set dest) { } } } - + + // fills the dest set public static void parseArrayProps(String val, Set dest) { // postgresql 2D array: {{a,b},{c,d}} if (val == null || val.isEmpty()) { From c1e02401a4a8294f057d97c08f6ac60e2d3bda54 Mon Sep 17 00:00:00 2001 From: Patrick Dowler Date: Thu, 1 Jun 2023 13:24:09 -0700 Subject: [PATCH 18/52] vault: NodePersistenceImpl implements extracted interface --- .../opencadc/vault/NodePersistenceImpl.java | 299 +++++++++++++----- 1 file changed, 217 insertions(+), 82 deletions(-) diff --git a/vault/src/main/java/org/opencadc/vault/NodePersistenceImpl.java b/vault/src/main/java/org/opencadc/vault/NodePersistenceImpl.java index afa32fc85..706499811 100644 --- a/vault/src/main/java/org/opencadc/vault/NodePersistenceImpl.java +++ b/vault/src/main/java/org/opencadc/vault/NodePersistenceImpl.java @@ -67,22 +67,37 @@ package org.opencadc.vault; +import ca.nrc.cadc.auth.AuthenticationUtil; +import ca.nrc.cadc.auth.IdentityManager; +import ca.nrc.cadc.auth.PrincipalExtractor; +import ca.nrc.cadc.auth.X509CertificateChain; +import ca.nrc.cadc.date.DateUtil; import ca.nrc.cadc.io.ResourceIterator; import ca.nrc.cadc.net.TransientException; +import ca.nrc.cadc.util.InvalidConfigException; import ca.nrc.cadc.util.MultiValuedProperties; import java.io.IOException; import java.net.URI; +import java.security.Principal; +import java.text.DateFormat; +import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Set; import java.util.TreeMap; import java.util.TreeSet; import java.util.UUID; +import javax.security.auth.Subject; +import javax.security.auth.x500.X500Principal; import org.apache.log4j.Logger; +import org.opencadc.inventory.Artifact; import org.opencadc.inventory.Namespace; +import org.opencadc.inventory.db.ArtifactDAO; import org.opencadc.inventory.db.SQLGenerator; import org.opencadc.vospace.ContainerNode; import org.opencadc.vospace.DataNode; +import org.opencadc.vospace.DeletedNodeEvent; +import org.opencadc.vospace.LinkNode; import org.opencadc.vospace.Node; import org.opencadc.vospace.NodeNotSupportedException; import org.opencadc.vospace.NodeProperty; @@ -93,16 +108,18 @@ * * @author pdowler */ -public class NodePersistenceImpl { //implements NodePersistence { +public class NodePersistenceImpl implements NodePersistence { private static final Logger log = Logger.getLogger(NodePersistenceImpl.class); private final Map nodeDaoConfig = new TreeMap<>(); private final ContainerNode root; private final ContainerNode trash; private final Namespace storageNamespace; - private final Set nonWritableProps = new TreeSet<>(); + private final Set immutableProps = new TreeSet<>(); + + private final Set artifactProps = new TreeSet<>(); + private final Set rootAdminContainerProps = new TreeSet<>(); private final Set rootAdminProps = new TreeSet<>(); - private final Set directNodeProps = new TreeSet<>(); public NodePersistenceImpl() { MultiValuedProperties config = VaultInitAction.getConfig(); @@ -115,48 +132,65 @@ public NodePersistenceImpl() { nodeDaoConfig.put("vosSchema", vospaceSchema); + final String owner = config.getFirstPropertyValue(VaultInitAction.ROOT_OWNER); + if (owner == null) { + throw new InvalidConfigException(VaultInitAction.ROOT_OWNER + " cannot be null"); + } + IdentityManager im = AuthenticationUtil.getIdentityManager(); + Subject rawOwnerSubject = AuthenticationUtil.getSubject(new PrincipalExtractor() { + @Override + public Set getPrincipals() { + Set ret = new TreeSet<>(); + ret.add(new X500Principal(owner)); + return ret; + } + + @Override + public X509CertificateChain getCertificateChain() { + return null; + } + }); + + // root node UUID rootID = new UUID(0L, 0L); this.root = new ContainerNode(rootID, "", false); - String owner = config.getFirstPropertyValue(VaultInitAction.ROOT_OWNER); - //root.owner = ?? // subject needed to authorize admin requests - //root.ownerID not needed + root.owner = im.augment(rawOwnerSubject); + root.ownerID = im.toDisplayString(root.owner); + // trash node // TODO: do this setup in a txn with a lock on something NodeDAO dao = getDAO(); ContainerNode tn = (ContainerNode) dao.get(root, ".trash"); if (tn == null) { tn = new ContainerNode(".trash", false); - tn.ownerID = root.ownerID; - tn.owner = root.owner; - tn.isPublic = false; - dao.put(tn); } + // always reset props to current config + tn.ownerID = root.ownerID; + tn.owner = root.owner; + tn.isPublic = false; + dao.put(tn); this.trash = tn; String ns = config.getFirstPropertyValue(VaultInitAction.STORAGE_NAMESPACE_KEY); this.storageNamespace = new Namespace(ns); - // node properties that match immutable Artifact fields - nonWritableProps.add(VOS.PROPERTY_URI_CONTENTLENGTH); // data nodes - nonWritableProps.add(VOS.PROPERTY_URI_CONTENTMD5); // data nodes - nonWritableProps.add(VOS.PROPERTY_URI_CREATION_DATE); // no touch - // computed properties - nonWritableProps.add(VOS.PROPERTY_URI_WRITABLE); // prediction for current caller + // VOS.PROPERTY_URI_AVAILABLESPACE // container nodes + // VOS.PROPERTY_URI_WRITABLE // prediction for current caller - // props only the root admin can modify - rootAdminProps.add(VOS.PROPERTY_URI_AVAILABLESPACE); - rootAdminProps.add(VOS.PROPERTY_URI_CONTENTLENGTH); // container nodes - rootAdminProps.add(VOS.PROPERTY_URI_QUOTA); + // props only the admin (root owner) can modify rootAdminProps.add(VOS.PROPERTY_URI_CREATOR); // owner + rootAdminContainerProps.add(VOS.PROPERTY_URI_CONTENTLENGTH); // container nodes + rootAdminContainerProps.add(VOS.PROPERTY_URI_QUOTA); // container nodes - // props that are stored as Node fields - directNodeProps.add(VOS.PROPERTY_URI_GROUPREAD); - directNodeProps.add(VOS.PROPERTY_URI_GROUPWRITE); - directNodeProps.add(VOS.PROPERTY_URI_INHERIT_PERMISSIONS); - directNodeProps.add(VOS.PROPERTY_URI_ISLOCKED); - directNodeProps.add(VOS.PROPERTY_URI_ISPUBLIC); + // node properties that match immutable Artifact fields + immutableProps.add(VOS.PROPERTY_URI_CONTENTLENGTH); // immutable + immutableProps.add(VOS.PROPERTY_URI_CONTENTMD5); // immutable + immutableProps.add(VOS.PROPERTY_URI_CREATION_DATE); // immutable + artifactProps.addAll(immutableProps); + artifactProps.add(VOS.PROPERTY_URI_CONTENTENCODING); // mutable + artifactProps.add(VOS.PROPERTY_URI_TYPE); // mutable } private NodeDAO getDAO() { @@ -165,6 +199,12 @@ private NodeDAO getDAO() { return instance; } + private ArtifactDAO getArtifactDAO() { + ArtifactDAO instance = new ArtifactDAO(true); // origin==true? + instance.setConfig(nodeDaoConfig); + return instance; + } + private URI generateStorageID() { UUID id = UUID.randomUUID(); URI ret = URI.create(storageNamespace.getNamespace() + id.toString()); @@ -178,26 +218,51 @@ private URI generateStorageID() { * * @return the root container node */ + @Override public ContainerNode getRootNode() { return root; } - + + @Override + public Set getImmutableProps() { + return immutableProps; + } + /** - * Get a node by name. Concept: The caller uses this to navigate the path from the root - * node to the target, checking permissions and deciding what to do about - * LinkNode(s) along the way. + * Get a node by name. Concept: The caller uses this to navigate the path + * from the root node to the target, checking permissions and deciding what + * to do about LinkNode(s) along the way. * * @param parent parent node, may be special root node but not null * @param name relative name of the child node * @return the child node or null if it does not exist * @throws TransientException */ + @Override public Node get(ContainerNode parent, String name) throws TransientException { if (parent == null || name == null) { throw new IllegalArgumentException("args cannot be null: parent, name"); } NodeDAO dao = getDAO(); Node ret = dao.get(parent, name); + if (ret instanceof DataNode) { + DataNode dn = (DataNode) ret; + ArtifactDAO artifactDAO = getArtifactDAO(); + Artifact a = artifactDAO.get(dn.storageID); + if (a != null) { + DateFormat df = DateUtil.getDateFormat(DateUtil.IVOA_DATE_FORMAT, DateUtil.UTC); + ret.getProperties().add(new NodeProperty(VOS.PROPERTY_URI_CONTENTLENGTH, a.getContentLength().toString())); + // assume MD5 + ret.getProperties().add(new NodeProperty(VOS.PROPERTY_URI_CONTENTMD5, a.getContentChecksum().getSchemeSpecificPart())); + ret.getProperties().add(new NodeProperty(VOS.PROPERTY_URI_DATE, df.format(a.getContentLastModified()))); + if (a.contentEncoding != null) { + ret.getProperties().add(new NodeProperty(VOS.PROPERTY_URI_CONTENTENCODING, a.contentEncoding)); + } + if (a.contentType != null) { + ret.getProperties().add(new NodeProperty(VOS.PROPERTY_URI_TYPE, a.contentType)); + } + } + } return ret; } @@ -212,6 +277,7 @@ public Node get(ContainerNode parent, String name) throws TransientException { * @param start first node in order to consider, may be null * @return iterator of matching child nodes, may be empty */ + @Override public ResourceIterator iterator(ContainerNode parent, Integer limit, String start) { if (parent == null) { throw new IllegalArgumentException("arg cannot be null: parent"); @@ -228,8 +294,9 @@ public ResourceIterator iterator(ContainerNode parent, Integer limit, Stri * @param node * @throws TransientException */ + @Override public void getProperties(Node node) throws TransientException { - throw new UnsupportedOperationException(); + // no-op } /** @@ -243,6 +310,7 @@ public void getProperties(Node node) throws TransientException { * @throws NodeNotSupportedException * @throws TransientException */ + @Override public Node put(Node node) throws NodeNotSupportedException, TransientException { if (node == null) { throw new IllegalArgumentException("arg cannot be null: node"); @@ -266,56 +334,46 @@ public Node put(Node node) throws NodeNotSupportedException, TransientException // once someone puts the file to minoc, so Node.storageID == Artifact.uri // but the artifact may or may not exist dn.storageID = generateStorageID(); - } - } - NodeDAO dao = getDAO(); - dao.put(node); - return node; - } - - /** - * Update properties of a node. This method is responsible for accepting/rejecting changes - * based on whether a node property is read-only or writable by the caller (implementation - * specific). - * - * @param node the node to update - * @param props new property values to set - * @return the modified node - * @throws TransientException - */ - public Node updateProperties(Node node, List props) throws TransientException { - if (node == null || props == null) { - throw new IllegalArgumentException("args cannot be null: node, props"); - } - - // merge props -> node and/or node.properties - for (NodeProperty np : props) { - if (nonWritableProps.contains(np.getKey())) { - log.debug("updateProperties: skip " + np.getKey()); } else { - // some props only root admin can modify - if (rootAdminProps.contains(np.getKey())) { - throw new UnsupportedOperationException("TODO: allow admin to modify prop: " + np.getKey()); - } - - // some props are directly in the node - if (VOS.PROPERTY_URI_FORMAT.equals(np.getKey()) - || VOS.PROPERTY_URI_CONTENTENCODING.equals(np.getKey())) { - throw new UnsupportedOperationException("TODO: set mutable artifact prop: " + np.getKey()); - } + // update existing data node + // need to remove all artifact props from the node.getProperties() + // and use artifactDAO to set the mutable ones + NodeProperty contentType = null; + NodeProperty contentEncoding = null; - // some props are directly in the artifact - if (directNodeProps.contains(np.getKey())) { - throw new UnsupportedOperationException("TODO: set mutable node prop: " + np.getKey()); - } - - // generic key-value props - if (node.properties.contains(np)) { - log.debug("updateProperties: remove previous " + np.getKey()); - node.properties.remove(np); + Iterator i = dn.getProperties().iterator(); + while (i.hasNext()) { + NodeProperty np = i.next(); + if (VOS.PROPERTY_URI_TYPE.equals(np.getKey())) { + contentType = np; + } else if (VOS.PROPERTY_URI_CONTENTENCODING.equals(np.getKey())) { + contentEncoding = np; + } + + if (artifactProps.contains(np.getKey())) { + i.remove(); + } } - if (!np.isMarkedForDeletion()) { - node.properties.add(np); + if (contentType != null || contentEncoding != null) { // optimization + ArtifactDAO artifactDAO = getArtifactDAO(); + Artifact a = artifactDAO.get(dn.storageID); + if (a != null) { + if (contentType != null) { + if (contentType.isMarkedForDeletion()) { + a.contentType = null; + } else { + a.contentType = contentType.getValue(); + } + } + if (contentEncoding != null) { + if (contentEncoding.isMarkedForDeletion()) { + a.contentEncoding = null; + } else { + a.contentEncoding = contentEncoding.getValue(); + } + } + artifactDAO.put(a); + } } } } @@ -323,7 +381,6 @@ public Node updateProperties(Node node, List props) throws Transie NodeDAO dao = getDAO(); dao.put(node); return node; - } /** @@ -332,23 +389,47 @@ public Node updateProperties(Node node, List props) throws Transie * @param node the node to delete * @throws TransientException */ + @Override public void delete(Node node) throws TransientException { if (node == null) { throw new IllegalArgumentException("arg cannot be null: node"); } NodeDAO dao = getDAO(); - boolean moveToTrash = false; + + // TODO: do the following in a transaction, acquire lock on target node + + boolean moveToTrash = true; // default + URI storageID = null; if (node instanceof ContainerNode) { ContainerNode cn = (ContainerNode) node; try (ResourceIterator iter = dao.iterator(cn, 1, null)) { - moveToTrash = !iter.hasNext(); + moveToTrash = iter.hasNext(); // empty } catch (IOException ex) { throw new TransientException("database IO failure", ex); } + } else if (node instanceof LinkNode) { + moveToTrash = false; + } else if (node instanceof DataNode) { + DataNode dn = (DataNode) node; + NodeProperty len = dn.getProperty(VOS.PROPERTY_URI_CONTENTLENGTH); + if (len == null) { + // artifact does not exist + moveToTrash = false; + } else { + storageID = dn.storageID; + } } - // TODO: DeletedNodeEvent + // TODO: create DeletedNodeEvent + // what about DNE for all child nodes, which also got deleted? or would sync of a + // DNE involve calling NodePersistence.delete(DeletedNodeEvent) to replay this same + // deletion logic in a mirror?? TBD + // persisting the DNE means that recovery from trash has to re-assign IDs + + DeletedNodeEvent dne = new DeletedNodeEvent(node.getID(), node.getClass(), storageID); + // TODO: need DeletedNodeDAO + if (moveToTrash) { node.parentID = trash.getID(); dao.put(node); @@ -358,4 +439,58 @@ public void delete(Node node) throws TransientException { } + + // this code is incomplete but shows the logic of merging the requested property changes + // into the node before put(node)... keeping it here for reference + public Node updateProperties(Node node, List props) + throws TransientException, NodeNotSupportedException { + if (node == null || props == null) { + throw new IllegalArgumentException("args cannot be null: node, props"); + } + + // merge props -> node and/or node.properties and/or mutable artifact + for (NodeProperty np : props) { + boolean writable = (node instanceof ContainerNode && this.rootAdminContainerProps.contains(np.getKey())); + writable = writable || rootAdminProps.contains(np.getKey()); + if (writable) { + // some props are directly in the node + if (artifactProps.contains(np.getKey())) { + throw new UnsupportedOperationException("TODO: set mutable artifact prop: " + np.getKey()); + } + + // some props are directly in the node + if (VOS.PROPERTY_URI_GROUPREAD.equals(np.getKey())) { + String raw = np.getValue(); + throw new UnsupportedOperationException(); + } else if (VOS.PROPERTY_URI_GROUPWRITE.equals(np.getKey())) { + String raw = np.getValue(); + throw new UnsupportedOperationException(); + } else if (VOS.PROPERTY_URI_INHERIT_PERMISSIONS.equals(np.getKey())) { + String raw = np.getValue(); + throw new UnsupportedOperationException(); + } else if (VOS.PROPERTY_URI_ISLOCKED.equals(np.getKey())) { + String raw = np.getValue(); + throw new UnsupportedOperationException(); + } else if (VOS.PROPERTY_URI_ISPUBLIC.equals(np.getKey())) { + String raw = np.getValue(); + throw new UnsupportedOperationException(); + } + + // generic key-value props + if (node.getProperties().contains(np)) { + // remove previous; covers mark for deletion case + node.getProperties().remove(np); + } + + if (!np.isMarkedForDeletion()) { + // add new + node.getProperties().add(np); + } + } else { + log.debug("updateProperties: skip non-writable " + np.getKey()); + } + } + + return put(node); + } } From dc1e726de11a502612f8092fe3c9c6c43313e808 Mon Sep 17 00:00:00 2001 From: Adrian Damian Date: Fri, 16 Jun 2023 10:40:52 +0800 Subject: [PATCH 19/52] Checkpoint --- vault/Dockerfile | 5 +- vault/build.gradle | 25 +++- .../opencadc/vault/NodePersistenceImpl.java | 6 +- vault/src/main/webapp/WEB-INF/web.xml | 117 +++++++++++------- vault/src/main/webapp/capabilities.xml | 11 ++ 5 files changed, 108 insertions(+), 56 deletions(-) diff --git a/vault/Dockerfile b/vault/Dockerfile index f3bc94674..4f35f9ff5 100644 --- a/vault/Dockerfile +++ b/vault/Dockerfile @@ -1,3 +1,4 @@ -FROM cadc-tomcat:1 +FROM images.opencadc.org/library/cadc-tomcat:1 + +COPY build/libs/vault.war /usr/share/tomcat/webapps/vault.war -COPY build/libs/vault.war /usr/share/tomcat/webapps/ diff --git a/vault/build.gradle b/vault/build.gradle index 94f622694..a07901381 100644 --- a/vault/build.gradle +++ b/vault/build.gradle @@ -24,21 +24,36 @@ war { include 'VERSION' } } +description = 'OpenCADC VOSpace server' +def git_url = 'https://github.com/opencadc/vos' dependencies { - compile 'org.opencadc:cadc-util:[1.9,2.0)' - compile 'org.opencadc:cadc-log:[1.1.6,2.0)' - compile 'org.opencadc:cadc-vosi:[1.4.3,2.0)' - compile 'org.opencadc:cadc-vos:[2.0,3.0)' - //compile 'org.opencadc:cadc-vos-server:[2.0,3.0)' + compile 'javax.servlet:javax.servlet-api:[3.1,4.0)' + + compile 'org.opencadc:cadc-util:[1.9.5,2.0)' + compile 'org.opencadc:cadc-gms:[1.0.0,)' + compile 'org.opencadc:cadc-rest:[1.3.16,)' + compile 'org.opencadc:cadc-vos:[2.0,)' + compile 'org.opencadc:cadc-vos-server-2.0:[2.0,)' + compile 'org.opencadc:cadc-vosi:[1.3.2,)' + compile 'org.opencadc:cadc-uws:[1.0,)' + compile 'org.opencadc:cadc-uws-server:[1.2.12,)' + compile 'org.opencadc:cadc-access-control:[1.1.1,2.0)' + compile 'org.opencadc:cadc-cdp:[1.2.3,)' + compile 'org.opencadc:cadc-registry:[1.5.15,)' compile 'org.opencadc:cadc-inventory:[0.9.4,1.0)' compile 'org.opencadc:cadc-inventory-db:[0.15.0,1.0)' testCompile 'junit:junit:[4.0,)' + testCompile 'org.easymock:easymock:3.6' + testCompile 'org.skyscreamer:jsonassert:[1.0,)' runtime 'org.opencadc:cadc-access-control-identity:[1.2.0,)' runtime 'org.opencadc:cadc-gms:[1.0.4,)' + // JDBC drivers + intTestRuntime 'org.postgresql:postgresql:[42.2.8,)' + intTestRuntime 'net.sourceforge.jtds:jtds:[1.0,)' intTestCompile 'org.opencadc:cadc-test-vosi:[1.0.11,)' } diff --git a/vault/src/main/java/org/opencadc/vault/NodePersistenceImpl.java b/vault/src/main/java/org/opencadc/vault/NodePersistenceImpl.java index afa32fc85..885ab980c 100644 --- a/vault/src/main/java/org/opencadc/vault/NodePersistenceImpl.java +++ b/vault/src/main/java/org/opencadc/vault/NodePersistenceImpl.java @@ -310,12 +310,12 @@ public Node updateProperties(Node node, List props) throws Transie } // generic key-value props - if (node.properties.contains(np)) { + if (node.getProperties().contains(np)) { log.debug("updateProperties: remove previous " + np.getKey()); - node.properties.remove(np); + node.getProperties().remove(np); } if (!np.isMarkedForDeletion()) { - node.properties.add(np); + node.getProperties().add(np); } } } diff --git a/vault/src/main/webapp/WEB-INF/web.xml b/vault/src/main/webapp/WEB-INF/web.xml index 2d519aa70..d28360085 100644 --- a/vault/src/main/webapp/WEB-INF/web.xml +++ b/vault/src/main/webapp/WEB-INF/web.xml @@ -5,7 +5,6 @@ "http://java.sun.com/j2ee/dtds/web-app_2_3.dtd"> - vault @@ -20,6 +19,7 @@ org.opencadc.vault ca.nrc.cadc.db + org.opencadc.vospace ca.nrc.cadc.rest ca.nrc.cadc.util ca.nrc.cadc.vosi @@ -34,51 +34,71 @@ NodesServlet - ca.nrc.cadc.rest.RestServlet - - init - org.opencadc.vault.VaultInitAction - - 2 - + ca.nrc.cadc.rest.RestServlet + + augmentSubject + false + + + init + org.opencadc.vault.VaultInitAction + + + get + org.opencadc.vospace.server.web.actions.GetNodeAction + + + put + org.opencadc.vospace.server.web.actions.CreateNodeAction + + + post + org.opencadc.vospace.server.web.actions.UpdateNodeAction + + + delete + org.opencadc.vospace.server.web.actions.DeleteNodeAction + + 2 + + + + + CapabilitiesServlet + ca.nrc.cadc.rest.RestServlet + + init + ca.nrc.cadc.vosi.CapInitAction + + + head + ca.nrc.cadc.vosi.CapHeadAction + + + get + ca.nrc.cadc.vosi.CapGetAction + + + input + /capabilities.xml + + 3 + - - - CapabilitiesServlet - ca.nrc.cadc.rest.RestServlet - - init - ca.nrc.cadc.vosi.CapInitAction - - - head - ca.nrc.cadc.vosi.CapHeadAction - - - get - ca.nrc.cadc.vosi.CapGetAction - - - input - /capabilities.xml - - 3 - - - - - AvailabilityServlet - ca.nrc.cadc.vosi.AvailabilityServlet - - ca.nrc.cadc.vosi.AvailabilityPlugin - org.opencadc.vault.ServiceAvailability - - - availabilityProperties - vault-availability.properties - - 4 - + + + AvailabilityServlet + ca.nrc.cadc.vosi.AvailabilityServlet + + ca.nrc.cadc.vosi.AvailabilityPlugin + org.opencadc.vospace.ServiceAvailability + + + availabilityProperties + vault-availability.properties + + 4 + @@ -86,7 +106,7 @@ NodesServlet /nodes/* - + AvailabilityServlet @@ -105,4 +125,9 @@ /logControl + + index.html + + + diff --git a/vault/src/main/webapp/capabilities.xml b/vault/src/main/webapp/capabilities.xml index 0a6205003..9663ecec2 100644 --- a/vault/src/main/webapp/capabilities.xml +++ b/vault/src/main/webapp/capabilities.xml @@ -4,6 +4,16 @@ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:vs="http://www.ivoa.net/xml/VODataService/v1.1"> + + + https://replace.com/vault/nodes + + + + + + + https://replace.com/vault/capabilities @@ -26,3 +36,4 @@ + From 6a8e3542d22f44b3519a66e8bb8f60c50c8f4444 Mon Sep 17 00:00:00 2001 From: Patrick Dowler Date: Thu, 29 Jun 2023 11:18:58 -0700 Subject: [PATCH 20/52] vault: add cadc-registry link to README --- vault/README.md | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/vault/README.md b/vault/README.md index 67d202847..c4615fc99 100644 --- a/vault/README.md +++ b/vault/README.md @@ -28,9 +28,8 @@ for system properties related to the deployment environment. See cadc-util for common system properties. -`minoc` includes multiple IdentityManager implementations to support authenticated access: +`vault` includes multiple IdentityManager implementations to support authenticated access: - See cadc-access-control-identity for CADC access-control system support. - - See cadc-gms for OIDC token support. `vault` requires a connection pool to the local database: @@ -41,11 +40,15 @@ org.opencadc.vault.nodes.username={username for vospace pool} org.opencadc.vault.nodes.password={password for vospace pool} org.opencadc.vault.nodes.url=jdbc:postgresql://{server}/{database} ``` -The `nodes` account owns and manages (create, alter, drop) vospace database objects and manages +The _nodes_ account owns and manages (create, alter, drop) vospace database objects and manages all the content (insert, update, delete). The database is specified in the JDBC URL and the schema name is specified in the vault.properties (below). Failure to connect or initialize the database will show up in logs and in the VOSI-availability output. +### cadc-registry.properties + +See cadc-registry. + ### vault.properties A vault.properties file in /config is required to run this service. The following keys are required: ``` From 5403a9a13d6302a6f2a05e4695e34a9eeb87169a Mon Sep 17 00:00:00 2001 From: Patrick Dowler Date: Mon, 10 Jul 2023 16:30:45 -0700 Subject: [PATCH 21/52] cadc-inventory-db: node iterator bug fix --- .../org/opencadc/vospace/db/NodeDAOTest.java | 42 ++++++++++++++++++- .../opencadc/inventory/db/SQLGenerator.java | 2 +- 2 files changed, 42 insertions(+), 2 deletions(-) diff --git a/cadc-inventory-db/src/intTest/java/org/opencadc/vospace/db/NodeDAOTest.java b/cadc-inventory-db/src/intTest/java/org/opencadc/vospace/db/NodeDAOTest.java index b3aa5d1ef..ae9938908 100644 --- a/cadc-inventory-db/src/intTest/java/org/opencadc/vospace/db/NodeDAOTest.java +++ b/cadc-inventory-db/src/intTest/java/org/opencadc/vospace/db/NodeDAOTest.java @@ -601,8 +601,8 @@ public void testContainerNodeIterator() throws IOException { c2 = iter.next(); Assert.assertTrue(iter.hasNext()); c3 = iter.next(); + Assert.assertFalse(iter.hasNext()); } - // default order: alpha Assert.assertEquals(cont.getID(), c1.getID()); Assert.assertEquals(cont.getName(), c1.getName()); @@ -613,6 +613,46 @@ public void testContainerNodeIterator() throws IOException { Assert.assertEquals(link.getID(), c3.getID()); Assert.assertEquals(link.getName(), c3.getName()); + // iterate with limit + try (ResourceIterator iter = nodeDAO.iterator(orig, 2, null)) { + Assert.assertNotNull(iter); + Assert.assertTrue(iter.hasNext()); + c1 = iter.next(); + Assert.assertTrue(iter.hasNext()); + c2 = iter.next(); + Assert.assertFalse(iter.hasNext()); + } + Assert.assertEquals(cont.getID(), c1.getID()); + Assert.assertEquals(cont.getName(), c1.getName()); + + Assert.assertEquals(data.getID(), c2.getID()); + Assert.assertEquals(data.getName(), c2.getName()); + + // iterate with start + try (ResourceIterator iter = nodeDAO.iterator(orig, null, c2.getName())) { + Assert.assertNotNull(iter); + Assert.assertTrue(iter.hasNext()); + c2 = iter.next(); + Assert.assertTrue(iter.hasNext()); + c3 = iter.next(); + Assert.assertFalse(iter.hasNext()); + } + Assert.assertEquals(data.getID(), c2.getID()); + Assert.assertEquals(data.getName(), c2.getName()); + + Assert.assertEquals(link.getID(), c3.getID()); + Assert.assertEquals(link.getName(), c3.getName()); + + // iteratoe with limit and start + try (ResourceIterator iter = nodeDAO.iterator(orig, 1, c2.getName())) { + Assert.assertNotNull(iter); + Assert.assertTrue(iter.hasNext()); + c2 = iter.next(); + Assert.assertFalse(iter.hasNext()); + } + Assert.assertEquals(data.getID(), c2.getID()); + Assert.assertEquals(data.getName(), c2.getName()); + // depth first delete required but not enforced by DAO nodeDAO.delete(cont.getID()); nodeDAO.delete(data.getID()); diff --git a/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/SQLGenerator.java b/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/SQLGenerator.java index 97d27e985..131cb274b 100644 --- a/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/SQLGenerator.java +++ b/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/SQLGenerator.java @@ -915,7 +915,7 @@ public ResourceIterator query(DataSource ds) { ps.setFetchDirection(ResultSet.FETCH_FORWARD); int col = 1; - ps.setObject(1, parent.getID()); + ps.setObject(col++, parent.getID()); log.debug("parentID = " + parent.getID()); if (start != null) { ps.setString(col++, start); From 5092f91c40fa72ce233b6c3c0ad12bd1a0db0fe1 Mon Sep 17 00:00:00 2001 From: Patrick Dowler Date: Mon, 10 Jul 2023 16:32:56 -0700 Subject: [PATCH 22/52] typo --- .../src/intTest/java/org/opencadc/vospace/db/NodeDAOTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cadc-inventory-db/src/intTest/java/org/opencadc/vospace/db/NodeDAOTest.java b/cadc-inventory-db/src/intTest/java/org/opencadc/vospace/db/NodeDAOTest.java index ae9938908..1c41e5da4 100644 --- a/cadc-inventory-db/src/intTest/java/org/opencadc/vospace/db/NodeDAOTest.java +++ b/cadc-inventory-db/src/intTest/java/org/opencadc/vospace/db/NodeDAOTest.java @@ -643,7 +643,7 @@ public void testContainerNodeIterator() throws IOException { Assert.assertEquals(link.getID(), c3.getID()); Assert.assertEquals(link.getName(), c3.getName()); - // iteratoe with limit and start + // iterate with limit and start try (ResourceIterator iter = nodeDAO.iterator(orig, 1, c2.getName())) { Assert.assertNotNull(iter); Assert.assertTrue(iter.hasNext()); From a57647969af2e49ba89a8a9259ba14a7c967f6f5 Mon Sep 17 00:00:00 2001 From: Patrick Dowler Date: Tue, 11 Jul 2023 07:59:42 -0700 Subject: [PATCH 23/52] cadc-inventory-db: improve txn close in catch to avoid leak --- .../org/opencadc/vospace/db/NodeDAOTest.java | 4 ++-- .../org/opencadc/inventory/db/SQLGenerator.java | 16 ++++++++-------- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/cadc-inventory-db/src/intTest/java/org/opencadc/vospace/db/NodeDAOTest.java b/cadc-inventory-db/src/intTest/java/org/opencadc/vospace/db/NodeDAOTest.java index 1c41e5da4..d079ba329 100644 --- a/cadc-inventory-db/src/intTest/java/org/opencadc/vospace/db/NodeDAOTest.java +++ b/cadc-inventory-db/src/intTest/java/org/opencadc/vospace/db/NodeDAOTest.java @@ -103,10 +103,10 @@ public class NodeDAOTest { static { Log4jInit.setLevel("org.opencadc.inventory", Level.INFO); - Log4jInit.setLevel("org.opencadc.inventory.db", Level.DEBUG); + Log4jInit.setLevel("org.opencadc.inventory.db", Level.INFO); Log4jInit.setLevel("ca.nrc.cadc.db", Level.INFO); Log4jInit.setLevel("org.opencadc.vospace", Level.INFO); - Log4jInit.setLevel("org.opencadc.vospace.db", Level.DEBUG); + Log4jInit.setLevel("org.opencadc.vospace.db", Level.INFO); } NodeDAO nodeDAO; diff --git a/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/SQLGenerator.java b/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/SQLGenerator.java index 131cb274b..40b35fd55 100644 --- a/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/SQLGenerator.java +++ b/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/SQLGenerator.java @@ -222,7 +222,7 @@ protected void init() { this.columnMap.put(HarvestState.class, cols); // optional vospace - log.warn("vosSchema: " + vosSchema); + log.debug("vosSchema: " + vosSchema); if (vosSchema != null) { pref = vosSchema + "."; tableMap.put(Node.class, pref + Node.class.getSimpleName()); @@ -1575,10 +1575,10 @@ public Artifact next() { if (hasRow) { log.debug("ArtifactResultSetIterator: " + super.toString() + " ResultSet.next() FAILED - setAutoCommit(true)"); try { - close(); + con.setAutoCommit(false); hasRow = false; - } catch (IOException unexpected) { - log.debug("BUG: unexpected IOException from close", unexpected); + } catch (SQLException unexpected) { + log.error("unexpected SQLException trying to close txn", unexpected); } } throw new RuntimeException("BUG: artifact list query failed while iterating", ex); @@ -1609,7 +1609,7 @@ public NodeResultSetIterator(Connection con, ResultSet rs, ContainerNode parent) @Override public void close() throws IOException { if (hasRow) { - log.debug("NodeResultSetIterator: " + super.toString() + " ctor - setAutoCommit(true)"); + log.debug("NodeResultSetIterator: " + super.toString() + " close - setAutoCommit(true)"); try { con.setAutoCommit(true); hasRow = false; @@ -1638,10 +1638,10 @@ public Node next() { if (hasRow) { log.debug("NodeResultSetIterator: " + super.toString() + " ResultSet.next() FAILED - setAutoCommit(true)"); try { - close(); + con.setAutoCommit(true); hasRow = false; - } catch (IOException unexpected) { - log.debug("BUG: unexpected IOException from close", unexpected); + } catch (SQLException unexpected) { + log.error("unexpected SQLException trying to close txn", unexpected); } } throw new RuntimeException("BUG: node list query failed while iterating", ex); From fa011e01440865beed7c7b7a02e9da61dec132d5 Mon Sep 17 00:00:00 2001 From: Adrian Damian Date: Wed, 12 Jul 2023 18:11:58 +0300 Subject: [PATCH 24/52] Fixed small issues --- vault/src/main/java/org/opencadc/vault/VaultInitAction.java | 3 +-- vault/src/main/webapp/WEB-INF/web.xml | 6 ++---- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/vault/src/main/java/org/opencadc/vault/VaultInitAction.java b/vault/src/main/java/org/opencadc/vault/VaultInitAction.java index e65aa14c6..a6c2195ad 100644 --- a/vault/src/main/java/org/opencadc/vault/VaultInitAction.java +++ b/vault/src/main/java/org/opencadc/vault/VaultInitAction.java @@ -135,7 +135,6 @@ static MultiValuedProperties getConfig() { sb.append("incomplete config: "); boolean ok = true; - System.out.println("looking for " + RESOURCE_ID_KEY); String rid = mvp.getFirstPropertyValue(RESOURCE_ID_KEY); sb.append("\n\t" + RESOURCE_ID_KEY + ": "); if (rid == null) { @@ -223,7 +222,7 @@ protected void initNodePersistence() { try { ctx.unbind(jndiNodePersistence); } catch (NamingException ignore) { - log.debug("unbind previous JobManager failed... ignoring"); + log.debug("unbind previous JNDI key (" + jndiNodePersistence + ") failed... ignoring"); } NodePersistence npi = new NodePersistenceImpl(resourceID); ctx.bind(jndiNodePersistence, npi); diff --git a/vault/src/main/webapp/WEB-INF/web.xml b/vault/src/main/webapp/WEB-INF/web.xml index 0759ae033..5d371d0b4 100644 --- a/vault/src/main/webapp/WEB-INF/web.xml +++ b/vault/src/main/webapp/WEB-INF/web.xml @@ -12,7 +12,7 @@ ca.nrc.cadc.log.LogControlServlet logLevel - debug + info logLevelPackages @@ -20,9 +20,7 @@ org.opencadc.vault ca.nrc.cadc.db org.opencadc.vospace - + ca.nrc.cadc.rest ca.nrc.cadc.util ca.nrc.cadc.vosi From 3fb92010039239d24f9453f654771ed497b9bdd6 Mon Sep 17 00:00:00 2001 From: Patrick Dowler Date: Wed, 12 Jul 2023 11:23:32 -0700 Subject: [PATCH 25/52] cadc-inventory-db: add connection.close() calls in NodeIterator to return con to pool also added to ArtifactIterator for completness, but disabled by default because they are currently used in a non-pool way (critwall, ratik, tantar) --- .../org/opencadc/vospace/db/NodeDAOTest.java | 12 ++-- .../opencadc/inventory/db/SQLGenerator.java | 67 ++++++++++++++----- 2 files changed, 59 insertions(+), 20 deletions(-) diff --git a/cadc-inventory-db/src/intTest/java/org/opencadc/vospace/db/NodeDAOTest.java b/cadc-inventory-db/src/intTest/java/org/opencadc/vospace/db/NodeDAOTest.java index d079ba329..ba8cc20d6 100644 --- a/cadc-inventory-db/src/intTest/java/org/opencadc/vospace/db/NodeDAOTest.java +++ b/cadc-inventory-db/src/intTest/java/org/opencadc/vospace/db/NodeDAOTest.java @@ -76,6 +76,7 @@ import java.net.URI; import java.security.MessageDigest; import java.security.NoSuchAlgorithmException; +import java.sql.Connection; import java.util.Map; import java.util.TreeMap; import java.util.UUID; @@ -104,7 +105,7 @@ public class NodeDAOTest { static { Log4jInit.setLevel("org.opencadc.inventory", Level.INFO); Log4jInit.setLevel("org.opencadc.inventory.db", Level.INFO); - Log4jInit.setLevel("ca.nrc.cadc.db", Level.INFO); + Log4jInit.setLevel("ca.nrc.cadc.db", Level.DEBUG); Log4jInit.setLevel("org.opencadc.vospace", Level.INFO); Log4jInit.setLevel("org.opencadc.vospace.db", Level.INFO); } @@ -115,11 +116,12 @@ public NodeDAOTest() throws Exception { try { DBConfig dbrc = new DBConfig(); ConnectionConfig cc = dbrc.getConnectionConfig(TestUtil.SERVER, TestUtil.DATABASE); - DBUtil.createJNDIDataSource("jdbc/ArtifactDAOTest", cc); + DBUtil.PoolConfig pool = new DBUtil.PoolConfig(cc, 1, 6000L, "select 123"); + DBUtil.createJNDIDataSource("jdbc/NodeDAOTest", pool); Map config = new TreeMap<>(); config.put(SQLGenerator.class.getName(), SQLGenerator.class); - config.put("jndiDataSourceName", "jdbc/ArtifactDAOTest"); + config.put("jndiDataSourceName", "jdbc/NodeDAOTest"); config.put("database", TestUtil.DATABASE); config.put("schema", TestUtil.SCHEMA); config.put("vosSchema", TestUtil.VOS_SCHEMA); @@ -145,7 +147,9 @@ public void init_cleanup() throws Exception { DataSource ds = nodeDAO.getDataSource(); String sql = "delete from " + gen.getTable(ContainerNode.class); log.info("pre-test cleanup: " + sql); - ds.getConnection().createStatement().execute(sql); + Connection con = ds.getConnection(); + con.createStatement().execute(sql); + con.close(); log.info("clearing old content... OK"); } diff --git a/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/SQLGenerator.java b/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/SQLGenerator.java index 40b35fd55..8a22ddb8a 100644 --- a/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/SQLGenerator.java +++ b/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/SQLGenerator.java @@ -1531,6 +1531,7 @@ private class ArtifactResultSetIterator implements ResourceIterator { private final Connection con; private final ResultSet rs; boolean hasRow; + boolean closeWhenDone = false; // not a pooled connection ArtifactResultSetIterator(Connection con, ResultSet rs) throws SQLException { this.con = con; @@ -1539,7 +1540,14 @@ private class ArtifactResultSetIterator implements ResourceIterator { log.debug("ArtifactResultSetIterator: " + super.toString() + " ctor " + hasRow); if (!hasRow) { log.debug("ArtifactResultSetIterator: " + super.toString() + " ctor - setAutoCommit(true)"); - con.setAutoCommit(true); + try { + con.setAutoCommit(true); // commit txn + if (closeWhenDone) { + con.close(); // return to pool + } + } catch (SQLException unexpected) { + log.error("Connection.setAutoCommit(true) & close() failed", unexpected); + } } } @@ -1548,10 +1556,13 @@ public void close() throws IOException { if (hasRow) { log.debug("ArtifactResultSetIterator: " + super.toString() + " ctor - setAutoCommit(true)"); try { - con.setAutoCommit(true); - hasRow = false; - } catch (SQLException ex) { - throw new RuntimeException("BUG: artifact list query failed during close()", ex); + con.setAutoCommit(true); // commit txn + if (closeWhenDone) { + con.close(); // return to pool + } + hasRow = false; + } catch (SQLException unexpected) { + log.error("Connection.setAutoCommit(true) & close() failed", unexpected); } } } @@ -1568,17 +1579,27 @@ public Artifact next() { hasRow = rs.next(); if (!hasRow) { log.debug("ArtifactResultSetIterator: " + super.toString() + " DONE - setAutoCommit(true)"); - con.setAutoCommit(true); + try { + con.setAutoCommit(true); // commit txn + if (closeWhenDone) { + con.close(); // return to pool + } + } catch (SQLException unexpected) { + log.error("Connection.setAutoCommit(true) & close() failed", unexpected); + } } return ret; } catch (Exception ex) { if (hasRow) { log.debug("ArtifactResultSetIterator: " + super.toString() + " ResultSet.next() FAILED - setAutoCommit(true)"); try { - con.setAutoCommit(false); + con.setAutoCommit(true); // commit txn + if (closeWhenDone) { + con.close(); // return to pool + } hasRow = false; } catch (SQLException unexpected) { - log.error("unexpected SQLException trying to close txn", unexpected); + log.error("Connection.setAutoCommit(true) & close() failed", unexpected); } } throw new RuntimeException("BUG: artifact list query failed while iterating", ex); @@ -1602,7 +1623,13 @@ public NodeResultSetIterator(Connection con, ResultSet rs, ContainerNode parent) log.debug("NodeResultSetIterator: " + super.toString() + " ctor " + hasRow); if (!hasRow) { log.debug("NodeResultSetIterator: " + super.toString() + " ctor - setAutoCommit(true)"); - con.setAutoCommit(true); + + try { + con.setAutoCommit(true); // commit txn + con.close(); // return to pool + } catch (SQLException ignore) { + log.error("Connection.setAutoCommit(true) & close() failed", ignore); + } } } @@ -1611,10 +1638,11 @@ public void close() throws IOException { if (hasRow) { log.debug("NodeResultSetIterator: " + super.toString() + " close - setAutoCommit(true)"); try { - con.setAutoCommit(true); + con.setAutoCommit(true); // commit txn + con.close(); // return to pool hasRow = false; - } catch (SQLException ex) { - throw new RuntimeException("BUG: node list query failed during close()", ex); + } catch (SQLException ignore) { + log.error("Connection.setAutoCommit(true) & close() failed", ignore); } } } @@ -1631,17 +1659,24 @@ public Node next() { hasRow = rs.next(); if (!hasRow) { log.debug("NodeResultSetIterator: " + super.toString() + " DONE - setAutoCommit(true)"); - con.setAutoCommit(true); + try { + con.setAutoCommit(true); // commit txn + con.close(); // return to pool + hasRow = false; + } catch (SQLException ignore) { + log.error("Connection.setAutoCommit(true) & close() failed", ignore); + } } return ret; } catch (Exception ex) { if (hasRow) { log.debug("NodeResultSetIterator: " + super.toString() + " ResultSet.next() FAILED - setAutoCommit(true)"); try { - con.setAutoCommit(true); + con.setAutoCommit(true); // commit txn + con.close(); // return to pool hasRow = false; - } catch (SQLException unexpected) { - log.error("unexpected SQLException trying to close txn", unexpected); + } catch (SQLException ignore) { + log.error("Connection.setAutoCommit(true) & close() failed", ignore); } } throw new RuntimeException("BUG: node list query failed while iterating", ex); From ed3fa618ad63669b0897ac879e6d7cf6128267a4 Mon Sep 17 00:00:00 2001 From: Patrick Dowler Date: Wed, 12 Jul 2023 11:43:29 -0700 Subject: [PATCH 26/52] debug->info in test code --- .../src/intTest/java/org/opencadc/vospace/db/NodeDAOTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cadc-inventory-db/src/intTest/java/org/opencadc/vospace/db/NodeDAOTest.java b/cadc-inventory-db/src/intTest/java/org/opencadc/vospace/db/NodeDAOTest.java index ba8cc20d6..e5fedfb54 100644 --- a/cadc-inventory-db/src/intTest/java/org/opencadc/vospace/db/NodeDAOTest.java +++ b/cadc-inventory-db/src/intTest/java/org/opencadc/vospace/db/NodeDAOTest.java @@ -105,7 +105,7 @@ public class NodeDAOTest { static { Log4jInit.setLevel("org.opencadc.inventory", Level.INFO); Log4jInit.setLevel("org.opencadc.inventory.db", Level.INFO); - Log4jInit.setLevel("ca.nrc.cadc.db", Level.DEBUG); + Log4jInit.setLevel("ca.nrc.cadc.db", Level.INFO); Log4jInit.setLevel("org.opencadc.vospace", Level.INFO); Log4jInit.setLevel("org.opencadc.vospace.db", Level.INFO); } From bf26dde87f5f85691fce7512fda06ae2d9138364 Mon Sep 17 00:00:00 2001 From: Patrick Dowler Date: Wed, 12 Jul 2023 11:47:19 -0700 Subject: [PATCH 27/52] raven: limit cadc-inventory-db version to avoid cadc-vos-2.0 until ready to adapt --- raven/build.gradle | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/raven/build.gradle b/raven/build.gradle index 1026ab9ed..6d2cbba4e 100644 --- a/raven/build.gradle +++ b/raven/build.gradle @@ -34,7 +34,8 @@ dependencies { compile 'org.opencadc:cadc-cdp:[1.0,)' compile 'org.opencadc:cadc-gms:[1.0.4,)' compile 'org.opencadc:cadc-inventory:[0.9.4,2.0)' - compile 'org.opencadc:cadc-inventory-db:[0.14.5,1.0)' + // temporarily limit db lib + compile 'org.opencadc:cadc-inventory-db:[0.14.5,0.15.0)' compile 'org.opencadc:cadc-inventory-server:[0.2.1,)' compile 'org.opencadc:cadc-permissions:[0.3.1,)' compile 'org.opencadc:cadc-permissions-client:[0.3,)' From ce0898f62fe9272d7e116e0895b54f78214c0d16 Mon Sep 17 00:00:00 2001 From: Adrian Damian Date: Wed, 12 Jul 2023 23:04:42 +0300 Subject: [PATCH 28/52] Re-factored the library --- vault/src/main/webapp/WEB-INF/web.xml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/vault/src/main/webapp/WEB-INF/web.xml b/vault/src/main/webapp/WEB-INF/web.xml index 5d371d0b4..a671c0448 100644 --- a/vault/src/main/webapp/WEB-INF/web.xml +++ b/vault/src/main/webapp/WEB-INF/web.xml @@ -12,7 +12,7 @@ ca.nrc.cadc.log.LogControlServlet logLevel - info + debug logLevelPackages @@ -45,19 +45,19 @@ get - org.opencadc.vospace.server.web.actions.GetNodeAction + org.opencadc.vospace.server.actions.GetNodeAction put - org.opencadc.vospace.server.web.actions.CreateNodeAction + org.opencadc.vospace.server.actions.CreateNodeAction post - org.opencadc.vospace.server.web.actions.UpdateNodeAction + org.opencadc.vospace.server.actions.UpdateNodeAction delete - org.opencadc.vospace.server.web.actions.DeleteNodeAction + org.opencadc.vospace.server.actions.DeleteNodeAction 2 From 5b4e2727b08e042552f413e0c2eb567f1e8ea798 Mon Sep 17 00:00:00 2001 From: Adrian Damian Date: Thu, 13 Jul 2023 21:10:36 +0300 Subject: [PATCH 29/52] Small fixes --- vault/build.gradle | 4 ---- .../src/main/java/org/opencadc/vault/NodePersistenceImpl.java | 3 +-- vault/src/main/webapp/WEB-INF/web.xml | 2 +- 3 files changed, 2 insertions(+), 7 deletions(-) diff --git a/vault/build.gradle b/vault/build.gradle index bf307a578..96ecd7f1c 100644 --- a/vault/build.gradle +++ b/vault/build.gradle @@ -46,15 +46,11 @@ dependencies { compile 'org.opencadc:cadc-inventory-db:[0.15.0,1.0)' testCompile 'junit:junit:[4.0,)' - testCompile 'org.easymock:easymock:3.6' - testCompile 'org.skyscreamer:jsonassert:[1.0,)' runtime 'org.opencadc:cadc-access-control-identity:[1.2.0,)' runtime 'org.opencadc:cadc-gms:[1.0.4,)' // JDBC drivers - intTestRuntime 'org.postgresql:postgresql:[42.2.8,)' - intTestRuntime 'net.sourceforge.jtds:jtds:[1.0,)' intTestCompile 'org.opencadc:cadc-test-vosi:[1.0.11,)' } diff --git a/vault/src/main/java/org/opencadc/vault/NodePersistenceImpl.java b/vault/src/main/java/org/opencadc/vault/NodePersistenceImpl.java index 80e809d9d..d36965c0b 100644 --- a/vault/src/main/java/org/opencadc/vault/NodePersistenceImpl.java +++ b/vault/src/main/java/org/opencadc/vault/NodePersistenceImpl.java @@ -81,7 +81,6 @@ import java.net.URI; import java.security.Principal; import java.text.DateFormat; -import java.util.HashSet; import java.util.Iterator; import java.util.List; import java.util.Map; @@ -147,7 +146,7 @@ public NodePersistenceImpl(URI resourceID) { Subject rawOwnerSubject = AuthenticationUtil.getSubject(new PrincipalExtractor() { @Override public Set getPrincipals() { - Set ret = new HashSet<>(); + Set ret = new TreeSet<>(); ret.add(new X500Principal(owner)); return ret; } diff --git a/vault/src/main/webapp/WEB-INF/web.xml b/vault/src/main/webapp/WEB-INF/web.xml index a671c0448..af0394c10 100644 --- a/vault/src/main/webapp/WEB-INF/web.xml +++ b/vault/src/main/webapp/WEB-INF/web.xml @@ -12,7 +12,7 @@ ca.nrc.cadc.log.LogControlServlet logLevel - debug + info logLevelPackages From 1c885d7dc96810c39e7191701b5a27469f1da9e6 Mon Sep 17 00:00:00 2001 From: Adrian Damian Date: Thu, 13 Jul 2023 21:11:41 +0300 Subject: [PATCH 30/52] Small fixes --- vault/build.gradle | 1 - 1 file changed, 1 deletion(-) diff --git a/vault/build.gradle b/vault/build.gradle index 96ecd7f1c..787a86b9b 100644 --- a/vault/build.gradle +++ b/vault/build.gradle @@ -50,7 +50,6 @@ dependencies { runtime 'org.opencadc:cadc-access-control-identity:[1.2.0,)' runtime 'org.opencadc:cadc-gms:[1.0.4,)' - // JDBC drivers intTestCompile 'org.opencadc:cadc-test-vosi:[1.0.11,)' } From a2c63e8f83b36693d4d667e635af2126945e4b41 Mon Sep 17 00:00:00 2001 From: Adrian Damian Date: Thu, 13 Jul 2023 22:23:17 +0300 Subject: [PATCH 31/52] Renamed cadc-vos-server dependency --- vault/build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vault/build.gradle b/vault/build.gradle index 787a86b9b..eb52ef101 100644 --- a/vault/build.gradle +++ b/vault/build.gradle @@ -35,7 +35,7 @@ dependencies { compile 'org.opencadc:cadc-gms:[1.0.0,)' compile 'org.opencadc:cadc-rest:[1.3.16,)' compile 'org.opencadc:cadc-vos:[2.0,)' - compile 'org.opencadc:cadc-vos-server-2.0:[2.0,)' + compile 'org.opencadc:cadc-vos-server-alt:[2.0,)' compile 'org.opencadc:cadc-vosi:[1.3.2,)' compile 'org.opencadc:cadc-uws:[1.0,)' compile 'org.opencadc:cadc-uws-server:[1.2.12,)' From d364005888ec288e653c32c856765a031e9ea62d Mon Sep 17 00:00:00 2001 From: Adrian Damian Date: Thu, 13 Jul 2023 23:41:37 +0300 Subject: [PATCH 32/52] Fixed bug --- .../src/main/java/org/opencadc/vault/NodePersistenceImpl.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/vault/src/main/java/org/opencadc/vault/NodePersistenceImpl.java b/vault/src/main/java/org/opencadc/vault/NodePersistenceImpl.java index d36965c0b..80e809d9d 100644 --- a/vault/src/main/java/org/opencadc/vault/NodePersistenceImpl.java +++ b/vault/src/main/java/org/opencadc/vault/NodePersistenceImpl.java @@ -81,6 +81,7 @@ import java.net.URI; import java.security.Principal; import java.text.DateFormat; +import java.util.HashSet; import java.util.Iterator; import java.util.List; import java.util.Map; @@ -146,7 +147,7 @@ public NodePersistenceImpl(URI resourceID) { Subject rawOwnerSubject = AuthenticationUtil.getSubject(new PrincipalExtractor() { @Override public Set getPrincipals() { - Set ret = new TreeSet<>(); + Set ret = new HashSet<>(); ret.add(new X500Principal(owner)); return ret; } From e6ca6ec93970a5cf63342c95f792a852e255fda9 Mon Sep 17 00:00:00 2001 From: Patrick Dowler Date: Fri, 4 Aug 2023 12:16:45 -0700 Subject: [PATCH 33/52] vault: added NodesTest integration test using cadc-test-vos-2.0 --- vault/build.gradle | 1 + .../java/org/opencadc/vault/NodesTest.java | 82 +++++++++++++++++++ 2 files changed, 83 insertions(+) create mode 100644 vault/src/intTest/java/org/opencadc/vault/NodesTest.java diff --git a/vault/build.gradle b/vault/build.gradle index eb52ef101..ca56af16c 100644 --- a/vault/build.gradle +++ b/vault/build.gradle @@ -51,6 +51,7 @@ dependencies { runtime 'org.opencadc:cadc-gms:[1.0.4,)' intTestCompile 'org.opencadc:cadc-test-vosi:[1.0.11,)' + intTestCompile 'org.opencadc:cadc-test-vos:[2.0,3.0)' } configurations { diff --git a/vault/src/intTest/java/org/opencadc/vault/NodesTest.java b/vault/src/intTest/java/org/opencadc/vault/NodesTest.java new file mode 100644 index 000000000..a8afb4078 --- /dev/null +++ b/vault/src/intTest/java/org/opencadc/vault/NodesTest.java @@ -0,0 +1,82 @@ +/* +************************************************************************ +******************* CANADIAN ASTRONOMY DATA CENTRE ******************* +************** CENTRE CANADIEN DE DONNÉES ASTRONOMIQUES ************** +* +* (c) 2023. (c) 2023. +* Government of Canada Gouvernement du Canada +* National Research Council Conseil national de recherches +* Ottawa, Canada, K1A 0R6 Ottawa, Canada, K1A 0R6 +* All rights reserved Tous droits réservés +* +* NRC disclaims any warranties, Le CNRC dénie toute garantie +* expressed, implied, or énoncée, implicite ou légale, +* statutory, of any kind with de quelque nature que ce +* respect to the software, soit, concernant le logiciel, +* including without limitation y compris sans restriction +* any warranty of merchantability toute garantie de valeur +* or fitness for a particular marchande ou de pertinence +* purpose. NRC shall not be pour un usage particulier. +* liable in any event for any Le CNRC ne pourra en aucun cas +* damages, whether direct or être tenu responsable de tout +* indirect, special or general, dommage, direct ou indirect, +* consequential or incidental, particulier ou général, +* arising from the use of the accessoire ou fortuit, résultant +* software. Neither the name de l'utilisation du logiciel. Ni +* of the National Research le nom du Conseil National de +* Council of Canada nor the Recherches du Canada ni les noms +* names of its contributors may de ses participants ne peuvent +* be used to endorse or promote être utilisés pour approuver ou +* products derived from this promouvoir les produits dérivés +* software without specific prior de ce logiciel sans autorisation +* written permission. préalable et particulière +* par écrit. +* +* This file is part of the Ce fichier fait partie du projet +* OpenCADC project. OpenCADC. +* +* OpenCADC is free software: OpenCADC est un logiciel libre ; +* you can redistribute it and/or vous pouvez le redistribuer ou le +* modify it under the terms of modifier suivant les termes de +* the GNU Affero General Public la “GNU Affero General Public +* License as published by the License” telle que publiée +* Free Software Foundation, par la Free Software Foundation +* either version 3 of the : soit la version 3 de cette +* License, or (at your option) licence, soit (à votre gré) +* any later version. toute version ultérieure. +* +* OpenCADC is distributed in the OpenCADC est distribué +* hope that it will be useful, dans l’espoir qu’il vous +* but WITHOUT ANY WARRANTY; sera utile, mais SANS AUCUNE +* without even the implied GARANTIE : sans même la garantie +* warranty of MERCHANTABILITY implicite de COMMERCIALISABILITÉ +* or FITNESS FOR A PARTICULAR ni d’ADÉQUATION À UN OBJECTIF +* PURPOSE. See the GNU Affero PARTICULIER. Consultez la Licence +* General Public License for Générale Publique GNU Affero +* more details. pour plus de détails. +* +* You should have received Vous devriez avoir reçu une +* a copy of the GNU Affero copie de la Licence Générale +* General Public License along Publique GNU Affero avec +* with OpenCADC. If not, see OpenCADC ; si ce n’est +* . pas le cas, consultez : +* . +* +************************************************************************ +*/ + +package org.opencadc.vault; + +import org.apache.log4j.Logger; + +/** + * Test the nodes endpoint. + * + * @author pdowler + */ +public class NodesTest extends org.opencadc.conformance.vos.NodesTest { + private static final Logger log = Logger.getLogger(NodesTest.class); + + public NodesTest() { + } +} From 2b45eaa068f05e10a3cd513bd80372ad18739937 Mon Sep 17 00:00:00 2001 From: Patrick Dowler Date: Fri, 4 Aug 2023 12:17:22 -0700 Subject: [PATCH 34/52] vault: comment out unimplemented endpoints from capabilities --- vault/src/main/webapp/capabilities.xml | 50 ++++++++++---------------- 1 file changed, 18 insertions(+), 32 deletions(-) diff --git a/vault/src/main/webapp/capabilities.xml b/vault/src/main/webapp/capabilities.xml index f36f7a0f1..fb145630e 100644 --- a/vault/src/main/webapp/capabilities.xml +++ b/vault/src/main/webapp/capabilities.xml @@ -23,11 +23,13 @@ + @@ -37,12 +39,9 @@ - - https://replace.me.com/vault/auth/nodes - - + + + + + + + + From 3efead89371a5415b9edfe0c875976527f390522 Mon Sep 17 00:00:00 2001 From: Patrick Dowler Date: Fri, 4 Aug 2023 12:18:09 -0700 Subject: [PATCH 35/52] vault: tweak root and trash node settings in NodePersistenceImpl --- .../main/java/org/opencadc/vault/NodePersistenceImpl.java | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/vault/src/main/java/org/opencadc/vault/NodePersistenceImpl.java b/vault/src/main/java/org/opencadc/vault/NodePersistenceImpl.java index 80e809d9d..d15725966 100644 --- a/vault/src/main/java/org/opencadc/vault/NodePersistenceImpl.java +++ b/vault/src/main/java/org/opencadc/vault/NodePersistenceImpl.java @@ -162,7 +162,11 @@ public X509CertificateChain getCertificateChain() { UUID rootID = new UUID(0L, 0L); this.root = new ContainerNode(rootID, "", false); root.owner = im.augment(rawOwnerSubject); - root.ownerID = im.toOwner(root.owner); + log.warn("ROOT owner: " + root.owner); + root.ownerID = im.toOwner(rawOwnerSubject); + log.warn("ROOT ownerID: " + root.ownerID + " rtype: " + root.ownerID.getClass().getName()); + root.isPublic = true; + root.inheritPermissions = false; // trash node // TODO: do this setup in a txn with a lock on something @@ -175,6 +179,7 @@ public X509CertificateChain getCertificateChain() { tn.ownerID = root.ownerID; tn.owner = root.owner; tn.isPublic = false; + tn.inheritPermissions = false; tn.parentID = rootID; dao.put(tn); this.trash = tn; From 60651cb494b204578d52a42c151b54ab35a0086c Mon Sep 17 00:00:00 2001 From: Patrick Dowler Date: Fri, 4 Aug 2023 12:25:10 -0700 Subject: [PATCH 36/52] vault: update dependencies --- vault/build.gradle | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/vault/build.gradle b/vault/build.gradle index ca56af16c..86b736b0c 100644 --- a/vault/build.gradle +++ b/vault/build.gradle @@ -32,7 +32,7 @@ dependencies { compile 'org.opencadc:cadc-util:[1.9.5,2.0)' compile 'org.opencadc:cadc-log:[1.1.6,2.0)' - compile 'org.opencadc:cadc-gms:[1.0.0,)' + compile 'org.opencadc:cadc-gms:[1.0.5,)' compile 'org.opencadc:cadc-rest:[1.3.16,)' compile 'org.opencadc:cadc-vos:[2.0,)' compile 'org.opencadc:cadc-vos-server-alt:[2.0,)' @@ -47,8 +47,8 @@ dependencies { testCompile 'junit:junit:[4.0,)' - runtime 'org.opencadc:cadc-access-control-identity:[1.2.0,)' - runtime 'org.opencadc:cadc-gms:[1.0.4,)' + runtime 'org.opencadc:cadc-access-control-identity:[1.2.1,)' + runtime 'org.opencadc:cadc-gms:[1.0.5,)' intTestCompile 'org.opencadc:cadc-test-vosi:[1.0.11,)' intTestCompile 'org.opencadc:cadc-test-vos:[2.0,3.0)' From 0531edc6618e1afa566b30668fc758a320e4d577 Mon Sep 17 00:00:00 2001 From: Patrick Dowler Date: Fri, 4 Aug 2023 14:26:17 -0700 Subject: [PATCH 37/52] vault: cleanup in NodePersistenceImpl --- .../opencadc/vault/NodePersistenceImpl.java | 81 +------------------ 1 file changed, 4 insertions(+), 77 deletions(-) diff --git a/vault/src/main/java/org/opencadc/vault/NodePersistenceImpl.java b/vault/src/main/java/org/opencadc/vault/NodePersistenceImpl.java index d15725966..0d63898a8 100644 --- a/vault/src/main/java/org/opencadc/vault/NodePersistenceImpl.java +++ b/vault/src/main/java/org/opencadc/vault/NodePersistenceImpl.java @@ -119,10 +119,7 @@ public class NodePersistenceImpl implements NodePersistence { private final ContainerNode trash; private final Namespace storageNamespace; private final Set immutableProps = new TreeSet<>(); - private final Set artifactProps = new TreeSet<>(); - private final Set rootAdminContainerProps = new TreeSet<>(); - private final Set rootAdminProps = new TreeSet<>(); private URI resourceID; public NodePersistenceImpl(URI resourceID) { @@ -191,10 +188,10 @@ public X509CertificateChain getCertificateChain() { // VOS.PROPERTY_URI_AVAILABLESPACE // container nodes // VOS.PROPERTY_URI_WRITABLE // prediction for current caller - // props only the admin (root owner) can modify - rootAdminProps.add(VOS.PROPERTY_URI_CREATOR); // owner - rootAdminContainerProps.add(VOS.PROPERTY_URI_CONTENTLENGTH); // container nodes - rootAdminContainerProps.add(VOS.PROPERTY_URI_QUOTA); // container nodes + // props only the admin (root owner) can modify? + // VOS.PROPERTY_URI_CREATOR // owner + // VOS.PROPERTY_URI_CONTENTLENGTH // container nodes + // VOS.PROPERTY_URI_QUOTA // container nodes // node properties that match immutable Artifact fields immutableProps.add(VOS.PROPERTY_URI_CONTENTLENGTH); // immutable @@ -457,74 +454,4 @@ public void delete(Node node) throws TransientException { } } - - - // this code is incomplete but shows the logic of merging the requested property changes - // into the node before put(node)... keeping it here for reference - public Node updateProperties(Node node, List props) - throws TransientException, NodeNotSupportedException { - if (node == null || props == null) { - throw new IllegalArgumentException("args cannot be null: node, props"); - } - - // merge props -> node and/or node.properties and/or mutable artifact - for (NodeProperty np : props) { - boolean writable = (node instanceof ContainerNode && this.rootAdminContainerProps.contains(np.getKey())); - writable = writable || rootAdminProps.contains(np.getKey()); - if (writable) { - // some props are directly in the node - if (artifactProps.contains(np.getKey())) { - throw new UnsupportedOperationException("TODO: set mutable artifact prop: " + np.getKey()); - } - - // some props are directly in the node - if (VOS.PROPERTY_URI_GROUPREAD.equals(np.getKey())) { - String raw = np.getValue(); - throw new UnsupportedOperationException(); - } else if (VOS.PROPERTY_URI_GROUPWRITE.equals(np.getKey())) { - String raw = np.getValue(); - throw new UnsupportedOperationException(); - } else if (VOS.PROPERTY_URI_INHERIT_PERMISSIONS.equals(np.getKey())) { - String raw = np.getValue(); - throw new UnsupportedOperationException(); - } else if (VOS.PROPERTY_URI_ISLOCKED.equals(np.getKey())) { - String raw = np.getValue(); - throw new UnsupportedOperationException(); - } else if (VOS.PROPERTY_URI_ISPUBLIC.equals(np.getKey())) { - String raw = np.getValue(); - throw new UnsupportedOperationException(); - } - - // generic key-value props - if (node.getProperties().contains(np)) { - // remove previous; covers mark for deletion case - node.getProperties().remove(np); - } - - if (!np.isMarkedForDeletion()) { - // add new - node.getProperties().add(np); - } - } else { - log.debug("updateProperties: skip non-writable " + np.getKey()); - } - } - - return put(node); - } - - @Override - public void setFileMetadata(DataNode dataNode, FileMetadata fileMetadata, boolean b) throws TransientException { - - } - - @Override - public void move(Node node, ContainerNode containerNode) throws TransientException { - - } - - @Override - public void copy(Node node, ContainerNode containerNode) throws TransientException { - - } } From a79673b6aa1340c393246162f62c96b72865e349 Mon Sep 17 00:00:00 2001 From: Patrick Dowler Date: Tue, 8 Aug 2023 16:38:28 -0700 Subject: [PATCH 38/52] move assign of owner and parent details to NodePersistenceImpl --- .../org/opencadc/vospace/db/NodeDAOTest.java | 14 +++-- .../opencadc/inventory/db/SQLGenerator.java | 7 --- .../opencadc/vault/NodePersistenceImpl.java | 53 +++++++++++++++++-- 3 files changed, 55 insertions(+), 19 deletions(-) diff --git a/cadc-inventory-db/src/intTest/java/org/opencadc/vospace/db/NodeDAOTest.java b/cadc-inventory-db/src/intTest/java/org/opencadc/vospace/db/NodeDAOTest.java index e5fedfb54..24291b403 100644 --- a/cadc-inventory-db/src/intTest/java/org/opencadc/vospace/db/NodeDAOTest.java +++ b/cadc-inventory-db/src/intTest/java/org/opencadc/vospace/db/NodeDAOTest.java @@ -199,8 +199,8 @@ public void testPutGetUpdateDeleteContainerNode() throws InterruptedException, Assert.assertEquals(orig.getID(), aa.getID()); Assert.assertEquals(orig.getName(), aa.getName()); Assert.assertEquals(root.getID(), a.parentID); - Assert.assertNotNull(aa.parent); - Assert.assertEquals(root.getID(), aa.parent.getID()); + Assert.assertNotNull(aa.parentID); + Assert.assertEquals(root.getID(), aa.parentID); Assert.assertNull(a.parent); // get-node-by-id: comes pack without parent Assert.assertEquals(orig.getName(), a.getName()); @@ -290,6 +290,7 @@ public void testPutGetUpdateDeleteContainerNodeMax() throws InterruptedException log.info("found by id: " + a.getID() + " aka " + a); Assert.assertEquals(orig.getID(), a.getID()); Assert.assertEquals(orig.getName(), a.getName()); + Assert.assertNotNull(a.parentID); Assert.assertEquals(root.getID(), a.parentID); // get-by-path @@ -298,8 +299,7 @@ public void testPutGetUpdateDeleteContainerNodeMax() throws InterruptedException log.info("found by path: " + aa.getID() + " aka " + aa); Assert.assertEquals(orig.getID(), aa.getID()); Assert.assertEquals(orig.getName(), aa.getName()); - Assert.assertNotNull(aa.parent); - Assert.assertEquals(root.getID(), aa.parent.getID()); + Assert.assertNotNull(aa.parentID); Assert.assertEquals(root.getID(), aa.parentID); Assert.assertNull(a.parent); // get-node-by-id: comes pack without parent @@ -389,8 +389,7 @@ public void testPutGetUpdateDeleteDataNode() throws InterruptedException, log.info("found: " + aa.getID() + " aka " + aa); Assert.assertEquals(orig.getID(), aa.getID()); Assert.assertEquals(orig.getName(), aa.getName()); - Assert.assertNotNull(aa.parent); - Assert.assertEquals(root.getID(), aa.parent.getID()); + Assert.assertNotNull(aa.parentID); Assert.assertEquals(root.getID(), aa.parentID); Assert.assertNull(a.parent); // get-node-by-id: comes pack without parent @@ -481,8 +480,7 @@ public void testPutGetUpdateDeleteLinkNode() throws InterruptedException, log.info("found: " + aa.getID() + " aka " + aa); Assert.assertEquals(orig.getID(), aa.getID()); Assert.assertEquals(orig.getName(), aa.getName()); - Assert.assertNotNull(aa.parent); - Assert.assertEquals(root.getID(), aa.parent.getID()); + Assert.assertNotNull(aa.parentID); Assert.assertEquals(root.getID(), aa.parentID); Assert.assertNull(a.parent); // get-node-by-id: comes pack without parent diff --git a/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/SQLGenerator.java b/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/SQLGenerator.java index 8a22ddb8a..bd5b2eb42 100644 --- a/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/SQLGenerator.java +++ b/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/SQLGenerator.java @@ -1764,13 +1764,6 @@ private Node mapRowToNode(ResultSet rs, Calendar utc, ContainerNode parent) thro InventoryUtil.assignLastModified(ret, lastModified); InventoryUtil.assignMetaChecksum(ret, metaChecksum); - if (parent != null) { - if (!parent.getID().equals(parentID)) { - throw new RuntimeException("BUG: expected parentID=" + parent.getID() + " but got: " + parentID); - } - ret.parent = parent; - } - return ret; } diff --git a/vault/src/main/java/org/opencadc/vault/NodePersistenceImpl.java b/vault/src/main/java/org/opencadc/vault/NodePersistenceImpl.java index 0d63898a8..ab960c6fc 100644 --- a/vault/src/main/java/org/opencadc/vault/NodePersistenceImpl.java +++ b/vault/src/main/java/org/opencadc/vault/NodePersistenceImpl.java @@ -67,6 +67,7 @@ package org.opencadc.vault; +import sun.rmi.rmic.IndentingWriter; import ca.nrc.cadc.auth.AuthenticationUtil; import ca.nrc.cadc.auth.IdentityManager; import ca.nrc.cadc.auth.PrincipalExtractor; @@ -122,6 +123,8 @@ public class NodePersistenceImpl implements NodePersistence { private final Set artifactProps = new TreeSet<>(); private URI resourceID; + private IdentityManager identityManager = AuthenticationUtil.getIdentityManager(); + public NodePersistenceImpl(URI resourceID) { if (resourceID == null) { throw new IllegalArgumentException("resource ID required"); @@ -140,7 +143,6 @@ public NodePersistenceImpl(URI resourceID) { if (owner == null) { throw new InvalidConfigException(VaultInitAction.ROOT_OWNER + " cannot be null"); } - IdentityManager im = AuthenticationUtil.getIdentityManager(); Subject rawOwnerSubject = AuthenticationUtil.getSubject(new PrincipalExtractor() { @Override public Set getPrincipals() { @@ -158,9 +160,10 @@ public X509CertificateChain getCertificateChain() { // root node UUID rootID = new UUID(0L, 0L); this.root = new ContainerNode(rootID, "", false); - root.owner = im.augment(rawOwnerSubject); + root.owner = identityManager.augment(rawOwnerSubject); + root.ownerDisplay = identityManager.toDisplayString(root.owner); log.warn("ROOT owner: " + root.owner); - root.ownerID = im.toOwner(rawOwnerSubject); + root.ownerID = identityManager.toOwner(rawOwnerSubject); log.warn("ROOT ownerID: " + root.ownerID + " rtype: " + root.ownerID.getClass().getName()); root.isPublic = true; root.inheritPermissions = false; @@ -260,6 +263,10 @@ public Node get(ContainerNode parent, String name) throws TransientException { } NodeDAO dao = getDAO(); Node ret = dao.get(parent, name); + ret.parent = parent; + ret.owner = identityManager.toSubject(ret.ownerID); + ret.ownerDisplay = identityManager.toDisplayString(ret.owner); + if (ret instanceof DataNode) { DataNode dn = (DataNode) ret; ArtifactDAO artifactDAO = getArtifactDAO(); @@ -299,7 +306,38 @@ public ResourceIterator iterator(ContainerNode parent, Integer limit, Stri } NodeDAO dao = getDAO(); ResourceIterator ret = dao.iterator(parent, limit, start); - return ret; + return new IdentWrapper(parent, ret); + } + + private class IdentWrapper implements ResourceIterator { + + private final ContainerNode parent; + private final ResourceIterator childIter; + + IdentWrapper(ContainerNode parent, ResourceIterator childIter) { + this.parent = parent; + this.childIter = childIter; + } + + @Override + public boolean hasNext() { + return childIter.hasNext(); + } + + @Override + public Node next() { + Node ret = childIter.next(); + ret.parent = parent; + ret.owner = identityManager.toSubject(ret.ownerID); + ret.ownerDisplay = identityManager.toDisplayString(ret.owner); + return ret; + } + + @Override + public void close() throws IOException { + childIter.close(); + } + } /** @@ -336,6 +374,13 @@ public Node put(Node node) throws NodeNotSupportedException, TransientException } node.parentID = node.parent.getID(); } + if (node.ownerID == null) { + if (node.owner == null) { + throw new RuntimeException("BUG: cannot persist node without owner: " + node); + } + node.ownerID = identityManager.toOwner(node.owner); + } + if (node instanceof DataNode) { DataNode dn = (DataNode) node; if (dn.storageID == null) { From 4f455a8ad816e7a08582ab29b3a5cfa9370f038f Mon Sep 17 00:00:00 2001 From: Patrick Dowler Date: Wed, 9 Aug 2023 15:58:06 -0700 Subject: [PATCH 39/52] vault: formatting --- .../org/opencadc/vault/VaultInitAction.java | 41 +++++++++---------- 1 file changed, 20 insertions(+), 21 deletions(-) diff --git a/vault/src/main/java/org/opencadc/vault/VaultInitAction.java b/vault/src/main/java/org/opencadc/vault/VaultInitAction.java index a6c2195ad..bc9dc89f5 100644 --- a/vault/src/main/java/org/opencadc/vault/VaultInitAction.java +++ b/vault/src/main/java/org/opencadc/vault/VaultInitAction.java @@ -63,7 +63,7 @@ * . * ************************************************************************ -*/ + */ package org.opencadc.vault; @@ -89,8 +89,9 @@ * @author pdowler */ public class VaultInitAction extends InitAction { + private static final Logger log = Logger.getLogger(VaultInitAction.class); - + static final String JNDI_DATASOURCE = "jdbc/nodes"; // context.xml // config keys @@ -98,15 +99,15 @@ public class VaultInitAction extends InitAction { static final String RESOURCE_ID_KEY = VAULT_KEY + ".resourceID"; static final String INVENTORY_SCHEMA_KEY = VAULT_KEY + ".inventory.schema"; static final String VOSPACE_SCHEMA_KEY = VAULT_KEY + ".vospace.schema"; - + static final String ROOT_OWNER = VAULT_KEY + ".root.owner"; // numeric? - + static final String STORAGE_NAMESPACE_KEY = VAULT_KEY + ".storage.namespace"; - + MultiValuedProperties props; private URI resourceID; private Namespace storageNamespace; - private Map daoConfig; + private Map daoConfig; private String jndiNodePersistence; @@ -120,10 +121,10 @@ public void doInit() { initDatabase(); initNodePersistence(); } - + /** * Read config file and verify that all required entries are present. - * + * * @return MultiValuedProperties containing the application config * @throws IllegalStateException if required config items are missing */ @@ -152,7 +153,7 @@ static MultiValuedProperties getConfig() { } else { sb.append("OK"); } - + String vosSchema = mvp.getFirstPropertyValue(VOSPACE_SCHEMA_KEY); sb.append("\n\t").append(VOSPACE_SCHEMA_KEY).append(": "); if (vosSchema == null) { @@ -161,7 +162,7 @@ static MultiValuedProperties getConfig() { } else { sb.append("OK"); } - + String ns = mvp.getFirstPropertyValue(STORAGE_NAMESPACE_KEY); sb.append("\n\t").append(STORAGE_NAMESPACE_KEY).append(": "); if (ns == null) { @@ -170,22 +171,22 @@ static MultiValuedProperties getConfig() { } else { sb.append("OK"); } - + if (!ok) { throw new IllegalStateException(sb.toString()); } return mvp; } - - static Map getDaoConfig(MultiValuedProperties props) { + + static Map getDaoConfig(MultiValuedProperties props) { Map ret = new TreeMap<>(); ret.put("jndiDataSourceName", org.opencadc.vault.VaultInitAction.JNDI_DATASOURCE); ret.put("schema", props.getFirstPropertyValue(org.opencadc.vault.VaultInitAction.INVENTORY_SCHEMA_KEY)); ret.put("vosSchema", props.getFirstPropertyValue(org.opencadc.vault.VaultInitAction.VOSPACE_SCHEMA_KEY)); return ret; } - + private void initConfig() { log.info("initConfig: START"); this.props = getConfig(); @@ -200,7 +201,7 @@ private void initConfig() { throw new IllegalStateException("invalid config: " + RESOURCE_ID_KEY + " must be a valid URI"); } } - + private void initDatabase() { log.info("initDatabase: START"); try { @@ -216,8 +217,7 @@ private void initDatabase() { protected void initNodePersistence() { jndiNodePersistence = componentID + ".nodePersistence"; - try - { + try { Context ctx = new InitialContext(); try { ctx.unbind(jndiNodePersistence); @@ -227,15 +227,14 @@ protected void initNodePersistence() { NodePersistence npi = new NodePersistenceImpl(resourceID); ctx.bind(jndiNodePersistence, npi); - log.info("created JNDI key" + jndiNodePersistence + " "); - } catch(Exception ex) { + log.info("created JNDI key: " + jndiNodePersistence + " impl: " + npi.getClass().getName()); + } catch (Exception ex) { log.error("Failed to create JNDI Key " + jndiNodePersistence, ex); } } @Override - public void doShutdown() - { + public void doShutdown() { try { Context ctx = new InitialContext(); ctx.unbind(jndiNodePersistence); From 679b3ee488abfce249f0c010e6d4835d8ff8c197 Mon Sep 17 00:00:00 2001 From: Patrick Dowler Date: Fri, 11 Aug 2023 16:02:50 -0700 Subject: [PATCH 40/52] cadc-inventory-db: adapt to ContainerNode api change --- .../org/opencadc/vospace/db/NodeDAOTest.java | 23 ++++++++++--------- .../opencadc/inventory/db/SQLGenerator.java | 4 +++- 2 files changed, 15 insertions(+), 12 deletions(-) diff --git a/cadc-inventory-db/src/intTest/java/org/opencadc/vospace/db/NodeDAOTest.java b/cadc-inventory-db/src/intTest/java/org/opencadc/vospace/db/NodeDAOTest.java index 24291b403..230c70392 100644 --- a/cadc-inventory-db/src/intTest/java/org/opencadc/vospace/db/NodeDAOTest.java +++ b/cadc-inventory-db/src/intTest/java/org/opencadc/vospace/db/NodeDAOTest.java @@ -162,12 +162,12 @@ public void testGetByID_NotFound() { @Test public void testGetByPath_NotFound() { - ContainerNode parent = new ContainerNode("not-found", false); + ContainerNode parent = new ContainerNode("not-found"); Node a = nodeDAO.get(parent, "not-found"); Assert.assertNull(a); UUID rootID = new UUID(0L, 0L); - ContainerNode root = new ContainerNode(rootID, "root", false); + ContainerNode root = new ContainerNode(rootID, "root"); a = nodeDAO.get(root, "not-found"); Assert.assertNull(a); } @@ -176,10 +176,10 @@ public void testGetByPath_NotFound() { public void testPutGetUpdateDeleteContainerNode() throws InterruptedException, NoSuchAlgorithmException { UUID rootID = new UUID(0L, 0L); - ContainerNode root = new ContainerNode(rootID, "root", false); + ContainerNode root = new ContainerNode(rootID, "root"); // put - ContainerNode orig = new ContainerNode("container-test", false); + ContainerNode orig = new ContainerNode("container-test"); orig.parent = root; orig.ownerID = "the-owner"; nodeDAO.put(orig); @@ -228,6 +228,7 @@ public void testPutGetUpdateDeleteContainerNode() throws InterruptedException, orig.getReadWriteGroup().add(URI.create("ivo://opencadc.org/gms?g3")); orig.getProperties().add(new NodeProperty(VOS.PROPERTY_URI_CONTENTLENGTH, "123")); orig.isPublic = true; + orig.inheritPermissions = true; nodeDAO.put(orig); Node updated = nodeDAO.get(orig.getID()); Assert.assertNotNull(updated); @@ -259,11 +260,11 @@ public void testPutGetUpdateDeleteContainerNode() throws InterruptedException, public void testPutGetUpdateDeleteContainerNodeMax() throws InterruptedException, NoSuchAlgorithmException { UUID rootID = new UUID(0L, 0L); - ContainerNode root = new ContainerNode(rootID, "root", false); + ContainerNode root = new ContainerNode(rootID, "root"); // TODO: use get-by-path to find and remove the test node - ContainerNode orig = new ContainerNode("container-test", false); + ContainerNode orig = new ContainerNode("container-test"); orig.parent = root; orig.ownerID = "the-owner"; orig.isPublic = true; @@ -363,7 +364,7 @@ public void testPutGetUpdateDeleteContainerNodeMax() throws InterruptedException public void testPutGetUpdateDeleteDataNode() throws InterruptedException, NoSuchAlgorithmException { UUID rootID = new UUID(0L, 0L); - ContainerNode root = new ContainerNode(rootID, "root", false); + ContainerNode root = new ContainerNode(rootID, "root"); DataNode orig = new DataNode(UUID.randomUUID(), "data-test", URI.create("cadc:vault/" + UUID.randomUUID())); orig.parent = root; @@ -454,7 +455,7 @@ public void testPutGetUpdateDeleteDataNode() throws InterruptedException, public void testPutGetUpdateDeleteLinkNode() throws InterruptedException, NoSuchAlgorithmException { UUID rootID = new UUID(0L, 0L); - ContainerNode root = new ContainerNode(rootID, "root", false); + ContainerNode root = new ContainerNode(rootID, "root"); // TODO: use get-by-path to find and remove the test node @@ -543,9 +544,9 @@ public void testPutGetUpdateDeleteLinkNode() throws InterruptedException, @Test public void testContainerNodeIterator() throws IOException { UUID rootID = new UUID(0L, 0L); - ContainerNode root = new ContainerNode(rootID, "root", false); + ContainerNode root = new ContainerNode(rootID, "root"); - ContainerNode orig = new ContainerNode("container-test", false); + ContainerNode orig = new ContainerNode("container-test"); orig.parent = root; orig.ownerID = "the-owner"; nodeDAO.put(orig); @@ -576,7 +577,7 @@ public void testContainerNodeIterator() throws IOException { Assert.assertEquals(orig.getID(), top.getID()); // add children - ContainerNode cont = new ContainerNode("container1", false); + ContainerNode cont = new ContainerNode("container1"); cont.parent = orig; cont.ownerID = orig.ownerID; DataNode data = new DataNode(UUID.randomUUID(), "data1", URI.create("cadc:vault/" + UUID.randomUUID())); diff --git a/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/SQLGenerator.java b/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/SQLGenerator.java index bd5b2eb42..76baf96de 100644 --- a/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/SQLGenerator.java +++ b/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/SQLGenerator.java @@ -1738,7 +1738,9 @@ private Node mapRowToNode(ResultSet rs, Calendar utc, ContainerNode parent) thro Node ret; if (nodeType.equals("C")) { - ret = new ContainerNode(id, name, inheritPermissions); + ContainerNode cn = new ContainerNode(id, name); + cn.inheritPermissions = inheritPermissions; + ret = cn; } else if (nodeType.equals("D")) { ret = new DataNode(id, name, storageID); } else if (nodeType.equals("L")) { From 62645da74dc390200826c3b6f486420a79bcd298 Mon Sep 17 00:00:00 2001 From: Patrick Dowler Date: Fri, 11 Aug 2023 16:03:17 -0700 Subject: [PATCH 41/52] vault: quick hack fix for delete(Node) to allow repeated create/delete to move to trash --- .../java/org/opencadc/vault/NodesTest.java | 11 ++++++++++- .../org/opencadc/vault/NodePersistenceImpl.java | 17 ++++++++++------- 2 files changed, 20 insertions(+), 8 deletions(-) diff --git a/vault/src/intTest/java/org/opencadc/vault/NodesTest.java b/vault/src/intTest/java/org/opencadc/vault/NodesTest.java index a8afb4078..71bb1178b 100644 --- a/vault/src/intTest/java/org/opencadc/vault/NodesTest.java +++ b/vault/src/intTest/java/org/opencadc/vault/NodesTest.java @@ -67,6 +67,9 @@ package org.opencadc.vault; +import ca.nrc.cadc.util.Log4jInit; +import java.net.URI; +import org.apache.log4j.Level; import org.apache.log4j.Logger; /** @@ -77,6 +80,12 @@ public class NodesTest extends org.opencadc.conformance.vos.NodesTest { private static final Logger log = Logger.getLogger(NodesTest.class); - public NodesTest() { + static { + Log4jInit.setLevel("org.opencadc.conformance.vos", Level.DEBUG); + Log4jInit.setLevel("org.opencadc.vospace", Level.DEBUG); + } + + public NodesTest() { + super(URI.create("ivo://opencadc.org/vault"), "vault-test.pem"); } } diff --git a/vault/src/main/java/org/opencadc/vault/NodePersistenceImpl.java b/vault/src/main/java/org/opencadc/vault/NodePersistenceImpl.java index ab960c6fc..07cfde569 100644 --- a/vault/src/main/java/org/opencadc/vault/NodePersistenceImpl.java +++ b/vault/src/main/java/org/opencadc/vault/NodePersistenceImpl.java @@ -159,12 +159,11 @@ public X509CertificateChain getCertificateChain() { // root node UUID rootID = new UUID(0L, 0L); - this.root = new ContainerNode(rootID, "", false); + this.root = new ContainerNode(rootID, ""); root.owner = identityManager.augment(rawOwnerSubject); root.ownerDisplay = identityManager.toDisplayString(root.owner); log.warn("ROOT owner: " + root.owner); root.ownerID = identityManager.toOwner(rawOwnerSubject); - log.warn("ROOT ownerID: " + root.ownerID + " rtype: " + root.ownerID.getClass().getName()); root.isPublic = true; root.inheritPermissions = false; @@ -173,7 +172,7 @@ public X509CertificateChain getCertificateChain() { NodeDAO dao = getDAO(); ContainerNode tn = (ContainerNode) dao.get(root, ".trash"); if (tn == null) { - tn = new ContainerNode(".trash", false); + tn = new ContainerNode(".trash"); } // always reset props to current config tn.ownerID = root.ownerID; @@ -263,6 +262,9 @@ public Node get(ContainerNode parent, String name) throws TransientException { } NodeDAO dao = getDAO(); Node ret = dao.get(parent, name); + if (ret == null) { + return null; + } ret.parent = parent; ret.owner = identityManager.toSubject(ret.ownerID); ret.ownerDisplay = identityManager.toDisplayString(ret.owner); @@ -482,17 +484,18 @@ public void delete(Node node) throws TransientException { } } - // TODO: create DeletedNodeEvent + // TODO: create DeletedNodeEvent? + // need DeletedNodeDAO // what about DNE for all child nodes, which also got deleted? or would sync of a // DNE involve calling NodePersistence.delete(DeletedNodeEvent) to replay this same // deletion logic in a mirror?? TBD // persisting the DNE means that recovery from trash has to re-assign IDs - - DeletedNodeEvent dne = new DeletedNodeEvent(node.getID(), node.getClass(), storageID); - // TODO: need DeletedNodeDAO + + // TODO: if DataNode (storageID != null): delete artifact and create DeletedArtifactEvent? if (moveToTrash) { node.parentID = trash.getID(); + node.setName(node.getName() + "-" + UUID.randomUUID().toString()); dao.put(node); } else { dao.delete(node.getID()); From 8962bde9fe15a66b8316ea4d227d5f43efa288a0 Mon Sep 17 00:00:00 2001 From: Patrick Dowler Date: Tue, 15 Aug 2023 11:19:16 -0700 Subject: [PATCH 42/52] vault: fix imports --- .../src/main/java/org/opencadc/vault/NodePersistenceImpl.java | 4 ---- 1 file changed, 4 deletions(-) diff --git a/vault/src/main/java/org/opencadc/vault/NodePersistenceImpl.java b/vault/src/main/java/org/opencadc/vault/NodePersistenceImpl.java index 07cfde569..f89d04c57 100644 --- a/vault/src/main/java/org/opencadc/vault/NodePersistenceImpl.java +++ b/vault/src/main/java/org/opencadc/vault/NodePersistenceImpl.java @@ -67,7 +67,6 @@ package org.opencadc.vault; -import sun.rmi.rmic.IndentingWriter; import ca.nrc.cadc.auth.AuthenticationUtil; import ca.nrc.cadc.auth.IdentityManager; import ca.nrc.cadc.auth.PrincipalExtractor; @@ -75,7 +74,6 @@ import ca.nrc.cadc.date.DateUtil; import ca.nrc.cadc.io.ResourceIterator; import ca.nrc.cadc.net.TransientException; -import ca.nrc.cadc.util.FileMetadata; import ca.nrc.cadc.util.InvalidConfigException; import ca.nrc.cadc.util.MultiValuedProperties; import java.io.IOException; @@ -84,7 +82,6 @@ import java.text.DateFormat; import java.util.HashSet; import java.util.Iterator; -import java.util.List; import java.util.Map; import java.util.Set; import java.util.TreeMap; @@ -99,7 +96,6 @@ import org.opencadc.inventory.db.SQLGenerator; import org.opencadc.vospace.ContainerNode; import org.opencadc.vospace.DataNode; -import org.opencadc.vospace.DeletedNodeEvent; import org.opencadc.vospace.LinkNode; import org.opencadc.vospace.Node; import org.opencadc.vospace.NodeNotSupportedException; From 0420a6b718511513bafa0ee8fc9ef2f8c372eacd Mon Sep 17 00:00:00 2001 From: Patrick Dowler Date: Wed, 23 Aug 2023 11:18:10 -0700 Subject: [PATCH 43/52] vault: added indentity cache to child iterator --- .../org/opencadc/vault/NodePersistenceImpl.java | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/vault/src/main/java/org/opencadc/vault/NodePersistenceImpl.java b/vault/src/main/java/org/opencadc/vault/NodePersistenceImpl.java index f89d04c57..bfdb7ba04 100644 --- a/vault/src/main/java/org/opencadc/vault/NodePersistenceImpl.java +++ b/vault/src/main/java/org/opencadc/vault/NodePersistenceImpl.java @@ -311,10 +311,15 @@ private class IdentWrapper implements ResourceIterator { private final ContainerNode parent; private final ResourceIterator childIter; + private Map identCache = new TreeMap<>(); IdentWrapper(ContainerNode parent, ResourceIterator childIter) { this.parent = parent; this.childIter = childIter; + // prime cache with caller + Subject caller = AuthenticationUtil.getCurrentSubject(); + Object ownerID = identityManager.toOwner(caller); + identCache.put(ownerID, caller); } @Override @@ -326,7 +331,12 @@ public boolean hasNext() { public Node next() { Node ret = childIter.next(); ret.parent = parent; - ret.owner = identityManager.toSubject(ret.ownerID); + Subject s = identCache.get(ret.ownerID); + if (s == null) { + s = identityManager.toSubject(ret.ownerID); + identCache.put(ret.ownerID, s); + } + ret.owner = s; ret.ownerDisplay = identityManager.toDisplayString(ret.owner); return ret; } @@ -334,6 +344,7 @@ public Node next() { @Override public void close() throws IOException { childIter.close(); + identCache.clear(); } } From 1c269619c44065be7195c36aac48a780b6cb5c4e Mon Sep 17 00:00:00 2001 From: Patrick Dowler Date: Wed, 23 Aug 2023 13:17:22 -0700 Subject: [PATCH 44/52] vault: fix class cast issue with ident cache in iterator delete(Node) now rejects deleting non-empty container untested: delete(node) also delete Artifact and generate DeletedArtifactEvent --- .../opencadc/vault/NodePersistenceImpl.java | 64 +++++++++++-------- 1 file changed, 36 insertions(+), 28 deletions(-) diff --git a/vault/src/main/java/org/opencadc/vault/NodePersistenceImpl.java b/vault/src/main/java/org/opencadc/vault/NodePersistenceImpl.java index bfdb7ba04..5484d14dd 100644 --- a/vault/src/main/java/org/opencadc/vault/NodePersistenceImpl.java +++ b/vault/src/main/java/org/opencadc/vault/NodePersistenceImpl.java @@ -91,8 +91,10 @@ import javax.security.auth.x500.X500Principal; import org.apache.log4j.Logger; import org.opencadc.inventory.Artifact; +import org.opencadc.inventory.DeletedArtifactEvent; import org.opencadc.inventory.Namespace; import org.opencadc.inventory.db.ArtifactDAO; +import org.opencadc.inventory.db.DeletedArtifactEventDAO; import org.opencadc.inventory.db.SQLGenerator; import org.opencadc.vospace.ContainerNode; import org.opencadc.vospace.DataNode; @@ -119,8 +121,6 @@ public class NodePersistenceImpl implements NodePersistence { private final Set artifactProps = new TreeSet<>(); private URI resourceID; - private IdentityManager identityManager = AuthenticationUtil.getIdentityManager(); - public NodePersistenceImpl(URI resourceID) { if (resourceID == null) { throw new IllegalArgumentException("resource ID required"); @@ -152,6 +152,7 @@ public X509CertificateChain getCertificateChain() { return null; } }); + IdentityManager identityManager = AuthenticationUtil.getIdentityManager(); // root node UUID rootID = new UUID(0L, 0L); @@ -262,9 +263,12 @@ public Node get(ContainerNode parent, String name) throws TransientException { return null; } ret.parent = parent; + IdentityManager identityManager = AuthenticationUtil.getIdentityManager(); ret.owner = identityManager.toSubject(ret.ownerID); ret.ownerDisplay = identityManager.toDisplayString(ret.owner); + // in principle we could have queried vospace.Node join inventory.Artifact above + // and avoid this query.... simplicity for now if (ret instanceof DataNode) { DataNode dn = (DataNode) ret; ArtifactDAO artifactDAO = getArtifactDAO(); @@ -311,6 +315,8 @@ private class IdentWrapper implements ResourceIterator { private final ContainerNode parent; private final ResourceIterator childIter; + + private IdentityManager identityManager = AuthenticationUtil.getIdentityManager(); private Map identCache = new TreeMap<>(); IdentWrapper(ContainerNode parent, ResourceIterator childIter) { @@ -318,8 +324,14 @@ private class IdentWrapper implements ResourceIterator { this.childIter = childIter; // prime cache with caller Subject caller = AuthenticationUtil.getCurrentSubject(); - Object ownerID = identityManager.toOwner(caller); - identCache.put(ownerID, caller); + if (caller != null) { + Object ownerID = identityManager.toOwner(caller); + if (ownerID != null) { + // HACK: NodeDAO returns ownerID as String and relies on the IM + // to convert to a number (eg) + identCache.put(ownerID.toString(), caller); + } + } } @Override @@ -387,6 +399,7 @@ public Node put(Node node) throws NodeNotSupportedException, TransientException if (node.owner == null) { throw new RuntimeException("BUG: cannot persist node without owner: " + node); } + IdentityManager identityManager = AuthenticationUtil.getIdentityManager(); node.ownerID = identityManager.toOwner(node.owner); } @@ -465,48 +478,43 @@ public void delete(Node node) throws TransientException { throw new IllegalArgumentException("arg cannot be null: node"); } - NodeDAO dao = getDAO(); + final NodeDAO dao = getDAO(); + final ArtifactDAO artifactDAO = getArtifactDAO(); // TODO: do the following in a transaction, acquire lock on target node - boolean moveToTrash = true; // default URI storageID = null; if (node instanceof ContainerNode) { ContainerNode cn = (ContainerNode) node; try (ResourceIterator iter = dao.iterator(cn, 1, null)) { - moveToTrash = iter.hasNext(); // empty + if (iter.hasNext()) { + throw new IllegalArgumentException("container node '" + node.getName() + "' is not empty"); + } } catch (IOException ex) { throw new TransientException("database IO failure", ex); } - } else if (node instanceof LinkNode) { - moveToTrash = false; } else if (node instanceof DataNode) { DataNode dn = (DataNode) node; NodeProperty len = dn.getProperty(VOS.PROPERTY_URI_CONTENTLENGTH); - if (len == null) { - // artifact does not exist - moveToTrash = false; - } else { + if (len != null) { + // artifact exists storageID = dn.storageID; } - } + } // else: LinkNode can always be deleted - // TODO: create DeletedNodeEvent? - // need DeletedNodeDAO - // what about DNE for all child nodes, which also got deleted? or would sync of a - // DNE involve calling NodePersistence.delete(DeletedNodeEvent) to replay this same - // deletion logic in a mirror?? TBD - // persisting the DNE means that recovery from trash has to re-assign IDs - - // TODO: if DataNode (storageID != null): delete artifact and create DeletedArtifactEvent? + // TODO: need DeletedNodeDAO to create DeletedNodeEvent - if (moveToTrash) { - node.parentID = trash.getID(); - node.setName(node.getName() + "-" + UUID.randomUUID().toString()); - dao.put(node); - } else { - dao.delete(node.getID()); + if (storageID != null) { + Artifact a = artifactDAO.get(storageID); + if (a != null) { + DeletedArtifactEventDAO daeDAO = new DeletedArtifactEventDAO(artifactDAO); + DeletedArtifactEvent dae = new DeletedArtifactEvent(a.getID()); + daeDAO.put(dae); + artifactDAO.delete(a.getID()); + } } + dao.delete(node.getID()); + // TODO: commit transaction } } From e6ef466abc65630e8bc56635ae58c813cbe45538 Mon Sep 17 00:00:00 2001 From: Adrian Damian Date: Wed, 6 Sep 2023 14:10:22 -0700 Subject: [PATCH 45/52] Changed type of group sets in Node to GroupURI --- cadc-inventory-db/build.gradle | 1 + .../org/opencadc/vospace/db/NodeDAOTest.java | 32 +++++++++---------- .../opencadc/inventory/db/SQLGenerator.java | 16 ++++------ .../java/org/opencadc/inventory/db/Util.java | 11 +++---- .../org/opencadc/inventory/db/UtilTest.java | 13 ++++---- 5 files changed, 34 insertions(+), 39 deletions(-) diff --git a/cadc-inventory-db/build.gradle b/cadc-inventory-db/build.gradle index 2f1566ca2..d96867abb 100644 --- a/cadc-inventory-db/build.gradle +++ b/cadc-inventory-db/build.gradle @@ -26,6 +26,7 @@ mainClassName = 'org.opencadc.inventory.db.version.Main' dependencies { compile 'org.opencadc:cadc-util:[1.9.5,2.0)' + compile 'org.opencadc:cadc-gms:[1.0.0,)' compile 'org.opencadc:cadc-inventory:[0.9.4,)' compile 'org.opencadc:cadc-vos:[2.0,3.0)' diff --git a/cadc-inventory-db/src/intTest/java/org/opencadc/vospace/db/NodeDAOTest.java b/cadc-inventory-db/src/intTest/java/org/opencadc/vospace/db/NodeDAOTest.java index 230c70392..10c8e9e46 100644 --- a/cadc-inventory-db/src/intTest/java/org/opencadc/vospace/db/NodeDAOTest.java +++ b/cadc-inventory-db/src/intTest/java/org/opencadc/vospace/db/NodeDAOTest.java @@ -86,6 +86,7 @@ import org.junit.Assert; import org.junit.Before; import org.junit.Test; +import org.opencadc.gms.GroupURI; import org.opencadc.inventory.db.SQLGenerator; import org.opencadc.inventory.db.TestUtil; import org.opencadc.vospace.ContainerNode; @@ -224,8 +225,8 @@ public void testPutGetUpdateDeleteContainerNode() throws InterruptedException, // update Thread.sleep(10L); - orig.getReadOnlyGroup().add(URI.create("ivo://opencadc.org/gms?g1")); - orig.getReadWriteGroup().add(URI.create("ivo://opencadc.org/gms?g3")); + orig.getReadOnlyGroup().add(new GroupURI(URI.create("ivo://opencadc.org/gms?g1"))); + orig.getReadWriteGroup().add(new GroupURI(URI.create("ivo://opencadc.org/gms?g3"))); orig.getProperties().add(new NodeProperty(VOS.PROPERTY_URI_CONTENTLENGTH, "123")); orig.isPublic = true; orig.inheritPermissions = true; @@ -270,14 +271,13 @@ public void testPutGetUpdateDeleteContainerNodeMax() throws InterruptedException orig.isPublic = true; orig.isLocked = false; orig.inheritPermissions = false; - orig.getReadOnlyGroup().add(URI.create("ivo://opencadc.org/gms?g1")); - orig.getReadOnlyGroup().add(URI.create("ivo://opencadc.org/gms?g2")); - orig.getReadWriteGroup().add(URI.create("ivo://opencadc.org/gms?g3")); - orig.getReadWriteGroup().add(URI.create("ivo://opencadc.org/gms?g4,g5")); - orig.getReadWriteGroup().add(URI.create("ivo://opencadc.org/gms?g6-g7")); - orig.getReadWriteGroup().add(URI.create("ivo://opencadc.org/gms?g6.g7")); - orig.getReadWriteGroup().add(URI.create("ivo://opencadc.org/gms?g6_g7")); - orig.getReadWriteGroup().add(URI.create("ivo://opencadc.org/gms?g6~g7")); + orig.getReadOnlyGroup().add(new GroupURI(URI.create("ivo://opencadc.org/gms?g1"))); + orig.getReadOnlyGroup().add(new GroupURI(URI.create("ivo://opencadc.org/gms?g2"))); + orig.getReadWriteGroup().add(new GroupURI(URI.create("ivo://opencadc.org/gms?g3"))); + orig.getReadWriteGroup().add(new GroupURI(URI.create("ivo://opencadc.org/gms?g6-g7"))); + orig.getReadWriteGroup().add(new GroupURI(URI.create("ivo://opencadc.org/gms?g6.g7"))); + orig.getReadWriteGroup().add(new GroupURI(URI.create("ivo://opencadc.org/gms?g6_g7"))); + orig.getReadWriteGroup().add(new GroupURI(URI.create("ivo://opencadc.org/gms?g6~g7"))); orig.getProperties().add(new NodeProperty(VOS.PROPERTY_URI_CONTENTLENGTH, "123")); orig.getProperties().add(new NodeProperty(URI.create("custom:prop"), "spaces in value")); @@ -328,9 +328,9 @@ public void testPutGetUpdateDeleteContainerNodeMax() throws InterruptedException orig.isPublic = false; orig.isLocked = true; orig.getReadOnlyGroup().clear(); - orig.getReadOnlyGroup().add(URI.create("ivo://opencadc.org/gms?g1")); + orig.getReadOnlyGroup().add(new GroupURI(URI.create("ivo://opencadc.org/gms?g1"))); orig.getReadWriteGroup().clear(); - orig.getReadWriteGroup().add(URI.create("ivo://opencadc.org/gms?g3")); + orig.getReadWriteGroup().add(new GroupURI(URI.create("ivo://opencadc.org/gms?g3"))); orig.getProperties().clear(); orig.getProperties().add(new NodeProperty(VOS.PROPERTY_URI_CONTENTLENGTH, "123")); orig.inheritPermissions = true; @@ -418,9 +418,9 @@ public void testPutGetUpdateDeleteDataNode() throws InterruptedException, orig.isPublic = false; orig.isLocked = true; orig.getReadOnlyGroup().clear(); - orig.getReadOnlyGroup().add(URI.create("ivo://opencadc.org/gms?g1")); + orig.getReadOnlyGroup().add(new GroupURI(URI.create("ivo://opencadc.org/gms?g1"))); orig.getReadWriteGroup().clear(); - orig.getReadWriteGroup().add(URI.create("ivo://opencadc.org/gms?g3")); + orig.getReadWriteGroup().add(new GroupURI(URI.create("ivo://opencadc.org/gms?g3"))); orig.getProperties().clear(); orig.getProperties().add(new NodeProperty(VOS.PROPERTY_URI_CONTENTLENGTH, "123")); // don't change storageID @@ -509,9 +509,9 @@ public void testPutGetUpdateDeleteLinkNode() throws InterruptedException, orig.isPublic = false; orig.isLocked = true; orig.getReadOnlyGroup().clear(); - orig.getReadOnlyGroup().add(URI.create("ivo://opencadc.org/gms?g1")); + orig.getReadOnlyGroup().add(new GroupURI(URI.create("ivo://opencadc.org/gms?g1"))); orig.getReadWriteGroup().clear(); - orig.getReadWriteGroup().add(URI.create("ivo://opencadc.org/gms?g3")); + orig.getReadWriteGroup().add(new GroupURI(URI.create("ivo://opencadc.org/gms?g3"))); orig.getProperties().clear(); orig.getProperties().add(new NodeProperty(VOS.PROPERTY_URI_CONTENTLENGTH, "123")); // don't change target diff --git a/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/SQLGenerator.java b/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/SQLGenerator.java index 76baf96de..32a65631b 100644 --- a/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/SQLGenerator.java +++ b/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/SQLGenerator.java @@ -72,8 +72,6 @@ import ca.nrc.cadc.util.StringUtil; import java.io.IOException; import java.net.URI; -import java.net.URISyntaxException; -import java.sql.Array; import java.sql.Connection; import java.sql.PreparedStatement; import java.sql.ResultSet; @@ -83,16 +81,15 @@ import java.util.Calendar; import java.util.Comparator; import java.util.Date; -import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Set; -import java.util.StringTokenizer; import java.util.TreeMap; import java.util.TreeSet; import java.util.UUID; import javax.sql.DataSource; import org.apache.log4j.Logger; +import org.opencadc.gms.GroupURI; import org.opencadc.inventory.Artifact; import org.opencadc.inventory.DeletedArtifactEvent; import org.opencadc.inventory.DeletedStorageLocationEvent; @@ -109,7 +106,6 @@ import org.opencadc.vospace.LinkNode; import org.opencadc.vospace.Node; import org.opencadc.vospace.NodeProperty; -import org.opencadc.vospace.VOS; import org.springframework.dao.DataAccessException; import org.springframework.jdbc.core.JdbcTemplate; import org.springframework.jdbc.core.ResultSetExtractor; @@ -979,14 +975,14 @@ private void safeSetTimestamp(PreparedStatement prep, int col, Timestamp value, } } - private void safeSetArray(PreparedStatement prep, int col, Set values) throws SQLException { + private void safeSetArray(PreparedStatement prep, int col, Set values) throws SQLException { if (values != null && !values.isEmpty()) { log.debug("safeSetArray: " + col + " " + values.size()); String[] array1d = new String[values.size()]; int i = 0; - for (URI u : values) { - array1d[i] = u.toASCIIString(); + for (GroupURI u : values) { + array1d[i] = u.getURI().toASCIIString(); i++; } java.sql.Array arr = prep.getConnection().createArrayOf("text", array1d); @@ -1754,10 +1750,10 @@ private Node mapRowToNode(ResultSet rs, Calendar utc, ContainerNode parent) thro ret.isLocked = isLocked; if (rawROG != null) { - Util.parseArrayURI(rawROG, ret.getReadOnlyGroup()); + Util.parseArrayGroupURI(rawROG, ret.getReadOnlyGroup()); } if (rawRWG != null) { - Util.parseArrayURI(rawRWG, ret.getReadWriteGroup()); + Util.parseArrayGroupURI(rawRWG, ret.getReadWriteGroup()); } if (rawProps != null) { Util.parseArrayProps(rawProps, ret.getProperties()); diff --git a/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/Util.java b/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/Util.java index 2e7e197b8..782cb8a6f 100644 --- a/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/Util.java +++ b/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/Util.java @@ -75,14 +75,13 @@ import java.sql.Array; import java.sql.ResultSet; import java.sql.SQLException; -import java.util.ArrayList; import java.util.Calendar; import java.util.Date; -import java.util.List; import java.util.Set; import java.util.StringTokenizer; import java.util.UUID; import org.apache.log4j.Logger; +import org.opencadc.gms.GroupURI; import org.opencadc.vospace.NodeProperty; /** @@ -384,7 +383,7 @@ public static byte[] getByteArray(ResultSet rs, int col) } // fills the dest set - public static void parseArrayURI(String val, Set dest) { + public static void parseArrayGroupURI(String val, Set dest) { // postgresql 1D array: {a,"b,c"} if (val == null || val.isEmpty()) { return; @@ -407,14 +406,14 @@ public static void parseArrayURI(String val, Set dest) { handleToken(token, dest); } - private static void handleToken(String token, Set dest) { + private static void handleToken(String token, Set dest) { if (token.startsWith("ivo://")) { - dest.add(URI.create(token)); + dest.add(new GroupURI(URI.create(token))); } else { StringTokenizer st = new StringTokenizer(token, "{,}"); while (st.hasMoreTokens()) { String s = st.nextToken(); - dest.add(URI.create(s)); + dest.add(new GroupURI(URI.create(s))); } } } diff --git a/cadc-inventory-db/src/test/java/org/opencadc/inventory/db/UtilTest.java b/cadc-inventory-db/src/test/java/org/opencadc/inventory/db/UtilTest.java index adf255402..c26cf3f26 100644 --- a/cadc-inventory-db/src/test/java/org/opencadc/inventory/db/UtilTest.java +++ b/cadc-inventory-db/src/test/java/org/opencadc/inventory/db/UtilTest.java @@ -68,12 +68,12 @@ package org.opencadc.inventory.db; import ca.nrc.cadc.util.Log4jInit; -import java.net.URI; import java.util.Set; import java.util.TreeSet; import org.apache.log4j.Level; import org.apache.log4j.Logger; import org.junit.Test; +import org.opencadc.gms.GroupURI; import org.opencadc.vospace.NodeProperty; /** @@ -91,19 +91,18 @@ public UtilTest() { } @Test - public void testParseArrayURI() throws Exception { + public void testParseArrayGroupURI() throws Exception { String str = "{ivo://opencadc.org/gms?g3," - + "\"ivo://opencadc.org/gms?g4,g5\"," + "ivo://opencadc.org/gms?g6-g7," + "ivo://opencadc.org/gms?g6.g7," + "ivo://opencadc.org/gms?g6_g7," + "ivo://opencadc.org/gms?g6~g7}"; - Set dest = new TreeSet<>(); - Util.parseArrayURI(str, dest); - for (URI u : dest) { - log.info("uri: " + u); + Set dest = new TreeSet<>(); + Util.parseArrayGroupURI(str, dest); + for (GroupURI u : dest) { + log.info("uri: " + u.getURI()); } } From ddde9d923a62e980b39b2d6ec92135233fcae79e Mon Sep 17 00:00:00 2001 From: Patrick Dowler Date: Thu, 7 Sep 2023 12:17:50 -0700 Subject: [PATCH 46/52] cadc-inventory-db: fix NodeDAO.lock for subclass support add NodeDAO.isEmpty(ContainerNode) to support check in delete code --- .../org/opencadc/vospace/db/NodeDAOTest.java | 34 +++++++++++++++++++ .../opencadc/inventory/db/SQLGenerator.java | 34 ++++++++++++++++--- .../java/org/opencadc/vospace/db/NodeDAO.java | 29 ++++++++++++++++ 3 files changed, 93 insertions(+), 4 deletions(-) diff --git a/cadc-inventory-db/src/intTest/java/org/opencadc/vospace/db/NodeDAOTest.java b/cadc-inventory-db/src/intTest/java/org/opencadc/vospace/db/NodeDAOTest.java index 10c8e9e46..c24341c6e 100644 --- a/cadc-inventory-db/src/intTest/java/org/opencadc/vospace/db/NodeDAOTest.java +++ b/cadc-inventory-db/src/intTest/java/org/opencadc/vospace/db/NodeDAOTest.java @@ -541,6 +541,35 @@ public void testPutGetUpdateDeleteLinkNode() throws InterruptedException, Assert.assertNull(gone); } + @Test + public void testGetWithLock() { + UUID rootID = new UUID(0L, 0L); + ContainerNode root = new ContainerNode(rootID, "root"); + + // put + ContainerNode orig = new ContainerNode("container-test"); + orig.parent = root; + orig.ownerID = "the-owner"; + nodeDAO.put(orig); + + // get-by-id + Node a = nodeDAO.get(orig.getID()); + Assert.assertNotNull(a); + log.info("found by id: " + a.getID() + " aka " + a); + Assert.assertEquals(orig.getID(), a.getID()); + Assert.assertEquals(orig.getName(), a.getName()); + Assert.assertEquals(root.getID(), a.parentID); + + // get with lock + Node locked = nodeDAO.lock(a); + Assert.assertNotNull(locked); + log.info("locked: " + a.getID() + " aka " + a); + + nodeDAO.delete(orig.getID()); + Node gone = nodeDAO.get(orig.getID()); + Assert.assertNull(gone); + } + @Test public void testContainerNodeIterator() throws IOException { UUID rootID = new UUID(0L, 0L); @@ -558,6 +587,8 @@ public void testContainerNodeIterator() throws IOException { Assert.assertEquals(orig.getName(), a.getName()); Assert.assertTrue(a instanceof ContainerNode); + ContainerNode cn = (ContainerNode) a; + Assert.assertTrue(nodeDAO.isEmpty(cn)); // these are set in put Assert.assertEquals(orig.getMetaChecksum(), a.getMetaChecksum()); @@ -588,10 +619,13 @@ public void testContainerNodeIterator() throws IOException { link.ownerID = orig.ownerID; log.info("put child: " + cont + " of " + cont.parent); nodeDAO.put(cont); + Assert.assertFalse(nodeDAO.isEmpty(cn)); log.info("put child: " + data + " of " + data.parent); nodeDAO.put(data); + Assert.assertFalse(nodeDAO.isEmpty(cn)); log.info("put child: " + link + " of " + link.parent); nodeDAO.put(link); + Assert.assertFalse(nodeDAO.isEmpty(cn)); Node c1; Node c2; diff --git a/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/SQLGenerator.java b/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/SQLGenerator.java index 32a65631b..6ac3dcbb0 100644 --- a/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/SQLGenerator.java +++ b/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/SQLGenerator.java @@ -108,6 +108,7 @@ import org.opencadc.vospace.NodeProperty; import org.springframework.dao.DataAccessException; import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.jdbc.core.PreparedStatementCreator; import org.springframework.jdbc.core.ResultSetExtractor; import org.springframework.jdbc.core.RowMapper; @@ -304,17 +305,15 @@ public EntityGet getEntityGet(Class c, boolean forUpdate) { return new StorageSiteGet(forUpdate); } - if (Node.class.equals(c) || Node.class.isInstance(c)) { + if (Node.class.equals(c)) { return new NodeGet(forUpdate); } - if (DeletedNodeEvent.class.equals(c)) { - //return new DeletedNodeGet(); - } if (forUpdate) { throw new UnsupportedOperationException("entity-get + forUpdate: " + c.getSimpleName()); } + // raw events are never locked for update if (DeletedArtifactEvent.class.equals(c)) { return new DeletedArtifactEventGet(); } @@ -327,6 +326,11 @@ public EntityGet getEntityGet(Class c, boolean forUpdate) { if (ObsoleteStorageLocation.class.equals(c)) { return new ObsoleteStorageLocationGet(); } + + if (DeletedNodeEvent.class.equals(c)) { + //return new DeletedNodeGet(); + } + if (HarvestState.class.equals(c)) { return new HarvestStateGet(); } @@ -334,6 +338,28 @@ public EntityGet getEntityGet(Class c, boolean forUpdate) { throw new UnsupportedOperationException("entity-get: " + c.getName()); } + public NodeCount getNodeCount() { + return new NodeCount(); + } + + public class NodeCount { + private UUID id; + + public void setID(UUID id) { + this.id = id; + } + + public int execute(JdbcTemplate jdbc) { + StringBuilder sb = new StringBuilder(); + sb.append("SELECT count(*) FROM ").append(getTable(Node.class)); + sb.append(" WHERE parentID = '").append(id.toString()).append("'"); + String sql = sb.toString(); + log.debug("NodeCount: " + sql); + int ret = jdbc.queryForObject(sql, Integer.class); + return ret; + } + } + public EntityIteratorQuery getEntityIteratorQuery(Class c) { if (Artifact.class.equals(c)) { return new ArtifactIteratorQuery(); diff --git a/cadc-inventory-db/src/main/java/org/opencadc/vospace/db/NodeDAO.java b/cadc-inventory-db/src/main/java/org/opencadc/vospace/db/NodeDAO.java index 6fc157862..acf81b961 100644 --- a/cadc-inventory-db/src/main/java/org/opencadc/vospace/db/NodeDAO.java +++ b/cadc-inventory-db/src/main/java/org/opencadc/vospace/db/NodeDAO.java @@ -98,6 +98,15 @@ public void put(Node val) { super.put(val); } + @Override + public Node lock(Node n) { + if (n == null) { + throw new IllegalArgumentException("entity cannot be null"); + } + // override because Node has subclasses: force base class here + return super.lock(Node.class, n.getID()); + } + public Node get(UUID id) { return super.get(Node.class, id); } @@ -121,6 +130,26 @@ public Node get(ContainerNode parent, String name) { throw new RuntimeException("BUG: handleInternalFail did not throw"); } + public boolean isEmpty(ContainerNode parent) { + checkInit(); + log.debug("isEmpty: " + parent.getID()); + long t = System.currentTimeMillis(); + + try { + JdbcTemplate jdbc = new JdbcTemplate(dataSource); + SQLGenerator.NodeCount count = (SQLGenerator.NodeCount) gen.getNodeCount(); + count.setID(parent.getID()); + int num = count.execute(jdbc); + return (num == 0); + } catch (BadSqlGrammarException ex) { + handleInternalFail(ex); + } finally { + long dt = System.currentTimeMillis() - t; + log.debug("isEmpty: " + parent.getID() + " " + dt + "ms"); + } + throw new RuntimeException("BUG: handleInternalFail did not throw"); + } + public void delete(UUID id) { super.delete(Node.class, id); } From b3f51aa38c8ac0e82cbb51b3b8adbaab3b5daf0c Mon Sep 17 00:00:00 2001 From: Patrick Dowler Date: Thu, 7 Sep 2023 12:18:53 -0700 Subject: [PATCH 47/52] vault: reject deletion of non-empty containers, perform delete actions in txn --- .../opencadc/vault/NodePersistenceImpl.java | 86 ++++++++++++------- 1 file changed, 56 insertions(+), 30 deletions(-) diff --git a/vault/src/main/java/org/opencadc/vault/NodePersistenceImpl.java b/vault/src/main/java/org/opencadc/vault/NodePersistenceImpl.java index 5484d14dd..63bf5ec96 100644 --- a/vault/src/main/java/org/opencadc/vault/NodePersistenceImpl.java +++ b/vault/src/main/java/org/opencadc/vault/NodePersistenceImpl.java @@ -72,6 +72,7 @@ import ca.nrc.cadc.auth.PrincipalExtractor; import ca.nrc.cadc.auth.X509CertificateChain; import ca.nrc.cadc.date.DateUtil; +import ca.nrc.cadc.db.TransactionManager; import ca.nrc.cadc.io.ResourceIterator; import ca.nrc.cadc.net.TransientException; import ca.nrc.cadc.util.InvalidConfigException; @@ -480,41 +481,66 @@ public void delete(Node node) throws TransientException { final NodeDAO dao = getDAO(); final ArtifactDAO artifactDAO = getArtifactDAO(); + TransactionManager txn = dao.getTransactionManager(); - // TODO: do the following in a transaction, acquire lock on target node - - URI storageID = null; - if (node instanceof ContainerNode) { - ContainerNode cn = (ContainerNode) node; - try (ResourceIterator iter = dao.iterator(cn, 1, null)) { - if (iter.hasNext()) { - throw new IllegalArgumentException("container node '" + node.getName() + "' is not empty"); + try { + log.debug("starting transaction"); + txn.startTransaction(); + log.debug("start txn: OK"); + + Node locked = dao.lock(node); + if (locked != null) { + node = locked; // safer than having two vars and accidentally using the wrong one + URI storageID = null; + if (node instanceof ContainerNode) { + ContainerNode cn = (ContainerNode) node; + boolean empty = dao.isEmpty(cn); + if (!empty) { + log.debug("commit txn..."); + txn.commitTransaction(); + log.debug("commit txn: OK"); + throw new IllegalArgumentException("container node '" + node.getName() + "' is not empty"); + } + } else if (node instanceof DataNode) { + DataNode dn = (DataNode) node; + NodeProperty len = dn.getProperty(VOS.PROPERTY_URI_CONTENTLENGTH); + if (len != null) { + // artifact exists + storageID = dn.storageID; + } + } // else: LinkNode can always be deleted + + if (storageID != null) { + Artifact a = artifactDAO.get(storageID); + if (a != null) { + DeletedArtifactEventDAO daeDAO = new DeletedArtifactEventDAO(artifactDAO); + DeletedArtifactEvent dae = new DeletedArtifactEvent(a.getID()); + daeDAO.put(dae); + artifactDAO.delete(a.getID()); + } } - } catch (IOException ex) { - throw new TransientException("database IO failure", ex); + // TODO: need DeletedNodeDAO to create DeletedNodeEvent + dao.delete(node.getID()); + } else { + log.debug("failed to lock node " + node.getID() + " - assume deleted by another process"); } - } else if (node instanceof DataNode) { - DataNode dn = (DataNode) node; - NodeProperty len = dn.getProperty(VOS.PROPERTY_URI_CONTENTLENGTH); - if (len != null) { - // artifact exists - storageID = dn.storageID; + + log.debug("commit txn..."); + txn.commitTransaction(); + log.debug("commit txn: OK"); + } catch (Exception ex) { + if (txn.isOpen()) { + log.error("failed to delete " + node.getID() + " aka " + node.getName(), ex); + txn.rollbackTransaction(); + log.debug("rollback txn: OK"); } - } // else: LinkNode can always be deleted - - // TODO: need DeletedNodeDAO to create DeletedNodeEvent - - if (storageID != null) { - Artifact a = artifactDAO.get(storageID); - if (a != null) { - DeletedArtifactEventDAO daeDAO = new DeletedArtifactEventDAO(artifactDAO); - DeletedArtifactEvent dae = new DeletedArtifactEvent(a.getID()); - daeDAO.put(dae); - artifactDAO.delete(a.getID()); + throw ex; + } finally { + if (txn.isOpen()) { + log.error("BUG - open transaction in finally"); + txn.rollbackTransaction(); + log.error("rollback txn: OK"); } } - dao.delete(node.getID()); - - // TODO: commit transaction } } From 2911e7b2cc98cb4ca071c82523027088b466e5a0 Mon Sep 17 00:00:00 2001 From: Patrick Dowler Date: Fri, 8 Sep 2023 12:55:41 -0700 Subject: [PATCH 48/52] vault-quota: README and design doc --- vault-quota/Design.md | 66 +++++++++++++++++++++++++++++++++++++++++++ vault-quota/README.md | 59 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 125 insertions(+) create mode 100644 vault-quota/Design.md create mode 100644 vault-quota/README.md diff --git a/vault-quota/Design.md b/vault-quota/Design.md new file mode 100644 index 000000000..435360fce --- /dev/null +++ b/vault-quota/Design.md @@ -0,0 +1,66 @@ +# vault quota design/algorithms + +The definitive source of content-length (file size) of a DataNode coems from the +`inventory.Artifact` table and it not known until a PUT to storage is completed. +In the case of a `vault` service co-located with a single storage site (`minoc`), +the new Artifact is visible in the database as soon as the PUT to `minoc` is +completed. In the case of a `vault` service co-located with a global SI, the new +Artifact is visible in the database once it is synced from the site of the PUT to +`minoc` to the global database by `fenwick` (or worst case: `ratik`). + +## TODO +The design below only takes into account incremental propagation of space used +by stored files. It is not complete/verified until we also come up with a validation +algorithm that can detect and fix discrepancies in a live `vault`. + +## Event watcher algorithm: +``` +track progress using HarvestState (name: `Artifact`, source: `db:{bucket range}`) +incremental query for new artifacts in lastModified order +for each new Artifact: + query for DataNode (storageID = artifact.uri) + if Artifact.contentLength != Node.size: + start txn + lock datanode + compute delta + lock parent + apply delta to parent.delta + set dataNode.size + update HarvestState + commit txn +``` +The above sequence does the first step of propagation from DataNode to parent ContainerNode. +This can be done in parallel by using bucket ranges (smaller than 0-f). + +## Container size propagation algorithm: +``` +query for ContainerNode with non-zero delta +for each ContainerNode: + start txn + lock containernode + re-check delta + lock parent + apply delta to parent.delta + apply delta containernode.size, set containernode.delta=0 + commit txn +``` +The above sequence finds candidate propagations, locks (order: child-then-parent as above), +and applies the propagation. This moves the outstanding delta up the tree one level. If the +sequence acts on multiple child containers before the parent, the delta(s) naturally +_merge_ and there are fewer larger delta propagations in the upper part of the tree. It would +be optimal to do propagations depth-first but it doesn't seem practical to forcibly accomplish +that ordering. + +Container size propagation will be implemented as a single sequence (thread). We could add +something to the vospace.Node table to support subdividing work and enable multiple threads, +but there is nothing there right now. + +## database changes required +note: all field and column names TBD +* add `size` and `delta` fields to ContainerNode (transient) +* add `size` field to DataNode (transient) +* add `size` to the `vospace.Node` table +* add `delta` to the `vospace.Node` table +* incremental sync query/iterator (ArtifactDAO?) +* lookup DataNode by storageID (ArtifactDAO?) + diff --git a/vault-quota/README.md b/vault-quota/README.md new file mode 100644 index 000000000..c0f2db0eb --- /dev/null +++ b/vault-quota/README.md @@ -0,0 +1,59 @@ +# Storage Inventory VOSpace quota support process (vault-quota) + +Process to maintain container node sizes so that quota limits can be enforced by the +main `vault` service. This process runs in incremental mode (single process running +continuously) to update a local vospace database. + +`vault-quota` is an optional process that is only needed if `vault` is configured to +enforce quotas, although it could be used to maintain container node sizes without +quota enforcement. + +## configuration +See the [cadc-java](https://github.com/opencadc/docker-base/tree/master/cadc-java) image +docs for general config requirements. + +Runtime configuration must be made available via the `/config` directory. + +### vault-quota.properties +``` +org.opencadc.vault.quota.logging = {info|debug} + +# inventory database settings +org.opencadc.inventory.db.SQLGenerator=org.opencadc.inventory.db.SQLGenerator +org.opencadc.vault.quota.nodes.schema={schema for inventory database objects} +org.opencadc.vault.quota.nodes.username={username for inventory admin} +org.opencadc.vault.quota.nodes.password={password for inventory admin} +org.opencadc.vault.quota.nodes.url=jdbc:postgresql://{server}/{database} + +org.opencadc.vault.quota.threads={number of threads to watch for artifact events} + +# storage namespace +org.opencadc.vault.storage.namespace = {a storage inventory namespace to use} +``` +The _nodes_ account owns and manages (create, alter, drop) vospace database objects and updates +content in the vospace schema. The database is specified in the JDBC URL. Failure to connect or +initialize the database will show up in logs. + +The _threads_ key configures the number of threads that watch for new Artifact events and initiate +the propagation of sizes to parent containers. These threads each monitor a subset of artifacts using +`Artifact.uriBucket` filtering; for simplicity, the following values are allowed: 1, 2, 4, 8, 16. + +In addition to the above threads, there is one additional thread that propagates size changes up +the tree of container nodes to the container node(s) where quotas are specified. + +## building it +``` +gradle clean build +docker build -t vault-quota -f Dockerfile . +``` + +## checking it +``` +docker run -it vault-quota:latest /bin/bash +``` + +## running it +``` +docker run --user opencadc:opencadc -v /path/to/external/config:/config:ro --name vault-quota vault-quota:latest +``` + From e15cbd0f4205a863e32f95c37f86eeb3a83ab993 Mon Sep 17 00:00:00 2001 From: Patrick Dowler Date: Mon, 11 Sep 2023 09:37:23 -0700 Subject: [PATCH 49/52] Update Design.md --- vault-quota/Design.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/vault-quota/Design.md b/vault-quota/Design.md index 435360fce..af2e188eb 100644 --- a/vault-quota/Design.md +++ b/vault-quota/Design.md @@ -1,6 +1,6 @@ # vault quota design/algorithms -The definitive source of content-length (file size) of a DataNode coems from the +The definitive source of content-length (file size) of a DataNode comes from the `inventory.Artifact` table and it not known until a PUT to storage is completed. In the case of a `vault` service co-located with a single storage site (`minoc`), the new Artifact is visible in the database as soon as the PUT to `minoc` is @@ -63,4 +63,5 @@ note: all field and column names TBD * add `delta` to the `vospace.Node` table * incremental sync query/iterator (ArtifactDAO?) * lookup DataNode by storageID (ArtifactDAO?) +* indices to support new queries From 07d759f59d81e8e39184c60a34accc5e30b1f175 Mon Sep 17 00:00:00 2001 From: Patrick Dowler Date: Mon, 11 Sep 2023 12:13:30 -0700 Subject: [PATCH 50/52] Update Design.md --- vault-quota/Design.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vault-quota/Design.md b/vault-quota/Design.md index af2e188eb..f6d6b137d 100644 --- a/vault-quota/Design.md +++ b/vault-quota/Design.md @@ -15,7 +15,7 @@ algorithm that can detect and fix discrepancies in a live `vault`. ## Event watcher algorithm: ``` -track progress using HarvestState (name: `Artifact`, source: `db:{bucket range}`) +track progress using HarvestState (source: `db:{bucket range}`, name: TBD) incremental query for new artifacts in lastModified order for each new Artifact: query for DataNode (storageID = artifact.uri) From 53363891932112e3dadf7dbb8f2b4b95d8b23d58 Mon Sep 17 00:00:00 2001 From: Patrick Dowler Date: Tue, 12 Sep 2023 13:16:52 -0700 Subject: [PATCH 51/52] vault-quota: update design doc with some validate ideas --- vault-quota/Design.md | 50 +++++++++++++++++++++++++++++++++++++++---- 1 file changed, 46 insertions(+), 4 deletions(-) diff --git a/vault-quota/Design.md b/vault-quota/Design.md index f6d6b137d..b2b43b920 100644 --- a/vault-quota/Design.md +++ b/vault-quota/Design.md @@ -13,7 +13,9 @@ The design below only takes into account incremental propagation of space used by stored files. It is not complete/verified until we also come up with a validation algorithm that can detect and fix discrepancies in a live `vault`. -## Event watcher algorithm: +## DataNode size algorithm: +This is an event watcher that gets Artifact events (after a PUT) and intiates the +propagation of sizes (space used). ``` track progress using HarvestState (source: `db:{bucket range}`, name: TBD) incremental query for new artifacts in lastModified order @@ -29,10 +31,12 @@ for each new Artifact: update HarvestState commit txn ``` -The above sequence does the first step of propagation from DataNode to parent ContainerNode. -This can be done in parallel by using bucket ranges (smaller than 0-f). +Optimization: The above sequence does the first step of propagation from DataNode to +parent ContainerNode so the maximum work can be done in parallel using bucket ranges +(smaller than 0-f). It also means the propagation below only has to consider +ContainerNode.delta since DataNode(s) never have a delta. -## Container size propagation algorithm: +## ContainerNode size propagation algorithm: ``` query for ContainerNode with non-zero delta for each ContainerNode: @@ -55,12 +59,50 @@ Container size propagation will be implemented as a single sequence (thread). We something to the vospace.Node table to support subdividing work and enable multiple threads, but there is nothing there right now. +## validation + +### DataNode vs Artifact discrepancies +These can be validated in parallel by multiple threads, subdivide work by bucket. + +``` +discrepancy 1: Artifact exists but DataNode does not +explanation: DataNode created, transfer negotiated, DataNode removed, transfer executed +evidence: check for DeletedNodeEvent +action: remove artifact, create DeletedArtifactEvent +else: ?? + +discrepancy 2: DataNode exists but Artifact does not +explanation: DataNode created, Artifact never (successfully) put +evidence: dataNode.size == 0 +action: none + +discrepancy 3: DataNode exists but Artifact does not +explanation: deleted or lost Artifact +evidence: DataNode.size != 0 (deleted vs lost: DeletedArtifactEvent exists) +action: fix DataNode.size + +discrepancy 4: DataNode.size != Artifact.contentLength +explanation: pending/missed Artifact event +action: fix DataNode and propagate delta to parent ContainerNode (same as incremental) +``` + +This could be accomplished with a single query on on inventory.Artifact full outer join +vospace.Node to get all the pairs. The more generic approach would be to do a merge join +of two iterators: + +Iterator aiter = artifactDAO.iterator(vaultNamespace, bucket); +Iterator niter = nodeDAO.iterator(vaultNamespace, bucket); + +The more generic dual iterator approach could be made to work if the inventory and vospace +content are in different PG database or server - TBD. + ## database changes required note: all field and column names TBD * add `size` and `delta` fields to ContainerNode (transient) * add `size` field to DataNode (transient) * add `size` to the `vospace.Node` table * add `delta` to the `vospace.Node` table +* add `storageBucket` to `vospace.Node` table (validation) * incremental sync query/iterator (ArtifactDAO?) * lookup DataNode by storageID (ArtifactDAO?) * indices to support new queries From dec5ed8b358195a1576a65d69ea66116cb3870af Mon Sep 17 00:00:00 2001 From: Adrian Damian Date: Fri, 6 Oct 2023 16:06:05 -0700 Subject: [PATCH 52/52] Support for recursive deletes --- vault/build.gradle | 6 ++-- .../java/org/opencadc/vault/NodesTest.java | 2 ++ .../org/opencadc/vault/VaultInitAction.java | 17 +++++++++++ vault/src/main/webapp/META-INF/context.xml | 13 ++++++++ vault/src/main/webapp/WEB-INF/web.xml | 30 +++++++++++++++++++ vault/src/main/webapp/capabilities.xml | 10 +++++++ 6 files changed, 75 insertions(+), 3 deletions(-) diff --git a/vault/build.gradle b/vault/build.gradle index 86b736b0c..7fd069259 100644 --- a/vault/build.gradle +++ b/vault/build.gradle @@ -30,7 +30,7 @@ def git_url = 'https://github.com/opencadc/vos' dependencies { compile 'javax.servlet:javax.servlet-api:[3.1,4.0)' - compile 'org.opencadc:cadc-util:[1.9.5,2.0)' + compile 'org.opencadc:cadc-util:[1.9.10,2.0)' compile 'org.opencadc:cadc-log:[1.1.6,2.0)' compile 'org.opencadc:cadc-gms:[1.0.5,)' compile 'org.opencadc:cadc-rest:[1.3.16,)' @@ -38,10 +38,10 @@ dependencies { compile 'org.opencadc:cadc-vos-server-alt:[2.0,)' compile 'org.opencadc:cadc-vosi:[1.3.2,)' compile 'org.opencadc:cadc-uws:[1.0,)' - compile 'org.opencadc:cadc-uws-server:[1.2.12,)' + compile 'org.opencadc:cadc-uws-server:[1.2.19,)' compile 'org.opencadc:cadc-access-control:[1.1.1,2.0)' compile 'org.opencadc:cadc-cdp:[1.2.3,)' - compile 'org.opencadc:cadc-registry:[1.5.15,)' + compile 'org.opencadc:cadc-registry:[1.7.4,)' compile 'org.opencadc:cadc-inventory:[0.9.4,1.0)' compile 'org.opencadc:cadc-inventory-db:[0.15.0,1.0)' diff --git a/vault/src/intTest/java/org/opencadc/vault/NodesTest.java b/vault/src/intTest/java/org/opencadc/vault/NodesTest.java index 71bb1178b..2a4a317da 100644 --- a/vault/src/intTest/java/org/opencadc/vault/NodesTest.java +++ b/vault/src/intTest/java/org/opencadc/vault/NodesTest.java @@ -71,6 +71,7 @@ import java.net.URI; import org.apache.log4j.Level; import org.apache.log4j.Logger; +import org.opencadc.gms.GroupURI; /** * Test the nodes endpoint. @@ -86,6 +87,7 @@ public class NodesTest extends org.opencadc.conformance.vos.NodesTest { } public NodesTest() { + //super(URI.create("ivo://opencadc.org/vault"), "vault-test.pem", new GroupURI(URI.create("ivo://cadc.nrc.ca/gms?CADC_TEST_GROUP2")), "vault-test-auth.pem"); super(URI.create("ivo://opencadc.org/vault"), "vault-test.pem"); } } diff --git a/vault/src/main/java/org/opencadc/vault/VaultInitAction.java b/vault/src/main/java/org/opencadc/vault/VaultInitAction.java index bc9dc89f5..c22083be8 100644 --- a/vault/src/main/java/org/opencadc/vault/VaultInitAction.java +++ b/vault/src/main/java/org/opencadc/vault/VaultInitAction.java @@ -71,6 +71,7 @@ import ca.nrc.cadc.rest.InitAction; import ca.nrc.cadc.util.MultiValuedProperties; import ca.nrc.cadc.util.PropertiesReader; +import ca.nrc.cadc.uws.server.impl.InitDatabaseUWS; import java.net.URI; import java.net.URISyntaxException; import java.util.Map; @@ -93,6 +94,7 @@ public class VaultInitAction extends InitAction { private static final Logger log = Logger.getLogger(VaultInitAction.class); static final String JNDI_DATASOURCE = "jdbc/nodes"; // context.xml + static final String JNDI_UWS_DATASOURCE = "jdbc/uws"; // context.xml // config keys private static final String VAULT_KEY = "org.opencadc.vault"; @@ -119,6 +121,7 @@ public VaultInitAction() { public void doInit() { initConfig(); initDatabase(); + initUWSDatabase(); initNodePersistence(); } @@ -215,6 +218,20 @@ private void initDatabase() { } } + private void initUWSDatabase() { + log.info("initUWSDatabase: START"); + try { + // Init UWS database + DataSource uws = DBUtil.findJNDIDataSource(JNDI_UWS_DATASOURCE); + InitDatabaseUWS uwsi = new InitDatabaseUWS(uws, null, "uws"); + uwsi.doInit(); + log.info("initDatabase: " + JNDI_UWS_DATASOURCE + " uws OK"); + + } catch (Exception ex) { + throw new RuntimeException("check/init uws database failed", ex); + } + } + protected void initNodePersistence() { jndiNodePersistence = componentID + ".nodePersistence"; try { diff --git a/vault/src/main/webapp/META-INF/context.xml b/vault/src/main/webapp/META-INF/context.xml index cbc72f2e0..e5775723b 100644 --- a/vault/src/main/webapp/META-INF/context.xml +++ b/vault/src/main/webapp/META-INF/context.xml @@ -15,4 +15,17 @@ removeAbandoned="false" testOnBorrow="true" validationQuery="select 123" /> + + + diff --git a/vault/src/main/webapp/WEB-INF/web.xml b/vault/src/main/webapp/WEB-INF/web.xml index af0394c10..90476792e 100644 --- a/vault/src/main/webapp/WEB-INF/web.xml +++ b/vault/src/main/webapp/WEB-INF/web.xml @@ -23,6 +23,7 @@ ca.nrc.cadc.rest ca.nrc.cadc.util ca.nrc.cadc.vosi + ca.nrc.cadc.uws @@ -62,6 +63,30 @@ 2 + + + RecursiveDeleteNodeServlet + ca.nrc.cadc.uws.server.JobServlet + + get + ca.nrc.cadc.uws.web.GetAction + + + post + ca.nrc.cadc.uws.web.PostAction + + + delete + ca.nrc.cadc.uws.web.DeleteAction + + + ca.nrc.cadc.uws.server.JobManager + org.opencadc.vault.RecursiveDeleteNodeJobManager + + 3 + + + CapabilitiesServlet @@ -107,6 +132,11 @@ /nodes/* + + RecursiveDeleteNodeServlet + /recursiveDelete/* + + AvailabilityServlet diff --git a/vault/src/main/webapp/capabilities.xml b/vault/src/main/webapp/capabilities.xml index fb145630e..d24f1a220 100644 --- a/vault/src/main/webapp/capabilities.xml +++ b/vault/src/main/webapp/capabilities.xml @@ -41,6 +41,16 @@ + + + https://replace.me.com/vault/recursiveDelete + + + + + + +