From 6dd4bb16435fa4a7f9c0aa4a4f6244076c496263 Mon Sep 17 00:00:00 2001 From: Patrick Dowler Date: Mon, 18 Mar 2024 14:57:15 -0700 Subject: [PATCH 01/18] vault: integrate ArtifactSyncWorker --- .../vospace/db/ArtifactSyncWorker.java | 45 +++++-- .../org/opencadc/vault/VaultInitAction.java | 26 ++-- .../opencadc/vault/metadata/ArtifactSync.java | 121 ++++++++++++------ vault/src/main/webapp/META-INF/context.xml | 2 +- 4 files changed, 136 insertions(+), 58 deletions(-) diff --git a/cadc-inventory-db/src/main/java/org/opencadc/vospace/db/ArtifactSyncWorker.java b/cadc-inventory-db/src/main/java/org/opencadc/vospace/db/ArtifactSyncWorker.java index 29e80384..33bf2094 100644 --- a/cadc-inventory-db/src/main/java/org/opencadc/vospace/db/ArtifactSyncWorker.java +++ b/cadc-inventory-db/src/main/java/org/opencadc/vospace/db/ArtifactSyncWorker.java @@ -67,9 +67,11 @@ package org.opencadc.vospace.db; +import ca.nrc.cadc.date.DateUtil; import ca.nrc.cadc.db.TransactionManager; import ca.nrc.cadc.io.ResourceIterator; import java.io.IOException; +import java.text.DateFormat; import org.apache.log4j.Logger; import org.opencadc.inventory.Artifact; import org.opencadc.inventory.Namespace; @@ -77,10 +79,10 @@ import org.opencadc.inventory.db.HarvestState; import org.opencadc.inventory.db.HarvestStateDAO; import org.opencadc.vospace.DataNode; -import org.opencadc.vospace.db.NodeDAO; /** - * This class performs the work of synchronizing the size of Data Nodes between backend storage and Node Persistence + * This class performs the work of synchronizing the size of Data Nodes from + * inventory (Artifact) to vopsace (Node). * * @author adriand */ @@ -93,8 +95,16 @@ public class ArtifactSyncWorker implements Runnable { private final HarvestStateDAO harvestStateDAO; private final Namespace storageNamespace; - public ArtifactSyncWorker(HarvestStateDAO harvestStateDAO, HarvestState harvestState, ArtifactDAO artifactDAO, - Namespace namespace) { + /** + * Worker constructor. + * + * @param harvestStateDAO DAO class to persist progress in the vospace database + * @param harvestState current HarvestState instance + * @param artifactDAO DAO class to query for artifacts + * @param namespace artifact namespace + */ + public ArtifactSyncWorker(HarvestStateDAO harvestStateDAO, HarvestState harvestState, + ArtifactDAO artifactDAO, Namespace namespace) { this.harvestState = harvestState; this.harvestStateDAO = harvestStateDAO; this.nodeDAO = new NodeDAO(harvestStateDAO); @@ -104,11 +114,17 @@ public ArtifactSyncWorker(HarvestStateDAO harvestStateDAO, HarvestState harvestS @Override public void run() { - log.debug("Start harvesting " + harvestState.toString() + " at " + harvestState.curLastModified); + DateFormat df = DateUtil.getDateFormat(DateUtil.IVOA_DATE_FORMAT, DateUtil.UTC); + if (harvestState.curLastModified != null) { + log.info("ArtifactSyncWorker.artifactQuery source=" + harvestState.getResourceID() + + " instance=" + harvestState.instanceID + " start=" + df.format(harvestState.curLastModified)); + } else { + log.info("ArtifactSyncWorker.artifactQuery source=" + harvestState.getResourceID() + " instance=" + harvestState.instanceID); + } - TransactionManager tm = nodeDAO.getTransactionManager(); - try (final ResourceIterator iter = artifactDAO.iterator(storageNamespace, null, - harvestState.curLastModified, true)) { + String uriBucket = null; // process all artifacts in a single thread + try (final ResourceIterator iter = artifactDAO.iterator(storageNamespace, uriBucket, harvestState.curLastModified, true)) { + TransactionManager tm = nodeDAO.getTransactionManager(); while (iter.hasNext()) { Artifact artifact = iter.next(); DataNode node = nodeDAO.getDataNode(artifact.getURI()); @@ -122,7 +138,8 @@ public void run() { node.bytesUsed = artifact.getContentLength(); nodeDAO.put(node); tm.commitTransaction(); - log.debug("Updated size of data node " + node.getName()); + log.debug("ArtifactSyncWorker.updateDataNode id=" + node.getID() + + " bytesUsed=" + node.bytesUsed + " artifact.lastModified=" + df.format(artifact.getLastModified())); } catch (Exception ex) { log.debug("Failed to update data node size for " + node.getName(), ex); tm.rollbackTransaction(); @@ -137,13 +154,19 @@ public void run() { } } harvestState.curLastModified = artifact.getLastModified(); - harvestState.curID = node.getID(); + harvestState.curID = artifact.getID(); harvestStateDAO.put(harvestState); } } catch (IOException ex) { log.error("Error closing iterator", ex); throw new RuntimeException("error while closing ResourceIterator", ex); } - log.debug("End harvesting " + harvestState.toString() + " at " + harvestState.curLastModified); + if (harvestState.curLastModified != null) { + log.info("ArtifactSyncWorker.artifactQuery source=" + harvestState.getResourceID() + + " instance=" + harvestState.instanceID + " end=" + df.format(harvestState.curLastModified)); + } else { + log.info("ArtifactSyncWorker.artifactQuery source=" + harvestState.getResourceID() + + " instance=" + harvestState.instanceID + " end=true"); + } } } diff --git a/vault/src/main/java/org/opencadc/vault/VaultInitAction.java b/vault/src/main/java/org/opencadc/vault/VaultInitAction.java index e4e51b41..12e1f2e0 100644 --- a/vault/src/main/java/org/opencadc/vault/VaultInitAction.java +++ b/vault/src/main/java/org/opencadc/vault/VaultInitAction.java @@ -88,6 +88,7 @@ import org.apache.log4j.Logger; import org.opencadc.inventory.Namespace; import org.opencadc.inventory.PreauthKeyPair; +import org.opencadc.inventory.db.ArtifactDAO; import org.opencadc.inventory.db.HarvestStateDAO; import org.opencadc.inventory.db.PreauthKeyPairDAO; import org.opencadc.inventory.db.SQLGenerator; @@ -111,6 +112,7 @@ public class VaultInitAction extends InitAction { static final String JNDI_VOS_DATASOURCE = "jdbc/nodes"; // context.xml static final String JNDI_INV_DATASOURCE = "jdbc/inventory"; // context.xml + static final String JNDI_INV_ITER_DATASOURCE = "jdbc/inventory-iterator"; // context.xml static final String JNDI_UWS_DATASOURCE = "jdbc/uws"; // context.xml // config keys @@ -290,16 +292,17 @@ static Map getInvConfig(MultiValuedProperties props) { return ret; } - static Map getKeyPairConfig(MultiValuedProperties props) { - return getDaoConfig(props); - /* + static Map getIteratorConfig(MultiValuedProperties props) { Map ret = new TreeMap<>(); ret.put(SQLGenerator.class.getName(), SQLGenerator.class); // not configurable right now - ret.put("jndiDataSourceName", JNDI_VOS_DATASOURCE); - ret.put("invSchema", props.getFirstPropertyValue(INVENTORY_SCHEMA_KEY)); // requied but unused - ret.put("genSchema", props.getFirstPropertyValue(VOSPACE_SCHEMA_KEY)); + ret.put("jndiDataSourceName", JNDI_INV_ITER_DATASOURCE); + ret.put("invSchema", props.getFirstPropertyValue(INVENTORY_SCHEMA_KEY)); + ret.put("genSchema", props.getFirstPropertyValue(INVENTORY_SCHEMA_KEY)); // for complete init return ret; - */ + } + + static Map getKeyPairConfig(MultiValuedProperties props) { + return getDaoConfig(props); } private void initConfig() { @@ -453,10 +456,15 @@ private void terminateAvailabilityCheck() { private void initBackgroundWorkers() { HarvestStateDAO hsDAO = new HarvestStateDAO(); - hsDAO.setConfig(getDaoConfig(props)); + hsDAO.setConfig(vosDaoConfig); + + ArtifactDAO artifactDAO = new ArtifactDAO(); + Map iterprops = getIteratorConfig(props); + log.warn("iterator pool: " + iterprops.get("jndiDataSourceName")); + artifactDAO.setConfig(iterprops); terminateBackgroundWorkers(); - this.artifactSync = new Thread(new ArtifactSync(hsDAO)); + this.artifactSync = new Thread(new ArtifactSync(hsDAO, artifactDAO, storageNamespace)); artifactSync.setDaemon(true); artifactSync.start(); } diff --git a/vault/src/main/java/org/opencadc/vault/metadata/ArtifactSync.java b/vault/src/main/java/org/opencadc/vault/metadata/ArtifactSync.java index ffd1879e..c0f7d38f 100644 --- a/vault/src/main/java/org/opencadc/vault/metadata/ArtifactSync.java +++ b/vault/src/main/java/org/opencadc/vault/metadata/ArtifactSync.java @@ -72,8 +72,11 @@ import java.util.UUID; import org.apache.log4j.Logger; import org.opencadc.inventory.Artifact; +import org.opencadc.inventory.Namespace; +import org.opencadc.inventory.db.ArtifactDAO; import org.opencadc.inventory.db.HarvestState; import org.opencadc.inventory.db.HarvestStateDAO; +import org.opencadc.vospace.db.ArtifactSyncWorker; /** * Main artifact-sync agent that enables incremental sync of Artifact @@ -84,67 +87,91 @@ public class ArtifactSync implements Runnable { private static final Logger log = Logger.getLogger(ArtifactSync.class); - private static final long SHORT_SLEEP = 12000L; - private static final long LONG_SLEEP = 2 * SHORT_SLEEP; - private static final long EVICT_AGE = 3 * LONG_SLEEP; + private static final long ROUNDS = 6000L; // 6 sec + private static final long SHORT_SLEEP = 2 * ROUNDS; + private static final long LONG_SLEEP = 5 * ROUNDS; + private static final long EVICT_AGE = 12 * ROUNDS; + + private static final long FAIL_SLEEP = 10 * ROUNDS; // slightly smaller than evict private final UUID instanceID = UUID.randomUUID(); - private final HarvestStateDAO dao; - private String name = Artifact.class.getSimpleName(); - private URI resourceID = URI.create("jdbc/inventory"); + private final HarvestStateDAO stateDAO; + private final ArtifactDAO artifactDAO; + private final Namespace artifactNamespace; - public ArtifactSync(HarvestStateDAO dao) { - this.dao = dao; + public ArtifactSync(HarvestStateDAO stateDAO, ArtifactDAO artifactDAO, Namespace artifactNamespace) { + this.stateDAO = stateDAO; + this.artifactDAO = artifactDAO; + this.artifactNamespace = artifactNamespace; // fenwick setup for production workload: //dao.setUpdateBufferCount(99); // buffer 99 updates, do every 100 //dao.setMaintCount(999); // buffer 999 so every 1000 real updates aka every 1e5 events - // here, we need timestamp updates to retain leader status, so - // dao.setMaintCount(9999); // every 1e4 + // we need continuous timestamp updates to retain leader status, so only schedule maintenance + stateDAO.setUpdateBufferCount(0); + stateDAO.setMaintCount(9999); // every 1e4 } @Override public void run() { + String name = Artifact.class.getSimpleName(); + URI resourceID = URI.create("jdbc:inventory"); try { Thread.sleep(SHORT_SLEEP); while (true) { - boolean leader = false; log.debug("check leader " + instanceID); - HarvestState state = dao.get(name, resourceID); + HarvestState state = stateDAO.get(name, resourceID); log.debug("check leader " + instanceID + " found: " + state); if (state.instanceID == null) { state.instanceID = instanceID; - dao.put(state); - state = dao.get(state.getID()); + stateDAO.put(state); + state = stateDAO.get(state.getID()); log.debug("created: " + state); } - if (instanceID.equals(state.instanceID)) { - log.debug("still the leader..."); - dao.put(state, true); - leader = true; - } else { - // see if we should perform a coup... - Date now = new Date(); - long age = now.getTime() - state.getLastModified().getTime(); - if (age > EVICT_AGE) { - - state.instanceID = instanceID; - dao.put(state); - state = dao.get(state.getID()); - leader = true; - log.debug("EVICTED " + state.instanceID + " because age " + age + " > " + EVICT_AGE); - } - } + + // determine leader + boolean leader = checkLeaderStatus(state); if (leader) { - log.debug("leader " + state.instanceID + " starting worker..."); - // TODO - dao.flushBufferedState(); - Thread.sleep(SHORT_SLEEP / 2L); // for testing - log.debug("idle leader " + state.instanceID + " sleep=" + SHORT_SLEEP); - Thread.sleep(SHORT_SLEEP); + log.info("LEADER " + state.instanceID); + boolean fail = false; + try { + ArtifactSyncWorker worker = new ArtifactSyncWorker(stateDAO, state, artifactDAO, artifactNamespace); + worker.run(); + } catch (Exception ex) { + log.error("unexpected worker fail", ex); + fail = true; + } + + leader = checkLeaderStatus(state); + if (!fail && leader) { + try { + stateDAO.flushBufferedState(); + } catch (Exception ex) { + log.error("unexpected HarvestState flush fail", ex); + fail = true; + } + } + + if (!fail && leader) { + try { + // update leader timestamp before sleeping + stateDAO.put(state, true); + } catch (Exception ex) { + log.error("unexpected HarvestState force update fail", ex); + fail = true; + } + } + + if (fail) { + log.debug("failed leader " + state.instanceID + " sleep=" + FAIL_SLEEP); + Thread.sleep(FAIL_SLEEP); + } else { + log.debug("idle leader " + state.instanceID + " sleep=" + SHORT_SLEEP); + Thread.sleep(SHORT_SLEEP); + } } else { log.debug("not leader: sleep=" + LONG_SLEEP); Thread.sleep(LONG_SLEEP); @@ -154,4 +181,24 @@ public void run() { log.debug("interrupted - assuming shutdown", ex); } } + + private boolean checkLeaderStatus(HarvestState state) { + boolean leader = false; + if (instanceID.equals(state.instanceID)) { + stateDAO.put(state, true); + leader = true; + } else { + // see if we should perform a coup... + Date now = new Date(); + long age = now.getTime() - state.getLastModified().getTime(); + if (age > EVICT_AGE) { + log.info("EVICTING " + state.instanceID + " because age " + age + " > " + EVICT_AGE); + state.instanceID = instanceID; + stateDAO.put(state); + state = stateDAO.get(state.getID()); + leader = true; + } + } + return leader; + } } diff --git a/vault/src/main/webapp/META-INF/context.xml b/vault/src/main/webapp/META-INF/context.xml index 94ec7b42..0e787176 100644 --- a/vault/src/main/webapp/META-INF/context.xml +++ b/vault/src/main/webapp/META-INF/context.xml @@ -28,7 +28,7 @@ testOnBorrow="true" validationQuery="select 123" /> - Date: Mon, 18 Mar 2024 15:43:04 -0700 Subject: [PATCH 02/18] vault: update DataNode.bytesUsed from artifact in get use transaction and lock node before update in move --- .../opencadc/vault/NodePersistenceImpl.java | 119 +++++++++++++----- 1 file changed, 86 insertions(+), 33 deletions(-) diff --git a/vault/src/main/java/org/opencadc/vault/NodePersistenceImpl.java b/vault/src/main/java/org/opencadc/vault/NodePersistenceImpl.java index e3e98cee..b22ae52b 100644 --- a/vault/src/main/java/org/opencadc/vault/NodePersistenceImpl.java +++ b/vault/src/main/java/org/opencadc/vault/NodePersistenceImpl.java @@ -358,10 +358,6 @@ public Node get(ContainerNode parent, String name) throws TransientException { if (ret == null) { return null; } - ret.parent = parent; - IdentityManager identityManager = AuthenticationUtil.getIdentityManager(); - ret.owner = identityManager.toSubject(ret.ownerID); - ret.ownerDisplay = identityManager.toDisplayString(ret.owner); // in principle we could have queried vospace.Node join inventory.Artifact above // and avoid this query.... simplicity for now @@ -371,6 +367,46 @@ public Node get(ContainerNode parent, String name) throws TransientException { Artifact a = artifactDAO.get(dn.storageID); DateFormat df = NodeWriter.getDateFormat(); if (a != null) { + // DataNode.bytesUsed is an optimization (cache): + // if DataNode.bytesUsed != Artifact.contentLength we update the cache + // this retains put+get consistency in a single-site deployed (with minoc) + // and may help hide some inconsistencies in child listing sizes + if (!a.getContentLength().equals(dn.bytesUsed)) { + TransactionManager txn = dao.getTransactionManager(); + try { + log.debug("starting node transaction"); + txn.startTransaction(); + log.debug("start txn: OK"); + + DataNode locked = (DataNode) dao.lock(dn); + if (locked != null) { + dn = locked; // safer than accidentally using the wrong variable + dn.bytesUsed = a.getContentLength(); + dao.put(dn); + ret = dn; + } + + log.debug("commit txn..."); + txn.commitTransaction(); + log.debug("commit txn: OK"); + if (locked == null) { + return null; // gone + } + } catch (Exception ex) { + if (txn.isOpen()) { + log.error("failed to update bytesUsed on " + dn.getID() + " aka " + dn.getName(), ex); + txn.rollbackTransaction(); + log.debug("rollback txn: OK"); + } + } finally { + if (txn.isOpen()) { + log.error("BUG - open transaction in finally"); + txn.rollbackTransaction(); + log.error("rollback txn: OK"); + } + } + } + Date d = ret.getLastModified(); Date cd = null; if (ret.getLastModified().before(a.getLastModified())) { @@ -396,19 +432,17 @@ public Node get(ContainerNode parent, String name) throws TransientException { if (a.contentType != null) { ret.getProperties().add(new NodeProperty(VOS.PROPERTY_URI_TYPE, a.contentType)); } - // TODO: a.getContentLength() is correct, but might differ from dn.bytesUsed due to eventual consistency - // child listing iterator can only report dn.bytesUsed - // correct or consistency with listing?? - - // currently needed for consistency-requiring FilesTest - log.warn("DataNode.bytesUsed: " + dn.bytesUsed + " -> " + a.getContentLength()); - dn.bytesUsed = a.getContentLength(); } if (dn.bytesUsed == null) { - log.warn("DataNode.bytesUsed: 0 (no artifact)"); dn.bytesUsed = 0L; // no data stored } } + + ret.parent = parent; + IdentityManager identityManager = AuthenticationUtil.getIdentityManager(); + ret.owner = identityManager.toSubject(ret.ownerID); + ret.ownerDisplay = identityManager.toDisplayString(ret.owner); + return ret; } @@ -645,7 +679,7 @@ public void move(Node node, ContainerNode dest, String newName) { if (node.parent == null || dest.parent == null) { throw new IllegalArgumentException("args must both be peristent nodes before move"); } - // try to detect attempt to dsconnect from path to root: node is a parent of dest + // try to detect attempt to disconnect from path to root: node is a parent of dest ContainerNode cur = dest; while (!cur.getID().equals(root.getID())) { cur = cur.parent; @@ -654,28 +688,48 @@ public void move(Node node, ContainerNode dest, String newName) { } } - - Subject caller = AuthenticationUtil.getCurrentSubject(); - node.owner = caller; - node.ownerID = null; - node.ownerDisplay = null; - node.parent = dest; - node.parentID = null; - if (newName != null) { - node.setName(newName); - } + + NodeDAO dao = getDAO(); + TransactionManager txn = dao.getTransactionManager(); try { - Node result = put(node); - log.debug("moved: " + result); - } catch (NodeNotSupportedException ex) { - LocalServiceURI loc = new LocalServiceURI(getResourceID()); - VOSURI srcURI = loc.getURI(node); - throw new RuntimeException("BUG: failed to move node because of detected type change: " - + srcURI.getPath() + " type=" + node.getClass().getSimpleName(), ex); + log.debug("starting node transaction"); + txn.startTransaction(); + log.debug("start txn: OK"); + + // lock the source node + Node locked = dao.lock(node); + if (locked != null) { + node = locked; // safer than having two vars and accidentally using the wrong one + Subject caller = AuthenticationUtil.getCurrentSubject(); + node.owner = caller; + node.ownerID = null; + node.ownerDisplay = null; + node.parent = dest; + node.parentID = null; + if (newName != null) { + node.setName(newName); + } + Node result = put(node); + log.debug("moved: " + result); + } + log.debug("commit txn..."); + txn.commitTransaction(); + log.debug("commit txn: OK"); + } catch (Exception ex) { + if (txn.isOpen()) { + log.error("failed to move " + node.getID() + " aka " + node.getName(), ex); + txn.rollbackTransaction(); + log.debug("rollback txn: OK"); + } + } finally { + if (txn.isOpen()) { + log.error("BUG - open transaction in finally"); + txn.rollbackTransaction(); + log.error("rollback txn: OK"); + } } } - /** * Delete the specified node. * @@ -721,8 +775,7 @@ public void delete(Node node) throws TransientException { } } else if (node instanceof DataNode) { DataNode dn = (DataNode) node; - NodeProperty len = dn.getProperty(VOS.PROPERTY_URI_CONTENTLENGTH); - if (len != null) { + if (dn.bytesUsed != null) { // artifact exists storageID = dn.storageID; } From e284c6ed25aa3b320ae06b1df87690288a8563b1 Mon Sep 17 00:00:00 2001 From: Patrick Dowler Date: Tue, 19 Mar 2024 12:10:53 -0700 Subject: [PATCH 03/18] vault: enable availability modes to disable and enable background ArtifactSync thread --- .../vospace/db/ArtifactSyncWorker.java | 12 ++-- .../opencadc/vault/NodePersistenceImpl.java | 10 ++- .../opencadc/vault/ServiceAvailability.java | 45 ++++++++++++- .../org/opencadc/vault/VaultInitAction.java | 64 +++++++++++++------ .../vault/VaultTransferGenerator.java | 5 +- .../org/opencadc/vault/files/GetAction.java | 2 - .../opencadc/vault/metadata/ArtifactSync.java | 20 ++++-- vault/src/main/webapp/META-INF/context.xml | 5 +- vault/src/main/webapp/WEB-INF/web.xml | 4 -- 9 files changed, 125 insertions(+), 42 deletions(-) diff --git a/cadc-inventory-db/src/main/java/org/opencadc/vospace/db/ArtifactSyncWorker.java b/cadc-inventory-db/src/main/java/org/opencadc/vospace/db/ArtifactSyncWorker.java index 33bf2094..96709c05 100644 --- a/cadc-inventory-db/src/main/java/org/opencadc/vospace/db/ArtifactSyncWorker.java +++ b/cadc-inventory-db/src/main/java/org/opencadc/vospace/db/ArtifactSyncWorker.java @@ -117,9 +117,11 @@ public void run() { DateFormat df = DateUtil.getDateFormat(DateUtil.IVOA_DATE_FORMAT, DateUtil.UTC); if (harvestState.curLastModified != null) { log.info("ArtifactSyncWorker.artifactQuery source=" + harvestState.getResourceID() - + " instance=" + harvestState.instanceID + " start=" + df.format(harvestState.curLastModified)); + + " instance=" + harvestState.instanceID + + " start=" + df.format(harvestState.curLastModified)); } else { - log.info("ArtifactSyncWorker.artifactQuery source=" + harvestState.getResourceID() + " instance=" + harvestState.instanceID); + log.info("ArtifactSyncWorker.artifactQuery source=" + harvestState.getResourceID() + + " instance=" + harvestState.instanceID); } String uriBucket = null; // process all artifacts in a single thread @@ -163,10 +165,12 @@ public void run() { } if (harvestState.curLastModified != null) { log.info("ArtifactSyncWorker.artifactQuery source=" + harvestState.getResourceID() - + " instance=" + harvestState.instanceID + " end=" + df.format(harvestState.curLastModified)); + + " instance=" + harvestState.instanceID + + " end=" + df.format(harvestState.curLastModified)); } else { log.info("ArtifactSyncWorker.artifactQuery source=" + harvestState.getResourceID() - + " instance=" + harvestState.instanceID + " end=true"); + + " instance=" + harvestState.instanceID + + " end=true"); } } } diff --git a/vault/src/main/java/org/opencadc/vault/NodePersistenceImpl.java b/vault/src/main/java/org/opencadc/vault/NodePersistenceImpl.java index b22ae52b..bf2fbb38 100644 --- a/vault/src/main/java/org/opencadc/vault/NodePersistenceImpl.java +++ b/vault/src/main/java/org/opencadc/vault/NodePersistenceImpl.java @@ -166,18 +166,22 @@ public class NodePersistenceImpl implements NodePersistence { private final Namespace storageNamespace; private final boolean localGroupsOnly; - private URI resourceID; + private final URI resourceID; private final boolean preventNotFound; + final String appName; // access by VaultTransferGenerator + // possibly temporary hack so migration tool can set this to false and // preserve lastModified timestamps on nodes public boolean nodeOrigin = true; - public NodePersistenceImpl(URI resourceID) { + public NodePersistenceImpl(URI resourceID, String appName) { if (resourceID == null) { throw new IllegalArgumentException("resource ID required"); } this.resourceID = resourceID; + this.appName = appName; + MultiValuedProperties config = VaultInitAction.getConfig(); this.nodeDaoConfig = VaultInitAction.getDaoConfig(config); this.invDaoConfig = VaultInitAction.getInvConfig(config); @@ -259,7 +263,7 @@ public TransferGenerator getTransferGenerator() { keyDAO.setConfig(kpDaoConfig); PreauthKeyPair kp = keyDAO.get(VaultInitAction.KEY_PAIR_NAME); TokenTool tt = new TokenTool(kp.getPublicKey(), kp.getPrivateKey()); - return new VaultTransferGenerator(this, getArtifactDAO(), tt, preventNotFound); + return new VaultTransferGenerator(this, appName, getArtifactDAO(), tt, preventNotFound); } private NodeDAO getDAO() { diff --git a/vault/src/main/java/org/opencadc/vault/ServiceAvailability.java b/vault/src/main/java/org/opencadc/vault/ServiceAvailability.java index 0b0005a4..e750e2be 100644 --- a/vault/src/main/java/org/opencadc/vault/ServiceAvailability.java +++ b/vault/src/main/java/org/opencadc/vault/ServiceAvailability.java @@ -67,11 +67,17 @@ package org.opencadc.vault; +import ca.nrc.cadc.db.DBUtil; import ca.nrc.cadc.rest.RestAction; import ca.nrc.cadc.vosi.Availability; import ca.nrc.cadc.vosi.AvailabilityPlugin; +import ca.nrc.cadc.vosi.avail.CheckDataSource; +import javax.naming.InitialContext; +import javax.naming.NamingException; +import javax.sql.DataSource; import org.apache.log4j.Logger; +import org.opencadc.vault.metadata.ArtifactSync; /** * This class performs the work of determining if the executing artifact @@ -130,7 +136,31 @@ public Availability getStatus() { return new Availability(false, RestAction.STATE_READ_ONLY_MSG); } - //TODO add availability checks for dependent services + // check database pools + DataSource ds; + String testSQL; + CheckDataSource cds; + + ds = DBUtil.findJNDIDataSource("jdbc/nodes"); + testSQL = "select * from vospace.ModelVersion"; + cds = new CheckDataSource(ds, testSQL); + cds.check(); + + ds = DBUtil.findJNDIDataSource("jdbc/inventory"); + testSQL = "select * from inventory.Artifact limit 1"; + cds = new CheckDataSource(ds, testSQL); + cds.check(); + + ds = DBUtil.findJNDIDataSource("jdbc/inventory-iterator"); + testSQL = "select * from inventory.Artifact limit 1"; + cds = new CheckDataSource(ds, testSQL); + cds.check(); + + ds = DBUtil.findJNDIDataSource("jdbc/uws"); + testSQL = "select * from uws.Job limit 1"; + cds = new CheckDataSource(ds, testSQL); + cds.check(); + } catch (Throwable t) { // the test itself failed log.debug("failure", t); @@ -149,10 +179,13 @@ public void setState(String state) { String key = appName + RestAction.STATE_MODE_KEY; if (RestAction.STATE_OFFLINE.equalsIgnoreCase(state)) { System.setProperty(key, RestAction.STATE_OFFLINE); + setOffline(true); } else if (RestAction.STATE_READ_ONLY.equalsIgnoreCase(state)) { System.setProperty(key, RestAction.STATE_READ_ONLY); + setOffline(true); } else if (RestAction.STATE_READ_WRITE.equalsIgnoreCase(state)) { System.setProperty(key, RestAction.STATE_READ_WRITE); + setOffline(false); } else { throw new IllegalArgumentException("invalid state: " + state + " expected: " + RestAction.STATE_READ_WRITE + "|" @@ -170,4 +203,14 @@ private String getState() { return ret; } + private void setOffline(boolean offline) { + String jndiArtifactSync = appName + "-" + ArtifactSync.class.getName(); + try { + InitialContext initialContext = new InitialContext(); + ArtifactSync async = (ArtifactSync) initialContext.lookup(jndiArtifactSync); + async.setOffline(offline); + } catch (NamingException e) { + log.debug(String.format("unable to unbind %s - %s", jndiArtifactSync, e.getMessage())); + } + } } diff --git a/vault/src/main/java/org/opencadc/vault/VaultInitAction.java b/vault/src/main/java/org/opencadc/vault/VaultInitAction.java index 12e1f2e0..f10298f7 100644 --- a/vault/src/main/java/org/opencadc/vault/VaultInitAction.java +++ b/vault/src/main/java/org/opencadc/vault/VaultInitAction.java @@ -3,7 +3,7 @@ ******************* CANADIAN ASTRONOMY DATA CENTRE ******************* ************** CENTRE CANADIEN DE DONNÉES ASTRONOMIQUES ************** * -* (c) 2023. (c) 2023. +* (c) 2024. (c) 2024. * Government of Canada Gouvernement du Canada * National Research Council Conseil national de recherches * Ottawa, Canada, K1A 0R6 Ottawa, Canada, K1A 0R6 @@ -134,10 +134,11 @@ public class VaultInitAction extends InitAction { private List allocationParents = new ArrayList<>(); private String jndiNodePersistence; - private String jndiPreauthKeys; + private String jndiPreauthKeys; // store pubkey in JNDI for download via GetKeyAction + private String jndiArtifactSync; // store in JNDI to support availability mode change private String jndiSiteAvailabilities; private Thread availabilityCheck; - private Thread artifactSync; + private Thread artifactSyncThread; public VaultInitAction() { super(); @@ -369,7 +370,7 @@ private void initNodePersistence() { } catch (NamingException ignore) { log.debug("unbind previous JNDI key (" + jndiNodePersistence + ") failed... ignoring"); } - NodePersistence npi = new NodePersistenceImpl(resourceID); + NodePersistence npi = new NodePersistenceImpl(resourceID, appName); ctx.bind(jndiNodePersistence, npi); log.info("initNodePersistence: created JNDI key: " + jndiNodePersistence + " impl: " + npi.getClass().getName()); @@ -455,31 +456,54 @@ private void terminateAvailabilityCheck() { } private void initBackgroundWorkers() { - HarvestStateDAO hsDAO = new HarvestStateDAO(); - hsDAO.setConfig(vosDaoConfig); - - ArtifactDAO artifactDAO = new ArtifactDAO(); - Map iterprops = getIteratorConfig(props); - log.warn("iterator pool: " + iterprops.get("jndiDataSourceName")); - artifactDAO.setConfig(iterprops); - - terminateBackgroundWorkers(); - this.artifactSync = new Thread(new ArtifactSync(hsDAO, artifactDAO, storageNamespace)); - artifactSync.setDaemon(true); - artifactSync.start(); + try { + HarvestStateDAO hsDAO = new HarvestStateDAO(); + hsDAO.setConfig(vosDaoConfig); + + ArtifactDAO artifactDAO = new ArtifactDAO(); + Map iterprops = getIteratorConfig(props); + log.warn("iterator pool: " + iterprops.get("jndiDataSourceName")); + artifactDAO.setConfig(iterprops); + + terminateBackgroundWorkers(); + ArtifactSync async = new ArtifactSync(hsDAO, artifactDAO, storageNamespace); + this.artifactSyncThread = new Thread(async); + artifactSyncThread.setDaemon(true); + artifactSyncThread.start(); + + // store in JNDI so availability can set offline + String jndiArtifactSync = appName + "-" + ArtifactSync.class.getName(); + InitialContext ctx = new InitialContext(); + try { + ctx.unbind(jndiArtifactSync); + } catch (NamingException ignore) { + log.debug("unbind previous JNDI key (" + jndiPreauthKeys + ") failed... ignoring"); + } + ctx.bind(jndiArtifactSync, async); + log.info("initBackgroundWorkers: created JNDI key: " + jndiArtifactSync); + } catch (Exception ex) { + throw new RuntimeException("check/init ArtifactSync failed", ex); + } } private void terminateBackgroundWorkers() { - if (this.artifactSync != null) { + if (this.artifactSyncThread != null) { try { log.info("terminating ArtifactSync Thread..."); - this.artifactSync.interrupt(); - this.artifactSync.join(); + this.artifactSyncThread.interrupt(); + this.artifactSyncThread.join(); log.info("terminating ArtifactSync Thread... [OK]"); } catch (Throwable t) { log.info("failed to terminate ArtifactSync thread", t); } finally { - this.artifactSync = null; + this.artifactSyncThread = null; + } + + try { + InitialContext initialContext = new InitialContext(); + initialContext.unbind(this.jndiArtifactSync); + } catch (NamingException e) { + log.debug(String.format("unable to unbind %s - %s", this.jndiArtifactSync, e.getMessage())); } } } diff --git a/vault/src/main/java/org/opencadc/vault/VaultTransferGenerator.java b/vault/src/main/java/org/opencadc/vault/VaultTransferGenerator.java index 51fec42a..3b088488 100644 --- a/vault/src/main/java/org/opencadc/vault/VaultTransferGenerator.java +++ b/vault/src/main/java/org/opencadc/vault/VaultTransferGenerator.java @@ -117,7 +117,8 @@ public class VaultTransferGenerator implements TransferGenerator { private final Map siteAvailabilities; @SuppressWarnings("unchecked") - public VaultTransferGenerator(NodePersistenceImpl nodePersistence, ArtifactDAO artifactDAO, TokenTool tokenTool, boolean preventNotFound) { + public VaultTransferGenerator(NodePersistenceImpl nodePersistence, String appName, + ArtifactDAO artifactDAO, TokenTool tokenTool, boolean preventNotFound) { this.nodePersistence = nodePersistence; this.authorizer = new VOSpaceAuthorizer(nodePersistence); this.artifactDAO = artifactDAO; @@ -125,7 +126,7 @@ public VaultTransferGenerator(NodePersistenceImpl nodePersistence, ArtifactDAO a this.preventNotFound = preventNotFound; // TODO: get appname from ??? - String siteAvailabilitiesKey = "vault" + "-" + StorageSiteAvailabilityCheck.class.getName(); + String siteAvailabilitiesKey = appName + "-" + StorageSiteAvailabilityCheck.class.getName(); log.debug("siteAvailabilitiesKey: " + siteAvailabilitiesKey); try { Context initContext = new InitialContext(); diff --git a/vault/src/main/java/org/opencadc/vault/files/GetAction.java b/vault/src/main/java/org/opencadc/vault/files/GetAction.java index d3967e5c..231cad51 100644 --- a/vault/src/main/java/org/opencadc/vault/files/GetAction.java +++ b/vault/src/main/java/org/opencadc/vault/files/GetAction.java @@ -70,12 +70,10 @@ import ca.nrc.cadc.auth.AuthenticationUtil; import ca.nrc.cadc.net.TransientException; import java.net.HttpURLConnection; -import java.net.URI; import java.util.List; import javax.security.auth.Subject; import org.apache.log4j.Logger; import org.opencadc.vospace.DataNode; -import org.opencadc.vospace.NodeProperty; import org.opencadc.vospace.VOS; import org.opencadc.vospace.VOSURI; import org.opencadc.vospace.server.NodeFault; diff --git a/vault/src/main/java/org/opencadc/vault/metadata/ArtifactSync.java b/vault/src/main/java/org/opencadc/vault/metadata/ArtifactSync.java index c0f7d38f..023a2d2b 100644 --- a/vault/src/main/java/org/opencadc/vault/metadata/ArtifactSync.java +++ b/vault/src/main/java/org/opencadc/vault/metadata/ArtifactSync.java @@ -88,8 +88,8 @@ public class ArtifactSync implements Runnable { private static final Logger log = Logger.getLogger(ArtifactSync.class); private static final long ROUNDS = 6000L; // 6 sec - private static final long SHORT_SLEEP = 2 * ROUNDS; - private static final long LONG_SLEEP = 5 * ROUNDS; + private static final long SHORT_SLEEP = 5 * ROUNDS; + private static final long LONG_SLEEP = 10 * ROUNDS; private static final long EVICT_AGE = 12 * ROUNDS; private static final long FAIL_SLEEP = 10 * ROUNDS; // slightly smaller than evict @@ -99,6 +99,8 @@ public class ArtifactSync implements Runnable { private final ArtifactDAO artifactDAO; private final Namespace artifactNamespace; + private boolean offline = false; + public ArtifactSync(HarvestStateDAO stateDAO, ArtifactDAO artifactDAO, Namespace artifactNamespace) { this.stateDAO = stateDAO; this.artifactDAO = artifactDAO; @@ -113,14 +115,24 @@ public ArtifactSync(HarvestStateDAO stateDAO, ArtifactDAO artifactDAO, Namespace stateDAO.setMaintCount(9999); // every 1e4 } + public void setOffline(boolean offline) { + this.offline = offline; + + } + @Override public void run() { String name = Artifact.class.getSimpleName(); URI resourceID = URI.create("jdbc:inventory"); try { - Thread.sleep(SHORT_SLEEP); + Thread.sleep(1 * ROUNDS); // delay startup a bit while (true) { + while (offline) { + log.warn("disabled: sleep=" + LONG_SLEEP); + Thread.sleep(LONG_SLEEP); + } + log.debug("check leader " + instanceID); HarvestState state = stateDAO.get(name, resourceID); log.debug("check leader " + instanceID + " found: " + state); @@ -135,7 +147,7 @@ public void run() { boolean leader = checkLeaderStatus(state); if (leader) { - log.info("LEADER " + state.instanceID); + log.debug("leader: " + state); boolean fail = false; try { ArtifactSyncWorker worker = new ArtifactSyncWorker(stateDAO, state, artifactDAO, artifactNamespace); diff --git a/vault/src/main/webapp/META-INF/context.xml b/vault/src/main/webapp/META-INF/context.xml index 0e787176..02f5f709 100644 --- a/vault/src/main/webapp/META-INF/context.xml +++ b/vault/src/main/webapp/META-INF/context.xml @@ -27,14 +27,15 @@ removeAbandoned="false" testOnBorrow="true" validationQuery="select 123" /> - + ca.nrc.cadc.vosi.AvailabilityPlugin org.opencadc.vault.ServiceAvailability - - availabilityProperties - vault-availability.properties - 3 From ea52a02eb6d4909d86a1e90cc6a02002176ce4f9 Mon Sep 17 00:00:00 2001 From: Patrick Dowler Date: Tue, 19 Mar 2024 15:21:58 -0700 Subject: [PATCH 04/18] update VERSIONs for service images and add change log --- ChangeLog.md | 70 ++++++++++++++++++++++++++++++++++++++++++++++++ luskan/README.md | 4 +++ luskan/VERSION | 2 +- minoc/README.md | 25 +++-------------- minoc/VERSION | 2 +- raven/README.md | 4 +++ raven/VERSION | 2 +- vault/VERSION | 2 +- 8 files changed, 85 insertions(+), 26 deletions(-) create mode 100644 ChangeLog.md diff --git a/ChangeLog.md b/ChangeLog.md new file mode 100644 index 00000000..7fca1ef3 --- /dev/null +++ b/ChangeLog.md @@ -0,0 +1,70 @@ +# Change Log + +This is a cursory summary of changes to various storage-inventory components. +Check the README in specific modules for details. + +## minoc:1.0.0 +``` +added optional `org.opencadc.minoc.trust.preauth` config +removed optional `org.opencadc.minoc.publicKeyFile` config +``` +A `minoc` instance will download a public key from each trusted service and +use the public key(s) to validate URLs that include a _preauth_ token. + +``` +added optional `org.opencadc.minoc.readable` and `org.opencadc.minoc.writable` config +``` +A `minoc` service will advertise (via inventory.StorageSite record in the database) the +_readable_ and _writable_ status; this information is synced to global inventory and +used by `raven` to determine if it should generate PUT or GET URLs that use the `minoc` +service(s) at that site. By default, the configuration of _readGrantProvider_(s) and +_writeGrantProvider_(s) is used to determine the default status; configuration of any +_trust.preauth_ will also make make the status _readable_ and _writable_ even if the +service is only intending to accept one or the other. + +The explicit _readable_ and _writable_ configuration options will override all other +logic and set the status accordingly. This is currently optional but may be required +in a future version. + +``` +added optional config file: cadc-log.properties +added optional config file: cadc-vosi.properties +``` + +## raven:1.0.0 +``` +added org.opencadc.raven.inventory connection pool +``` +A `raven` service uses this pool to perform database initialization. This pool is +configured in the `catalina.properties` file. + +``` +added optional `org.opencadc.raven.keys.preauth` config +removed optional `org.opencadc.raven.publicKeyFile` and `org.opencadc.minoc.privateKeyFile` config +``` +When configured to do so, a `raven` service will generate it's own public/private key pair +and use the private key to _sign_ `minoc` URLs. All the `minoc` services known to the global +`raven` service must also be configured to _trust_ `raven`. + +``` +added optional config file: cadc-log.properties +added optional config file: cadc-vosi.properties +``` + +## luskan:1.0.0 +``` +changed config file: cadc-tap-tmp.properties +``` +A `luskan` service now uses the DelegatingStorageManager` so this config file must +specify which storage manager implementation to use along with existing +implementation-specific configuration options. + +``` +added optional config file: cadc-log.properties +added optional config file: cadc-vosi.properties +``` + +## vault:0.5.0 (NEW) +This is a new service that implements the IVOA VOSpace 2.1 REST API to provide user-managed +hierarchical storage. `vault` is deployed with it's own database and an associated inventory +database, uses inventory services (`minoc`) for file storage and management. diff --git a/luskan/README.md b/luskan/README.md index 0d4dd209..90f4ccdf 100644 --- a/luskan/README.md +++ b/luskan/README.md @@ -104,6 +104,10 @@ Assuming instances are restarted regularly, this would cause rollover approximat See cadc-log for common dynamic logging control. +### cadc-vosi.properties (optional) +See cadc-vosi for common +service state control. + ### cadcproxy.pem (optional) This client certificate is used to make authenticated server-to-server calls for system-level A&A purposes. diff --git a/luskan/VERSION b/luskan/VERSION index e894843e..2b866458 100644 --- a/luskan/VERSION +++ b/luskan/VERSION @@ -1,6 +1,6 @@ ## deployable containers have a semantic and build tag # semantic version tag: major.minor # build version tag: timestamp -VER=0.6.8 +VER=1.0.0 TAGS="${VER} ${VER}-$(date --utc +"%Y%m%dT%H%M%S")" unset VER diff --git a/minoc/README.md b/minoc/README.md index 0f27e2b0..e0292b94 100644 --- a/minoc/README.md +++ b/minoc/README.md @@ -144,16 +144,9 @@ will be able to read files. See cadc-log for common dynamic logging control. -### minoc-availability.properties (optional) -WARN: This config file name is going to change to a common one for consistency across multiple services. - -The minoc-availability.properties file specifies which users have the authority to change the availability state of the minoc service. Each entry consists of a key=value pair. The key is always "users". The value is the x500 canonical user name. - -Example: -``` -users = {user identity} -``` -`users` specifies the user(s) who are authorized to make calls to the service. The value is a list of user identities (X500 distingushed name), one line per user. Optional: if the `minoc-availability.properties` is not found or does not list any `users`, the service will function in the default mode (ReadWrite) and the state will not be changeable. +### cadc-vosi.properties (optional) +See cadc-vosi for common +service state control. ### cadcproxy.pem (optional) This client certificate is used to make authenticated server-to-server calls for system-level A&A purposes. @@ -174,15 +167,3 @@ docker run --rm -it minoc:latest /bin/bash docker run --rm --user tomcat:tomcat --volume=/path/to/external/config:/config:ro --name minoc minoc:latest ``` -## using it - -Using `cURL` is possible with Minoc to put a file for testing. - -**Note:** The `content-type` header is important! -```bash -$ curl -v -X PUT \ - --header "content-type: application/fits" \ - --data-binary @myfile.fits \ - -E ~/.ssl/cadcproxy.pem \ - https://myhost.com/minoc/files/test:TEST/myfile.fits -``` diff --git a/minoc/VERSION b/minoc/VERSION index a8d5aa3a..3d4fab56 100644 --- a/minoc/VERSION +++ b/minoc/VERSION @@ -4,6 +4,6 @@ # tags with and without build number so operators use the versioned # tag but we always keep a timestamped tag in case a semantic tag gets # replaced accidentally -VER=0.9.10 +VER=1.0.0 TAGS="${VER} ${VER}-$(date --utc +"%Y%m%dT%H%M%S")" unset VER diff --git a/raven/README.md b/raven/README.md index e4ea4f0d..cac828dd 100644 --- a/raven/README.md +++ b/raven/README.md @@ -112,6 +112,10 @@ will be able to read files. See cadc-log for common dynamic logging control. +### cadc-vosi.properties (optional) +See cadc-vosi for common +service state control. + ### cadcproxy.pem (optional) This client certificate is used to make authenticated server-to-server calls for system-level A&A purposes. diff --git a/raven/VERSION b/raven/VERSION index ee967b01..6a39bdba 100644 --- a/raven/VERSION +++ b/raven/VERSION @@ -2,6 +2,6 @@ # semantic version tag: major.minor[.patch] # build version tag: timestamp # tag: {semantic}-{build} -VER=0.8.0 +VER=1.0.0 TAGS="${VER} ${VER}-$(date --utc +"%Y%m%dT%H%M%S")" unset VER diff --git a/vault/VERSION b/vault/VERSION index 9d05f397..17ac68c1 100644 --- a/vault/VERSION +++ b/vault/VERSION @@ -4,6 +4,6 @@ # tags with and without build number so operators use the versioned # tag but we always keep a timestamped tag in case a semantic tag gets # replaced accidentally -VER=0.2.0 +VER=0.5.0 TAGS="${VER} ${VER}-$(date --utc +"%Y%m%dT%H%M%S")" unset VER From ba3bd5cc62462c274ef8b66cb4cf42ea80a04f72 Mon Sep 17 00:00:00 2001 From: Patrick Dowler Date: Wed, 20 Mar 2024 09:55:46 -0700 Subject: [PATCH 05/18] update image versions --- critwall/VERSION | 2 +- fenwick/VERSION | 2 +- ratik/VERSION | 2 +- ringhold/VERSION | 4 +++- tantar/VERSION | 2 +- 5 files changed, 7 insertions(+), 5 deletions(-) diff --git a/critwall/VERSION b/critwall/VERSION index ba99f57c..7fbddd11 100644 --- a/critwall/VERSION +++ b/critwall/VERSION @@ -1,6 +1,6 @@ ## deployable containers have a semantic and build tag # semantic version tag: major.minor[.patch] # build version tag: timestamp -VER=0.4.5 +VER=0.5.0 TAGS="${VER} ${VER}-$(date --utc +"%Y%m%dT%H%M%S")" unset VER diff --git a/fenwick/VERSION b/fenwick/VERSION index fed2d595..a7748476 100644 --- a/fenwick/VERSION +++ b/fenwick/VERSION @@ -4,6 +4,6 @@ # tags with and without build number so operators use the versioned # tag but we always keep a timestamped tag in case a semantic tag gets # replaced accidentally -VER=0.5.7 +VER=0.6.0 TAGS="${VER} ${VER}-$(date --utc +"%Y%m%dT%H%M%S")" unset VER diff --git a/ratik/VERSION b/ratik/VERSION index 79fec590..d9e5d604 100644 --- a/ratik/VERSION +++ b/ratik/VERSION @@ -4,6 +4,6 @@ # tags with and without build number so operators use the versioned # tag but we always keep a timestamped tag in case a semantic tag gets # replaced accidentally -VER=0.1.9 +VER=0.2.0 TAGS="${VER} ${VER}-$(date --utc +"%Y%m%dT%H%M%S")" unset VER diff --git a/ringhold/VERSION b/ringhold/VERSION index 51807fa8..b3562c8a 100644 --- a/ringhold/VERSION +++ b/ringhold/VERSION @@ -1,4 +1,6 @@ ## deployable containers have a semantic and build tag # semantic version tag: major.minor[.patch] # build version tag: timestamp -TAGS="0.2-$(date --utc +"%Y%m%dT%H%M%S")" +VER=0.2.0 +TAGS="${VER} ${VER}-$(date --utc +"%Y%m%dT%H%M%S")" +unset VER diff --git a/tantar/VERSION b/tantar/VERSION index ba99f57c..7fbddd11 100644 --- a/tantar/VERSION +++ b/tantar/VERSION @@ -1,6 +1,6 @@ ## deployable containers have a semantic and build tag # semantic version tag: major.minor[.patch] # build version tag: timestamp -VER=0.4.5 +VER=0.5.0 TAGS="${VER} ${VER}-$(date --utc +"%Y%m%dT%H%M%S")" unset VER From 3ca95773baa4a75a8ba7960338b0ac34f81edd01 Mon Sep 17 00:00:00 2001 From: Patrick Dowler Date: Wed, 20 Mar 2024 14:01:29 -0700 Subject: [PATCH 06/18] cadc-inventory and cadc-inventory-db versions 1.0.0 inventory and vospace DDL version 1.0.0 remove intermediate vospace DDL upgrade minoc: fix pubkey loading bug --- cadc-inventory-db/build.gradle | 6 +- .../inventory/db/version/InitDatabaseSI.java | 4 +- .../opencadc/vospace/db/InitDatabaseVOS.java | 7 +-- ...e-0.15.sql => inventory.upgrade-1.0.0.sql} | 0 .../main/resources/vospace.upgrade-0.15.sql | 7 --- cadc-inventory/build.gradle | 4 +- .../java/org/opencadc/minoc/MinocConfig.java | 59 +++++++++---------- tantar/build.gradle | 4 +- vault/build.gradle | 8 +-- 9 files changed, 43 insertions(+), 56 deletions(-) rename cadc-inventory-db/src/main/resources/{inventory.upgrade-0.15.sql => inventory.upgrade-1.0.0.sql} (100%) delete mode 100644 cadc-inventory-db/src/main/resources/vospace.upgrade-0.15.sql diff --git a/cadc-inventory-db/build.gradle b/cadc-inventory-db/build.gradle index 1b5a7ca8..91e5e51d 100644 --- a/cadc-inventory-db/build.gradle +++ b/cadc-inventory-db/build.gradle @@ -17,7 +17,7 @@ sourceCompatibility = 1.8 group = 'org.opencadc' -version = '0.15.0' +version = '1.0.0' description = 'OpenCADC Storage Inventory database library' def git_url = 'https://github.com/opencadc/storage-inventory' @@ -27,8 +27,8 @@ mainClassName = 'org.opencadc.inventory.db.version.Main' dependencies { compile 'org.opencadc:cadc-util:[1.11.0,2.0)' compile 'org.opencadc:cadc-gms:[1.0.0,)' - compile 'org.opencadc:cadc-inventory:[0.9.4,)' - compile 'org.opencadc:cadc-vos:[2.0.5,3.0)' + compile 'org.opencadc:cadc-inventory:[1.0.0,)' + compile 'org.opencadc:cadc-vos:[2.0.6,3.0)' testCompile 'junit:junit:[4.0,)' diff --git a/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/version/InitDatabaseSI.java b/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/version/InitDatabaseSI.java index 6e810b8f..f59d7d76 100644 --- a/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/version/InitDatabaseSI.java +++ b/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/version/InitDatabaseSI.java @@ -80,7 +80,7 @@ public class InitDatabaseSI extends InitDatabase { private static final Logger log = Logger.getLogger(InitDatabaseSI.class); public static final String MODEL_NAME = "storage-inventory"; - public static final String MODEL_VERSION = "0.15"; + public static final String MODEL_VERSION = "1.0.0"; public static final String PREV_MODEL_VERSION = "0.14"; //public static final String PREV_MODEL_VERSION = "DO-NOT_UPGRADE-BY-ACCIDENT"; @@ -98,7 +98,7 @@ public class InitDatabaseSI extends InitDatabase { }; static String[] UPGRADE_SQL = new String[] { - "inventory.upgrade-0.15.sql", + "inventory.upgrade-1.0.0.sql", "generic.PreauthKeyPair.sql", "generic.permissions.sql" }; diff --git a/cadc-inventory-db/src/main/java/org/opencadc/vospace/db/InitDatabaseVOS.java b/cadc-inventory-db/src/main/java/org/opencadc/vospace/db/InitDatabaseVOS.java index 05d8c45b..40d7f29d 100644 --- a/cadc-inventory-db/src/main/java/org/opencadc/vospace/db/InitDatabaseVOS.java +++ b/cadc-inventory-db/src/main/java/org/opencadc/vospace/db/InitDatabaseVOS.java @@ -79,9 +79,9 @@ public class InitDatabaseVOS extends ca.nrc.cadc.db.version.InitDatabase { private static final Logger log = Logger.getLogger(InitDatabaseVOS.class); - public static final String MODEL_NAME = "vospace-inventory"; - public static final String MODEL_VERSION = "0.15"; - public static final String PREV_MODEL_VERSION = "0.3"; + public static final String MODEL_NAME = "storage-vospace"; + public static final String MODEL_VERSION = "1.0.0"; + public static final String PREV_MODEL_VERSION = "n/a"; static String[] CREATE_SQL = new String[] { "generic.ModelVersion.sql", @@ -93,7 +93,6 @@ public class InitDatabaseVOS extends ca.nrc.cadc.db.version.InitDatabase { }; static String[] UPGRADE_SQL = new String[] { - "vospace.upgrade-0.15.sql", "generic.permissions.sql" }; diff --git a/cadc-inventory-db/src/main/resources/inventory.upgrade-0.15.sql b/cadc-inventory-db/src/main/resources/inventory.upgrade-1.0.0.sql similarity index 100% rename from cadc-inventory-db/src/main/resources/inventory.upgrade-0.15.sql rename to cadc-inventory-db/src/main/resources/inventory.upgrade-1.0.0.sql diff --git a/cadc-inventory-db/src/main/resources/vospace.upgrade-0.15.sql b/cadc-inventory-db/src/main/resources/vospace.upgrade-0.15.sql deleted file mode 100644 index bfb33166..00000000 --- a/cadc-inventory-db/src/main/resources/vospace.upgrade-0.15.sql +++ /dev/null @@ -1,7 +0,0 @@ - -alter table .Node - add column bytesUsed bigint, - add column storageBucket varchar(5) -; - -create unique index node_storageID on .Node(storageID); \ No newline at end of file diff --git a/cadc-inventory/build.gradle b/cadc-inventory/build.gradle index 55f43e50..cfe77269 100644 --- a/cadc-inventory/build.gradle +++ b/cadc-inventory/build.gradle @@ -14,13 +14,13 @@ sourceCompatibility = 1.8 group = 'org.opencadc' -version = '0.10.0' +version = '1.0.0' description = 'OpenCADC Storage Inventory core library' def git_url = 'https://github.com/opencadc/storage-inventory' dependencies { - compile 'org.opencadc:cadc-util:[1.10.6,2.0)' + compile 'org.opencadc:cadc-util:[1.11.0,2.0)' testCompile 'junit:junit:[4.0,)' } diff --git a/minoc/src/main/java/org/opencadc/minoc/MinocConfig.java b/minoc/src/main/java/org/opencadc/minoc/MinocConfig.java index d4b27d9b..437804b4 100644 --- a/minoc/src/main/java/org/opencadc/minoc/MinocConfig.java +++ b/minoc/src/main/java/org/opencadc/minoc/MinocConfig.java @@ -116,7 +116,6 @@ public class MinocConfig { private final MultiValuedProperties configProperties; private final Map trustedServices = new TreeMap<>(); - private boolean trustedServiceKeySync = false; private final List readGrantServices = new ArrayList<>(); private final List writeGrantServices = new ArrayList<>(); private final boolean readable; @@ -180,7 +179,7 @@ public MinocConfig() { throw new IllegalStateException("invalid config: " + TRUST_KEY + "=" + s + " INVALID", ex); } } - trustedServiceKeySync = false; + // try to sync keys on startup syncKeys(); } @@ -351,44 +350,40 @@ public MultiValuedProperties getProperties() { } public Map getTrustedServices() { + // check and try to sync missing keys before request syncKeys(); return trustedServices; } private void syncKeys() { - if (!trustedServiceKeySync) { - int numFails = 0; - RegistryClient reg = new RegistryClient(); - // check map for null keys and try to retrieve them - for (Map.Entry me : trustedServices.entrySet()) { - if (me.getValue() == null) { - try { - log.info("get trusted pubkey: " + me.getKey()); - URL capURL = reg.getAccessURL(RegistryClient.Query.CAPABILITIES, me.getKey()); - String s = capURL.toExternalForm().replace("/capabilities", "/pubkey"); - URL keyURL = new URL(s); - log.info("get trusted pubkey: " + me.getKey() + " -> " + keyURL); - ByteArrayOutputStream bos = new ByteArrayOutputStream(); - HttpGet get = new HttpGet(keyURL, bos); - get.setConnectionTimeout(6000); - get.setReadTimeout(6000); - get.setRetry(0, 0, HttpTransfer.RetryReason.NONE); - get.run(); - if (get.getThrowable() != null) { - throw (Exception) get.getThrowable(); - } - byte[] key = bos.toByteArray(); - trustedServices.put(me.getKey(), key); - log.info("get trusted pubkey: " + me.getKey() + " OK"); - } catch (Exception ex) { - log.warn("failed to get public key from " + me.getKey() + ": " + ex); - numFails++; + RegistryClient reg = new RegistryClient(); + // check map for null keys and try to retrieve them + // ASSUMPTION: keys never change once generated so if they do then minoc + // needs to be restarted + for (Map.Entry me : trustedServices.entrySet()) { + if (me.getValue() == null) { + try { + log.info("get trusted pubkey: " + me.getKey()); + URL capURL = reg.getAccessURL(RegistryClient.Query.CAPABILITIES, me.getKey()); + String s = capURL.toExternalForm().replace("/capabilities", "/pubkey"); + URL keyURL = new URL(s); + log.info("get trusted pubkey: " + me.getKey() + " -> " + keyURL); + ByteArrayOutputStream bos = new ByteArrayOutputStream(); + HttpGet get = new HttpGet(keyURL, bos); + get.setConnectionTimeout(6000); + get.setReadTimeout(6000); + get.setRetry(0, 0, HttpTransfer.RetryReason.NONE); + get.run(); + if (get.getThrowable() != null) { + throw (Exception) get.getThrowable(); } + byte[] key = bos.toByteArray(); + trustedServices.put(me.getKey(), key); + log.info("get trusted pubkey: " + me.getKey() + " OK"); + } catch (Exception ex) { + log.warn("failed to get public key from " + me.getKey() + ": " + ex); } } - if (numFails == 0) { - trustedServiceKeySync = true; - } } } diff --git a/tantar/build.gradle b/tantar/build.gradle index c99751a6..154e5309 100644 --- a/tantar/build.gradle +++ b/tantar/build.gradle @@ -21,8 +21,8 @@ mainClassName = 'org.opencadc.tantar.Main' dependencies { compile 'org.opencadc:cadc-util:[1.10.2,2.0)' compile 'org.opencadc:cadc-log:[1.1.2,2.0)' - compile 'org.opencadc:cadc-inventory:[0.10,)' - compile 'org.opencadc:cadc-inventory-db:[0.15.0,1.0)' + compile 'org.opencadc:cadc-inventory:[1.0.0,2.0)' + compile 'org.opencadc:cadc-inventory-db:[1.0.0,2.0)' compile 'org.opencadc:cadc-inventory-util:[0.1.8,1.0)' compile 'org.opencadc:cadc-storage-adapter:[0.11.1,1.0)' diff --git a/vault/build.gradle b/vault/build.gradle index a1154999..e6ac8689 100644 --- a/vault/build.gradle +++ b/vault/build.gradle @@ -31,11 +31,11 @@ def git_url = 'https://github.com/opencadc/storage-inventory' dependencies { compile 'javax.servlet:javax.servlet-api:[3.1,4.0)' - compile 'org.opencadc:cadc-util:[1.9.10,2.0)' + compile 'org.opencadc:cadc-util:[1.11.0,2.0)' compile 'org.opencadc:cadc-log:[1.1.6,2.0)' compile 'org.opencadc:cadc-gms:[1.0.5,)' compile 'org.opencadc:cadc-rest:[1.3.16,)' - compile 'org.opencadc:cadc-vos:[2.0.3,)' + compile 'org.opencadc:cadc-vos:[2.0.6,)' compile 'org.opencadc:cadc-vos-server:[2.0.9,)' compile 'org.opencadc:cadc-vosi:[1.3.2,)' compile 'org.opencadc:cadc-uws:[1.0,)' @@ -43,8 +43,8 @@ dependencies { compile 'org.opencadc:cadc-access-control:[1.1.1,2.0)' compile 'org.opencadc:cadc-cdp:[1.2.3,)' compile 'org.opencadc:cadc-registry:[1.7.6,)' - compile 'org.opencadc:cadc-inventory:[0.9.4,1.0)' - compile 'org.opencadc:cadc-inventory-db:[0.15.0,1.0)' + compile 'org.opencadc:cadc-inventory:[1.0.0,2.0)' + compile 'org.opencadc:cadc-inventory-db:[1.0.0,2.0)' compile 'org.opencadc:cadc-inventory-server:[0.3,1.0)' compile 'org.opencadc:cadc-permissions:[0.3.5,1.0)' From 8a729871bc4f2f905e0206921dd199bf91658607 Mon Sep 17 00:00:00 2001 From: Patrick Dowler Date: Wed, 20 Mar 2024 14:59:48 -0700 Subject: [PATCH 07/18] update cadc-inventory lib dependencies for 1.0 milestone --- critwall/build.gradle | 5 ++--- fenwick/build.gradle | 5 ++--- minoc/build.gradle | 4 ++-- ratik/build.gradle | 5 ++--- raven/build.gradle | 4 ++-- ringhold/build.gradle | 4 ++-- 6 files changed, 12 insertions(+), 15 deletions(-) diff --git a/critwall/build.gradle b/critwall/build.gradle index 9b9a74cd..83716cb1 100644 --- a/critwall/build.gradle +++ b/critwall/build.gradle @@ -21,9 +21,8 @@ mainClassName = 'org.opencadc.critwall.Main' dependencies { compile 'org.opencadc:cadc-storage-adapter:[0.8,1.0)' compile 'org.opencadc:cadc-util:[1.10.2,2.0)' - compile 'org.opencadc:cadc-inventory:[0.10,)' - // cadc-inventory-db-0.15 is in the vos2 feature branch - compile 'org.opencadc:cadc-inventory-db:[0.15.0,)' + compile 'org.opencadc:cadc-inventory:[1.0,2.0)' + compile 'org.opencadc:cadc-inventory-db:[1.0,2.0)' compile 'org.opencadc:cadc-registry:[1.7,2.0)' compile 'org.opencadc:cadc-vosi:[1.3.6,2.0)' compile 'org.opencadc:cadc-vos:[2.0,)' diff --git a/fenwick/build.gradle b/fenwick/build.gradle index eb9cce1e..f7feee09 100644 --- a/fenwick/build.gradle +++ b/fenwick/build.gradle @@ -16,9 +16,8 @@ group = 'org.opencadc' dependencies { compile 'org.opencadc:cadc-util:[1.10.2,2.0)' - compile 'org.opencadc:cadc-inventory:[0.9.4,2.0)' - // cadc-inventory-db-0.15.0 is from the vos2 feature branch - compile 'org.opencadc:cadc-inventory-db:[0.15,)' + compile 'org.opencadc:cadc-inventory:[1.0,2.0)' + compile 'org.opencadc:cadc-inventory-db:[1.0,2.0)' compile 'org.opencadc:cadc-inventory-util:[0.1.8,1.0)' compile 'org.opencadc:cadc-registry:[1.5,2.0)' compile 'org.opencadc:cadc-tap:[1.1.14,1.2)' // 1.2 upper bound is correct #reasons diff --git a/minoc/build.gradle b/minoc/build.gradle index 45705d4f..add0a28c 100644 --- a/minoc/build.gradle +++ b/minoc/build.gradle @@ -34,8 +34,8 @@ dependencies { compile 'org.opencadc:cadc-cdp:[1.0,)' compile 'org.opencadc:cadc-data-ops-fits:[0.3.0,)' compile 'org.opencadc:cadc-gms:[1.0.0,)' - compile 'org.opencadc:cadc-inventory:[0.9.4,2.0)' - compile 'org.opencadc:cadc-inventory-db:[0.14.5,0.15)' + compile 'org.opencadc:cadc-inventory:[1.0,2.0)' + compile 'org.opencadc:cadc-inventory-db:[1.0,2.0)' compile 'org.opencadc:cadc-inventory-server:[0.2.1,)' compile 'org.opencadc:cadc-soda-server:[1.2.0,2.0.0)' compile 'org.opencadc:cadc-storage-adapter:[0.11.2,)' diff --git a/ratik/build.gradle b/ratik/build.gradle index 6eef59f9..79137835 100644 --- a/ratik/build.gradle +++ b/ratik/build.gradle @@ -16,9 +16,8 @@ group = 'org.opencadc' dependencies { compile 'org.opencadc:cadc-util:[1.10.2,2.0)' - compile 'org.opencadc:cadc-inventory:[0.9.4,2.0)' - // cadc-inventory-db-0.15.0 is from the vos2 feature branch - compile 'org.opencadc:cadc-inventory-db:[0.15,)' + compile 'org.opencadc:cadc-inventory:[1.0,2.0)' + compile 'org.opencadc:cadc-inventory-db:[1.0,2.0)' compile 'org.opencadc:cadc-inventory-util:[0.1.8,1.0)' compile 'org.opencadc:cadc-registry:[1.5,2.0)' compile 'org.opencadc:cadc-tap:[1.1.15,2.0)' diff --git a/raven/build.gradle b/raven/build.gradle index a20a7319..fec3cf4f 100644 --- a/raven/build.gradle +++ b/raven/build.gradle @@ -33,8 +33,8 @@ dependencies { compile 'org.opencadc:cadc-rest:[1.3.14,)' compile 'org.opencadc:cadc-cdp:[1.0,)' compile 'org.opencadc:cadc-gms:[1.0.4,)' - compile 'org.opencadc:cadc-inventory:[0.9.4,2.0)' - compile 'org.opencadc:cadc-inventory-db:[0.15.0,)' + compile 'org.opencadc:cadc-inventory:[1.0,2.0)' + compile 'org.opencadc:cadc-inventory-db:[1.0,2.0)' compile 'org.opencadc:cadc-inventory-server:[0.3.0,)' compile 'org.opencadc:cadc-permissions:[0.3.5,)' compile 'org.opencadc:cadc-permissions-client:[0.3,)' diff --git a/ringhold/build.gradle b/ringhold/build.gradle index e29f2e50..685a569d 100644 --- a/ringhold/build.gradle +++ b/ringhold/build.gradle @@ -16,8 +16,8 @@ group = 'org.opencadc' dependencies { compile 'org.opencadc:cadc-util:[1.6,2.0)' - compile 'org.opencadc:cadc-inventory:[0.10,)' - compile 'org.opencadc:cadc-inventory-db:[0.15,1.0)' + compile 'org.opencadc:cadc-inventory:[1.0,2.0)' + compile 'org.opencadc:cadc-inventory-db:[1.0,2.0)' testCompile 'junit:junit:[4.12,5.0)' } From d762437a8ee9c20e57798ce5c8c0f9a886155b0c Mon Sep 17 00:00:00 2001 From: Patrick Dowler Date: Wed, 20 Mar 2024 15:41:59 -0700 Subject: [PATCH 08/18] rename ArtifactSync to DataNodeSizeSync for clarity --- ...rTest.java => DataNodeSizeWorkerTest.java} | 10 ++--- ...yncWorker.java => DataNodeSizeWorker.java} | 15 +++---- .../opencadc/vault/ServiceAvailability.java | 6 +-- .../org/opencadc/vault/VaultInitAction.java | 42 ++++++++++--------- ...rtifactSync.java => DataNodeSizeSync.java} | 10 ++--- 5 files changed, 43 insertions(+), 40 deletions(-) rename cadc-inventory-db/src/intTest/java/org/opencadc/vospace/db/{ArtifactSyncWorkerTest.java => DataNodeSizeWorkerTest.java} (97%) rename cadc-inventory-db/src/main/java/org/opencadc/vospace/db/{ArtifactSyncWorker.java => DataNodeSizeWorker.java} (93%) rename vault/src/main/java/org/opencadc/vault/metadata/{ArtifactSync.java => DataNodeSizeSync.java} (95%) diff --git a/cadc-inventory-db/src/intTest/java/org/opencadc/vospace/db/ArtifactSyncWorkerTest.java b/cadc-inventory-db/src/intTest/java/org/opencadc/vospace/db/DataNodeSizeWorkerTest.java similarity index 97% rename from cadc-inventory-db/src/intTest/java/org/opencadc/vospace/db/ArtifactSyncWorkerTest.java rename to cadc-inventory-db/src/intTest/java/org/opencadc/vospace/db/DataNodeSizeWorkerTest.java index fa18912c..649800b3 100644 --- a/cadc-inventory-db/src/intTest/java/org/opencadc/vospace/db/ArtifactSyncWorkerTest.java +++ b/cadc-inventory-db/src/intTest/java/org/opencadc/vospace/db/DataNodeSizeWorkerTest.java @@ -84,7 +84,7 @@ import org.junit.Before; import org.junit.Test; import org.opencadc.inventory.Artifact; -import org.opencadc.vospace.db.ArtifactSyncWorker; +import org.opencadc.vospace.db.DataNodeSizeWorker; import org.opencadc.inventory.Namespace; import org.opencadc.inventory.db.ArtifactDAO; import org.opencadc.inventory.db.HarvestState; @@ -101,8 +101,8 @@ * * @author adriand */ -public class ArtifactSyncWorkerTest { - private static final Logger log = Logger.getLogger(ArtifactSyncWorkerTest.class); +public class DataNodeSizeWorkerTest { + private static final Logger log = Logger.getLogger(DataNodeSizeWorkerTest.class); static { Log4jInit.setLevel("org.opencadc.inventory", Level.INFO); @@ -117,7 +117,7 @@ public class ArtifactSyncWorkerTest { ArtifactDAO artifactDAO; - public ArtifactSyncWorkerTest() throws Exception { + public DataNodeSizeWorkerTest() throws Exception { DBConfig dbrc = new DBConfig(); ConnectionConfig cc = dbrc.getConnectionConfig(TestUtil.SERVER, TestUtil.DATABASE); DBUtil.PoolConfig pool = new DBUtil.PoolConfig(cc, 1, 6000L, "select 123"); @@ -219,7 +219,7 @@ public void testSyncArtifact() throws Exception { harvestStateDAO.put(hs); hs = harvestStateDAO.get(hsName, resourceID); - ArtifactSyncWorker asWorker = new ArtifactSyncWorker(harvestStateDAO, hs, artifactDAO, siNamespace); + DataNodeSizeWorker asWorker = new DataNodeSizeWorker(harvestStateDAO, hs, artifactDAO, siNamespace); asWorker.run(); actual = (DataNode)nodeDAO.get(orig.getID()); diff --git a/cadc-inventory-db/src/main/java/org/opencadc/vospace/db/ArtifactSyncWorker.java b/cadc-inventory-db/src/main/java/org/opencadc/vospace/db/DataNodeSizeWorker.java similarity index 93% rename from cadc-inventory-db/src/main/java/org/opencadc/vospace/db/ArtifactSyncWorker.java rename to cadc-inventory-db/src/main/java/org/opencadc/vospace/db/DataNodeSizeWorker.java index 96709c05..0b40f848 100644 --- a/cadc-inventory-db/src/main/java/org/opencadc/vospace/db/ArtifactSyncWorker.java +++ b/cadc-inventory-db/src/main/java/org/opencadc/vospace/db/DataNodeSizeWorker.java @@ -86,8 +86,8 @@ * * @author adriand */ -public class ArtifactSyncWorker implements Runnable { - private static final Logger log = Logger.getLogger(ArtifactSyncWorker.class); +public class DataNodeSizeWorker implements Runnable { + private static final Logger log = Logger.getLogger(DataNodeSizeWorker.class); private final HarvestState harvestState; private final NodeDAO nodeDAO; @@ -103,7 +103,7 @@ public class ArtifactSyncWorker implements Runnable { * @param artifactDAO DAO class to query for artifacts * @param namespace artifact namespace */ - public ArtifactSyncWorker(HarvestStateDAO harvestStateDAO, HarvestState harvestState, + public DataNodeSizeWorker(HarvestStateDAO harvestStateDAO, HarvestState harvestState, ArtifactDAO artifactDAO, Namespace namespace) { this.harvestState = harvestState; this.harvestStateDAO = harvestStateDAO; @@ -114,13 +114,14 @@ public ArtifactSyncWorker(HarvestStateDAO harvestStateDAO, HarvestState harvestS @Override public void run() { + String opName = DataNodeSizeWorker.class.getSimpleName() + ".artifactQuery"; DateFormat df = DateUtil.getDateFormat(DateUtil.IVOA_DATE_FORMAT, DateUtil.UTC); if (harvestState.curLastModified != null) { - log.info("ArtifactSyncWorker.artifactQuery source=" + harvestState.getResourceID() + log.info(opName + " source=" + harvestState.getResourceID() + " instance=" + harvestState.instanceID + " start=" + df.format(harvestState.curLastModified)); } else { - log.info("ArtifactSyncWorker.artifactQuery source=" + harvestState.getResourceID() + log.info(opName + " source=" + harvestState.getResourceID() + " instance=" + harvestState.instanceID); } @@ -164,11 +165,11 @@ public void run() { throw new RuntimeException("error while closing ResourceIterator", ex); } if (harvestState.curLastModified != null) { - log.info("ArtifactSyncWorker.artifactQuery source=" + harvestState.getResourceID() + log.info(opName + " source=" + harvestState.getResourceID() + " instance=" + harvestState.instanceID + " end=" + df.format(harvestState.curLastModified)); } else { - log.info("ArtifactSyncWorker.artifactQuery source=" + harvestState.getResourceID() + log.info(opName + " source=" + harvestState.getResourceID() + " instance=" + harvestState.instanceID + " end=true"); } diff --git a/vault/src/main/java/org/opencadc/vault/ServiceAvailability.java b/vault/src/main/java/org/opencadc/vault/ServiceAvailability.java index e750e2be..6e22d8c8 100644 --- a/vault/src/main/java/org/opencadc/vault/ServiceAvailability.java +++ b/vault/src/main/java/org/opencadc/vault/ServiceAvailability.java @@ -77,7 +77,7 @@ import javax.naming.NamingException; import javax.sql.DataSource; import org.apache.log4j.Logger; -import org.opencadc.vault.metadata.ArtifactSync; +import org.opencadc.vault.metadata.DataNodeSizeSync; /** * This class performs the work of determining if the executing artifact @@ -204,10 +204,10 @@ private String getState() { } private void setOffline(boolean offline) { - String jndiArtifactSync = appName + "-" + ArtifactSync.class.getName(); + String jndiArtifactSync = appName + "-" + DataNodeSizeSync.class.getName(); try { InitialContext initialContext = new InitialContext(); - ArtifactSync async = (ArtifactSync) initialContext.lookup(jndiArtifactSync); + DataNodeSizeSync async = (DataNodeSizeSync) initialContext.lookup(jndiArtifactSync); async.setOffline(offline); } catch (NamingException e) { log.debug(String.format("unable to unbind %s - %s", jndiArtifactSync, e.getMessage())); diff --git a/vault/src/main/java/org/opencadc/vault/VaultInitAction.java b/vault/src/main/java/org/opencadc/vault/VaultInitAction.java index f10298f7..b61f3470 100644 --- a/vault/src/main/java/org/opencadc/vault/VaultInitAction.java +++ b/vault/src/main/java/org/opencadc/vault/VaultInitAction.java @@ -95,7 +95,7 @@ import org.opencadc.inventory.db.StorageSiteDAO; import org.opencadc.inventory.db.version.InitDatabaseSI; import org.opencadc.inventory.transfer.StorageSiteAvailabilityCheck; -import org.opencadc.vault.metadata.ArtifactSync; +import org.opencadc.vault.metadata.DataNodeSizeSync; import org.opencadc.vospace.db.InitDatabaseVOS; import org.opencadc.vospace.server.NodePersistence; import org.springframework.dao.DataIntegrityViolationException; @@ -135,10 +135,12 @@ public class VaultInitAction extends InitAction { private String jndiNodePersistence; private String jndiPreauthKeys; // store pubkey in JNDI for download via GetKeyAction - private String jndiArtifactSync; // store in JNDI to support availability mode change + private String jndiSiteAvailabilities; private Thread availabilityCheck; - private Thread artifactSyncThread; + + private String jndiDataNodeSizeSync; // store in JNDI to support availability mode change + private Thread dataNodeSizeSyncThread; public VaultInitAction() { super(); @@ -466,44 +468,44 @@ private void initBackgroundWorkers() { artifactDAO.setConfig(iterprops); terminateBackgroundWorkers(); - ArtifactSync async = new ArtifactSync(hsDAO, artifactDAO, storageNamespace); - this.artifactSyncThread = new Thread(async); - artifactSyncThread.setDaemon(true); - artifactSyncThread.start(); + DataNodeSizeSync async = new DataNodeSizeSync(hsDAO, artifactDAO, storageNamespace); + this.dataNodeSizeSyncThread = new Thread(async); + dataNodeSizeSyncThread.setDaemon(true); + dataNodeSizeSyncThread.start(); // store in JNDI so availability can set offline - String jndiArtifactSync = appName + "-" + ArtifactSync.class.getName(); + this.jndiDataNodeSizeSync = appName + "-" + DataNodeSizeSync.class.getName(); InitialContext ctx = new InitialContext(); try { - ctx.unbind(jndiArtifactSync); + ctx.unbind(jndiDataNodeSizeSync); } catch (NamingException ignore) { log.debug("unbind previous JNDI key (" + jndiPreauthKeys + ") failed... ignoring"); } - ctx.bind(jndiArtifactSync, async); - log.info("initBackgroundWorkers: created JNDI key: " + jndiArtifactSync); + ctx.bind(jndiDataNodeSizeSync, async); + log.info("initBackgroundWorkers: created JNDI key: " + jndiDataNodeSizeSync); } catch (Exception ex) { throw new RuntimeException("check/init ArtifactSync failed", ex); } } private void terminateBackgroundWorkers() { - if (this.artifactSyncThread != null) { + if (this.dataNodeSizeSyncThread != null) { try { - log.info("terminating ArtifactSync Thread..."); - this.artifactSyncThread.interrupt(); - this.artifactSyncThread.join(); - log.info("terminating ArtifactSync Thread... [OK]"); + log.info("terminating " + DataNodeSizeSync.class.getSimpleName() + " Thread..."); + this.dataNodeSizeSyncThread.interrupt(); + this.dataNodeSizeSyncThread.join(); + log.info("terminating " + DataNodeSizeSync.class.getSimpleName() + " Thread... [OK]"); } catch (Throwable t) { - log.info("failed to terminate ArtifactSync thread", t); + log.info("failed to terminate " + DataNodeSizeSync.class.getSimpleName() + " thread", t); } finally { - this.artifactSyncThread = null; + this.dataNodeSizeSyncThread = null; } try { InitialContext initialContext = new InitialContext(); - initialContext.unbind(this.jndiArtifactSync); + initialContext.unbind(this.jndiDataNodeSizeSync); } catch (NamingException e) { - log.debug(String.format("unable to unbind %s - %s", this.jndiArtifactSync, e.getMessage())); + log.debug(String.format("unable to unbind %s - %s", this.jndiDataNodeSizeSync, e.getMessage())); } } } diff --git a/vault/src/main/java/org/opencadc/vault/metadata/ArtifactSync.java b/vault/src/main/java/org/opencadc/vault/metadata/DataNodeSizeSync.java similarity index 95% rename from vault/src/main/java/org/opencadc/vault/metadata/ArtifactSync.java rename to vault/src/main/java/org/opencadc/vault/metadata/DataNodeSizeSync.java index 023a2d2b..2bee5411 100644 --- a/vault/src/main/java/org/opencadc/vault/metadata/ArtifactSync.java +++ b/vault/src/main/java/org/opencadc/vault/metadata/DataNodeSizeSync.java @@ -76,7 +76,7 @@ import org.opencadc.inventory.db.ArtifactDAO; import org.opencadc.inventory.db.HarvestState; import org.opencadc.inventory.db.HarvestStateDAO; -import org.opencadc.vospace.db.ArtifactSyncWorker; +import org.opencadc.vospace.db.DataNodeSizeWorker; /** * Main artifact-sync agent that enables incremental sync of Artifact @@ -84,8 +84,8 @@ * * @author pdowler */ -public class ArtifactSync implements Runnable { - private static final Logger log = Logger.getLogger(ArtifactSync.class); +public class DataNodeSizeSync implements Runnable { + private static final Logger log = Logger.getLogger(DataNodeSizeSync.class); private static final long ROUNDS = 6000L; // 6 sec private static final long SHORT_SLEEP = 5 * ROUNDS; @@ -101,7 +101,7 @@ public class ArtifactSync implements Runnable { private boolean offline = false; - public ArtifactSync(HarvestStateDAO stateDAO, ArtifactDAO artifactDAO, Namespace artifactNamespace) { + public DataNodeSizeSync(HarvestStateDAO stateDAO, ArtifactDAO artifactDAO, Namespace artifactNamespace) { this.stateDAO = stateDAO; this.artifactDAO = artifactDAO; this.artifactNamespace = artifactNamespace; @@ -150,7 +150,7 @@ public void run() { log.debug("leader: " + state); boolean fail = false; try { - ArtifactSyncWorker worker = new ArtifactSyncWorker(stateDAO, state, artifactDAO, artifactNamespace); + DataNodeSizeWorker worker = new DataNodeSizeWorker(stateDAO, state, artifactDAO, artifactNamespace); worker.run(); } catch (Exception ex) { log.error("unexpected worker fail", ex); From 5bb6d74f9529139f3628c172725b3525ee8b1654 Mon Sep 17 00:00:00 2001 From: Patrick Dowler Date: Mon, 25 Mar 2024 10:00:35 -0700 Subject: [PATCH 09/18] fix open connection leak in HarvestStateDAO table maintenance code --- .../opencadc/inventory/db/HarvestStateDAO.java | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/HarvestStateDAO.java b/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/HarvestStateDAO.java index 5df56335..a0f8d8f7 100644 --- a/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/HarvestStateDAO.java +++ b/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/HarvestStateDAO.java @@ -68,6 +68,7 @@ package org.opencadc.inventory.db; import java.net.URI; +import java.sql.Connection; import java.sql.SQLException; import java.util.UUID; import org.apache.log4j.Logger; @@ -194,13 +195,15 @@ public void put(HarvestState val, boolean forceTimestampUpdate) { if (curMaintCount == maintCount) { String sql = "VACUUM " + gen.getTable(HarvestState.class); log.warn("maintenance: " + curMaintCount + "==" + maintCount + " " + sql); - //JdbcTemplate jdbc = new JdbcTemplate(dataSource); - //jdbc.execute(sql); try { - dataSource.getConnection().createStatement().execute(sql); - } catch (SQLException ex) { - log.error("ERROR: " + sql + " FAILED", ex); - // yes, log and proceed + try (Connection c = dataSource.getConnection()) { + c.createStatement().execute(sql); + } catch (SQLException ex) { + log.error("maintenance failed: " + sql, ex); + // yes, log and proceed + } // auto-close to return to pool + } catch (Exception ex) { + log.error("failed to close connection after maintenance: " + sql, ex); } curMaintCount = 0; } else { From 209a95eb5e41108ab0f4e71bdb71575ed78fb1ef Mon Sep 17 00:00:00 2001 From: Patrick Dowler Date: Mon, 25 Mar 2024 10:10:17 -0700 Subject: [PATCH 10/18] import cleanup --- vault/src/main/java/org/opencadc/vault/ServiceAvailability.java | 1 - 1 file changed, 1 deletion(-) diff --git a/vault/src/main/java/org/opencadc/vault/ServiceAvailability.java b/vault/src/main/java/org/opencadc/vault/ServiceAvailability.java index 6e22d8c8..b1e53f10 100644 --- a/vault/src/main/java/org/opencadc/vault/ServiceAvailability.java +++ b/vault/src/main/java/org/opencadc/vault/ServiceAvailability.java @@ -71,7 +71,6 @@ import ca.nrc.cadc.rest.RestAction; import ca.nrc.cadc.vosi.Availability; import ca.nrc.cadc.vosi.AvailabilityPlugin; - import ca.nrc.cadc.vosi.avail.CheckDataSource; import javax.naming.InitialContext; import javax.naming.NamingException; From af92038185258d5dbb0f8445c2ef5107afbca5b8 Mon Sep 17 00:00:00 2001 From: Patrick Dowler Date: Mon, 25 Mar 2024 10:50:10 -0700 Subject: [PATCH 11/18] temporarily remove ringhold from CI --- .github/workflows/gradle.yml | 16 ++++------------ 1 file changed, 4 insertions(+), 12 deletions(-) diff --git a/.github/workflows/gradle.yml b/.github/workflows/gradle.yml index 25d39bd6..64748cf5 100644 --- a/.github/workflows/gradle.yml +++ b/.github/workflows/gradle.yml @@ -66,19 +66,11 @@ jobs: - name: java build -- raven run: cd raven && ../gradlew --info clean build javadoc checkstyleMain - - - name: java build -- ringhold - run: cd ringhold && ../gradlew --info clean build javadoc checkstyleMain - name: java build -- ratik run: cd ratik && ../gradlew --info clean build javadoc checkstyleMain -## TODO: docker build depends on cadc-tomcat base image from docker-base.git -# - name: docker build -- baldur -# run: cd baldur && docker build . --file Dockerfile --tag baldur:$(date +%s) -# - name: docker build -- minoc -# run: cd minoc && docker build . --file Dockerfile --tag minoc:$(date +%s) -# - name: docker build -- luskan -# run: cd luskan && docker build . --file Dockerfile --tag luskan:$(date +%s) -# - name: docker build -- raven -# run: cd raven && docker build . --file Dockerfile --tag raven:$(date +%s) +## disabled until updated for cadc-inventory-db API changes +# - name: java build -- ringhold +# run: cd ringhold && ../gradlew --info clean build javadoc checkstyleMain + From f737deecf57e6156c61dd7d3cde4b2d54f9c7353 Mon Sep 17 00:00:00 2001 From: Patrick Dowler Date: Mon, 25 Mar 2024 11:15:33 -0700 Subject: [PATCH 12/18] vault code cleanup --- .../java/org/opencadc/vault/metadata/DataNodeSizeSync.java | 5 ----- 1 file changed, 5 deletions(-) diff --git a/vault/src/main/java/org/opencadc/vault/metadata/DataNodeSizeSync.java b/vault/src/main/java/org/opencadc/vault/metadata/DataNodeSizeSync.java index 2bee5411..9b917b45 100644 --- a/vault/src/main/java/org/opencadc/vault/metadata/DataNodeSizeSync.java +++ b/vault/src/main/java/org/opencadc/vault/metadata/DataNodeSizeSync.java @@ -106,10 +106,6 @@ public DataNodeSizeSync(HarvestStateDAO stateDAO, ArtifactDAO artifactDAO, Names this.artifactDAO = artifactDAO; this.artifactNamespace = artifactNamespace; - // fenwick setup for production workload: - //dao.setUpdateBufferCount(99); // buffer 99 updates, do every 100 - //dao.setMaintCount(999); // buffer 999 so every 1000 real updates aka every 1e5 events - // we need continuous timestamp updates to retain leader status, so only schedule maintenance stateDAO.setUpdateBufferCount(0); stateDAO.setMaintCount(9999); // every 1e4 @@ -117,7 +113,6 @@ public DataNodeSizeSync(HarvestStateDAO stateDAO, ArtifactDAO artifactDAO, Names public void setOffline(boolean offline) { this.offline = offline; - } @Override From c2c650b3f9ac3cf44dcc10cbc4359d3b7235d056 Mon Sep 17 00:00:00 2001 From: Patrick Dowler Date: Mon, 25 Mar 2024 12:52:03 -0700 Subject: [PATCH 13/18] DataNodeSizeSync logging in machine readable format move info logging from DataNodeSizeWorker to DataNodeSizeSync --- .../vospace/db/DataNodeSizeWorker.java | 16 ++++++++--- .../vault/metadata/DataNodeSizeSync.java | 27 ++++++++++++++----- 2 files changed, 32 insertions(+), 11 deletions(-) diff --git a/cadc-inventory-db/src/main/java/org/opencadc/vospace/db/DataNodeSizeWorker.java b/cadc-inventory-db/src/main/java/org/opencadc/vospace/db/DataNodeSizeWorker.java index 0b40f848..aeabf6e3 100644 --- a/cadc-inventory-db/src/main/java/org/opencadc/vospace/db/DataNodeSizeWorker.java +++ b/cadc-inventory-db/src/main/java/org/opencadc/vospace/db/DataNodeSizeWorker.java @@ -94,6 +94,8 @@ public class DataNodeSizeWorker implements Runnable { private final ArtifactDAO artifactDAO; private final HarvestStateDAO harvestStateDAO; private final Namespace storageNamespace; + + private long numArtifactsProcessed; /** * Worker constructor. @@ -112,16 +114,21 @@ public DataNodeSizeWorker(HarvestStateDAO harvestStateDAO, HarvestState harvestS this.storageNamespace = namespace; } + public long getNumArtifactsProcessed() { + return numArtifactsProcessed; + } + @Override public void run() { + this.numArtifactsProcessed = 0L; String opName = DataNodeSizeWorker.class.getSimpleName() + ".artifactQuery"; DateFormat df = DateUtil.getDateFormat(DateUtil.IVOA_DATE_FORMAT, DateUtil.UTC); if (harvestState.curLastModified != null) { - log.info(opName + " source=" + harvestState.getResourceID() + log.debug(opName + " source=" + harvestState.getResourceID() + " instance=" + harvestState.instanceID + " start=" + df.format(harvestState.curLastModified)); } else { - log.info(opName + " source=" + harvestState.getResourceID() + log.debug(opName + " source=" + harvestState.getResourceID() + " instance=" + harvestState.instanceID); } @@ -159,17 +166,18 @@ public void run() { harvestState.curLastModified = artifact.getLastModified(); harvestState.curID = artifact.getID(); harvestStateDAO.put(harvestState); + numArtifactsProcessed++; } } catch (IOException ex) { log.error("Error closing iterator", ex); throw new RuntimeException("error while closing ResourceIterator", ex); } if (harvestState.curLastModified != null) { - log.info(opName + " source=" + harvestState.getResourceID() + log.debug(opName + " source=" + harvestState.getResourceID() + " instance=" + harvestState.instanceID + " end=" + df.format(harvestState.curLastModified)); } else { - log.info(opName + " source=" + harvestState.getResourceID() + log.debug(opName + " source=" + harvestState.getResourceID() + " instance=" + harvestState.instanceID + " end=true"); } diff --git a/vault/src/main/java/org/opencadc/vault/metadata/DataNodeSizeSync.java b/vault/src/main/java/org/opencadc/vault/metadata/DataNodeSizeSync.java index 9b917b45..8abf2b8a 100644 --- a/vault/src/main/java/org/opencadc/vault/metadata/DataNodeSizeSync.java +++ b/vault/src/main/java/org/opencadc/vault/metadata/DataNodeSizeSync.java @@ -138,15 +138,25 @@ public void run() { log.debug("created: " + state); } + long t1 = System.currentTimeMillis(); + BackgroundLogInfo logInfo = new BackgroundLogInfo(instanceID.toString()); + logInfo.setSuccess(false); + // determine leader boolean leader = checkLeaderStatus(state); - + logInfo.leader = leader; + + log.info(logInfo.start()); + long sleep = LONG_SLEEP; // default for not leader if (leader) { log.debug("leader: " + state); boolean fail = false; try { DataNodeSizeWorker worker = new DataNodeSizeWorker(stateDAO, state, artifactDAO, artifactNamespace); worker.run(); + logInfo.setLastModified(state.curLastModified); + logInfo.processed = worker.getNumArtifactsProcessed(); + logInfo.setSuccess(true); } catch (Exception ex) { log.error("unexpected worker fail", ex); fail = true; @@ -173,16 +183,19 @@ public void run() { } if (fail) { - log.debug("failed leader " + state.instanceID + " sleep=" + FAIL_SLEEP); - Thread.sleep(FAIL_SLEEP); + sleep = FAIL_SLEEP; } else { - log.debug("idle leader " + state.instanceID + " sleep=" + SHORT_SLEEP); - Thread.sleep(SHORT_SLEEP); + sleep = SHORT_SLEEP; } + logInfo.setSuccess(!fail); + logInfo.setElapsedTime(System.currentTimeMillis() - t1); } else { - log.debug("not leader: sleep=" + LONG_SLEEP); - Thread.sleep(LONG_SLEEP); + // not leader success + logInfo.setSuccess(true); } + logInfo.sleep = sleep; + log.info(logInfo.end()); + Thread.sleep(sleep); } } catch (InterruptedException ex) { log.debug("interrupted - assuming shutdown", ex); From 0c4c36ec0f575e5bf5d8773ea1c23e3a59fdf4b7 Mon Sep 17 00:00:00 2001 From: Patrick Dowler Date: Mon, 25 Mar 2024 13:38:13 -0700 Subject: [PATCH 14/18] vault: machine-readable logging in DataNodeSizeSync incremental lookback in DataNodeSizeWorker --- .../vospace/db/DataNodeSizeWorker.java | 32 +++++- .../vault/metadata/BackgroundLogInfo.java | 106 ++++++++++++++++++ .../vault/metadata/DataNodeSizeSync.java | 9 +- 3 files changed, 142 insertions(+), 5 deletions(-) create mode 100644 vault/src/main/java/org/opencadc/vault/metadata/BackgroundLogInfo.java diff --git a/cadc-inventory-db/src/main/java/org/opencadc/vospace/db/DataNodeSizeWorker.java b/cadc-inventory-db/src/main/java/org/opencadc/vospace/db/DataNodeSizeWorker.java index aeabf6e3..236da5ca 100644 --- a/cadc-inventory-db/src/main/java/org/opencadc/vospace/db/DataNodeSizeWorker.java +++ b/cadc-inventory-db/src/main/java/org/opencadc/vospace/db/DataNodeSizeWorker.java @@ -72,6 +72,7 @@ import ca.nrc.cadc.io.ResourceIterator; import java.io.IOException; import java.text.DateFormat; +import java.util.Date; import org.apache.log4j.Logger; import org.opencadc.inventory.Artifact; import org.opencadc.inventory.Namespace; @@ -89,6 +90,10 @@ public class DataNodeSizeWorker implements Runnable { private static final Logger log = Logger.getLogger(DataNodeSizeWorker.class); + // lookback when doing incremental harvest because head of sequence is + // not monotonic over short timescales (events arrive out of sequence) + private static final long LOOKBACK_TIME_MS = 60 * 1000L; + private final HarvestState harvestState; private final NodeDAO nodeDAO; private final ArtifactDAO artifactDAO; @@ -132,12 +137,21 @@ public void run() { + " instance=" + harvestState.instanceID); } + final Date now = new Date(); + final Date lookBack = new Date(now.getTime() - LOOKBACK_TIME_MS); + Date startTime = getQueryLowerBound(lookBack, harvestState.curLastModified); + if (lookBack != null && harvestState.curLastModified != null) { + log.debug("lookBack=" + df.format(lookBack) + " curLastModified=" + df.format(harvestState.curLastModified) + + " -> " + df.format(startTime)); + } + String uriBucket = null; // process all artifacts in a single thread - try (final ResourceIterator iter = artifactDAO.iterator(storageNamespace, uriBucket, harvestState.curLastModified, true)) { + try (final ResourceIterator iter = artifactDAO.iterator(storageNamespace, uriBucket, startTime, true)) { TransactionManager tm = nodeDAO.getTransactionManager(); while (iter.hasNext()) { Artifact artifact = iter.next(); DataNode node = nodeDAO.getDataNode(artifact.getURI()); + log.debug(artifact.getURI() + " len=" + artifact.getContentLength() + " -> " + node.getName()); if (node != null && !artifact.getContentLength().equals(node.bytesUsed)) { tm.startTransaction(); try { @@ -182,4 +196,20 @@ public void run() { + " end=true"); } } + + private Date getQueryLowerBound(Date lookBack, Date lastModified) { + if (lookBack == null) { + // feature not enabled + return lastModified; + } + if (lastModified == null) { + // first harvest + return null; + } + if (lookBack.before(lastModified)) { + return lookBack; + } + return lastModified; + + } } diff --git a/vault/src/main/java/org/opencadc/vault/metadata/BackgroundLogInfo.java b/vault/src/main/java/org/opencadc/vault/metadata/BackgroundLogInfo.java new file mode 100644 index 00000000..88ac11a9 --- /dev/null +++ b/vault/src/main/java/org/opencadc/vault/metadata/BackgroundLogInfo.java @@ -0,0 +1,106 @@ +/* +************************************************************************ +******************* CANADIAN ASTRONOMY DATA CENTRE ******************* +************** CENTRE CANADIEN DE DONNÉES ASTRONOMIQUES ************** +* +* (c) 2024. (c) 2024. +* Government of Canada Gouvernement du Canada +* National Research Council Conseil national de recherches +* Ottawa, Canada, K1A 0R6 Ottawa, Canada, K1A 0R6 +* All rights reserved Tous droits réservés +* +* NRC disclaims any warranties, Le CNRC dénie toute garantie +* expressed, implied, or énoncée, implicite ou légale, +* statutory, of any kind with de quelque nature que ce +* respect to the software, soit, concernant le logiciel, +* including without limitation y compris sans restriction +* any warranty of merchantability toute garantie de valeur +* or fitness for a particular marchande ou de pertinence +* purpose. NRC shall not be pour un usage particulier. +* liable in any event for any Le CNRC ne pourra en aucun cas +* damages, whether direct or être tenu responsable de tout +* indirect, special or general, dommage, direct ou indirect, +* consequential or incidental, particulier ou général, +* arising from the use of the accessoire ou fortuit, résultant +* software. Neither the name de l'utilisation du logiciel. Ni +* of the National Research le nom du Conseil National de +* Council of Canada nor the Recherches du Canada ni les noms +* names of its contributors may de ses participants ne peuvent +* be used to endorse or promote être utilisés pour approuver ou +* products derived from this promouvoir les produits dérivés +* software without specific prior de ce logiciel sans autorisation +* written permission. préalable et particulière +* par écrit. +* +* This file is part of the Ce fichier fait partie du projet +* OpenCADC project. OpenCADC. +* +* OpenCADC is free software: OpenCADC est un logiciel libre ; +* you can redistribute it and/or vous pouvez le redistribuer ou le +* modify it under the terms of modifier suivant les termes de +* the GNU Affero General Public la “GNU Affero General Public +* License as published by the License” telle que publiée +* Free Software Foundation, par la Free Software Foundation +* either version 3 of the : soit la version 3 de cette +* License, or (at your option) licence, soit (à votre gré) +* any later version. toute version ultérieure. +* +* OpenCADC is distributed in the OpenCADC est distribué +* hope that it will be useful, dans l’espoir qu’il vous +* but WITHOUT ANY WARRANTY; sera utile, mais SANS AUCUNE +* without even the implied GARANTIE : sans même la garantie +* warranty of MERCHANTABILITY implicite de COMMERCIALISABILITÉ +* or FITNESS FOR A PARTICULAR ni d’ADÉQUATION À UN OBJECTIF +* PURPOSE. See the GNU Affero PARTICULIER. Consultez la Licence +* General Public License for Générale Publique GNU Affero +* more details. pour plus de détails. +* +* You should have received Vous devriez avoir reçu une +* a copy of the GNU Affero copie de la Licence Générale +* General Public License along Publique GNU Affero avec +* with OpenCADC. If not, see OpenCADC ; si ce n’est +* . pas le cas, consultez : +* . +* +************************************************************************ +*/ + +package org.opencadc.vault.metadata; + +import ca.nrc.cadc.date.DateUtil; +import ca.nrc.cadc.log.WebServiceLogInfo; +import java.text.DateFormat; +import java.util.Date; +import org.apache.log4j.Logger; + +/** + * Log structure for background threads. + * + * @author pdowler + */ +public class BackgroundLogInfo extends WebServiceLogInfo { + private static final Logger log = Logger.getLogger(BackgroundLogInfo.class); + + public Boolean leader; + public String instance; + public String lastmodified; + public Long processed; + public Long sleep; + + + public BackgroundLogInfo(String instance) { + super.serviceName = "vault"; + this.instance = instance; + } + + public void setOperation(String op) { + super.method = op; + } + + public void setLastModified(Date ts) { + DateFormat df = DateUtil.getDateFormat(DateUtil.IVOA_DATE_FORMAT, DateUtil.UTC); + this.lastmodified = df.format(ts); + } + + +} diff --git a/vault/src/main/java/org/opencadc/vault/metadata/DataNodeSizeSync.java b/vault/src/main/java/org/opencadc/vault/metadata/DataNodeSizeSync.java index 8abf2b8a..a63eb83c 100644 --- a/vault/src/main/java/org/opencadc/vault/metadata/DataNodeSizeSync.java +++ b/vault/src/main/java/org/opencadc/vault/metadata/DataNodeSizeSync.java @@ -140,17 +140,18 @@ public void run() { long t1 = System.currentTimeMillis(); BackgroundLogInfo logInfo = new BackgroundLogInfo(instanceID.toString()); + logInfo.setOperation(DataNodeSizeWorker.class.getSimpleName()); logInfo.setSuccess(false); // determine leader boolean leader = checkLeaderStatus(state); logInfo.leader = leader; - - log.info(logInfo.start()); - long sleep = LONG_SLEEP; // default for not leader + logInfo.setLastModified(state.curLastModified); + long sleep = LONG_SLEEP; if (leader) { log.debug("leader: " + state); boolean fail = false; + log.info(logInfo.start()); try { DataNodeSizeWorker worker = new DataNodeSizeWorker(stateDAO, state, artifactDAO, artifactNamespace); worker.run(); @@ -191,6 +192,7 @@ public void run() { logInfo.setElapsedTime(System.currentTimeMillis() - t1); } else { // not leader success + sleep = LONG_SLEEP; logInfo.setSuccess(true); } logInfo.sleep = sleep; @@ -215,7 +217,6 @@ private boolean checkLeaderStatus(HarvestState state) { log.info("EVICTING " + state.instanceID + " because age " + age + " > " + EVICT_AGE); state.instanceID = instanceID; stateDAO.put(state); - state = stateDAO.get(state.getID()); leader = true; } } From 419ef8306cdfbe8818afcb21b290864ed209bdfa Mon Sep 17 00:00:00 2001 From: Patrick Dowler Date: Tue, 26 Mar 2024 14:19:01 -0700 Subject: [PATCH 15/18] vault: cleanup and doc update --- vault/README.md | 37 ++++------ .../opencadc/vault/ServiceAvailability.java | 8 +-- .../org/opencadc/vault/VaultInitAction.java | 29 +++++--- vault/src/main/webapp/WEB-INF/web.xml | 70 +++++++++---------- 4 files changed, 74 insertions(+), 70 deletions(-) diff --git a/vault/README.md b/vault/README.md index 15323212..f2ef1086 100644 --- a/vault/README.md +++ b/vault/README.md @@ -41,7 +41,6 @@ org.opencadc.vault.nodes.password={password for vospace pool} org.opencadc.vault.nodes.url=jdbc:postgresql://{server}/{database} org.opencadc.vault.inventory.maxActive={max connections for inventory pool} -# optional: config for separate inventory pool org.opencadc.vault.inventory.username={username for inventory pool} org.opencadc.vault.inventory.password={password for inventory pool} org.opencadc.vault.inventory.url=jdbc:postgresql://{server}/{database} @@ -58,13 +57,12 @@ VOSI-availability output. The _inventory_ account owns and manages (create, alter, drop) inventory database objects and manages all the content (update and delete Artifact, insert DeletedArtifactEvent). The database is specified -in the JDBC URL and the schema name is specified in the minoc.properties (below). Failure to connect or +in the JDBC URL and the schema name is specified in the vault.properties (below). Failure to connect or initialize the database will show up in logs and in the VOSI-availability output. The _inventory_ content may be in the same database as the _nodes_, in a different database in the same server, or in a different -server entirely. See `org.opencadc.vault.singlePool` below for the pros and cons. Note: it is a good -idea to set `maxActive` to a valid integer (e.g. 1 because the tomcat connection pool doesn't like 0 and -decides to make it 100 instead) when using a single pool; this avoids an ugly but meaningless stack trace -in the logs at startup. +server entirely. See `org.opencadc.vault.singlePool` below for the pros and cons. The _inventory_ pool must +be functional for initialization, availability checks (`maxActive` = 1 with `singlePool` is sufficient), and +the connection information is re-used by an internal background thread that synchronizes data node sizes. The _uws_ account owns and manages (create, alter, drop) uws database objects in the `uws` schema and manages all the content (insert, update, delete). The database is specified in the JDBC URLFailure to connect or initialize the @@ -128,28 +126,21 @@ DeletedArtifactEvent are done in a separate transaction and if that fails the Ar orphaned until the vault validation (see ???) runs and fixes such a discrepancy. However, _singlePool_ = `false` allows the content to be stored in two separate databases or servers. -The _root.owner_ owns the root node and has full read and write permission in the root container, so it can -create and delete container nodes at the root and assign container node properties that are normally read-only -to normal users: owner, quota, etc. This must be set to the username of the admin. +The _root.owner_ key configures the owner of the root node; the owner and has full read and write permission +in the root container, so it can create and delete container nodes at the root and assign container node properties +that are normally read-only to normal users: owner, quota, etc. This must be set to the username of the admin. -The _storage.namespace_ configures `vault` to use the specified namespace in storage-inventory to store files. +The _storage.namespace_ key configures `vault` to use the specified namespace in storage-inventory to store files. This only applies to new data nodes that are created and will not effect previously created nodes and artifacts. Probably don't want to change this... prevent change? TBD. -### vault-availability.properties (optional) +### cadc-log.properties (optional) +See cadc-log for common +dynamic logging control. -The vault-availability.properties file specifies which users have the authority to change the availability state of -the vault service. Each entry consists of a key=value pair. The key is always "users". The value is the x500 canonical -user name. - -Example: -``` -users = {user identity} -``` -`users` specifies the user(s) who are authorized to make calls to the service. The value is a list of user -identities (X500 distingushed name), one line per user. Optional: if the `vault-availability.properties` is -not found or does not list any `users`, the service will function in the default mode (ReadWrite) and the -state will not be changeable. +### cadc-vosi.properties (optional) +See cadc-vosi for common +service state control. ## building it ``` diff --git a/vault/src/main/java/org/opencadc/vault/ServiceAvailability.java b/vault/src/main/java/org/opencadc/vault/ServiceAvailability.java index b1e53f10..5a76fb1b 100644 --- a/vault/src/main/java/org/opencadc/vault/ServiceAvailability.java +++ b/vault/src/main/java/org/opencadc/vault/ServiceAvailability.java @@ -201,15 +201,15 @@ private String getState() { } return ret; } - + private void setOffline(boolean offline) { - String jndiArtifactSync = appName + "-" + DataNodeSizeSync.class.getName(); + String jndiKey = appName + "-" + DataNodeSizeSync.class.getName(); try { InitialContext initialContext = new InitialContext(); - DataNodeSizeSync async = (DataNodeSizeSync) initialContext.lookup(jndiArtifactSync); + DataNodeSizeSync async = (DataNodeSizeSync) initialContext.lookup(jndiKey); async.setOffline(offline); } catch (NamingException e) { - log.debug(String.format("unable to unbind %s - %s", jndiArtifactSync, e.getMessage())); + log.debug(String.format("unable to find %s - %s", jndiKey, e.getMessage())); } } } diff --git a/vault/src/main/java/org/opencadc/vault/VaultInitAction.java b/vault/src/main/java/org/opencadc/vault/VaultInitAction.java index b61f3470..a80f0452 100644 --- a/vault/src/main/java/org/opencadc/vault/VaultInitAction.java +++ b/vault/src/main/java/org/opencadc/vault/VaultInitAction.java @@ -69,6 +69,7 @@ import ca.nrc.cadc.db.DBUtil; import ca.nrc.cadc.rest.InitAction; +import ca.nrc.cadc.rest.RestAction; import ca.nrc.cadc.util.InvalidConfigException; import ca.nrc.cadc.util.MultiValuedProperties; import ca.nrc.cadc.util.PropertiesReader; @@ -131,12 +132,11 @@ public class VaultInitAction extends InitAction { private Namespace storageNamespace; private Map vosDaoConfig; private Map invDaoConfig; - private List allocationParents = new ArrayList<>(); - private String jndiNodePersistence; + private String jndiNodePersistence; // store in JNDI for cadc-vos-server lib private String jndiPreauthKeys; // store pubkey in JNDI for download via GetKeyAction - private String jndiSiteAvailabilities; + private String jndiSiteAvailabilities; // store in JNDI to share with ProtocolsGenerator private Thread availabilityCheck; private String jndiDataNodeSizeSync; // store in JNDI to support availability mode change @@ -149,8 +149,9 @@ public VaultInitAction() { @Override public void doInit() { initConfig(); - initDatabase(); - initUWSDatabase(); + initDatabaseVOS(); + initDatabaseINV(); + initDatabaseUWS(); initNodePersistence(); initKeyPair(); initAvailabilityCheck(); @@ -324,7 +325,7 @@ private void initConfig() { } } - private void initDatabase() { + private void initDatabaseVOS() { try { String dsname = (String) vosDaoConfig.get("jndiDataSourceName"); String schema = (String) vosDaoConfig.get("vosSchema"); @@ -336,7 +337,9 @@ private void initDatabase() { } catch (Exception ex) { throw new IllegalStateException("check/init vospace database failed", ex); } - + } + + private void initDatabaseINV() { try { String dsname = (String) invDaoConfig.get("jndiDataSourceName"); String schema = (String) invDaoConfig.get("invSchema"); @@ -350,7 +353,7 @@ private void initDatabase() { } } - private void initUWSDatabase() { + private void initDatabaseUWS() { try { log.info("initDatabase: " + JNDI_UWS_DATASOURCE + " uws START"); DataSource uws = DBUtil.findJNDIDataSource(JNDI_UWS_DATASOURCE); @@ -466,9 +469,19 @@ private void initBackgroundWorkers() { Map iterprops = getIteratorConfig(props); log.warn("iterator pool: " + iterprops.get("jndiDataSourceName")); artifactDAO.setConfig(iterprops); + + // determine startup mode + boolean offline = false; // normal + String key = appName + RestAction.STATE_MODE_KEY; + String ret = System.getProperty(key); + if (ret != null + && (RestAction.STATE_READ_ONLY.equals(ret) || RestAction.STATE_OFFLINE.equals(ret))) { + offline = true; + } terminateBackgroundWorkers(); DataNodeSizeSync async = new DataNodeSizeSync(hsDAO, artifactDAO, storageNamespace); + async.setOffline(offline); this.dataNodeSizeSyncThread = new Thread(async); dataNodeSizeSyncThread.setDaemon(true); dataNodeSizeSyncThread.start(); diff --git a/vault/src/main/webapp/WEB-INF/web.xml b/vault/src/main/webapp/WEB-INF/web.xml index fffacd30..f57bad3b 100644 --- a/vault/src/main/webapp/WEB-INF/web.xml +++ b/vault/src/main/webapp/WEB-INF/web.xml @@ -37,6 +37,40 @@ 1 + + + AvailabilityServlet + ca.nrc.cadc.vosi.AvailabilityServlet + + ca.nrc.cadc.vosi.AvailabilityPlugin + org.opencadc.vault.ServiceAvailability + + 2 + + + + + CapabilitiesServlet + ca.nrc.cadc.rest.RestServlet + + init + ca.nrc.cadc.vosi.CapInitAction + + + head + ca.nrc.cadc.vosi.CapHeadAction + + + get + ca.nrc.cadc.vosi.CapGetAction + + + input + /capabilities.xml + + 2 + + NodesServlet ca.nrc.cadc.rest.RestServlet @@ -64,7 +98,7 @@ delete org.opencadc.vospace.server.actions.DeleteNodeAction - 2 + 3 @@ -208,40 +242,6 @@ - - - CapabilitiesServlet - ca.nrc.cadc.rest.RestServlet - - init - ca.nrc.cadc.vosi.CapInitAction - - - head - ca.nrc.cadc.vosi.CapHeadAction - - - get - ca.nrc.cadc.vosi.CapGetAction - - - input - /capabilities.xml - - 3 - - - - - AvailabilityServlet - ca.nrc.cadc.vosi.AvailabilityServlet - - ca.nrc.cadc.vosi.AvailabilityPlugin - org.opencadc.vault.ServiceAvailability - - 3 - - From 15b86dd22b10145f8457d2388a8ce9b63fb5beb6 Mon Sep 17 00:00:00 2001 From: Patrick Dowler Date: Thu, 28 Mar 2024 12:35:36 -0700 Subject: [PATCH 16/18] vault: improve server header info for files endpoint --- .../main/java/org/opencadc/vault/files/HeadAction.java | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/vault/src/main/java/org/opencadc/vault/files/HeadAction.java b/vault/src/main/java/org/opencadc/vault/files/HeadAction.java index 69b68f00..902c4c9a 100644 --- a/vault/src/main/java/org/opencadc/vault/files/HeadAction.java +++ b/vault/src/main/java/org/opencadc/vault/files/HeadAction.java @@ -70,6 +70,7 @@ import ca.nrc.cadc.net.ResourceNotFoundException; import ca.nrc.cadc.rest.InlineContentHandler; import ca.nrc.cadc.rest.RestAction; +import ca.nrc.cadc.rest.Version; import java.net.URI; import java.net.URISyntaxException; import java.util.Date; @@ -107,6 +108,13 @@ protected final InlineContentHandler getInlineContentHandler() { return null; } + @Override + protected String getServerImpl() { + // no null version checking because fail to build correctly can't get past basic testing + Version v = getVersionFromResource(); + return "storage-inventory/vault-" + v.getMajorMinor(); + } + @Override public void initAction() throws Exception { String jndiNodePersistence = super.appName + "-" + NodePersistence.class.getName(); From 8c91699d7bedca367c39d4abcacb316ef27eddfa Mon Sep 17 00:00:00 2001 From: Patrick Dowler Date: Tue, 2 Apr 2024 12:48:14 -0700 Subject: [PATCH 17/18] rework after review --- ChangeLog.md | 12 ++++++------ .../org/opencadc/vospace/db/DataNodeSizeWorker.java | 5 +++-- minoc/README.md | 9 +++++---- .../opencadc/vault/metadata/DataNodeSizeSync.java | 1 - 4 files changed, 14 insertions(+), 13 deletions(-) diff --git a/ChangeLog.md b/ChangeLog.md index 7fca1ef3..77d824e3 100644 --- a/ChangeLog.md +++ b/ChangeLog.md @@ -17,13 +17,13 @@ added optional `org.opencadc.minoc.readable` and `org.opencadc.minoc.writable` c A `minoc` service will advertise (via inventory.StorageSite record in the database) the _readable_ and _writable_ status; this information is synced to global inventory and used by `raven` to determine if it should generate PUT or GET URLs that use the `minoc` -service(s) at that site. By default, the configuration of _readGrantProvider_(s) and -_writeGrantProvider_(s) is used to determine the default status; configuration of any -_trust.preauth_ will also make make the status _readable_ and _writable_ even if the -service is only intending to accept one or the other. +service(s) at that site. The configuration of _readGrantProvider_(s) and +_writeGrantProvider_(s) implicitly determines the status (_readable_ and _writable_ +respectively); configuration of any _trust.preauth_ will also implicitly make make the +status _readable_ and _writable_. -The explicit _readable_ and _writable_ configuration options will override all other -logic and set the status accordingly. This is currently optional but may be required +The explicit _readable_ and _writable_ configuration options will override the above +implicit logic and set the status accordingly. This is currently optional but may be required in a future version. ``` diff --git a/cadc-inventory-db/src/main/java/org/opencadc/vospace/db/DataNodeSizeWorker.java b/cadc-inventory-db/src/main/java/org/opencadc/vospace/db/DataNodeSizeWorker.java index 236da5ca..5bfcb30e 100644 --- a/cadc-inventory-db/src/main/java/org/opencadc/vospace/db/DataNodeSizeWorker.java +++ b/cadc-inventory-db/src/main/java/org/opencadc/vospace/db/DataNodeSizeWorker.java @@ -134,7 +134,8 @@ public void run() { + " start=" + df.format(harvestState.curLastModified)); } else { log.debug(opName + " source=" + harvestState.getResourceID() - + " instance=" + harvestState.instanceID); + + " instance=" + harvestState.instanceID + + " start=null"); } final Date now = new Date(); @@ -193,7 +194,7 @@ public void run() { } else { log.debug(opName + " source=" + harvestState.getResourceID() + " instance=" + harvestState.instanceID - + " end=true"); + + " end=null"); } } diff --git a/minoc/README.md b/minoc/README.md index e0292b94..99831467 100644 --- a/minoc/README.md +++ b/minoc/README.md @@ -91,7 +91,8 @@ org.opencadc.minoc.recoverableNamespace = {namespace} The optional _trust.preauth_ key(s) configure `minoc` to trust external service(s) to have performed authorization checks. Such services may include a signed token in the URL and `minoc` will validate the request using a public key retrieved from the service instead of performing authorization checks -itself. Example: +itself. Currently, only `raven` and `vault` can generate such URLs and provide access to their +public keys. Example: ``` # trust a SI global inventory org.opencadc.minoc.trust.preauth = ivo://example.net/raven @@ -99,13 +100,13 @@ org.opencadc.minoc.trust.preauth = ivo://example.net/raven # trust a SI VOSpace service org.opencadc.minoc.trust.preauth = ivo://example.net/vault ``` -Setting _trust.preauth_ one or more times also implies _readable_ and _writable_ are _true_. +Setting _trust.preauth_ one or more times implicitly sets _readable_ and _writable_ to _true_. The optional _readGrantProvider_ and _writeGrantProvider_ keys configure minoc to call other services to get grants (permissions) for operations. Multiple values of the granting service resourceID(s) may be provided by including multiple property settings (one per line). All services will be consulted but a single positive -result is sufficient to grant permission for an action. Setting these values also sets the implied _readable_ -and _writable_ is _true_ respectively. +result is sufficient to grant permission for an action. Setting these values implicitly sets _readable_ +and _writable_ to _true_ respectively. The optional _readable_ and _writable_ keys configure minoc explicitly rather than relying on one or more of the above trust or grant provider settings. For example, this allows one to configure a read-only minoc diff --git a/vault/src/main/java/org/opencadc/vault/metadata/DataNodeSizeSync.java b/vault/src/main/java/org/opencadc/vault/metadata/DataNodeSizeSync.java index a63eb83c..3bce32e9 100644 --- a/vault/src/main/java/org/opencadc/vault/metadata/DataNodeSizeSync.java +++ b/vault/src/main/java/org/opencadc/vault/metadata/DataNodeSizeSync.java @@ -157,7 +157,6 @@ public void run() { worker.run(); logInfo.setLastModified(state.curLastModified); logInfo.processed = worker.getNumArtifactsProcessed(); - logInfo.setSuccess(true); } catch (Exception ex) { log.error("unexpected worker fail", ex); fail = true; From 47800127fbf01bb0ad06a2d867776333a29f7446 Mon Sep 17 00:00:00 2001 From: Patrick Dowler Date: Tue, 2 Apr 2024 13:48:28 -0700 Subject: [PATCH 18/18] remaining SI image version to 1.0.0 --- critwall/VERSION | 2 +- fenwick/VERSION | 2 +- ratik/VERSION | 2 +- tantar/VERSION | 2 +- vault/VERSION | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/critwall/VERSION b/critwall/VERSION index 7fbddd11..d849b8f3 100644 --- a/critwall/VERSION +++ b/critwall/VERSION @@ -1,6 +1,6 @@ ## deployable containers have a semantic and build tag # semantic version tag: major.minor[.patch] # build version tag: timestamp -VER=0.5.0 +VER=1.0.0 TAGS="${VER} ${VER}-$(date --utc +"%Y%m%dT%H%M%S")" unset VER diff --git a/fenwick/VERSION b/fenwick/VERSION index a7748476..38b1547a 100644 --- a/fenwick/VERSION +++ b/fenwick/VERSION @@ -4,6 +4,6 @@ # tags with and without build number so operators use the versioned # tag but we always keep a timestamped tag in case a semantic tag gets # replaced accidentally -VER=0.6.0 +VER=1.0.0 TAGS="${VER} ${VER}-$(date --utc +"%Y%m%dT%H%M%S")" unset VER diff --git a/ratik/VERSION b/ratik/VERSION index d9e5d604..38b1547a 100644 --- a/ratik/VERSION +++ b/ratik/VERSION @@ -4,6 +4,6 @@ # tags with and without build number so operators use the versioned # tag but we always keep a timestamped tag in case a semantic tag gets # replaced accidentally -VER=0.2.0 +VER=1.0.0 TAGS="${VER} ${VER}-$(date --utc +"%Y%m%dT%H%M%S")" unset VER diff --git a/tantar/VERSION b/tantar/VERSION index 7fbddd11..d849b8f3 100644 --- a/tantar/VERSION +++ b/tantar/VERSION @@ -1,6 +1,6 @@ ## deployable containers have a semantic and build tag # semantic version tag: major.minor[.patch] # build version tag: timestamp -VER=0.5.0 +VER=1.0.0 TAGS="${VER} ${VER}-$(date --utc +"%Y%m%dT%H%M%S")" unset VER diff --git a/vault/VERSION b/vault/VERSION index 17ac68c1..3d4fab56 100644 --- a/vault/VERSION +++ b/vault/VERSION @@ -4,6 +4,6 @@ # tags with and without build number so operators use the versioned # tag but we always keep a timestamped tag in case a semantic tag gets # replaced accidentally -VER=0.5.0 +VER=1.0.0 TAGS="${VER} ${VER}-$(date --utc +"%Y%m%dT%H%M%S")" unset VER