diff --git a/.github/workflows/gradle.yml b/.github/workflows/gradle.yml index 25d39bd68..64748cf5b 100644 --- a/.github/workflows/gradle.yml +++ b/.github/workflows/gradle.yml @@ -66,19 +66,11 @@ jobs: - name: java build -- raven run: cd raven && ../gradlew --info clean build javadoc checkstyleMain - - - name: java build -- ringhold - run: cd ringhold && ../gradlew --info clean build javadoc checkstyleMain - name: java build -- ratik run: cd ratik && ../gradlew --info clean build javadoc checkstyleMain -## TODO: docker build depends on cadc-tomcat base image from docker-base.git -# - name: docker build -- baldur -# run: cd baldur && docker build . --file Dockerfile --tag baldur:$(date +%s) -# - name: docker build -- minoc -# run: cd minoc && docker build . --file Dockerfile --tag minoc:$(date +%s) -# - name: docker build -- luskan -# run: cd luskan && docker build . --file Dockerfile --tag luskan:$(date +%s) -# - name: docker build -- raven -# run: cd raven && docker build . --file Dockerfile --tag raven:$(date +%s) +## disabled until updated for cadc-inventory-db API changes +# - name: java build -- ringhold +# run: cd ringhold && ../gradlew --info clean build javadoc checkstyleMain + diff --git a/ChangeLog.md b/ChangeLog.md new file mode 100644 index 000000000..77d824e33 --- /dev/null +++ b/ChangeLog.md @@ -0,0 +1,70 @@ +# Change Log + +This is a cursory summary of changes to various storage-inventory components. +Check the README in specific modules for details. + +## minoc:1.0.0 +``` +added optional `org.opencadc.minoc.trust.preauth` config +removed optional `org.opencadc.minoc.publicKeyFile` config +``` +A `minoc` instance will download a public key from each trusted service and +use the public key(s) to validate URLs that include a _preauth_ token. + +``` +added optional `org.opencadc.minoc.readable` and `org.opencadc.minoc.writable` config +``` +A `minoc` service will advertise (via inventory.StorageSite record in the database) the +_readable_ and _writable_ status; this information is synced to global inventory and +used by `raven` to determine if it should generate PUT or GET URLs that use the `minoc` +service(s) at that site. The configuration of _readGrantProvider_(s) and +_writeGrantProvider_(s) implicitly determines the status (_readable_ and _writable_ +respectively); configuration of any _trust.preauth_ will also implicitly make make the +status _readable_ and _writable_. + +The explicit _readable_ and _writable_ configuration options will override the above +implicit logic and set the status accordingly. This is currently optional but may be required +in a future version. + +``` +added optional config file: cadc-log.properties +added optional config file: cadc-vosi.properties +``` + +## raven:1.0.0 +``` +added org.opencadc.raven.inventory connection pool +``` +A `raven` service uses this pool to perform database initialization. This pool is +configured in the `catalina.properties` file. + +``` +added optional `org.opencadc.raven.keys.preauth` config +removed optional `org.opencadc.raven.publicKeyFile` and `org.opencadc.minoc.privateKeyFile` config +``` +When configured to do so, a `raven` service will generate it's own public/private key pair +and use the private key to _sign_ `minoc` URLs. All the `minoc` services known to the global +`raven` service must also be configured to _trust_ `raven`. + +``` +added optional config file: cadc-log.properties +added optional config file: cadc-vosi.properties +``` + +## luskan:1.0.0 +``` +changed config file: cadc-tap-tmp.properties +``` +A `luskan` service now uses the DelegatingStorageManager` so this config file must +specify which storage manager implementation to use along with existing +implementation-specific configuration options. + +``` +added optional config file: cadc-log.properties +added optional config file: cadc-vosi.properties +``` + +## vault:0.5.0 (NEW) +This is a new service that implements the IVOA VOSpace 2.1 REST API to provide user-managed +hierarchical storage. `vault` is deployed with it's own database and an associated inventory +database, uses inventory services (`minoc`) for file storage and management. diff --git a/cadc-inventory-db/build.gradle b/cadc-inventory-db/build.gradle index 1b5a7ca8f..91e5e51d8 100644 --- a/cadc-inventory-db/build.gradle +++ b/cadc-inventory-db/build.gradle @@ -17,7 +17,7 @@ sourceCompatibility = 1.8 group = 'org.opencadc' -version = '0.15.0' +version = '1.0.0' description = 'OpenCADC Storage Inventory database library' def git_url = 'https://github.com/opencadc/storage-inventory' @@ -27,8 +27,8 @@ mainClassName = 'org.opencadc.inventory.db.version.Main' dependencies { compile 'org.opencadc:cadc-util:[1.11.0,2.0)' compile 'org.opencadc:cadc-gms:[1.0.0,)' - compile 'org.opencadc:cadc-inventory:[0.9.4,)' - compile 'org.opencadc:cadc-vos:[2.0.5,3.0)' + compile 'org.opencadc:cadc-inventory:[1.0.0,)' + compile 'org.opencadc:cadc-vos:[2.0.6,3.0)' testCompile 'junit:junit:[4.0,)' diff --git a/cadc-inventory-db/src/intTest/java/org/opencadc/vospace/db/ArtifactSyncWorkerTest.java b/cadc-inventory-db/src/intTest/java/org/opencadc/vospace/db/DataNodeSizeWorkerTest.java similarity index 97% rename from cadc-inventory-db/src/intTest/java/org/opencadc/vospace/db/ArtifactSyncWorkerTest.java rename to cadc-inventory-db/src/intTest/java/org/opencadc/vospace/db/DataNodeSizeWorkerTest.java index fa18912c6..649800b31 100644 --- a/cadc-inventory-db/src/intTest/java/org/opencadc/vospace/db/ArtifactSyncWorkerTest.java +++ b/cadc-inventory-db/src/intTest/java/org/opencadc/vospace/db/DataNodeSizeWorkerTest.java @@ -84,7 +84,7 @@ import org.junit.Before; import org.junit.Test; import org.opencadc.inventory.Artifact; -import org.opencadc.vospace.db.ArtifactSyncWorker; +import org.opencadc.vospace.db.DataNodeSizeWorker; import org.opencadc.inventory.Namespace; import org.opencadc.inventory.db.ArtifactDAO; import org.opencadc.inventory.db.HarvestState; @@ -101,8 +101,8 @@ * * @author adriand */ -public class ArtifactSyncWorkerTest { - private static final Logger log = Logger.getLogger(ArtifactSyncWorkerTest.class); +public class DataNodeSizeWorkerTest { + private static final Logger log = Logger.getLogger(DataNodeSizeWorkerTest.class); static { Log4jInit.setLevel("org.opencadc.inventory", Level.INFO); @@ -117,7 +117,7 @@ public class ArtifactSyncWorkerTest { ArtifactDAO artifactDAO; - public ArtifactSyncWorkerTest() throws Exception { + public DataNodeSizeWorkerTest() throws Exception { DBConfig dbrc = new DBConfig(); ConnectionConfig cc = dbrc.getConnectionConfig(TestUtil.SERVER, TestUtil.DATABASE); DBUtil.PoolConfig pool = new DBUtil.PoolConfig(cc, 1, 6000L, "select 123"); @@ -219,7 +219,7 @@ public void testSyncArtifact() throws Exception { harvestStateDAO.put(hs); hs = harvestStateDAO.get(hsName, resourceID); - ArtifactSyncWorker asWorker = new ArtifactSyncWorker(harvestStateDAO, hs, artifactDAO, siNamespace); + DataNodeSizeWorker asWorker = new DataNodeSizeWorker(harvestStateDAO, hs, artifactDAO, siNamespace); asWorker.run(); actual = (DataNode)nodeDAO.get(orig.getID()); diff --git a/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/HarvestStateDAO.java b/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/HarvestStateDAO.java index 5df563354..a0f8d8f74 100644 --- a/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/HarvestStateDAO.java +++ b/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/HarvestStateDAO.java @@ -68,6 +68,7 @@ package org.opencadc.inventory.db; import java.net.URI; +import java.sql.Connection; import java.sql.SQLException; import java.util.UUID; import org.apache.log4j.Logger; @@ -194,13 +195,15 @@ public void put(HarvestState val, boolean forceTimestampUpdate) { if (curMaintCount == maintCount) { String sql = "VACUUM " + gen.getTable(HarvestState.class); log.warn("maintenance: " + curMaintCount + "==" + maintCount + " " + sql); - //JdbcTemplate jdbc = new JdbcTemplate(dataSource); - //jdbc.execute(sql); try { - dataSource.getConnection().createStatement().execute(sql); - } catch (SQLException ex) { - log.error("ERROR: " + sql + " FAILED", ex); - // yes, log and proceed + try (Connection c = dataSource.getConnection()) { + c.createStatement().execute(sql); + } catch (SQLException ex) { + log.error("maintenance failed: " + sql, ex); + // yes, log and proceed + } // auto-close to return to pool + } catch (Exception ex) { + log.error("failed to close connection after maintenance: " + sql, ex); } curMaintCount = 0; } else { diff --git a/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/version/InitDatabaseSI.java b/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/version/InitDatabaseSI.java index 6e810b8f3..f59d7d767 100644 --- a/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/version/InitDatabaseSI.java +++ b/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/version/InitDatabaseSI.java @@ -80,7 +80,7 @@ public class InitDatabaseSI extends InitDatabase { private static final Logger log = Logger.getLogger(InitDatabaseSI.class); public static final String MODEL_NAME = "storage-inventory"; - public static final String MODEL_VERSION = "0.15"; + public static final String MODEL_VERSION = "1.0.0"; public static final String PREV_MODEL_VERSION = "0.14"; //public static final String PREV_MODEL_VERSION = "DO-NOT_UPGRADE-BY-ACCIDENT"; @@ -98,7 +98,7 @@ public class InitDatabaseSI extends InitDatabase { }; static String[] UPGRADE_SQL = new String[] { - "inventory.upgrade-0.15.sql", + "inventory.upgrade-1.0.0.sql", "generic.PreauthKeyPair.sql", "generic.permissions.sql" }; diff --git a/cadc-inventory-db/src/main/java/org/opencadc/vospace/db/ArtifactSyncWorker.java b/cadc-inventory-db/src/main/java/org/opencadc/vospace/db/DataNodeSizeWorker.java similarity index 64% rename from cadc-inventory-db/src/main/java/org/opencadc/vospace/db/ArtifactSyncWorker.java rename to cadc-inventory-db/src/main/java/org/opencadc/vospace/db/DataNodeSizeWorker.java index 29e80384f..5bfcb30e3 100644 --- a/cadc-inventory-db/src/main/java/org/opencadc/vospace/db/ArtifactSyncWorker.java +++ b/cadc-inventory-db/src/main/java/org/opencadc/vospace/db/DataNodeSizeWorker.java @@ -67,9 +67,12 @@ package org.opencadc.vospace.db; +import ca.nrc.cadc.date.DateUtil; import ca.nrc.cadc.db.TransactionManager; import ca.nrc.cadc.io.ResourceIterator; import java.io.IOException; +import java.text.DateFormat; +import java.util.Date; import org.apache.log4j.Logger; import org.opencadc.inventory.Artifact; import org.opencadc.inventory.Namespace; @@ -77,24 +80,38 @@ import org.opencadc.inventory.db.HarvestState; import org.opencadc.inventory.db.HarvestStateDAO; import org.opencadc.vospace.DataNode; -import org.opencadc.vospace.db.NodeDAO; /** - * This class performs the work of synchronizing the size of Data Nodes between backend storage and Node Persistence + * This class performs the work of synchronizing the size of Data Nodes from + * inventory (Artifact) to vopsace (Node). * * @author adriand */ -public class ArtifactSyncWorker implements Runnable { - private static final Logger log = Logger.getLogger(ArtifactSyncWorker.class); +public class DataNodeSizeWorker implements Runnable { + private static final Logger log = Logger.getLogger(DataNodeSizeWorker.class); + + // lookback when doing incremental harvest because head of sequence is + // not monotonic over short timescales (events arrive out of sequence) + private static final long LOOKBACK_TIME_MS = 60 * 1000L; private final HarvestState harvestState; private final NodeDAO nodeDAO; private final ArtifactDAO artifactDAO; private final HarvestStateDAO harvestStateDAO; private final Namespace storageNamespace; + + private long numArtifactsProcessed; - public ArtifactSyncWorker(HarvestStateDAO harvestStateDAO, HarvestState harvestState, ArtifactDAO artifactDAO, - Namespace namespace) { + /** + * Worker constructor. + * + * @param harvestStateDAO DAO class to persist progress in the vospace database + * @param harvestState current HarvestState instance + * @param artifactDAO DAO class to query for artifacts + * @param namespace artifact namespace + */ + public DataNodeSizeWorker(HarvestStateDAO harvestStateDAO, HarvestState harvestState, + ArtifactDAO artifactDAO, Namespace namespace) { this.harvestState = harvestState; this.harvestStateDAO = harvestStateDAO; this.nodeDAO = new NodeDAO(harvestStateDAO); @@ -102,16 +119,40 @@ public ArtifactSyncWorker(HarvestStateDAO harvestStateDAO, HarvestState harvestS this.storageNamespace = namespace; } + public long getNumArtifactsProcessed() { + return numArtifactsProcessed; + } + @Override public void run() { - log.debug("Start harvesting " + harvestState.toString() + " at " + harvestState.curLastModified); + this.numArtifactsProcessed = 0L; + String opName = DataNodeSizeWorker.class.getSimpleName() + ".artifactQuery"; + DateFormat df = DateUtil.getDateFormat(DateUtil.IVOA_DATE_FORMAT, DateUtil.UTC); + if (harvestState.curLastModified != null) { + log.debug(opName + " source=" + harvestState.getResourceID() + + " instance=" + harvestState.instanceID + + " start=" + df.format(harvestState.curLastModified)); + } else { + log.debug(opName + " source=" + harvestState.getResourceID() + + " instance=" + harvestState.instanceID + + " start=null"); + } + + final Date now = new Date(); + final Date lookBack = new Date(now.getTime() - LOOKBACK_TIME_MS); + Date startTime = getQueryLowerBound(lookBack, harvestState.curLastModified); + if (lookBack != null && harvestState.curLastModified != null) { + log.debug("lookBack=" + df.format(lookBack) + " curLastModified=" + df.format(harvestState.curLastModified) + + " -> " + df.format(startTime)); + } - TransactionManager tm = nodeDAO.getTransactionManager(); - try (final ResourceIterator iter = artifactDAO.iterator(storageNamespace, null, - harvestState.curLastModified, true)) { + String uriBucket = null; // process all artifacts in a single thread + try (final ResourceIterator iter = artifactDAO.iterator(storageNamespace, uriBucket, startTime, true)) { + TransactionManager tm = nodeDAO.getTransactionManager(); while (iter.hasNext()) { Artifact artifact = iter.next(); DataNode node = nodeDAO.getDataNode(artifact.getURI()); + log.debug(artifact.getURI() + " len=" + artifact.getContentLength() + " -> " + node.getName()); if (node != null && !artifact.getContentLength().equals(node.bytesUsed)) { tm.startTransaction(); try { @@ -122,7 +163,8 @@ public void run() { node.bytesUsed = artifact.getContentLength(); nodeDAO.put(node); tm.commitTransaction(); - log.debug("Updated size of data node " + node.getName()); + log.debug("ArtifactSyncWorker.updateDataNode id=" + node.getID() + + " bytesUsed=" + node.bytesUsed + " artifact.lastModified=" + df.format(artifact.getLastModified())); } catch (Exception ex) { log.debug("Failed to update data node size for " + node.getName(), ex); tm.rollbackTransaction(); @@ -137,13 +179,38 @@ public void run() { } } harvestState.curLastModified = artifact.getLastModified(); - harvestState.curID = node.getID(); + harvestState.curID = artifact.getID(); harvestStateDAO.put(harvestState); + numArtifactsProcessed++; } } catch (IOException ex) { log.error("Error closing iterator", ex); throw new RuntimeException("error while closing ResourceIterator", ex); } - log.debug("End harvesting " + harvestState.toString() + " at " + harvestState.curLastModified); + if (harvestState.curLastModified != null) { + log.debug(opName + " source=" + harvestState.getResourceID() + + " instance=" + harvestState.instanceID + + " end=" + df.format(harvestState.curLastModified)); + } else { + log.debug(opName + " source=" + harvestState.getResourceID() + + " instance=" + harvestState.instanceID + + " end=null"); + } + } + + private Date getQueryLowerBound(Date lookBack, Date lastModified) { + if (lookBack == null) { + // feature not enabled + return lastModified; + } + if (lastModified == null) { + // first harvest + return null; + } + if (lookBack.before(lastModified)) { + return lookBack; + } + return lastModified; + } } diff --git a/cadc-inventory-db/src/main/java/org/opencadc/vospace/db/InitDatabaseVOS.java b/cadc-inventory-db/src/main/java/org/opencadc/vospace/db/InitDatabaseVOS.java index 05d8c45b4..40d7f29de 100644 --- a/cadc-inventory-db/src/main/java/org/opencadc/vospace/db/InitDatabaseVOS.java +++ b/cadc-inventory-db/src/main/java/org/opencadc/vospace/db/InitDatabaseVOS.java @@ -79,9 +79,9 @@ public class InitDatabaseVOS extends ca.nrc.cadc.db.version.InitDatabase { private static final Logger log = Logger.getLogger(InitDatabaseVOS.class); - public static final String MODEL_NAME = "vospace-inventory"; - public static final String MODEL_VERSION = "0.15"; - public static final String PREV_MODEL_VERSION = "0.3"; + public static final String MODEL_NAME = "storage-vospace"; + public static final String MODEL_VERSION = "1.0.0"; + public static final String PREV_MODEL_VERSION = "n/a"; static String[] CREATE_SQL = new String[] { "generic.ModelVersion.sql", @@ -93,7 +93,6 @@ public class InitDatabaseVOS extends ca.nrc.cadc.db.version.InitDatabase { }; static String[] UPGRADE_SQL = new String[] { - "vospace.upgrade-0.15.sql", "generic.permissions.sql" }; diff --git a/cadc-inventory-db/src/main/resources/inventory.upgrade-0.15.sql b/cadc-inventory-db/src/main/resources/inventory.upgrade-1.0.0.sql similarity index 100% rename from cadc-inventory-db/src/main/resources/inventory.upgrade-0.15.sql rename to cadc-inventory-db/src/main/resources/inventory.upgrade-1.0.0.sql diff --git a/cadc-inventory-db/src/main/resources/vospace.upgrade-0.15.sql b/cadc-inventory-db/src/main/resources/vospace.upgrade-0.15.sql deleted file mode 100644 index bfb33166e..000000000 --- a/cadc-inventory-db/src/main/resources/vospace.upgrade-0.15.sql +++ /dev/null @@ -1,7 +0,0 @@ - -alter table .Node - add column bytesUsed bigint, - add column storageBucket varchar(5) -; - -create unique index node_storageID on .Node(storageID); \ No newline at end of file diff --git a/cadc-inventory/build.gradle b/cadc-inventory/build.gradle index 55f43e500..cfe772697 100644 --- a/cadc-inventory/build.gradle +++ b/cadc-inventory/build.gradle @@ -14,13 +14,13 @@ sourceCompatibility = 1.8 group = 'org.opencadc' -version = '0.10.0' +version = '1.0.0' description = 'OpenCADC Storage Inventory core library' def git_url = 'https://github.com/opencadc/storage-inventory' dependencies { - compile 'org.opencadc:cadc-util:[1.10.6,2.0)' + compile 'org.opencadc:cadc-util:[1.11.0,2.0)' testCompile 'junit:junit:[4.0,)' } diff --git a/critwall/VERSION b/critwall/VERSION index ba99f57c6..d849b8f34 100644 --- a/critwall/VERSION +++ b/critwall/VERSION @@ -1,6 +1,6 @@ ## deployable containers have a semantic and build tag # semantic version tag: major.minor[.patch] # build version tag: timestamp -VER=0.4.5 +VER=1.0.0 TAGS="${VER} ${VER}-$(date --utc +"%Y%m%dT%H%M%S")" unset VER diff --git a/critwall/build.gradle b/critwall/build.gradle index 9b9a74cd9..83716cb15 100644 --- a/critwall/build.gradle +++ b/critwall/build.gradle @@ -21,9 +21,8 @@ mainClassName = 'org.opencadc.critwall.Main' dependencies { compile 'org.opencadc:cadc-storage-adapter:[0.8,1.0)' compile 'org.opencadc:cadc-util:[1.10.2,2.0)' - compile 'org.opencadc:cadc-inventory:[0.10,)' - // cadc-inventory-db-0.15 is in the vos2 feature branch - compile 'org.opencadc:cadc-inventory-db:[0.15.0,)' + compile 'org.opencadc:cadc-inventory:[1.0,2.0)' + compile 'org.opencadc:cadc-inventory-db:[1.0,2.0)' compile 'org.opencadc:cadc-registry:[1.7,2.0)' compile 'org.opencadc:cadc-vosi:[1.3.6,2.0)' compile 'org.opencadc:cadc-vos:[2.0,)' diff --git a/fenwick/VERSION b/fenwick/VERSION index fed2d5959..38b1547ae 100644 --- a/fenwick/VERSION +++ b/fenwick/VERSION @@ -4,6 +4,6 @@ # tags with and without build number so operators use the versioned # tag but we always keep a timestamped tag in case a semantic tag gets # replaced accidentally -VER=0.5.7 +VER=1.0.0 TAGS="${VER} ${VER}-$(date --utc +"%Y%m%dT%H%M%S")" unset VER diff --git a/fenwick/build.gradle b/fenwick/build.gradle index eb9cce1e1..f7feee09f 100644 --- a/fenwick/build.gradle +++ b/fenwick/build.gradle @@ -16,9 +16,8 @@ group = 'org.opencadc' dependencies { compile 'org.opencadc:cadc-util:[1.10.2,2.0)' - compile 'org.opencadc:cadc-inventory:[0.9.4,2.0)' - // cadc-inventory-db-0.15.0 is from the vos2 feature branch - compile 'org.opencadc:cadc-inventory-db:[0.15,)' + compile 'org.opencadc:cadc-inventory:[1.0,2.0)' + compile 'org.opencadc:cadc-inventory-db:[1.0,2.0)' compile 'org.opencadc:cadc-inventory-util:[0.1.8,1.0)' compile 'org.opencadc:cadc-registry:[1.5,2.0)' compile 'org.opencadc:cadc-tap:[1.1.14,1.2)' // 1.2 upper bound is correct #reasons diff --git a/luskan/README.md b/luskan/README.md index 0d4dd2093..90f4ccdf9 100644 --- a/luskan/README.md +++ b/luskan/README.md @@ -104,6 +104,10 @@ Assuming instances are restarted regularly, this would cause rollover approximat See cadc-log for common dynamic logging control. +### cadc-vosi.properties (optional) +See cadc-vosi for common +service state control. + ### cadcproxy.pem (optional) This client certificate is used to make authenticated server-to-server calls for system-level A&A purposes. diff --git a/luskan/VERSION b/luskan/VERSION index e894843e4..2b8664584 100644 --- a/luskan/VERSION +++ b/luskan/VERSION @@ -1,6 +1,6 @@ ## deployable containers have a semantic and build tag # semantic version tag: major.minor # build version tag: timestamp -VER=0.6.8 +VER=1.0.0 TAGS="${VER} ${VER}-$(date --utc +"%Y%m%dT%H%M%S")" unset VER diff --git a/minoc/README.md b/minoc/README.md index 0f27e2b08..99831467e 100644 --- a/minoc/README.md +++ b/minoc/README.md @@ -91,7 +91,8 @@ org.opencadc.minoc.recoverableNamespace = {namespace} The optional _trust.preauth_ key(s) configure `minoc` to trust external service(s) to have performed authorization checks. Such services may include a signed token in the URL and `minoc` will validate the request using a public key retrieved from the service instead of performing authorization checks -itself. Example: +itself. Currently, only `raven` and `vault` can generate such URLs and provide access to their +public keys. Example: ``` # trust a SI global inventory org.opencadc.minoc.trust.preauth = ivo://example.net/raven @@ -99,13 +100,13 @@ org.opencadc.minoc.trust.preauth = ivo://example.net/raven # trust a SI VOSpace service org.opencadc.minoc.trust.preauth = ivo://example.net/vault ``` -Setting _trust.preauth_ one or more times also implies _readable_ and _writable_ are _true_. +Setting _trust.preauth_ one or more times implicitly sets _readable_ and _writable_ to _true_. The optional _readGrantProvider_ and _writeGrantProvider_ keys configure minoc to call other services to get grants (permissions) for operations. Multiple values of the granting service resourceID(s) may be provided by including multiple property settings (one per line). All services will be consulted but a single positive -result is sufficient to grant permission for an action. Setting these values also sets the implied _readable_ -and _writable_ is _true_ respectively. +result is sufficient to grant permission for an action. Setting these values implicitly sets _readable_ +and _writable_ to _true_ respectively. The optional _readable_ and _writable_ keys configure minoc explicitly rather than relying on one or more of the above trust or grant provider settings. For example, this allows one to configure a read-only minoc @@ -144,16 +145,9 @@ will be able to read files. See cadc-log for common dynamic logging control. -### minoc-availability.properties (optional) -WARN: This config file name is going to change to a common one for consistency across multiple services. - -The minoc-availability.properties file specifies which users have the authority to change the availability state of the minoc service. Each entry consists of a key=value pair. The key is always "users". The value is the x500 canonical user name. - -Example: -``` -users = {user identity} -``` -`users` specifies the user(s) who are authorized to make calls to the service. The value is a list of user identities (X500 distingushed name), one line per user. Optional: if the `minoc-availability.properties` is not found or does not list any `users`, the service will function in the default mode (ReadWrite) and the state will not be changeable. +### cadc-vosi.properties (optional) +See cadc-vosi for common +service state control. ### cadcproxy.pem (optional) This client certificate is used to make authenticated server-to-server calls for system-level A&A purposes. @@ -174,15 +168,3 @@ docker run --rm -it minoc:latest /bin/bash docker run --rm --user tomcat:tomcat --volume=/path/to/external/config:/config:ro --name minoc minoc:latest ``` -## using it - -Using `cURL` is possible with Minoc to put a file for testing. - -**Note:** The `content-type` header is important! -```bash -$ curl -v -X PUT \ - --header "content-type: application/fits" \ - --data-binary @myfile.fits \ - -E ~/.ssl/cadcproxy.pem \ - https://myhost.com/minoc/files/test:TEST/myfile.fits -``` diff --git a/minoc/VERSION b/minoc/VERSION index a8d5aa3a2..3d4fab565 100644 --- a/minoc/VERSION +++ b/minoc/VERSION @@ -4,6 +4,6 @@ # tags with and without build number so operators use the versioned # tag but we always keep a timestamped tag in case a semantic tag gets # replaced accidentally -VER=0.9.10 +VER=1.0.0 TAGS="${VER} ${VER}-$(date --utc +"%Y%m%dT%H%M%S")" unset VER diff --git a/minoc/build.gradle b/minoc/build.gradle index 45705d4f2..add0a28c7 100644 --- a/minoc/build.gradle +++ b/minoc/build.gradle @@ -34,8 +34,8 @@ dependencies { compile 'org.opencadc:cadc-cdp:[1.0,)' compile 'org.opencadc:cadc-data-ops-fits:[0.3.0,)' compile 'org.opencadc:cadc-gms:[1.0.0,)' - compile 'org.opencadc:cadc-inventory:[0.9.4,2.0)' - compile 'org.opencadc:cadc-inventory-db:[0.14.5,0.15)' + compile 'org.opencadc:cadc-inventory:[1.0,2.0)' + compile 'org.opencadc:cadc-inventory-db:[1.0,2.0)' compile 'org.opencadc:cadc-inventory-server:[0.2.1,)' compile 'org.opencadc:cadc-soda-server:[1.2.0,2.0.0)' compile 'org.opencadc:cadc-storage-adapter:[0.11.2,)' diff --git a/minoc/src/main/java/org/opencadc/minoc/MinocConfig.java b/minoc/src/main/java/org/opencadc/minoc/MinocConfig.java index d4b27d9b9..437804b4a 100644 --- a/minoc/src/main/java/org/opencadc/minoc/MinocConfig.java +++ b/minoc/src/main/java/org/opencadc/minoc/MinocConfig.java @@ -116,7 +116,6 @@ public class MinocConfig { private final MultiValuedProperties configProperties; private final Map trustedServices = new TreeMap<>(); - private boolean trustedServiceKeySync = false; private final List readGrantServices = new ArrayList<>(); private final List writeGrantServices = new ArrayList<>(); private final boolean readable; @@ -180,7 +179,7 @@ public MinocConfig() { throw new IllegalStateException("invalid config: " + TRUST_KEY + "=" + s + " INVALID", ex); } } - trustedServiceKeySync = false; + // try to sync keys on startup syncKeys(); } @@ -351,44 +350,40 @@ public MultiValuedProperties getProperties() { } public Map getTrustedServices() { + // check and try to sync missing keys before request syncKeys(); return trustedServices; } private void syncKeys() { - if (!trustedServiceKeySync) { - int numFails = 0; - RegistryClient reg = new RegistryClient(); - // check map for null keys and try to retrieve them - for (Map.Entry me : trustedServices.entrySet()) { - if (me.getValue() == null) { - try { - log.info("get trusted pubkey: " + me.getKey()); - URL capURL = reg.getAccessURL(RegistryClient.Query.CAPABILITIES, me.getKey()); - String s = capURL.toExternalForm().replace("/capabilities", "/pubkey"); - URL keyURL = new URL(s); - log.info("get trusted pubkey: " + me.getKey() + " -> " + keyURL); - ByteArrayOutputStream bos = new ByteArrayOutputStream(); - HttpGet get = new HttpGet(keyURL, bos); - get.setConnectionTimeout(6000); - get.setReadTimeout(6000); - get.setRetry(0, 0, HttpTransfer.RetryReason.NONE); - get.run(); - if (get.getThrowable() != null) { - throw (Exception) get.getThrowable(); - } - byte[] key = bos.toByteArray(); - trustedServices.put(me.getKey(), key); - log.info("get trusted pubkey: " + me.getKey() + " OK"); - } catch (Exception ex) { - log.warn("failed to get public key from " + me.getKey() + ": " + ex); - numFails++; + RegistryClient reg = new RegistryClient(); + // check map for null keys and try to retrieve them + // ASSUMPTION: keys never change once generated so if they do then minoc + // needs to be restarted + for (Map.Entry me : trustedServices.entrySet()) { + if (me.getValue() == null) { + try { + log.info("get trusted pubkey: " + me.getKey()); + URL capURL = reg.getAccessURL(RegistryClient.Query.CAPABILITIES, me.getKey()); + String s = capURL.toExternalForm().replace("/capabilities", "/pubkey"); + URL keyURL = new URL(s); + log.info("get trusted pubkey: " + me.getKey() + " -> " + keyURL); + ByteArrayOutputStream bos = new ByteArrayOutputStream(); + HttpGet get = new HttpGet(keyURL, bos); + get.setConnectionTimeout(6000); + get.setReadTimeout(6000); + get.setRetry(0, 0, HttpTransfer.RetryReason.NONE); + get.run(); + if (get.getThrowable() != null) { + throw (Exception) get.getThrowable(); } + byte[] key = bos.toByteArray(); + trustedServices.put(me.getKey(), key); + log.info("get trusted pubkey: " + me.getKey() + " OK"); + } catch (Exception ex) { + log.warn("failed to get public key from " + me.getKey() + ": " + ex); } } - if (numFails == 0) { - trustedServiceKeySync = true; - } } } diff --git a/ratik/VERSION b/ratik/VERSION index 79fec5904..38b1547ae 100644 --- a/ratik/VERSION +++ b/ratik/VERSION @@ -4,6 +4,6 @@ # tags with and without build number so operators use the versioned # tag but we always keep a timestamped tag in case a semantic tag gets # replaced accidentally -VER=0.1.9 +VER=1.0.0 TAGS="${VER} ${VER}-$(date --utc +"%Y%m%dT%H%M%S")" unset VER diff --git a/ratik/build.gradle b/ratik/build.gradle index 6eef59f99..79137835c 100644 --- a/ratik/build.gradle +++ b/ratik/build.gradle @@ -16,9 +16,8 @@ group = 'org.opencadc' dependencies { compile 'org.opencadc:cadc-util:[1.10.2,2.0)' - compile 'org.opencadc:cadc-inventory:[0.9.4,2.0)' - // cadc-inventory-db-0.15.0 is from the vos2 feature branch - compile 'org.opencadc:cadc-inventory-db:[0.15,)' + compile 'org.opencadc:cadc-inventory:[1.0,2.0)' + compile 'org.opencadc:cadc-inventory-db:[1.0,2.0)' compile 'org.opencadc:cadc-inventory-util:[0.1.8,1.0)' compile 'org.opencadc:cadc-registry:[1.5,2.0)' compile 'org.opencadc:cadc-tap:[1.1.15,2.0)' diff --git a/raven/README.md b/raven/README.md index e4ea4f0d9..cac828ddc 100644 --- a/raven/README.md +++ b/raven/README.md @@ -112,6 +112,10 @@ will be able to read files. See cadc-log for common dynamic logging control. +### cadc-vosi.properties (optional) +See cadc-vosi for common +service state control. + ### cadcproxy.pem (optional) This client certificate is used to make authenticated server-to-server calls for system-level A&A purposes. diff --git a/raven/VERSION b/raven/VERSION index ee967b01d..6a39bdba3 100644 --- a/raven/VERSION +++ b/raven/VERSION @@ -2,6 +2,6 @@ # semantic version tag: major.minor[.patch] # build version tag: timestamp # tag: {semantic}-{build} -VER=0.8.0 +VER=1.0.0 TAGS="${VER} ${VER}-$(date --utc +"%Y%m%dT%H%M%S")" unset VER diff --git a/raven/build.gradle b/raven/build.gradle index a20a7319a..fec3cf4f1 100644 --- a/raven/build.gradle +++ b/raven/build.gradle @@ -33,8 +33,8 @@ dependencies { compile 'org.opencadc:cadc-rest:[1.3.14,)' compile 'org.opencadc:cadc-cdp:[1.0,)' compile 'org.opencadc:cadc-gms:[1.0.4,)' - compile 'org.opencadc:cadc-inventory:[0.9.4,2.0)' - compile 'org.opencadc:cadc-inventory-db:[0.15.0,)' + compile 'org.opencadc:cadc-inventory:[1.0,2.0)' + compile 'org.opencadc:cadc-inventory-db:[1.0,2.0)' compile 'org.opencadc:cadc-inventory-server:[0.3.0,)' compile 'org.opencadc:cadc-permissions:[0.3.5,)' compile 'org.opencadc:cadc-permissions-client:[0.3,)' diff --git a/ringhold/VERSION b/ringhold/VERSION index 51807fa89..b3562c8a8 100644 --- a/ringhold/VERSION +++ b/ringhold/VERSION @@ -1,4 +1,6 @@ ## deployable containers have a semantic and build tag # semantic version tag: major.minor[.patch] # build version tag: timestamp -TAGS="0.2-$(date --utc +"%Y%m%dT%H%M%S")" +VER=0.2.0 +TAGS="${VER} ${VER}-$(date --utc +"%Y%m%dT%H%M%S")" +unset VER diff --git a/ringhold/build.gradle b/ringhold/build.gradle index e29f2e50c..685a569da 100644 --- a/ringhold/build.gradle +++ b/ringhold/build.gradle @@ -16,8 +16,8 @@ group = 'org.opencadc' dependencies { compile 'org.opencadc:cadc-util:[1.6,2.0)' - compile 'org.opencadc:cadc-inventory:[0.10,)' - compile 'org.opencadc:cadc-inventory-db:[0.15,1.0)' + compile 'org.opencadc:cadc-inventory:[1.0,2.0)' + compile 'org.opencadc:cadc-inventory-db:[1.0,2.0)' testCompile 'junit:junit:[4.12,5.0)' } diff --git a/tantar/VERSION b/tantar/VERSION index ba99f57c6..d849b8f34 100644 --- a/tantar/VERSION +++ b/tantar/VERSION @@ -1,6 +1,6 @@ ## deployable containers have a semantic and build tag # semantic version tag: major.minor[.patch] # build version tag: timestamp -VER=0.4.5 +VER=1.0.0 TAGS="${VER} ${VER}-$(date --utc +"%Y%m%dT%H%M%S")" unset VER diff --git a/tantar/build.gradle b/tantar/build.gradle index c99751a61..154e5309d 100644 --- a/tantar/build.gradle +++ b/tantar/build.gradle @@ -21,8 +21,8 @@ mainClassName = 'org.opencadc.tantar.Main' dependencies { compile 'org.opencadc:cadc-util:[1.10.2,2.0)' compile 'org.opencadc:cadc-log:[1.1.2,2.0)' - compile 'org.opencadc:cadc-inventory:[0.10,)' - compile 'org.opencadc:cadc-inventory-db:[0.15.0,1.0)' + compile 'org.opencadc:cadc-inventory:[1.0.0,2.0)' + compile 'org.opencadc:cadc-inventory-db:[1.0.0,2.0)' compile 'org.opencadc:cadc-inventory-util:[0.1.8,1.0)' compile 'org.opencadc:cadc-storage-adapter:[0.11.1,1.0)' diff --git a/vault/README.md b/vault/README.md index 153232129..f2ef10863 100644 --- a/vault/README.md +++ b/vault/README.md @@ -41,7 +41,6 @@ org.opencadc.vault.nodes.password={password for vospace pool} org.opencadc.vault.nodes.url=jdbc:postgresql://{server}/{database} org.opencadc.vault.inventory.maxActive={max connections for inventory pool} -# optional: config for separate inventory pool org.opencadc.vault.inventory.username={username for inventory pool} org.opencadc.vault.inventory.password={password for inventory pool} org.opencadc.vault.inventory.url=jdbc:postgresql://{server}/{database} @@ -58,13 +57,12 @@ VOSI-availability output. The _inventory_ account owns and manages (create, alter, drop) inventory database objects and manages all the content (update and delete Artifact, insert DeletedArtifactEvent). The database is specified -in the JDBC URL and the schema name is specified in the minoc.properties (below). Failure to connect or +in the JDBC URL and the schema name is specified in the vault.properties (below). Failure to connect or initialize the database will show up in logs and in the VOSI-availability output. The _inventory_ content may be in the same database as the _nodes_, in a different database in the same server, or in a different -server entirely. See `org.opencadc.vault.singlePool` below for the pros and cons. Note: it is a good -idea to set `maxActive` to a valid integer (e.g. 1 because the tomcat connection pool doesn't like 0 and -decides to make it 100 instead) when using a single pool; this avoids an ugly but meaningless stack trace -in the logs at startup. +server entirely. See `org.opencadc.vault.singlePool` below for the pros and cons. The _inventory_ pool must +be functional for initialization, availability checks (`maxActive` = 1 with `singlePool` is sufficient), and +the connection information is re-used by an internal background thread that synchronizes data node sizes. The _uws_ account owns and manages (create, alter, drop) uws database objects in the `uws` schema and manages all the content (insert, update, delete). The database is specified in the JDBC URLFailure to connect or initialize the @@ -128,28 +126,21 @@ DeletedArtifactEvent are done in a separate transaction and if that fails the Ar orphaned until the vault validation (see ???) runs and fixes such a discrepancy. However, _singlePool_ = `false` allows the content to be stored in two separate databases or servers. -The _root.owner_ owns the root node and has full read and write permission in the root container, so it can -create and delete container nodes at the root and assign container node properties that are normally read-only -to normal users: owner, quota, etc. This must be set to the username of the admin. +The _root.owner_ key configures the owner of the root node; the owner and has full read and write permission +in the root container, so it can create and delete container nodes at the root and assign container node properties +that are normally read-only to normal users: owner, quota, etc. This must be set to the username of the admin. -The _storage.namespace_ configures `vault` to use the specified namespace in storage-inventory to store files. +The _storage.namespace_ key configures `vault` to use the specified namespace in storage-inventory to store files. This only applies to new data nodes that are created and will not effect previously created nodes and artifacts. Probably don't want to change this... prevent change? TBD. -### vault-availability.properties (optional) +### cadc-log.properties (optional) +See cadc-log for common +dynamic logging control. -The vault-availability.properties file specifies which users have the authority to change the availability state of -the vault service. Each entry consists of a key=value pair. The key is always "users". The value is the x500 canonical -user name. - -Example: -``` -users = {user identity} -``` -`users` specifies the user(s) who are authorized to make calls to the service. The value is a list of user -identities (X500 distingushed name), one line per user. Optional: if the `vault-availability.properties` is -not found or does not list any `users`, the service will function in the default mode (ReadWrite) and the -state will not be changeable. +### cadc-vosi.properties (optional) +See cadc-vosi for common +service state control. ## building it ``` diff --git a/vault/VERSION b/vault/VERSION index 9d05f397f..3d4fab565 100644 --- a/vault/VERSION +++ b/vault/VERSION @@ -4,6 +4,6 @@ # tags with and without build number so operators use the versioned # tag but we always keep a timestamped tag in case a semantic tag gets # replaced accidentally -VER=0.2.0 +VER=1.0.0 TAGS="${VER} ${VER}-$(date --utc +"%Y%m%dT%H%M%S")" unset VER diff --git a/vault/build.gradle b/vault/build.gradle index a11549995..e6ac8689f 100644 --- a/vault/build.gradle +++ b/vault/build.gradle @@ -31,11 +31,11 @@ def git_url = 'https://github.com/opencadc/storage-inventory' dependencies { compile 'javax.servlet:javax.servlet-api:[3.1,4.0)' - compile 'org.opencadc:cadc-util:[1.9.10,2.0)' + compile 'org.opencadc:cadc-util:[1.11.0,2.0)' compile 'org.opencadc:cadc-log:[1.1.6,2.0)' compile 'org.opencadc:cadc-gms:[1.0.5,)' compile 'org.opencadc:cadc-rest:[1.3.16,)' - compile 'org.opencadc:cadc-vos:[2.0.3,)' + compile 'org.opencadc:cadc-vos:[2.0.6,)' compile 'org.opencadc:cadc-vos-server:[2.0.9,)' compile 'org.opencadc:cadc-vosi:[1.3.2,)' compile 'org.opencadc:cadc-uws:[1.0,)' @@ -43,8 +43,8 @@ dependencies { compile 'org.opencadc:cadc-access-control:[1.1.1,2.0)' compile 'org.opencadc:cadc-cdp:[1.2.3,)' compile 'org.opencadc:cadc-registry:[1.7.6,)' - compile 'org.opencadc:cadc-inventory:[0.9.4,1.0)' - compile 'org.opencadc:cadc-inventory-db:[0.15.0,1.0)' + compile 'org.opencadc:cadc-inventory:[1.0.0,2.0)' + compile 'org.opencadc:cadc-inventory-db:[1.0.0,2.0)' compile 'org.opencadc:cadc-inventory-server:[0.3,1.0)' compile 'org.opencadc:cadc-permissions:[0.3.5,1.0)' diff --git a/vault/src/main/java/org/opencadc/vault/NodePersistenceImpl.java b/vault/src/main/java/org/opencadc/vault/NodePersistenceImpl.java index e3e98ceec..bf2fbb388 100644 --- a/vault/src/main/java/org/opencadc/vault/NodePersistenceImpl.java +++ b/vault/src/main/java/org/opencadc/vault/NodePersistenceImpl.java @@ -166,18 +166,22 @@ public class NodePersistenceImpl implements NodePersistence { private final Namespace storageNamespace; private final boolean localGroupsOnly; - private URI resourceID; + private final URI resourceID; private final boolean preventNotFound; + final String appName; // access by VaultTransferGenerator + // possibly temporary hack so migration tool can set this to false and // preserve lastModified timestamps on nodes public boolean nodeOrigin = true; - public NodePersistenceImpl(URI resourceID) { + public NodePersistenceImpl(URI resourceID, String appName) { if (resourceID == null) { throw new IllegalArgumentException("resource ID required"); } this.resourceID = resourceID; + this.appName = appName; + MultiValuedProperties config = VaultInitAction.getConfig(); this.nodeDaoConfig = VaultInitAction.getDaoConfig(config); this.invDaoConfig = VaultInitAction.getInvConfig(config); @@ -259,7 +263,7 @@ public TransferGenerator getTransferGenerator() { keyDAO.setConfig(kpDaoConfig); PreauthKeyPair kp = keyDAO.get(VaultInitAction.KEY_PAIR_NAME); TokenTool tt = new TokenTool(kp.getPublicKey(), kp.getPrivateKey()); - return new VaultTransferGenerator(this, getArtifactDAO(), tt, preventNotFound); + return new VaultTransferGenerator(this, appName, getArtifactDAO(), tt, preventNotFound); } private NodeDAO getDAO() { @@ -358,10 +362,6 @@ public Node get(ContainerNode parent, String name) throws TransientException { if (ret == null) { return null; } - ret.parent = parent; - IdentityManager identityManager = AuthenticationUtil.getIdentityManager(); - ret.owner = identityManager.toSubject(ret.ownerID); - ret.ownerDisplay = identityManager.toDisplayString(ret.owner); // in principle we could have queried vospace.Node join inventory.Artifact above // and avoid this query.... simplicity for now @@ -371,6 +371,46 @@ public Node get(ContainerNode parent, String name) throws TransientException { Artifact a = artifactDAO.get(dn.storageID); DateFormat df = NodeWriter.getDateFormat(); if (a != null) { + // DataNode.bytesUsed is an optimization (cache): + // if DataNode.bytesUsed != Artifact.contentLength we update the cache + // this retains put+get consistency in a single-site deployed (with minoc) + // and may help hide some inconsistencies in child listing sizes + if (!a.getContentLength().equals(dn.bytesUsed)) { + TransactionManager txn = dao.getTransactionManager(); + try { + log.debug("starting node transaction"); + txn.startTransaction(); + log.debug("start txn: OK"); + + DataNode locked = (DataNode) dao.lock(dn); + if (locked != null) { + dn = locked; // safer than accidentally using the wrong variable + dn.bytesUsed = a.getContentLength(); + dao.put(dn); + ret = dn; + } + + log.debug("commit txn..."); + txn.commitTransaction(); + log.debug("commit txn: OK"); + if (locked == null) { + return null; // gone + } + } catch (Exception ex) { + if (txn.isOpen()) { + log.error("failed to update bytesUsed on " + dn.getID() + " aka " + dn.getName(), ex); + txn.rollbackTransaction(); + log.debug("rollback txn: OK"); + } + } finally { + if (txn.isOpen()) { + log.error("BUG - open transaction in finally"); + txn.rollbackTransaction(); + log.error("rollback txn: OK"); + } + } + } + Date d = ret.getLastModified(); Date cd = null; if (ret.getLastModified().before(a.getLastModified())) { @@ -396,19 +436,17 @@ public Node get(ContainerNode parent, String name) throws TransientException { if (a.contentType != null) { ret.getProperties().add(new NodeProperty(VOS.PROPERTY_URI_TYPE, a.contentType)); } - // TODO: a.getContentLength() is correct, but might differ from dn.bytesUsed due to eventual consistency - // child listing iterator can only report dn.bytesUsed - // correct or consistency with listing?? - - // currently needed for consistency-requiring FilesTest - log.warn("DataNode.bytesUsed: " + dn.bytesUsed + " -> " + a.getContentLength()); - dn.bytesUsed = a.getContentLength(); } if (dn.bytesUsed == null) { - log.warn("DataNode.bytesUsed: 0 (no artifact)"); dn.bytesUsed = 0L; // no data stored } } + + ret.parent = parent; + IdentityManager identityManager = AuthenticationUtil.getIdentityManager(); + ret.owner = identityManager.toSubject(ret.ownerID); + ret.ownerDisplay = identityManager.toDisplayString(ret.owner); + return ret; } @@ -645,7 +683,7 @@ public void move(Node node, ContainerNode dest, String newName) { if (node.parent == null || dest.parent == null) { throw new IllegalArgumentException("args must both be peristent nodes before move"); } - // try to detect attempt to dsconnect from path to root: node is a parent of dest + // try to detect attempt to disconnect from path to root: node is a parent of dest ContainerNode cur = dest; while (!cur.getID().equals(root.getID())) { cur = cur.parent; @@ -654,28 +692,48 @@ public void move(Node node, ContainerNode dest, String newName) { } } - - Subject caller = AuthenticationUtil.getCurrentSubject(); - node.owner = caller; - node.ownerID = null; - node.ownerDisplay = null; - node.parent = dest; - node.parentID = null; - if (newName != null) { - node.setName(newName); - } + + NodeDAO dao = getDAO(); + TransactionManager txn = dao.getTransactionManager(); try { - Node result = put(node); - log.debug("moved: " + result); - } catch (NodeNotSupportedException ex) { - LocalServiceURI loc = new LocalServiceURI(getResourceID()); - VOSURI srcURI = loc.getURI(node); - throw new RuntimeException("BUG: failed to move node because of detected type change: " - + srcURI.getPath() + " type=" + node.getClass().getSimpleName(), ex); + log.debug("starting node transaction"); + txn.startTransaction(); + log.debug("start txn: OK"); + + // lock the source node + Node locked = dao.lock(node); + if (locked != null) { + node = locked; // safer than having two vars and accidentally using the wrong one + Subject caller = AuthenticationUtil.getCurrentSubject(); + node.owner = caller; + node.ownerID = null; + node.ownerDisplay = null; + node.parent = dest; + node.parentID = null; + if (newName != null) { + node.setName(newName); + } + Node result = put(node); + log.debug("moved: " + result); + } + log.debug("commit txn..."); + txn.commitTransaction(); + log.debug("commit txn: OK"); + } catch (Exception ex) { + if (txn.isOpen()) { + log.error("failed to move " + node.getID() + " aka " + node.getName(), ex); + txn.rollbackTransaction(); + log.debug("rollback txn: OK"); + } + } finally { + if (txn.isOpen()) { + log.error("BUG - open transaction in finally"); + txn.rollbackTransaction(); + log.error("rollback txn: OK"); + } } } - /** * Delete the specified node. * @@ -721,8 +779,7 @@ public void delete(Node node) throws TransientException { } } else if (node instanceof DataNode) { DataNode dn = (DataNode) node; - NodeProperty len = dn.getProperty(VOS.PROPERTY_URI_CONTENTLENGTH); - if (len != null) { + if (dn.bytesUsed != null) { // artifact exists storageID = dn.storageID; } diff --git a/vault/src/main/java/org/opencadc/vault/ServiceAvailability.java b/vault/src/main/java/org/opencadc/vault/ServiceAvailability.java index 0b0005a42..5a76fb1bc 100644 --- a/vault/src/main/java/org/opencadc/vault/ServiceAvailability.java +++ b/vault/src/main/java/org/opencadc/vault/ServiceAvailability.java @@ -67,11 +67,16 @@ package org.opencadc.vault; +import ca.nrc.cadc.db.DBUtil; import ca.nrc.cadc.rest.RestAction; import ca.nrc.cadc.vosi.Availability; import ca.nrc.cadc.vosi.AvailabilityPlugin; - +import ca.nrc.cadc.vosi.avail.CheckDataSource; +import javax.naming.InitialContext; +import javax.naming.NamingException; +import javax.sql.DataSource; import org.apache.log4j.Logger; +import org.opencadc.vault.metadata.DataNodeSizeSync; /** * This class performs the work of determining if the executing artifact @@ -130,7 +135,31 @@ public Availability getStatus() { return new Availability(false, RestAction.STATE_READ_ONLY_MSG); } - //TODO add availability checks for dependent services + // check database pools + DataSource ds; + String testSQL; + CheckDataSource cds; + + ds = DBUtil.findJNDIDataSource("jdbc/nodes"); + testSQL = "select * from vospace.ModelVersion"; + cds = new CheckDataSource(ds, testSQL); + cds.check(); + + ds = DBUtil.findJNDIDataSource("jdbc/inventory"); + testSQL = "select * from inventory.Artifact limit 1"; + cds = new CheckDataSource(ds, testSQL); + cds.check(); + + ds = DBUtil.findJNDIDataSource("jdbc/inventory-iterator"); + testSQL = "select * from inventory.Artifact limit 1"; + cds = new CheckDataSource(ds, testSQL); + cds.check(); + + ds = DBUtil.findJNDIDataSource("jdbc/uws"); + testSQL = "select * from uws.Job limit 1"; + cds = new CheckDataSource(ds, testSQL); + cds.check(); + } catch (Throwable t) { // the test itself failed log.debug("failure", t); @@ -149,10 +178,13 @@ public void setState(String state) { String key = appName + RestAction.STATE_MODE_KEY; if (RestAction.STATE_OFFLINE.equalsIgnoreCase(state)) { System.setProperty(key, RestAction.STATE_OFFLINE); + setOffline(true); } else if (RestAction.STATE_READ_ONLY.equalsIgnoreCase(state)) { System.setProperty(key, RestAction.STATE_READ_ONLY); + setOffline(true); } else if (RestAction.STATE_READ_WRITE.equalsIgnoreCase(state)) { System.setProperty(key, RestAction.STATE_READ_WRITE); + setOffline(false); } else { throw new IllegalArgumentException("invalid state: " + state + " expected: " + RestAction.STATE_READ_WRITE + "|" @@ -169,5 +201,15 @@ private String getState() { } return ret; } - + + private void setOffline(boolean offline) { + String jndiKey = appName + "-" + DataNodeSizeSync.class.getName(); + try { + InitialContext initialContext = new InitialContext(); + DataNodeSizeSync async = (DataNodeSizeSync) initialContext.lookup(jndiKey); + async.setOffline(offline); + } catch (NamingException e) { + log.debug(String.format("unable to find %s - %s", jndiKey, e.getMessage())); + } + } } diff --git a/vault/src/main/java/org/opencadc/vault/VaultInitAction.java b/vault/src/main/java/org/opencadc/vault/VaultInitAction.java index e4e51b418..a80f04526 100644 --- a/vault/src/main/java/org/opencadc/vault/VaultInitAction.java +++ b/vault/src/main/java/org/opencadc/vault/VaultInitAction.java @@ -3,7 +3,7 @@ ******************* CANADIAN ASTRONOMY DATA CENTRE ******************* ************** CENTRE CANADIEN DE DONNÉES ASTRONOMIQUES ************** * -* (c) 2023. (c) 2023. +* (c) 2024. (c) 2024. * Government of Canada Gouvernement du Canada * National Research Council Conseil national de recherches * Ottawa, Canada, K1A 0R6 Ottawa, Canada, K1A 0R6 @@ -69,6 +69,7 @@ import ca.nrc.cadc.db.DBUtil; import ca.nrc.cadc.rest.InitAction; +import ca.nrc.cadc.rest.RestAction; import ca.nrc.cadc.util.InvalidConfigException; import ca.nrc.cadc.util.MultiValuedProperties; import ca.nrc.cadc.util.PropertiesReader; @@ -88,13 +89,14 @@ import org.apache.log4j.Logger; import org.opencadc.inventory.Namespace; import org.opencadc.inventory.PreauthKeyPair; +import org.opencadc.inventory.db.ArtifactDAO; import org.opencadc.inventory.db.HarvestStateDAO; import org.opencadc.inventory.db.PreauthKeyPairDAO; import org.opencadc.inventory.db.SQLGenerator; import org.opencadc.inventory.db.StorageSiteDAO; import org.opencadc.inventory.db.version.InitDatabaseSI; import org.opencadc.inventory.transfer.StorageSiteAvailabilityCheck; -import org.opencadc.vault.metadata.ArtifactSync; +import org.opencadc.vault.metadata.DataNodeSizeSync; import org.opencadc.vospace.db.InitDatabaseVOS; import org.opencadc.vospace.server.NodePersistence; import org.springframework.dao.DataIntegrityViolationException; @@ -111,6 +113,7 @@ public class VaultInitAction extends InitAction { static final String JNDI_VOS_DATASOURCE = "jdbc/nodes"; // context.xml static final String JNDI_INV_DATASOURCE = "jdbc/inventory"; // context.xml + static final String JNDI_INV_ITER_DATASOURCE = "jdbc/inventory-iterator"; // context.xml static final String JNDI_UWS_DATASOURCE = "jdbc/uws"; // context.xml // config keys @@ -129,13 +132,15 @@ public class VaultInitAction extends InitAction { private Namespace storageNamespace; private Map vosDaoConfig; private Map invDaoConfig; - private List allocationParents = new ArrayList<>(); - private String jndiNodePersistence; - private String jndiPreauthKeys; - private String jndiSiteAvailabilities; + private String jndiNodePersistence; // store in JNDI for cadc-vos-server lib + private String jndiPreauthKeys; // store pubkey in JNDI for download via GetKeyAction + + private String jndiSiteAvailabilities; // store in JNDI to share with ProtocolsGenerator private Thread availabilityCheck; - private Thread artifactSync; + + private String jndiDataNodeSizeSync; // store in JNDI to support availability mode change + private Thread dataNodeSizeSyncThread; public VaultInitAction() { super(); @@ -144,8 +149,9 @@ public VaultInitAction() { @Override public void doInit() { initConfig(); - initDatabase(); - initUWSDatabase(); + initDatabaseVOS(); + initDatabaseINV(); + initDatabaseUWS(); initNodePersistence(); initKeyPair(); initAvailabilityCheck(); @@ -290,16 +296,17 @@ static Map getInvConfig(MultiValuedProperties props) { return ret; } - static Map getKeyPairConfig(MultiValuedProperties props) { - return getDaoConfig(props); - /* + static Map getIteratorConfig(MultiValuedProperties props) { Map ret = new TreeMap<>(); ret.put(SQLGenerator.class.getName(), SQLGenerator.class); // not configurable right now - ret.put("jndiDataSourceName", JNDI_VOS_DATASOURCE); - ret.put("invSchema", props.getFirstPropertyValue(INVENTORY_SCHEMA_KEY)); // requied but unused - ret.put("genSchema", props.getFirstPropertyValue(VOSPACE_SCHEMA_KEY)); + ret.put("jndiDataSourceName", JNDI_INV_ITER_DATASOURCE); + ret.put("invSchema", props.getFirstPropertyValue(INVENTORY_SCHEMA_KEY)); + ret.put("genSchema", props.getFirstPropertyValue(INVENTORY_SCHEMA_KEY)); // for complete init return ret; - */ + } + + static Map getKeyPairConfig(MultiValuedProperties props) { + return getDaoConfig(props); } private void initConfig() { @@ -318,7 +325,7 @@ private void initConfig() { } } - private void initDatabase() { + private void initDatabaseVOS() { try { String dsname = (String) vosDaoConfig.get("jndiDataSourceName"); String schema = (String) vosDaoConfig.get("vosSchema"); @@ -330,7 +337,9 @@ private void initDatabase() { } catch (Exception ex) { throw new IllegalStateException("check/init vospace database failed", ex); } - + } + + private void initDatabaseINV() { try { String dsname = (String) invDaoConfig.get("jndiDataSourceName"); String schema = (String) invDaoConfig.get("invSchema"); @@ -344,7 +353,7 @@ private void initDatabase() { } } - private void initUWSDatabase() { + private void initDatabaseUWS() { try { log.info("initDatabase: " + JNDI_UWS_DATASOURCE + " uws START"); DataSource uws = DBUtil.findJNDIDataSource(JNDI_UWS_DATASOURCE); @@ -366,7 +375,7 @@ private void initNodePersistence() { } catch (NamingException ignore) { log.debug("unbind previous JNDI key (" + jndiNodePersistence + ") failed... ignoring"); } - NodePersistence npi = new NodePersistenceImpl(resourceID); + NodePersistence npi = new NodePersistenceImpl(resourceID, appName); ctx.bind(jndiNodePersistence, npi); log.info("initNodePersistence: created JNDI key: " + jndiNodePersistence + " impl: " + npi.getClass().getName()); @@ -452,26 +461,64 @@ private void terminateAvailabilityCheck() { } private void initBackgroundWorkers() { - HarvestStateDAO hsDAO = new HarvestStateDAO(); - hsDAO.setConfig(getDaoConfig(props)); - - terminateBackgroundWorkers(); - this.artifactSync = new Thread(new ArtifactSync(hsDAO)); - artifactSync.setDaemon(true); - artifactSync.start(); + try { + HarvestStateDAO hsDAO = new HarvestStateDAO(); + hsDAO.setConfig(vosDaoConfig); + + ArtifactDAO artifactDAO = new ArtifactDAO(); + Map iterprops = getIteratorConfig(props); + log.warn("iterator pool: " + iterprops.get("jndiDataSourceName")); + artifactDAO.setConfig(iterprops); + + // determine startup mode + boolean offline = false; // normal + String key = appName + RestAction.STATE_MODE_KEY; + String ret = System.getProperty(key); + if (ret != null + && (RestAction.STATE_READ_ONLY.equals(ret) || RestAction.STATE_OFFLINE.equals(ret))) { + offline = true; + } + + terminateBackgroundWorkers(); + DataNodeSizeSync async = new DataNodeSizeSync(hsDAO, artifactDAO, storageNamespace); + async.setOffline(offline); + this.dataNodeSizeSyncThread = new Thread(async); + dataNodeSizeSyncThread.setDaemon(true); + dataNodeSizeSyncThread.start(); + + // store in JNDI so availability can set offline + this.jndiDataNodeSizeSync = appName + "-" + DataNodeSizeSync.class.getName(); + InitialContext ctx = new InitialContext(); + try { + ctx.unbind(jndiDataNodeSizeSync); + } catch (NamingException ignore) { + log.debug("unbind previous JNDI key (" + jndiPreauthKeys + ") failed... ignoring"); + } + ctx.bind(jndiDataNodeSizeSync, async); + log.info("initBackgroundWorkers: created JNDI key: " + jndiDataNodeSizeSync); + } catch (Exception ex) { + throw new RuntimeException("check/init ArtifactSync failed", ex); + } } private void terminateBackgroundWorkers() { - if (this.artifactSync != null) { + if (this.dataNodeSizeSyncThread != null) { try { - log.info("terminating ArtifactSync Thread..."); - this.artifactSync.interrupt(); - this.artifactSync.join(); - log.info("terminating ArtifactSync Thread... [OK]"); + log.info("terminating " + DataNodeSizeSync.class.getSimpleName() + " Thread..."); + this.dataNodeSizeSyncThread.interrupt(); + this.dataNodeSizeSyncThread.join(); + log.info("terminating " + DataNodeSizeSync.class.getSimpleName() + " Thread... [OK]"); } catch (Throwable t) { - log.info("failed to terminate ArtifactSync thread", t); + log.info("failed to terminate " + DataNodeSizeSync.class.getSimpleName() + " thread", t); } finally { - this.artifactSync = null; + this.dataNodeSizeSyncThread = null; + } + + try { + InitialContext initialContext = new InitialContext(); + initialContext.unbind(this.jndiDataNodeSizeSync); + } catch (NamingException e) { + log.debug(String.format("unable to unbind %s - %s", this.jndiDataNodeSizeSync, e.getMessage())); } } } diff --git a/vault/src/main/java/org/opencadc/vault/VaultTransferGenerator.java b/vault/src/main/java/org/opencadc/vault/VaultTransferGenerator.java index 51fec42a9..3b0884887 100644 --- a/vault/src/main/java/org/opencadc/vault/VaultTransferGenerator.java +++ b/vault/src/main/java/org/opencadc/vault/VaultTransferGenerator.java @@ -117,7 +117,8 @@ public class VaultTransferGenerator implements TransferGenerator { private final Map siteAvailabilities; @SuppressWarnings("unchecked") - public VaultTransferGenerator(NodePersistenceImpl nodePersistence, ArtifactDAO artifactDAO, TokenTool tokenTool, boolean preventNotFound) { + public VaultTransferGenerator(NodePersistenceImpl nodePersistence, String appName, + ArtifactDAO artifactDAO, TokenTool tokenTool, boolean preventNotFound) { this.nodePersistence = nodePersistence; this.authorizer = new VOSpaceAuthorizer(nodePersistence); this.artifactDAO = artifactDAO; @@ -125,7 +126,7 @@ public VaultTransferGenerator(NodePersistenceImpl nodePersistence, ArtifactDAO a this.preventNotFound = preventNotFound; // TODO: get appname from ??? - String siteAvailabilitiesKey = "vault" + "-" + StorageSiteAvailabilityCheck.class.getName(); + String siteAvailabilitiesKey = appName + "-" + StorageSiteAvailabilityCheck.class.getName(); log.debug("siteAvailabilitiesKey: " + siteAvailabilitiesKey); try { Context initContext = new InitialContext(); diff --git a/vault/src/main/java/org/opencadc/vault/files/GetAction.java b/vault/src/main/java/org/opencadc/vault/files/GetAction.java index d3967e5c2..231cad51e 100644 --- a/vault/src/main/java/org/opencadc/vault/files/GetAction.java +++ b/vault/src/main/java/org/opencadc/vault/files/GetAction.java @@ -70,12 +70,10 @@ import ca.nrc.cadc.auth.AuthenticationUtil; import ca.nrc.cadc.net.TransientException; import java.net.HttpURLConnection; -import java.net.URI; import java.util.List; import javax.security.auth.Subject; import org.apache.log4j.Logger; import org.opencadc.vospace.DataNode; -import org.opencadc.vospace.NodeProperty; import org.opencadc.vospace.VOS; import org.opencadc.vospace.VOSURI; import org.opencadc.vospace.server.NodeFault; diff --git a/vault/src/main/java/org/opencadc/vault/files/HeadAction.java b/vault/src/main/java/org/opencadc/vault/files/HeadAction.java index 69b68f00a..902c4c9a6 100644 --- a/vault/src/main/java/org/opencadc/vault/files/HeadAction.java +++ b/vault/src/main/java/org/opencadc/vault/files/HeadAction.java @@ -70,6 +70,7 @@ import ca.nrc.cadc.net.ResourceNotFoundException; import ca.nrc.cadc.rest.InlineContentHandler; import ca.nrc.cadc.rest.RestAction; +import ca.nrc.cadc.rest.Version; import java.net.URI; import java.net.URISyntaxException; import java.util.Date; @@ -107,6 +108,13 @@ protected final InlineContentHandler getInlineContentHandler() { return null; } + @Override + protected String getServerImpl() { + // no null version checking because fail to build correctly can't get past basic testing + Version v = getVersionFromResource(); + return "storage-inventory/vault-" + v.getMajorMinor(); + } + @Override public void initAction() throws Exception { String jndiNodePersistence = super.appName + "-" + NodePersistence.class.getName(); diff --git a/vault/src/main/java/org/opencadc/vault/metadata/ArtifactSync.java b/vault/src/main/java/org/opencadc/vault/metadata/BackgroundLogInfo.java similarity index 55% rename from vault/src/main/java/org/opencadc/vault/metadata/ArtifactSync.java rename to vault/src/main/java/org/opencadc/vault/metadata/BackgroundLogInfo.java index ffd1879e6..88ac11a93 100644 --- a/vault/src/main/java/org/opencadc/vault/metadata/ArtifactSync.java +++ b/vault/src/main/java/org/opencadc/vault/metadata/BackgroundLogInfo.java @@ -67,91 +67,40 @@ package org.opencadc.vault.metadata; -import java.net.URI; +import ca.nrc.cadc.date.DateUtil; +import ca.nrc.cadc.log.WebServiceLogInfo; +import java.text.DateFormat; import java.util.Date; -import java.util.UUID; import org.apache.log4j.Logger; -import org.opencadc.inventory.Artifact; -import org.opencadc.inventory.db.HarvestState; -import org.opencadc.inventory.db.HarvestStateDAO; /** - * Main artifact-sync agent that enables incremental sync of Artifact - * metadata to Node. + * Log structure for background threads. * * @author pdowler */ -public class ArtifactSync implements Runnable { - private static final Logger log = Logger.getLogger(ArtifactSync.class); +public class BackgroundLogInfo extends WebServiceLogInfo { + private static final Logger log = Logger.getLogger(BackgroundLogInfo.class); - private static final long SHORT_SLEEP = 12000L; - private static final long LONG_SLEEP = 2 * SHORT_SLEEP; - private static final long EVICT_AGE = 3 * LONG_SLEEP; + public Boolean leader; + public String instance; + public String lastmodified; + public Long processed; + public Long sleep; - private final UUID instanceID = UUID.randomUUID(); - private final HarvestStateDAO dao; - private String name = Artifact.class.getSimpleName(); - private URI resourceID = URI.create("jdbc/inventory"); - public ArtifactSync(HarvestStateDAO dao) { - this.dao = dao; - - // fenwick setup for production workload: - //dao.setUpdateBufferCount(99); // buffer 99 updates, do every 100 - //dao.setMaintCount(999); // buffer 999 so every 1000 real updates aka every 1e5 events - - // here, we need timestamp updates to retain leader status, so - // dao.setMaintCount(9999); // every 1e4 + public BackgroundLogInfo(String instance) { + super.serviceName = "vault"; + this.instance = instance; } - - @Override - public void run() { - try { - Thread.sleep(SHORT_SLEEP); - - while (true) { - boolean leader = false; - log.debug("check leader " + instanceID); - HarvestState state = dao.get(name, resourceID); - log.debug("check leader " + instanceID + " found: " + state); - if (state.instanceID == null) { - state.instanceID = instanceID; - dao.put(state); - state = dao.get(state.getID()); - log.debug("created: " + state); - } - if (instanceID.equals(state.instanceID)) { - log.debug("still the leader..."); - dao.put(state, true); - leader = true; - } else { - // see if we should perform a coup... - Date now = new Date(); - long age = now.getTime() - state.getLastModified().getTime(); - if (age > EVICT_AGE) { - - state.instanceID = instanceID; - dao.put(state); - state = dao.get(state.getID()); - leader = true; - log.debug("EVICTED " + state.instanceID + " because age " + age + " > " + EVICT_AGE); - } - } - - if (leader) { - log.debug("leader " + state.instanceID + " starting worker..."); - // TODO - dao.flushBufferedState(); - Thread.sleep(SHORT_SLEEP / 2L); // for testing - log.debug("idle leader " + state.instanceID + " sleep=" + SHORT_SLEEP); - Thread.sleep(SHORT_SLEEP); - } else { - log.debug("not leader: sleep=" + LONG_SLEEP); - Thread.sleep(LONG_SLEEP); - } - } - } catch (InterruptedException ex) { - log.debug("interrupted - assuming shutdown", ex); - } + + public void setOperation(String op) { + super.method = op; } + + public void setLastModified(Date ts) { + DateFormat df = DateUtil.getDateFormat(DateUtil.IVOA_DATE_FORMAT, DateUtil.UTC); + this.lastmodified = df.format(ts); + } + + } diff --git a/vault/src/main/java/org/opencadc/vault/metadata/DataNodeSizeSync.java b/vault/src/main/java/org/opencadc/vault/metadata/DataNodeSizeSync.java new file mode 100644 index 000000000..3bce32e99 --- /dev/null +++ b/vault/src/main/java/org/opencadc/vault/metadata/DataNodeSizeSync.java @@ -0,0 +1,224 @@ +/* +************************************************************************ +******************* CANADIAN ASTRONOMY DATA CENTRE ******************* +************** CENTRE CANADIEN DE DONNÉES ASTRONOMIQUES ************** +* +* (c) 2024. (c) 2024. +* Government of Canada Gouvernement du Canada +* National Research Council Conseil national de recherches +* Ottawa, Canada, K1A 0R6 Ottawa, Canada, K1A 0R6 +* All rights reserved Tous droits réservés +* +* NRC disclaims any warranties, Le CNRC dénie toute garantie +* expressed, implied, or énoncée, implicite ou légale, +* statutory, of any kind with de quelque nature que ce +* respect to the software, soit, concernant le logiciel, +* including without limitation y compris sans restriction +* any warranty of merchantability toute garantie de valeur +* or fitness for a particular marchande ou de pertinence +* purpose. NRC shall not be pour un usage particulier. +* liable in any event for any Le CNRC ne pourra en aucun cas +* damages, whether direct or être tenu responsable de tout +* indirect, special or general, dommage, direct ou indirect, +* consequential or incidental, particulier ou général, +* arising from the use of the accessoire ou fortuit, résultant +* software. Neither the name de l'utilisation du logiciel. Ni +* of the National Research le nom du Conseil National de +* Council of Canada nor the Recherches du Canada ni les noms +* names of its contributors may de ses participants ne peuvent +* be used to endorse or promote être utilisés pour approuver ou +* products derived from this promouvoir les produits dérivés +* software without specific prior de ce logiciel sans autorisation +* written permission. préalable et particulière +* par écrit. +* +* This file is part of the Ce fichier fait partie du projet +* OpenCADC project. OpenCADC. +* +* OpenCADC is free software: OpenCADC est un logiciel libre ; +* you can redistribute it and/or vous pouvez le redistribuer ou le +* modify it under the terms of modifier suivant les termes de +* the GNU Affero General Public la “GNU Affero General Public +* License as published by the License” telle que publiée +* Free Software Foundation, par la Free Software Foundation +* either version 3 of the : soit la version 3 de cette +* License, or (at your option) licence, soit (à votre gré) +* any later version. toute version ultérieure. +* +* OpenCADC is distributed in the OpenCADC est distribué +* hope that it will be useful, dans l’espoir qu’il vous +* but WITHOUT ANY WARRANTY; sera utile, mais SANS AUCUNE +* without even the implied GARANTIE : sans même la garantie +* warranty of MERCHANTABILITY implicite de COMMERCIALISABILITÉ +* or FITNESS FOR A PARTICULAR ni d’ADÉQUATION À UN OBJECTIF +* PURPOSE. See the GNU Affero PARTICULIER. Consultez la Licence +* General Public License for Générale Publique GNU Affero +* more details. pour plus de détails. +* +* You should have received Vous devriez avoir reçu une +* a copy of the GNU Affero copie de la Licence Générale +* General Public License along Publique GNU Affero avec +* with OpenCADC. If not, see OpenCADC ; si ce n’est +* . pas le cas, consultez : +* . +* +************************************************************************ +*/ + +package org.opencadc.vault.metadata; + +import java.net.URI; +import java.util.Date; +import java.util.UUID; +import org.apache.log4j.Logger; +import org.opencadc.inventory.Artifact; +import org.opencadc.inventory.Namespace; +import org.opencadc.inventory.db.ArtifactDAO; +import org.opencadc.inventory.db.HarvestState; +import org.opencadc.inventory.db.HarvestStateDAO; +import org.opencadc.vospace.db.DataNodeSizeWorker; + +/** + * Main artifact-sync agent that enables incremental sync of Artifact + * metadata to Node. + * + * @author pdowler + */ +public class DataNodeSizeSync implements Runnable { + private static final Logger log = Logger.getLogger(DataNodeSizeSync.class); + + private static final long ROUNDS = 6000L; // 6 sec + private static final long SHORT_SLEEP = 5 * ROUNDS; + private static final long LONG_SLEEP = 10 * ROUNDS; + private static final long EVICT_AGE = 12 * ROUNDS; + + private static final long FAIL_SLEEP = 10 * ROUNDS; // slightly smaller than evict + + private final UUID instanceID = UUID.randomUUID(); + private final HarvestStateDAO stateDAO; + private final ArtifactDAO artifactDAO; + private final Namespace artifactNamespace; + + private boolean offline = false; + + public DataNodeSizeSync(HarvestStateDAO stateDAO, ArtifactDAO artifactDAO, Namespace artifactNamespace) { + this.stateDAO = stateDAO; + this.artifactDAO = artifactDAO; + this.artifactNamespace = artifactNamespace; + + // we need continuous timestamp updates to retain leader status, so only schedule maintenance + stateDAO.setUpdateBufferCount(0); + stateDAO.setMaintCount(9999); // every 1e4 + } + + public void setOffline(boolean offline) { + this.offline = offline; + } + + @Override + public void run() { + String name = Artifact.class.getSimpleName(); + URI resourceID = URI.create("jdbc:inventory"); + try { + Thread.sleep(1 * ROUNDS); // delay startup a bit + + while (true) { + while (offline) { + log.warn("disabled: sleep=" + LONG_SLEEP); + Thread.sleep(LONG_SLEEP); + } + + log.debug("check leader " + instanceID); + HarvestState state = stateDAO.get(name, resourceID); + log.debug("check leader " + instanceID + " found: " + state); + if (state.instanceID == null) { + state.instanceID = instanceID; + stateDAO.put(state); + state = stateDAO.get(state.getID()); + log.debug("created: " + state); + } + + long t1 = System.currentTimeMillis(); + BackgroundLogInfo logInfo = new BackgroundLogInfo(instanceID.toString()); + logInfo.setOperation(DataNodeSizeWorker.class.getSimpleName()); + logInfo.setSuccess(false); + + // determine leader + boolean leader = checkLeaderStatus(state); + logInfo.leader = leader; + logInfo.setLastModified(state.curLastModified); + long sleep = LONG_SLEEP; + if (leader) { + log.debug("leader: " + state); + boolean fail = false; + log.info(logInfo.start()); + try { + DataNodeSizeWorker worker = new DataNodeSizeWorker(stateDAO, state, artifactDAO, artifactNamespace); + worker.run(); + logInfo.setLastModified(state.curLastModified); + logInfo.processed = worker.getNumArtifactsProcessed(); + } catch (Exception ex) { + log.error("unexpected worker fail", ex); + fail = true; + } + + leader = checkLeaderStatus(state); + if (!fail && leader) { + try { + stateDAO.flushBufferedState(); + } catch (Exception ex) { + log.error("unexpected HarvestState flush fail", ex); + fail = true; + } + } + + if (!fail && leader) { + try { + // update leader timestamp before sleeping + stateDAO.put(state, true); + } catch (Exception ex) { + log.error("unexpected HarvestState force update fail", ex); + fail = true; + } + } + + if (fail) { + sleep = FAIL_SLEEP; + } else { + sleep = SHORT_SLEEP; + } + logInfo.setSuccess(!fail); + logInfo.setElapsedTime(System.currentTimeMillis() - t1); + } else { + // not leader success + sleep = LONG_SLEEP; + logInfo.setSuccess(true); + } + logInfo.sleep = sleep; + log.info(logInfo.end()); + Thread.sleep(sleep); + } + } catch (InterruptedException ex) { + log.debug("interrupted - assuming shutdown", ex); + } + } + + private boolean checkLeaderStatus(HarvestState state) { + boolean leader = false; + if (instanceID.equals(state.instanceID)) { + stateDAO.put(state, true); + leader = true; + } else { + // see if we should perform a coup... + Date now = new Date(); + long age = now.getTime() - state.getLastModified().getTime(); + if (age > EVICT_AGE) { + log.info("EVICTING " + state.instanceID + " because age " + age + " > " + EVICT_AGE); + state.instanceID = instanceID; + stateDAO.put(state); + leader = true; + } + } + return leader; + } +} diff --git a/vault/src/main/webapp/META-INF/context.xml b/vault/src/main/webapp/META-INF/context.xml index 94ec7b42f..02f5f7090 100644 --- a/vault/src/main/webapp/META-INF/context.xml +++ b/vault/src/main/webapp/META-INF/context.xml @@ -27,14 +27,15 @@ removeAbandoned="false" testOnBorrow="true" validationQuery="select 123" /> - - + 1 + + + AvailabilityServlet + ca.nrc.cadc.vosi.AvailabilityServlet + + ca.nrc.cadc.vosi.AvailabilityPlugin + org.opencadc.vault.ServiceAvailability + + 2 + + + + + CapabilitiesServlet + ca.nrc.cadc.rest.RestServlet + + init + ca.nrc.cadc.vosi.CapInitAction + + + head + ca.nrc.cadc.vosi.CapHeadAction + + + get + ca.nrc.cadc.vosi.CapGetAction + + + input + /capabilities.xml + + 2 + + NodesServlet ca.nrc.cadc.rest.RestServlet @@ -64,7 +98,7 @@ delete org.opencadc.vospace.server.actions.DeleteNodeAction - 2 + 3 @@ -208,44 +242,6 @@ - - - CapabilitiesServlet - ca.nrc.cadc.rest.RestServlet - - init - ca.nrc.cadc.vosi.CapInitAction - - - head - ca.nrc.cadc.vosi.CapHeadAction - - - get - ca.nrc.cadc.vosi.CapGetAction - - - input - /capabilities.xml - - 3 - - - - - AvailabilityServlet - ca.nrc.cadc.vosi.AvailabilityServlet - - ca.nrc.cadc.vosi.AvailabilityPlugin - org.opencadc.vault.ServiceAvailability - - - availabilityProperties - vault-availability.properties - - 3 - -