Skip to content

Commit

Permalink
Merge pull request #557 from pdowler/vos2
Browse files Browse the repository at this point in the history 
cadc-inventory-db update to support node size metadata and workflow
  • Loading branch information
@pdowler
pdowler authored Mar 13, 2024
2 parents 3081552 + 22718c8 commit 347773c
Show file tree
Hide file tree
Showing 25 changed files with 958 additions and 215 deletions.
4 changes: 2 additions & 2 deletions cadc-inventory-db/build.gradle
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,10 +25,10 @@ def git_url = 'https://github.com/opencadc/storage-inventory'
mainClassName = 'org.opencadc.inventory.db.version.Main'

dependencies {
compile 'org.opencadc:cadc-util:[1.10.3,2.0)'
compile 'org.opencadc:cadc-util:[1.11.0,2.0)'
compile 'org.opencadc:cadc-gms:[1.0.0,)'
compile 'org.opencadc:cadc-inventory:[0.9.4,)'
compile 'org.opencadc:cadc-vos:[2.0,3.0)'
compile 'org.opencadc:cadc-vos:[2.0.5,3.0)'

testCompile 'junit:junit:[4.0,)'

Expand Down
67 changes: 60 additions & 7 deletions cadc-inventory-db/src/intTest/java/org/opencadc/inventory/db/ArtifactDAOTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -94,6 +94,7 @@
import org.junit.Test;
import org.opencadc.inventory.Artifact;
import org.opencadc.inventory.InventoryUtil;
import org.opencadc.inventory.Namespace;
import org.opencadc.inventory.SiteLocation;
import org.opencadc.inventory.StorageLocation;
import org.opencadc.inventory.StoredArtifactComparator;
Expand Down Expand Up @@ -955,6 +956,9 @@ public void testIteratorClose() {
public void testArtifactIterator() {
int num = 10;
try {
final Date startDate = new Date();
Thread.sleep(10L);

int numArtifacts = 0;
int numStuffExpected = 0;
// artifacts with storageLocation
Expand All @@ -977,6 +981,7 @@ public void testArtifactIterator() {
numStuffExpected++;
}
}

// some artifacts with no storageLocation
collection = "STUFF";
for (int i = num; i < 2 * num; i++) {
Expand All @@ -996,6 +1001,12 @@ public void testArtifactIterator() {
numStuffExpected++;
}
}

Thread.sleep(10L);
final Date midDate = new Date();
Thread.sleep(10L);
final int midNumStuff = numStuffExpected;

// some artifacts with siteLocations
UUID siteID = UUID.randomUUID();
int numSiteExpected = 0;
Expand All @@ -1018,6 +1029,8 @@ public void testArtifactIterator() {
numStuffExpected++;
}
}
Thread.sleep(10L);
final Date endDate = new Date();
log.info("added: " + numArtifacts);

log.info("count all...");
Expand All @@ -1032,13 +1045,14 @@ public void testArtifactIterator() {
Assert.assertEquals("count", numArtifacts, count);

log.info("count with criteria...");
final Namespace ns = new Namespace("cadc:STUFF/");
count = 0;
try (ResourceIterator<Artifact> iter = originDAO.iterator("uri like 'cadc:STUFF/%'", null, false)) {
try (ResourceIterator<Artifact> iter = originDAO.iterator(ns, null, false)) {
while (iter.hasNext()) {
Artifact actual = iter.next();
count++;
log.info("found: " + actual.getURI());
Assert.assertTrue("STUFF", actual.getURI().toASCIIString().startsWith("cadc:STUFF/"));
Assert.assertTrue("STUFF", actual.getURI().toASCIIString().startsWith(ns.getNamespace()));
}
}
Assert.assertEquals("count", numStuffExpected, count);
Expand All @@ -1065,7 +1079,8 @@ public void testArtifactIterator() {
while (iter.hasNext()) {
Artifact actual = iter.next();
count++;
log.info("found: " + actual.getURI());
log.info("found: " + actual.getBucket() + " " + actual.getURI());
Assert.assertTrue(actual.getBucket().startsWith(bpre));
}
}
}
Expand All @@ -1075,18 +1090,56 @@ public void testArtifactIterator() {
count = 0;
for (byte b = 0; b < 16; b++) {
String bpre = HexUtil.toHex(b).substring(1);
log.debug("bucket prefix: " + bpre);
try (ResourceIterator<Artifact> iter = originDAO.iterator("uri like 'cadc:STUFF/%'", bpre, false)) {
log.info("bucket prefix: " + bpre);
try (ResourceIterator<Artifact> iter = originDAO.iterator(ns, bpre, false)) {
while (iter.hasNext()) {
Artifact actual = iter.next();
count++;
log.info("found: " + actual.getURI());
Assert.assertTrue("STUFF", actual.getURI().toASCIIString().startsWith("cadc:STUFF/"));
log.info("found: " + actual.getBucket() + " " + actual.getURI());
Assert.assertTrue(actual.getBucket().startsWith(bpre));
Assert.assertTrue("STUFF", actual.getURI().toASCIIString().startsWith(ns.getNamespace()));
}
}
}
Assert.assertEquals("count", numStuffExpected, count);

log.info("count vs Namespace incremental from start...");
DateFormat df = DateUtil.getDateFormat(DateUtil.IVOA_DATE_FORMAT, DateUtil.UTC);
count = 0;
try (ResourceIterator<Artifact> iter = originDAO.iterator(ns, null, startDate, true)) {
while (iter.hasNext()) {
Artifact actual = iter.next();
count++;
log.info("found: " + actual.getBucket() + " " + actual.getURI() + " " + df.format(actual.getContentLastModified()));
Assert.assertTrue("STUFF", actual.getURI().toASCIIString().startsWith(ns.getNamespace()));
}
}
Assert.assertEquals("count", numStuffExpected, count);

log.info("count vs Namespace incremental from mid...");
count = 0;
try (ResourceIterator<Artifact> iter = originDAO.iterator(ns, null, midDate, true)) {
while (iter.hasNext()) {
Artifact actual = iter.next();
count++;
log.info("found: " + actual.getBucket() + " " + actual.getURI() + " " + df.format(actual.getContentLastModified()));
Assert.assertTrue("STUFF", actual.getURI().toASCIIString().startsWith(ns.getNamespace()));
}
}
Assert.assertEquals("count", midNumStuff, count);

log.info("count vs Namespace incremental from end...");
count = 0;
try (ResourceIterator<Artifact> iter = originDAO.iterator(ns, null, endDate, true)) {
while (iter.hasNext()) {
Artifact actual = iter.next();
count++;
log.info("found: " + actual.getBucket() + " " + actual.getURI() + " " + df.format(actual.getContentLastModified()));
Assert.assertTrue("STUFF", actual.getURI().toASCIIString().startsWith(ns.getNamespace()));
}
}
Assert.assertEquals("count", 0, count);

} catch (Exception unexpected) {
log.error("unexpected exception", unexpected);
Assert.fail("unexpected exception: " + unexpected);
Expand Down
Loading

0 comments on commit 347773c

Please sign in to comment.