From 23f607741776dec17ce04ce24c1d38e511c6156f Mon Sep 17 00:00:00 2001 From: Patrick Dowler Date: Wed, 10 Apr 2024 12:43:44 -0700 Subject: [PATCH 1/2] cadc-inventory-server: expose resolved artifact for preventNotFound --- cadc-inventory-server/build.gradle | 2 +- .../inventory/transfer/ProtocolsGenerator.java | 15 ++++++++++++--- 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/cadc-inventory-server/build.gradle b/cadc-inventory-server/build.gradle index 8c071669..cfe7d425 100644 --- a/cadc-inventory-server/build.gradle +++ b/cadc-inventory-server/build.gradle @@ -12,7 +12,7 @@ repositories { sourceCompatibility = 1.8 group = 'org.opencadc' -version = '0.3.0' +version = '0.3.1' description = 'OpenCADC Storage Inventory server utility library' def git_url = 'https://github.com/opencadc/storage-inventory' diff --git a/cadc-inventory-server/src/main/java/org/opencadc/inventory/transfer/ProtocolsGenerator.java b/cadc-inventory-server/src/main/java/org/opencadc/inventory/transfer/ProtocolsGenerator.java index d8fca2f0..79ee7f3f 100644 --- a/cadc-inventory-server/src/main/java/org/opencadc/inventory/transfer/ProtocolsGenerator.java +++ b/cadc-inventory-server/src/main/java/org/opencadc/inventory/transfer/ProtocolsGenerator.java @@ -159,6 +159,11 @@ public class ProtocolsGenerator { // for use by FilesAction subclasses to enhance logging boolean storageResolverAdded = false; + + /** + * The resolved Artifact from the database or due to preventNotFound actions. + */ + public Artifact resolvedArtifact; public ProtocolsGenerator(ArtifactDAO artifactDAO, Map siteAvailabilities, Map siteRules) { this.artifactDAO = artifactDAO; @@ -241,7 +246,9 @@ public Artifact getUnsyncedArtifact(URI artifactURI, Transfer transfer, Set doPullFrom(URI artifactURI, Transfer transfer, String authToken, } } log.debug(artifactURI + " found: " + artifact); + this.resolvedArtifact = artifact; List storageSites = new ArrayList<>(); if (artifact != null) { @@ -411,7 +420,7 @@ List doPullFrom(URI artifactURI, Transfer transfer, String authToken, log.debug("added: " + p); // add a plain anon URL - if (authToken != null && !requirePreauthAnon && Standards.SECURITY_METHOD_ANON.equals(sec)) { + if (!requirePreauthAnon && Standards.SECURITY_METHOD_ANON.equals(sec)) { sb = new StringBuilder(); sb.append(baseURL.toExternalForm()).append("/"); sb.append(artifactURI.toASCIIString()); From b54083a3cdf1be0fef3833c1a2456fa1c1c1da76 Mon Sep 17 00:00:00 2001 From: Patrick Dowler Date: Wed, 10 Apr 2024 12:44:18 -0700 Subject: [PATCH 2/2] vault: make preventNotFound work in files endpoint --- .../opencadc/vault/NodePersistenceImpl.java | 4 +- .../vault/VaultTransferGenerator.java | 4 + .../org/opencadc/vault/files/GetAction.java | 31 +++--- .../org/opencadc/vault/files/HeadAction.java | 94 +++++++++++++++---- 4 files changed, 99 insertions(+), 34 deletions(-) diff --git a/vault/src/main/java/org/opencadc/vault/NodePersistenceImpl.java b/vault/src/main/java/org/opencadc/vault/NodePersistenceImpl.java index bf2fbb38..91ed9b69 100644 --- a/vault/src/main/java/org/opencadc/vault/NodePersistenceImpl.java +++ b/vault/src/main/java/org/opencadc/vault/NodePersistenceImpl.java @@ -167,7 +167,7 @@ public class NodePersistenceImpl implements NodePersistence { private final boolean localGroupsOnly; private final URI resourceID; - private final boolean preventNotFound; + public final boolean preventNotFound; // GetAction needs to see this final String appName; // access by VaultTransferGenerator @@ -258,7 +258,7 @@ public Views getViews() { } @Override - public TransferGenerator getTransferGenerator() { + public VaultTransferGenerator getTransferGenerator() { PreauthKeyPairDAO keyDAO = new PreauthKeyPairDAO(); keyDAO.setConfig(kpDaoConfig); PreauthKeyPair kp = keyDAO.get(VaultInitAction.KEY_PAIR_NAME); diff --git a/vault/src/main/java/org/opencadc/vault/VaultTransferGenerator.java b/vault/src/main/java/org/opencadc/vault/VaultTransferGenerator.java index 3b088488..dfd96025 100644 --- a/vault/src/main/java/org/opencadc/vault/VaultTransferGenerator.java +++ b/vault/src/main/java/org/opencadc/vault/VaultTransferGenerator.java @@ -85,6 +85,7 @@ import javax.naming.NamingException; import javax.security.auth.Subject; import org.apache.log4j.Logger; +import org.opencadc.inventory.Artifact; import org.opencadc.inventory.db.ArtifactDAO; import org.opencadc.inventory.transfer.ProtocolsGenerator; import org.opencadc.inventory.transfer.StorageSiteAvailabilityCheck; @@ -116,6 +117,8 @@ public class VaultTransferGenerator implements TransferGenerator { private final Map siteRules = new HashMap<>(); private final Map siteAvailabilities; + public Artifact resolvedArtifact; + @SuppressWarnings("unchecked") public VaultTransferGenerator(NodePersistenceImpl nodePersistence, String appName, ArtifactDAO artifactDAO, TokenTool tokenTool, boolean preventNotFound) { @@ -205,6 +208,7 @@ private List handleDataNode(DataNode node, String filename, Transfer t for (Protocol p : ret) { log.debug(p.getEndpoint() + " using " + p.getSecurityMethod()); } + this.resolvedArtifact = pg.resolvedArtifact; return ret; } catch (ResourceNotFoundException ex) { return new ArrayList<>(); diff --git a/vault/src/main/java/org/opencadc/vault/files/GetAction.java b/vault/src/main/java/org/opencadc/vault/files/GetAction.java index bc5722ee..8fc2bca5 100644 --- a/vault/src/main/java/org/opencadc/vault/files/GetAction.java +++ b/vault/src/main/java/org/opencadc/vault/files/GetAction.java @@ -73,12 +73,12 @@ import java.util.List; import javax.security.auth.Subject; import org.apache.log4j.Logger; +import org.opencadc.vault.VaultTransferGenerator; import org.opencadc.vospace.DataNode; import org.opencadc.vospace.VOS; import org.opencadc.vospace.VOSURI; import org.opencadc.vospace.server.NodeFault; import org.opencadc.vospace.server.Utils; -import org.opencadc.vospace.server.transfers.TransferGenerator; import org.opencadc.vospace.transfer.Direction; import org.opencadc.vospace.transfer.Protocol; import org.opencadc.vospace.transfer.Transfer; @@ -97,7 +97,8 @@ public GetAction() { @Override public void doAction() throws Exception { // don't set content-length header since we plan to redirect - DataNode node = resolveAndSetMetadata(false); + ResolvedNode rn = resolveAndSetMetadata(false); + DataNode node = rn.node; Subject caller = AuthenticationUtil.getCurrentSubject(); if (!voSpaceAuthorizer.hasSingleNodeReadPermission(node, caller)) { @@ -105,23 +106,29 @@ public void doAction() throws Exception { throw NodeFault.PermissionDenied.getStatus(syncInput.getPath()); } - if (node.bytesUsed == null || node.bytesUsed == 0L) { - // empty file + boolean noArtifact = node.bytesUsed == null || node.bytesUsed == 0L; + noArtifact = noArtifact && nodePersistence.preventNotFound && rn.artifact == null; + if (noArtifact) { + // no file syncOutput.setCode(HttpURLConnection.HTTP_NO_CONTENT); return; } - VOSURI targetURI = localServiceURI.getURI(node); - Transfer pullTransfer = new Transfer(targetURI.getURI(), Direction.pullFromVoSpace); - pullTransfer.version = VOS.VOSPACE_21; - pullTransfer.getProtocols().add(new Protocol(VOS.PROTOCOL_HTTPS_GET)); // anon, preauth + if (rn.protos == null) { + VOSURI targetURI = localServiceURI.getURI(node); + Transfer pullTransfer = new Transfer(targetURI.getURI(), Direction.pullFromVoSpace); + pullTransfer.version = VOS.VOSPACE_21; + pullTransfer.getProtocols().add(new Protocol(VOS.PROTOCOL_HTTPS_GET)); // anon, preauth - TransferGenerator tg = nodePersistence.getTransferGenerator(); - List protos = tg.getEndpoints(targetURI, pullTransfer, null); - if (protos.isEmpty()) { + VaultTransferGenerator tg = nodePersistence.getTransferGenerator(); + rn.protos = tg.getEndpoints(targetURI, pullTransfer, null); + rn.artifact = tg.resolvedArtifact; // currently unused at this point + } + + if (rn.protos.isEmpty()) { throw new TransientException("No location found for file " + Utils.getPath(node)); } - Protocol proto = protos.get(0); + Protocol proto = rn.protos.get(0); String loc = proto.getEndpoint(); log.debug("Location: " + loc); syncOutput.setHeader("Location", loc); diff --git a/vault/src/main/java/org/opencadc/vault/files/HeadAction.java b/vault/src/main/java/org/opencadc/vault/files/HeadAction.java index 58a85b3f..77d798ee 100644 --- a/vault/src/main/java/org/opencadc/vault/files/HeadAction.java +++ b/vault/src/main/java/org/opencadc/vault/files/HeadAction.java @@ -74,9 +74,13 @@ import java.net.URI; import java.net.URISyntaxException; import java.util.Date; +import java.util.List; import javax.naming.Context; import javax.naming.InitialContext; import org.apache.log4j.Logger; +import org.opencadc.inventory.Artifact; +import org.opencadc.vault.NodePersistenceImpl; +import org.opencadc.vault.VaultTransferGenerator; import org.opencadc.vospace.DataNode; import org.opencadc.vospace.Node; import org.opencadc.vospace.VOS; @@ -87,6 +91,9 @@ import org.opencadc.vospace.server.PathResolver; import org.opencadc.vospace.server.Utils; import org.opencadc.vospace.server.auth.VOSpaceAuthorizer; +import org.opencadc.vospace.transfer.Direction; +import org.opencadc.vospace.transfer.Protocol; +import org.opencadc.vospace.transfer.Transfer; /** * Class to handle a HEAD request to a DataNode @@ -96,13 +103,19 @@ public class HeadAction extends RestAction { protected static Logger log = Logger.getLogger(HeadAction.class); protected VOSpaceAuthorizer voSpaceAuthorizer; - protected NodePersistence nodePersistence; + protected NodePersistenceImpl nodePersistence; // need impl in GetAction protected LocalServiceURI localServiceURI; public HeadAction() { super(); } + protected class ResolvedNode { + DataNode node; + Artifact artifact; + List protos; + } + @Override protected final InlineContentHandler getInlineContentHandler() { return null; @@ -120,7 +133,7 @@ public void initAction() throws Exception { String jndiNodePersistence = super.appName + "-" + NodePersistence.class.getName(); try { Context ctx = new InitialContext(); - this.nodePersistence = ((NodePersistence) ctx.lookup(jndiNodePersistence)); + this.nodePersistence = ((NodePersistenceImpl) ctx.lookup(jndiNodePersistence)); this.voSpaceAuthorizer = new VOSpaceAuthorizer(nodePersistence); localServiceURI = new LocalServiceURI(nodePersistence.getResourceID()); } catch (Exception oops) { @@ -135,7 +148,7 @@ public void doAction() throws Exception { resolveAndSetMetadata(true); } - DataNode resolveAndSetMetadata(boolean includeContentLength) throws Exception { + ResolvedNode resolveAndSetMetadata(boolean includeContentLength) throws Exception { PathResolver pathResolver = new PathResolver(nodePersistence, voSpaceAuthorizer); String filePath = syncInput.getPath(); Node node = pathResolver.getNode(filePath, true); @@ -150,29 +163,70 @@ DataNode resolveAndSetMetadata(boolean includeContentLength) throws Exception { log.debug("node path resolved: " + node.getName() + " type: " + node.getClass().getName()); - DataNode dn = (DataNode) node; - if (includeContentLength) { - syncOutput.setHeader("Content-Length", dn.bytesUsed); - } syncOutput.setHeader("Content-Disposition", "inline; filename=\"" + node.getName() + "\""); - syncOutput.setHeader("Content-Type", node.getPropertyValue(VOS.PROPERTY_URI_TYPE)); - syncOutput.setHeader("Content-Encoding", node.getPropertyValue(VOS.PROPERTY_URI_CONTENTENCODING)); - if (node.getPropertyValue(VOS.PROPERTY_URI_DATE) != null) { - Date lastMod = NodeWriter.getDateFormat().parse(node.getPropertyValue(VOS.PROPERTY_URI_DATE)); - syncOutput.setLastModified(lastMod); - } + + ResolvedNode ret = new ResolvedNode(); + ret.node = (DataNode) node; + DataNode dn = ret.node; + + // determine if data node is up to date or we need to find artifact + if (nodePersistence.preventNotFound && (dn.bytesUsed == null || dn.bytesUsed == 0)) { + VOSURI targetURI = localServiceURI.getURI(node); + Transfer pullTransfer = new Transfer(targetURI.getURI(), Direction.pullFromVoSpace); + pullTransfer.version = VOS.VOSPACE_21; + pullTransfer.getProtocols().add(new Protocol(VOS.PROTOCOL_HTTPS_GET)); // anon, preauth + VaultTransferGenerator tg = nodePersistence.getTransferGenerator(); + ret.protos = tg.getEndpoints(targetURI, pullTransfer, null); + ret.artifact = tg.resolvedArtifact; + } + + URI contentChecksum = null; String contentMD5 = node.getPropertyValue(VOS.PROPERTY_URI_CONTENTMD5); if (contentMD5 != null) { - try { - URI md5 = new URI("md5:" + contentMD5); - syncOutput.setDigest(md5); - } catch (URISyntaxException ex) { - throw new RuntimeException("BUG: invalid " + VOS.PROPERTY_URI_CONTENTMD5 + " value " + contentMD5); - } + contentChecksum = URI.create("md5:" + contentMD5); + } + String contentLength = null; + if (dn.bytesUsed != null) { + contentLength = dn.bytesUsed.toString(); + } + String contentType = node.getPropertyValue(VOS.PROPERTY_URI_TYPE); + String contentEncoding = node.getPropertyValue(VOS.PROPERTY_URI_CONTENTENCODING); + String clm = node.getPropertyValue(VOS.PROPERTY_URI_CONTENTDATE); // use if present + if (clm == null) { + clm = node.getPropertyValue(VOS.PROPERTY_URI_DATE); } + Date contentLastModified = null; + if (clm != null) { + contentLastModified = NodeWriter.getDateFormat().parse(clm); + } + if (ret.artifact != null) { + // preventNotFound + contentChecksum = ret.artifact.getContentChecksum(); + contentLength = ret.artifact.getContentLength().toString(); + contentLastModified = ret.artifact.getContentLastModified(); + contentType = ret.artifact.contentType; + contentEncoding = ret.artifact.contentEncoding; + } + + if (includeContentLength && contentLength != null) { + syncOutput.setHeader("Content-Length", contentLength); + } + if (contentType != null) { + syncOutput.setHeader("Content-Type", contentType); + } + if (contentEncoding != null) { + syncOutput.setHeader("Content-Encoding", contentEncoding); + } + if (contentLastModified != null) { + syncOutput.setLastModified(contentLastModified); + } + if (contentChecksum != null) { + syncOutput.setDigest(contentChecksum); + } + syncOutput.setCode(200); - return dn; + return ret; } }