Skip to content

Commit

Permalink
Merge pull request #576 from pdowler/master
Browse files Browse the repository at this point in the history 
vault: preventNotFound in files endpoint
  • Loading branch information
@pdowler
pdowler authored Apr 10, 2024
2 parents 933a3e3 + b54083a commit 01b6866
Show file tree
Hide file tree
Showing 6 changed files with 112 additions and 38 deletions.
2 changes: 1 addition & 1 deletion cadc-inventory-server/build.gradle
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ repositories {

sourceCompatibility = 1.8
group = 'org.opencadc'
version = '0.3.0'
version = '0.3.1'

description = 'OpenCADC Storage Inventory server utility library'
def git_url = 'https://github.com/opencadc/storage-inventory'
Expand Down
15 changes: 12 additions & 3 deletions cadc-inventory-server/src/main/java/org/opencadc/inventory/transfer/ProtocolsGenerator.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -159,6 +159,11 @@ public class ProtocolsGenerator {

// for use by FilesAction subclasses to enhance logging
boolean storageResolverAdded = false;

/**
* The resolved Artifact from the database or due to preventNotFound actions.
*/
public Artifact resolvedArtifact;

public ProtocolsGenerator(ArtifactDAO artifactDAO, Map<URI, Availability> siteAvailabilities, Map<URI, StorageSiteRule> siteRules) {
this.artifactDAO = artifactDAO;
Expand Down Expand Up @@ -241,7 +246,9 @@ public Artifact getUnsyncedArtifact(URI artifactURI, Transfer transfer, Set<Stor
URL pa = new URL(sb.toString() + "/" + authToken + "/" + artifactURI.toASCIIString());
urls.add(pa);
}
urls.add(new URL(sb.append(artifactURI.toASCIIString()).toString()));
if (!requirePreauthAnon) {
urls.add(new URL(sb.append(artifactURI.toASCIIString()).toString()));
}
} catch (MalformedURLException ex) {
throw new RuntimeException("BUG: Malformed URL to the site", ex);
}
Expand Down Expand Up @@ -277,7 +284,8 @@ Artifact getRemoteArtifact(URL location, URI artifactURI) {
try {
HttpGet head = new HttpGet(location, true);
head.setHeadOnly(true);
head.setReadTimeout(10000);
head.setConnectionTimeout(6000);
head.setReadTimeout(9000);
head.run();
if (head.getResponseCode() != 200) {
// caught at the end of the method
Expand Down Expand Up @@ -348,6 +356,7 @@ List<Protocol> doPullFrom(URI artifactURI, Transfer transfer, String authToken,
}
}
log.debug(artifactURI + " found: " + artifact);
this.resolvedArtifact = artifact;

List<StorageSite> storageSites = new ArrayList<>();
if (artifact != null) {
Expand Down Expand Up @@ -411,7 +420,7 @@ List<Protocol> doPullFrom(URI artifactURI, Transfer transfer, String authToken,
log.debug("added: " + p);

// add a plain anon URL
if (authToken != null && !requirePreauthAnon && Standards.SECURITY_METHOD_ANON.equals(sec)) {
if (!requirePreauthAnon && Standards.SECURITY_METHOD_ANON.equals(sec)) {
sb = new StringBuilder();
sb.append(baseURL.toExternalForm()).append("/");
sb.append(artifactURI.toASCIIString());
Expand Down
4 changes: 2 additions & 2 deletions vault/src/main/java/org/opencadc/vault/NodePersistenceImpl.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -167,7 +167,7 @@ public class NodePersistenceImpl implements NodePersistence {

private final boolean localGroupsOnly;
private final URI resourceID;
private final boolean preventNotFound;
public final boolean preventNotFound; // GetAction needs to see this

final String appName; // access by VaultTransferGenerator

Expand Down Expand Up @@ -258,7 +258,7 @@ public Views getViews() {
}

@Override
public TransferGenerator getTransferGenerator() {
public VaultTransferGenerator getTransferGenerator() {
PreauthKeyPairDAO keyDAO = new PreauthKeyPairDAO();
keyDAO.setConfig(kpDaoConfig);
PreauthKeyPair kp = keyDAO.get(VaultInitAction.KEY_PAIR_NAME);
Expand Down
4 changes: 4 additions & 0 deletions vault/src/main/java/org/opencadc/vault/VaultTransferGenerator.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -85,6 +85,7 @@
import javax.naming.NamingException;
import javax.security.auth.Subject;
import org.apache.log4j.Logger;
import org.opencadc.inventory.Artifact;
import org.opencadc.inventory.db.ArtifactDAO;
import org.opencadc.inventory.transfer.ProtocolsGenerator;
import org.opencadc.inventory.transfer.StorageSiteAvailabilityCheck;
Expand Down Expand Up @@ -116,6 +117,8 @@ public class VaultTransferGenerator implements TransferGenerator {
private final Map<URI, StorageSiteRule> siteRules = new HashMap<>();
private final Map<URI, Availability> siteAvailabilities;

public Artifact resolvedArtifact;

@SuppressWarnings("unchecked")
public VaultTransferGenerator(NodePersistenceImpl nodePersistence, String appName,
ArtifactDAO artifactDAO, TokenTool tokenTool, boolean preventNotFound) {
Expand Down Expand Up @@ -205,6 +208,7 @@ private List<Protocol> handleDataNode(DataNode node, String filename, Transfer t
for (Protocol p : ret) {
log.debug(p.getEndpoint() + " using " + p.getSecurityMethod());
}
this.resolvedArtifact = pg.resolvedArtifact;
return ret;
} catch (ResourceNotFoundException ex) {
return new ArrayList<>();
Expand Down
31 changes: 19 additions & 12 deletions vault/src/main/java/org/opencadc/vault/files/GetAction.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -73,12 +73,12 @@
import java.util.List;
import javax.security.auth.Subject;
import org.apache.log4j.Logger;
import org.opencadc.vault.VaultTransferGenerator;
import org.opencadc.vospace.DataNode;
import org.opencadc.vospace.VOS;
import org.opencadc.vospace.VOSURI;
import org.opencadc.vospace.server.NodeFault;
import org.opencadc.vospace.server.Utils;
import org.opencadc.vospace.server.transfers.TransferGenerator;
import org.opencadc.vospace.transfer.Direction;
import org.opencadc.vospace.transfer.Protocol;
import org.opencadc.vospace.transfer.Transfer;
Expand All @@ -97,31 +97,38 @@ public GetAction() {
@Override
public void doAction() throws Exception {
// don't set content-length header since we plan to redirect
DataNode node = resolveAndSetMetadata(false);
ResolvedNode rn = resolveAndSetMetadata(false);
DataNode node = rn.node;

Subject caller = AuthenticationUtil.getCurrentSubject();
if (!voSpaceAuthorizer.hasSingleNodeReadPermission(node, caller)) {
// TODO: should output requested vos URI here
throw NodeFault.PermissionDenied.getStatus(syncInput.getPath());
}

if (node.bytesUsed == null || node.bytesUsed == 0L) {
// empty file
boolean noArtifact = node.bytesUsed == null || node.bytesUsed == 0L;
noArtifact = noArtifact && nodePersistence.preventNotFound && rn.artifact == null;
if (noArtifact) {
// no file
syncOutput.setCode(HttpURLConnection.HTTP_NO_CONTENT);
return;
}

VOSURI targetURI = localServiceURI.getURI(node);
Transfer pullTransfer = new Transfer(targetURI.getURI(), Direction.pullFromVoSpace);
pullTransfer.version = VOS.VOSPACE_21;
pullTransfer.getProtocols().add(new Protocol(VOS.PROTOCOL_HTTPS_GET)); // anon, preauth
if (rn.protos == null) {
VOSURI targetURI = localServiceURI.getURI(node);
Transfer pullTransfer = new Transfer(targetURI.getURI(), Direction.pullFromVoSpace);
pullTransfer.version = VOS.VOSPACE_21;
pullTransfer.getProtocols().add(new Protocol(VOS.PROTOCOL_HTTPS_GET)); // anon, preauth

TransferGenerator tg = nodePersistence.getTransferGenerator();
List<Protocol> protos = tg.getEndpoints(targetURI, pullTransfer, null);
if (protos.isEmpty()) {
VaultTransferGenerator tg = nodePersistence.getTransferGenerator();
rn.protos = tg.getEndpoints(targetURI, pullTransfer, null);
rn.artifact = tg.resolvedArtifact; // currently unused at this point
}

if (rn.protos.isEmpty()) {
throw new TransientException("No location found for file " + Utils.getPath(node));
}
Protocol proto = protos.get(0);
Protocol proto = rn.protos.get(0);
String loc = proto.getEndpoint();
log.debug("Location: " + loc);
syncOutput.setHeader("Location", loc);
Expand Down
94 changes: 74 additions & 20 deletions vault/src/main/java/org/opencadc/vault/files/HeadAction.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -74,9 +74,13 @@
import java.net.URI;
import java.net.URISyntaxException;
import java.util.Date;
import java.util.List;
import javax.naming.Context;
import javax.naming.InitialContext;
import org.apache.log4j.Logger;
import org.opencadc.inventory.Artifact;
import org.opencadc.vault.NodePersistenceImpl;
import org.opencadc.vault.VaultTransferGenerator;
import org.opencadc.vospace.DataNode;
import org.opencadc.vospace.Node;
import org.opencadc.vospace.VOS;
Expand All @@ -87,6 +91,9 @@
import org.opencadc.vospace.server.PathResolver;
import org.opencadc.vospace.server.Utils;
import org.opencadc.vospace.server.auth.VOSpaceAuthorizer;
import org.opencadc.vospace.transfer.Direction;
import org.opencadc.vospace.transfer.Protocol;
import org.opencadc.vospace.transfer.Transfer;

/**
* Class to handle a HEAD request to a DataNode
Expand All @@ -96,13 +103,19 @@ public class HeadAction extends RestAction {
protected static Logger log = Logger.getLogger(HeadAction.class);

protected VOSpaceAuthorizer voSpaceAuthorizer;
protected NodePersistence nodePersistence;
protected NodePersistenceImpl nodePersistence; // need impl in GetAction
protected LocalServiceURI localServiceURI;

public HeadAction() {
super();
}

protected class ResolvedNode {
DataNode node;
Artifact artifact;
List<Protocol> protos;
}

@Override
protected final InlineContentHandler getInlineContentHandler() {
return null;
Expand All @@ -120,7 +133,7 @@ public void initAction() throws Exception {
String jndiNodePersistence = super.appName + "-" + NodePersistence.class.getName();
try {
Context ctx = new InitialContext();
this.nodePersistence = ((NodePersistence) ctx.lookup(jndiNodePersistence));
this.nodePersistence = ((NodePersistenceImpl) ctx.lookup(jndiNodePersistence));
this.voSpaceAuthorizer = new VOSpaceAuthorizer(nodePersistence);
localServiceURI = new LocalServiceURI(nodePersistence.getResourceID());
} catch (Exception oops) {
Expand All @@ -135,7 +148,7 @@ public void doAction() throws Exception {
resolveAndSetMetadata(true);
}

DataNode resolveAndSetMetadata(boolean includeContentLength) throws Exception {
ResolvedNode resolveAndSetMetadata(boolean includeContentLength) throws Exception {
PathResolver pathResolver = new PathResolver(nodePersistence, voSpaceAuthorizer);
String filePath = syncInput.getPath();
Node node = pathResolver.getNode(filePath, true);
Expand All @@ -150,29 +163,70 @@ DataNode resolveAndSetMetadata(boolean includeContentLength) throws Exception {

log.debug("node path resolved: " + node.getName() + " type: " + node.getClass().getName());

DataNode dn = (DataNode) node;
if (includeContentLength) {
syncOutput.setHeader("Content-Length", dn.bytesUsed);
}
syncOutput.setHeader("Content-Disposition", "inline; filename=\"" + node.getName() + "\"");
syncOutput.setHeader("Content-Type", node.getPropertyValue(VOS.PROPERTY_URI_TYPE));
syncOutput.setHeader("Content-Encoding", node.getPropertyValue(VOS.PROPERTY_URI_CONTENTENCODING));
if (node.getPropertyValue(VOS.PROPERTY_URI_DATE) != null) {
Date lastMod = NodeWriter.getDateFormat().parse(node.getPropertyValue(VOS.PROPERTY_URI_DATE));
syncOutput.setLastModified(lastMod);
}

ResolvedNode ret = new ResolvedNode();
ret.node = (DataNode) node;
DataNode dn = ret.node;

// determine if data node is up to date or we need to find artifact
if (nodePersistence.preventNotFound && (dn.bytesUsed == null || dn.bytesUsed == 0)) {
VOSURI targetURI = localServiceURI.getURI(node);
Transfer pullTransfer = new Transfer(targetURI.getURI(), Direction.pullFromVoSpace);
pullTransfer.version = VOS.VOSPACE_21;
pullTransfer.getProtocols().add(new Protocol(VOS.PROTOCOL_HTTPS_GET)); // anon, preauth

VaultTransferGenerator tg = nodePersistence.getTransferGenerator();
ret.protos = tg.getEndpoints(targetURI, pullTransfer, null);
ret.artifact = tg.resolvedArtifact;
}

URI contentChecksum = null;
String contentMD5 = node.getPropertyValue(VOS.PROPERTY_URI_CONTENTMD5);
if (contentMD5 != null) {
try {
URI md5 = new URI("md5:" + contentMD5);
syncOutput.setDigest(md5);
} catch (URISyntaxException ex) {
throw new RuntimeException("BUG: invalid " + VOS.PROPERTY_URI_CONTENTMD5 + " value " + contentMD5);
}
contentChecksum = URI.create("md5:" + contentMD5);
}
String contentLength = null;
if (dn.bytesUsed != null) {
contentLength = dn.bytesUsed.toString();
}
String contentType = node.getPropertyValue(VOS.PROPERTY_URI_TYPE);
String contentEncoding = node.getPropertyValue(VOS.PROPERTY_URI_CONTENTENCODING);
String clm = node.getPropertyValue(VOS.PROPERTY_URI_CONTENTDATE); // use if present
if (clm == null) {
clm = node.getPropertyValue(VOS.PROPERTY_URI_DATE);
}
Date contentLastModified = null;
if (clm != null) {
contentLastModified = NodeWriter.getDateFormat().parse(clm);
}
if (ret.artifact != null) {
// preventNotFound
contentChecksum = ret.artifact.getContentChecksum();
contentLength = ret.artifact.getContentLength().toString();
contentLastModified = ret.artifact.getContentLastModified();
contentType = ret.artifact.contentType;
contentEncoding = ret.artifact.contentEncoding;
}

if (includeContentLength && contentLength != null) {
syncOutput.setHeader("Content-Length", contentLength);
}
if (contentType != null) {
syncOutput.setHeader("Content-Type", contentType);
}
if (contentEncoding != null) {
syncOutput.setHeader("Content-Encoding", contentEncoding);
}
if (contentLastModified != null) {
syncOutput.setLastModified(contentLastModified);
}
if (contentChecksum != null) {
syncOutput.setDigest(contentChecksum);
}

syncOutput.setCode(200);
return dn;
return ret;
}

}

0 comments on commit 01b6866

Please sign in to comment.