Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

vault: preventNotFound in files endpoint #576

Merged
merged 2 commits into from
Apr 10, 2024
Merged

vault: preventNotFound in files endpoint #576

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
23f6077
cadc-inventory-server: expose resolved artifact for preventNotFound
pdowler Apr 10, 2024
b54083a
vault: make preventNotFound work in files endpoint
pdowler Apr 10, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion cadc-inventory-server/build.gradle
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ repositories {

sourceCompatibility = 1.8
group = 'org.opencadc'
version = '0.3.0'
version = '0.3.1'

description = 'OpenCADC Storage Inventory server utility library'
def git_url = 'https://github.com/opencadc/storage-inventory'
Expand Down
15 changes: 12 additions & 3 deletions cadc-inventory-server/src/main/java/org/opencadc/inventory/transfer/ProtocolsGenerator.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -159,6 +159,11 @@ public class ProtocolsGenerator {

// for use by FilesAction subclasses to enhance logging
boolean storageResolverAdded = false;

/**
* The resolved Artifact from the database or due to preventNotFound actions.
*/
public Artifact resolvedArtifact;

public ProtocolsGenerator(ArtifactDAO artifactDAO, Map<URI, Availability> siteAvailabilities, Map<URI, StorageSiteRule> siteRules) {
this.artifactDAO = artifactDAO;
Expand Down Expand Up @@ -241,7 +246,9 @@ public Artifact getUnsyncedArtifact(URI artifactURI, Transfer transfer, Set<Stor
URL pa = new URL(sb.toString() + "/" + authToken + "/" + artifactURI.toASCIIString());
urls.add(pa);
}
urls.add(new URL(sb.append(artifactURI.toASCIIString()).toString()));
if (!requirePreauthAnon) {
urls.add(new URL(sb.append(artifactURI.toASCIIString()).toString()));
}
} catch (MalformedURLException ex) {
throw new RuntimeException("BUG: Malformed URL to the site", ex);
}
Expand Down Expand Up @@ -277,7 +284,8 @@ Artifact getRemoteArtifact(URL location, URI artifactURI) {
try {
HttpGet head = new HttpGet(location, true);
head.setHeadOnly(true);
head.setReadTimeout(10000);
head.setConnectionTimeout(6000);
head.setReadTimeout(9000);
head.run();
if (head.getResponseCode() != 200) {
// caught at the end of the method
Expand Down Expand Up @@ -348,6 +356,7 @@ List<Protocol> doPullFrom(URI artifactURI, Transfer transfer, String authToken,
}
}
log.debug(artifactURI + " found: " + artifact);
this.resolvedArtifact = artifact;

List<StorageSite> storageSites = new ArrayList<>();
if (artifact != null) {
Expand Down Expand Up @@ -411,7 +420,7 @@ List<Protocol> doPullFrom(URI artifactURI, Transfer transfer, String authToken,
log.debug("added: " + p);

// add a plain anon URL
if (authToken != null && !requirePreauthAnon && Standards.SECURITY_METHOD_ANON.equals(sec)) {
if (!requirePreauthAnon && Standards.SECURITY_METHOD_ANON.equals(sec)) {
sb = new StringBuilder();
sb.append(baseURL.toExternalForm()).append("/");
sb.append(artifactURI.toASCIIString());
Expand Down
4 changes: 2 additions & 2 deletions vault/src/main/java/org/opencadc/vault/NodePersistenceImpl.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -167,7 +167,7 @@ public class NodePersistenceImpl implements NodePersistence {

private final boolean localGroupsOnly;
private final URI resourceID;
private final boolean preventNotFound;
public final boolean preventNotFound; // GetAction needs to see this

final String appName; // access by VaultTransferGenerator

Expand Down Expand Up @@ -258,7 +258,7 @@ public Views getViews() {
}

@Override
public TransferGenerator getTransferGenerator() {
public VaultTransferGenerator getTransferGenerator() {
PreauthKeyPairDAO keyDAO = new PreauthKeyPairDAO();
keyDAO.setConfig(kpDaoConfig);
PreauthKeyPair kp = keyDAO.get(VaultInitAction.KEY_PAIR_NAME);
Expand Down
4 changes: 4 additions & 0 deletions vault/src/main/java/org/opencadc/vault/VaultTransferGenerator.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -85,6 +85,7 @@
import javax.naming.NamingException;
import javax.security.auth.Subject;
import org.apache.log4j.Logger;
import org.opencadc.inventory.Artifact;
import org.opencadc.inventory.db.ArtifactDAO;
import org.opencadc.inventory.transfer.ProtocolsGenerator;
import org.opencadc.inventory.transfer.StorageSiteAvailabilityCheck;
Expand Down Expand Up @@ -116,6 +117,8 @@ public class VaultTransferGenerator implements TransferGenerator {
private final Map<URI, StorageSiteRule> siteRules = new HashMap<>();
private final Map<URI, Availability> siteAvailabilities;

public Artifact resolvedArtifact;

@SuppressWarnings("unchecked")
public VaultTransferGenerator(NodePersistenceImpl nodePersistence, String appName,
ArtifactDAO artifactDAO, TokenTool tokenTool, boolean preventNotFound) {
Expand Down Expand Up @@ -205,6 +208,7 @@ private List<Protocol> handleDataNode(DataNode node, String filename, Transfer t
for (Protocol p : ret) {
log.debug(p.getEndpoint() + " using " + p.getSecurityMethod());
}
this.resolvedArtifact = pg.resolvedArtifact;
return ret;
} catch (ResourceNotFoundException ex) {
return new ArrayList<>();
Expand Down
31 changes: 19 additions & 12 deletions vault/src/main/java/org/opencadc/vault/files/GetAction.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -73,12 +73,12 @@
import java.util.List;
import javax.security.auth.Subject;
import org.apache.log4j.Logger;
import org.opencadc.vault.VaultTransferGenerator;
import org.opencadc.vospace.DataNode;
import org.opencadc.vospace.VOS;
import org.opencadc.vospace.VOSURI;
import org.opencadc.vospace.server.NodeFault;
import org.opencadc.vospace.server.Utils;
import org.opencadc.vospace.server.transfers.TransferGenerator;
import org.opencadc.vospace.transfer.Direction;
import org.opencadc.vospace.transfer.Protocol;
import org.opencadc.vospace.transfer.Transfer;
Expand All @@ -97,31 +97,38 @@ public GetAction() {
@Override
public void doAction() throws Exception {
// don't set content-length header since we plan to redirect
DataNode node = resolveAndSetMetadata(false);
ResolvedNode rn = resolveAndSetMetadata(false);
DataNode node = rn.node;

Subject caller = AuthenticationUtil.getCurrentSubject();
if (!voSpaceAuthorizer.hasSingleNodeReadPermission(node, caller)) {
// TODO: should output requested vos URI here
throw NodeFault.PermissionDenied.getStatus(syncInput.getPath());
}

if (node.bytesUsed == null || node.bytesUsed == 0L) {
// empty file
boolean noArtifact = node.bytesUsed == null || node.bytesUsed == 0L;
noArtifact = noArtifact && nodePersistence.preventNotFound && rn.artifact == null;
if (noArtifact) {
// no file
syncOutput.setCode(HttpURLConnection.HTTP_NO_CONTENT);
return;
}

VOSURI targetURI = localServiceURI.getURI(node);
Transfer pullTransfer = new Transfer(targetURI.getURI(), Direction.pullFromVoSpace);
pullTransfer.version = VOS.VOSPACE_21;
pullTransfer.getProtocols().add(new Protocol(VOS.PROTOCOL_HTTPS_GET)); // anon, preauth
if (rn.protos == null) {
VOSURI targetURI = localServiceURI.getURI(node);
Transfer pullTransfer = new Transfer(targetURI.getURI(), Direction.pullFromVoSpace);
pullTransfer.version = VOS.VOSPACE_21;
pullTransfer.getProtocols().add(new Protocol(VOS.PROTOCOL_HTTPS_GET)); // anon, preauth

TransferGenerator tg = nodePersistence.getTransferGenerator();
List<Protocol> protos = tg.getEndpoints(targetURI, pullTransfer, null);
if (protos.isEmpty()) {
VaultTransferGenerator tg = nodePersistence.getTransferGenerator();
rn.protos = tg.getEndpoints(targetURI, pullTransfer, null);
rn.artifact = tg.resolvedArtifact; // currently unused at this point
}

if (rn.protos.isEmpty()) {
throw new TransientException("No location found for file " + Utils.getPath(node));
}
Protocol proto = protos.get(0);
Protocol proto = rn.protos.get(0);
String loc = proto.getEndpoint();
log.debug("Location: " + loc);
syncOutput.setHeader("Location", loc);
Expand Down
94 changes: 74 additions & 20 deletions vault/src/main/java/org/opencadc/vault/files/HeadAction.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -74,9 +74,13 @@
import java.net.URI;
import java.net.URISyntaxException;
import java.util.Date;
import java.util.List;
import javax.naming.Context;
import javax.naming.InitialContext;
import org.apache.log4j.Logger;
import org.opencadc.inventory.Artifact;
import org.opencadc.vault.NodePersistenceImpl;
import org.opencadc.vault.VaultTransferGenerator;
import org.opencadc.vospace.DataNode;
import org.opencadc.vospace.Node;
import org.opencadc.vospace.VOS;
Expand All @@ -87,6 +91,9 @@
import org.opencadc.vospace.server.PathResolver;
import org.opencadc.vospace.server.Utils;
import org.opencadc.vospace.server.auth.VOSpaceAuthorizer;
import org.opencadc.vospace.transfer.Direction;
import org.opencadc.vospace.transfer.Protocol;
import org.opencadc.vospace.transfer.Transfer;

/**
* Class to handle a HEAD request to a DataNode
Expand All @@ -96,13 +103,19 @@ public class HeadAction extends RestAction {
protected static Logger log = Logger.getLogger(HeadAction.class);

protected VOSpaceAuthorizer voSpaceAuthorizer;
protected NodePersistence nodePersistence;
protected NodePersistenceImpl nodePersistence; // need impl in GetAction
protected LocalServiceURI localServiceURI;

public HeadAction() {
super();
}

protected class ResolvedNode {
DataNode node;
Artifact artifact;
List<Protocol> protos;
}

@Override
protected final InlineContentHandler getInlineContentHandler() {
return null;
Expand All @@ -120,7 +133,7 @@ public void initAction() throws Exception {
String jndiNodePersistence = super.appName + "-" + NodePersistence.class.getName();
try {
Context ctx = new InitialContext();
this.nodePersistence = ((NodePersistence) ctx.lookup(jndiNodePersistence));
this.nodePersistence = ((NodePersistenceImpl) ctx.lookup(jndiNodePersistence));
this.voSpaceAuthorizer = new VOSpaceAuthorizer(nodePersistence);
localServiceURI = new LocalServiceURI(nodePersistence.getResourceID());
} catch (Exception oops) {
Expand All @@ -135,7 +148,7 @@ public void doAction() throws Exception {
resolveAndSetMetadata(true);
}

DataNode resolveAndSetMetadata(boolean includeContentLength) throws Exception {
ResolvedNode resolveAndSetMetadata(boolean includeContentLength) throws Exception {
PathResolver pathResolver = new PathResolver(nodePersistence, voSpaceAuthorizer);
String filePath = syncInput.getPath();
Node node = pathResolver.getNode(filePath, true);
Expand All @@ -150,29 +163,70 @@ DataNode resolveAndSetMetadata(boolean includeContentLength) throws Exception {

log.debug("node path resolved: " + node.getName() + " type: " + node.getClass().getName());

DataNode dn = (DataNode) node;
if (includeContentLength) {
syncOutput.setHeader("Content-Length", dn.bytesUsed);
}
syncOutput.setHeader("Content-Disposition", "inline; filename=\"" + node.getName() + "\"");
syncOutput.setHeader("Content-Type", node.getPropertyValue(VOS.PROPERTY_URI_TYPE));
syncOutput.setHeader("Content-Encoding", node.getPropertyValue(VOS.PROPERTY_URI_CONTENTENCODING));
if (node.getPropertyValue(VOS.PROPERTY_URI_DATE) != null) {
Date lastMod = NodeWriter.getDateFormat().parse(node.getPropertyValue(VOS.PROPERTY_URI_DATE));
syncOutput.setLastModified(lastMod);
}

ResolvedNode ret = new ResolvedNode();
ret.node = (DataNode) node;
DataNode dn = ret.node;

// determine if data node is up to date or we need to find artifact
if (nodePersistence.preventNotFound && (dn.bytesUsed == null || dn.bytesUsed == 0)) {
VOSURI targetURI = localServiceURI.getURI(node);
Transfer pullTransfer = new Transfer(targetURI.getURI(), Direction.pullFromVoSpace);
pullTransfer.version = VOS.VOSPACE_21;
pullTransfer.getProtocols().add(new Protocol(VOS.PROTOCOL_HTTPS_GET)); // anon, preauth

VaultTransferGenerator tg = nodePersistence.getTransferGenerator();
ret.protos = tg.getEndpoints(targetURI, pullTransfer, null);
ret.artifact = tg.resolvedArtifact;
}

URI contentChecksum = null;
String contentMD5 = node.getPropertyValue(VOS.PROPERTY_URI_CONTENTMD5);
if (contentMD5 != null) {
try {
URI md5 = new URI("md5:" + contentMD5);
syncOutput.setDigest(md5);
} catch (URISyntaxException ex) {
throw new RuntimeException("BUG: invalid " + VOS.PROPERTY_URI_CONTENTMD5 + " value " + contentMD5);
}
contentChecksum = URI.create("md5:" + contentMD5);
}
String contentLength = null;
if (dn.bytesUsed != null) {
contentLength = dn.bytesUsed.toString();
}
String contentType = node.getPropertyValue(VOS.PROPERTY_URI_TYPE);
String contentEncoding = node.getPropertyValue(VOS.PROPERTY_URI_CONTENTENCODING);
String clm = node.getPropertyValue(VOS.PROPERTY_URI_CONTENTDATE); // use if present
if (clm == null) {
clm = node.getPropertyValue(VOS.PROPERTY_URI_DATE);
}
Date contentLastModified = null;
if (clm != null) {
contentLastModified = NodeWriter.getDateFormat().parse(clm);
}
if (ret.artifact != null) {
// preventNotFound
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This potentially overrides the values set in the previous lines which might not be very obvious. Maybe re-org so that it's more clear where the specific value originates from (artifact first then node). Just a comment.

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

basically, if preventNotFound logic was triggered above and an artifact was found, it's always right to report that metadata as it's the definitive file metadata

the "else" is potentially getting metadata for an old version of the file, which can't practically be detected

I tried several sequences and this was the most clear to me

contentChecksum = ret.artifact.getContentChecksum();
contentLength = ret.artifact.getContentLength().toString();
contentLastModified = ret.artifact.getContentLastModified();
contentType = ret.artifact.contentType;
contentEncoding = ret.artifact.contentEncoding;
}

if (includeContentLength && contentLength != null) {
syncOutput.setHeader("Content-Length", contentLength);
}
if (contentType != null) {
syncOutput.setHeader("Content-Type", contentType);
}
if (contentEncoding != null) {
syncOutput.setHeader("Content-Encoding", contentEncoding);
}
if (contentLastModified != null) {
syncOutput.setLastModified(contentLastModified);
}
if (contentChecksum != null) {
syncOutput.setDigest(contentChecksum);
}

syncOutput.setCode(200);
return dn;
return ret;
}

}
Loading