diff --git a/vault/build.gradle b/vault/build.gradle index 6a637f96..ce2424e2 100644 --- a/vault/build.gradle +++ b/vault/build.gradle @@ -44,6 +44,8 @@ dependencies { compile 'org.opencadc:cadc-registry:[1.7.4,)' compile 'org.opencadc:cadc-inventory:[0.9.4,1.0)' compile 'org.opencadc:cadc-inventory-db:[0.15.0,1.0)' + compile 'org.opencadc:cadc-inventory-server:[0.3,1.0)' + compile 'org.opencadc:cadc-permissions:[0.3.5,1.0)' testCompile 'junit:junit:[4.0,)' diff --git a/vault/src/main/java/org/opencadc/vault/NodePersistenceImpl.java b/vault/src/main/java/org/opencadc/vault/NodePersistenceImpl.java index 1fb0905d..dcfe9805 100644 --- a/vault/src/main/java/org/opencadc/vault/NodePersistenceImpl.java +++ b/vault/src/main/java/org/opencadc/vault/NodePersistenceImpl.java @@ -70,8 +70,6 @@ import ca.nrc.cadc.auth.AuthenticationUtil; import ca.nrc.cadc.auth.HttpPrincipal; import ca.nrc.cadc.auth.IdentityManager; -import ca.nrc.cadc.auth.PrincipalExtractor; -import ca.nrc.cadc.auth.X509CertificateChain; import ca.nrc.cadc.date.DateUtil; import ca.nrc.cadc.db.TransactionManager; import ca.nrc.cadc.io.ResourceIterator; @@ -82,11 +80,9 @@ import ca.nrc.cadc.util.MultiValuedProperties; import java.io.IOException; import java.net.URI; -import java.security.Principal; import java.text.DateFormat; import java.util.Arrays; import java.util.Collections; -import java.util.HashSet; import java.util.Iterator; import java.util.Map; import java.util.NoSuchElementException; @@ -95,22 +91,26 @@ import java.util.TreeSet; import java.util.UUID; import javax.security.auth.Subject; -import javax.security.auth.x500.X500Principal; import org.apache.log4j.Logger; import org.opencadc.gms.GroupURI; import org.opencadc.inventory.Artifact; import org.opencadc.inventory.DeletedArtifactEvent; import org.opencadc.inventory.Namespace; +import org.opencadc.inventory.PreauthKeyPair; import org.opencadc.inventory.db.ArtifactDAO; import org.opencadc.inventory.db.DeletedArtifactEventDAO; +import org.opencadc.inventory.db.PreauthKeyPairDAO; import org.opencadc.inventory.db.SQLGenerator; +import org.opencadc.permissions.TokenTool; import org.opencadc.vospace.ContainerNode; import org.opencadc.vospace.DataNode; import org.opencadc.vospace.Node; import org.opencadc.vospace.NodeNotSupportedException; import org.opencadc.vospace.NodeProperty; import org.opencadc.vospace.VOS; +import org.opencadc.vospace.VOSURI; import org.opencadc.vospace.db.NodeDAO; +import org.opencadc.vospace.server.LocalServiceURI; import org.opencadc.vospace.server.NodePersistence; import org.opencadc.vospace.server.Views; import org.opencadc.vospace.server.transfers.TransferGenerator; @@ -225,7 +225,11 @@ public Views getViews() { @Override public TransferGenerator getTransferGenerator() { - throw new UnsupportedOperationException(); + PreauthKeyPairDAO keyDAO = new PreauthKeyPairDAO(); + keyDAO.setConfig(nodeDaoConfig); + PreauthKeyPair kp = keyDAO.get(VaultInitAction.KEY_PAIR_NAME); + TokenTool tt = new TokenTool(kp.getPublicKey(), kp.getPrivateKey()); + return new VaultTransferGenerator(this, getArtifactDAO(), tt); } private NodeDAO getDAO() { @@ -526,7 +530,31 @@ public Node put(Node node) throws NodeNotSupportedException, TransientException @Override public void move(Node node, ContainerNode dest, String newName) { - throw new UnsupportedOperationException(); + if (node == null || dest == null) { + throw new IllegalArgumentException("args cannot be null"); + } + if (node.parent == null || dest.parent == null) { + throw new IllegalArgumentException("args must both be peristent nodes before move"); + } + + Subject caller = AuthenticationUtil.getCurrentSubject(); + node.owner = caller; + node.ownerID = null; + node.ownerDisplay = null; + node.parent = dest; + node.parentID = null; + if (newName != null) { + node.setName(newName); + } + try { + Node result = put(node); + log.debug("moved: " + result); + } catch (NodeNotSupportedException ex) { + LocalServiceURI loc = new LocalServiceURI(getResourceID()); + VOSURI srcURI = loc.getURI(node); + throw new RuntimeException("BUG: failed to move node because of detected type change: " + + srcURI.getPath() + " type=" + node.getClass().getSimpleName(), ex); + } } diff --git a/vault/src/main/java/org/opencadc/vault/VaultInitAction.java b/vault/src/main/java/org/opencadc/vault/VaultInitAction.java index b33bb9d8..95b80323 100644 --- a/vault/src/main/java/org/opencadc/vault/VaultInitAction.java +++ b/vault/src/main/java/org/opencadc/vault/VaultInitAction.java @@ -71,9 +71,11 @@ import ca.nrc.cadc.rest.InitAction; import ca.nrc.cadc.util.MultiValuedProperties; import ca.nrc.cadc.util.PropertiesReader; +import ca.nrc.cadc.util.RsaSignatureGenerator; import ca.nrc.cadc.uws.server.impl.InitDatabaseUWS; import java.net.URI; import java.net.URISyntaxException; +import java.security.KeyPair; import java.util.Map; import java.util.TreeMap; import javax.naming.Context; @@ -82,8 +84,14 @@ import javax.sql.DataSource; import org.apache.log4j.Logger; import org.opencadc.inventory.Namespace; +import org.opencadc.inventory.PreauthKeyPair; +import org.opencadc.inventory.db.PreauthKeyPairDAO; +import org.opencadc.inventory.db.SQLGenerator; +import org.opencadc.inventory.db.StorageSiteDAO; +import org.opencadc.inventory.transfer.StorageSiteAvailabilityCheck; import org.opencadc.vospace.db.InitDatabaseVOS; import org.opencadc.vospace.server.NodePersistence; +import org.springframework.dao.DataIntegrityViolationException; /** * @@ -93,6 +101,8 @@ public class VaultInitAction extends InitAction { private static final Logger log = Logger.getLogger(VaultInitAction.class); + static String KEY_PAIR_NAME = "vault-preauth-keys"; + static final String JNDI_DATASOURCE = "jdbc/nodes"; // context.xml static final String JNDI_UWS_DATASOURCE = "jdbc/uws"; // context.xml @@ -112,6 +122,9 @@ public class VaultInitAction extends InitAction { private Map daoConfig; private String jndiNodePersistence; + private String jndiPreauthKeys; + private String jndiSiteAvailabilities; + private Thread availabilityCheck; public VaultInitAction() { super(); @@ -123,8 +136,29 @@ public void doInit() { initDatabase(); initUWSDatabase(); initNodePersistence(); + initKeyPair(); + initAvailabilityCheck(); } + @Override + public void doShutdown() { + try { + Context ctx = new InitialContext(); + ctx.unbind(jndiNodePersistence); + } catch (Exception oops) { + log.error("unbind failed during destroy", oops); + } + + try { + Context ctx = new InitialContext(); + ctx.unbind(jndiPreauthKeys); + } catch (Exception oops) { + log.error("unbind failed during destroy", oops); + } + + terminateAvailabilityCheck(); + } + /** * Read config file and verify that all required entries are present. * @@ -183,7 +217,9 @@ static MultiValuedProperties getConfig() { } static Map getDaoConfig(MultiValuedProperties props) { + Map ret = new TreeMap<>(); + ret.put(SQLGenerator.class.getName(), SQLGenerator.class); // not configurable right now ret.put("jndiDataSourceName", org.opencadc.vault.VaultInitAction.JNDI_DATASOURCE); ret.put("schema", props.getFirstPropertyValue(org.opencadc.vault.VaultInitAction.INVENTORY_SCHEMA_KEY)); ret.put("vosSchema", props.getFirstPropertyValue(org.opencadc.vault.VaultInitAction.VOSPACE_SCHEMA_KEY)); @@ -232,7 +268,7 @@ private void initUWSDatabase() { } } - protected void initNodePersistence() { + private void initNodePersistence() { jndiNodePersistence = appName + "-" + NodePersistence.class.getName(); try { Context ctx = new InitialContext(); @@ -249,15 +285,76 @@ protected void initNodePersistence() { log.error("Failed to create JNDI Key " + jndiNodePersistence, ex); } } - - @Override - public void doShutdown() { + + private void initKeyPair() { + log.info("initKeyPair: START"); + jndiPreauthKeys = appName + "-" + PreauthKeyPair.class.getName(); try { + PreauthKeyPairDAO dao = new PreauthKeyPairDAO(); + dao.setConfig(daoConfig); + PreauthKeyPair keys = dao.get(KEY_PAIR_NAME); + if (keys == null) { + KeyPair kp = RsaSignatureGenerator.getKeyPair(4096); + keys = new PreauthKeyPair(KEY_PAIR_NAME, kp.getPublic().getEncoded(), kp.getPrivate().getEncoded()); + try { + dao.put(keys); + log.info("initKeyPair: new keys created - OK"); + + } catch (DataIntegrityViolationException oops) { + log.warn("persist new " + PreauthKeyPair.class.getSimpleName() + " failed (" + oops + ") -- probably race condition"); + keys = dao.get(KEY_PAIR_NAME); + if (keys != null) { + log.info("race condition confirmed: another instance created keys - OK"); + } else { + throw new RuntimeException("check/init " + KEY_PAIR_NAME + " failed", oops); + } + } + } else { + log.info("initKeyPair: re-use existing keys - OK"); + } Context ctx = new InitialContext(); - ctx.unbind(jndiNodePersistence); - } catch (Exception oops) { - log.error("unbind failed during destroy", oops); + try { + ctx.unbind(jndiPreauthKeys); + } catch (NamingException ignore) { + log.debug("unbind previous JNDI key (" + jndiPreauthKeys + ") failed... ignoring"); + } + ctx.bind(jndiPreauthKeys, keys); + log.info("initKeyPair: created JNDI key: " + jndiPreauthKeys); + } catch (Exception ex) { + throw new RuntimeException("check/init " + KEY_PAIR_NAME + " failed", ex); } } + + void initAvailabilityCheck() { + StorageSiteDAO storageSiteDAO = new StorageSiteDAO(); + storageSiteDAO.setConfig(getDaoConfig(props)); + this.jndiSiteAvailabilities = appName + "-" + StorageSiteAvailabilityCheck.class.getName(); + terminateAvailabilityCheck(); + this.availabilityCheck = new Thread(new StorageSiteAvailabilityCheck(storageSiteDAO, this.jndiSiteAvailabilities)); + this.availabilityCheck.setDaemon(true); + this.availabilityCheck.start(); + } + + private final void terminateAvailabilityCheck() { + if (this.availabilityCheck != null) { + try { + log.info("terminating AvailabilityCheck Thread..."); + this.availabilityCheck.interrupt(); + this.availabilityCheck.join(); + log.info("terminating AvailabilityCheck Thread... [OK]"); + } catch (Throwable t) { + log.info("failed to terminate AvailabilityCheck thread", t); + } finally { + this.availabilityCheck = null; + } + } + try { + InitialContext initialContext = new InitialContext(); + initialContext.unbind(this.jndiSiteAvailabilities); + } catch (NamingException e) { + log.debug(String.format("unable to unbind %s - %s", this.jndiSiteAvailabilities, e.getMessage())); + } + } + } diff --git a/vault/src/main/java/org/opencadc/vault/VaultTransferGenerator.java b/vault/src/main/java/org/opencadc/vault/VaultTransferGenerator.java index abba154c..f0c8184a 100644 --- a/vault/src/main/java/org/opencadc/vault/VaultTransferGenerator.java +++ b/vault/src/main/java/org/opencadc/vault/VaultTransferGenerator.java @@ -86,16 +86,11 @@ import javax.naming.NamingException; import javax.security.auth.Subject; import org.apache.log4j.Logger; -import org.opencadc.inventory.KeyPair; import org.opencadc.inventory.db.ArtifactDAO; -import org.opencadc.inventory.db.KeyPairDAO; import org.opencadc.inventory.transfer.ProtocolsGenerator; import org.opencadc.inventory.transfer.StorageSiteAvailabilityCheck; import org.opencadc.inventory.transfer.StorageSiteRule; -import org.opencadc.permissions.Grant; -import org.opencadc.permissions.ReadGrant; import org.opencadc.permissions.TokenTool; -import org.opencadc.permissions.WriteGrant; import org.opencadc.vospace.ContainerNode; import org.opencadc.vospace.DataNode; import org.opencadc.vospace.LinkingException; @@ -116,21 +111,19 @@ public class VaultTransferGenerator implements TransferGenerator { private static final Logger log = Logger.getLogger(VaultTransferGenerator.class); - static String KEY_PAIR_NAME = "vault-preauth-keys"; - private final NodePersistenceImpl nodePersistence; private final VOSpaceAuthorizer authorizer; private final ArtifactDAO artifactDAO; - private final KeyPairDAO keyDAO; + private final TokenTool tokenTool; private Map siteRules = new HashMap<>(); private Map siteAvailabilities; - public VaultTransferGenerator(NodePersistenceImpl nodePersistence, ArtifactDAO artifactDAO, KeyPairDAO keyDAO) { + public VaultTransferGenerator(NodePersistenceImpl nodePersistence, ArtifactDAO artifactDAO, TokenTool tokenTool) { this.nodePersistence = nodePersistence; this.authorizer = new VOSpaceAuthorizer(nodePersistence); this.artifactDAO = artifactDAO; - this.keyDAO = keyDAO; + this.tokenTool = tokenTool; // TODO: get appname from ??? String siteAvailabilitiesKey = "vault" + "-" + StorageSiteAvailabilityCheck.class.getName(); @@ -190,29 +183,22 @@ private List handleDataNode(DataNode node, Transfer trans, Subject s) throws IOException, ResourceNotFoundException { log.debug("handleDataNode: " + node); - Direction dir = trans.getDirection(); - final Map params = new TreeMap<>(); // empty for now - - // vault only supports preauth URLs using persistent key pair - //KeyPair keys = keyDAO.get(KEY_PAIR_NAME); - //TokenTool tokenGen = new TokenTool(keys.getPublicKey(), keys.getPrivateKey()); - TokenTool tokenGen = null; - IdentityManager im = AuthenticationUtil.getIdentityManager(); Subject caller = AuthenticationUtil.getCurrentSubject(); Object userObject = im.toOwner(caller); String callingUser = (userObject == null ? null : userObject.toString()); - ProtocolsGenerator pg = new ProtocolsGenerator(artifactDAO, tokenGen, callingUser, siteAvailabilities, siteRules); + ProtocolsGenerator pg = new ProtocolsGenerator(artifactDAO, siteAvailabilities, siteRules); + pg.tokenGen = tokenTool; + pg.user = callingUser; + pg.requirePreauthAnon = true; //pg.preventNotFound = true; - Transfer artifactTrans = new Transfer(node.storageID, dir); + Transfer artifactTrans = new Transfer(node.storageID, trans.getDirection()); for (Protocol p : trans.getProtocols()) { log.debug("requested protocol: " + p); URI secM = p.getSecurityMethod(); - // TODO: request preauth URL specifically, not anon if (secM == null || secM.equals(Standards.SECURITY_METHOD_ANON)) { - p.setSecurityMethod(ProtocolsGenerator.SECURITY_EMBEDDED_TOKEN); artifactTrans.getProtocols().add(p); } } diff --git a/vault/src/main/webapp/WEB-INF/web.xml b/vault/src/main/webapp/WEB-INF/web.xml index 5d183135..2d479308 100644 --- a/vault/src/main/webapp/WEB-INF/web.xml +++ b/vault/src/main/webapp/WEB-INF/web.xml @@ -18,8 +18,9 @@ logLevelPackages org.opencadc.vault - ca.nrc.cadc.db + org.opencadc.inventory org.opencadc.vospace + ca.nrc.cadc.db ca.nrc.cadc.rest ca.nrc.cadc.util ca.nrc.cadc.vosi @@ -33,6 +34,19 @@ 1 + + PubKeyServlet + ca.nrc.cadc.rest.RestServlet + + augmentSubject + false + + + get + org.opencadc.vault.GetKeyAction + + + NodesServlet ca.nrc.cadc.rest.RestServlet @@ -63,6 +77,64 @@ 2 + + + SyncTransferServlet + ca.nrc.cadc.uws.server.JobServlet + + get + ca.nrc.cadc.uws.web.SyncGetAction + + + post + ca.nrc.cadc.uws.web.SyncPostAction + + + delete + ca.nrc.cadc.uws.web.DeleteAction + + + ca.nrc.cadc.uws.web.SyncPostAction.execOnPOST + true + + + ca.nrc.cadc.uws.server.JobManager + org.opencadc.vault.uws.SyncTransferManager + + + ca.nrc.cadc.rest.InlineContentHandler + org.opencadc.vospace.server.transfers.InlineTransferHandler + + 3 + + + + + AsyncTransferServlet + ca.nrc.cadc.uws.server.JobServlet + + get + ca.nrc.cadc.uws.web.GetAction + + + post + ca.nrc.cadc.uws.web.PostAction + + + delete + ca.nrc.cadc.uws.web.DeleteAction + + + ca.nrc.cadc.uws.server.JobManager + org.opencadc.vault.uws.AsyncTransferManager + + + ca.nrc.cadc.rest.InlineContentHandler + org.opencadc.vospace.server.transfers.InlineTransferHandler + + 3 + + RecursiveDeleteNodeServlet @@ -149,11 +221,28 @@ availabilityProperties vault-availability.properties - 4 + 3 + + + + TransferDetailsServlet + org.opencadc.vospace.server.transfers.TransferDetailsServlet + + ca.nrc.cadc.uws.server.JobManager + org.opencadc.vault.uws.SyncTransferManager + + 3 + + + + PubKeyServlet + /pubkey + + NodesServlet /nodes/* @@ -168,7 +257,22 @@ RecursiveNodePropsServlet /async-setprops/* - + + + AsyncTransferServlet + /transfers/* + + + + SyncTransferServlet + /synctrans/* + + + + TransferDetailsServlet + /xfer/* + + AvailabilityServlet diff --git a/vault/src/main/webapp/capabilities.xml b/vault/src/main/webapp/capabilities.xml index 28cfb09b..bd833eb7 100644 --- a/vault/src/main/webapp/capabilities.xml +++ b/vault/src/main/webapp/capabilities.xml @@ -61,7 +61,6 @@ - - - - - + - - - - - -