From 6e5639608ead99a934336364a133c5ccb240c635 Mon Sep 17 00:00:00 2001 From: Dustin Jenkins Date: Wed, 24 Apr 2024 09:48:51 -0700 Subject: [PATCH 1/2] Configured to use Package endpoints. --- VERSION | 2 +- .../web/resources/FileItemServerResource.java | 7 +- .../web/resources/PackageServerResource.java | 164 +++++++++--------- .../resources/StorageItemServerResource.java | 7 + src/main/webapp/js/filemanager.js | 135 ++++---------- .../webapp/scripts/filemanager.config.json | 16 +- src/main/webapp/themes/canfar/_main.ftl | 12 +- src/main/webapp/themes/src/_main.ftl | 5 +- 8 files changed, 150 insertions(+), 198 deletions(-) diff --git a/VERSION b/VERSION index 5fb90bc..d988d01 100644 --- a/VERSION +++ b/VERSION @@ -1,6 +1,6 @@ ## deployable containers have a semantic and build tag # semantic version tag: major.minor # build version tag: timestamp -VERSION="1.1.5" +VERSION="1.1.6" TAGS="${VERSION} ${VERSION}-$(date -u +"%Y%m%dT%H%M%S")" unset VERSION diff --git a/src/main/java/net/canfar/storage/web/resources/FileItemServerResource.java b/src/main/java/net/canfar/storage/web/resources/FileItemServerResource.java index 2902e86..68a53b6 100644 --- a/src/main/java/net/canfar/storage/web/resources/FileItemServerResource.java +++ b/src/main/java/net/canfar/storage/web/resources/FileItemServerResource.java @@ -197,7 +197,7 @@ private URL lookupDownloadEndpoint(final URI serviceURI, final AuthMethod authMe }; for (final URI uri : downloadEndpointStandards) { - final URL serviceURL = lookupDownloadEndpoint(serviceURI, uri, authMethod); + final URL serviceURL = lookupEndpoint(serviceURI, uri, authMethod); if (serviceURL != null) { return serviceURL; } @@ -208,11 +208,6 @@ private URL lookupDownloadEndpoint(final URI serviceURI, final AuthMethod authMe + Arrays.toString(downloadEndpointStandards) + ")"); } - private URL lookupDownloadEndpoint(final URI serviceURI, final URI capabilityStandardURI, - final AuthMethod authMethod) { - return getRegistryClient().getServiceURL(serviceURI, capabilityStandardURI, authMethod); - } - String toEndpoint(final URI downloadURI) { final Transfer transfer = new Transfer(downloadURI, Direction.pullFromVoSpace); transfer.setView(new View(VOS.VIEW_DEFAULT)); diff --git a/src/main/java/net/canfar/storage/web/resources/PackageServerResource.java b/src/main/java/net/canfar/storage/web/resources/PackageServerResource.java index 3c95518..97587b1 100644 --- a/src/main/java/net/canfar/storage/web/resources/PackageServerResource.java +++ b/src/main/java/net/canfar/storage/web/resources/PackageServerResource.java @@ -68,33 +68,38 @@ package net.canfar.storage.web.resources; +import ca.nrc.cadc.auth.AuthMethod; +import ca.nrc.cadc.auth.RunnableAction; +import ca.nrc.cadc.net.FileContent; +import ca.nrc.cadc.net.HttpGet; +import ca.nrc.cadc.net.HttpPost; import ca.nrc.cadc.reg.Standards; -import ca.nrc.cadc.util.StringUtil; import net.canfar.storage.web.config.VOSpaceServiceConfig; -import org.opencadc.vospace.client.ClientTransfer; import org.opencadc.vospace.client.VOSpaceClient; -import net.canfar.storage.web.restlet.JSONRepresentation; import org.opencadc.vospace.VOS; import org.opencadc.vospace.View; +import java.io.OutputStream; +import java.io.StringWriter; import java.net.URI; -import java.util.ArrayList; -import java.util.List; -import java.util.Set; +import java.net.URL; +import java.util.*; +import java.util.stream.Collectors; + import org.apache.log4j.Logger; -import org.json.JSONArray; -import org.json.JSONException; -import org.json.JSONObject; -import org.json.JSONWriter; import org.opencadc.vospace.transfer.Direction; import org.opencadc.vospace.transfer.Protocol; import org.opencadc.vospace.transfer.Transfer; -import org.restlet.data.Status; -import org.restlet.ext.json.JsonRepresentation; +import org.opencadc.vospace.transfer.TransferWriter; +import org.restlet.data.Form; +import org.restlet.data.MediaType; +import org.restlet.representation.OutputRepresentation; import org.restlet.representation.Representation; import org.restlet.resource.Get; import org.restlet.resource.Post; +import javax.security.auth.Subject; + public class PackageServerResource extends StorageItemServerResource { private static final Logger LOGGER = Logger.getLogger(PackageServerResource.class); @@ -111,98 +116,95 @@ public PackageServerResource() { @Get("json") public Representation notSupported() { - getResponse().setStatus(Status.CLIENT_ERROR_METHOD_NOT_ALLOWED); - return new JSONRepresentation() { - @Override - public void write(final JSONWriter jsonWriter) - throws JSONException { - jsonWriter.object() - .key("msg").value("GET not supported.") - .endObject(); - } - }; + throw new UnsupportedOperationException("GET not supported."); } - @Post("json") - public Representation getPackage(final JsonRepresentation payload) throws Exception { - final JSONObject jsonObject = payload.getJsonObject(); - LOGGER.debug("getPackage input: " + jsonObject); - - List targetList = new ArrayList<>(); - final Set keySet = jsonObject.keySet(); - - String responseFormat; - if (keySet.contains("responseformat")) { - responseFormat = jsonObject.getString("responseformat"); - } else { - // default response format - responseFormat = "application/zip"; - } + @Post + public Representation getPackage(final Representation payload) throws Exception { + final Form form = new Form(payload); + final String responseFormat = determineContentType(form); + final String[] uris = form.getValuesArray("uri", true); - if (!keySet.contains("targets")) { - getResponse().setStatus(Status.CLIENT_ERROR_BAD_REQUEST); - return new JSONRepresentation() { - @Override - public void write(final JSONWriter jsonWriter) - throws JSONException { - jsonWriter.object() - .key("msg").value("no targets found.") - .endObject(); - } - }; + if (uris == null || uris.length == 0) { + throw new IllegalArgumentException("Nothing specified to download."); } else { + LOGGER.debug("Determined content type of " + responseFormat); // build target list to add to transfer - JSONArray targets = jsonObject.getJSONArray("targets"); - for (int i = 0; i < targets.length(); i++) { - URI targetURI = new URI(targets.getString(i)); - targetList.add(targetURI); + final List targetList = Arrays.stream(uris).map(URI::create).collect(Collectors.toList()); + + if (LOGGER.isDebugEnabled()) { + targetList.forEach(target -> LOGGER.debug("Sending target " + target)); } // Create the Transfer. - Transfer transfer = new Transfer(Direction.pullFromVoSpace); + final Transfer transfer = new Transfer(Direction.pullFromVoSpace); transfer.getTargets().addAll(targetList); + transfer.version = VOS.VOSPACE_21; - final List protocols = new ArrayList<>(); - protocols.add(new Protocol(VOS.PROTOCOL_HTTPS_GET)); - transfer.getProtocols().addAll(protocols); + final Protocol protocol = new Protocol(VOS.PROTOCOL_HTTPS_GET); + protocol.setSecurityMethod(Standards.SECURITY_METHOD_COOKIE); + transfer.getProtocols().add(protocol); // Add package view to request using responseFormat provided - final View packageView = new View(new URI(Standards.PKG_10.toString())); + final View packageView = new View(Standards.PKG_10); packageView.getParameters().add(new View.Parameter(VOS.PROPERTY_URI_FORMAT, responseFormat)); + transfer.setView(packageView); - transfer.version = VOS.VOSPACE_21; + final TransferWriter transferWriter = new TransferWriter(); + final StringWriter sw = new StringWriter(); + transferWriter.write(transfer, sw); + LOGGER.debug("transfer XML: " + sw); - final ClientTransfer ct = voSpaceClient.createTransfer(transfer); - // There should be one protocol in the transfer, with an endpoint - // like '/vault/pkg/{jobid}/run'. - final String packageEndpoint = ct.getTransfer().getProtocols().get(0).getEndpoint(); + final Subject currentSubject = getCurrentSubject(); + final URL dataURL = getRedirect(currentSubject, sw.toString().getBytes()); - if (StringUtil.hasLength(packageEndpoint)) { - getResponse().setStatus(Status.SUCCESS_OK); - return new JSONRepresentation() { - @Override - public void write(final JSONWriter jsonWriter) - throws JSONException { - jsonWriter.object() - .key("endpoint").value(packageEndpoint) - .key("msg").value("successfully generated package file.") - .endObject(); - } - }; + if (dataURL == null) { + throw new IllegalArgumentException("No package endpoint available."); } else { - getResponse().setStatus(Status.SUCCESS_NO_CONTENT); - return new JSONRepresentation() { + return new OutputRepresentation(new MediaType(responseFormat)) { @Override - public void write(final JSONWriter jsonWriter) - throws JSONException { - jsonWriter.object() - .key("errMsg").value("package endpoint not generated.") - .endObject(); + public void write(OutputStream outputStream) { + final HttpGet httpGet = new HttpGet(dataURL, outputStream); + Subject.doAs(currentSubject, new RunnableAction(httpGet)); } }; } } } + private URL getRedirect(final Subject currentSubject, final byte[] payload) { + // POST the transfer to synctrans + final FileContent fileContent = new FileContent(payload, "text/xml"); + final URL synctransServiceURL = lookupEndpoint(this.currentService.getResourceID(), Standards.VOSPACE_SYNC_21, + AuthMethod.ANON); + final HttpPost post = new HttpPost(synctransServiceURL, fileContent, false); + + Subject.doAs(currentSubject, new RunnableAction(post)); + return post.getRedirectURL(); + } + + private String determineContentType(final Form form) { + final String packageType = form.getFirst("method").getValue(); + return PackageTypes.fromLabel(packageType).contentType; + } + + private enum PackageTypes { + ZIP("ZIP Package", "application/zip"), + TAR("TAR Package", "application/x-tar"); + + final String label; + final String contentType; + + PackageTypes(String label, String contentType) { + this.label = label; + this.contentType = contentType; + } + + static PackageTypes fromLabel(final String label) { + return Arrays.stream(values()).filter(packageType -> packageType.label.equals(label)) + .findFirst() + .orElseThrow(() -> new NoSuchElementException("No Package with label " + label)); + } + } } diff --git a/src/main/java/net/canfar/storage/web/resources/StorageItemServerResource.java b/src/main/java/net/canfar/storage/web/resources/StorageItemServerResource.java index 8e48cc4..ca76b31 100644 --- a/src/main/java/net/canfar/storage/web/resources/StorageItemServerResource.java +++ b/src/main/java/net/canfar/storage/web/resources/StorageItemServerResource.java @@ -68,6 +68,7 @@ package net.canfar.storage.web.resources; +import ca.nrc.cadc.auth.AuthMethod; import ca.nrc.cadc.auth.NotAuthenticatedException; import ca.nrc.cadc.net.RemoteServiceException; import ca.nrc.cadc.util.StringUtil; @@ -94,6 +95,7 @@ import javax.servlet.ServletContext; import java.io.*; import java.net.URI; +import java.net.URL; import java.nio.file.Path; import java.nio.file.Paths; import java.security.*; @@ -174,6 +176,11 @@ private void initializeStorageItemFactory() { this.currentService); } + URL lookupEndpoint(final URI serviceURI, final URI capabilityStandardURI, + final AuthMethod authMethod) { + return getRegistryClient().getServiceURL(serviceURI, capabilityStandardURI, authMethod); + } + Path getCurrentPath() { if (getRequestAttributes().containsKey("path")) { final String pathInRequest = getRequestAttribute("path"); diff --git a/src/main/webapp/js/filemanager.js b/src/main/webapp/js/filemanager.js index 619c278..a056f34 100644 --- a/src/main/webapp/js/filemanager.js +++ b/src/main/webapp/js/filemanager.js @@ -2547,111 +2547,52 @@ function fileManager( } $(document).on('click', '.download-dropdown-menu > li > a', function() { - var $thisLink = $(this) - - // ZIP file is disabled for now - // 2023.05.10 - // jenkinsd - // - if ($thisLink.attr('class') === 'download-zip-file') { - - var postData = {} - postData.responseformat = 'application/zip' - var targetList = new Array(); - - $.each( - $dt - .rows({ - selected: true - }) - .data(), - function (key, itemData) { - targetList.push(itemData[10]) - } - ) + let $thisLink = $(this) - postData.targets = targetList - - $.ajax({ - method: 'POST', - url: - contextPath + - vospaceServicePath + - config.options.packageConnector, - data: JSON.stringify(postData), - contentType: 'application/json', - success: function (data, textStatus, jqXHR) { - var infoMsg = data.msg; + // Package download (ZIP or TAR) are handled separately. Default to Download Manager, otherwise. + const formAction = $thisLink.attr('class').includes('download-package') + ? `${contextPath}${vospaceServicePath}${config.options.packageConnector}` + : '/downloadManager/download' - var $anchor = $('') - .attr('href', data.endpoint) - .attr('display', 'none') + const form = document.createElement('form') + form.setAttribute('method', 'POST') + form.setAttribute('action', formAction) - // Append to the document body temporarily - // so package url can be clicked and the package - // file download handled by the browser - $('#main_section').append($anchor) - // Must trigger the native click event manually - jquery - // blocks this for links. - $anchor.get(0).click() - $anchor.remove() + const downloadMethod = config.download.methods[$thisLink.data('download-type')] + const methodHiddenField = document.createElement('input') + methodHiddenField.setAttribute('type', 'hidden') + methodHiddenField.setAttribute('name', 'method') + methodHiddenField.setAttribute('value', downloadMethod.id) - // timeout is in ms - $.prompt(lg.package_generate, {'timeout': 3500 }) - - }, - error: function (jqXHR, textStatus, errorThrown) { - var errMsg = '' - - switch (jqXHR.status) { - default: - errMsg = getErrorMsg(jqXHR, errorThrown) - } - $.prompt(errMsg) - } - }) + form.appendChild(methodHiddenField) - } else { - var downloadMethod = config.download.methods[$thisLink.attr('class')] - var form = document.createElement('form') - var formAction = '/downloadManager/download' - form.setAttribute('method', 'POST') - form.setAttribute('action', formAction) - - var methodHiddenField = document.createElement('input') - methodHiddenField.setAttribute('type', 'hidden') - methodHiddenField.setAttribute('name', 'method') - methodHiddenField.setAttribute('value', downloadMethod.id) - - form.appendChild(methodHiddenField) - - //Move the submit function to another variable - //so that it doesn't get overwritten. - form._submit_function_ = form.submit - - $.each( - $dt - .rows({ - selected: true - }) - .data(), - function (key, itemData) { - var hiddenField = document.createElement('input') - hiddenField.setAttribute('type', 'hidden') - hiddenField.setAttribute('name', 'uri') - hiddenField.setAttribute( - 'value', - config.download.vos_prefix + itemData[9] - ) + //Move the submit function to another variable + //so that it doesn't get overwritten. + form._submit_function_ = form.submit - form.appendChild(hiddenField) - } - ) + $.each( + $dt + .rows({ + selected: true + }) + .data(), + (_key, itemData) => { + const hiddenField = document.createElement('input') + hiddenField.setAttribute('type', 'hidden') + hiddenField.setAttribute('name', 'uri') + hiddenField.setAttribute( + 'value', + // config.download.vos_prefix + itemData[9] + itemData[10] + ) + + form.appendChild(hiddenField) + } + ) - document.body.appendChild(form) + document.body.appendChild(form) - form._submit_function_() - } + form._submit_function_() }) // Display an 'edit' link for editable files diff --git a/src/main/webapp/scripts/filemanager.config.json b/src/main/webapp/scripts/filemanager.config.json index a0b010b..8433088 100644 --- a/src/main/webapp/scripts/filemanager.config.json +++ b/src/main/webapp/scripts/filemanager.config.json @@ -75,18 +75,22 @@ "vos_prefix": "vos://cadc.nrc.ca~vault", "connector": "batch-download", "methods": { - "download-url-list": { + "url-list": { "id": "URL List" }, - "download-shell-script": { + "shell-script": { "id": "Shell Script" }, - "download-html-list": - { + "html-list": { "id": "HTML List" }, - "download-download-manager": - { + "package-zip": { + "id": "ZIP Package" + }, + "package-tar": { + "id": "TAR Package" + }, + "download-manager": { "id": "Java Webstart" } } diff --git a/src/main/webapp/themes/canfar/_main.ftl b/src/main/webapp/themes/canfar/_main.ftl index fd1ad98..9287034 100644 --- a/src/main/webapp/themes/canfar/_main.ftl +++ b/src/main/webapp/themes/canfar/_main.ftl @@ -94,10 +94,14 @@ diff --git a/src/main/webapp/themes/src/_main.ftl b/src/main/webapp/themes/src/_main.ftl index 9c9f850..2072b57 100644 --- a/src/main/webapp/themes/src/_main.ftl +++ b/src/main/webapp/themes/src/_main.ftl @@ -94,9 +94,8 @@ From 4e46500c4773ae49f4073a22ca58df804d890484 Mon Sep 17 00:00:00 2001 From: Dustin Jenkins Date: Tue, 26 Nov 2024 21:16:17 +0000 Subject: [PATCH 2/2] feat: add download options for zip and tar --- .dockerignore | 10 + Dockerfile | 16 +- VERSION | 2 +- build.gradle | 73 ++++++ .../web/resources/FileItemServerResource.java | 139 ++++++------ .../web/resources/PackageServerResource.java | 210 ------------------ .../web/resources/SecureServerResource.java | 67 ++---- .../resources/StorageItemServerResource.java | 86 ++++--- .../web/restlet/PackageRepresentation.java | 139 ------------ .../web/restlet/StorageApplication.java | 105 ++++----- .../org/opencadc/storage/StorageAction.java | 114 ++++++++++ .../org/opencadc/storage/pkg/PostAction.java | 164 ++++++++++++++ src/main/webapp/WEB-INF/web.xml | 14 ++ src/main/webapp/js/filemanager.js | 11 +- src/main/webapp/themes/canfar/_main.ftl | 1 - src/main/webapp/themes/default/_main.ftl | 6 +- src/main/webapp/themes/src/_main.ftl | 1 - .../opencadc/storage/pkg/PostActionTest.java | 77 +++++++ 18 files changed, 645 insertions(+), 590 deletions(-) create mode 100644 .dockerignore delete mode 100644 src/main/java/net/canfar/storage/web/resources/PackageServerResource.java delete mode 100644 src/main/java/net/canfar/storage/web/restlet/PackageRepresentation.java create mode 100644 src/main/java/org/opencadc/storage/StorageAction.java create mode 100644 src/main/java/org/opencadc/storage/pkg/PostAction.java create mode 100644 src/test/java/org/opencadc/storage/pkg/PostActionTest.java diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..369db8f --- /dev/null +++ b/.dockerignore @@ -0,0 +1,10 @@ +Dockerfile* +bin +build +.idea +.vscode +.gradle +.github +.gitlab-ci.yml +.gitignore +*.md diff --git a/Dockerfile b/Dockerfile index ab11d82..68168cc 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,3 +1,15 @@ -FROM images.opencadc.org/library/cadc-tomcat:1.2.1 +FROM eclipse-temurin:11-alpine AS base -COPY build/libs/storage.war /usr/share/tomcat/webapps/ +FROM base AS builder + +COPY . /storage-ui +WORKDIR /storage-ui +RUN apk --no-cache add git \ + && git fetch origin main \ + && ./gradlew -i clean spotlessCheck build test --no-daemon + +FROM images.opencadc.org/library/cadc-tomcat:1.3 AS production + +RUN mkdir -p /usr/share/tomcat/config + +COPY --from=builder /storage-ui/build/libs/storage.war /usr/share/tomcat/webapps/ diff --git a/VERSION b/VERSION index 832dc54..235468d 100644 --- a/VERSION +++ b/VERSION @@ -1,6 +1,6 @@ ## deployable containers have a semantic and build tag # semantic version tag: major.minor # build version tag: timestamp -VERSION="1.1.8" +VERSION="1.2.0" TAGS="${VERSION} ${VERSION}-$(date -u +"%Y%m%dT%H%M%S")" unset VERSION diff --git a/build.gradle b/build.gradle index 1aed153..c1ce823 100644 --- a/build.gradle +++ b/build.gradle @@ -1,8 +1,12 @@ plugins { id 'war' + id "com.github.node-gradle.node" version "3.0.1" + id 'com.diffplug.spotless' version '6.25.0' // IntelliJ IDEA plugin here to allow integration tests to appear properly in IDEs. id 'idea' + id 'jacoco' + id 'org.jetbrains.dokka' version '1.6.0' } repositories { @@ -38,6 +42,8 @@ dependencies { implementation 'org.restlet.jee:org.restlet.ext.json:[2.4.3,2.4.99)' testImplementation 'junit:junit:[4.12,5.0)' + testImplementation 'org.xmlunit:xmlunit-core:2.10.0' + testImplementation 'org.xmlunit:xmlunit-assertj:2.10.0' testImplementation 'org.opencadc:cadc-web-test:[2.1.0,3.0.0)' testImplementation 'org.mockito:mockito-core:[3.9.0,4.0.0)' testImplementation 'org.seleniumhq.selenium:selenium-java:[3.14,4.0)' @@ -45,6 +51,73 @@ dependencies { sourceCompatibility = 11 +spotless { + // optional: only format files which have changed since origin/main + ratchetFrom 'origin/main' + + java { + // Pass spotless:on or spotless:off + toggleOffOn() + + // Use the default importOrder configuration + importOrder() + // Remove unused imports + removeUnusedImports() + // Google Java Format, Android Open Source Project style which uses 4 spaces for indentation + palantirJavaFormat('2.50.0').formatJavadoc(true) + // Format annotations on a single line + formatAnnotations() + } + format 'misc', { + target '*.gradle' + trimTrailingWhitespace() + indentWithSpaces(4) + endWithNewline() + } +} +check.dependsOn spotlessCheck + +// Create Java Code Coverage Reports +jacocoTestReport { + reports { + xml.enabled true + html.enabled true + } +} +check.dependsOn jacocoTestReport + +// Create JavaDoc +javadoc { + destinationDir = file("${buildDir}/docs/javadoc") +} + +// Create Java Documentation using Dokka for Github Markdown and HTML +tasks.dokkaGfm.configure { + outputDirectory.set(file("${buildDir}/docs/dokka/gfm")) + dokkaSourceSets { + register("main") { + sourceRoots.from(file("src/main/java")) + } + } +} + +tasks.dokkaHtml.configure { + outputDirectory.set(file("${buildDir}/docs/dokka/html")) + dokkaSourceSets { + register("main") { + sourceRoots.from(file("src/main/java")) + } + configureEach { + jdkVersion.set(11) + sourceLink { + localDirectory.set(file("src/main/java")) + remoteUrl.set("https://github.com/opencadc/storage-ui/tree/main/src/main/java") + } + } + } +} + + war { archiveName 'storage.war' } diff --git a/src/main/java/net/canfar/storage/web/resources/FileItemServerResource.java b/src/main/java/net/canfar/storage/web/resources/FileItemServerResource.java index 611546b..7118ccd 100644 --- a/src/main/java/net/canfar/storage/web/resources/FileItemServerResource.java +++ b/src/main/java/net/canfar/storage/web/resources/FileItemServerResource.java @@ -74,15 +74,24 @@ import ca.nrc.cadc.net.ResourceNotFoundException; import ca.nrc.cadc.reg.Standards; import ca.nrc.cadc.util.StringUtil; +import java.io.IOException; +import java.io.InputStream; +import java.net.URI; +import java.net.URL; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.Arrays; +import java.util.List; +import java.util.Objects; +import java.util.Set; +import java.util.stream.Collectors; +import javax.security.auth.Subject; import net.canfar.storage.PathUtils; import net.canfar.storage.web.*; import net.canfar.storage.web.config.StorageConfiguration; import net.canfar.storage.web.config.VOSpaceServiceConfig; import net.canfar.storage.web.config.VOSpaceServiceConfigManager; import net.canfar.storage.web.restlet.JSONRepresentation; - -import javax.security.auth.Subject; - import org.apache.commons.fileupload.FileItemIterator; import org.apache.commons.fileupload.FileItemStream; import org.apache.commons.fileupload.FileUploadException; @@ -112,19 +121,6 @@ import org.restlet.resource.Put; import org.restlet.resource.ResourceException; -import java.io.IOException; -import java.io.InputStream; -import java.net.URI; -import java.net.URL; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.util.Arrays; -import java.util.List; -import java.util.Objects; -import java.util.Set; -import java.util.stream.Collectors; - - public class FileItemServerResource extends StorageItemServerResource { private static final Logger LOGGER = Logger.getLogger(FileItemServerResource.class); @@ -135,18 +131,17 @@ public class FileItemServerResource extends StorageItemServerResource { private final UploadVerifier uploadVerifier; private final FileValidator fileValidator; - private final static AuthMethod[] PROTOCOL_AUTH_METHODS = new AuthMethod[] { - AuthMethod.ANON, - AuthMethod.CERT, - AuthMethod.COOKIE - }; - - - FileItemServerResource(StorageConfiguration storageConfiguration, - VOSpaceServiceConfigManager voSpaceServiceConfigManager, - StorageItemFactory storageItemFactory, VOSpaceClient voSpaceClient, - VOSpaceServiceConfig serviceConfig, UploadVerifier uploadVerifier, - FileValidator fileValidator) { + private static final AuthMethod[] PROTOCOL_AUTH_METHODS = + new AuthMethod[] {AuthMethod.ANON, AuthMethod.CERT, AuthMethod.COOKIE}; + + FileItemServerResource( + StorageConfiguration storageConfiguration, + VOSpaceServiceConfigManager voSpaceServiceConfigManager, + StorageItemFactory storageItemFactory, + VOSpaceClient voSpaceClient, + VOSpaceServiceConfig serviceConfig, + UploadVerifier uploadVerifier, + FileValidator fileValidator) { super(storageConfiguration, voSpaceServiceConfigManager, storageItemFactory, voSpaceClient, serviceConfig); this.uploadVerifier = uploadVerifier; this.fileValidator = fileValidator; @@ -185,16 +180,14 @@ void download(final DataNode dataNode) throws Exception { /** * Check both the new prototype and old Files service lookup endpoints. - * @param serviceURI The URI that identifies the Service to use. - * @param authMethod The AuthMethod interface to pick out. - * @return URL, never null. + * + * @param serviceURI The URI that identifies the Service to use. + * @param authMethod The AuthMethod interface to pick out. + * @return URL, never null. * @throws IllegalStateException if no URL can be found. */ private URL lookupDownloadEndpoint(final URI serviceURI, final AuthMethod authMethod) { - final URI[] downloadEndpointStandards = new URI[] { - Standards.VOSPACE_FILES, - Standards.VOSPACE_FILES_20 - }; + final URI[] downloadEndpointStandards = new URI[] {Standards.VOSPACE_FILES, Standards.VOSPACE_FILES_20}; for (final URI uri : downloadEndpointStandards) { final URL serviceURL = lookupEndpoint(serviceURI, uri, authMethod); @@ -204,8 +197,8 @@ private URL lookupDownloadEndpoint(final URI serviceURI, final AuthMethod authMe } throw new IllegalStateException("Incomplete configuration in the registry. No endpoint for " - + serviceURI + " could be found from (" - + Arrays.toString(downloadEndpointStandards) + ")"); + + serviceURI + " could be found from (" + + Arrays.toString(downloadEndpointStandards) + ")"); } String toEndpoint(final URI downloadURI) { @@ -235,8 +228,7 @@ public void accept(final Representation payload) throws Exception { if (!fileItemIterator.hasNext()) { // Some problem occurs, sent back a simple line of text. - uploadError(Status.CLIENT_ERROR_BAD_REQUEST, - "Unable to upload corrupted or incompatible data."); + uploadError(Status.CLIENT_ERROR_BAD_REQUEST, "Unable to upload corrupted or incompatible data."); } else { upload(fileItemIterator); } @@ -297,9 +289,8 @@ Path upload(final FileItemStream fileItemStream) throws Exception { return PathUtils.toPath(dataNode); } else { - throw new ResourceException(new IllegalArgumentException( - String.format("Invalid file name: %s -- File name must match %s.", filename, - fileValidator.getRule()))); + throw new ResourceException(new IllegalArgumentException(String.format( + "Invalid file name: %s -- File name must match %s.", filename, fileValidator.getRule()))); } } @@ -307,13 +298,13 @@ Path upload(final FileItemStream fileItemStream) throws Exception { * Do the secure upload. * * @param inputStream The InputStream to pull from. - * @param dataNode The DataNode to upload to. - * @param contentType The file content type. + * @param dataNode The DataNode to upload to. + * @param contentType The file content type. */ protected void upload(final InputStream inputStream, final DataNode dataNode, final String contentType) throws Exception { - final UploadOutputStreamWrapper outputStreamWrapper = new UploadOutputStreamWrapperImpl(inputStream, - BUFFER_SIZE); + final UploadOutputStreamWrapper outputStreamWrapper = + new UploadOutputStreamWrapperImpl(inputStream, BUFFER_SIZE); try { // Due to a bug in VOSpace that returns a 400 while checking @@ -327,7 +318,7 @@ protected void upload(final InputStream inputStream, final DataNode dataNode, fi if (cause instanceof IllegalStateException) { final Throwable illegalStateCause = cause.getCause(); if ((illegalStateCause instanceof NodeNotFoundException) - || (illegalStateCause instanceof ResourceNotFoundException)) { + || (illegalStateCause instanceof ResourceNotFoundException)) { createNode(dataNode); } else { throw new ResourceException(Status.CLIENT_ERROR_BAD_REQUEST, e.getCause()); @@ -360,22 +351,24 @@ protected void upload(final InputStream inputStream, final DataNode dataNode, fi } /** - * Abstract away the Transfer stuff. It's cumbersome. + * Abstract away the Transfer stuff. It's cumbersome. * * @param outputStreamWrapper The OutputStream wrapper. - * @param dataNode The node to upload. - * @param contentType The file content type. + * @param dataNode The node to upload. + * @param contentType The file content type. * @throws Exception To capture transfer and upload failures. */ void upload(final UploadOutputStreamWrapper outputStreamWrapper, final DataNode dataNode, final String contentType) throws Exception { final VOSURI dataNodeVOSURI = toURI(dataNode); - final List protocols = Arrays.stream(FileItemServerResource.PROTOCOL_AUTH_METHODS).map(authMethod -> { - final Protocol httpsAuth = new Protocol(VOS.PROTOCOL_HTTPS_PUT); - httpsAuth.setSecurityMethod(Standards.getSecurityMethod(authMethod)); - return httpsAuth; - }).collect(Collectors.toList()); + final List protocols = Arrays.stream(FileItemServerResource.PROTOCOL_AUTH_METHODS) + .map(authMethod -> { + final Protocol httpsAuth = new Protocol(VOS.PROTOCOL_HTTPS_PUT); + httpsAuth.setSecurityMethod(Standards.getSecurityMethod(authMethod)); + return httpsAuth; + }) + .collect(Collectors.toList()); final Transfer transfer = new Transfer(dataNodeVOSURI.getURI(), Direction.pushToVoSpace); transfer.setView(new View(VOS.VIEW_DEFAULT)); @@ -391,8 +384,9 @@ void upload(final UploadOutputStreamWrapper outputStreamWrapper, final DataNode VOSClientUtil.checkTransferFailure(ct); if (ct.getHttpTransferDetails().getDigest() != null) { - uploadVerifier.verifyMD5(outputStreamWrapper.getCalculatedMD5(), - ct.getHttpTransferDetails().getDigest().getSchemeSpecificPart()); + uploadVerifier.verifyMD5( + outputStreamWrapper.getCalculatedMD5(), + ct.getHttpTransferDetails().getDigest().getSchemeSpecificPart()); } uploadSuccess(); @@ -401,7 +395,7 @@ void upload(final UploadOutputStreamWrapper outputStreamWrapper, final DataNode /** * Parse the representation into a Map for easier access to Form elements. * - * @return Map of field names to File Items, or empty Map. Never null. + * @return Map of field names to File Items, or empty Map. Never null. */ private ServletFileUpload parseRepresentation() { // 1/ Create a factory for disk-based file items @@ -422,26 +416,21 @@ private ServletFileUpload createFileUpload(final DiskFileItemFactory factory) { return new ServletFileUpload(factory); } - private void uploadError(final Status status, final String message) { - writeResponse(status, - new JSONRepresentation() { - @Override - public void write(final JSONWriter jsonWriter) - throws JSONException { - jsonWriter.object().key("error").value(message).endObject(); - } - }); + writeResponse(status, new JSONRepresentation() { + @Override + public void write(final JSONWriter jsonWriter) throws JSONException { + jsonWriter.object().key("error").value(message).endObject(); + } + }); } private void uploadSuccess() { - writeResponse(Status.SUCCESS_CREATED, - new JSONRepresentation() { - @Override - public void write(final JSONWriter jsonWriter) - throws JSONException { - jsonWriter.object().key("code").value(0).endObject(); - } - }); + writeResponse(Status.SUCCESS_CREATED, new JSONRepresentation() { + @Override + public void write(final JSONWriter jsonWriter) throws JSONException { + jsonWriter.object().key("code").value(0).endObject(); + } + }); } } diff --git a/src/main/java/net/canfar/storage/web/resources/PackageServerResource.java b/src/main/java/net/canfar/storage/web/resources/PackageServerResource.java deleted file mode 100644 index 97587b1..0000000 --- a/src/main/java/net/canfar/storage/web/resources/PackageServerResource.java +++ /dev/null @@ -1,210 +0,0 @@ -/* - ************************************************************************ - ******************* CANADIAN ASTRONOMY DATA CENTRE ******************* - ************** CENTRE CANADIEN DE DONNÉES ASTRONOMIQUES ************** - * - * (c) 2022. (c) 2022. - * Government of Canada Gouvernement du Canada - * National Research Council Conseil national de recherches - * Ottawa, Canada, K1A 0R6 Ottawa, Canada, K1A 0R6 - * All rights reserved Tous droits réservés - * - * NRC disclaims any warranties, Le CNRC dénie toute garantie - * expressed, implied, or énoncée, implicite ou légale, - * statutory, of any kind with de quelque nature que ce - * respect to the software, soit, concernant le logiciel, - * including without limitation y compris sans restriction - * any warranty of merchantability toute garantie de valeur - * or fitness for a particular marchande ou de pertinence - * purpose. NRC shall not be pour un usage particulier. - * liable in any event for any Le CNRC ne pourra en aucun cas - * damages, whether direct or être tenu responsable de tout - * indirect, special or general, dommage, direct ou indirect, - * consequential or incidental, particulier ou général, - * arising from the use of the accessoire ou fortuit, résultant - * software. Neither the name de l'utilisation du logiciel. Ni - * of the National Research le nom du Conseil National de - * Council of Canada nor the Recherches du Canada ni les noms - * names of its contributors may de ses participants ne peuvent - * be used to endorse or promote être utilisés pour approuver ou - * products derived from this promouvoir les produits dérivés - * software without specific prior de ce logiciel sans autorisation - * written permission. préalable et particulière - * par écrit. - * - * This file is part of the Ce fichier fait partie du projet - * OpenCADC project. OpenCADC. - * - * OpenCADC is free software: OpenCADC est un logiciel libre ; - * you can redistribute it and/or vous pouvez le redistribuer ou le - * modify it under the terms of modifier suivant les termes de - * the GNU Affero General Public la “GNU Affero General Public - * License as published by the License” telle que publiée - * Free Software Foundation, par la Free Software Foundation - * either version 3 of the : soit la version 3 de cette - * License, or (at your option) licence, soit (à votre gré) - * any later version. toute version ultérieure. - * - * OpenCADC is distributed in the OpenCADC est distribué - * hope that it will be useful, dans l’espoir qu’il vous - * but WITHOUT ANY WARRANTY; sera utile, mais SANS AUCUNE - * without even the implied GARANTIE : sans même la garantie - * warranty of MERCHANTABILITY implicite de COMMERCIALISABILITÉ - * or FITNESS FOR A PARTICULAR ni d’ADÉQUATION À UN OBJECTIF - * PURPOSE. See the GNU Affero PARTICULIER. Consultez la Licence - * General Public License for Générale Publique GNU Affero - * more details. pour plus de détails. - * - * You should have received Vous devriez avoir reçu une - * a copy of the GNU Affero copie de la Licence Générale - * General Public License along Publique GNU Affero avec - * with OpenCADC. If not, see OpenCADC ; si ce n’est - * . pas le cas, consultez : - * . - * - * - ************************************************************************ - */ - -package net.canfar.storage.web.resources; - -import ca.nrc.cadc.auth.AuthMethod; -import ca.nrc.cadc.auth.RunnableAction; -import ca.nrc.cadc.net.FileContent; -import ca.nrc.cadc.net.HttpGet; -import ca.nrc.cadc.net.HttpPost; -import ca.nrc.cadc.reg.Standards; -import net.canfar.storage.web.config.VOSpaceServiceConfig; -import org.opencadc.vospace.client.VOSpaceClient; -import org.opencadc.vospace.VOS; -import org.opencadc.vospace.View; - -import java.io.OutputStream; -import java.io.StringWriter; -import java.net.URI; -import java.net.URL; -import java.util.*; -import java.util.stream.Collectors; - -import org.apache.log4j.Logger; -import org.opencadc.vospace.transfer.Direction; -import org.opencadc.vospace.transfer.Protocol; -import org.opencadc.vospace.transfer.Transfer; -import org.opencadc.vospace.transfer.TransferWriter; -import org.restlet.data.Form; -import org.restlet.data.MediaType; -import org.restlet.representation.OutputRepresentation; -import org.restlet.representation.Representation; -import org.restlet.resource.Get; -import org.restlet.resource.Post; - -import javax.security.auth.Subject; - - -public class PackageServerResource extends StorageItemServerResource { - private static final Logger LOGGER = Logger.getLogger(PackageServerResource.class); - - /** - * Empty constructor needed for Restlet to manage it. - */ - public PackageServerResource() { - } - - PackageServerResource(final VOSpaceClient voSpaceClient, final VOSpaceServiceConfig serviceConfig) { - super(voSpaceClient, serviceConfig); - } - - @Get("json") - public Representation notSupported() { - throw new UnsupportedOperationException("GET not supported."); - } - - @Post - public Representation getPackage(final Representation payload) throws Exception { - final Form form = new Form(payload); - final String responseFormat = determineContentType(form); - final String[] uris = form.getValuesArray("uri", true); - - if (uris == null || uris.length == 0) { - throw new IllegalArgumentException("Nothing specified to download."); - } else { - LOGGER.debug("Determined content type of " + responseFormat); - // build target list to add to transfer - final List targetList = Arrays.stream(uris).map(URI::create).collect(Collectors.toList()); - - if (LOGGER.isDebugEnabled()) { - targetList.forEach(target -> LOGGER.debug("Sending target " + target)); - } - - // Create the Transfer. - final Transfer transfer = new Transfer(Direction.pullFromVoSpace); - transfer.getTargets().addAll(targetList); - transfer.version = VOS.VOSPACE_21; - - final Protocol protocol = new Protocol(VOS.PROTOCOL_HTTPS_GET); - protocol.setSecurityMethod(Standards.SECURITY_METHOD_COOKIE); - transfer.getProtocols().add(protocol); - - // Add package view to request using responseFormat provided - final View packageView = new View(Standards.PKG_10); - packageView.getParameters().add(new View.Parameter(VOS.PROPERTY_URI_FORMAT, responseFormat)); - - transfer.setView(packageView); - - final TransferWriter transferWriter = new TransferWriter(); - final StringWriter sw = new StringWriter(); - transferWriter.write(transfer, sw); - LOGGER.debug("transfer XML: " + sw); - - final Subject currentSubject = getCurrentSubject(); - final URL dataURL = getRedirect(currentSubject, sw.toString().getBytes()); - - if (dataURL == null) { - throw new IllegalArgumentException("No package endpoint available."); - } else { - return new OutputRepresentation(new MediaType(responseFormat)) { - @Override - public void write(OutputStream outputStream) { - final HttpGet httpGet = new HttpGet(dataURL, outputStream); - Subject.doAs(currentSubject, new RunnableAction(httpGet)); - } - }; - } - } - } - - private URL getRedirect(final Subject currentSubject, final byte[] payload) { - // POST the transfer to synctrans - final FileContent fileContent = new FileContent(payload, "text/xml"); - final URL synctransServiceURL = lookupEndpoint(this.currentService.getResourceID(), Standards.VOSPACE_SYNC_21, - AuthMethod.ANON); - final HttpPost post = new HttpPost(synctransServiceURL, fileContent, false); - - Subject.doAs(currentSubject, new RunnableAction(post)); - return post.getRedirectURL(); - } - - private String determineContentType(final Form form) { - final String packageType = form.getFirst("method").getValue(); - return PackageTypes.fromLabel(packageType).contentType; - } - - private enum PackageTypes { - ZIP("ZIP Package", "application/zip"), - TAR("TAR Package", "application/x-tar"); - - final String label; - final String contentType; - - PackageTypes(String label, String contentType) { - this.label = label; - this.contentType = contentType; - } - - static PackageTypes fromLabel(final String label) { - return Arrays.stream(values()).filter(packageType -> packageType.label.equals(label)) - .findFirst() - .orElseThrow(() -> new NoSuchElementException("No Package with label " + label)); - } - } -} diff --git a/src/main/java/net/canfar/storage/web/resources/SecureServerResource.java b/src/main/java/net/canfar/storage/web/resources/SecureServerResource.java index 95d8119..fb48ef9 100644 --- a/src/main/java/net/canfar/storage/web/resources/SecureServerResource.java +++ b/src/main/java/net/canfar/storage/web/resources/SecureServerResource.java @@ -73,13 +73,16 @@ import ca.nrc.cadc.auth.AuthenticationUtil; import ca.nrc.cadc.auth.AuthorizationToken; import ca.nrc.cadc.auth.AuthorizationTokenPrincipal; -import ca.nrc.cadc.auth.SSOCookieCredential; import ca.nrc.cadc.reg.client.RegistryClient; +import java.io.IOException; import java.net.URL; +import java.util.Collections; +import java.util.Map; +import java.util.NoSuchElementException; +import javax.security.auth.Subject; +import javax.servlet.ServletContext; import net.canfar.storage.web.config.StorageConfiguration; -import net.canfar.storage.web.config.VOSpaceServiceConfigManager; import net.canfar.storage.web.restlet.StorageApplication; - import org.opencadc.token.Client; import org.restlet.Response; import org.restlet.data.Cookie; @@ -87,15 +90,6 @@ import org.restlet.representation.Representation; import org.restlet.resource.ServerResource; -import javax.security.auth.Subject; -import javax.servlet.ServletContext; -import java.io.IOException; -import java.util.Collections; -import java.util.Map; -import java.util.NoSuchElementException; -import java.util.Set; - - class SecureServerResource extends ServerResource { final StorageConfiguration storageConfiguration; @@ -104,7 +98,7 @@ public SecureServerResource() { } /** - * Full constructor. Useful for testing, or overriding default configuration. + * Full constructor. Useful for testing, or overriding default configuration. * * @param storageConfiguration The main configuration. */ @@ -137,16 +131,20 @@ Subject getCallingSubject(final URL target) throws Exception { try { final String accessToken = getOIDCClient().getAccessToken(firstPartyCookie.getValue()); - subject.getPrincipals().add(new AuthorizationTokenPrincipal(AuthenticationUtil.AUTHORIZATION_HEADER, - AuthenticationUtil.CHALLENGE_TYPE_BEARER - + " " + accessToken)); - subject.getPublicCredentials().add( - new AuthorizationToken(AuthenticationUtil.CHALLENGE_TYPE_BEARER, accessToken, Collections.singletonList(target.getHost()))); + subject.getPrincipals() + .add(new AuthorizationTokenPrincipal( + AuthenticationUtil.AUTHORIZATION_HEADER, + AuthenticationUtil.CHALLENGE_TYPE_BEARER + " " + accessToken)); + subject.getPublicCredentials() + .add(new AuthorizationToken( + AuthenticationUtil.CHALLENGE_TYPE_BEARER, + accessToken, + Collections.singletonList(target.getHost()))); if (!subject.getPrincipals(AuthorizationTokenPrincipal.class).isEmpty()) { // Ensure it's clean first. - subject.getPublicCredentials(AuthMethod.class) - .forEach(authMethod -> subject.getPublicCredentials().remove(authMethod)); + subject.getPublicCredentials(AuthMethod.class).forEach(authMethod -> subject.getPublicCredentials() + .remove(authMethod)); subject.getPublicCredentials().add(AuthMethod.TOKEN); return AuthenticationUtil.getIdentityManager().validate(subject); @@ -154,27 +152,6 @@ Subject getCallingSubject(final URL target) throws Exception { } catch (NoSuchElementException noSuchElementException) { // No Asset found } - } else { - // Ensure the valid backend Domains is added in. - // TODO: Is this insecure? It seems like a bit of a bastardization, but the assumption is that if the - // TODO: caller has a cookie, then they must be valid, right? - // TODO: jenkinsd 2024.01.24 - // - final Set cookieCredentialSet = - subject.getPublicCredentials(SSOCookieCredential.class); - if (!cookieCredentialSet.isEmpty()) { - final SSOCookieCredential firstCredential = cookieCredentialSet.stream().findFirst().get(); - final VOSpaceServiceConfigManager voSpaceServiceConfigManager = - new VOSpaceServiceConfigManager(this.storageConfiguration); - voSpaceServiceConfigManager.getServiceList().forEach(serviceName -> { - final String domain = voSpaceServiceConfigManager.getServiceConfig(serviceName) - .getResourceID().getHost(); - - subject.getPublicCredentials().add(new SSOCookieCredential(firstCredential.getSsoCookieValue(), - domain, - firstCredential.getExpiryDate())); - }); - } } return subject; @@ -186,14 +163,14 @@ ServletContext getServletContext() { } /** - * Set a default context path when this is not running in a servlet - * container. + * Set a default context path when this is not running in a servlet container. * * @return String path. */ String getContextPath() { return (getServletContext() == null) - ? StorageApplication.DEFAULT_CONTEXT_PATH : getServletContext().getContextPath(); + ? StorageApplication.DEFAULT_CONTEXT_PATH + : getServletContext().getContextPath(); } Client getOIDCClient() throws IOException { @@ -207,7 +184,7 @@ AccessControlClient getAccessControlClient() { /** * Write out the given status and representation body to the response. * - * @param status The Status to set. + * @param status The Status to set. * @param representation The representation used for the body of the response. */ void writeResponse(final Status status, final Representation representation) { diff --git a/src/main/java/net/canfar/storage/web/resources/StorageItemServerResource.java b/src/main/java/net/canfar/storage/web/resources/StorageItemServerResource.java index fddac06..7e01ed3 100644 --- a/src/main/java/net/canfar/storage/web/resources/StorageItemServerResource.java +++ b/src/main/java/net/canfar/storage/web/resources/StorageItemServerResource.java @@ -121,9 +121,7 @@ public class StorageItemServerResource extends SecureServerResource { VOSpaceClient voSpaceClient; VOSpaceServiceConfig currentService; - /** - * Empty constructor needed for Restlet to manage it. Needs to be public. - */ + /** Empty constructor needed for Restlet to manage it. Needs to be public. */ public StorageItemServerResource() { this.voSpaceServiceConfigManager = new VOSpaceServiceConfigManager(storageConfiguration); } @@ -131,16 +129,18 @@ public StorageItemServerResource() { /** * Only used for testing as no Request is coming through to initialize it as it would in Production. * - * @param storageConfiguration The StorageConfiguration object. + * @param storageConfiguration The StorageConfiguration object. * @param voSpaceServiceConfigManager The VOSpaceServiceConfigManager object. - * @param storageItemFactory The StorageItemFactory object. - * @param voSpaceClient The VOSpaceClient object. - * @param serviceConfig The current VOSpace Service. + * @param storageItemFactory The StorageItemFactory object. + * @param voSpaceClient The VOSpaceClient object. + * @param serviceConfig The current VOSpace Service. */ - StorageItemServerResource(StorageConfiguration storageConfiguration, - VOSpaceServiceConfigManager voSpaceServiceConfigManager, - StorageItemFactory storageItemFactory, - VOSpaceClient voSpaceClient, VOSpaceServiceConfig serviceConfig) { + StorageItemServerResource( + StorageConfiguration storageConfiguration, + VOSpaceServiceConfigManager voSpaceServiceConfigManager, + StorageItemFactory storageItemFactory, + VOSpaceClient voSpaceClient, + VOSpaceServiceConfig serviceConfig) { super(storageConfiguration); this.voSpaceServiceConfigManager = voSpaceServiceConfigManager; this.storageItemFactory = storageItemFactory; @@ -160,12 +160,10 @@ public StorageItemServerResource() { initializeStorageItemFactory(); } - /** - * Set-up method. This ensures there is a context first before pulling - * out some necessary objects for further work. - *

- * Tester + * Set-up method. This ensures there is a context first before pulling out some necessary objects for further work. + * + *

Tester */ @Override protected void doInit() throws ResourceException { @@ -177,18 +175,16 @@ protected void doInit() throws ResourceException { private void initializeStorageItemFactory() { final ServletContext servletContext = getServletContext(); - this.storageItemFactory = new StorageItemFactory((servletContext == null) - ? StorageApplication.DEFAULT_CONTEXT_PATH - : servletContext.getContextPath(), - this.currentService); + this.storageItemFactory = new StorageItemFactory( + (servletContext == null) ? StorageApplication.DEFAULT_CONTEXT_PATH : servletContext.getContextPath(), + this.currentService); } - URL lookupEndpoint(final URI serviceURI, final URI capabilityStandardURI, - final AuthMethod authMethod) { + URL lookupEndpoint(final URI serviceURI, final URI capabilityStandardURI, final AuthMethod authMethod) { return getRegistryClient().getServiceURL(serviceURI, capabilityStandardURI, authMethod); } - Path getCurrentPath() { + Path getCurrentPath() { if (getRequestAttributes().containsKey("path")) { final String pathInRequest = getRequestAttribute("path"); return Paths.get(pathInRequest); @@ -222,7 +218,8 @@ List getVOSpaceServiceList() { } VOSURI getCurrentItemURI() { - return new VOSURI(URI.create(this.currentService.getNodeResourceID() + getCurrentPath().toString())); + return new VOSURI(URI.create( + this.currentService.getNodeResourceID() + getCurrentPath().toString())); } String getCurrentName() { @@ -239,7 +236,7 @@ final T getCurrentNode(final VOS.Detail detail) { @SuppressWarnings("unchecked") T getNode(final Path nodePath, final VOS.Detail detail, final Integer limit) - throws ResourceException { + throws ResourceException { final Map queryPayload = new HashMap<>(); if (limit != null) { queryPayload.put("limit", limit); @@ -250,8 +247,8 @@ T getNode(final Path nodePath, final VOS.Detail detail, final I } final String query = queryPayload.entrySet().stream() - .map(entry -> entry.getKey() + "=" + entry.getValue()) - .collect(Collectors.joining("&")); + .map(entry -> entry.getKey() + "=" + entry.getValue()) + .collect(Collectors.joining("&")); try { final T currentNode = executeSecurely(() -> (T) voSpaceClient.getNode(nodePath.toString(), query)); @@ -261,8 +258,7 @@ T getNode(final Path nodePath, final VOS.Detail detail, final I return currentNode; } catch (IllegalArgumentException e) { // Very specific hack to try again without the (possibly) unsupported limit parameter. - if (limit != null - && e.getMessage().startsWith("OptionNotSupported")) { + if (limit != null && e.getMessage().startsWith("OptionNotSupported")) { return getNode(nodePath, detail, null); } else { throw new ResourceException(e); @@ -282,8 +278,8 @@ T getNode(final Path nodePath, final VOS.Detail detail, final I // Working around a bug where the RegistryClient improperly handles an unauthenticated request. if (runtimeException.getCause() != null) { if (runtimeException.getCause() instanceof IOException - && runtimeException.getCause().getCause() != null - && (runtimeException.getCause().getCause() instanceof NotAuthenticatedException)) { + && runtimeException.getCause().getCause() != null + && (runtimeException.getCause().getCause() instanceof NotAuthenticatedException)) { throw new ResourceException(runtimeException.getCause().getCause()); } else if (runtimeException.getCause().getMessage().contains("PosixMapperClient.augment(Subject)")) { // More hacks for the base call being unauthenticated. @@ -350,11 +346,10 @@ void setInheritedPermissions(final Path nodePath) throws Exception { } /** - * Resolve this link Node's target to its final destination. This method - * will follow the target of the provided LinkNode, and continue to do so - * until an external URL is found, or Node that is not a Link Node. - *

- * Finally, this method will redirect to the appropriate endpoint. + * Resolve this link Node's target to its final destination. This method will follow the target of the provided + * LinkNode, and continue to do so until an external URL is found, or Node that is not a Link Node. + * + *

Finally, this method will redirect to the appropriate endpoint. * * @throws Exception For any parsing errors. */ @@ -403,9 +398,8 @@ private URI resolveLink(final LinkNode linkNode) throws NodeNotFoundException { } /** - * Perform the HTTPS command to recursively set permissions for a node. - * Returns when job is complete, OR a maximum of (15) seconds has elapsed. - * If timeout has been reached, job will continue to run until is cancelled. + * Perform the HTTPS command to recursively set permissions for a node. Returns when job is complete, OR a maximum + * of (15) seconds has elapsed. If timeout has been reached, job will continue to run until is cancelled. * * @param newNode The Node whose permissions are to be recursively set */ @@ -425,7 +419,6 @@ private void setNodeRecursiveSecure(final Node newNode) throws Exception { } } - /** * Perform the HTTPS command. * @@ -438,7 +431,6 @@ private void setNodeSecure(final Node newNode) throws Exception { }); } - void createLink(final URI target) throws Exception { createNode(toLinkNode(target)); } @@ -491,9 +483,11 @@ String getDisplayName() throws Exception { } /** - * Convenience method to obtain a Subject targeted for the current VOSpace backend. When using Tokens, for example, the AuthenticationToken instance - * in the Subject's Public Credentials will contain the domain of the backend VOSpace API. - * @return Subject instance. Never null. + * Convenience method to obtain a Subject targeted for the current VOSpace backend. When using Tokens, for example, + * the AuthenticationToken instance in the Subject's Public Credentials will contain the domain of the backend + * VOSpace API. + * + * @return Subject instance. Never null. */ Subject getVOSpaceCallingSubject() throws Exception { return super.getCallingSubject(new URL(this.voSpaceClient.getBaseURL())); @@ -525,7 +519,7 @@ public void update(final JsonRepresentation payload) throws Exception { currentNode.getReadOnlyGroup().clear(); if (keySet.contains("readGroup") && StringUtil.hasText(jsonObject.getString("readGroup"))) { final GroupURI newReadGroupURI = - new GroupURI(storageConfiguration.getGroupURI(jsonObject.getString("readGroup"))); + new GroupURI(storageConfiguration.getGroupURI(jsonObject.getString("readGroup"))); currentNode.getReadOnlyGroup().add(newReadGroupURI); } else { currentNode.clearReadOnlyGroups = true; @@ -534,7 +528,7 @@ public void update(final JsonRepresentation payload) throws Exception { currentNode.getReadWriteGroup().clear(); if (keySet.contains("writeGroup") && StringUtil.hasText(jsonObject.getString("writeGroup"))) { final GroupURI newReadWriteGroupURI = - new GroupURI(storageConfiguration.getGroupURI(jsonObject.getString("writeGroup"))); + new GroupURI(storageConfiguration.getGroupURI(jsonObject.getString("writeGroup"))); currentNode.getReadWriteGroup().add(newReadWriteGroupURI); } else { currentNode.clearReadWriteGroups = true; diff --git a/src/main/java/net/canfar/storage/web/restlet/PackageRepresentation.java b/src/main/java/net/canfar/storage/web/restlet/PackageRepresentation.java deleted file mode 100644 index f36e0eb..0000000 --- a/src/main/java/net/canfar/storage/web/restlet/PackageRepresentation.java +++ /dev/null @@ -1,139 +0,0 @@ -/* - ************************************************************************ - ******************* CANADIAN ASTRONOMY DATA CENTRE ******************* - ************** CENTRE CANADIEN DE DONNÉES ASTRONOMIQUES ************** - * - * (c) 2022. (c) 2022. - * Government of Canada Gouvernement du Canada - * National Research Council Conseil national de recherches - * Ottawa, Canada, K1A 0R6 Ottawa, Canada, K1A 0R6 - * All rights reserved Tous droits réservés - * - * NRC disclaims any warranties, Le CNRC dénie toute garantie - * expressed, implied, or énoncée, implicite ou légale, - * statutory, of any kind with de quelque nature que ce - * respect to the software, soit, concernant le logiciel, - * including without limitation y compris sans restriction - * any warranty of merchantability toute garantie de valeur - * or fitness for a particular marchande ou de pertinence - * purpose. NRC shall not be pour un usage particulier. - * liable in any event for any Le CNRC ne pourra en aucun cas - * damages, whether direct or être tenu responsable de tout - * indirect, special or general, dommage, direct ou indirect, - * consequential or incidental, particulier ou général, - * arising from the use of the accessoire ou fortuit, résultant - * software. Neither the name de l'utilisation du logiciel. Ni - * of the National Research le nom du Conseil National de - * Council of Canada nor the Recherches du Canada ni les noms - * names of its contributors may de ses participants ne peuvent - * be used to endorse or promote être utilisés pour approuver ou - * products derived from this promouvoir les produits dérivés - * software without specific prior de ce logiciel sans autorisation - * written permission. préalable et particulière - * par écrit. - * - * This file is part of the Ce fichier fait partie du projet - * OpenCADC project. OpenCADC. - * - * OpenCADC is free software: OpenCADC est un logiciel libre ; - * you can redistribute it and/or vous pouvez le redistribuer ou le - * modify it under the terms of modifier suivant les termes de - * the GNU Affero General Public la “GNU Affero General Public - * License as published by the License” telle que publiée - * Free Software Foundation, par la Free Software Foundation - * either version 3 of the : soit la version 3 de cette - * License, or (at your option) licence, soit (à votre gré) - * any later version. toute version ultérieure. - * - * OpenCADC is distributed in the OpenCADC est distribué - * hope that it will be useful, dans l’espoir qu’il vous - * but WITHOUT ANY WARRANTY; sera utile, mais SANS AUCUNE - * without even the implied GARANTIE : sans même la garantie - * warranty of MERCHANTABILITY implicite de COMMERCIALISABILITÉ - * or FITNESS FOR A PARTICULAR ni d’ADÉQUATION À UN OBJECTIF - * PURPOSE. See the GNU Affero PARTICULIER. Consultez la Licence - * General Public License for Générale Publique GNU Affero - * more details. pour plus de détails. - * - * You should have received Vous devriez avoir reçu une - * a copy of the GNU Affero copie de la Licence Générale - * General Public License along Publique GNU Affero avec - * with OpenCADC. If not, see OpenCADC ; si ce n’est - * . pas le cas, consultez : - * . - * - * - ************************************************************************ - */ - -package net.canfar.storage.web.restlet; - -import ca.nrc.cadc.io.ByteLimitExceededException; -import ca.nrc.cadc.io.MultiBufferIO; -import ca.nrc.cadc.io.ReadException; -import ca.nrc.cadc.io.WriteException; -import ca.nrc.cadc.net.HttpGet; -import ca.nrc.cadc.net.ResourceAlreadyExistsException; -import ca.nrc.cadc.net.ResourceNotFoundException; -import ca.nrc.cadc.net.TransientException; -import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; -import java.net.URL; -import org.restlet.data.MediaType; -import org.restlet.representation.OutputRepresentation; - -/** - * Potential class for downloading packages. Depending on how front end solution - * to accessing /vault/pkg is accepted, this class may be needed. (If not, it can be - * deleted as part of the code review.) - */ -public class PackageRepresentation extends OutputRepresentation -{ - URL packageURL; - InputStream getIOStream; - /** - * Constructor. - * - * @param mediaType The representation's mediaType. - */ - public PackageRepresentation(MediaType mediaType, URL pkgURL) - { - super(mediaType); - this.packageURL = pkgURL; - } - - @Override - public void write(OutputStream outStream) throws IOException { - - try { - HttpGet get = new HttpGet(packageURL, true); - get.prepare(); - - // Copy the get InputStream to the package OutputStream - // this is 'writing' the content of the file - InputStream getIOStream = get.getInputStream(); - MultiBufferIO multiBufferIO = new MultiBufferIO(); - multiBufferIO.copy(getIOStream, outStream); - - } catch (ResourceNotFoundException e) { - // TODO: put some reasonable messages out from here - although - e.printStackTrace(); - } catch (ByteLimitExceededException e) { - e.printStackTrace(); - } catch (TransientException e) { - e.printStackTrace(); - } catch (WriteException e) { - e.printStackTrace(); - } catch (ResourceAlreadyExistsException e) { - e.printStackTrace(); - } catch (ReadException e) { - e.printStackTrace(); - } catch (IOException e) { - e.printStackTrace(); - } catch (InterruptedException e) { - e.printStackTrace(); - } - } - -} diff --git a/src/main/java/net/canfar/storage/web/restlet/StorageApplication.java b/src/main/java/net/canfar/storage/web/restlet/StorageApplication.java index caad3b8..ab129d6 100644 --- a/src/main/java/net/canfar/storage/web/restlet/StorageApplication.java +++ b/src/main/java/net/canfar/storage/web/restlet/StorageApplication.java @@ -69,23 +69,20 @@ package net.canfar.storage.web.restlet; import freemarker.cache.URLTemplateLoader; +import java.net.MalformedURLException; +import java.net.URL; +import java.util.HashMap; +import java.util.Map; +import javax.servlet.ServletContext; import net.canfar.storage.web.config.StorageConfiguration; import net.canfar.storage.web.resources.*; import net.canfar.storage.web.view.FreeMarkerConfiguration; - import org.apache.log4j.Logger; import org.restlet.*; import org.restlet.routing.Router; import org.restlet.routing.TemplateRoute; import org.restlet.routing.Variable; -import javax.servlet.ServletContext; -import java.net.MalformedURLException; -import java.net.URL; -import java.util.HashMap; -import java.util.Map; - - public class StorageApplication extends Application { private static final Logger log = Logger.getLogger(StorageApplication.class); @@ -94,26 +91,21 @@ public class StorageApplication extends Application { public static final String SERVLET_CONTEXT_ATTRIBUTE_KEY = "org.restlet.ext.servlet.ServletContext"; public static final String DEFAULT_CONTEXT_PATH = "/storage"; - /** * Constructor. * - * @param context The context to use based on parent component context. This - * context should be created using the - * {@link Context#createChildContext()} method to ensure a proper - * isolation with the other applications. + * @param context The context to use based on parent component context. This context should be created using the + * {@link Context#createChildContext()} method to ensure a proper isolation with the other applications. */ public StorageApplication(Context context) { super(context); setStatusService(new VOSpaceStatusService()); } - /** - * Creates an inbound root Restlet that will receive all incoming calls. In - * general, instances of Router, Filter or Finder classes will be used as - * initial application Restlet. The default implementation returns null by - * default. This method is intended to be overridden by subclasses. + * Creates an inbound root Restlet that will receive all incoming calls. In general, instances of Router, Filter or + * Finder classes will be used as initial application Restlet. The default implementation returns null by default. + * This method is intended to be overridden by subclasses. * * @return The inbound root Restlet. */ @@ -144,28 +136,19 @@ public Restlet createInboundRoot() { router.attach(contextPath + "/list/", MainPageServerResource.class); router.attach(contextPath + "/page", PageServerResource.class); - final TemplateRoute pageRoute = router.attach(contextPath + "/page{path}", - PageServerResource.class); + final TemplateRoute pageRoute = router.attach(contextPath + "/page{path}", PageServerResource.class); // Generic endpoint for files, folders, or links. - final TemplateRoute itemRoute = router.attach(contextPath + "/item{path}", - StorageItemServerResource.class); - final TemplateRoute folderRoute = router.attach(contextPath + "/folder{path}", - FolderItemServerResource.class); - final TemplateRoute fileRoute = router.attach(contextPath + "/file{path}", - FileItemServerResource.class); - final TemplateRoute linkRoute = router.attach(contextPath + "/link{path}", - LinkItemServerResource.class); - final TemplateRoute listRoute = router.attach(contextPath + "/list{path}", - MainPageServerResource.class); - final TemplateRoute nodeRoute = router.attach(contextPath + "/access{path}", - NodeServerResource.class); - final TemplateRoute rawRoute = router.attach(contextPath + "/raw{path}", - MainPageServerResource.class); - final TemplateRoute oidcCallbackRoute = router.attach(contextPath + "/oidc-callback", - OIDCCallbackServerResource.class); - final TemplateRoute oidcLoginRoute = router.attach(contextPath + "/oidc-login", - OIDCLoginServerResource.class); + final TemplateRoute itemRoute = router.attach(contextPath + "/item{path}", StorageItemServerResource.class); + final TemplateRoute folderRoute = router.attach(contextPath + "/folder{path}", FolderItemServerResource.class); + final TemplateRoute fileRoute = router.attach(contextPath + "/file{path}", FileItemServerResource.class); + final TemplateRoute linkRoute = router.attach(contextPath + "/link{path}", LinkItemServerResource.class); + final TemplateRoute listRoute = router.attach(contextPath + "/list{path}", MainPageServerResource.class); + final TemplateRoute nodeRoute = router.attach(contextPath + "/access{path}", NodeServerResource.class); + final TemplateRoute rawRoute = router.attach(contextPath + "/raw{path}", MainPageServerResource.class); + final TemplateRoute oidcCallbackRoute = + router.attach(contextPath + "/oidc-callback", OIDCCallbackServerResource.class); + final TemplateRoute oidcLoginRoute = router.attach(contextPath + "/oidc-login", OIDCLoginServerResource.class); itemRoute.getTemplate().getVariables().putAll(routeVariables); folderRoute.getTemplate().getVariables().putAll(routeVariables); @@ -180,33 +163,27 @@ public Restlet createInboundRoot() { // Support for routes with {svc} in URL router.attach(contextPath + "/{svc}/page", PageServerResource.class); - final TemplateRoute svcPageRoute = router.attach(contextPath + "/{svc}/page{path}", - PageServerResource.class); + final TemplateRoute svcPageRoute = router.attach(contextPath + "/{svc}/page{path}", PageServerResource.class); // Allow for an empty path to be the root. - final TemplateRoute svcListRouteNoPath = router.attach( - contextPath + "/{svc}/list", MainPageServerResource.class); - final TemplateRoute svcListRouteNoPath2 = router.attach( - contextPath + "/{svc}/list/", MainPageServerResource.class); + final TemplateRoute svcListRouteNoPath = + router.attach(contextPath + "/{svc}/list", MainPageServerResource.class); + final TemplateRoute svcListRouteNoPath2 = + router.attach(contextPath + "/{svc}/list/", MainPageServerResource.class); // Generic endpoint for files, folders, or links. - final TemplateRoute svcItemRoute = router.attach(contextPath + "/{svc}/item{path}", - StorageItemServerResource.class); - final TemplateRoute svcFolderRoute = router.attach(contextPath + "/{svc}/folder{path}", - FolderItemServerResource.class); - final TemplateRoute svcFileRoute = router.attach(contextPath + "/{svc}/file{path}", - FileItemServerResource.class); - final TemplateRoute svcLinkRoute = router.attach(contextPath + "/{svc}/link{path}", - LinkItemServerResource.class); - final TemplateRoute svcListRoute = router.attach(contextPath + "/{svc}/list{path}", - MainPageServerResource.class); - final TemplateRoute svcNodeRoute = router.attach(contextPath + "/{svc}/access{path}", - NodeServerResource.class); - final TemplateRoute svcRawRoute = router.attach(contextPath + "/{svc}/raw{path}", - MainPageServerResource.class); - final TemplateRoute pkgRawRoute = router.attach(contextPath + "/{svc}/pkg", - PackageServerResource.class); - + final TemplateRoute svcItemRoute = + router.attach(contextPath + "/{svc}/item{path}", StorageItemServerResource.class); + final TemplateRoute svcFolderRoute = + router.attach(contextPath + "/{svc}/folder{path}", FolderItemServerResource.class); + final TemplateRoute svcFileRoute = + router.attach(contextPath + "/{svc}/file{path}", FileItemServerResource.class); + final TemplateRoute svcLinkRoute = + router.attach(contextPath + "/{svc}/link{path}", LinkItemServerResource.class); + final TemplateRoute svcListRoute = + router.attach(contextPath + "/{svc}/list{path}", MainPageServerResource.class); + final TemplateRoute svcNodeRoute = router.attach(contextPath + "/{svc}/access{path}", NodeServerResource.class); + final TemplateRoute svcRawRoute = router.attach(contextPath + "/{svc}/raw{path}", MainPageServerResource.class); // Set route variables to all the templates svcItemRoute.getTemplate().getVariables().putAll(routeVariables); @@ -220,7 +197,6 @@ public Restlet createInboundRoot() { svcListRouteNoPath2.getTemplate().getVariables().putAll(routeVariables); svcRawRoute.getTemplate().getVariables().putAll(routeVariables); svcNodeRoute.getTemplate().getVariables().putAll(routeVariables); - pkgRawRoute.getTemplate().getVariables().putAll(routeVariables); router.setContext(getContext()); return router; @@ -243,8 +219,9 @@ public FreeMarkerConfiguration createFreemarkerConfig() { if (storageConfiguration.getThemeName().equalsIgnoreCase("canfar")) { final Map uriTemplateLoader = new HashMap<>(); try { - uriTemplateLoader.put("themes/canfar/canfar-application-header", - new URL("https://www.canfar.net/canfar/includes/_application_header.shtml")); + uriTemplateLoader.put( + "themes/canfar/canfar-application-header", + new URL("https://www.canfar.net/canfar/includes/_application_header.shtml")); } catch (MalformedURLException urlException) { // Should NEVER happen. throw new IllegalStateException(urlException.getMessage(), urlException); diff --git a/src/main/java/org/opencadc/storage/StorageAction.java b/src/main/java/org/opencadc/storage/StorageAction.java new file mode 100644 index 0000000..b2cfff2 --- /dev/null +++ b/src/main/java/org/opencadc/storage/StorageAction.java @@ -0,0 +1,114 @@ +package org.opencadc.storage; + +import ca.nrc.cadc.auth.AuthMethod; +import ca.nrc.cadc.auth.AuthenticationUtil; +import ca.nrc.cadc.auth.AuthorizationToken; +import ca.nrc.cadc.auth.AuthorizationTokenPrincipal; +import ca.nrc.cadc.auth.SSOCookieCredential; +import ca.nrc.cadc.reg.client.RegistryClient; +import ca.nrc.cadc.rest.RestAction; +import ca.nrc.cadc.util.StringUtil; +import java.io.IOException; +import java.net.URI; +import java.net.URL; +import java.util.Arrays; +import java.util.Collections; +import java.util.NoSuchElementException; +import java.util.Set; +import javax.security.auth.Subject; +import net.canfar.storage.web.config.StorageConfiguration; +import net.canfar.storage.web.config.VOSpaceServiceConfig; +import net.canfar.storage.web.config.VOSpaceServiceConfigManager; +import org.apache.log4j.Logger; +import org.opencadc.token.Client; + +public abstract class StorageAction extends RestAction { + private static final Logger LOGGER = Logger.getLogger(StorageAction.class); + final StorageConfiguration storageConfiguration; + final VOSpaceServiceConfigManager voSpaceServiceConfigManager; + + public StorageAction() { + this.storageConfiguration = new StorageConfiguration(); + this.voSpaceServiceConfigManager = new VOSpaceServiceConfigManager(this.storageConfiguration); + } + + protected StorageAction( + final StorageConfiguration storageConfiguration, + final VOSpaceServiceConfigManager voSpaceServiceConfigManager) { + this.storageConfiguration = storageConfiguration; + this.voSpaceServiceConfigManager = voSpaceServiceConfigManager; + } + + protected Subject getCurrentSubject(final URL targetURL) throws Exception { + final String rawCookieHeader = this.syncInput.getHeader("cookie"); + final Subject subject = AuthenticationUtil.getCurrentSubject(); + + if (StringUtil.hasText(rawCookieHeader)) { + final String[] firstPartyCookies = Arrays.stream(rawCookieHeader.split(";")) + .map(String::trim) + .filter(cookieString -> cookieString.startsWith(StorageConfiguration.FIRST_PARTY_COOKIE_NAME)) + .toArray(String[]::new); + + if (firstPartyCookies.length > 0 && storageConfiguration.isOIDCConfigured()) { + for (final String cookie : firstPartyCookies) { + // Only split on the first "=" symbol, and trim any wrapping double quotes + final String encryptedCookieValue = cookie.split("=", 2)[1].replaceAll("\"", ""); + + try { + final String accessToken = getOIDCClient().getAccessToken(encryptedCookieValue); + + subject.getPrincipals() + .add(new AuthorizationTokenPrincipal( + AuthenticationUtil.AUTHORIZATION_HEADER, + AuthenticationUtil.CHALLENGE_TYPE_BEARER + " " + accessToken)); + subject.getPublicCredentials() + .add(new AuthorizationToken( + AuthenticationUtil.CHALLENGE_TYPE_BEARER, + accessToken, + Collections.singletonList(targetURL.getHost()))); + } catch (NoSuchElementException noTokenForKeyInCacheException) { + LOGGER.warn("Cookie found and decrypted but no value in cache. Ignoring cookie..."); + } + } + + if (!subject.getPrincipals(AuthorizationTokenPrincipal.class).isEmpty()) { + // Ensure it's clean first. + subject.getPublicCredentials(AuthMethod.class).forEach(authMethod -> subject.getPublicCredentials() + .remove(authMethod)); + subject.getPublicCredentials().add(AuthMethod.TOKEN); + } + } else if (AuthenticationUtil.getAuthMethod(subject) == AuthMethod.COOKIE) { + final Set publicCookieCredentials = + subject.getPublicCredentials(SSOCookieCredential.class); + if (!publicCookieCredentials.isEmpty()) { + final SSOCookieCredential publicCookieCredential = + publicCookieCredentials.toArray(new SSOCookieCredential[0])[0]; + subject.getPublicCredentials() + .add(new SSOCookieCredential( + publicCookieCredential.getSsoCookieValue(), + targetURL.getHost(), + publicCookieCredential.getExpiryDate())); + } + } + } + + return subject; + } + + Client getOIDCClient() throws IOException { + return this.storageConfiguration.getOIDCClient(); + } + + protected URL lookupEndpoint(final URI serviceURI, final URI capabilityStandardURI, final AuthMethod authMethod) { + return new RegistryClient().getServiceURL(serviceURI, capabilityStandardURI, authMethod); + } + + protected final VOSpaceServiceConfig getCurrentService() { + final String providedServiceName = this.syncInput.getParameter("service"); + final String serviceName = StringUtil.hasText(providedServiceName) + ? providedServiceName + : this.voSpaceServiceConfigManager.getDefaultServiceName(); + LOGGER.debug("Service name: " + serviceName); + return this.voSpaceServiceConfigManager.getServiceConfig(serviceName); + } +} diff --git a/src/main/java/org/opencadc/storage/pkg/PostAction.java b/src/main/java/org/opencadc/storage/pkg/PostAction.java new file mode 100644 index 0000000..35fcebe --- /dev/null +++ b/src/main/java/org/opencadc/storage/pkg/PostAction.java @@ -0,0 +1,164 @@ +package org.opencadc.storage.pkg; + +import ca.nrc.cadc.auth.AuthMethod; +import ca.nrc.cadc.auth.RunnableAction; +import ca.nrc.cadc.net.FileContent; +import ca.nrc.cadc.net.HttpGet; +import ca.nrc.cadc.net.HttpPost; +import ca.nrc.cadc.reg.Standards; +import ca.nrc.cadc.rest.InlineContentHandler; +import ca.nrc.cadc.rest.SyncInput; +import ca.nrc.cadc.rest.SyncOutput; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.io.StringWriter; +import java.net.URI; +import java.net.URL; +import java.security.PrivilegedExceptionAction; +import java.util.Arrays; +import java.util.List; +import java.util.NoSuchElementException; +import java.util.stream.Collectors; +import javax.security.auth.Subject; +import net.canfar.storage.web.config.StorageConfiguration; +import net.canfar.storage.web.config.VOSpaceServiceConfig; +import net.canfar.storage.web.config.VOSpaceServiceConfigManager; +import org.apache.log4j.Logger; +import org.opencadc.storage.StorageAction; +import org.opencadc.vospace.VOS; +import org.opencadc.vospace.View; +import org.opencadc.vospace.transfer.Direction; +import org.opencadc.vospace.transfer.Protocol; +import org.opencadc.vospace.transfer.Transfer; +import org.opencadc.vospace.transfer.TransferWriter; + +public class PostAction extends StorageAction { + private static final Logger LOGGER = Logger.getLogger(PostAction.class); + + public PostAction() { + super(); + } + + PostAction( + final StorageConfiguration storageConfiguration, + final VOSpaceServiceConfigManager voSpaceServiceConfigManager, + final SyncInput syncInput, + final SyncOutput syncOutput) { + super(storageConfiguration, voSpaceServiceConfigManager); + this.syncInput = syncInput; + this.syncOutput = syncOutput; + } + + @Override + protected InlineContentHandler getInlineContentHandler() { + return null; + } + + @Override + public void doAction() throws Exception { + final List targetList = getTargetURIs(); + final String responseFormat = determineContentType(); + LOGGER.debug("Determined content type of " + responseFormat); + + if (LOGGER.isDebugEnabled()) { + targetList.forEach(target -> LOGGER.debug("Sending target " + target)); + } + + final Subject currentSubject = getCurrentSubject( + lookupEndpoint(getCurrentService().getResourceID(), Standards.VOSPACE_NODES_20, AuthMethod.TOKEN)); + final URL transferRunURL = getTransferRunURL(currentSubject, getTransferRunPayload(targetList, responseFormat)); + + final HttpGet httpGet = new HttpGet(transferRunURL, true); + Subject.doAs(currentSubject, (PrivilegedExceptionAction) () -> { + httpGet.prepare(); + return null; + }); + + this.syncOutput.addHeader("content-disposition", httpGet.getResponseHeader("content-disposition")); + this.syncOutput.addHeader("content-length", httpGet.getContentLength()); + this.syncOutput.addHeader("content-type", httpGet.getContentType()); + + final byte[] buffer = new byte[8192]; + final InputStream inputStream = httpGet.getInputStream(); + final OutputStream outputStream = this.syncOutput.getOutputStream(); + int bytesRead; + while ((bytesRead = inputStream.read(buffer)) != -1) { + outputStream.write(buffer, 0, bytesRead); + } + + outputStream.flush(); + } + + List getTargetURIs() { + final List requestedURIs = this.syncInput.getParameters("uri"); + + if (requestedURIs == null || requestedURIs.isEmpty()) { + throw new IllegalArgumentException("Nothing specified to download."); + } else { + return requestedURIs.stream().map(URI::create).collect(Collectors.toList()); + } + } + + byte[] getTransferRunPayload(final List targetList, final String responseFormat) throws IOException { + // Create the Transfer. + final Transfer transfer = new Transfer(Direction.pullFromVoSpace); + transfer.getTargets().addAll(targetList); + transfer.version = VOS.VOSPACE_21; + + final Protocol protocol = new Protocol(VOS.PROTOCOL_HTTPS_GET); + protocol.setSecurityMethod(Standards.SECURITY_METHOD_COOKIE); + transfer.getProtocols().add(protocol); + + // Add package view to request using responseFormat provided + final View packageView = new View(Standards.PKG_10); + packageView.getParameters().add(new View.Parameter(VOS.PROPERTY_URI_FORMAT, responseFormat)); + transfer.setView(packageView); + + final TransferWriter transferWriter = new TransferWriter(); + final StringWriter sw = new StringWriter(); + transferWriter.write(transfer, sw); + LOGGER.debug("transfer XML: " + sw); + + return sw.toString().getBytes(); + } + + private URL getTransferRunURL(final Subject currentSubject, final byte[] payload) { + // POST the transfer to synctrans + final VOSpaceServiceConfig currentService = getCurrentService(); + final FileContent fileContent = new FileContent(payload, "text/xml"); + final URL synctransServiceURL = + lookupEndpoint(currentService.getResourceID(), Standards.VOSPACE_SYNC_21, AuthMethod.TOKEN); + final HttpPost post = new HttpPost(synctransServiceURL, fileContent, false); + + Subject.doAs(currentSubject, new RunnableAction(post)); + return post.getRedirectURL(); + } + + private String determineContentType() { + final String packageType = this.syncInput.getParameter("method"); + return PackageTypes.fromLabel(packageType).contentType; + } + + enum PackageTypes { + ZIP("ZIP Package", "application/zip", "zip"), + TAR("TAR Package", "application/x-tar", "tar"); + + final String label; + final String contentType; + final String fileExtension; + + PackageTypes(String label, String contentType, String fileExtension) { + this.label = label; + this.contentType = contentType; + this.fileExtension = fileExtension; + } + + static PackageTypes fromLabel(final String label) { + return Arrays.stream(PackageTypes.values()) + .filter(packageType -> packageType.label.equals(label)) + .findFirst() + .orElseThrow(() -> new NoSuchElementException("No Package with label " + label)); + } + } +} diff --git a/src/main/webapp/WEB-INF/web.xml b/src/main/webapp/WEB-INF/web.xml index 513a222..3c6b736 100644 --- a/src/main/webapp/WEB-INF/web.xml +++ b/src/main/webapp/WEB-INF/web.xml @@ -41,6 +41,15 @@ 1 + + PackageServlet + ca.nrc.cadc.rest.RestServlet + + post + org.opencadc.storage.pkg.PostAction + + + Restlet org.restlet.ext.servlet.ServerServlet @@ -71,6 +80,11 @@ Restlet /* + + + PackageServlet + /pkg + default diff --git a/src/main/webapp/js/filemanager.js b/src/main/webapp/js/filemanager.js index 7cdb5a9..83da01e 100644 --- a/src/main/webapp/js/filemanager.js +++ b/src/main/webapp/js/filemanager.js @@ -2549,9 +2549,9 @@ function fileManager( $(document).on('click', '.download-dropdown-menu > li > a', function() { let $thisLink = $(this) - // Package download (ZIP or TAR) are handled separately. Default to Download Manager, otherwise. + // Package download (ZIP or TAR) are handled separately. const formAction = $thisLink.attr('class').includes('download-package') - ? `${contextPath}${vospaceServicePath}${config.options.packageConnector}` + ? `${contextPath}${config.options.packageConnector}` : '/downloadManager/download' const form = document.createElement('form') @@ -2590,6 +2590,13 @@ function fileManager( } ) + const serviceHiddenField = document.createElement('input') + serviceHiddenField.setAttribute('type', 'hidden') + serviceHiddenField.setAttribute('name', 'service') + serviceHiddenField.setAttribute('value', vospaceServicePath.replaceAll('/', '')) + + form.appendChild(serviceHiddenField) + document.body.appendChild(form) form._submit_function_() diff --git a/src/main/webapp/themes/canfar/_main.ftl b/src/main/webapp/themes/canfar/_main.ftl index 9287034..a23d00b 100644 --- a/src/main/webapp/themes/canfar/_main.ftl +++ b/src/main/webapp/themes/canfar/_main.ftl @@ -101,7 +101,6 @@ -->

  •  ZIP Package
  •  TAR Package
    • -
  •  Download Manager
    • diff --git a/src/main/webapp/themes/default/_main.ftl b/src/main/webapp/themes/default/_main.ftl index 9c9f850..ad1aaff 100644 --- a/src/main/webapp/themes/default/_main.ftl +++ b/src/main/webapp/themes/default/_main.ftl @@ -94,10 +94,8 @@ diff --git a/src/main/webapp/themes/src/_main.ftl b/src/main/webapp/themes/src/_main.ftl index 2072b57..ad1aaff 100644 --- a/src/main/webapp/themes/src/_main.ftl +++ b/src/main/webapp/themes/src/_main.ftl @@ -96,7 +96,6 @@ diff --git a/src/test/java/org/opencadc/storage/pkg/PostActionTest.java b/src/test/java/org/opencadc/storage/pkg/PostActionTest.java new file mode 100644 index 0000000..f2f2942 --- /dev/null +++ b/src/test/java/org/opencadc/storage/pkg/PostActionTest.java @@ -0,0 +1,77 @@ +package org.opencadc.storage.pkg; + +import ca.nrc.cadc.rest.SyncInput; +import ca.nrc.cadc.rest.SyncOutput; +import java.net.URI; +import java.util.Arrays; +import org.junit.Assert; +import org.junit.Test; +import org.mockito.Mockito; +import org.xmlunit.assertj.XmlAssert; + +public class PostActionTest { + @Test + public void testTransferRunXMLZIP() throws Exception { + final PostAction postAction = new PostAction(null, null, null, null); + + final String responseFormat = PostAction.PackageTypes.ZIP.contentType; + final URI[] targets = new URI[] {URI.create("vos://test1"), URI.create("vos://test2")}; + + final String xmlOutput = new String(postAction.getTransferRunPayload(Arrays.asList(targets), responseFormat)); + final String expectedXML = "\n" + + "\n" + + " vos://test1\n" + + " vos://test2\n" + + " pullFromVoSpace\n" + + " \n" + + " application/zip\n" + + " \n" + + " \n" + + " \n" + + " \n" + + " true\n" + + "\n"; + + XmlAssert.assertThat(xmlOutput).and(expectedXML).ignoreWhitespace().areSimilar(); + } + + @Test + public void testTransferRunXMLTAR() throws Exception { + final PostAction postAction = new PostAction(null, null, null, null); + + final String responseFormat = PostAction.PackageTypes.TAR.contentType; + final URI[] targets = new URI[] {URI.create("vos://test/tar/1"), URI.create("vos://test/tar/2")}; + + final String xmlOutput = new String(postAction.getTransferRunPayload(Arrays.asList(targets), responseFormat)); + final String expectedXML = "\n" + + "\n" + + " vos://test/tar/1\n" + + " vos://test/tar/2\n" + + " pullFromVoSpace\n" + + " \n" + + " application/x-tar\n" + + " \n" + + " \n" + + " \n" + + " \n" + + " true\n" + + "\n"; + + XmlAssert.assertThat(xmlOutput).and(expectedXML).ignoreWhitespace().areSimilar(); + } + + @Test + public void testGetTargetURIs() { + final SyncInput mockSyncInput = Mockito.mock(SyncInput.class); + final SyncOutput mockSyncOutput = Mockito.mock(SyncOutput.class); + final PostAction postAction = new PostAction(null, null, mockSyncInput, mockSyncOutput); + + Mockito.when(mockSyncInput.getParameters("uri")).thenReturn(Arrays.asList("vos://test/1", "vos://test/2")); + + final URI[] actualURIs = postAction.getTargetURIs().toArray(new URI[0]); + final URI[] expectedURIs = new URI[] {URI.create("vos://test/1"), URI.create("vos://test/2")}; + + Assert.assertArrayEquals("Wrong URIs.", expectedURIs, actualURIs); + Mockito.verify(mockSyncInput, Mockito.times(1)).getParameters("uri"); + } +}