From 158744042176b0e7d5c435db829c1386ead8f35c Mon Sep 17 00:00:00 2001 From: Dustin Jenkins Date: Wed, 10 Apr 2024 06:34:06 -0700 Subject: [PATCH 1/3] Fix for proper content length checking --- VERSION | 2 +- src/main/java/net/canfar/storage/web/UploadVerifier.java | 8 +++----- .../storage/web/resources/FileItemServerResource.java | 2 +- 3 files changed, 5 insertions(+), 7 deletions(-) diff --git a/VERSION b/VERSION index 831f6d4..00216a8 100644 --- a/VERSION +++ b/VERSION @@ -1,6 +1,6 @@ ## deployable containers have a semantic and build tag # semantic version tag: major.minor # build version tag: timestamp -VERSION="1.1.2" +VERSION="1.1.3" TAGS="${VERSION} ${VERSION}-$(date -u +"%Y%m%dT%H%M%S")" unset VERSION diff --git a/src/main/java/net/canfar/storage/web/UploadVerifier.java b/src/main/java/net/canfar/storage/web/UploadVerifier.java index c139f85..b5db69f 100644 --- a/src/main/java/net/canfar/storage/web/UploadVerifier.java +++ b/src/main/java/net/canfar/storage/web/UploadVerifier.java @@ -70,6 +70,7 @@ import ca.nrc.cadc.util.HexUtil; import ca.nrc.cadc.util.StringUtil; +import org.opencadc.vospace.DataNode; import org.opencadc.vospace.Node; import org.opencadc.vospace.NodeProperty; import org.opencadc.vospace.VOS; @@ -87,7 +88,7 @@ public class UploadVerifier { * @param node The node to verify. * @throws UploadVerificationFailedException Any upload error, such as bad filename. */ - public void verifyByteCount(final long byteCount, final Node node) + public void verifyByteCount(final long byteCount, final DataNode node) throws UploadVerificationFailedException { if (byteCount < 0) { throw new IllegalArgumentException("The given byte count cannot be a negative value."); @@ -95,10 +96,7 @@ public void verifyByteCount(final long byteCount, final Node node) throw new IllegalArgumentException("The given Node cannot be null."); } - final NodeProperty contentLengthProperty = node.getProperty(VOS.PROPERTY_URI_CONTENTLENGTH); - final long contentLength = contentLengthProperty == null - ? 0L - : Long.parseLong(contentLengthProperty.getValue()); + final long contentLength = node.bytesUsed == null ? 0L : node.bytesUsed; if (byteCount != contentLength) { throw new UploadVerificationFailedException("** ERROR ** - Upload did not succeed: " diff --git a/src/main/java/net/canfar/storage/web/resources/FileItemServerResource.java b/src/main/java/net/canfar/storage/web/resources/FileItemServerResource.java index 10ed7e2..2c746fe 100644 --- a/src/main/java/net/canfar/storage/web/resources/FileItemServerResource.java +++ b/src/main/java/net/canfar/storage/web/resources/FileItemServerResource.java @@ -409,7 +409,7 @@ void upload(final UploadOutputStreamWrapper outputStreamWrapper, final DataNode // Check uws job status VOSClientUtil.checkTransferFailure(ct); - final Node uploadedNode = getNode(Paths.get(dataNodeVOSURI.getPath()), VOS.Detail.properties); + final DataNode uploadedNode = getNode(Paths.get(dataNodeVOSURI.getPath()), VOS.Detail.properties); uploadVerifier.verifyByteCount(outputStreamWrapper.getByteCount(), uploadedNode); uploadVerifier.verifyMD5(outputStreamWrapper.getCalculatedMD5(), uploadedNode); From 20a08d7a39000d28437cc58763a3385207d4a3b3 Mon Sep 17 00:00:00 2001 From: Dustin Jenkins Date: Wed, 10 Apr 2024 10:46:56 -0700 Subject: [PATCH 2/3] Correct how upload information is checked. --- build.gradle | 2 +- .../canfar/storage/web/UploadVerifier.java | 44 +++++++------------ .../web/resources/FileItemServerResource.java | 44 +++++++------------ .../resources/FileItemServerResourceTest.java | 3 +- 4 files changed, 34 insertions(+), 59 deletions(-) diff --git a/build.gradle b/build.gradle index d8988ed..aff5734 100644 --- a/build.gradle +++ b/build.gradle @@ -24,7 +24,7 @@ dependencies { implementation 'org.opencadc:cadc-access-control-identity:[1.1.0,)' implementation 'org.opencadc:cadc-log:[1.1.7,2.0)' implementation 'org.opencadc:cadc-vos:[2.0.1,3.0)' - implementation 'org.opencadc:cadc-vos-client:[2.0.1,3.0)' + implementation 'org.opencadc:cadc-vos-client:[2.0.3,3.0)' implementation 'org.opencadc:cadc-vos-server:[2.0.4,3.0)' implementation 'org.opencadc:cadc-vosi:[1.3.12,2.0.0)' implementation 'org.opencadc:cadc-registry:[1.7.6,2.0)' diff --git a/src/main/java/net/canfar/storage/web/UploadVerifier.java b/src/main/java/net/canfar/storage/web/UploadVerifier.java index b5db69f..1535f06 100644 --- a/src/main/java/net/canfar/storage/web/UploadVerifier.java +++ b/src/main/java/net/canfar/storage/web/UploadVerifier.java @@ -70,40 +70,32 @@ import ca.nrc.cadc.util.HexUtil; import ca.nrc.cadc.util.StringUtil; -import org.opencadc.vospace.DataNode; -import org.opencadc.vospace.Node; -import org.opencadc.vospace.NodeProperty; -import org.opencadc.vospace.VOS; import net.canfar.storage.UploadVerificationFailedException; -import java.net.URI; - public class UploadVerifier { /** * Verify that each byte is accounted for on the server side. * - * @param byteCount The count of bytes. - * @param node The node to verify. + * @param calculatedByteCount The count of bytes. + * @param serverByteCount The server reported byte count. * @throws UploadVerificationFailedException Any upload error, such as bad filename. */ - public void verifyByteCount(final long byteCount, final DataNode node) + public void verifyByteCount(final long calculatedByteCount, final long serverByteCount) throws UploadVerificationFailedException { - if (byteCount < 0) { + if (calculatedByteCount < 0) { throw new IllegalArgumentException("The given byte count cannot be a negative value."); - } else if (node == null) { - throw new IllegalArgumentException("The given Node cannot be null."); + } else if (serverByteCount < 0) { + throw new IllegalArgumentException("The server byte count cannot be a negative value."); } - final long contentLength = node.bytesUsed == null ? 0L : node.bytesUsed; - - if (byteCount != contentLength) { + if (calculatedByteCount != serverByteCount) { throw new UploadVerificationFailedException("** ERROR ** - Upload did not succeed: " + String.format("File length counted [%d] does not " + "match what the service said it " - + "should be [%d]", byteCount, - contentLength)); + + "should be [%d]", calculatedByteCount, + serverByteCount)); } } @@ -114,29 +106,25 @@ public void verifyByteCount(final long byteCount, final DataNode node) * the string will be compared to what the returned Node provided. * * @param calculatedMD5 The byte array of the calculated MD5. - * @param node The node to verify against. + * @param serverMD5 The server reported MD5. * @throws UploadVerificationFailedException Any upload error, such as bad filename. */ - public void verifyMD5(final byte[] calculatedMD5, final Node node) throws UploadVerificationFailedException { + public void verifyMD5(final byte[] calculatedMD5, final String serverMD5) + throws UploadVerificationFailedException { if (calculatedMD5 == null) { throw new IllegalArgumentException("The calculated MD5 cannot be null."); - } else if (node == null) { - throw new IllegalArgumentException("The given Node cannot be null."); + } else if (serverMD5 == null) { + throw new IllegalArgumentException("The server MD5 cannot be null."); } - final NodeProperty MD5Property = node.getProperty(VOS.PROPERTY_URI_CONTENTMD5); - final String serverMD5String = MD5Property == null - ? null - : MD5Property.getValue(); - - if (!StringUtil.hasLength(serverMD5String)) { + if (!StringUtil.hasLength(serverMD5)) { throw new UploadVerificationFailedException("** ERROR YOUR UPLOAD DID NOT SUCCEED ** " + "MD5 checksum was not produced by " + "service! This was not expected, please " + "contact canfarhelp@nrc-cnrc.gc.ca for " + "assistance."); } else { - if (!HexUtil.toHex(calculatedMD5).equals(serverMD5String)) { + if (!HexUtil.toHex(calculatedMD5).equals(serverMD5)) { throw new UploadVerificationFailedException( "** ERROR ** - Upload did not succeed: " + "MD5 checksum failed."); diff --git a/src/main/java/net/canfar/storage/web/resources/FileItemServerResource.java b/src/main/java/net/canfar/storage/web/resources/FileItemServerResource.java index 2c746fe..4a912e2 100644 --- a/src/main/java/net/canfar/storage/web/resources/FileItemServerResource.java +++ b/src/main/java/net/canfar/storage/web/resources/FileItemServerResource.java @@ -73,7 +73,6 @@ import ca.nrc.cadc.auth.AuthorizationToken; import ca.nrc.cadc.net.ResourceNotFoundException; import ca.nrc.cadc.reg.Standards; -import ca.nrc.cadc.reg.client.RegistryClient; import ca.nrc.cadc.util.StringUtil; import net.canfar.storage.PathUtils; import net.canfar.storage.web.*; @@ -93,9 +92,7 @@ import org.json.JSONException; import org.json.JSONWriter; import org.opencadc.vospace.DataNode; -import org.opencadc.vospace.Node; import org.opencadc.vospace.NodeNotFoundException; -import org.opencadc.vospace.NodeProperty; import org.opencadc.vospace.VOS; import org.opencadc.vospace.VOSURI; import org.opencadc.vospace.View; @@ -122,7 +119,6 @@ import java.nio.file.Path; import java.nio.file.Paths; import java.util.Arrays; -import java.util.HashSet; import java.util.List; import java.util.Objects; import java.util.Set; @@ -298,17 +294,10 @@ Path upload(final FileItemStream fileItemStream) throws Exception { final DataNode dataNode = new DataNode(filename); PathUtils.augmentParents(Paths.get(getCurrentPath().toString(), filename), dataNode); - // WebRT 19564: Add content type to the response of uploaded items. - final Set properties = new HashSet<>(); - - properties.add(new NodeProperty(VOS.PROPERTY_URI_TYPE, fileItemStream.getContentType())); - properties.add(new NodeProperty(VOS.PROPERTY_URI_CONTENTLENGTH, - Long.toString(getRequest().getEntity().getSize()))); - - dataNode.getProperties().addAll(properties); + final String contentType = fileItemStream.getContentType(); try (final InputStream inputStream = fileItemStream.openStream()) { - upload(inputStream, dataNode); + upload(inputStream, dataNode, contentType); } return PathUtils.toPath(dataNode); @@ -324,10 +313,12 @@ Path upload(final FileItemStream fileItemStream) throws Exception { * * @param inputStream The InputStream to pull from. * @param dataNode The DataNode to upload to. + * @param contentType The file content type. */ - protected void upload(final InputStream inputStream, final DataNode dataNode) throws Exception { - final UploadOutputStreamWrapper outputStreamWrapper = - new UploadOutputStreamWrapperImpl(inputStream, BUFFER_SIZE); + protected void upload(final InputStream inputStream, final DataNode dataNode, final String contentType) + throws Exception { + final UploadOutputStreamWrapper outputStreamWrapper = new UploadOutputStreamWrapperImpl(inputStream, + BUFFER_SIZE); try { // Due to a bug in VOSpace that returns a 400 while checking @@ -355,7 +346,7 @@ protected void upload(final InputStream inputStream, final DataNode dataNode) th try { executeSecurely(() -> { - upload(outputStreamWrapper, dataNode); + upload(outputStreamWrapper, dataNode, contentType); return null; }); } catch (Exception e) { @@ -378,18 +369,13 @@ protected void upload(final InputStream inputStream, final DataNode dataNode) th * * @param outputStreamWrapper The OutputStream wrapper. * @param dataNode The node to upload. + * @param contentType The file content type. * @throws Exception To capture transfer and upload failures. */ - void upload(final UploadOutputStreamWrapper outputStreamWrapper, final DataNode dataNode) throws Exception { - final RegistryClient registryClient = new RegistryClient(); - final Subject subject = AuthenticationUtil.getCurrentSubject(); - final AuthMethod am = AuthenticationUtil.getAuthMethodFromCredentials(subject); + void upload(final UploadOutputStreamWrapper outputStreamWrapper, final DataNode dataNode, final String contentType) + throws Exception { final VOSURI dataNodeVOSURI = toURI(dataNode); - final URL baseURL = registryClient.getServiceURL(dataNodeVOSURI.getServiceURI(), - Standards.VOSPACE_TRANSFERS_20, am); - LOGGER.debug("uploadURL: " + baseURL); - final List protocols = Arrays.stream(FileItemServerResource.PROTOCOL_AUTH_METHODS).map(authMethod -> { final Protocol httpsAuth = new Protocol(VOS.PROTOCOL_HTTPS_PUT); httpsAuth.setSecurityMethod(Standards.getSecurityMethod(authMethod)); @@ -402,16 +388,16 @@ void upload(final UploadOutputStreamWrapper outputStreamWrapper, final DataNode transfer.version = VOS.VOSPACE_21; final ClientTransfer ct = voSpaceClient.createTransfer(transfer); + ct.setRequestProperty("content-type", contentType); ct.setOutputStreamWrapper(outputStreamWrapper); - ct.runTransfer(); // Check uws job status VOSClientUtil.checkTransferFailure(ct); - final DataNode uploadedNode = getNode(Paths.get(dataNodeVOSURI.getPath()), VOS.Detail.properties); - uploadVerifier.verifyByteCount(outputStreamWrapper.getByteCount(), uploadedNode); - uploadVerifier.verifyMD5(outputStreamWrapper.getCalculatedMD5(), uploadedNode); + uploadVerifier.verifyByteCount(outputStreamWrapper.getByteCount(), + ct.getHttpTransferDetails().getContentLength()); + uploadVerifier.verifyMD5(outputStreamWrapper.getCalculatedMD5(), ct.getHttpTransferDetails().getContentMD5()); uploadSuccess(); } diff --git a/src/test/java/net/canfar/storage/web/resources/FileItemServerResourceTest.java b/src/test/java/net/canfar/storage/web/resources/FileItemServerResourceTest.java index ced774f..722fe8a 100644 --- a/src/test/java/net/canfar/storage/web/resources/FileItemServerResourceTest.java +++ b/src/test/java/net/canfar/storage/web/resources/FileItemServerResourceTest.java @@ -251,9 +251,10 @@ VOSURI getCurrentItemURI() { * * @param outputStreamWrapper The OutputStream wrapper. * @param dataNode The node to upload. + * @param contentType The file content type */ @Override - void upload(UploadOutputStreamWrapper outputStreamWrapper, DataNode dataNode) { + void upload(UploadOutputStreamWrapper outputStreamWrapper, DataNode dataNode, String contentType) { // Do nothing. } From 590073b2026a23beb5699377aee536a7e8dead76 Mon Sep 17 00:00:00 2001 From: Dustin Jenkins Date: Wed, 10 Apr 2024 11:05:43 -0700 Subject: [PATCH 3/3] Properly check MD5. --- .../canfar/storage/web/resources/FileItemServerResource.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/main/java/net/canfar/storage/web/resources/FileItemServerResource.java b/src/main/java/net/canfar/storage/web/resources/FileItemServerResource.java index 4a912e2..14ca274 100644 --- a/src/main/java/net/canfar/storage/web/resources/FileItemServerResource.java +++ b/src/main/java/net/canfar/storage/web/resources/FileItemServerResource.java @@ -397,7 +397,10 @@ void upload(final UploadOutputStreamWrapper outputStreamWrapper, final DataNode uploadVerifier.verifyByteCount(outputStreamWrapper.getByteCount(), ct.getHttpTransferDetails().getContentLength()); - uploadVerifier.verifyMD5(outputStreamWrapper.getCalculatedMD5(), ct.getHttpTransferDetails().getContentMD5()); + if (ct.getHttpTransferDetails().getDigest() != null) { + uploadVerifier.verifyMD5(outputStreamWrapper.getCalculatedMD5(), + ct.getHttpTransferDetails().getDigest().getSchemeSpecificPart()); + } uploadSuccess(); }