Skip to content

Commit

Permalink
Merge pull request #15 from at88mph/size-fix
Browse files Browse the repository at this point in the history
Fix for proper content length checking
  • Loading branch information
at88mph authored Apr 10, 2024
2 parents b8cbee4 + 590073b commit 9331be2
Show file tree
Hide file tree
Showing 5 changed files with 38 additions and 62 deletions.
2 changes: 1 addition & 1 deletion VERSION
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
## deployable containers have a semantic and build tag
# semantic version tag: major.minor
# build version tag: timestamp
VERSION="1.1.2"
VERSION="1.1.3"
TAGS="${VERSION} ${VERSION}-$(date -u +"%Y%m%dT%H%M%S")"
unset VERSION
2 changes: 1 addition & 1 deletion build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@ dependencies {
implementation 'org.opencadc:cadc-access-control-identity:[1.1.0,)'
implementation 'org.opencadc:cadc-log:[1.1.7,2.0)'
implementation 'org.opencadc:cadc-vos:[2.0.1,3.0)'
implementation 'org.opencadc:cadc-vos-client:[2.0.1,3.0)'
implementation 'org.opencadc:cadc-vos-client:[2.0.3,3.0)'
implementation 'org.opencadc:cadc-vos-server:[2.0.4,3.0)'
implementation 'org.opencadc:cadc-vosi:[1.3.12,2.0.0)'
implementation 'org.opencadc:cadc-registry:[1.7.6,2.0)'
Expand Down
46 changes: 16 additions & 30 deletions src/main/java/net/canfar/storage/web/UploadVerifier.java
Original file line number Diff line number Diff line change
Expand Up @@ -70,42 +70,32 @@

import ca.nrc.cadc.util.HexUtil;
import ca.nrc.cadc.util.StringUtil;
import org.opencadc.vospace.Node;
import org.opencadc.vospace.NodeProperty;
import org.opencadc.vospace.VOS;
import net.canfar.storage.UploadVerificationFailedException;

import java.net.URI;


public class UploadVerifier {

/**
* Verify that each byte is accounted for on the server side.
*
* @param byteCount The count of bytes.
* @param node The node to verify.
* @param calculatedByteCount The count of bytes.
* @param serverByteCount The server reported byte count.
* @throws UploadVerificationFailedException Any upload error, such as bad filename.
*/
public void verifyByteCount(final long byteCount, final Node node)
public void verifyByteCount(final long calculatedByteCount, final long serverByteCount)
throws UploadVerificationFailedException {
if (byteCount < 0) {
if (calculatedByteCount < 0) {
throw new IllegalArgumentException("The given byte count cannot be a negative value.");
} else if (node == null) {
throw new IllegalArgumentException("The given Node cannot be null.");
} else if (serverByteCount < 0) {
throw new IllegalArgumentException("The server byte count cannot be a negative value.");
}

final NodeProperty contentLengthProperty = node.getProperty(VOS.PROPERTY_URI_CONTENTLENGTH);
final long contentLength = contentLengthProperty == null
? 0L
: Long.parseLong(contentLengthProperty.getValue());

if (byteCount != contentLength) {
if (calculatedByteCount != serverByteCount) {
throw new UploadVerificationFailedException("** ERROR ** - Upload did not succeed: "
+ String.format("File length counted [%d] does not "
+ "match what the service said it "
+ "should be [%d]", byteCount,
contentLength));
+ "should be [%d]", calculatedByteCount,
serverByteCount));
}
}

Expand All @@ -116,29 +106,25 @@ public void verifyByteCount(final long byteCount, final Node node)
* the string will be compared to what the returned Node provided.
*
* @param calculatedMD5 The byte array of the calculated MD5.
* @param node The node to verify against.
* @param serverMD5 The server reported MD5.
* @throws UploadVerificationFailedException Any upload error, such as bad filename.
*/
public void verifyMD5(final byte[] calculatedMD5, final Node node) throws UploadVerificationFailedException {
public void verifyMD5(final byte[] calculatedMD5, final String serverMD5)
throws UploadVerificationFailedException {
if (calculatedMD5 == null) {
throw new IllegalArgumentException("The calculated MD5 cannot be null.");
} else if (node == null) {
throw new IllegalArgumentException("The given Node cannot be null.");
} else if (serverMD5 == null) {
throw new IllegalArgumentException("The server MD5 cannot be null.");
}

final NodeProperty MD5Property = node.getProperty(VOS.PROPERTY_URI_CONTENTMD5);
final String serverMD5String = MD5Property == null
? null
: MD5Property.getValue();

if (!StringUtil.hasLength(serverMD5String)) {
if (!StringUtil.hasLength(serverMD5)) {
throw new UploadVerificationFailedException("** ERROR YOUR UPLOAD DID NOT SUCCEED ** "
+ "MD5 checksum was not produced by "
+ "service! This was not expected, please "
+ "contact [email protected] for "
+ "assistance.");
} else {
if (!HexUtil.toHex(calculatedMD5).equals(serverMD5String)) {
if (!HexUtil.toHex(calculatedMD5).equals(serverMD5)) {
throw new UploadVerificationFailedException(
"** ERROR ** - Upload did not succeed: "
+ "MD5 checksum failed.");
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -73,7 +73,6 @@
import ca.nrc.cadc.auth.AuthorizationToken;
import ca.nrc.cadc.net.ResourceNotFoundException;
import ca.nrc.cadc.reg.Standards;
import ca.nrc.cadc.reg.client.RegistryClient;
import ca.nrc.cadc.util.StringUtil;
import net.canfar.storage.PathUtils;
import net.canfar.storage.web.*;
Expand All @@ -93,9 +92,7 @@
import org.json.JSONException;
import org.json.JSONWriter;
import org.opencadc.vospace.DataNode;
import org.opencadc.vospace.Node;
import org.opencadc.vospace.NodeNotFoundException;
import org.opencadc.vospace.NodeProperty;
import org.opencadc.vospace.VOS;
import org.opencadc.vospace.VOSURI;
import org.opencadc.vospace.View;
Expand All @@ -122,7 +119,6 @@
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.Arrays;
import java.util.HashSet;
import java.util.List;
import java.util.Objects;
import java.util.Set;
Expand Down Expand Up @@ -298,17 +294,10 @@ Path upload(final FileItemStream fileItemStream) throws Exception {
final DataNode dataNode = new DataNode(filename);
PathUtils.augmentParents(Paths.get(getCurrentPath().toString(), filename), dataNode);

// WebRT 19564: Add content type to the response of uploaded items.
final Set<NodeProperty> properties = new HashSet<>();

properties.add(new NodeProperty(VOS.PROPERTY_URI_TYPE, fileItemStream.getContentType()));
properties.add(new NodeProperty(VOS.PROPERTY_URI_CONTENTLENGTH,
Long.toString(getRequest().getEntity().getSize())));

dataNode.getProperties().addAll(properties);
final String contentType = fileItemStream.getContentType();

try (final InputStream inputStream = fileItemStream.openStream()) {
upload(inputStream, dataNode);
upload(inputStream, dataNode, contentType);
}

return PathUtils.toPath(dataNode);
Expand All @@ -324,10 +313,12 @@ Path upload(final FileItemStream fileItemStream) throws Exception {
*
* @param inputStream The InputStream to pull from.
* @param dataNode The DataNode to upload to.
* @param contentType The file content type.
*/
protected void upload(final InputStream inputStream, final DataNode dataNode) throws Exception {
final UploadOutputStreamWrapper outputStreamWrapper =
new UploadOutputStreamWrapperImpl(inputStream, BUFFER_SIZE);
protected void upload(final InputStream inputStream, final DataNode dataNode, final String contentType)
throws Exception {
final UploadOutputStreamWrapper outputStreamWrapper = new UploadOutputStreamWrapperImpl(inputStream,
BUFFER_SIZE);

try {
// Due to a bug in VOSpace that returns a 400 while checking
Expand Down Expand Up @@ -355,7 +346,7 @@ protected void upload(final InputStream inputStream, final DataNode dataNode) th

try {
executeSecurely(() -> {
upload(outputStreamWrapper, dataNode);
upload(outputStreamWrapper, dataNode, contentType);
return null;
});
} catch (Exception e) {
Expand All @@ -378,18 +369,13 @@ protected void upload(final InputStream inputStream, final DataNode dataNode) th
*
* @param outputStreamWrapper The OutputStream wrapper.
* @param dataNode The node to upload.
* @param contentType The file content type.
* @throws Exception To capture transfer and upload failures.
*/
void upload(final UploadOutputStreamWrapper outputStreamWrapper, final DataNode dataNode) throws Exception {
final RegistryClient registryClient = new RegistryClient();
final Subject subject = AuthenticationUtil.getCurrentSubject();
final AuthMethod am = AuthenticationUtil.getAuthMethodFromCredentials(subject);
void upload(final UploadOutputStreamWrapper outputStreamWrapper, final DataNode dataNode, final String contentType)
throws Exception {
final VOSURI dataNodeVOSURI = toURI(dataNode);

final URL baseURL = registryClient.getServiceURL(dataNodeVOSURI.getServiceURI(),
Standards.VOSPACE_TRANSFERS_20, am);
LOGGER.debug("uploadURL: " + baseURL);

final List<Protocol> protocols = Arrays.stream(FileItemServerResource.PROTOCOL_AUTH_METHODS).map(authMethod -> {
final Protocol httpsAuth = new Protocol(VOS.PROTOCOL_HTTPS_PUT);
httpsAuth.setSecurityMethod(Standards.getSecurityMethod(authMethod));
Expand All @@ -402,16 +388,19 @@ void upload(final UploadOutputStreamWrapper outputStreamWrapper, final DataNode
transfer.version = VOS.VOSPACE_21;

final ClientTransfer ct = voSpaceClient.createTransfer(transfer);
ct.setRequestProperty("content-type", contentType);
ct.setOutputStreamWrapper(outputStreamWrapper);

ct.runTransfer();

// Check uws job status
VOSClientUtil.checkTransferFailure(ct);

final Node uploadedNode = getNode(Paths.get(dataNodeVOSURI.getPath()), VOS.Detail.properties);
uploadVerifier.verifyByteCount(outputStreamWrapper.getByteCount(), uploadedNode);
uploadVerifier.verifyMD5(outputStreamWrapper.getCalculatedMD5(), uploadedNode);
uploadVerifier.verifyByteCount(outputStreamWrapper.getByteCount(),
ct.getHttpTransferDetails().getContentLength());
if (ct.getHttpTransferDetails().getDigest() != null) {
uploadVerifier.verifyMD5(outputStreamWrapper.getCalculatedMD5(),
ct.getHttpTransferDetails().getDigest().getSchemeSpecificPart());
}

uploadSuccess();
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -251,9 +251,10 @@ VOSURI getCurrentItemURI() {
*
* @param outputStreamWrapper The OutputStream wrapper.
* @param dataNode The node to upload.
* @param contentType The file content type
*/
@Override
void upload(UploadOutputStreamWrapper outputStreamWrapper, DataNode dataNode) {
void upload(UploadOutputStreamWrapper outputStreamWrapper, DataNode dataNode, String contentType) {
// Do nothing.
}

Expand Down

0 comments on commit 9331be2

Please sign in to comment.