Skip to content

Commit

Permalink
Merge pull request #151 from wujinhu/master
Browse files Browse the repository at this point in the history 
support select object operation
  • Loading branch information
@Shallwewu
Shallwewu authored Sep 15, 2018
2 parents 196cf71 + 333dfe2 commit 94031b1
Show file tree
Hide file tree
Showing 32 changed files with 1,487 additions and 122 deletions.
2 changes: 1 addition & 1 deletion pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@

<groupId>com.aliyun.oss</groupId>
<artifactId>aliyun-sdk-oss</artifactId>
<version>3.1.0-SNAPSHOT</version>
<version>3.3.0-SNAPSHOT</version>
<packaging>jar</packaging>
<name>Aliyun OSS SDK for Java</name>
<description>The Aliyun OSS SDK for Java used for accessing Aliyun Object Storage Service</description>
Expand Down
4 changes: 2 additions & 2 deletions src/main/java/com/aliyun/oss/ClientException.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -154,7 +154,7 @@ public String getRequestId() {

@Override
public String getMessage() {
return getErrorMessage() + "\n[ErrorCode]: " + errorCode != null ? errorCode
: "" + "\n[RequestId]: " + requestId != null ? requestId : "";
return getErrorMessage() + "\n[ErrorCode]: " + (errorCode != null ? errorCode
: "") + "\n[RequestId]: " + (requestId != null ? requestId : "");
}
}
41 changes: 34 additions & 7 deletions src/main/java/com/aliyun/oss/OSS.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@
* <p>
* Object Store Service (a.k.a OSS) is the massive, secure, low cost and highly
* reliable public storage which could be accessed from anywhere at anytime via
* REST APIs, SDKs or web console. <br />
* REST APIs, SDKs or web console. <br>
* Developers could use OSS to create any services that need huge data storage
* and access throughput, such as media sharing web apps, cloud storage service
* or enterprise or personal data backup.
Expand Down Expand Up @@ -270,8 +270,8 @@ public interface OSS {
* @param bucketName
* Bucket name.
* @param tags
* The dictionary that contains the tags in the form of <key,
* value> pairs
* The dictionary that contains the tags in the form of &lt;key,
* value&gt; pairs
*/
public void setBucketTagging(String bucketName, Map<String, String> tags) throws OSSException, ClientException;

Expand All @@ -281,8 +281,8 @@ public interface OSS {
* @param bucketName
* Bucket name.
* @param tagSet
* {@link TagSet} instance that has the tags in the form of <key,
* value> paris.
* {@link TagSet} instance that has the tags in the form of &lt;key,
* value&gt; paris.
*/
public void setBucketTagging(String bucketName, TagSet tagSet) throws OSSException, ClientException;

Expand Down Expand Up @@ -612,6 +612,23 @@ public CopyObjectResult copyObject(String sourceBucketName, String sourceKey, St
*/
public OSSObject getObject(GetObjectRequest getObjectRequest) throws OSSException, ClientException;

/**
* Select the {@link OSSObject} from the bucket specified in
* {@link SelectObjectRequest} parameter
* @param selectObjectRequest
* A {@link SelectObjectRequest} instance which specifies the
* bucket name
* object key
* filter expression
* input serialization
* output serialization
* @return A {@link OSSObject} instance will be returned. The caller is
* responsible to close the connection after usage.
* @throws OSSException
* @throws ClientException
*/
public OSSObject selectObject(SelectObjectRequest selectObjectRequest) throws OSSException, ClientException;

/**
* Gets the {@link OSSObject} from the signed Url.
*
Expand Down Expand Up @@ -679,6 +696,16 @@ public SimplifiedObjectMeta getSimplifiedObjectMeta(GenericRequest genericReques
*/
public ObjectMetadata getObjectMetadata(GenericRequest genericRequest) throws OSSException, ClientException;

/**
* Create select object metadata(create metadata if not exists or overwrite flag set in {@link CreateSelectObjectMetadataRequest})
*
* @param createSelectObjectMetadataRequest
* {@link CreateSelectObjectMetadataRequest} create select object metadata request.
*
* @return The {@link SelectObjectMetadata} instance.
*/
public SelectObjectMetadata createSelectObjectMetadata(CreateSelectObjectMetadataRequest createSelectObjectMetadataRequest) throws OSSException, ClientException;

/**
* Append the data to the appendable object specified in
* {@link AppendObjectRequest}. It's not applicable to normal OSS object.
Expand Down Expand Up @@ -1826,7 +1853,7 @@ public void setBucketStorageCapacity(SetBucketStorageCapacityRequest setBucketSt
*
* @param uploadFileRequest
* A {@link UploadFileRequest} instance that specifies the bucket
* name, object key, file path ,part size (>100K) and thread
* name, object key, file path ,part size (&gt; 100K) and thread
* count (from 1 to 1000) and checkpoint file.
* @return A {@link UploadFileRequest} instance which has the new uploaded
* file's key, ETag, location.
Expand All @@ -1848,7 +1875,7 @@ public void setBucketStorageCapacity(SetBucketStorageCapacityRequest setBucketSt
*
* @param downloadFileRequest
* A {@link DownloadFileRequest} instance that specifies the
* bucket name, object key, file path, part size (>100K) and
* bucket name, object key, file path, part size (&gt; 100K) and
* thread count (from 1 to 1000) and checkpoint file. Also it
* could have the ETag and ModifiedSince constraints.
* @return A {@link DownloadFileResult} instance that has the
Expand Down
10 changes: 10 additions & 0 deletions src/main/java/com/aliyun/oss/OSSClient.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -561,6 +561,11 @@ public OSSObject getObject(URL signedUrl, Map<String, String> requestHeaders) th
return objectOperation.getObject(getObjectRequest);
}

@Override
public OSSObject selectObject(SelectObjectRequest selectObjectRequest) throws OSSException, ClientException {
return objectOperation.selectObject(selectObjectRequest);
}

@Override
public SimplifiedObjectMeta getSimplifiedObjectMeta(String bucketName, String key)
throws OSSException, ClientException {
Expand All @@ -578,6 +583,11 @@ public ObjectMetadata getObjectMetadata(String bucketName, String key) throws OS
return this.getObjectMetadata(new GenericRequest(bucketName, key));
}

@Override
public SelectObjectMetadata createSelectObjectMetadata(CreateSelectObjectMetadataRequest createSelectObjectMetadataRequest) throws OSSException, ClientException {
return objectOperation.createSelectObjectMetadata(createSelectObjectMetadataRequest);
}

@Override
public ObjectMetadata getObjectMetadata(GenericRequest genericRequest) throws OSSException, ClientException {
return objectOperation.getObjectMetadata(genericRequest);
Expand Down
115 changes: 86 additions & 29 deletions src/main/java/com/aliyun/oss/common/parser/RequestMarshallers.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,42 +29,15 @@

import com.aliyun.oss.ClientException;
import com.aliyun.oss.common.comm.io.FixedLengthInputStream;
import com.aliyun.oss.common.utils.BinaryUtil;
import com.aliyun.oss.common.utils.DateUtil;
import com.aliyun.oss.internal.RequestParameters;
import com.aliyun.oss.model.*;
import com.aliyun.oss.model.AddBucketReplicationRequest.ReplicationAction;
import com.aliyun.oss.model.BucketReferer;
import com.aliyun.oss.model.CompleteMultipartUploadRequest;
import com.aliyun.oss.model.CreateBucketRequest;
import com.aliyun.oss.model.CreateLiveChannelRequest;
import com.aliyun.oss.model.CreateUdfApplicationRequest;
import com.aliyun.oss.model.CreateUdfRequest;
import com.aliyun.oss.model.DeleteBucketCnameRequest;
import com.aliyun.oss.model.DeleteObjectsRequest;
import com.aliyun.oss.model.ImageProcess;
import com.aliyun.oss.model.LifecycleRule;
import com.aliyun.oss.model.LifecycleRule.AbortMultipartUpload;
import com.aliyun.oss.model.LifecycleRule.RuleStatus;
import com.aliyun.oss.model.LifecycleRule.StorageTransition;
import com.aliyun.oss.model.LiveChannelTarget;
import com.aliyun.oss.model.PartETag;
import com.aliyun.oss.model.ProcessObjectRequest;
import com.aliyun.oss.model.PutBucketImageRequest;
import com.aliyun.oss.model.PutImageStyleRequest;
import com.aliyun.oss.model.ResizeUdfApplicationRequest;
import com.aliyun.oss.model.SetBucketCORSRequest;
import com.aliyun.oss.model.SetBucketCORSRequest.CORSRule;
import com.aliyun.oss.model.DeleteBucketReplicationRequest;
import com.aliyun.oss.model.RoutingRule;
import com.aliyun.oss.model.AddBucketCnameRequest;
import com.aliyun.oss.model.SetBucketLifecycleRequest;
import com.aliyun.oss.model.SetBucketLoggingRequest;
import com.aliyun.oss.model.AddBucketReplicationRequest;
import com.aliyun.oss.model.SetBucketTaggingRequest;
import com.aliyun.oss.model.SetBucketWebsiteRequest;
import com.aliyun.oss.model.TagSet;
import com.aliyun.oss.model.UdfApplicationConfiguration;
import com.aliyun.oss.model.UpgradeUdfApplicationRequest;
import com.aliyun.oss.model.UserQos;

/**
* A collection of marshallers that marshall HTTP request into crossponding
Expand Down Expand Up @@ -99,6 +72,9 @@ public final class RequestMarshallers {
public static final ResizeUdfApplicationRequestMarshaller resizeUdfApplicationRequestMarshaller = new ResizeUdfApplicationRequestMarshaller();
public static final ProcessObjectRequestMarshaller processObjectRequestMarshaller = new ProcessObjectRequestMarshaller();

public static final CreateSelectObjectMetadataRequestMarshaller createSelectObjectMetadataRequestMarshaller = new CreateSelectObjectMetadataRequestMarshaller();
public static final SelectObjectRequestMarshaller selectObjectRequestMarshaller = new SelectObjectRequestMarshaller();

public interface RequestMarshaller<R> extends Marshaller<FixedLengthInputStream, R> {

}
Expand Down Expand Up @@ -501,6 +477,87 @@ public FixedLengthInputStream marshall(CompleteMultipartUploadRequest request) {

}

public static final class CreateSelectObjectMetadataRequestMarshaller
implements RequestMarshaller2<CreateSelectObjectMetadataRequest> {

@Override
public byte[] marshall(CreateSelectObjectMetadataRequest request) {
StringBuffer xmlBody = new StringBuffer();
InputSerialization inputSerialization = request.getInputSerialization();
CSVFormat csvFormat = inputSerialization.getCsvInputFormat();
xmlBody.append("<CsvMetaRequest>");
xmlBody.append("<InputSerialization>");
xmlBody.append("<CompressionType>" + inputSerialization.getCompressionType() + "</CompressionType>");
xmlBody.append("<CSV>");
xmlBody.append("<RecordDelimiter>" + BinaryUtil.toBase64String(csvFormat.getRecordDelimiter().getBytes()) + "</RecordDelimiter>");
xmlBody.append("<FieldDelimiter>" + BinaryUtil.toBase64String(csvFormat.getFieldDelimiter().toString().getBytes()) + "</FieldDelimiter>");
xmlBody.append("<QuoteCharacter>" + BinaryUtil.toBase64String(csvFormat.getQuoteChar().toString().getBytes()) + "</QuoteCharacter>");
xmlBody.append("</CSV>");
xmlBody.append("</InputSerialization>");
xmlBody.append("<OverwriteIfExists>" + request.isOverwrite() + "</OverwriteIfExists>");
xmlBody.append("</CsvMetaRequest>");

try {
return xmlBody.toString().getBytes(DEFAULT_CHARSET_NAME);
} catch (UnsupportedEncodingException e) {
throw new ClientException("Unsupported encoding " + e.getMessage(), e);
}
}
}

public static final class SelectObjectRequestMarshaller implements RequestMarshaller2<SelectObjectRequest> {

@Override
public byte[] marshall(SelectObjectRequest request) {
StringBuffer xmlBody = new StringBuffer();
xmlBody.append("<SelectRequest>");

xmlBody.append("<Expression>" + BinaryUtil.toBase64String(request.getExpression().getBytes()) + "</Expression>");
xmlBody.append("<Options>");
xmlBody.append("<SkipPartialDataRecord>" + request.isSkipPartialDataRecord() + "</SkipPartialDataRecord>");
xmlBody.append("</Options>");
InputSerialization inputSerialization = request.getInputSerialization();
CSVFormat csvInputFormat = inputSerialization.getCsvInputFormat();
xmlBody.append("<InputSerialization>");
xmlBody.append("<CompressionType>" + inputSerialization.getCompressionType() + "</CompressionType>");
xmlBody.append("<CSV>");
xmlBody.append("<FileHeaderInfo>" + csvInputFormat.getHeaderInfo() + "</FileHeaderInfo>");
xmlBody.append("<RecordDelimiter>" + BinaryUtil.toBase64String(csvInputFormat.getRecordDelimiter().getBytes()) + "</RecordDelimiter>");
xmlBody.append("<FieldDelimiter>" + BinaryUtil.toBase64String(csvInputFormat.getFieldDelimiter().toString().getBytes()) + "</FieldDelimiter>");
xmlBody.append("<QuoteCharacter>" + BinaryUtil.toBase64String(csvInputFormat.getQuoteChar().toString().getBytes()) + "</QuoteCharacter>");
xmlBody.append("<CommentCharacter>" + BinaryUtil.toBase64String(csvInputFormat.getCommentChar().toString().getBytes()) + "</CommentCharacter>");

if (request.getLineRange() != null) {
xmlBody.append("<Range>" + request.lineRangeToString(request.getLineRange()) + "</Range>");
}
if (request.getSplitRange() != null) {
xmlBody.append("<Range>" + request.splitRangeToString(request.getSplitRange()) + "</Range>");
}
xmlBody.append("</CSV>");
xmlBody.append("</InputSerialization>");
OutputSerialization outputSerialization = request.getOutputSerialization();
CSVFormat csvOutputFormat = outputSerialization.getCsvOutputFormat();
xmlBody.append("<OutputSerialization>");
xmlBody.append("<CSV>");
xmlBody.append("<RecordDelimiter>" + BinaryUtil.toBase64String(csvOutputFormat.getRecordDelimiter().getBytes()) + "</RecordDelimiter>");
xmlBody.append("<FieldDelimiter>" + BinaryUtil.toBase64String(csvOutputFormat.getFieldDelimiter().toString().getBytes()) + "</FieldDelimiter>");
xmlBody.append("<QuoteCharacter>" + BinaryUtil.toBase64String(csvOutputFormat.getQuoteChar().toString().getBytes()) + "</QuoteCharacter>");
xmlBody.append("</CSV>");
xmlBody.append("<KeepAllColumns>" + outputSerialization.isKeepAllColumns() + "</KeepAllColumns>");
xmlBody.append("<OutputRawData>" + outputSerialization.isOutputRawData() + "</OutputRawData>");
xmlBody.append("<OutputHeader>" + outputSerialization.isOutputHeader() + "</OutputHeader>");
xmlBody.append("<EnablePayloadCrc>" + outputSerialization.isPayloadCrcEnabled() + "</EnablePayloadCrc>");
xmlBody.append("</OutputSerialization>");
xmlBody.append("</SelectRequest>");

try {
return xmlBody.toString().getBytes(DEFAULT_CHARSET_NAME);
} catch (UnsupportedEncodingException e) {
throw new ClientException("Unsupported encoding " + e.getMessage(), e);
}
}
}

public static final class DeleteObjectsRequestMarshaller implements RequestMarshaller2<DeleteObjectsRequest> {

@Override
Expand Down
7 changes: 6 additions & 1 deletion src/main/java/com/aliyun/oss/event/ProgressEventType.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -44,5 +44,10 @@ public enum ProgressEventType {
/**
* Transfer events.
*/
TRANSFER_PREPARING_EVENT, TRANSFER_STARTED_EVENT, TRANSFER_COMPLETED_EVENT, TRANSFER_FAILED_EVENT, TRANSFER_CANCELED_EVENT, TRANSFER_PART_STARTED_EVENT, TRANSFER_PART_COMPLETED_EVENT, TRANSFER_PART_FAILED_EVENT;
TRANSFER_PREPARING_EVENT, TRANSFER_STARTED_EVENT, TRANSFER_COMPLETED_EVENT, TRANSFER_FAILED_EVENT, TRANSFER_CANCELED_EVENT, TRANSFER_PART_STARTED_EVENT, TRANSFER_PART_COMPLETED_EVENT, TRANSFER_PART_FAILED_EVENT,

/**
* Select object events.
*/
SELECT_STARTED_EVENT, SELECT_SCAN_EVENT, SELECT_COMPLETED_EVENT, SELECT_FAILED_EVENT
}
8 changes: 8 additions & 0 deletions src/main/java/com/aliyun/oss/event/ProgressPublisher.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,14 @@ public static void publishProgress(final ProgressListener listener, final Progre
listener.progressChanged(new ProgressEvent(eventType));
}

public static void publishSelectProgress(final ProgressListener listener, final ProgressEventType eventType,
final long scannedBytes) {
if (listener == ProgressListener.NOOP || listener == null || eventType == null) {
return;
}
listener.progressChanged(new ProgressEvent(eventType, scannedBytes));
}

public static void publishRequestContentLength(final ProgressListener listener, final long bytes) {
publishByteCountEvent(listener, REQUEST_CONTENT_LENGTH_EVENT, bytes);
}
Expand Down
8 changes: 8 additions & 0 deletions src/main/java/com/aliyun/oss/internal/OSSHeaders.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -80,4 +80,12 @@ public interface OSSHeaders extends HttpHeaders {
static final String OSS_ONGOING_RESTORE = "ongoing-request=\"true\"";

static final String OSS_BUCKET_REGION = "x-oss-bucket-region";

static final String OSS_SELECT_PREFIX = "x-oss-select";
static final String OSS_SELECT_CSV_ROWS = OSS_SELECT_PREFIX + "-csv-rows";
static final String OSS_SELECT_OUTPUT_RAW = OSS_SELECT_PREFIX + "-output-raw";
static final String OSS_SELECT_CSV_SPLITS = OSS_SELECT_PREFIX + "-csv-splits";
static final String OSS_SELECT_INPUT_LINE_RANGE = OSS_SELECT_PREFIX + "-line-range";
static final String OSS_SELECT_INPUT_SPLIT_RANGE = OSS_SELECT_PREFIX + "-split-range";

}
Loading

0 comments on commit 94031b1

Please sign in to comment.