From 333dfe2b75335e6da2b77780b68fd820ae0f75e7 Mon Sep 17 00:00:00 2001 From: "jinhu.wjh" Date: Sat, 15 Sep 2018 16:17:11 +0800 Subject: [PATCH] add ut --- .../oss/common/parser/RequestMarshallers.java | 2 +- .../model/CreateSelectMetaInputStream.java | 2 +- .../aliyun/oss/model/OutputSerialization.java | 14 ----- .../integrationtests/SelectObjectTest.java | 55 +++++++++++++++---- 4 files changed, 45 insertions(+), 28 deletions(-) diff --git a/src/main/java/com/aliyun/oss/common/parser/RequestMarshallers.java b/src/main/java/com/aliyun/oss/common/parser/RequestMarshallers.java index 1e9bb561..78130373 100644 --- a/src/main/java/com/aliyun/oss/common/parser/RequestMarshallers.java +++ b/src/main/java/com/aliyun/oss/common/parser/RequestMarshallers.java @@ -525,7 +525,7 @@ public byte[] marshall(SelectObjectRequest request) { xmlBody.append("" + BinaryUtil.toBase64String(csvInputFormat.getRecordDelimiter().getBytes()) + ""); xmlBody.append("" + BinaryUtil.toBase64String(csvInputFormat.getFieldDelimiter().toString().getBytes()) + ""); xmlBody.append("" + BinaryUtil.toBase64String(csvInputFormat.getQuoteChar().toString().getBytes()) + ""); - xmlBody.append("" + BinaryUtil.toBase64String(csvInputFormat.getCommentChar().toString().getBytes()) + ""); + xmlBody.append("" + BinaryUtil.toBase64String(csvInputFormat.getCommentChar().toString().getBytes()) + ""); if (request.getLineRange() != null) { xmlBody.append("" + request.lineRangeToString(request.getLineRange()) + ""); diff --git a/src/main/java/com/aliyun/oss/model/CreateSelectMetaInputStream.java b/src/main/java/com/aliyun/oss/model/CreateSelectMetaInputStream.java index bf144332..478b0e64 100644 --- a/src/main/java/com/aliyun/oss/model/CreateSelectMetaInputStream.java +++ b/src/main/java/com/aliyun/oss/model/CreateSelectMetaInputStream.java @@ -147,7 +147,7 @@ private void readFrame() throws IOException { throw new IOException("Oss Select create meta encounter error code: " + status + ", message: " + error); } - selectObjectMetadata.setCsvObjectMetadata( + selectObjectMetadata.withCsvObjectMetadata( new SelectObjectMetadata.CSVObjectMetadata() .withSplits(ByteBuffer.wrap(splitBytes).getInt()) .withTotalLines(ByteBuffer.wrap(totalLineBytes).getLong())); diff --git a/src/main/java/com/aliyun/oss/model/OutputSerialization.java b/src/main/java/com/aliyun/oss/model/OutputSerialization.java index 73e6f604..93efe605 100644 --- a/src/main/java/com/aliyun/oss/model/OutputSerialization.java +++ b/src/main/java/com/aliyun/oss/model/OutputSerialization.java @@ -7,7 +7,6 @@ */ public class OutputSerialization implements Serializable { private CSVFormat csvOutputFormat = new CSVFormat(); - private String compressionType = CompressionType.NONE.name(); private boolean keepAllColumns = false; private boolean payloadCrcEnabled = false; private boolean outputRawData = false; @@ -26,19 +25,6 @@ public OutputSerialization withCsvOutputFormat(CSVFormat csvFormat) { return this; } - public String getCompressionType() { - return compressionType; - } - - public void setCompressionType(CompressionType compressionType) { - this.compressionType = compressionType.name(); - } - - public OutputSerialization withCompressionType(CompressionType compressionType) { - setCompressionType(compressionType); - return this; - } - public boolean isKeepAllColumns() { return keepAllColumns; } diff --git a/src/test/java/com/aliyun/oss/integrationtests/SelectObjectTest.java b/src/test/java/com/aliyun/oss/integrationtests/SelectObjectTest.java index 4c8722c4..373c75ac 100644 --- a/src/test/java/com/aliyun/oss/integrationtests/SelectObjectTest.java +++ b/src/test/java/com/aliyun/oss/integrationtests/SelectObjectTest.java @@ -1,5 +1,6 @@ package com.aliyun.oss.integrationtests; +import com.aliyun.oss.event.ProgressEvent; import com.aliyun.oss.model.*; import org.junit.Assert; import org.junit.Test; @@ -9,6 +10,12 @@ public class SelectObjectTest extends TestBase { + private static class CustomProgressListener implements com.aliyun.oss.event.ProgressListener { + public void progressChanged(ProgressEvent progressEvent) { + System.out.println(progressEvent.getBytes()); + } + } + @Test public void testGetSelectObjectMetadata() { final String valid = "get-select-object-metadata-valid"; @@ -20,6 +27,8 @@ public void testGetSelectObjectMetadata() { ossClient.putObject(bucketName, valid, new ByteArrayInputStream(validContent.getBytes())); SelectObjectMetadata validSelectObjectMetadata = ossClient.createSelectObjectMetadata( new CreateSelectObjectMetadataRequest(bucketName, valid) + .withOverwrite(true) + .withSelectProgressListener(new CustomProgressListener()) .withInputSerialization(new InputSerialization().withCsvInputFormat(new CSVFormat()))); Assert.assertEquals(5, validSelectObjectMetadata.getCsvObjectMetadata().getTotalLines()); Assert.assertEquals(1, validSelectObjectMetadata.getCsvObjectMetadata().getSplits()); @@ -47,6 +56,7 @@ public void testSelectObject() throws IOException { final String key = "get-select-object-metadata-valid"; final String content = "name,school,company,age\n" + "Lora Francis,School,Staples Inc,27\n" + + "#Lora Francis,School,Staples Inc,27\n" + "Eleanor Little,School,\"Conectiv, Inc\",43\n" + "Rosie Hughes,School,Western Gas Resources Inc,44\n" + "Lawrence Ross,School,MetLife Inc.,24\n"; @@ -54,43 +64,64 @@ public void testSelectObject() throws IOException { SelectObjectRequest selectObjectRequest = new SelectObjectRequest(bucketName, key) - .withInputSerialization(new InputSerialization().withCsvInputFormat( - new CSVFormat().withHeaderInfo(CSVFormat.Header.Ignore))) - .withOutputSerialization(new OutputSerialization().withCsvOutputFormat(new CSVFormat())); - selectObjectRequest.setExpression("select * from ossobject"); + .withSelectProgressListener(new CustomProgressListener()) + .withSkipPartialDataRecord(false) + .withInputSerialization(new InputSerialization() + .withCompressionType(CompressionType.NONE) + .withCsvInputFormat( + new CSVFormat().withRecordDelimiter("\n") + .withQuoteChar("\"") + .withFieldDelimiter(",") + .withCommentChar("#") + .withHeaderInfo(CSVFormat.Header.Ignore))) + .withOutputSerialization(new OutputSerialization() + .withOutputHeader(false) + .withOutputRawData(false) + .withCrcEnabled(true) + .withKeepAllColumns(true) + .withCsvOutputFormat(new CSVFormat())) + .withExpression("select * from ossobject"); OSSObject ossObject = ossClient.selectObject(selectObjectRequest); byte[] buffer = new byte[1024]; int bytesRead; int off = 0; - while ((bytesRead = ossObject.getObjectContent().read(buffer, off, 1024 - off)) != -1) { - off += bytesRead; + while ((bytesRead = ossObject.getObjectContent().read()) != -1) { + buffer[off++] = (byte)bytesRead; } - Assert.assertEquals(new String(buffer, 0, off), content.substring(content.indexOf("\n") + 1)); + Assert.assertEquals(new String(buffer, 0, off), content.substring(content.indexOf("#L") + 1)); ossClient.createSelectObjectMetadata( new CreateSelectObjectMetadataRequest(bucketName, key) .withInputSerialization(new InputSerialization().withCsvInputFormat(new CSVFormat()))); selectObjectRequest.setLineRange(1, 3); + selectObjectRequest.getOutputSerialization().withKeepAllColumns(true); + selectObjectRequest.getOutputSerialization().withCrcEnabled(false); OSSObject rangeOssObject = ossClient.selectObject(selectObjectRequest); + try { + rangeOssObject.getObjectContent().available(); + Assert.fail("select object input stream does not support available() operation"); + } catch (Exception e) { + + } + off = 0; - while ((bytesRead = rangeOssObject.getObjectContent().read(buffer, off, 1024 - off)) != -1) { + while ((bytesRead = rangeOssObject.getObjectContent().read(buffer)) != -1) { off += bytesRead; } Assert.assertEquals(new String(buffer, 0, off), "Lora Francis,School,Staples Inc,27\n" + - "Eleanor Little,School,\"Conectiv, Inc\",43\n" + - "Rosie Hughes,School,Western Gas Resources Inc,44\n"); + "Eleanor Little,School,\"Conectiv, Inc\",43\n"); - selectObjectRequest.setLineRange(5, 10); + selectObjectRequest.withLineRange(6, 10); try { ossClient.selectObject(selectObjectRequest); Assert.fail("invalid line range for select object request"); } catch (Exception e) { } - selectObjectRequest.setSplitRange(5, 10); + selectObjectRequest.withSplitRange(5, 10); try { ossClient.selectObject(selectObjectRequest); Assert.fail("both split range and line range have been set for select object request");