diff --git a/README.MD b/README.MD index 5e3777f..5484691 100644 --- a/README.MD +++ b/README.MD @@ -1,3 +1,15 @@ +Version 3.24.9 + +New Features: +1. Added list POSIX Object API. + +Documentation & Demo: + +Resolved Issues: +1. Optimize error log printing. + +----------------------------------------------------------------------------------- + Version 3.24.6 New Features: diff --git a/README_CN.MD b/README_CN.MD index a3937fa..ac3e62f 100644 --- a/README_CN.MD +++ b/README_CN.MD @@ -1,3 +1,16 @@ +Version 3.24.9 + +新特性： +1. 新增listPosixObject接口。 + +资料 & demo： + +修复问题： + +1. 优化错误日志打印。 + +----------------------------------------------------------------------------------- + Version 3.24.6 新特性： diff --git a/obs/client_object.go b/obs/client_object.go index 818f47b..b53509b 100644 --- a/obs/client_object.go +++ b/obs/client_object.go @@ -46,6 +46,32 @@ func (obsClient ObsClient) ListObjects(input *ListObjectsInput, extensions ...ex return } +// ListPosixObjects lists objects in a posix. +// +// You can use this API to list objects in a posix. By default, a maximum of 1000 objects are listed. +func (obsClient ObsClient) ListPosixObjects(input *ListPosixObjectsInput, extensions ...extensionOptions) (output *ListPosixObjectsOutput, err error) { + if input == nil { + return nil, errors.New("ListPosixObjects is nil") + } + output = &ListPosixObjectsOutput{} + err = obsClient.doActionWithBucket("ListPosixObjects", HTTP_GET, input.Bucket, input, output, extensions) + if err != nil { + output = nil + } else { + if location, ok := output.ResponseHeaders[HEADER_BUCKET_REGION]; ok { + output.Location = location[0] + } + if output.EncodingType == "url" { + err = decodeListPosixObjectsOutput(output) + if err != nil { + doLog(LEVEL_ERROR, "Failed to get ListPosixObjectsOutput with error: %v.", err) + output = nil + } + } + } + return +} + // ListVersions lists versioning objects in a bucket. // // You can use this API to list versioning objects in a bucket. By default, a maximum of 1000 versioning objects are listed. diff --git a/obs/client_part.go b/obs/client_part.go index 195f54b..98c84d5 100644 --- a/obs/client_part.go +++ b/obs/client_part.go @@ -106,6 +106,7 @@ func (obsClient ObsClient) UploadPart(_input *UploadPartInput, extensions ...ext input.PartNumber = _input.PartNumber input.UploadId = _input.UploadId input.ContentMD5 = _input.ContentMD5 + input.ContentSHA256 = _input.ContentSHA256 input.SourceFile = _input.SourceFile input.Offset = _input.Offset input.PartSize = _input.PartSize diff --git a/obs/const.go b/obs/const.go index 8e98ec8..1b11575 100644 --- a/obs/const.go +++ b/obs/const.go @@ -50,6 +50,7 @@ const ( HEADER_GRANT_FULL_CONTROL_DELIVERED_OBS = "grant-full-control-delivered" HEADER_REQUEST_ID = "request-id" HEADER_ERROR_CODE = "error-code" + HEADER_ERROR_INDICATOR = "x-reserved-indicator" HEADER_ERROR_MESSAGE = "error-message" HEADER_BUCKET_REGION = "bucket-region" HEADER_ACCESS_CONRTOL_ALLOW_ORIGIN = "access-control-allow-origin" @@ -121,6 +122,8 @@ const ( HEADER_HOST = "host" HEADER_AUTH_CAMEL = "Authorization" HEADER_MD5_CAMEL = "Content-MD5" + HEADER_SHA256_CAMEL = "Content-SHA256" + HEADER_SHA256 = "content-sha256" HEADER_LOCATION_CAMEL = "Location" HEADER_CONTENT_LENGTH_CAMEL = "Content-Length" HEADER_CONTENT_TYPE_CAML = "Content-Type" @@ -210,6 +213,7 @@ var ( allowedRequestHTTPHeaderMetadataNames = map[string]bool{ "content-type": true, "content-md5": true, + "content-sha256": true, "content-length": true, "content-language": true, "expires": true, @@ -235,12 +239,13 @@ var ( } allowedLogResponseHTTPHeaderNames = map[string]bool{ - "content-type": true, - "etag": true, - "connection": true, - "content-length": true, - "date": true, - "server": true, + "content-type": true, + "etag": true, + "connection": true, + "content-length": true, + "date": true, + "server": true, + "x-reserved-indicator": true, } allowedResourceParameterNames = map[string]bool{ @@ -291,6 +296,15 @@ var ( "customdomain": true, "mirrorbacktosource": true, "x-obs-accesslabel": true, + "object-lock": true, + "retention": true, + "x-obs-security-token": true, + "truncate": true, + "length": true, + "inventory": true, + "directcoldaccess": true, + "attname": true, + "cdnnotifyconfiguration": true, } obsStorageClasses = []string{ diff --git a/obs/convert.go b/obs/convert.go index b0fbd93..743f3f9 100644 --- a/obs/convert.go +++ b/obs/convert.go @@ -27,11 +27,16 @@ import ( "time" ) -func cleanHeaderPrefix(header http.Header) map[string][]string { +func cleanHeaderPrefix(header http.Header, isObs bool) map[string][]string { responseHeaders := make(map[string][]string) for key, value := range header { if len(value) > 0 { key = strings.ToLower(key) + + if !isObs && strings.HasSuffix(key, HEADER_EXPIRES_OBS) { + responseHeaders[key] = value + continue + } if strings.HasPrefix(key, HEADER_PREFIX) || strings.HasPrefix(key, HEADER_PREFIX_OBS) { key = key[len(HEADER_PREFIX):] } @@ -425,7 +430,7 @@ func convertAbortIncompleteMultipartUploadToXML(abortIncompleteMultipartUpload A } // ConvertLifecycleConfigurationToXml converts BucketLifecycleConfiguration value to XML data and returns it -func ConvertLifecycleConfigurationToXml(input BucketLifecycleConfiguration, returnMd5 bool, isObs bool) (data string, md5 string) { +func ConvertLifecycleConfigurationToXml(input BucketLifecycleConfiguration, returnMd5, isObs, enableSha256 bool) (data string, md5OrSha256 string) { xml := make([]string, 0, 2+len(input.LifecycleRules)*9) xml = append(xml, "") for _, lifecycleRule := range input.LifecycleRules { @@ -463,7 +468,7 @@ func ConvertLifecycleConfigurationToXml(input BucketLifecycleConfiguration, retu xml = append(xml, "") data = strings.Join(xml, "") if returnMd5 { - md5 = Base64Md5([]byte(data)) + md5OrSha256 = Base64Md5OrSha256([]byte(data), enableSha256) } return } @@ -921,10 +926,11 @@ func ParseGetObjectOutput(output *GetObjectOutput) { } // ConvertRequestToIoReaderV2 converts req to XML data -func ConvertRequestToIoReaderV2(req interface{}) (io.Reader, string, error) { +func ConvertRequestToIoReaderV2(req interface{}, enableSha256 bool) (io.Reader, string, error) { data, err := TransToXml(req) + md5OrSha256 := Base64Md5OrSha256(data, enableSha256) if err == nil { - return bytes.NewReader(data), Base64Md5(data), nil + return bytes.NewReader(data), md5OrSha256, nil } return nil, "", err } @@ -950,7 +956,7 @@ func parseResponseBodyOutput(s reflect.Type, baseModel IBaseModel, body []byte) // ParseCallbackResponseToBaseModel gets response from Callback Service func ParseCallbackResponseToBaseModel(resp *http.Response, baseModel IBaseModel, isObs bool) error { baseModel.setStatusCode(resp.StatusCode) - responseHeaders := cleanHeaderPrefix(resp.Header) + responseHeaders := cleanHeaderPrefix(resp.Header, isObs) baseModel.setResponseHeaders(responseHeaders) if values, ok := responseHeaders[HEADER_REQUEST_ID]; ok { baseModel.setRequestID(values[0]) @@ -976,11 +982,12 @@ func ParseResponseToBaseModel(resp *http.Response, baseModel IBaseModel, xmlResu var body []byte body, err = ioutil.ReadAll(resp.Body) if err == nil && len(body) > 0 { + + name := reflect.TypeOf(baseModel).Elem().Name() if xmlResult { err = ParseXml(body, baseModel) } else { s := reflect.TypeOf(baseModel).Elem() - name := reflect.TypeOf(baseModel).Elem().Name() if name == "GetBucketPolicyOutput" || name == "GetBucketMirrorBackToSourceOuput" { parseResponseBodyOutput(s, baseModel, body) } else { @@ -988,7 +995,13 @@ func ParseResponseToBaseModel(resp *http.Response, baseModel IBaseModel, xmlResu } } if err != nil { - doLog(LEVEL_ERROR, "Unmarshal error: %v", err) + doLog(LEVEL_ERROR, "body: %s", body) + if _, ok := baseModel.(*ObsError); !ok && name == "CopyObjectOutput" { + doLog(LEVEL_ERROR, "Unmarshal error: %v, try parse response to ObsError", err) + err = ParseResponseToObsError(resp, isObs) + } else { + doLog(LEVEL_ERROR, "Unmarshal error: %v", err) + } } } } else { @@ -996,7 +1009,7 @@ func ParseResponseToBaseModel(resp *http.Response, baseModel IBaseModel, xmlResu } baseModel.setStatusCode(resp.StatusCode) - responseHeaders := cleanHeaderPrefix(resp.Header) + responseHeaders := cleanHeaderPrefix(resp.Header, isObs) baseModel.setResponseHeaders(responseHeaders) if values, ok := responseHeaders[HEADER_REQUEST_ID]; ok { baseModel.setRequestID(values[0]) @@ -1017,13 +1030,16 @@ func ParseResponseToObsError(resp *http.Response, isObs bool) error { doLog(LEVEL_WARN, "Parse response to BaseModel with error: %v", respError) } obsError.Status = resp.Status - responseHeaders := cleanHeaderPrefix(resp.Header) + responseHeaders := cleanHeaderPrefix(resp.Header, isObs) if values, ok := responseHeaders[HEADER_ERROR_MESSAGE]; ok { obsError.Message = values[0] } if values, ok := responseHeaders[HEADER_ERROR_CODE]; ok { obsError.Code = values[0] } + if values, ok := responseHeaders[HEADER_ERROR_INDICATOR]; ok { + obsError.Indicator = values[0] + } return obsError } @@ -1115,6 +1131,38 @@ func decodeListObjectsOutput(output *ListObjectsOutput) (err error) { return } +func decodeListPosixObjectsOutput(output *ListPosixObjectsOutput) (err error) { + output.Delimiter, err = url.QueryUnescape(output.Delimiter) + if err != nil { + return + } + output.Marker, err = url.QueryUnescape(output.Marker) + if err != nil { + return + } + output.NextMarker, err = url.QueryUnescape(output.NextMarker) + if err != nil { + return + } + output.Prefix, err = url.QueryUnescape(output.Prefix) + if err != nil { + return + } + for index, value := range output.CommonPrefixes { + output.CommonPrefixes[index].Prefix, err = url.QueryUnescape(value.Prefix) + if err != nil { + return + } + } + for index, content := range output.Contents { + output.Contents[index].Key, err = url.QueryUnescape(content.Key) + if err != nil { + return + } + } + return +} + func decodeListVersionsOutput(output *ListVersionsOutput) (err error) { output.Delimiter, err = url.QueryUnescape(output.Delimiter) if err != nil { diff --git a/obs/error.go b/obs/error.go index 36ca583..f074288 100644 --- a/obs/error.go +++ b/obs/error.go @@ -20,16 +20,17 @@ import ( // ObsError defines error response from OBS type ObsError struct { BaseModel - Status string - XMLName xml.Name `xml:"Error"` - Code string `xml:"Code" json:"code"` - Message string `xml:"Message" json:"message"` - Resource string `xml:"Resource"` - HostId string `xml:"HostId"` + Status string + XMLName xml.Name `xml:"Error"` + Code string `xml:"Code" json:"code"` + Message string `xml:"Message" json:"message"` + Resource string `xml:"Resource"` + HostId string `xml:"HostId"` + Indicator string } // Format print obs error's log func (err ObsError) Error() string { - return fmt.Sprintf("obs: service returned error: Status=%s, Code=%s, Message=%s, RequestId=%s", - err.Status, err.Code, err.Message, err.RequestId) + return fmt.Sprintf("obs: service returned error: Status=%s, Code=%s, Message=%s, RequestId=%s, Indicator=%s.", + err.Status, err.Code, err.Message, err.RequestId, err.Indicator) } diff --git a/obs/extension.go b/obs/extension.go index 9d96463..8043247 100644 --- a/obs/extension.go +++ b/obs/extension.go @@ -56,7 +56,6 @@ func WithCustomHeader(key string, value string) extensionHeaders { if strings.TrimSpace(value) == "" { return fmt.Errorf("set header %s with empty value", key) } - allowedRequestHTTPHeaderMetadataNames[strings.ToLower(key)] = true headers[key] = []string{value} return nil } diff --git a/obs/model_bucket.go b/obs/model_bucket.go index 1b90f6b..e2660cd 100644 --- a/obs/model_bucket.go +++ b/obs/model_bucket.go @@ -183,6 +183,7 @@ type GetBucketPolicyOutput struct { type SetBucketCorsInput struct { Bucket string `xml:"-"` BucketCors + EnableSha256 bool `xml:"-"` } // GetBucketCorsOutput is the result of GetBucketCors function @@ -261,6 +262,7 @@ type BucketLifecycleConfiguration struct { type SetBucketLifecycleConfigurationInput struct { Bucket string `xml:"-"` BucketLifecycleConfiguration + EnableSha256 bool `xml:"-"` } // GetBucketLifecycleConfigurationOutput is the result of GetBucketLifecycleConfiguration function @@ -285,6 +287,7 @@ type GetBucketEncryptionOutput struct { type SetBucketTaggingInput struct { Bucket string `xml:"-"` BucketTagging + EnableSha256 bool `xml:"-"` } // GetBucketTaggingOutput is the result of GetBucketTagging function diff --git a/obs/model_object.go b/obs/model_object.go index 3305bb6..26d6421 100644 --- a/obs/model_object.go +++ b/obs/model_object.go @@ -35,6 +35,10 @@ type ListObjectsInput struct { Marker string } +type ListPosixObjectsInput struct { + ListObjectsInput +} + // ListObjectsOutput is the result of ListObjects function type ListObjectsOutput struct { BaseModel @@ -52,6 +56,20 @@ type ListObjectsOutput struct { EncodingType string `xml:"EncodingType,omitempty"` } +type ListPosixObjectsOutput struct { + ListObjectsOutput + CommonPrefixes []CommonPrefix `xml:"CommonPrefixes"` +} + +type CommonPrefix struct { + XMLName xml.Name `xml:"CommonPrefixes"` + Prefix string `xml:"Prefix"` + MTime string `xml:"MTime"` + Mode string `xml:"Mode"` + InodeNo string `xml:"InodeNo"` + LastModified time.Time `xml:"LastModified"` +} + // ListVersionsInput is the input parameter of ListVersions function type ListVersionsInput struct { ListObjsInput @@ -236,6 +254,7 @@ type PutObjectBasicInput struct { ObjectOperationInput HttpHeader ContentMD5 string + ContentSHA256 string ContentLength int64 } diff --git a/obs/model_part.go b/obs/model_part.go index 85bd3b2..16ef0d5 100644 --- a/obs/model_part.go +++ b/obs/model_part.go @@ -74,16 +74,17 @@ type InitiateMultipartUploadOutput struct { // UploadPartInput is the input parameter of UploadPart function type UploadPartInput struct { - Bucket string - Key string - PartNumber int - UploadId string - ContentMD5 string - SseHeader ISseHeader - Body io.Reader - SourceFile string - Offset int64 - PartSize int64 + Bucket string + Key string + PartNumber int + UploadId string + ContentMD5 string + ContentSHA256 string + SseHeader ISseHeader + Body io.Reader + SourceFile string + Offset int64 + PartSize int64 } // UploadPartOutput is the result of UploadPart function diff --git a/obs/trait_bucket.go b/obs/trait_bucket.go index e3063c6..3233ae0 100644 --- a/obs/trait_bucket.go +++ b/obs/trait_bucket.go @@ -163,11 +163,17 @@ func (input SetBucketPolicyInput) trans(isObs bool) (params map[string]string, h func (input SetBucketCorsInput) trans(isObs bool) (params map[string]string, headers map[string][]string, data interface{}, err error) { params = map[string]string{string(SubResourceCors): ""} - data, md5, err := ConvertRequestToIoReaderV2(input) + data, md5OrSha256, err := ConvertRequestToIoReaderV2(input, input.EnableSha256) if err != nil { return } - headers = map[string][]string{HEADER_MD5_CAMEL: {md5}} + + headerCheckAlgorithm := HEADER_MD5_CAMEL + if input.EnableSha256 { + headerCheckAlgorithm = HEADER_SHA256_CAMEL + } + + headers = map[string][]string{headerCheckAlgorithm: {md5OrSha256}} return } @@ -200,8 +206,15 @@ func (input SetBucketLoggingConfigurationInput) trans(isObs bool) (params map[st func (input SetBucketLifecycleConfigurationInput) trans(isObs bool) (params map[string]string, headers map[string][]string, data interface{}, err error) { params = map[string]string{string(SubResourceLifecycle): ""} - data, md5 := ConvertLifecycleConfigurationToXml(input.BucketLifecycleConfiguration, true, isObs) - headers = map[string][]string{HEADER_MD5_CAMEL: {md5}} + + data, md5OrSha256 := ConvertLifecycleConfigurationToXml(input.BucketLifecycleConfiguration, true, isObs, input.EnableSha256) + + headerCheckAlgorithm := HEADER_MD5_CAMEL + if input.EnableSha256 { + headerCheckAlgorithm = HEADER_SHA256_CAMEL + } + + headers = map[string][]string{headerCheckAlgorithm: {md5OrSha256}} return } @@ -213,11 +226,17 @@ func (input SetBucketEncryptionInput) trans(isObs bool) (params map[string]strin func (input SetBucketTaggingInput) trans(isObs bool) (params map[string]string, headers map[string][]string, data interface{}, err error) { params = map[string]string{string(SubResourceTagging): ""} - data, md5, err := ConvertRequestToIoReaderV2(input) + data, md5OrSha256, err := ConvertRequestToIoReaderV2(input, input.EnableSha256) if err != nil { return } - headers = map[string][]string{HEADER_MD5_CAMEL: {md5}} + + headerCheckAlgorithm := HEADER_MD5_CAMEL + if input.EnableSha256 { + headerCheckAlgorithm = HEADER_SHA256_CAMEL + } + + headers = map[string][]string{headerCheckAlgorithm: {md5OrSha256}} return } diff --git a/obs/trait_object.go b/obs/trait_object.go index ba93e9c..98cdf01 100644 --- a/obs/trait_object.go +++ b/obs/trait_object.go @@ -59,6 +59,17 @@ func (input ListObjectsInput) trans(isObs bool) (params map[string]string, heade return } +func (input ListPosixObjectsInput) trans(isObs bool) (params map[string]string, headers map[string][]string, data interface{}, err error) { + params, headers, data, err = input.ListObjsInput.trans(isObs) + if err != nil { + return + } + if input.Marker != "" { + params["marker"] = input.Marker + } + return +} + func (input ListVersionsInput) trans(isObs bool) (params map[string]string, headers map[string][]string, data interface{}, err error) { params, headers, data, err = input.ListObjsInput.trans(isObs) if err != nil { @@ -90,6 +101,7 @@ func (input DeleteObjectsInput) trans(isObs bool) (params map[string]string, hea } } data, md5 := convertDeleteObjectsToXML(input) + headers = map[string][]string{HEADER_MD5_CAMEL: {md5}} return } @@ -185,7 +197,6 @@ func (input SetObjectMetadataInput) prepareStorageClass(headers map[string][]str } func (input SetObjectMetadataInput) trans(isObs bool) (params map[string]string, headers map[string][]string, data interface{}, err error) { - params = make(map[string]string) params = map[string]string{string(SubResourceMetadata): ""} if input.VersionId != "" { params[PARAM_VERSION_ID] = input.VersionId @@ -322,6 +333,10 @@ func (input PutObjectBasicInput) trans(isObs bool) (params map[string]string, he headers[HEADER_MD5_CAMEL] = []string{input.ContentMD5} } + if input.ContentSHA256 != "" { + setHeaders(headers, HEADER_SHA256, []string{input.ContentSHA256}, isObs) + } + if input.ContentLength > 0 { headers[HEADER_CONTENT_LENGTH_CAMEL] = []string{Int64ToString(input.ContentLength)} } diff --git a/obs/trait_part.go b/obs/trait_part.go index 0769768..286dc8a 100644 --- a/obs/trait_part.go +++ b/obs/trait_part.go @@ -81,6 +81,9 @@ func (input UploadPartInput) trans(isObs bool) (params map[string]string, header if input.ContentMD5 != "" { headers[HEADER_MD5_CAMEL] = []string{input.ContentMD5} } + if input.ContentSHA256 != "" { + setHeaders(headers, HEADER_SHA256, []string{input.ContentSHA256}, isObs) + } if input.Body != nil { data = input.Body } diff --git a/obs/transfer.go b/obs/transfer.go index a1067e7..a353055 100644 --- a/obs/transfer.go +++ b/obs/transfer.go @@ -97,6 +97,9 @@ func (task *uploadPartTask) Run() interface{} { input.SourceFile = task.SourceFile input.Offset = task.Offset input.PartSize = task.PartSize + input.ContentMD5 = task.ContentMD5 + input.ContentSHA256 = task.ContentSHA256 + extensions := task.extensions var output *UploadPartOutput diff --git a/obs/util.go b/obs/util.go index efbb51b..aff0f6c 100644 --- a/obs/util.go +++ b/obs/util.go @@ -183,6 +183,19 @@ func Base64Md5(value []byte) string { return Base64Encode(Md5(value)) } +// Base64Md5OrSha256 returns the md5 or sha256 value of input with Base64Encode +func Base64Md5OrSha256(value []byte, enableSha256 bool) string { + if enableSha256 { + return Base64Sha256(value) + } + return Base64Md5(value) +} + +// Base64Sha256 returns the sha256 value of input with Base64Encode +func Base64Sha256(value []byte) string { + return Base64Encode(Sha256Hash(value)) +} + // Sha256Hash returns sha256 checksum func Sha256Hash(value []byte) []byte { hash := sha256.New()