From 60db5bdfe05e1d9fbcd3a96b5929ebef9b418936 Mon Sep 17 00:00:00 2001 From: Karl Mutch Date: Thu, 1 Jul 2021 19:28:42 -0700 Subject: [PATCH] fixes #416 --- cmd/runner/main.go | 2 + cmd/runner/processor.go | 39 +- docs/metadata.md | 3 +- go.mod | 9 +- go.sum | 25 +- .../studio-go-runner/reports/v1/reports.pb.go | 2 +- internal/runner/pythonenv.go | 5 +- licenses.manifest | 12 +- tools/queue-scaler/sqs_job.yaml | 1 + .../aws/aws-sdk-go/aws/endpoints/defaults.go | 3 + .../github.com/aws/aws-sdk-go/aws/version.go | 2 +- .../aws/aws-sdk-go/service/autoscaling/api.go | 83 +++-- .../aws/aws-sdk-go/service/ec2/api.go | 291 ++++++++++----- .../aws/aws-sdk-go/service/sqs/api.go | 255 +++++++------- .../aws/aws-sdk-go/service/sqs/doc.go | 21 +- .../minio/minio-go/v7/api-compose-object.go | 2 +- .../github.com/minio/minio-go/v7/api-list.go | 55 +-- .../minio-go/v7/api-put-object-multipart.go | 5 +- .../minio-go/v7/api-put-object-streaming.go | 4 +- .../minio/minio-go/v7/api-put-object.go | 2 +- vendor/github.com/minio/minio-go/v7/api.go | 2 +- vendor/github.com/minio/minio-go/v7/core.go | 8 +- .../minio/minio-go/v7/functional_tests.go | 332 +++++++++++++++++- vendor/github.com/minio/minio-go/v7/go.mod | 2 +- vendor/github.com/minio/minio-go/v7/go.sum | 5 +- .../shirou/gopsutil/cpu/cpu_windows.go | 16 +- vendor/modules.txt | 10 +- 27 files changed, 850 insertions(+), 346 deletions(-) diff --git a/cmd/runner/main.go b/cmd/runner/main.go index 3a8223db1..e82b88a3d 100644 --- a/cmd/runner/main.go +++ b/cmd/runner/main.go @@ -88,6 +88,8 @@ var ( rspnsEncrypt = &defense.PubkeyStore{} promAddrOpt = flag.String("prom-address", ":9090", "the address for the prometheus http server within the runner") + + captureOutputMD = flag.Bool("schema-logs", true, "automatically add experiment logs to metadata json") ) // GetRqstSigs returns the signing public key struct for diff --git a/cmd/runner/processor.go b/cmd/runner/processor.go index 6968ad7ef..de8d88a05 100644 --- a/cmd/runner/processor.go +++ b/cmd/runner/processor.go @@ -10,6 +10,7 @@ import ( "bufio" "context" "encoding/base64" + "encoding/json" "fmt" "io/ioutil" "os" @@ -94,6 +95,8 @@ const ( ExecPythonVEnv // ExecSingularity inidcates we are using the Singularity container packaging and runtime ExecSingularity + + fmtAddLog = `[{"op": "add", "path": "/studioml/log/-", "value": {"ts": "%s", "msg":"%s"}}]` ) func init() { @@ -411,6 +414,15 @@ func (p *processor) fetchAll(ctx context.Context) (err kv.Error) { return nil } +func jsonEscape(unescaped string) (escaped string, errGo error) { + b, errGo := json.Marshal(unescaped) + if errGo != nil { + return escaped, errGo + } + escaped = string(b) + return escaped[1 : len(escaped)-1], nil +} + // copyToMetaData is used to copy a file to the meta data area using the file naming semantics // of the metadata layout func (p *processor) copyToMetaData(src string, jsonDest string) (err kv.Error) { @@ -459,6 +471,8 @@ func (p *processor) copyToMetaData(src string, jsonDest string) (err kv.Error) { // Store any discovered json fragments for generating experiment documents as a single collection jsonDirectives := []string{} + autoCapture := *captureOutputMD + // Checkmarx code checking note. Checkmarx is for Web applications and is not a good fit general purpose server code. // It is also worth mentioning that if you are reading this message that Checkmarx does not understand Go package structure // and does not appear to use the Go AST to validate code so is not able to perform path and escape analysis which @@ -473,10 +487,23 @@ func (p *processor) copyToMetaData(src string, jsonDest string) (err kv.Error) { if len(line) <= 2 { continue } - if (line[0] != '{' || line[len(line)-1] != '}') && - (line[0] != '[' || line[len(line)-1] != ']') { - continue + + // See if this line is a sensible json fragment + if !((line[0] == '{' && line[len(line)-1] == '}') || + (line[0] == '[' && line[len(line)-1] == ']')) { + // If we dont have a fragment we check to see if in the line should be formatted as + // json and inserted using a command line switch + if !autoCapture { + continue + } + line, errGo = jsonEscape(line) + if errGo != nil { + logger.Trace("output json filter failed", "error", errGo, "line", line, "stack", stack.Trace().TrimRuntime()) + continue + } + line = fmt.Sprintf(fmtAddLog, time.Now().UTC().Format("2006-01-02T15:04:05.999999999-0700"), line) } + // After each line is scanned the json fragment is merged into a collection of all detected patches and merges that // have been output by the experiment if errGo = fastjson.Validate(line); errGo != nil { @@ -491,8 +518,14 @@ func (p *processor) copyToMetaData(src string, jsonDest string) (err kv.Error) { } } if len(jsonDirectives) == 0 { + logger.Debug("no json directives found", "stack", stack.Trace().TrimRuntime()) return nil } + // Zero copy prepend + jsonDirectives = append(jsonDirectives, " ") + copy(jsonDirectives[1:], jsonDirectives[0:]) + jsonDirectives[0] = `{"studioml": {"log": [{"ts": "0", "msg":"Init"},{"ts":"1", "msg":""}]}}` + result, err := runner.JSONEditor("", jsonDirectives) if err != nil { return err diff --git a/docs/metadata.md b/docs/metadata.md index d5630677c..a3c3a7e46 100644 --- a/docs/metadata.md +++ b/docs/metadata.md @@ -204,7 +204,7 @@ tStamp = datetime.datetime.utcnow().replace(tzinfo=datetime.timezone.utc).isofor print('[{"op": "add", "path": "/studioml/log/-", "value": {"ts": "{0}", "msg:"Example log message"}}]', tStamp) ``` -It is of course tedious if you have to decorate every log entry with a time stamp so as a convineance the go runner does have an option which will auto decorate an standard output and standard error messages with json directives for inclusion in the MLOps logs, --schemaLogs. Standard practiuve for using this option is to include it in your deployment configuration as an environment variable. +It is of course tedious if you have to decorate every log entry with a time stamp so as a convineance the go runner does have an option which will auto decorate an standard output and standard error messages with json directives for inclusion in the MLOps logs, --schema-logs. Standard practiuve for using this option is to include it in your deployment configuration as an environment variable. # Storage platforms and query capabilities @@ -246,7 +246,6 @@ The first step is to run after creating the Database and Table is to run a Studi ``` $ cat ~/.studioml/config.yaml - 0   1 metadata.md   2 kmutch@awsdev:~/project/src/github.com/leaf-ai/unileaf   3  zsh    ⇅ ☰ ✔ ?  29.06.2021  16:00  awsdev database: type: s3 endpoint: https://s3-us-west-2.amazonaws.com diff --git a/go.mod b/go.mod index 2424c1537..70fc16e7f 100644 --- a/go.mod +++ b/go.mod @@ -8,7 +8,7 @@ require ( github.com/Rhymond/go-money v1.0.2 github.com/StackExchange/wmi v0.0.0-20210224194228-fe8f1750fd46 // indirect github.com/awnumar/memguard v0.22.2 - github.com/aws/aws-sdk-go v1.38.68 + github.com/aws/aws-sdk-go v1.38.71 github.com/cenkalti/backoff/v4 v4.1.1 github.com/davecgh/go-spew v1.1.1 github.com/deckarep/golang-set v1.7.1 @@ -18,7 +18,6 @@ require ( github.com/dustin/go-humanize v1.0.0 github.com/ekalinin/github-markdown-toc.go v0.0.0-20201214100212-a3e410f71786 // indirect github.com/evanphx/json-patch v4.11.0+incompatible - github.com/evanphx/json-patch/v5 v5.5.0 // indirect github.com/fsnotify/fsnotify v1.4.9 github.com/go-enry/go-license-detector/v4 v4.2.0 github.com/go-ole/go-ole v1.2.5 // indirect @@ -50,7 +49,7 @@ require ( github.com/mholt/archiver/v3 v3.5.0 github.com/michaelklishin/rabbit-hole/v2 v2.10.0 github.com/minio/minio v0.0.0-20210507214158-ab7d5ee3d98e // indirect - github.com/minio/minio-go/v7 v7.0.11 + github.com/minio/minio-go/v7 v7.0.12 github.com/mitchellh/copystructure v1.2.0 github.com/montanaflynn/stats v0.6.6 // indirect github.com/odg0318/aws-ec2-price v0.0.0-20200327030318-88202a04e06d @@ -62,7 +61,7 @@ require ( github.com/prometheus/prom2json v1.3.0 github.com/rs/xid v1.3.0 github.com/russross/blackfriday/v2 v2.1.0 // indirect - github.com/shirou/gopsutil v3.21.5+incompatible + github.com/shirou/gopsutil v3.21.6+incompatible github.com/shirou/gopsutil/v3 v3.21.4 // indirect github.com/streadway/amqp v1.0.1-0.20200716223359-e6b33f460591 github.com/tebeka/atexit v0.3.0 @@ -79,7 +78,7 @@ require ( golang.org/x/tools v0.1.3 // indirect google.golang.org/genproto v0.0.0-20210224155714-063164c882e6 // indirect google.golang.org/grpc v1.36.0 // indirect - google.golang.org/protobuf v1.27.0 + google.golang.org/protobuf v1.27.1 k8s.io/api v0.21.2 k8s.io/apimachinery v0.21.2 k8s.io/client-go v0.21.2 diff --git a/go.sum b/go.sum index 85a02d0a7..477168189 100644 --- a/go.sum +++ b/go.sum @@ -135,12 +135,8 @@ github.com/aws/aws-sdk-go v1.27.0/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN github.com/aws/aws-sdk-go v1.35.20/go.mod h1:tlPOdRjfxPBpNIwqDj61rmsnA85v9jc0Ps9+muhnW+k= github.com/aws/aws-sdk-go v1.37.18/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro= github.com/aws/aws-sdk-go v1.38.45/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro= -github.com/aws/aws-sdk-go v1.38.64 h1:aE178SZNBpAT9T2U5hacKJiyiRE/Li2Hax6xddVuyGA= -github.com/aws/aws-sdk-go v1.38.64/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro= -github.com/aws/aws-sdk-go v1.38.66 h1:z1nEnVOb0ctiHURel3sFxZdXDrXnG6Fa+Dly3Kb0KVo= -github.com/aws/aws-sdk-go v1.38.66/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro= -github.com/aws/aws-sdk-go v1.38.68 h1:aOG8geU4SohNp659eKBHRBgbqSrZ6jNZlfimIuJAwL8= -github.com/aws/aws-sdk-go v1.38.68/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro= +github.com/aws/aws-sdk-go v1.38.71 h1:aWhtgoOiDhBCfaAj9XbxzcyvjEAKovbtv7d5mCVBZXw= +github.com/aws/aws-sdk-go v1.38.71/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro= github.com/aws/aws-sdk-go-v2 v0.18.0/go.mod h1:JWVYvqSMppoMJC0x5wdwiImzgXTI9FuZwxzkQq9wy+g= github.com/bcicen/jstream v1.0.1/go.mod h1:9ielPxqFry7Y4Tg3j4BfjPocfJ3TbsRtXOAYXYmRuAQ= github.com/beevik/ntp v0.3.0/go.mod h1:hIHWr+l3+/clUnF44zdK+CWW7fO8dR5cIylAQ76NRpg= @@ -246,12 +242,9 @@ github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.m github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= -github.com/evanphx/json-patch v4.9.0+incompatible h1:kLcOMZeuLAJvL2BPWLMIj5oaZQobrkAqrL+WFZwQses= github.com/evanphx/json-patch v4.9.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/evanphx/json-patch v4.11.0+incompatible h1:glyUF9yIYtMHzn8xaKw5rMhdWcwsYV8dZHIq5567/xs= github.com/evanphx/json-patch v4.11.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= -github.com/evanphx/json-patch/v5 v5.5.0 h1:bAmFiUJ+o0o2B4OiTFeE3MqCOtyo+jjPP9iZ0VRxYUc= -github.com/evanphx/json-patch/v5 v5.5.0/go.mod h1:G79N1coSVB93tBe7j6PhzjmR3/2VvlbKOFpnXhI9Bw4= github.com/facebookgo/clock v0.0.0-20150410010913-600d898af40a h1:yDWHCSQ40h88yih2JAcL6Ls/kVkSE8GFACTGVnMPruw= github.com/facebookgo/clock v0.0.0-20150410010913-600d898af40a/go.mod h1:7Ga40egUymuWXxAe151lTNnCv97MddSOVsjpPPkityA= github.com/facebookgo/ensure v0.0.0-20200202191622-63f1cf65ac4c h1:8ISkoahWXwZR41ois5lSJBSVw4D0OV19Ht/JSTzvSv0= @@ -685,8 +678,8 @@ github.com/minio/minio v0.0.0-20210507214158-ab7d5ee3d98e h1:1EzTJ9yFQOlghFVg147 github.com/minio/minio v0.0.0-20210507214158-ab7d5ee3d98e/go.mod h1:8HjbXAVfMFI2lVE76TUgW0GXUB8Vlmy75YlJHnwcOiY= github.com/minio/minio-go/v7 v7.0.10/go.mod h1:td4gW1ldOsj1PbSNS+WYK43j+P1XVhX/8W8awaYlBFo= github.com/minio/minio-go/v7 v7.0.11-0.20210302210017-6ae69c73ce78/go.mod h1:mTh2uJuAbEqdhMVl6CMIIZLUeiMiWtJR4JB8/5g2skw= -github.com/minio/minio-go/v7 v7.0.11 h1:7utSkCtMQPYYB1UB8FR3d0QSiOWE6F/JYXon29imYek= -github.com/minio/minio-go/v7 v7.0.11/go.mod h1:WoyW+ySKAKjY98B9+7ZbI8z8S3jaxaisdcvj9TGlazA= +github.com/minio/minio-go/v7 v7.0.12 h1:/4pxUdwn9w0QEryNkrrWaodIESPRX+NxpO0Q6hVdaAA= +github.com/minio/minio-go/v7 v7.0.12/go.mod h1:S23iSP5/gbMwtxeY5FM71R+TkAYyzEdoNEDDwpt8yWs= github.com/minio/parquet-go v1.0.0/go.mod h1:aQlkSOfOq2AtQKkuou3mosNVMwNokd+faTacxxk/oHA= github.com/minio/rpc v1.0.0/go.mod h1:b9xqF7J0xeMXr0cM4pnBlP7Te7PDsG5JrRxl5dG6Ldk= github.com/minio/selfupdate v0.3.1/go.mod h1:b8ThJzzH7u2MkF6PcIra7KaXO9Khf6alWPvMSyTDCFM= @@ -893,8 +886,8 @@ github.com/sergi/go-diff v1.1.0 h1:we8PVUC3FE2uYfodKH/nBHMSetSfHDR6scGdBi+erh0= github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= github.com/shirou/gopsutil v3.20.11+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA= github.com/shirou/gopsutil v3.21.4+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA= -github.com/shirou/gopsutil v3.21.5+incompatible h1:OloQyEerMi7JUrXiNzy8wQ5XN+baemxSl12QgIzt0jc= -github.com/shirou/gopsutil v3.21.5+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA= +github.com/shirou/gopsutil v3.21.6+incompatible h1:mmZtAlWSd8U2HeRTjswbnDLPxqsEoK01NK+GZ1P+nEM= +github.com/shirou/gopsutil v3.21.6+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA= github.com/shirou/gopsutil/v3 v3.21.1/go.mod h1:igHnfak0qnw1biGeI2qKQvu0ZkwvEkUcCLlYhZzdr/4= github.com/shirou/gopsutil/v3 v3.21.3/go.mod h1:ghfMypLDrFSWN2c9cDYFLHyynQ+QUht0cv/18ZqVczw= github.com/shirou/gopsutil/v3 v3.21.4 h1:XB/+p+kVnyYLuPHCfa99lxz2aJyvVhnyd+FxZqH/k7M= @@ -1066,6 +1059,7 @@ golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20201016220609-9e8e0b390897/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20201112155050-0c6587e931a9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20201124201722-c8d3bf9c5392/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= +golang.org/x/crypto v0.0.0-20201216223049-8b5274cf687f/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= golang.org/x/crypto v0.0.0-20210415154028-4f45737414dc/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= @@ -1430,10 +1424,9 @@ google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpAD google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4= google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= -google.golang.org/protobuf v1.26.0 h1:bxAC2xTBsZGibn2RTntX0oH50xLsqy1OxA9tTL3p/lk= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.27.0 h1:KhgSLlr/moiqjv0qUsSnLvdUL7NH7PHW8aZGn7Jpjko= -google.golang.org/protobuf v1.27.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.27.1 h1:SnqbnDw1V7RiZcXPx5MEeqPv2s79L9i7BJUlG/+RurQ= +google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= gopkg.in/alecthomas/kingpin.v2 v2.2.4/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/alecthomas/kingpin.v2 v2.2.6 h1:jMFz6MfLP0/4fUyZle81rXUoxOBFi19VUFKVDOQfozc= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= diff --git a/internal/gen/dev.cognizant_dev.ai/genproto/studio-go-runner/reports/v1/reports.pb.go b/internal/gen/dev.cognizant_dev.ai/genproto/studio-go-runner/reports/v1/reports.pb.go index fadf602e3..276502f3f 100644 --- a/internal/gen/dev.cognizant_dev.ai/genproto/studio-go-runner/reports/v1/reports.pb.go +++ b/internal/gen/dev.cognizant_dev.ai/genproto/studio-go-runner/reports/v1/reports.pb.go @@ -2,7 +2,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.27.0 +// protoc-gen-go v1.27.1 // protoc v3.15.8 // source: reports.proto diff --git a/internal/runner/pythonenv.go b/internal/runner/pythonenv.go index 4fdb4b59b..04c3cf9ca 100644 --- a/internal/runner/pythonenv.go +++ b/internal/runner/pythonenv.go @@ -239,6 +239,7 @@ func (p *VirtualEnv) Make(alloc *resources.Allocated, e interface{}) (err kv.Err // the python environment in a virtual env tmpl, errGo := template.New("pythonRunner").Parse( `#!/bin/bash -x +echo "{\"studioml\": {\"log\": [{\"ts\": \"` + "`" + `date -u -Ins` + "`" + `\", \"msg\":\"Init\"},{\"ts\":\"0\", \"msg\":\"\"}]}}" | jq -c '.' sleep 2 # Credit https://github.com/fernandoacorreia/azure-docker-registry/blob/master/tools/scripts/create-registry-server function fail { @@ -351,8 +352,8 @@ nvidia-smi mig -i 5 -cgi 14,14,14 -C || true nvidia-smi mig -i 6 -cgi 14,14,14 -C || true nvidia-smi mig -i 7 -cgi 14,14,14 -C || true nvidia-smi 2>/dev/null || true -echo "{\"studioml\": {\"log\": [{\"ts\": \"` + "`" + `date -u -Ins` + "`" + `\", \"msg\":\"Start\"},{\"ts\":\"0\", \"msg\":\"\"}]}}" | jq -c '.' -python {{.E.Request.Experiment.Filename}} {{range .E.Request.Experiment.Args}}{{.}} {{end}} +echo "[{\"op\": \"add\", \"path\": \"/studioml/log/-\", \"value\": {\"ts\": \"` + "`" + `date -u -Ins` + "`" + `\", \"msg\":\"Start\"}}]" | jq -c '.' +stdbuf -oL -eL python {{.E.Request.Experiment.Filename}} {{range .E.Request.Experiment.Args}}{{.}} {{end}} result=$? echo $result set +e diff --git a/licenses.manifest b/licenses.manifest index 5f9c3b8da..7dfebd34b 100644 --- a/licenses.manifest +++ b/licenses.manifest @@ -3,9 +3,9 @@ docs/slides,MIT-0,0.8050314 docs/slides/lib/font/source-sans-pro,OFL-1.0-no-RFN,0.757377 examples/aws/aws,OpenSSL,0.7797131 examples/aws/aws/dist/cryptography-2.8-py3.7.egg-info,BSD-3-Clause-Clear,0.83412325 -examples/aws/cpu,MPL-2.0-no-copyleft-exception,0.33333334 -examples/docker,deprecated_GPL-2.0-with-font-exception,0.33333334 -examples/local,deprecated_GPL-2.0-with-bison-exception,0.33333334 +examples/aws/cpu,ADSL,0.33333334 +examples/docker,CC-BY-NC-ND-2.0,0.33333334 +examples/local,CC-BY-NC-ND-2.0,0.33333334 vendor/cloud.google.com/go,SHL-0.5,0.85278857 vendor/github.com/Azure/go-autorest,ECL-2.0,0.83882034 vendor/github.com/Azure/go-autorest/autorest,ECL-2.0,0.83882034 @@ -129,7 +129,7 @@ vendor/github.com/pkg/errors,BSD-2-Clause-NetBSD,0.9076087 vendor/github.com/prometheus/client_golang,SHL-0.5,0.85278857 vendor/github.com/prometheus/client_model,SHL-0.5,0.85278857 vendor/github.com/prometheus/common,SHL-0.5,0.85278857 -vendor/github.com/prometheus/common/internal/bitbucket.org/ww/goautoneg,NOSL,0.33333334 +vendor/github.com/prometheus/common/internal/bitbucket.org/ww/goautoneg,OSL-1.1,0.33333334 vendor/github.com/prometheus/procfs,SHL-0.5,0.85278857 vendor/github.com/prometheus/prom2json,SHL-0.5,0.85278857 vendor/github.com/rs/xid,MIT-0,0.8125 @@ -184,6 +184,10 @@ vendor/k8s.io/client-go,SHL-0.5,0.85278857 vendor/k8s.io/klog/v2,SHL-0.5,0.8657289 vendor/k8s.io/utils,SHL-0.5,0.85278857 vendor/sigs.k8s.io/structured-merge-diff/v4,SHL-0.5,0.85214007 +/v4,SHL-0.5,0.85214007 +tured-merge-diff/v4,SHL-0.5,0.85214007 +4007 +7 07 s.io/structured-merge-diff/v4,SHL-0.5,0.85214007 rge-diff/v4,SHL-0.5,0.85214007 diff --git a/tools/queue-scaler/sqs_job.yaml b/tools/queue-scaler/sqs_job.yaml index 37eb8e802..0ba1aa285 100644 --- a/tools/queue-scaler/sqs_job.yaml +++ b/tools/queue-scaler/sqs_job.yaml @@ -18,6 +18,7 @@ data: CACHE_DIR: "/tmp/cache" CLEAR_TEXT_MESSAGES: "true" LIMIT_IDLE_DURATION: "15m" + SCHEMA_LOGS: "true" DEBUG: "true" --- apiVersion: batch/v1 diff --git a/vendor/github.com/aws/aws-sdk-go/aws/endpoints/defaults.go b/vendor/github.com/aws/aws-sdk-go/aws/endpoints/defaults.go index 887c21dcb..27797a72e 100644 --- a/vendor/github.com/aws/aws-sdk-go/aws/endpoints/defaults.go +++ b/vendor/github.com/aws/aws-sdk-go/aws/endpoints/defaults.go @@ -982,6 +982,7 @@ var awsPartition = partition{ "ap-east-1": endpoint{}, "ap-northeast-1": endpoint{}, "ap-northeast-2": endpoint{}, + "ap-northeast-3": endpoint{}, "ap-south-1": endpoint{}, "ap-southeast-1": endpoint{}, "ap-southeast-2": endpoint{}, @@ -3047,6 +3048,7 @@ var awsPartition = partition{ "ap-east-1": endpoint{}, "ap-northeast-1": endpoint{}, "ap-northeast-2": endpoint{}, + "ap-northeast-3": endpoint{}, "ap-south-1": endpoint{}, "ap-southeast-1": endpoint{}, "ap-southeast-2": endpoint{}, @@ -6737,6 +6739,7 @@ var awsPartition = partition{ "ap-east-1": endpoint{}, "ap-northeast-1": endpoint{}, "ap-northeast-2": endpoint{}, + "ap-northeast-3": endpoint{}, "ap-south-1": endpoint{}, "ap-southeast-1": endpoint{}, "ap-southeast-2": endpoint{}, diff --git a/vendor/github.com/aws/aws-sdk-go/aws/version.go b/vendor/github.com/aws/aws-sdk-go/aws/version.go index db6522049..95a41a03b 100644 --- a/vendor/github.com/aws/aws-sdk-go/aws/version.go +++ b/vendor/github.com/aws/aws-sdk-go/aws/version.go @@ -5,4 +5,4 @@ package aws const SDKName = "aws-sdk-go" // SDKVersion is the version of this SDK -const SDKVersion = "1.38.68" +const SDKVersion = "1.38.71" diff --git a/vendor/github.com/aws/aws-sdk-go/service/autoscaling/api.go b/vendor/github.com/aws/aws-sdk-go/service/autoscaling/api.go index fc332608b..10a4a166f 100644 --- a/vendor/github.com/aws/aws-sdk-go/service/autoscaling/api.go +++ b/vendor/github.com/aws/aws-sdk-go/service/autoscaling/api.go @@ -1751,8 +1751,10 @@ func (c *AutoScaling) DescribeAccountLimitsRequest(input *DescribeAccountLimitsI // // Describes the current Amazon EC2 Auto Scaling resource quotas for your account. // -// For information about requesting an increase, see Amazon EC2 Auto Scaling -// service quotas (https://docs.aws.amazon.com/autoscaling/ec2/userguide/as-account-limits.html) +// When you establish an account, the account has initial quotas on the maximum +// number of Auto Scaling groups and launch configurations that you can create +// in a given Region. For more information, see Amazon EC2 Auto Scaling service +// quotas (https://docs.aws.amazon.com/autoscaling/ec2/userguide/as-account-limits.html) // in the Amazon EC2 Auto Scaling User Guide. // // Returns awserr.Error for service API and SDK errors. Use runtime type assertions @@ -7054,6 +7056,9 @@ type CreateAutoScalingGroupInput struct { // in the Amazon EC2 Auto Scaling User Guide. CapacityRebalance *bool `type:"boolean"` + // Reserved. + Context *string `type:"string"` + // The amount of time, in seconds, after a scaling activity completes before // another scaling activity can start. The default value is 300. This setting // applies when using simple scaling policies, but not when using other scaling @@ -7318,6 +7323,12 @@ func (s *CreateAutoScalingGroupInput) SetCapacityRebalance(v bool) *CreateAutoSc return s } +// SetContext sets the Context field's value. +func (s *CreateAutoScalingGroupInput) SetContext(v string) *CreateAutoScalingGroupInput { + s.Context = &v + return s +} + // SetDefaultCooldown sets the DefaultCooldown field's value. func (s *CreateAutoScalingGroupInput) SetDefaultCooldown(v int64) *CreateAutoScalingGroupInput { s.DefaultCooldown = &v @@ -10427,15 +10438,16 @@ type Ebs struct { // If you are creating a volume from a snapshot, you cannot specify an encryption // value. Volumes that are created from encrypted snapshots are automatically // encrypted, and volumes that are created from unencrypted snapshots are automatically - // unencrypted. By default, encrypted snapshots use the AWS managed CMK that - // is used for EBS encryption, but you can specify a custom CMK when you create - // the snapshot. The ability to encrypt a snapshot during copying also allows - // you to apply a new CMK to an already-encrypted snapshot. Volumes restored - // from the resulting copy are only accessible using the new CMK. + // unencrypted. By default, encrypted snapshots use the Amazon Web Services + // managed CMK that is used for EBS encryption, but you can specify a custom + // CMK when you create the snapshot. The ability to encrypt a snapshot during + // copying also allows you to apply a new CMK to an already-encrypted snapshot. + // Volumes restored from the resulting copy are only accessible using the new + // CMK. // // Enabling encryption by default (https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/EBSEncryption.html#encryption-by-default) - // results in all EBS volumes being encrypted with the AWS managed CMK or a - // customer managed CMK, whether or not the snapshot was encrypted. + // results in all EBS volumes being encrypted with the Amazon Web Services managed + // CMK or a customer managed CMK, whether or not the snapshot was encrypted. // // For more information, see Using Encryption with EBS-Backed AMIs (https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/AMIEncryption.html) // in the Amazon EC2 User Guide for Linux Instances and Required CMK key policy @@ -11285,6 +11297,9 @@ type Group struct { // Indicates whether Capacity Rebalancing is enabled. CapacityRebalance *bool `type:"boolean"` + // Reserved. + Context *string `type:"string"` + // The date and time the group was created. // // CreatedTime is a required field @@ -11419,6 +11434,12 @@ func (s *Group) SetCapacityRebalance(v bool) *Group { return s } +// SetContext sets the Context field's value. +func (s *Group) SetContext(v string) *Group { + s.Context = &v + return s +} + // SetCreatedTime sets the CreatedTime field's value. func (s *Group) SetCreatedTime(v time.Time) *Group { s.CreatedTime = &v @@ -13452,14 +13473,19 @@ type PredefinedMetricSpecification struct { // PredefinedMetricType is a required field PredefinedMetricType *string `type:"string" required:"true" enum:"MetricType"` - // Identifies the resource associated with the metric type. You can't specify - // a resource label unless the metric type is ALBRequestCountPerTarget and there - // is a target group attached to the Auto Scaling group. + // A label that uniquely identifies a specific Application Load Balancer target + // group from which to determine the average request count served by your Auto + // Scaling group. You can't specify a resource label unless the target group + // is attached to the Auto Scaling group. // // You create the resource label by appending the final portion of the load // balancer ARN and the final portion of the target group ARN into a single - // value, separated by a forward slash (/). The format is app///targetgroup//, - // where: + // value, separated by a forward slash (/). The format of the resource label + // is: + // + // app/my-alb/778d41231b141a0f/targetgroup/my-alb-target-group/943f017f100becff. + // + // Where: // // * app// is the final portion of // the load balancer ARN @@ -13467,8 +13493,6 @@ type PredefinedMetricSpecification struct { // * targetgroup// is the final portion // of the target group ARN. // - // This is an example: app/EC2Co-EcsEl-1TKLTMITMM0EO/f37c06a68c1748aa/targetgroup/EC2Co-Defau-LDNM7Q3ZH1ZN/6d4ea56ca2d6a18d. - // // To find the ARN for an Application Load Balancer, use the DescribeLoadBalancers // (https://docs.aws.amazon.com/elasticloadbalancing/latest/APIReference/API_DescribeLoadBalancers.html) // API operation. To find the ARN for the target group, use the DescribeTargetGroups @@ -13778,7 +13802,7 @@ type PredictiveScalingPredefinedLoadMetric struct { // value, separated by a forward slash (/). The format of the resource label // is: // - // app/EC2Co-EcsEl-1TKLTMITMM0EO/f37c06a68c1748aa/targetgroup/EC2Co-Defau-LDNM7Q3ZH1ZN/6d4ea56ca2d6a18d. + // app/my-alb/778d41231b141a0f/targetgroup/my-alb-target-group/943f017f100becff. // // Where: // @@ -13848,16 +13872,16 @@ type PredictiveScalingPredefinedMetricPair struct { PredefinedMetricType *string `type:"string" required:"true" enum:"PredefinedMetricPairType"` // A label that uniquely identifies a specific Application Load Balancer target - // group from which to determine the request count served by your Auto Scaling - // group. You can't specify a resource label unless the target group is attached - // to the Auto Scaling group. + // group from which to determine the total and average request count served + // by your Auto Scaling group. You can't specify a resource label unless the + // target group is attached to the Auto Scaling group. // // You create the resource label by appending the final portion of the load // balancer ARN and the final portion of the target group ARN into a single // value, separated by a forward slash (/). The format of the resource label // is: // - // app/EC2Co-EcsEl-1TKLTMITMM0EO/f37c06a68c1748aa/targetgroup/EC2Co-Defau-LDNM7Q3ZH1ZN/6d4ea56ca2d6a18d. + // app/my-alb/778d41231b141a0f/targetgroup/my-alb-target-group/943f017f100becff. // // Where: // @@ -13927,16 +13951,16 @@ type PredictiveScalingPredefinedScalingMetric struct { PredefinedMetricType *string `type:"string" required:"true" enum:"PredefinedScalingMetricType"` // A label that uniquely identifies a specific Application Load Balancer target - // group from which to determine the request count served by your Auto Scaling - // group. You can't specify a resource label unless the target group is attached - // to the Auto Scaling group. + // group from which to determine the average request count served by your Auto + // Scaling group. You can't specify a resource label unless the target group + // is attached to the Auto Scaling group. // // You create the resource label by appending the final portion of the load // balancer ARN and the final portion of the target group ARN into a single // value, separated by a forward slash (/). The format of the resource label // is: // - // app/EC2Co-EcsEl-1TKLTMITMM0EO/f37c06a68c1748aa/targetgroup/EC2Co-Defau-LDNM7Q3ZH1ZN/6d4ea56ca2d6a18d. + // app/my-alb/778d41231b141a0f/targetgroup/my-alb-target-group/943f017f100becff. // // Where: // @@ -16379,6 +16403,9 @@ type UpdateAutoScalingGroupInput struct { // in the Amazon EC2 Auto Scaling User Guide. CapacityRebalance *bool `type:"boolean"` + // Reserved. + Context *string `type:"string"` + // The amount of time, in seconds, after a scaling activity completes before // another scaling activity can start. The default value is 300. This setting // applies when using simple scaling policies, but not when using other scaling @@ -16545,6 +16572,12 @@ func (s *UpdateAutoScalingGroupInput) SetCapacityRebalance(v bool) *UpdateAutoSc return s } +// SetContext sets the Context field's value. +func (s *UpdateAutoScalingGroupInput) SetContext(v string) *UpdateAutoScalingGroupInput { + s.Context = &v + return s +} + // SetDefaultCooldown sets the DefaultCooldown field's value. func (s *UpdateAutoScalingGroupInput) SetDefaultCooldown(v int64) *UpdateAutoScalingGroupInput { s.DefaultCooldown = &v diff --git a/vendor/github.com/aws/aws-sdk-go/service/ec2/api.go b/vendor/github.com/aws/aws-sdk-go/service/ec2/api.go index 42c06eba4..c85e7824c 100644 --- a/vendor/github.com/aws/aws-sdk-go/service/ec2/api.go +++ b/vendor/github.com/aws/aws-sdk-go/service/ec2/api.go @@ -6719,7 +6719,7 @@ func (c *EC2) CreateSpotDatafeedSubscriptionRequest(input *CreateSpotDatafeedSub // CreateSpotDatafeedSubscription API operation for Amazon Elastic Compute Cloud. // // Creates a data feed for Spot Instances, enabling you to view Spot Instance -// usage logs. You can create one data feed per AWS account. For more information, +// usage logs. You can create one data feed per account. For more information, // see Spot Instance data feed (https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/spot-data-feeds.html) // in the Amazon EC2 User Guide for Linux Instances. // @@ -14136,7 +14136,7 @@ func (c *EC2) DescribeCapacityReservationsRequest(input *DescribeCapacityReserva // DescribeCapacityReservations API operation for Amazon Elastic Compute Cloud. // // Describes one or more of your Capacity Reservations. The results describe -// only the Capacity Reservations in the AWS Region that you're currently using. +// only the Capacity Reservations in the Region that you're currently using. // // Returns awserr.Error for service API and SDK errors. Use runtime type assertions // with awserr.Error's Code and Message methods to get detailed information about @@ -31158,9 +31158,9 @@ func (c *EC2) GetCapacityReservationUsageRequest(input *GetCapacityReservationUs // // Gets usage information about a Capacity Reservation. If the Capacity Reservation // is shared, it shows usage information for the Capacity Reservation owner -// and each AWS account that is currently using the shared capacity. If the -// Capacity Reservation is not shared, it shows only the Capacity Reservation -// owner's usage. +// and each account that is currently using the shared capacity. If the Capacity +// Reservation is not shared, it shows only the Capacity Reservation owner's +// usage. // // Returns awserr.Error for service API and SDK errors. Use runtime type assertions // with awserr.Error's Code and Message methods to get detailed information about @@ -34059,14 +34059,14 @@ func (c *EC2) ModifyDefaultCreditSpecificationRequest(input *ModifyDefaultCredit // ModifyDefaultCreditSpecification API operation for Amazon Elastic Compute Cloud. // // Modifies the default credit option for CPU usage of burstable performance -// instances. The default credit option is set at the account level per AWS -// Region, and is specified per instance family. All new burstable performance -// instances in the account launch using the default credit option. +// instances. The default credit option is set at the account level per Region, +// and is specified per instance family. All new burstable performance instances +// in the account launch using the default credit option. // // ModifyDefaultCreditSpecification is an asynchronous operation, which works -// at an AWS Region level and modifies the credit option for each Availability -// Zone. All zones in a Region are updated within five minutes. But if instances -// are launched during this operation, they might not get the new credit option +// at an Region level and modifies the credit option for each Availability Zone. +// All zones in a Region are updated within five minutes. But if instances are +// launched during this operation, they might not get the new credit option // until the zone is updated. To verify whether the update has occurred, you // can call GetDefaultCreditSpecification and check DefaultCreditSpecification // for updates. @@ -41239,11 +41239,7 @@ func (c *EC2) StartInstancesRequest(input *StartInstancesInput) (req *request.Re // released and you are not billed for instance usage. However, your root partition // Amazon EBS volume remains and continues to persist your data, and you are // charged for Amazon EBS volume usage. You can restart your instance at any -// time. Every time you start your Windows instance, Amazon EC2 charges you -// for a full instance hour. If you stop and restart your Windows instance, -// a new instance hour begins and Amazon EC2 charges you for another full instance -// hour even if you are still within the same 60-minute period when it was stopped. -// Every time you start your Linux instance, Amazon EC2 charges a one-minute +// time. Every time you start your instance, Amazon EC2 charges a one-minute // minimum for instance usage, and thereafter charges per second for instance // usage. // @@ -41497,12 +41493,8 @@ func (c *EC2) StopInstancesRequest(input *StopInstancesInput) (req *request.Requ // We don't charge usage for a stopped instance, or data transfer fees; however, // your root partition Amazon EBS volume remains and continues to persist your // data, and you are charged for Amazon EBS volume usage. Every time you start -// your Windows instance, Amazon EC2 charges you for a full instance hour. If -// you stop and restart your Windows instance, a new instance hour begins and -// Amazon EC2 charges you for another full instance hour even if you are still -// within the same 60-minute period when it was stopped. Every time you start -// your Linux instance, Amazon EC2 charges a one-minute minimum for instance -// usage, and thereafter charges per second for instance usage. +// your instance, Amazon EC2 charges a one-minute minimum for instance usage, +// and thereafter charges per second for instance usage. // // You can't stop or hibernate instance store-backed instances. You can't use // the Stop action to hibernate Spot Instances, but you can specify that Amazon @@ -41686,6 +41678,36 @@ func (c *EC2) TerminateInstancesRequest(input *TerminateInstancesInput) (req *re // If you specify multiple instances and the request fails (for example, because // of a single incorrect instance ID), none of the instances are terminated. // +// If you terminate multiple instances across multiple Availability Zones, and +// one or more of the specified instances are enabled for termination protection, +// the request fails with the following results: +// +// * The specified instances that are in the same Availability Zone as the +// protected instance are not terminated. +// +// * The specified instances that are in different Availability Zones, where +// no other specified instances are protected, are successfully terminated. +// +// For example, say you have the following instances: +// +// * Instance A: us-east-1a; Not protected +// +// * Instance B: us-east-1a; Not protected +// +// * Instance C: us-east-1b; Protected +// +// * Instance D: us-east-1b; not protected +// +// If you attempt to terminate all of these instances in the same request, the +// request reports failure with the following results: +// +// * Instance A and Instance B are successfully terminated because none of +// the specified instances in us-east-1a are enabled for termination protection. +// +// * Instance C and Instance D fail to terminate because at least one of +// the specified instances in us-east-1b (Instance C) is enabled for termination +// protection. +// // Terminated instances remain visible after termination (for approximately // one hour). // @@ -47906,7 +47928,7 @@ type CapacityReservation struct { // was created. OutpostArn *string `locationName:"outpostArn" type:"string"` - // The ID of the AWS account that owns the Capacity Reservation. + // The ID of the account that owns the Capacity Reservation. OwnerId *string `locationName:"ownerId" type:"string"` // The date and time at which the Capacity Reservation was started. @@ -47940,10 +47962,10 @@ type CapacityReservation struct { // can have one of the following tenancy settings: // // * default - The Capacity Reservation is created on hardware that is shared - // with other AWS accounts. + // with other accounts. // // * dedicated - The Capacity Reservation is created on single-tenant hardware - // that is dedicated to a single AWS account. + // that is dedicated to a single account. Tenancy *string `locationName:"tenancy" type:"string" enum:"CapacityReservationTenancy"` // The total number of instances for which the Capacity Reservation reserves @@ -48088,7 +48110,7 @@ type CapacityReservationGroup struct { // The ARN of the resource group. GroupArn *string `locationName:"groupArn" type:"string"` - // The ID of the AWS account that owns the resource group. + // The ID of the account that owns the resource group. OwnerId *string `locationName:"ownerId" type:"string"` } @@ -49767,7 +49789,7 @@ func (s *ConfirmProductInstanceInput) SetProductCode(v string) *ConfirmProductIn type ConfirmProductInstanceOutput struct { _ struct{} `type:"structure"` - // The AWS account ID of the instance owner. This is only present if the product + // The account ID of the instance owner. This is only present if the product // code is attached to the instance. OwnerId *string `locationName:"ownerId" type:"string"` @@ -50721,10 +50743,10 @@ type CreateCapacityReservationInput struct { // can have one of the following tenancy settings: // // * default - The Capacity Reservation is created on hardware that is shared - // with other AWS accounts. + // with other accounts. // // * dedicated - The Capacity Reservation is created on single-tenant hardware - // that is dedicated to a single AWS account. + // that is dedicated to a single account. Tenancy *string `type:"string" enum:"CapacityReservationTenancy"` } @@ -51861,6 +51883,9 @@ type CreateFleetInput struct { // of the request. For more information, see Ensuring Idempotency (https://docs.aws.amazon.com/AWSEC2/latest/APIReference/Run_Instance_Idempotency.html). ClientToken *string `type:"string"` + // Reserved. + Context *string `type:"string"` + // Checks whether you have the required permissions for the action, without // actually making the request, and provides an error response. If you have // the required permissions, the error response is DryRunOperation. Otherwise, @@ -51889,10 +51914,15 @@ type CreateFleetInput struct { // Describes the configuration of Spot Instances in an EC2 Fleet. SpotOptions *SpotOptionsRequest `type:"structure"` - // The key-value pair for tagging the EC2 Fleet request on creation. The value - // for ResourceType must be fleet, otherwise the fleet request fails. To tag - // instances at launch, specify the tags in the launch template (https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-launch-templates.html#create-launch-template). - // For information about tagging after launch, see Tagging your resources (https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/Using_Tags.html#tag-resources). + // The key-value pair for tagging the EC2 Fleet request on creation. For more + // information, see Tagging your resources (https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/Using_Tags.html#tag-resources). + // + // If the fleet type is instant, specify a resource type of fleet to tag the + // fleet or instance to tag the instances at launch. + // + // If the fleet type is maintain or request, specify a resource type of fleet + // to tag the fleet. You cannot specify a resource type of instance. To tag + // instances at launch, specify the tags in a launch template (https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-launch-templates.html#create-launch-template). TagSpecifications []*TagSpecification `locationName:"TagSpecification" locationNameList:"item" type:"list"` // The number of units to request. @@ -51904,7 +51934,7 @@ type CreateFleetInput struct { // expires. TerminateInstancesWithExpiration *bool `type:"boolean"` - // The type of request. The default value is maintain. + // The fleet type. The default value is maintain. // // * maintain - The EC2 Fleet places an asynchronous request for your desired // capacity, and continues to maintain your desired Spot capacity by replenishing @@ -51980,6 +52010,12 @@ func (s *CreateFleetInput) SetClientToken(v string) *CreateFleetInput { return s } +// SetContext sets the Context field's value. +func (s *CreateFleetInput) SetContext(v string) *CreateFleetInput { + s.Context = &v + return s +} + // SetDryRun sets the DryRun field's value. func (s *CreateFleetInput) SetDryRun(v bool) *CreateFleetInput { s.DryRun = &v @@ -52119,14 +52155,14 @@ type CreateFleetOutput struct { _ struct{} `type:"structure"` // Information about the instances that could not be launched by the fleet. - // Valid only when Type is set to instant. + // Supported only for fleets of type instant. Errors []*CreateFleetError `locationName:"errorSet" locationNameList:"item" type:"list"` // The ID of the EC2 Fleet. FleetId *string `locationName:"fleetId" type:"string"` - // Information about the instances that were launched by the fleet. Valid only - // when Type is set to instant. + // Information about the instances that were launched by the fleet. Supported + // only for fleets of type instant. Instances []*CreateFleetInstance `locationName:"fleetInstanceSet" locationNameList:"item" type:"list"` } @@ -64214,7 +64250,7 @@ type DescribeCapacityReservationsInput struct { // * instance-type - The type of instance for which the Capacity Reservation // reserves capacity. // - // * owner-id - The ID of the AWS account that owns the Capacity Reservation. + // * owner-id - The ID of the account that owns the Capacity Reservation. // // * availability-zone-id - The Availability Zone ID of the Capacity Reservation. // @@ -64226,8 +64262,8 @@ type DescribeCapacityReservationsInput struct { // * tenancy - Indicates the tenancy of the Capacity Reservation. A Capacity // Reservation can have one of the following tenancy settings: default - // The Capacity Reservation is created on hardware that is shared with other - // AWS accounts. dedicated - The Capacity Reservation is created on single-tenant - // hardware that is dedicated to a single AWS account. + // accounts. dedicated - The Capacity Reservation is created on single-tenant + // hardware that is dedicated to a single account. // // * outpost-arn - The Amazon Resource Name (ARN) of the Outpost on which // the Capacity Reservation was created. @@ -68327,8 +68363,8 @@ type DescribeInstanceAttributeOutput struct { // Indicates whether enhanced networking with ENA is enabled. EnaSupport *AttributeBooleanValue `locationName:"enaSupport" type:"structure"` - // To enable the instance for AWS Nitro Enclaves, set this parameter to true; - // otherwise, set it to false. + // To enable the instance for Amazon Web Services Nitro Enclaves, set this parameter + // to true; otherwise, set it to false. EnclaveOptions *EnclaveOptions `locationName:"enclaveOptions" type:"structure"` // The security groups associated with the instance. @@ -69320,7 +69356,7 @@ type DescribeInstancesInput struct { // * network-interface.requester-id - The requester ID for the network interface. // // * network-interface.requester-managed - Indicates whether the network - // interface is being managed by AWS. + // interface is being managed by Amazon Web Services. // // * network-interface.status - The status of the network interface (available) // | in-use). @@ -69337,7 +69373,7 @@ type DescribeInstancesInput struct { // // * outpost-arn - The Amazon Resource Name (ARN) of the Outpost. // - // * owner-id - The AWS account ID of the instance owner. + // * owner-id - The account ID of the instance owner. // // * placement-group-name - The name of the placement group for the instance. // @@ -69362,7 +69398,7 @@ type DescribeInstancesInput struct { // Similar to the state-reason-code filter. // // * requester-id - The ID of the entity that launched the instance on your - // behalf (for example, AWS Management Console, Auto Scaling, and so on). + // behalf (for example, Management Console, Auto Scaling, and so on). // // * reservation-id - The ID of the instance's reservation. A reservation // ID is created any time you launch an instance. A reservation ID has a @@ -73015,8 +73051,8 @@ type DescribeReservedInstancesOfferingsInput struct { // // * marketplace - Set to true to show only Reserved Instance Marketplace // offerings. When this filter is not used, which is the default behavior, - // all offerings from both AWS and the Reserved Instance Marketplace are - // listed. + // all offerings from both Amazon Web Services and the Reserved Instance + // Marketplace are listed. // // * product-description - The Reserved Instance product platform description. // Instances that include (Amazon VPC) in the product platform description @@ -83069,12 +83105,13 @@ func (s *EnableVpcClassicLinkOutput) SetReturn(v bool) *EnableVpcClassicLinkOutp return s } -// Indicates whether the instance is enabled for AWS Nitro Enclaves. +// Indicates whether the instance is enabled for Amazon Web Services Nitro Enclaves. type EnclaveOptions struct { _ struct{} `type:"structure"` - // If this parameter is set to true, the instance is enabled for AWS Nitro Enclaves; - // otherwise, it is not enabled for AWS Nitro Enclaves. + // If this parameter is set to true, the instance is enabled for Amazon Web + // Services Nitro Enclaves; otherwise, it is not enabled for Amazon Web Services + // Nitro Enclaves. Enabled *bool `locationName:"enabled" type:"boolean"` } @@ -83094,13 +83131,14 @@ func (s *EnclaveOptions) SetEnabled(v bool) *EnclaveOptions { return s } -// Indicates whether the instance is enabled for AWS Nitro Enclaves. For more -// information, see What is AWS Nitro Enclaves? (https://docs.aws.amazon.com/enclaves/latest/user/nitro-enclave.html) -// in the AWS Nitro Enclaves User Guide. +// Indicates whether the instance is enabled for Amazon Web Services Nitro Enclaves. +// For more information, see What is Amazon Web Services Nitro Enclaves? (https://docs.aws.amazon.com/enclaves/latest/user/nitro-enclave.html) +// in the Amazon Web Services Nitro Enclaves User Guide. type EnclaveOptionsRequest struct { _ struct{} `type:"structure"` - // To enable the instance for AWS Nitro Enclaves, set this parameter to true. + // To enable the instance for Amazon Web Services Nitro Enclaves, set this parameter + // to true. Enabled *bool `type:"boolean"` } @@ -84660,6 +84698,9 @@ type FleetData struct { // Constraints: Maximum 64 ASCII characters ClientToken *string `locationName:"clientToken" type:"string"` + // Reserved. + Context *string `locationName:"context" type:"string"` + // The creation date and time of the EC2 Fleet. CreateTime *time.Time `locationName:"createTime" type:"timestamp"` @@ -84760,6 +84801,12 @@ func (s *FleetData) SetClientToken(v string) *FleetData { return s } +// SetContext sets the Context field's value. +func (s *FleetData) SetContext(v string) *FleetData { + s.Context = &v + return s +} + // SetCreateTime sets the CreateTime field's value. func (s *FleetData) SetCreateTime(v time.Time) *FleetData { s.CreateTime = &v @@ -88634,7 +88681,7 @@ type Host struct { // is true, the host is in a host resource group; otherwise, it is not. MemberOfServiceLinkedResourceGroup *bool `locationName:"memberOfServiceLinkedResourceGroup" type:"boolean"` - // The ID of the AWS account that owns the Dedicated Host. + // The ID of the account that owns the Dedicated Host. OwnerId *string `locationName:"ownerId" type:"string"` // The time that the Dedicated Host was released. @@ -88769,7 +88816,7 @@ type HostInstance struct { // The instance type (for example, m3.medium) of the running instance. InstanceType *string `locationName:"instanceType" type:"string"` - // The ID of the AWS account that owns the instance. + // The ID of the account that owns the instance. OwnerId *string `locationName:"ownerId" type:"string"` } @@ -91335,7 +91382,7 @@ type Instance struct { // Specifies whether enhanced networking with ENA is enabled. EnaSupport *bool `locationName:"enaSupport" type:"boolean"` - // Indicates whether the instance is enabled for AWS Nitro Enclaves. + // Indicates whether the instance is enabled for Amazon Web Services Nitro Enclaves. EnclaveOptions *EnclaveOptions `locationName:"enclaveOptions" type:"structure"` // Indicates whether the instance is enabled for hibernation. @@ -93577,10 +93624,10 @@ func (s *InstanceTypeOffering) SetLocationType(v string) *InstanceTypeOffering { type InstanceUsage struct { _ struct{} `type:"structure"` - // The ID of the AWS account that is making use of the Capacity Reservation. + // The ID of the account that is making use of the Capacity Reservation. AccountId *string `locationName:"accountId" type:"string"` - // The number of instances the AWS account currently has in the Capacity Reservation. + // The number of instances the account currently has in the Capacity Reservation. UsedInstanceCount *int64 `locationName:"usedInstanceCount" type:"integer"` } @@ -97988,6 +98035,9 @@ func (s *ModifyEbsDefaultKmsKeyIdOutput) SetKmsKeyId(v string) *ModifyEbsDefault type ModifyFleetInput struct { _ struct{} `type:"structure"` + // Reserved. + Context *string `type:"string"` + // Checks whether you have the required permissions for the action, without // actually making the request, and provides an error response. If you have // the required permissions, the error response is DryRunOperation. Otherwise, @@ -98049,6 +98099,12 @@ func (s *ModifyFleetInput) Validate() error { return nil } +// SetContext sets the Context field's value. +func (s *ModifyFleetInput) SetContext(v string) *ModifyFleetInput { + s.Context = &v + return s +} + // SetDryRun sets the DryRun field's value. func (s *ModifyFleetInput) SetDryRun(v bool) *ModifyFleetInput { s.DryRun = &v @@ -98771,9 +98827,9 @@ type ModifyInstanceAttributeInput struct { SriovNetSupport *AttributeValue `locationName:"sriovNetSupport" type:"structure"` // Changes the instance's user data to the specified value. If you are using - // an AWS SDK or command line tool, base64-encoding is performed for you, and - // you can load the text from a file. Otherwise, you must provide base64-encoded - // text. + // an Amazon Web Services SDK or command line tool, base64-encoding is performed + // for you, and you can load the text from a file. Otherwise, you must provide + // base64-encoded text. UserData *BlobAttributeValue `locationName:"userData" type:"structure"` // A new value for the attribute. Use only with the kernel, ramdisk, userData, @@ -99989,6 +100045,9 @@ func (s ModifySnapshotAttributeOutput) GoString() string { type ModifySpotFleetRequestInput struct { _ struct{} `type:"structure"` + // Reserved. + Context *string `type:"string"` + // Indicates whether running Spot Instances should be terminated if the target // capacity of the Spot Fleet request is decreased below the current size of // the Spot Fleet. @@ -100045,6 +100104,12 @@ func (s *ModifySpotFleetRequestInput) Validate() error { return nil } +// SetContext sets the Context field's value. +func (s *ModifySpotFleetRequestInput) SetContext(v string) *ModifySpotFleetRequestInput { + s.Context = &v + return s +} + // SetExcessCapacityTerminationPolicy sets the ExcessCapacityTerminationPolicy field's value. func (s *ModifySpotFleetRequestInput) SetExcessCapacityTerminationPolicy(v string) *ModifySpotFleetRequestInput { s.ExcessCapacityTerminationPolicy = &v @@ -106713,7 +106778,10 @@ func (s *PurchaseReservedInstancesOfferingInput) SetReservedInstancesOfferingId( type PurchaseReservedInstancesOfferingOutput struct { _ struct{} `type:"structure"` - // The IDs of the purchased Reserved Instances. + // The IDs of the purchased Reserved Instances. If your purchase crosses into + // a discounted pricing tier, the final Reserved Instances IDs might change. + // For more information, see Crossing pricing tiers (https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/concepts-reserved-instances-application.html#crossing-pricing-tiers) + // in the Amazon Elastic Compute Cloud User Guide. ReservedInstancesId *string `locationName:"reservedInstancesId" type:"string"` } @@ -109405,20 +109473,7 @@ type RequestSpotInstancesInput struct { // Default: Instances are launched in any available Availability Zone. AvailabilityZoneGroup *string `locationName:"availabilityZoneGroup" type:"string"` - // The required duration for the Spot Instances (also known as Spot blocks), - // in minutes. This value must be a multiple of 60 (60, 120, 180, 240, 300, - // or 360). - // - // The duration period starts as soon as your Spot Instance receives its instance - // ID. At the end of the duration period, Amazon EC2 marks the Spot Instance - // for termination and provides a Spot Instance termination notice, which gives - // the instance a two-minute warning before it terminates. - // - // You can't specify an Availability Zone group or a launch group if you specify - // a duration. - // - // New accounts or accounts with no previous billing history with AWS are not - // eligible for Spot Instances with a defined duration (also known as Spot blocks). + // Deprecated. BlockDurationMinutes *int64 `locationName:"blockDurationMinutes" type:"integer"` // Unique, case-sensitive identifier that you provide to ensure the idempotency @@ -109816,11 +109871,11 @@ type Reservation struct { // The instances. Instances []*Instance `locationName:"instancesSet" locationNameList:"item" type:"list"` - // The ID of the AWS account that owns the reservation. + // The ID of the account that owns the reservation. OwnerId *string `locationName:"ownerId" type:"string"` // The ID of the requester that launched the instances on your behalf (for example, - // AWS Management Console or Auto Scaling). + // Management Console or Auto Scaling). RequesterId *string `locationName:"requesterId" type:"string"` // The ID of the reservation. @@ -110509,8 +110564,8 @@ type ReservedInstancesOffering struct { InstanceType *string `locationName:"instanceType" type:"string" enum:"InstanceType"` // Indicates whether the offering is available through the Reserved Instance - // Marketplace (resale) or AWS. If it's a Reserved Instance Marketplace offering, - // this is true. + // Marketplace (resale) or Amazon Web Services. If it's a Reserved Instance + // Marketplace offering, this is true. Marketplace *bool `locationName:"marketplace" type:"boolean"` // If convertible it can be exchanged for Reserved Instances of the same or @@ -112486,18 +112541,20 @@ type RunInstancesInput struct { // You cannot specify accelerators from different generations in the same request. ElasticInferenceAccelerators []*ElasticInferenceAccelerator `locationName:"ElasticInferenceAccelerator" locationNameList:"item" type:"list"` - // Indicates whether the instance is enabled for AWS Nitro Enclaves. For more - // information, see What is AWS Nitro Enclaves? (https://docs.aws.amazon.com/enclaves/latest/user/nitro-enclave.html) - // in the AWS Nitro Enclaves User Guide. + // Indicates whether the instance is enabled for Amazon Web Services Nitro Enclaves. + // For more information, see What is Amazon Web Services Nitro Enclaves? (https://docs.aws.amazon.com/enclaves/latest/user/nitro-enclave.html) + // in the Amazon Web Services Nitro Enclaves User Guide. // - // You can't enable AWS Nitro Enclaves and hibernation on the same instance. + // You can't enable Amazon Web Services Nitro Enclaves and hibernation on the + // same instance. EnclaveOptions *EnclaveOptionsRequest `type:"structure"` // Indicates whether an instance is enabled for hibernation. For more information, // see Hibernate your instance (https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/Hibernate.html) // in the Amazon EC2 User Guide. // - // You can't enable hibernation and AWS Nitro Enclaves on the same instance. + // You can't enable hibernation and Amazon Web Services Nitro Enclaves on the + // same instance. HibernationOptions *HibernationOptionsRequest `type:"structure"` // The name or Amazon Resource Name (ARN) of an IAM instance profile. @@ -112619,8 +112676,8 @@ type RunInstancesInput struct { // The ID of the RAM disk to select. Some kernels require additional drivers // at launch. Check the kernel requirements for information about whether you - // need to specify a RAM disk. To find kernel requirements, go to the AWS Resource - // Center and search for the kernel ID. + // need to specify a RAM disk. To find kernel requirements, go to the Amazon + // Web Services Resource Center and search for the kernel ID. // // We recommend that you use PV-GRUB instead of kernels and RAM disks. For more // information, see PV-GRUB (https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/UserProvidedkernels.html) @@ -115809,7 +115866,7 @@ type SpotDatafeedSubscription struct { // The fault codes for the Spot Instance request, if any. Fault *SpotInstanceStateFault `locationName:"fault" type:"structure"` - // The AWS account ID of the account. + // The account ID of the account. OwnerId *string `locationName:"ownerId" type:"string"` // The prefix for the data feed files. @@ -115915,8 +115972,8 @@ type SpotFleetLaunchSpecification struct { // The ID of the RAM disk. Some kernels require additional drivers at launch. // Check the kernel requirements for information about whether you need to specify - // a RAM disk. To find kernel requirements, refer to the AWS Resource Center - // and search for the kernel ID. + // a RAM disk. To find kernel requirements, refer to the Amazon Web Services + // Resource Center and search for the kernel ID. RamdiskId *string `locationName:"ramdiskId" type:"string"` // One or more security groups. When requesting instances in a VPC, you must @@ -116199,6 +116256,9 @@ type SpotFleetRequestConfigData struct { // see Ensuring Idempotency (https://docs.aws.amazon.com/AWSEC2/latest/APIReference/Run_Instance_Idempotency.html). ClientToken *string `locationName:"clientToken" type:"string"` + // Reserved. + Context *string `locationName:"context" type:"string"` + // Indicates whether running Spot Instances should be terminated if you decrease // the target capacity of the Spot Fleet request below the current size of the // Spot Fleet. @@ -116208,7 +116268,7 @@ type SpotFleetRequestConfigData struct { // capacity. You cannot set this value. FulfilledCapacity *float64 `locationName:"fulfilledCapacity" type:"double"` - // The Amazon Resource Name (ARN) of an AWS Identity and Access Management (IAM) + // The Amazon Resource Name (ARN) of an Identity and Access Management (IAM) // role that grants the Spot Fleet the permission to request, launch, terminate, // and tag instances on your behalf. For more information, see Spot Fleet prerequisites // (https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/spot-fleet-requests.html#spot-fleet-prerequisites) @@ -116227,6 +116287,15 @@ type SpotFleetRequestConfigData struct { // Valid only when Spot AllocationStrategy is set to lowest-price. Spot Fleet // selects the cheapest Spot pools and evenly allocates your target Spot capacity // across the number of Spot pools that you specify. + // + // Note that Spot Fleet attempts to draw Spot Instances from the number of pools + // that you specify on a best effort basis. If a pool runs out of Spot capacity + // before fulfilling your target capacity, Spot Fleet will continue to fulfill + // your request by drawing from the next cheapest pool. To ensure that your + // target capacity is met, you might receive Spot Instances from more than the + // number of pools that you specified. Similarly, if most of the pools have + // no Spot capacity, you might receive your full target capacity from fewer + // than the number of pools that you specified. InstancePoolsToUseCount *int64 `locationName:"instancePoolsToUseCount" type:"integer"` // The launch specifications for the Spot Fleet request. If you specify LaunchSpecifications, @@ -116395,6 +116464,12 @@ func (s *SpotFleetRequestConfigData) SetClientToken(v string) *SpotFleetRequestC return s } +// SetContext sets the Context field's value. +func (s *SpotFleetRequestConfigData) SetContext(v string) *SpotFleetRequestConfigData { + s.Context = &v + return s +} + // SetExcessCapacityTerminationPolicy sets the ExcessCapacityTerminationPolicy field's value. func (s *SpotFleetRequestConfigData) SetExcessCapacityTerminationPolicy(v string) *SpotFleetRequestConfigData { s.ExcessCapacityTerminationPolicy = &v @@ -116566,8 +116641,7 @@ func (s *SpotFleetTagSpecification) SetTags(v []*Tag) *SpotFleetTagSpecification type SpotInstanceRequest struct { _ struct{} `type:"structure"` - // If you specified a duration and your Spot Instance request was fulfilled, - // this is the fixed hourly price in effect for the Spot Instance while it runs. + // Deprecated. ActualBlockHourlyPrice *string `locationName:"actualBlockHourlyPrice" type:"string"` // The Availability Zone group. If you specify the same Availability Zone group @@ -116575,7 +116649,7 @@ type SpotInstanceRequest struct { // Availability Zone. AvailabilityZoneGroup *string `locationName:"availabilityZoneGroup" type:"string"` - // The duration for the Spot Instance, in minutes. + // Deprecated. BlockDurationMinutes *int64 `locationName:"blockDurationMinutes" type:"integer"` // The date and time when the Spot Instance request was created, in UTC format @@ -116885,8 +116959,9 @@ type SpotMarketOptions struct { // You can't specify an Availability Zone group or a launch group if you specify // a duration. // - // New accounts or accounts with no previous billing history with AWS are not - // eligible for Spot Instances with a defined duration (also known as Spot blocks). + // New accounts or accounts with no previous billing history with Amazon Web + // Services are not eligible for Spot Instances with a defined duration (also + // known as Spot blocks). BlockDurationMinutes *int64 `type:"integer"` // The behavior when a Spot Instance is interrupted. The default is terminate. @@ -116986,6 +117061,15 @@ type SpotOptions struct { // Valid only when AllocationStrategy is set to lowest-price. EC2 Fleet selects // the cheapest Spot pools and evenly allocates your target Spot capacity across // the number of Spot pools that you specify. + // + // Note that EC2 Fleet attempts to draw Spot Instances from the number of pools + // that you specify on a best effort basis. If a pool runs out of Spot capacity + // before fulfilling your target capacity, EC2 Fleet will continue to fulfill + // your request by drawing from the next cheapest pool. To ensure that your + // target capacity is met, you might receive Spot Instances from more than the + // number of pools that you specified. Similarly, if most of the pools have + // no Spot capacity, you might receive your full target capacity from fewer + // than the number of pools that you specified. InstancePoolsToUseCount *int64 `locationName:"instancePoolsToUseCount" type:"integer"` // The strategies for managing your workloads on your Spot Instances that will @@ -117099,6 +117183,15 @@ type SpotOptionsRequest struct { // Valid only when Spot AllocationStrategy is set to lowest-price. EC2 Fleet // selects the cheapest Spot pools and evenly allocates your target Spot capacity // across the number of Spot pools that you specify. + // + // Note that EC2 Fleet attempts to draw Spot Instances from the number of pools + // that you specify on a best effort basis. If a pool runs out of Spot capacity + // before fulfilling your target capacity, EC2 Fleet will continue to fulfill + // your request by drawing from the next cheapest pool. To ensure that your + // target capacity is met, you might receive Spot Instances from more than the + // number of pools that you specified. Similarly, if most of the pools have + // no Spot capacity, you might receive your full target capacity from fewer + // than the number of pools that you specified. InstancePoolsToUseCount *int64 `type:"integer"` // The strategies for managing your Spot Instances that are at an elevated risk @@ -130001,6 +130094,9 @@ const ( // ResourceTypeNetworkInsightsAnalysis is a ResourceType enum value ResourceTypeNetworkInsightsAnalysis = "network-insights-analysis" + // ResourceTypeNetworkInsightsBoundary is a ResourceType enum value + ResourceTypeNetworkInsightsBoundary = "network-insights-boundary" + // ResourceTypeNetworkInsightsPath is a ResourceType enum value ResourceTypeNetworkInsightsPath = "network-insights-path" @@ -130098,6 +130194,7 @@ func ResourceType_Values() []string { ResourceTypeNetworkAcl, ResourceTypeNetworkInterface, ResourceTypeNetworkInsightsAnalysis, + ResourceTypeNetworkInsightsBoundary, ResourceTypeNetworkInsightsPath, ResourceTypePlacementGroup, ResourceTypeReservedInstances, diff --git a/vendor/github.com/aws/aws-sdk-go/service/sqs/api.go b/vendor/github.com/aws/aws-sdk-go/service/sqs/api.go index d58d7c034..a07d209e0 100644 --- a/vendor/github.com/aws/aws-sdk-go/service/sqs/api.go +++ b/vendor/github.com/aws/aws-sdk-go/service/sqs/api.go @@ -64,12 +64,12 @@ func (c *SQS) AddPermissionRequest(input *AddPermissionInput) (req *request.Requ // Only you, the owner of the queue, can grant or deny permissions to the queue. // For more information about these permissions, see Allow Developers to Write // Messages to a Shared Queue (https://docs.aws.amazon.com/AWSSimpleQueueService/latest/SQSDeveloperGuide/sqs-writing-an-sqs-policy.html#write-messages-to-shared-queue) -// in the Amazon Simple Queue Service Developer Guide. +// in the Amazon SQS Developer Guide. // // * AddPermission generates a policy for you. You can use SetQueueAttributes // to upload your policy. For more information, see Using Custom Policies // with the Amazon SQS Access Policy Language (https://docs.aws.amazon.com/AWSSimpleQueueService/latest/SQSDeveloperGuide/sqs-creating-custom-policies.html) -// in the Amazon Simple Queue Service Developer Guide. +// in the Amazon SQS Developer Guide. // // * An Amazon SQS policy can have a maximum of 7 actions. // @@ -87,7 +87,7 @@ func (c *SQS) AddPermissionRequest(input *AddPermissionInput) (req *request.Requ // // Cross-account permissions don't apply to this action. For more information, // see Grant cross-account permissions to a role and a user name (https://docs.aws.amazon.com/AWSSimpleQueueService/latest/SQSDeveloperGuide/sqs-customer-managed-policy-examples.html#grant-cross-account-permissions-to-role-and-user-name) -// in the Amazon Simple Queue Service Developer Guide. +// in the Amazon SQS Developer Guide. // // Returns awserr.Error for service API and SDK errors. Use runtime type assertions // with awserr.Error's Code and Message methods to get detailed information about @@ -174,7 +174,7 @@ func (c *SQS) ChangeMessageVisibilityRequest(input *ChangeMessageVisibilityInput // value. The default visibility timeout for a message is 30 seconds. The minimum // is 0 seconds. The maximum is 12 hours. For more information, see Visibility // Timeout (https://docs.aws.amazon.com/AWSSimpleQueueService/latest/SQSDeveloperGuide/sqs-visibility-timeout.html) -// in the Amazon Simple Queue Service Developer Guide. +// in the Amazon SQS Developer Guide. // // For example, you have a message with a visibility timeout of 5 minutes. After // 3 minutes, you call ChangeMessageVisibility with a timeout of 10 minutes. @@ -415,7 +415,7 @@ func (c *SQS) CreateQueueRequest(input *CreateQueueInput) (req *request.Request, // create a new FIFO queue for your application or delete your existing standard // queue and recreate it as a FIFO queue. For more information, see Moving // From a Standard Queue to a FIFO Queue (https://docs.aws.amazon.com/AWSSimpleQueueService/latest/SQSDeveloperGuide/FIFO-queues.html#FIFO-queues-moving) -// in the Amazon Simple Queue Service Developer Guide. +// in the Amazon SQS Developer Guide. // // * If you don't provide a value for an attribute, the queue is created // with the default value for the attribute. @@ -450,7 +450,7 @@ func (c *SQS) CreateQueueRequest(input *CreateQueueInput) (req *request.Request, // // Cross-account permissions don't apply to this action. For more information, // see Grant cross-account permissions to a role and a user name (https://docs.aws.amazon.com/AWSSimpleQueueService/latest/SQSDeveloperGuide/sqs-customer-managed-policy-examples.html#grant-cross-account-permissions-to-role-and-user-name) -// in the Amazon Simple Queue Service Developer Guide. +// in the Amazon SQS Developer Guide. // // Returns awserr.Error for service API and SDK errors. Use runtime type assertions // with awserr.Error's Code and Message methods to get detailed information about @@ -755,7 +755,7 @@ func (c *SQS) DeleteQueueRequest(input *DeleteQueueInput) (req *request.Request, // // Cross-account permissions don't apply to this action. For more information, // see Grant cross-account permissions to a role and a user name (https://docs.aws.amazon.com/AWSSimpleQueueService/latest/SQSDeveloperGuide/sqs-customer-managed-policy-examples.html#grant-cross-account-permissions-to-role-and-user-name) -// in the Amazon Simple Queue Service Developer Guide. +// in the Amazon SQS Developer Guide. // // Returns awserr.Error for service API and SDK errors. Use runtime type assertions // with awserr.Error's Code and Message methods to get detailed information about @@ -918,7 +918,7 @@ func (c *SQS) GetQueueUrlRequest(input *GetQueueUrlInput) (req *request.Request, // must grant you permission to access the queue. For more information about // shared queue access, see AddPermission or see Allow Developers to Write Messages // to a Shared Queue (https://docs.aws.amazon.com/AWSSimpleQueueService/latest/SQSDeveloperGuide/sqs-writing-an-sqs-policy.html#write-messages-to-shared-queue) -// in the Amazon Simple Queue Service Developer Guide. +// in the Amazon SQS Developer Guide. // // Returns awserr.Error for service API and SDK errors. Use runtime type assertions // with awserr.Error's Code and Message methods to get detailed information about @@ -1016,7 +1016,7 @@ func (c *SQS) ListDeadLetterSourceQueuesRequest(input *ListDeadLetterSourceQueue // // For more information about using dead-letter queues, see Using Amazon SQS // Dead-Letter Queues (https://docs.aws.amazon.com/AWSSimpleQueueService/latest/SQSDeveloperGuide/sqs-dead-letter-queues.html) -// in the Amazon Simple Queue Service Developer Guide. +// in the Amazon SQS Developer Guide. // // Returns awserr.Error for service API and SDK errors. Use runtime type assertions // with awserr.Error's Code and Message methods to get detailed information about @@ -1149,11 +1149,11 @@ func (c *SQS) ListQueueTagsRequest(input *ListQueueTagsInput) (req *request.Requ // // List all cost allocation tags added to the specified Amazon SQS queue. For // an overview, see Tagging Your Amazon SQS Queues (https://docs.aws.amazon.com/AWSSimpleQueueService/latest/SQSDeveloperGuide/sqs-queue-tags.html) -// in the Amazon Simple Queue Service Developer Guide. +// in the Amazon SQS Developer Guide. // // Cross-account permissions don't apply to this action. For more information, // see Grant cross-account permissions to a role and a user name (https://docs.aws.amazon.com/AWSSimpleQueueService/latest/SQSDeveloperGuide/sqs-customer-managed-policy-examples.html#grant-cross-account-permissions-to-role-and-user-name) -// in the Amazon Simple Queue Service Developer Guide. +// in the Amazon SQS Developer Guide. // // Returns awserr.Error for service API and SDK errors. Use runtime type assertions // with awserr.Error's Code and Message methods to get detailed information about @@ -1247,7 +1247,7 @@ func (c *SQS) ListQueuesRequest(input *ListQueuesInput) (req *request.Request, o // // Cross-account permissions don't apply to this action. For more information, // see Grant cross-account permissions to a role and a user name (https://docs.aws.amazon.com/AWSSimpleQueueService/latest/SQSDeveloperGuide/sqs-customer-managed-policy-examples.html#grant-cross-account-permissions-to-role-and-user-name) -// in the Amazon Simple Queue Service Developer Guide. +// in the Amazon SQS Developer Guide. // // Returns awserr.Error for service API and SDK errors. Use runtime type assertions // with awserr.Error's Code and Message methods to get detailed information about @@ -1473,7 +1473,7 @@ func (c *SQS) ReceiveMessageRequest(input *ReceiveMessageInput) (req *request.Re // Retrieves one or more messages (up to 10), from the specified queue. Using // the WaitTimeSeconds parameter enables long-poll support. For more information, // see Amazon SQS Long Polling (https://docs.aws.amazon.com/AWSSimpleQueueService/latest/SQSDeveloperGuide/sqs-long-polling.html) -// in the Amazon Simple Queue Service Developer Guide. +// in the Amazon SQS Developer Guide. // // Short poll is the default behavior where a weighted random set of machines // is sampled on a ReceiveMessage call. Thus, only the messages on the sampled @@ -1500,14 +1500,14 @@ func (c *SQS) ReceiveMessageRequest(input *ReceiveMessageInput) (req *request.Re // // The receipt handle is the identifier you must provide when deleting the message. // For more information, see Queue and Message Identifiers (https://docs.aws.amazon.com/AWSSimpleQueueService/latest/SQSDeveloperGuide/sqs-queue-message-identifiers.html) -// in the Amazon Simple Queue Service Developer Guide. +// in the Amazon SQS Developer Guide. // // You can provide the VisibilityTimeout parameter in your request. The parameter // is applied to the messages that Amazon SQS returns in the response. If you // don't include the parameter, the overall visibility timeout for the queue // is used for the returned messages. For more information, see Visibility Timeout // (https://docs.aws.amazon.com/AWSSimpleQueueService/latest/SQSDeveloperGuide/sqs-visibility-timeout.html) -// in the Amazon Simple Queue Service Developer Guide. +// in the Amazon SQS Developer Guide. // // A message that isn't deleted or a message whose visibility isn't extended // before the visibility timeout expires counts as a failed receive. Depending @@ -1606,7 +1606,7 @@ func (c *SQS) RemovePermissionRequest(input *RemovePermissionInput) (req *reques // // * Cross-account permissions don't apply to this action. For more information, // see Grant cross-account permissions to a role and a user name (https://docs.aws.amazon.com/AWSSimpleQueueService/latest/SQSDeveloperGuide/sqs-customer-managed-policy-examples.html#grant-cross-account-permissions-to-role-and-user-name) -// in the Amazon Simple Queue Service Developer Guide. +// in the Amazon SQS Developer Guide. // // * To remove the ability to change queue permissions, you must deny permission // to the AddPermission, RemovePermission, and SetQueueAttributes actions @@ -1910,7 +1910,7 @@ func (c *SQS) SetQueueAttributesRequest(input *SetQueueAttributesInput) (req *re // // * Cross-account permissions don't apply to this action. For more information, // see Grant cross-account permissions to a role and a user name (https://docs.aws.amazon.com/AWSSimpleQueueService/latest/SQSDeveloperGuide/sqs-customer-managed-policy-examples.html#grant-cross-account-permissions-to-role-and-user-name) -// in the Amazon Simple Queue Service Developer Guide. +// in the Amazon SQS Developer Guide. // // * To remove the ability to change queue permissions, you must deny permission // to the AddPermission, RemovePermission, and SetQueueAttributes actions @@ -1996,7 +1996,7 @@ func (c *SQS) TagQueueRequest(input *TagQueueInput) (req *request.Request, outpu // // Add cost allocation tags to the specified Amazon SQS queue. For an overview, // see Tagging Your Amazon SQS Queues (https://docs.aws.amazon.com/AWSSimpleQueueService/latest/SQSDeveloperGuide/sqs-queue-tags.html) -// in the Amazon Simple Queue Service Developer Guide. +// in the Amazon SQS Developer Guide. // // When you use queue tags, keep the following guidelines in mind: // @@ -2010,12 +2010,12 @@ func (c *SQS) TagQueueRequest(input *TagQueueInput) (req *request.Request, outpu // * A new tag with a key identical to that of an existing tag overwrites // the existing tag. // -// For a full list of tag restrictions, see Limits Related to Queues (https://docs.aws.amazon.com/AWSSimpleQueueService/latest/SQSDeveloperGuide/sqs-limits.html#limits-queues) -// in the Amazon Simple Queue Service Developer Guide. +// For a full list of tag restrictions, see Quotas related to queues (https://docs.aws.amazon.com/AWSSimpleQueueService/latest/SQSDeveloperGuide/sqs-limits.html#limits-queues) +// in the Amazon SQS Developer Guide. // // Cross-account permissions don't apply to this action. For more information, // see Grant cross-account permissions to a role and a user name (https://docs.aws.amazon.com/AWSSimpleQueueService/latest/SQSDeveloperGuide/sqs-customer-managed-policy-examples.html#grant-cross-account-permissions-to-role-and-user-name) -// in the Amazon Simple Queue Service Developer Guide. +// in the Amazon SQS Developer Guide. // // Returns awserr.Error for service API and SDK errors. Use runtime type assertions // with awserr.Error's Code and Message methods to get detailed information about @@ -2092,11 +2092,11 @@ func (c *SQS) UntagQueueRequest(input *UntagQueueInput) (req *request.Request, o // // Remove cost allocation tags from the specified Amazon SQS queue. For an overview, // see Tagging Your Amazon SQS Queues (https://docs.aws.amazon.com/AWSSimpleQueueService/latest/SQSDeveloperGuide/sqs-queue-tags.html) -// in the Amazon Simple Queue Service Developer Guide. +// in the Amazon SQS Developer Guide. // // Cross-account permissions don't apply to this action. For more information, // see Grant cross-account permissions to a role and a user name (https://docs.aws.amazon.com/AWSSimpleQueueService/latest/SQSDeveloperGuide/sqs-customer-managed-policy-examples.html#grant-cross-account-permissions-to-role-and-user-name) -// in the Amazon Simple Queue Service Developer Guide. +// in the Amazon SQS Developer Guide. // // Returns awserr.Error for service API and SDK errors. Use runtime type assertions // with awserr.Error's Code and Message methods to get detailed information about @@ -2129,11 +2129,10 @@ func (c *SQS) UntagQueueWithContext(ctx aws.Context, input *UntagQueueInput, opt type AddPermissionInput struct { _ struct{} `type:"structure"` - // The AWS account number of the principal (https://docs.aws.amazon.com/general/latest/gr/glos-chap.html#P) - // who is given permission. The principal must have an AWS account, but does - // not need to be signed up for Amazon SQS. For information about locating the - // AWS account identification, see Your AWS Identifiers (https://docs.aws.amazon.com/AWSSimpleQueueService/latest/SQSDeveloperGuide/sqs-making-api-requests.html#sqs-api-request-authentication) - // in the Amazon Simple Queue Service Developer Guide. + // The account numbers of the principals (https://docs.aws.amazon.com/general/latest/gr/glos-chap.html#P) + // who are to receive permission. For information about locating the account + // identification, see Your Amazon Web Services Identifiers (https://docs.aws.amazon.com/AWSSimpleQueueService/latest/SQSDeveloperGuide/sqs-making-api-requests.html#sqs-api-request-authentication) + // in the Amazon SQS Developer Guide. // // AWSAccountIds is a required field AWSAccountIds []*string `locationNameList:"AWSAccountId" type:"list" flattened:"true" required:"true"` @@ -2143,7 +2142,7 @@ type AddPermissionInput struct { // // For more information about these actions, see Overview of Managing Access // Permissions to Your Amazon Simple Queue Service Resource (https://docs.aws.amazon.com/AWSSimpleQueueService/latest/SQSDeveloperGuide/sqs-overview-of-managing-access.html) - // in the Amazon Simple Queue Service Developer Guide. + // in the Amazon SQS Developer Guide. // // Specifying SendMessage, DeleteMessage, or ChangeMessageVisibility for ActionName.n // also grants permissions for the corresponding batch versions of those actions: @@ -2607,8 +2606,9 @@ type CreateQueueInput struct { // Amazon SQS retains a message. Valid values: An integer from 60 seconds // (1 minute) to 1,209,600 seconds (14 days). Default: 345,600 (4 days). // - // * Policy – The queue's policy. A valid AWS policy. For more information - // about policy structure, see Overview of AWS IAM Policies (https://docs.aws.amazon.com/IAM/latest/UserGuide/PoliciesOverview.html) + // * Policy – The queue's policy. A valid Amazon Web Services policy. For + // more information about policy structure, see Overview of Amazon Web Services + // IAM Policies (https://docs.aws.amazon.com/IAM/latest/UserGuide/PoliciesOverview.html) // in the Amazon IAM User Guide. // // * ReceiveMessageWaitTimeSeconds – The length of time, in seconds, for @@ -2619,9 +2619,9 @@ type CreateQueueInput struct { // queue functionality of the source queue as a JSON object. For more information // about the redrive policy and dead-letter queues, see Using Amazon SQS // Dead-Letter Queues (https://docs.aws.amazon.com/AWSSimpleQueueService/latest/SQSDeveloperGuide/sqs-dead-letter-queues.html) - // in the Amazon Simple Queue Service Developer Guide. deadLetterTargetArn - // – The Amazon Resource Name (ARN) of the dead-letter queue to which Amazon - // SQS moves messages after the value of maxReceiveCount is exceeded. maxReceiveCount + // in the Amazon SQS Developer Guide. deadLetterTargetArn – The Amazon + // Resource Name (ARN) of the dead-letter queue to which Amazon SQS moves + // messages after the value of maxReceiveCount is exceeded. maxReceiveCount // – The number of times a message is delivered to the source queue before // being moved to the dead-letter queue. When the ReceiveCount for a message // exceeds the maxReceiveCount for a queue, Amazon SQS moves the message @@ -2633,25 +2633,26 @@ type CreateQueueInput struct { // Valid values: An integer from 0 to 43,200 (12 hours). Default: 30. For // more information about the visibility timeout, see Visibility Timeout // (https://docs.aws.amazon.com/AWSSimpleQueueService/latest/SQSDeveloperGuide/sqs-visibility-timeout.html) - // in the Amazon Simple Queue Service Developer Guide. + // in the Amazon SQS Developer Guide. // // The following attributes apply only to server-side-encryption (https://docs.aws.amazon.com/AWSSimpleQueueService/latest/SQSDeveloperGuide/sqs-server-side-encryption.html): // - // * KmsMasterKeyId – The ID of an AWS-managed customer master key (CMK) - // for Amazon SQS or a custom CMK. For more information, see Key Terms (https://docs.aws.amazon.com/AWSSimpleQueueService/latest/SQSDeveloperGuide/sqs-server-side-encryption.html#sqs-sse-key-terms). - // While the alias of the AWS-managed CMK for Amazon SQS is always alias/aws/sqs, - // the alias of a custom CMK can, for example, be alias/MyAlias . For more - // examples, see KeyId (https://docs.aws.amazon.com/kms/latest/APIReference/API_DescribeKey.html#API_DescribeKey_RequestParameters) - // in the AWS Key Management Service API Reference. + // * KmsMasterKeyId – The ID of an Amazon Web Services managed customer + // master key (CMK) for Amazon SQS or a custom CMK. For more information, + // see Key Terms (https://docs.aws.amazon.com/AWSSimpleQueueService/latest/SQSDeveloperGuide/sqs-server-side-encryption.html#sqs-sse-key-terms). + // While the alias of the Amazon Web Services managed CMK for Amazon SQS + // is always alias/aws/sqs, the alias of a custom CMK can, for example, be + // alias/MyAlias . For more examples, see KeyId (https://docs.aws.amazon.com/kms/latest/APIReference/API_DescribeKey.html#API_DescribeKey_RequestParameters) + // in the Key Management Service API Reference. // // * KmsDataKeyReusePeriodSeconds – The length of time, in seconds, for // which Amazon SQS can reuse a data key (https://docs.aws.amazon.com/kms/latest/developerguide/concepts.html#data-keys) - // to encrypt or decrypt messages before calling AWS KMS again. An integer - // representing seconds, between 60 seconds (1 minute) and 86,400 seconds - // (24 hours). Default: 300 (5 minutes). A shorter time period provides better - // security but results in more calls to KMS which might incur charges after - // Free Tier. For more information, see How Does the Data Key Reuse Period - // Work? (https://docs.aws.amazon.com/AWSSimpleQueueService/latest/SQSDeveloperGuide/sqs-server-side-encryption.html#sqs-how-does-the-data-key-reuse-period-work). + // to encrypt or decrypt messages before calling KMS again. An integer representing + // seconds, between 60 seconds (1 minute) and 86,400 seconds (24 hours). + // Default: 300 (5 minutes). A shorter time period provides better security + // but results in more calls to KMS which might incur charges after Free + // Tier. For more information, see How Does the Data Key Reuse Period Work? + // (https://docs.aws.amazon.com/AWSSimpleQueueService/latest/SQSDeveloperGuide/sqs-server-side-encryption.html#sqs-how-does-the-data-key-reuse-period-work). // // The following attributes apply only to FIFO (first-in-first-out) queues (https://docs.aws.amazon.com/AWSSimpleQueueService/latest/SQSDeveloperGuide/FIFO-queues.html): // @@ -2661,16 +2662,16 @@ type CreateQueueInput struct { // You can't change it for an existing queue. When you set this attribute, // you must also provide the MessageGroupId for your messages explicitly. // For more information, see FIFO queue logic (https://docs.aws.amazon.com/AWSSimpleQueueService/latest/SQSDeveloperGuide/FIFO-queues-understanding-logic.html) - // in the Amazon Simple Queue Service Developer Guide. + // in the Amazon SQS Developer Guide. // // * ContentBasedDeduplication – Enables content-based deduplication. Valid // values are true and false. For more information, see Exactly-once processing // (https://docs.aws.amazon.com/AWSSimpleQueueService/latest/SQSDeveloperGuide/FIFO-queues-exactly-once-processing.html) - // in the Amazon Simple Queue Service Developer Guide. Note the following: - // Every message must have a unique MessageDeduplicationId. You may provide - // a MessageDeduplicationId explicitly. If you aren't able to provide a MessageDeduplicationId - // and you enable ContentBasedDeduplication for your queue, Amazon SQS uses - // a SHA-256 hash to generate the MessageDeduplicationId using the body of + // in the Amazon SQS Developer Guide. Note the following: Every message must + // have a unique MessageDeduplicationId. You may provide a MessageDeduplicationId + // explicitly. If you aren't able to provide a MessageDeduplicationId and + // you enable ContentBasedDeduplication for your queue, Amazon SQS uses a + // SHA-256 hash to generate the MessageDeduplicationId using the body of // the message (but not the attributes of the message). If you don't provide // a MessageDeduplicationId and the queue doesn't have ContentBasedDeduplication // set, the action fails with an error. If the queue has ContentBasedDeduplication @@ -2704,7 +2705,7 @@ type CreateQueueInput struct { // as specified. // // For information on throughput quotas, see Quotas related to messages (https://docs.aws.amazon.com/AWSSimpleQueueService/latest/SQSDeveloperGuide/quotas-messages.html) - // in the Amazon Simple Queue Service Developer Guide. + // in the Amazon SQS Developer Guide. Attributes map[string]*string `locationName:"Attribute" locationNameKey:"Name" locationNameValue:"Value" type:"map" flattened:"true"` // The name of the new queue. The following limits apply to this name: @@ -2723,7 +2724,7 @@ type CreateQueueInput struct { // Add cost allocation tags to the specified Amazon SQS queue. For an overview, // see Tagging Your Amazon SQS Queues (https://docs.aws.amazon.com/AWSSimpleQueueService/latest/SQSDeveloperGuide/sqs-queue-tags.html) - // in the Amazon Simple Queue Service Developer Guide. + // in the Amazon SQS Developer Guide. // // When you use queue tags, keep the following guidelines in mind: // @@ -2737,15 +2738,15 @@ type CreateQueueInput struct { // * A new tag with a key identical to that of an existing tag overwrites // the existing tag. // - // For a full list of tag restrictions, see Limits Related to Queues (https://docs.aws.amazon.com/AWSSimpleQueueService/latest/SQSDeveloperGuide/sqs-limits.html#limits-queues) - // in the Amazon Simple Queue Service Developer Guide. + // For a full list of tag restrictions, see Quotas related to queues (https://docs.aws.amazon.com/AWSSimpleQueueService/latest/SQSDeveloperGuide/sqs-limits.html#limits-queues) + // in the Amazon SQS Developer Guide. // // To be able to tag a queue on creation, you must have the sqs:CreateQueue // and sqs:TagQueue permissions. // // Cross-account permissions don't apply to this action. For more information, // see Grant cross-account permissions to a role and a user name (https://docs.aws.amazon.com/AWSSimpleQueueService/latest/SQSDeveloperGuide/sqs-customer-managed-policy-examples.html#grant-cross-account-permissions-to-role-and-user-name) - // in the Amazon Simple Queue Service Developer Guide. + // in the Amazon SQS Developer Guide. Tags map[string]*string `locationName:"Tag" locationNameKey:"Key" locationNameValue:"Value" type:"map" flattened:"true"` } @@ -3129,6 +3130,9 @@ type GetQueueAttributesInput struct { // A list of attributes for which to retrieve information. // + // The AttributeName.N parameter is optional, but if you don't specify values + // for this parameter, the request returns empty results. + // // In the future, new attributes might be added. If you write code that calls // this action, we recommend that you structure your code so that it can handle // new attributes gracefully. @@ -3180,9 +3184,9 @@ type GetQueueAttributesInput struct { // queue functionality of the source queue as a JSON object. For more information // about the redrive policy and dead-letter queues, see Using Amazon SQS // Dead-Letter Queues (https://docs.aws.amazon.com/AWSSimpleQueueService/latest/SQSDeveloperGuide/sqs-dead-letter-queues.html) - // in the Amazon Simple Queue Service Developer Guide. deadLetterTargetArn - // – The Amazon Resource Name (ARN) of the dead-letter queue to which Amazon - // SQS moves messages after the value of maxReceiveCount is exceeded. maxReceiveCount + // in the Amazon SQS Developer Guide. deadLetterTargetArn – The Amazon + // Resource Name (ARN) of the dead-letter queue to which Amazon SQS moves + // messages after the value of maxReceiveCount is exceeded. maxReceiveCount // – The number of times a message is delivered to the source queue before // being moved to the dead-letter queue. When the ReceiveCount for a message // exceeds the maxReceiveCount for a queue, Amazon SQS moves the message @@ -3191,31 +3195,31 @@ type GetQueueAttributesInput struct { // * VisibilityTimeout – Returns the visibility timeout for the queue. // For more information about the visibility timeout, see Visibility Timeout // (https://docs.aws.amazon.com/AWSSimpleQueueService/latest/SQSDeveloperGuide/sqs-visibility-timeout.html) - // in the Amazon Simple Queue Service Developer Guide. + // in the Amazon SQS Developer Guide. // // The following attributes apply only to server-side-encryption (https://docs.aws.amazon.com/AWSSimpleQueueService/latest/SQSDeveloperGuide/sqs-server-side-encryption.html): // - // * KmsMasterKeyId – Returns the ID of an AWS-managed customer master - // key (CMK) for Amazon SQS or a custom CMK. For more information, see Key - // Terms (https://docs.aws.amazon.com/AWSSimpleQueueService/latest/SQSDeveloperGuide/sqs-server-side-encryption.html#sqs-sse-key-terms). + // * KmsMasterKeyId – Returns the ID of an Amazon Web Services managed + // customer master key (CMK) for Amazon SQS or a custom CMK. For more information, + // see Key Terms (https://docs.aws.amazon.com/AWSSimpleQueueService/latest/SQSDeveloperGuide/sqs-server-side-encryption.html#sqs-sse-key-terms). // // * KmsDataKeyReusePeriodSeconds – Returns the length of time, in seconds, // for which Amazon SQS can reuse a data key to encrypt or decrypt messages - // before calling AWS KMS again. For more information, see How Does the Data + // before calling KMS again. For more information, see How Does the Data // Key Reuse Period Work? (https://docs.aws.amazon.com/AWSSimpleQueueService/latest/SQSDeveloperGuide/sqs-server-side-encryption.html#sqs-how-does-the-data-key-reuse-period-work). // // The following attributes apply only to FIFO (first-in-first-out) queues (https://docs.aws.amazon.com/AWSSimpleQueueService/latest/SQSDeveloperGuide/FIFO-queues.html): // // * FifoQueue – Returns information about whether the queue is FIFO. For // more information, see FIFO queue logic (https://docs.aws.amazon.com/AWSSimpleQueueService/latest/SQSDeveloperGuide/FIFO-queues-understanding-logic.html) - // in the Amazon Simple Queue Service Developer Guide. To determine whether - // a queue is FIFO (https://docs.aws.amazon.com/AWSSimpleQueueService/latest/SQSDeveloperGuide/FIFO-queues.html), + // in the Amazon SQS Developer Guide. To determine whether a queue is FIFO + // (https://docs.aws.amazon.com/AWSSimpleQueueService/latest/SQSDeveloperGuide/FIFO-queues.html), // you can check whether QueueName ends with the .fifo suffix. // // * ContentBasedDeduplication – Returns whether content-based deduplication // is enabled for the queue. For more information, see Exactly-once processing // (https://docs.aws.amazon.com/AWSSimpleQueueService/latest/SQSDeveloperGuide/FIFO-queues-exactly-once-processing.html) - // in the Amazon Simple Queue Service Developer Guide. + // in the Amazon SQS Developer Guide. // // The following attributes apply only to high throughput for FIFO queues (https://docs.aws.amazon.com/AWSSimpleQueueService/latest/SQSDeveloperGuide/high-throughput-fifo.html): // @@ -3239,7 +3243,7 @@ type GetQueueAttributesInput struct { // as specified. // // For information on throughput quotas, see Quotas related to messages (https://docs.aws.amazon.com/AWSSimpleQueueService/latest/SQSDeveloperGuide/quotas-messages.html) - // in the Amazon Simple Queue Service Developer Guide. + // in the Amazon SQS Developer Guide. AttributeNames []*string `locationNameList:"AttributeName" type:"list" flattened:"true"` // The URL of the Amazon SQS queue whose attribute information is retrieved. @@ -3320,7 +3324,7 @@ type GetQueueUrlInput struct { // QueueName is a required field QueueName *string `type:"string" required:"true"` - // The AWS account ID of the account that created the queue. + // The account ID of the account that created the queue. QueueOwnerAWSAccountId *string `type:"string"` } @@ -3360,7 +3364,7 @@ func (s *GetQueueUrlInput) SetQueueOwnerAWSAccountId(v string) *GetQueueUrlInput } // For more information, see Interpreting Responses (https://docs.aws.amazon.com/AWSSimpleQueueService/latest/SQSDeveloperGuide/sqs-api-responses.html) -// in the Amazon Simple Queue Service Developer Guide. +// in the Amazon SQS Developer Guide. type GetQueueUrlOutput struct { _ struct{} `type:"structure"` @@ -3662,12 +3666,12 @@ type Message struct { MD5OfMessageAttributes *string `type:"string"` // Each message attribute consists of a Name, Type, and Value. For more information, - // see Amazon SQS Message Attributes (https://docs.aws.amazon.com/AWSSimpleQueueService/latest/SQSDeveloperGuide/sqs-message-metadata.html#sqs-message-attributes) - // in the Amazon Simple Queue Service Developer Guide. + // see Amazon SQS message attributes (https://docs.aws.amazon.com/AWSSimpleQueueService/latest/SQSDeveloperGuide/sqs-message-metadata.html#sqs-message-attributes) + // in the Amazon SQS Developer Guide. MessageAttributes map[string]*MessageAttributeValue `locationName:"MessageAttribute" locationNameKey:"Name" locationNameValue:"Value" type:"map" flattened:"true"` // A unique identifier for the message. A MessageIdis considered unique across - // all AWS accounts for an extended period of time. + // all accounts for an extended period of time. MessageId *string `type:"string"` // An identifier associated with the act of receiving the message. A new receipt @@ -3752,7 +3756,7 @@ type MessageAttributeValue struct { // // You can also append custom labels. For more information, see Amazon SQS Message // Attributes (https://docs.aws.amazon.com/AWSSimpleQueueService/latest/SQSDeveloperGuide/sqs-message-metadata.html#sqs-message-attributes) - // in the Amazon Simple Queue Service Developer Guide. + // in the Amazon SQS Developer Guide. // // DataType is a required field DataType *string `type:"string" required:"true"` @@ -3840,7 +3844,7 @@ type MessageSystemAttributeValue struct { // // You can also append custom labels. For more information, see Amazon SQS Message // Attributes (https://docs.aws.amazon.com/AWSSimpleQueueService/latest/SQSDeveloperGuide/sqs-message-metadata.html#sqs-message-attributes) - // in the Amazon Simple Queue Service Developer Guide. + // in the Amazon SQS Developer Guide. // // DataType is a required field DataType *string `type:"string" required:"true"` @@ -3975,7 +3979,7 @@ type ReceiveMessageInput struct { // * ApproximateReceiveCount – Returns the number of times a message has // been received across all queues but not deleted. // - // * AWSTraceHeader – Returns the AWS X-Ray trace header string. + // * AWSTraceHeader – Returns the X-Ray trace header string. // // * SenderId For an IAM user, returns the IAM user ID, for example ABCDEFGHI1JKLMNOPQ23R. // For an IAM role, returns the IAM role ID, for example ABCDE1F2GH3I4JK5LMNOP:i-a123b456. @@ -4052,15 +4056,14 @@ type ReceiveMessageInput struct { // return the same messages and receipt handles. If a retry occurs within // the deduplication interval, it resets the visibility timeout. For more // information, see Visibility Timeout (https://docs.aws.amazon.com/AWSSimpleQueueService/latest/SQSDeveloperGuide/sqs-visibility-timeout.html) - // in the Amazon Simple Queue Service Developer Guide. If a caller of the - // ReceiveMessage action still processes messages when the visibility timeout - // expires and messages become visible, another worker consuming from the - // same queue can receive the same messages and therefore process duplicates. - // Also, if a consumer whose message processing time is longer than the visibility - // timeout tries to delete the processed messages, the action fails with - // an error. To mitigate this effect, ensure that your application observes - // a safe threshold before the visibility timeout expires and extend the - // visibility timeout as necessary. + // in the Amazon SQS Developer Guide. If a caller of the ReceiveMessage action + // still processes messages when the visibility timeout expires and messages + // become visible, another worker consuming from the same queue can receive + // the same messages and therefore process duplicates. Also, if a consumer + // whose message processing time is longer than the visibility timeout tries + // to delete the processed messages, the action fails with an error. To mitigate + // this effect, ensure that your application observes a safe threshold before + // the visibility timeout expires and extend the visibility timeout as necessary. // // * While messages with a particular MessageGroupId are invisible, no more // messages belonging to the same MessageGroupId are returned until the visibility @@ -4076,7 +4079,7 @@ type ReceiveMessageInput struct { // // For best practices of using ReceiveRequestAttemptId, see Using the ReceiveRequestAttemptId // Request Parameter (https://docs.aws.amazon.com/AWSSimpleQueueService/latest/SQSDeveloperGuide/using-receiverequestattemptid-request-parameter.html) - // in the Amazon Simple Queue Service Developer Guide. + // in the Amazon SQS Developer Guide. ReceiveRequestAttemptId *string `type:"string"` // The duration (in seconds) that the received messages are hidden from subsequent @@ -4383,8 +4386,8 @@ type SendMessageBatchRequestEntry struct { Id *string `type:"string" required:"true"` // Each message attribute consists of a Name, Type, and Value. For more information, - // see Amazon SQS Message Attributes (https://docs.aws.amazon.com/AWSSimpleQueueService/latest/SQSDeveloperGuide/sqs-message-metadata.html#sqs-message-attributes) - // in the Amazon Simple Queue Service Developer Guide. + // see Amazon SQS message attributes (https://docs.aws.amazon.com/AWSSimpleQueueService/latest/SQSDeveloperGuide/sqs-message-metadata.html#sqs-message-attributes) + // in the Amazon SQS Developer Guide. MessageAttributes map[string]*MessageAttributeValue `locationName:"MessageAttribute" locationNameKey:"Name" locationNameValue:"Value" type:"map" flattened:"true"` // The body of the message. @@ -4398,7 +4401,7 @@ type SendMessageBatchRequestEntry struct { // interval. If a message with a particular MessageDeduplicationId is sent successfully, // subsequent messages with the same MessageDeduplicationId are accepted successfully // but aren't delivered. For more information, see Exactly-once processing (https://docs.aws.amazon.com/AWSSimpleQueueService/latest/SQSDeveloperGuide/FIFO-queues-exactly-once-processing.html) - // in the Amazon Simple Queue Service Developer Guide. + // in the Amazon SQS Developer Guide. // // * Every message must have a unique MessageDeduplicationId, You may provide // a MessageDeduplicationId explicitly. If you aren't able to provide a MessageDeduplicationId @@ -4433,7 +4436,7 @@ type SendMessageBatchRequestEntry struct { // // For best practices of using MessageDeduplicationId, see Using the MessageDeduplicationId // Property (https://docs.aws.amazon.com/AWSSimpleQueueService/latest/SQSDeveloperGuide/using-messagededuplicationid-property.html) - // in the Amazon Simple Queue Service Developer Guide. + // in the Amazon SQS Developer Guide. MessageDeduplicationId *string `type:"string"` // This parameter applies only to FIFO (first-in-first-out) queues. @@ -4458,7 +4461,7 @@ type SendMessageBatchRequestEntry struct { // // For best practices of using MessageGroupId, see Using the MessageGroupId // Property (https://docs.aws.amazon.com/AWSSimpleQueueService/latest/SQSDeveloperGuide/using-messagegroupid-property.html) - // in the Amazon Simple Queue Service Developer Guide. + // in the Amazon SQS Developer Guide. // // MessageGroupId is required for FIFO queues. You can't use it for Standard // queues. @@ -4468,8 +4471,8 @@ type SendMessageBatchRequestEntry struct { // of a Name, Type, and Value. // // * Currently, the only supported message system attribute is AWSTraceHeader. - // Its type must be String and its value must be a correctly formatted AWS - // X-Ray trace header string. + // Its type must be String and its value must be a correctly formatted X-Ray + // trace header string. // // * The size of a message system attribute doesn't count towards the total // size of a message. @@ -4666,8 +4669,8 @@ type SendMessageInput struct { DelaySeconds *int64 `type:"integer"` // Each message attribute consists of a Name, Type, and Value. For more information, - // see Amazon SQS Message Attributes (https://docs.aws.amazon.com/AWSSimpleQueueService/latest/SQSDeveloperGuide/sqs-message-metadata.html#sqs-message-attributes) - // in the Amazon Simple Queue Service Developer Guide. + // see Amazon SQS message attributes (https://docs.aws.amazon.com/AWSSimpleQueueService/latest/SQSDeveloperGuide/sqs-message-metadata.html#sqs-message-attributes) + // in the Amazon SQS Developer Guide. MessageAttributes map[string]*MessageAttributeValue `locationName:"MessageAttribute" locationNameKey:"Name" locationNameValue:"Value" type:"map" flattened:"true"` // The message to send. The minimum size is one character. The maximum size @@ -4691,7 +4694,7 @@ type SendMessageInput struct { // MessageDeduplicationId are accepted successfully but aren't delivered during // the 5-minute deduplication interval. For more information, see Exactly-once // processing (https://docs.aws.amazon.com/AWSSimpleQueueService/latest/SQSDeveloperGuide/FIFO-queues-exactly-once-processing.html) - // in the Amazon Simple Queue Service Developer Guide. + // in the Amazon SQS Developer Guide. // // * Every message must have a unique MessageDeduplicationId, You may provide // a MessageDeduplicationId explicitly. If you aren't able to provide a MessageDeduplicationId @@ -4726,7 +4729,7 @@ type SendMessageInput struct { // // For best practices of using MessageDeduplicationId, see Using the MessageDeduplicationId // Property (https://docs.aws.amazon.com/AWSSimpleQueueService/latest/SQSDeveloperGuide/using-messagededuplicationid-property.html) - // in the Amazon Simple Queue Service Developer Guide. + // in the Amazon SQS Developer Guide. MessageDeduplicationId *string `type:"string"` // This parameter applies only to FIFO (first-in-first-out) queues. @@ -4751,7 +4754,7 @@ type SendMessageInput struct { // // For best practices of using MessageGroupId, see Using the MessageGroupId // Property (https://docs.aws.amazon.com/AWSSimpleQueueService/latest/SQSDeveloperGuide/using-messagegroupid-property.html) - // in the Amazon Simple Queue Service Developer Guide. + // in the Amazon SQS Developer Guide. // // MessageGroupId is required for FIFO queues. You can't use it for Standard // queues. @@ -4761,8 +4764,8 @@ type SendMessageInput struct { // of a Name, Type, and Value. // // * Currently, the only supported message system attribute is AWSTraceHeader. - // Its type must be String and its value must be a correctly formatted AWS - // X-Ray trace header string. + // Its type must be String and its value must be a correctly formatted X-Ray + // trace header string. // // * The size of a message system attribute doesn't count towards the total // size of a message. @@ -4887,7 +4890,7 @@ type SendMessageOutput struct { // An attribute containing the MessageId of the message sent to the queue. For // more information, see Queue and Message Identifiers (https://docs.aws.amazon.com/AWSSimpleQueueService/latest/SQSDeveloperGuide/sqs-queue-message-identifiers.html) - // in the Amazon Simple Queue Service Developer Guide. + // in the Amazon SQS Developer Guide. MessageId *string `type:"string"` // This parameter applies only to FIFO (first-in-first-out) queues. @@ -4959,9 +4962,10 @@ type SetQueueAttributesInput struct { // Amazon SQS retains a message. Valid values: An integer representing seconds, // from 60 (1 minute) to 1,209,600 (14 days). Default: 345,600 (4 days). // - // * Policy – The queue's policy. A valid AWS policy. For more information - // about policy structure, see Overview of AWS IAM Policies (https://docs.aws.amazon.com/IAM/latest/UserGuide/PoliciesOverview.html) - // in the Amazon IAM User Guide. + // * Policy – The queue's policy. A valid Amazon Web Services policy. For + // more information about policy structure, see Overview of Amazon Web Services + // IAM Policies (https://docs.aws.amazon.com/IAM/latest/UserGuide/PoliciesOverview.html) + // in the Identity and Access Management User Guide. // // * ReceiveMessageWaitTimeSeconds – The length of time, in seconds, for // which a ReceiveMessage action waits for a message to arrive. Valid values: @@ -4971,9 +4975,9 @@ type SetQueueAttributesInput struct { // queue functionality of the source queue as a JSON object. For more information // about the redrive policy and dead-letter queues, see Using Amazon SQS // Dead-Letter Queues (https://docs.aws.amazon.com/AWSSimpleQueueService/latest/SQSDeveloperGuide/sqs-dead-letter-queues.html) - // in the Amazon Simple Queue Service Developer Guide. deadLetterTargetArn - // – The Amazon Resource Name (ARN) of the dead-letter queue to which Amazon - // SQS moves messages after the value of maxReceiveCount is exceeded. maxReceiveCount + // in the Amazon SQS Developer Guide. deadLetterTargetArn – The Amazon + // Resource Name (ARN) of the dead-letter queue to which Amazon SQS moves + // messages after the value of maxReceiveCount is exceeded. maxReceiveCount // – The number of times a message is delivered to the source queue before // being moved to the dead-letter queue. When the ReceiveCount for a message // exceeds the maxReceiveCount for a queue, Amazon SQS moves the message @@ -4985,36 +4989,37 @@ type SetQueueAttributesInput struct { // Valid values: An integer from 0 to 43,200 (12 hours). Default: 30. For // more information about the visibility timeout, see Visibility Timeout // (https://docs.aws.amazon.com/AWSSimpleQueueService/latest/SQSDeveloperGuide/sqs-visibility-timeout.html) - // in the Amazon Simple Queue Service Developer Guide. + // in the Amazon SQS Developer Guide. // // The following attributes apply only to server-side-encryption (https://docs.aws.amazon.com/AWSSimpleQueueService/latest/SQSDeveloperGuide/sqs-server-side-encryption.html): // - // * KmsMasterKeyId – The ID of an AWS-managed customer master key (CMK) - // for Amazon SQS or a custom CMK. For more information, see Key Terms (https://docs.aws.amazon.com/AWSSimpleQueueService/latest/SQSDeveloperGuide/sqs-server-side-encryption.html#sqs-sse-key-terms). + // * KmsMasterKeyId – The ID of an Amazon Web Services managed customer + // master key (CMK) for Amazon SQS or a custom CMK. For more information, + // see Key Terms (https://docs.aws.amazon.com/AWSSimpleQueueService/latest/SQSDeveloperGuide/sqs-server-side-encryption.html#sqs-sse-key-terms). // While the alias of the AWS-managed CMK for Amazon SQS is always alias/aws/sqs, // the alias of a custom CMK can, for example, be alias/MyAlias . For more // examples, see KeyId (https://docs.aws.amazon.com/kms/latest/APIReference/API_DescribeKey.html#API_DescribeKey_RequestParameters) - // in the AWS Key Management Service API Reference. + // in the Key Management Service API Reference. // // * KmsDataKeyReusePeriodSeconds – The length of time, in seconds, for // which Amazon SQS can reuse a data key (https://docs.aws.amazon.com/kms/latest/developerguide/concepts.html#data-keys) - // to encrypt or decrypt messages before calling AWS KMS again. An integer - // representing seconds, between 60 seconds (1 minute) and 86,400 seconds - // (24 hours). Default: 300 (5 minutes). A shorter time period provides better - // security but results in more calls to KMS which might incur charges after - // Free Tier. For more information, see How Does the Data Key Reuse Period - // Work? (https://docs.aws.amazon.com/AWSSimpleQueueService/latest/SQSDeveloperGuide/sqs-server-side-encryption.html#sqs-how-does-the-data-key-reuse-period-work). + // to encrypt or decrypt messages before calling KMS again. An integer representing + // seconds, between 60 seconds (1 minute) and 86,400 seconds (24 hours). + // Default: 300 (5 minutes). A shorter time period provides better security + // but results in more calls to KMS which might incur charges after Free + // Tier. For more information, see How Does the Data Key Reuse Period Work? + // (https://docs.aws.amazon.com/AWSSimpleQueueService/latest/SQSDeveloperGuide/sqs-server-side-encryption.html#sqs-how-does-the-data-key-reuse-period-work). // // The following attribute applies only to FIFO (first-in-first-out) queues // (https://docs.aws.amazon.com/AWSSimpleQueueService/latest/SQSDeveloperGuide/FIFO-queues.html): // // * ContentBasedDeduplication – Enables content-based deduplication. For // more information, see Exactly-once processing (https://docs.aws.amazon.com/AWSSimpleQueueService/latest/SQSDeveloperGuide/FIFO-queues-exactly-once-processing.html) - // in the Amazon Simple Queue Service Developer Guide. Note the following: - // Every message must have a unique MessageDeduplicationId. You may provide - // a MessageDeduplicationId explicitly. If you aren't able to provide a MessageDeduplicationId - // and you enable ContentBasedDeduplication for your queue, Amazon SQS uses - // a SHA-256 hash to generate the MessageDeduplicationId using the body of + // in the Amazon SQS Developer Guide. Note the following: Every message must + // have a unique MessageDeduplicationId. You may provide a MessageDeduplicationId + // explicitly. If you aren't able to provide a MessageDeduplicationId and + // you enable ContentBasedDeduplication for your queue, Amazon SQS uses a + // SHA-256 hash to generate the MessageDeduplicationId using the body of // the message (but not the attributes of the message). If you don't provide // a MessageDeduplicationId and the queue doesn't have ContentBasedDeduplication // set, the action fails with an error. If the queue has ContentBasedDeduplication @@ -5048,7 +5053,7 @@ type SetQueueAttributesInput struct { // as specified. // // For information on throughput quotas, see Quotas related to messages (https://docs.aws.amazon.com/AWSSimpleQueueService/latest/SQSDeveloperGuide/quotas-messages.html) - // in the Amazon Simple Queue Service Developer Guide. + // in the Amazon SQS Developer Guide. // // Attributes is a required field Attributes map[string]*string `locationName:"Attribute" locationNameKey:"Name" locationNameValue:"Value" type:"map" flattened:"true" required:"true"` diff --git a/vendor/github.com/aws/aws-sdk-go/service/sqs/doc.go b/vendor/github.com/aws/aws-sdk-go/service/sqs/doc.go index 854208bcc..57d7718cd 100644 --- a/vendor/github.com/aws/aws-sdk-go/service/sqs/doc.go +++ b/vendor/github.com/aws/aws-sdk-go/service/sqs/doc.go @@ -3,20 +3,19 @@ // Package sqs provides the client and types for making API // requests to Amazon Simple Queue Service. // -// Welcome to the Amazon Simple Queue Service API Reference. +// Welcome to the Amazon SQS API Reference. // -// Amazon Simple Queue Service (Amazon SQS) is a reliable, highly-scalable hosted -// queue for storing messages as they travel between applications or microservices. -// Amazon SQS moves data between distributed application components and helps -// you decouple these components. +// Amazon SQS is a reliable, highly-scalable hosted queue for storing messages +// as they travel between applications or microservices. Amazon SQS moves data +// between distributed application components and helps you decouple these components. // // For information on the permissions you need to use this API, see Identity // and access management (https://docs.aws.amazon.com/AWSSimpleQueueService/latest/SQSDeveloperGuide/sqs-authentication-and-access-control.html) -// in the Amazon Simple Queue Service Developer Guide. +// in the Amazon SQS Developer Guide. // -// You can use AWS SDKs (http://aws.amazon.com/tools/#sdk) to access Amazon -// SQS using your favorite programming language. The SDKs perform tasks such -// as the following automatically: +// You can use Amazon Web Services SDKs (http://aws.amazon.com/tools/#sdk) to +// access Amazon SQS using your favorite programming language. The SDKs perform +// tasks such as the following automatically: // // * Cryptographically sign your service requests // @@ -28,11 +27,11 @@ // // * Amazon SQS Product Page (http://aws.amazon.com/sqs/) // -// * Amazon Simple Queue Service Developer Guide Making API Requests (https://docs.aws.amazon.com/AWSSimpleQueueService/latest/SQSDeveloperGuide/sqs-making-api-requests.html) +// * Amazon SQS Developer Guide Making API Requests (https://docs.aws.amazon.com/AWSSimpleQueueService/latest/SQSDeveloperGuide/sqs-making-api-requests.html) // Amazon SQS Message Attributes (https://docs.aws.amazon.com/AWSSimpleQueueService/latest/SQSDeveloperGuide/sqs-message-metadata.html#sqs-message-attributes) // Amazon SQS Dead-Letter Queues (https://docs.aws.amazon.com/AWSSimpleQueueService/latest/SQSDeveloperGuide/sqs-dead-letter-queues.html) // -// * Amazon SQS in the AWS CLI Command Reference (http://docs.aws.amazon.com/cli/latest/reference/sqs/index.html) +// * Amazon SQS in the Command Line Interface (http://docs.aws.amazon.com/cli/latest/reference/sqs/index.html) // // * Amazon Web Services General Reference Regions and Endpoints (https://docs.aws.amazon.com/general/latest/gr/rande.html#sqs_region) // diff --git a/vendor/github.com/minio/minio-go/v7/api-compose-object.go b/vendor/github.com/minio/minio-go/v7/api-compose-object.go index dd597e46f..87dfae8d3 100644 --- a/vendor/github.com/minio/minio-go/v7/api-compose-object.go +++ b/vendor/github.com/minio/minio-go/v7/api-compose-object.go @@ -513,7 +513,7 @@ func (c Client) ComposeObject(ctx context.Context, dst CopyDestOptions, srcs ... // 4. Make final complete-multipart request. uploadInfo, err := c.completeMultipartUpload(ctx, dst.Bucket, dst.Object, uploadID, - completeMultipartUpload{Parts: objParts}) + completeMultipartUpload{Parts: objParts}, PutObjectOptions{}) if err != nil { return UploadInfo{}, err } diff --git a/vendor/github.com/minio/minio-go/v7/api-list.go b/vendor/github.com/minio/minio-go/v7/api-list.go index 7996c11e9..814ec29c8 100644 --- a/vendor/github.com/minio/minio-go/v7/api-list.go +++ b/vendor/github.com/minio/minio-go/v7/api-list.go @@ -58,12 +58,12 @@ func (c Client) ListBuckets(ctx context.Context) ([]BucketInfo, error) { /// Bucket Read Operations. -func (c Client) listObjectsV2(ctx context.Context, bucketName, objectPrefix string, recursive, metadata bool, maxKeys int) <-chan ObjectInfo { +func (c Client) listObjectsV2(ctx context.Context, bucketName string, opts ListObjectsOptions) <-chan ObjectInfo { // Allocate new list objects channel. objectStatCh := make(chan ObjectInfo, 1) // Default listing is delimited at "/" delimiter := "/" - if recursive { + if opts.Recursive { // If recursive we do not delimit. delimiter = "" } @@ -81,7 +81,7 @@ func (c Client) listObjectsV2(ctx context.Context, bucketName, objectPrefix stri } // Validate incoming object prefix. - if err := s3utils.CheckValidObjectNamePrefix(objectPrefix); err != nil { + if err := s3utils.CheckValidObjectNamePrefix(opts.Prefix); err != nil { defer close(objectStatCh) objectStatCh <- ObjectInfo{ Err: err, @@ -96,8 +96,8 @@ func (c Client) listObjectsV2(ctx context.Context, bucketName, objectPrefix stri var continuationToken string for { // Get list of objects a maximum of 1000 per request. - result, err := c.listObjectsV2Query(ctx, bucketName, objectPrefix, continuationToken, - fetchOwner, metadata, delimiter, maxKeys) + result, err := c.listObjectsV2Query(ctx, bucketName, opts.Prefix, continuationToken, + fetchOwner, opts.WithMetadata, delimiter, opts.MaxKeys, opts.headers) if err != nil { objectStatCh <- ObjectInfo{ Err: err, @@ -153,7 +153,7 @@ func (c Client) listObjectsV2(ctx context.Context, bucketName, objectPrefix stri // ?prefix - Limits the response to keys that begin with the specified prefix. // ?max-keys - Sets the maximum number of keys returned in the response body. // ?metadata - Specifies if we want metadata for the objects as part of list operation. -func (c Client) listObjectsV2Query(ctx context.Context, bucketName, objectPrefix, continuationToken string, fetchOwner, metadata bool, delimiter string, maxkeys int) (ListBucketV2Result, error) { +func (c Client) listObjectsV2Query(ctx context.Context, bucketName, objectPrefix, continuationToken string, fetchOwner, metadata bool, delimiter string, maxkeys int, headers http.Header) (ListBucketV2Result, error) { // Validate bucket name. if err := s3utils.CheckValidBucketName(bucketName); err != nil { return ListBucketV2Result{}, err @@ -202,6 +202,7 @@ func (c Client) listObjectsV2Query(ctx context.Context, bucketName, objectPrefix bucketName: bucketName, queryValues: urlValues, contentSHA256Hex: emptySHA256Hex, + customHeader: headers, }) defer closeResponse(resp) if err != nil { @@ -246,12 +247,12 @@ func (c Client) listObjectsV2Query(ctx context.Context, bucketName, objectPrefix return listBucketResult, nil } -func (c Client) listObjects(ctx context.Context, bucketName, objectPrefix string, recursive bool, maxKeys int) <-chan ObjectInfo { +func (c Client) listObjects(ctx context.Context, bucketName string, opts ListObjectsOptions) <-chan ObjectInfo { // Allocate new list objects channel. objectStatCh := make(chan ObjectInfo, 1) // Default listing is delimited at "/" delimiter := "/" - if recursive { + if opts.Recursive { // If recursive we do not delimit. delimiter = "" } @@ -264,7 +265,7 @@ func (c Client) listObjects(ctx context.Context, bucketName, objectPrefix string return objectStatCh } // Validate incoming object prefix. - if err := s3utils.CheckValidObjectNamePrefix(objectPrefix); err != nil { + if err := s3utils.CheckValidObjectNamePrefix(opts.Prefix); err != nil { defer close(objectStatCh) objectStatCh <- ObjectInfo{ Err: err, @@ -279,7 +280,7 @@ func (c Client) listObjects(ctx context.Context, bucketName, objectPrefix string marker := "" for { // Get list of objects a maximum of 1000 per request. - result, err := c.listObjectsQuery(ctx, bucketName, objectPrefix, marker, delimiter, maxKeys) + result, err := c.listObjectsQuery(ctx, bucketName, opts.Prefix, marker, delimiter, opts.MaxKeys, opts.headers) if err != nil { objectStatCh <- ObjectInfo{ Err: err, @@ -326,12 +327,12 @@ func (c Client) listObjects(ctx context.Context, bucketName, objectPrefix string return objectStatCh } -func (c Client) listObjectVersions(ctx context.Context, bucketName, prefix string, recursive bool, maxKeys int) <-chan ObjectInfo { +func (c Client) listObjectVersions(ctx context.Context, bucketName string, opts ListObjectsOptions) <-chan ObjectInfo { // Allocate new list objects channel. resultCh := make(chan ObjectInfo, 1) // Default listing is delimited at "/" delimiter := "/" - if recursive { + if opts.Recursive { // If recursive we do not delimit. delimiter = "" } @@ -346,7 +347,7 @@ func (c Client) listObjectVersions(ctx context.Context, bucketName, prefix strin } // Validate incoming object prefix. - if err := s3utils.CheckValidObjectNamePrefix(prefix); err != nil { + if err := s3utils.CheckValidObjectNamePrefix(opts.Prefix); err != nil { defer close(resultCh) resultCh <- ObjectInfo{ Err: err, @@ -365,7 +366,7 @@ func (c Client) listObjectVersions(ctx context.Context, bucketName, prefix strin for { // Get list of objects a maximum of 1000 per request. - result, err := c.listObjectVersionsQuery(ctx, bucketName, prefix, keyMarker, versionIDMarker, delimiter, maxKeys) + result, err := c.listObjectVersionsQuery(ctx, bucketName, opts.Prefix, keyMarker, versionIDMarker, delimiter, opts.MaxKeys, opts.headers) if err != nil { resultCh <- ObjectInfo{ Err: err, @@ -438,7 +439,7 @@ func (c Client) listObjectVersions(ctx context.Context, bucketName, prefix strin // ?delimiter - A delimiter is a character you use to group keys. // ?prefix - Limits the response to keys that begin with the specified prefix. // ?max-keys - Sets the maximum number of keys returned in the response body. -func (c Client) listObjectVersionsQuery(ctx context.Context, bucketName, prefix, keyMarker, versionIDMarker, delimiter string, maxkeys int) (ListVersionsResult, error) { +func (c Client) listObjectVersionsQuery(ctx context.Context, bucketName, prefix, keyMarker, versionIDMarker, delimiter string, maxkeys int, headers http.Header) (ListVersionsResult, error) { // Validate bucket name. if err := s3utils.CheckValidBucketName(bucketName); err != nil { return ListVersionsResult{}, err @@ -483,6 +484,7 @@ func (c Client) listObjectVersionsQuery(ctx context.Context, bucketName, prefix, bucketName: bucketName, queryValues: urlValues, contentSHA256Hex: emptySHA256Hex, + customHeader: headers, }) defer closeResponse(resp) if err != nil { @@ -534,7 +536,7 @@ func (c Client) listObjectVersionsQuery(ctx context.Context, bucketName, prefix, // ?delimiter - A delimiter is a character you use to group keys. // ?prefix - Limits the response to keys that begin with the specified prefix. // ?max-keys - Sets the maximum number of keys returned in the response body. -func (c Client) listObjectsQuery(ctx context.Context, bucketName, objectPrefix, objectMarker, delimiter string, maxkeys int) (ListBucketResult, error) { +func (c Client) listObjectsQuery(ctx context.Context, bucketName, objectPrefix, objectMarker, delimiter string, maxkeys int, headers http.Header) (ListBucketResult, error) { // Validate bucket name. if err := s3utils.CheckValidBucketName(bucketName); err != nil { return ListBucketResult{}, err @@ -571,6 +573,7 @@ func (c Client) listObjectsQuery(ctx context.Context, bucketName, objectPrefix, bucketName: bucketName, queryValues: urlValues, contentSHA256Hex: emptySHA256Hex, + customHeader: headers, }) defer closeResponse(resp) if err != nil { @@ -629,6 +632,18 @@ type ListObjectsOptions struct { // Use the deprecated list objects V1 API UseV1 bool + + headers http.Header +} + +// Set adds a key value pair to the options. The +// key-value pair will be part of the HTTP GET request +// headers. +func (o *ListObjectsOptions) Set(key, value string) { + if o.headers == nil { + o.headers = make(http.Header) + } + o.headers.Set(key, value) } // ListObjects returns objects list after evaluating the passed options. @@ -640,22 +655,22 @@ type ListObjectsOptions struct { // func (c Client) ListObjects(ctx context.Context, bucketName string, opts ListObjectsOptions) <-chan ObjectInfo { if opts.WithVersions { - return c.listObjectVersions(ctx, bucketName, opts.Prefix, opts.Recursive, opts.MaxKeys) + return c.listObjectVersions(ctx, bucketName, opts) } // Use legacy list objects v1 API if opts.UseV1 { - return c.listObjects(ctx, bucketName, opts.Prefix, opts.Recursive, opts.MaxKeys) + return c.listObjects(ctx, bucketName, opts) } // Check whether this is snowball region, if yes ListObjectsV2 doesn't work, fallback to listObjectsV1. if location, ok := c.bucketLocCache.Get(bucketName); ok { if location == "snowball" { - return c.listObjects(ctx, bucketName, opts.Prefix, opts.Recursive, opts.MaxKeys) + return c.listObjects(ctx, bucketName, opts) } } - return c.listObjectsV2(ctx, bucketName, opts.Prefix, opts.Recursive, opts.WithMetadata, opts.MaxKeys) + return c.listObjectsV2(ctx, bucketName, opts) } // ListIncompleteUploads - List incompletely uploaded multipart objects. diff --git a/vendor/github.com/minio/minio-go/v7/api-put-object-multipart.go b/vendor/github.com/minio/minio-go/v7/api-put-object-multipart.go index a70d7054f..873ec3878 100644 --- a/vendor/github.com/minio/minio-go/v7/api-put-object-multipart.go +++ b/vendor/github.com/minio/minio-go/v7/api-put-object-multipart.go @@ -176,7 +176,7 @@ func (c Client) putObjectMultipartNoStream(ctx context.Context, bucketName, obje // Sort all completed parts. sort.Sort(completedParts(complMultipartUpload.Parts)) - uploadInfo, err := c.completeMultipartUpload(ctx, bucketName, objectName, uploadID, complMultipartUpload) + uploadInfo, err := c.completeMultipartUpload(ctx, bucketName, objectName, uploadID, complMultipartUpload, PutObjectOptions{}) if err != nil { return UploadInfo{}, err } @@ -309,7 +309,7 @@ func (c Client) uploadPart(ctx context.Context, bucketName, objectName, uploadID // completeMultipartUpload - Completes a multipart upload by assembling previously uploaded parts. func (c Client) completeMultipartUpload(ctx context.Context, bucketName, objectName, uploadID string, - complete completeMultipartUpload) (UploadInfo, error) { + complete completeMultipartUpload, opts PutObjectOptions) (UploadInfo, error) { // Input validation. if err := s3utils.CheckValidBucketName(bucketName); err != nil { return UploadInfo{}, err @@ -336,6 +336,7 @@ func (c Client) completeMultipartUpload(ctx context.Context, bucketName, objectN contentBody: completeMultipartUploadBuffer, contentLength: int64(len(completeMultipartUploadBytes)), contentSHA256Hex: sum256Hex(completeMultipartUploadBytes), + customHeader: opts.Header(), } // Execute POST to complete multipart upload for an objectName. diff --git a/vendor/github.com/minio/minio-go/v7/api-put-object-streaming.go b/vendor/github.com/minio/minio-go/v7/api-put-object-streaming.go index 39e381e9e..f1cc9fbb9 100644 --- a/vendor/github.com/minio/minio-go/v7/api-put-object-streaming.go +++ b/vendor/github.com/minio/minio-go/v7/api-put-object-streaming.go @@ -231,7 +231,7 @@ func (c Client) putObjectMultipartStreamFromReadAt(ctx context.Context, bucketNa // Sort all completed parts. sort.Sort(completedParts(complMultipartUpload.Parts)) - uploadInfo, err := c.completeMultipartUpload(ctx, bucketName, objectName, uploadID, complMultipartUpload) + uploadInfo, err := c.completeMultipartUpload(ctx, bucketName, objectName, uploadID, complMultipartUpload, PutObjectOptions{}) if err != nil { return UploadInfo{}, err } @@ -358,7 +358,7 @@ func (c Client) putObjectMultipartStreamOptionalChecksum(ctx context.Context, bu // Sort all completed parts. sort.Sort(completedParts(complMultipartUpload.Parts)) - uploadInfo, err := c.completeMultipartUpload(ctx, bucketName, objectName, uploadID, complMultipartUpload) + uploadInfo, err := c.completeMultipartUpload(ctx, bucketName, objectName, uploadID, complMultipartUpload, PutObjectOptions{}) if err != nil { return UploadInfo{}, err } diff --git a/vendor/github.com/minio/minio-go/v7/api-put-object.go b/vendor/github.com/minio/minio-go/v7/api-put-object.go index 247e40a67..b89f96d53 100644 --- a/vendor/github.com/minio/minio-go/v7/api-put-object.go +++ b/vendor/github.com/minio/minio-go/v7/api-put-object.go @@ -360,7 +360,7 @@ func (c Client) putObjectMultipartStreamNoLength(ctx context.Context, bucketName // Sort all completed parts. sort.Sort(completedParts(complMultipartUpload.Parts)) - uploadInfo, err := c.completeMultipartUpload(ctx, bucketName, objectName, uploadID, complMultipartUpload) + uploadInfo, err := c.completeMultipartUpload(ctx, bucketName, objectName, uploadID, complMultipartUpload, PutObjectOptions{}) if err != nil { return UploadInfo{}, err } diff --git a/vendor/github.com/minio/minio-go/v7/api.go b/vendor/github.com/minio/minio-go/v7/api.go index 44660ab6f..3ca67f377 100644 --- a/vendor/github.com/minio/minio-go/v7/api.go +++ b/vendor/github.com/minio/minio-go/v7/api.go @@ -108,7 +108,7 @@ type Options struct { // Global constants. const ( libraryName = "minio-go" - libraryVersion = "v7.0.11" + libraryVersion = "v7.0.12" ) // User Agent should always following the below style. diff --git a/vendor/github.com/minio/minio-go/v7/core.go b/vendor/github.com/minio/minio-go/v7/core.go index 2bf4edf01..26931f22d 100644 --- a/vendor/github.com/minio/minio-go/v7/core.go +++ b/vendor/github.com/minio/minio-go/v7/core.go @@ -46,13 +46,13 @@ func NewCore(endpoint string, opts *Options) (*Core, error) { // ListObjects - List all the objects at a prefix, optionally with marker and delimiter // you can further filter the results. func (c Core) ListObjects(bucket, prefix, marker, delimiter string, maxKeys int) (result ListBucketResult, err error) { - return c.listObjectsQuery(context.Background(), bucket, prefix, marker, delimiter, maxKeys) + return c.listObjectsQuery(context.Background(), bucket, prefix, marker, delimiter, maxKeys, nil) } // ListObjectsV2 - Lists all the objects at a prefix, similar to ListObjects() but uses // continuationToken instead of marker to support iteration over the results. func (c Core) ListObjectsV2(bucketName, objectPrefix, continuationToken string, fetchOwner bool, delimiter string, maxkeys int) (ListBucketV2Result, error) { - return c.listObjectsV2Query(context.Background(), bucketName, objectPrefix, continuationToken, fetchOwner, false, delimiter, maxkeys) + return c.listObjectsV2Query(context.Background(), bucketName, objectPrefix, continuationToken, fetchOwner, false, delimiter, maxkeys, nil) } // CopyObject - copies an object from source object to destination object on server side. @@ -97,10 +97,10 @@ func (c Core) ListObjectParts(ctx context.Context, bucket, object, uploadID stri } // CompleteMultipartUpload - Concatenate uploaded parts and commit to an object. -func (c Core) CompleteMultipartUpload(ctx context.Context, bucket, object, uploadID string, parts []CompletePart) (string, error) { +func (c Core) CompleteMultipartUpload(ctx context.Context, bucket, object, uploadID string, parts []CompletePart, opts PutObjectOptions) (string, error) { res, err := c.completeMultipartUpload(ctx, bucket, object, uploadID, completeMultipartUpload{ Parts: parts, - }) + }, opts) return res.ETag, err } diff --git a/vendor/github.com/minio/minio-go/v7/functional_tests.go b/vendor/github.com/minio/minio-go/v7/functional_tests.go index ba7ff577c..6b329dd27 100644 --- a/vendor/github.com/minio/minio-go/v7/functional_tests.go +++ b/vendor/github.com/minio/minio-go/v7/functional_tests.go @@ -38,6 +38,7 @@ import ( "sort" "strconv" "strings" + "sync" "time" "github.com/dustin/go-humanize" @@ -1054,6 +1055,153 @@ func testGetObjectWithVersioning() { successLogger(testName, function, args, startTime).Info() } +func testPutObjectWithVersioning() { + // initialize logging params + startTime := time.Now() + testName := getFuncName() + function := "GetObject()" + args := map[string]interface{}{} + + // Seed random based on current time. + rand.Seed(time.Now().Unix()) + + // Instantiate new minio client object. + c, err := minio.New(os.Getenv(serverEndpoint), + &minio.Options{ + Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""), + Secure: mustParseBool(os.Getenv(enableHTTPS)), + }) + if err != nil { + logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) + return + } + + // Enable tracing, write to stderr. + // c.TraceOn(os.Stderr) + + // Set user agent. + c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0") + + // Generate a new random bucket name. + bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") + args["bucketName"] = bucketName + + // Make a new bucket. + err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1", ObjectLocking: true}) + if err != nil { + logError(testName, function, args, startTime, "", "Make bucket failed", err) + return + } + + err = c.EnableVersioning(context.Background(), bucketName) + if err != nil { + logError(testName, function, args, startTime, "", "Enable versioning failed", err) + return + } + + objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") + args["objectName"] = objectName + + const n = 10 + // Read input... + + // Save the data concurrently. + var wg sync.WaitGroup + wg.Add(n) + var buffers = make([][]byte, n) + var errs [n]error + for i := 0; i < n; i++ { + r := newRandomReader(int64((1<<20)*i+i), int64(i)) + buf, err := ioutil.ReadAll(r) + if err != nil { + logError(testName, function, args, startTime, "", "unexpected failure", err) + return + } + buffers[i] = buf + + go func(i int) { + defer wg.Done() + _, errs[i] = c.PutObject(context.Background(), bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{PartSize: 5 << 20}) + }(i) + } + wg.Wait() + for _, err := range errs { + if err != nil { + logError(testName, function, args, startTime, "", "PutObject failed", err) + return + } + } + + objectsInfo := c.ListObjects(context.Background(), bucketName, minio.ListObjectsOptions{WithVersions: true, Recursive: true}) + var results []minio.ObjectInfo + for info := range objectsInfo { + if info.Err != nil { + logError(testName, function, args, startTime, "", "Unexpected error during listing objects", err) + return + } + results = append(results, info) + } + + if len(results) != n { + logError(testName, function, args, startTime, "", "Unexpected number of Version elements in listing objects", nil) + return + } + + sort.Slice(results, func(i, j int) bool { + return results[i].Size < results[j].Size + }) + + sort.Slice(buffers, func(i, j int) bool { + return len(buffers[i]) < len(buffers[j]) + }) + + for i := 0; i < len(results); i++ { + opts := minio.GetObjectOptions{VersionID: results[i].VersionID} + reader, err := c.GetObject(context.Background(), bucketName, objectName, opts) + if err != nil { + logError(testName, function, args, startTime, "", "error during GET object", err) + return + } + statInfo, err := reader.Stat() + if err != nil { + logError(testName, function, args, startTime, "", "error during calling reader.Stat()", err) + return + } + if statInfo.ETag != results[i].ETag { + logError(testName, function, args, startTime, "", "error during HEAD object, unexpected ETag", err) + return + } + if statInfo.LastModified.Unix() != results[i].LastModified.Unix() { + logError(testName, function, args, startTime, "", "error during HEAD object, unexpected Last-Modified", err) + return + } + if statInfo.Size != results[i].Size { + logError(testName, function, args, startTime, "", "error during HEAD object, unexpected Content-Length", err) + return + } + + tmpBuffer := bytes.NewBuffer([]byte{}) + _, err = io.Copy(tmpBuffer, reader) + if err != nil { + logError(testName, function, args, startTime, "", "unexpected io.Copy()", err) + return + } + + if !bytes.Equal(tmpBuffer.Bytes(), buffers[i]) { + logError(testName, function, args, startTime, "", "unexpected content of GetObject()", err) + return + } + } + + // Delete all objects and their versions as long as the bucket itself + if err = cleanupVersionedBucket(bucketName, c); err != nil { + logError(testName, function, args, startTime, "", "CleanupBucket failed", err) + return + } + + successLogger(testName, function, args, startTime).Info() +} + func testCopyObjectWithVersioning() { // initialize logging params startTime := time.Now() @@ -1191,6 +1339,166 @@ func testCopyObjectWithVersioning() { successLogger(testName, function, args, startTime).Info() } +func testConcurrentCopyObjectWithVersioning() { + // initialize logging params + startTime := time.Now() + testName := getFuncName() + function := "CopyObject()" + args := map[string]interface{}{} + + // Seed random based on current time. + rand.Seed(time.Now().Unix()) + + // Instantiate new minio client object. + c, err := minio.New(os.Getenv(serverEndpoint), + &minio.Options{ + Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""), + Secure: mustParseBool(os.Getenv(enableHTTPS)), + }) + if err != nil { + logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) + return + } + + // Enable tracing, write to stderr. + // c.TraceOn(os.Stderr) + + // Set user agent. + c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0") + + // Generate a new random bucket name. + bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") + args["bucketName"] = bucketName + + // Make a new bucket. + err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1", ObjectLocking: true}) + if err != nil { + logError(testName, function, args, startTime, "", "Make bucket failed", err) + return + } + + err = c.EnableVersioning(context.Background(), bucketName) + if err != nil { + logError(testName, function, args, startTime, "", "Enable versioning failed", err) + return + } + + objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") + args["objectName"] = objectName + + var testFiles = []string{"datafile-10-kB"} + for _, testFile := range testFiles { + r := getDataReader(testFile) + buf, err := ioutil.ReadAll(r) + if err != nil { + logError(testName, function, args, startTime, "", "unexpected failure", err) + return + } + r.Close() + _, err = c.PutObject(context.Background(), bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{}) + if err != nil { + logError(testName, function, args, startTime, "", "PutObject failed", err) + return + } + } + + objectsInfo := c.ListObjects(context.Background(), bucketName, minio.ListObjectsOptions{WithVersions: true, Recursive: true}) + var infos []minio.ObjectInfo + for info := range objectsInfo { + if info.Err != nil { + logError(testName, function, args, startTime, "", "Unexpected error during listing objects", err) + return + } + infos = append(infos, info) + } + + sort.Slice(infos, func(i, j int) bool { + return infos[i].Size < infos[j].Size + }) + + reader, err := c.GetObject(context.Background(), bucketName, objectName, minio.GetObjectOptions{VersionID: infos[0].VersionID}) + if err != nil { + logError(testName, function, args, startTime, "", "GetObject of the oldest version content failed", err) + return + } + + oldestContent, err := ioutil.ReadAll(reader) + if err != nil { + logError(testName, function, args, startTime, "", "Reading the oldest object version failed", err) + return + } + + // Copy Source + srcOpts := minio.CopySrcOptions{ + Bucket: bucketName, + Object: objectName, + VersionID: infos[0].VersionID, + } + args["src"] = srcOpts + + dstOpts := minio.CopyDestOptions{ + Bucket: bucketName, + Object: objectName + "-copy", + } + args["dst"] = dstOpts + + // Perform the Copy concurrently + const n = 10 + var wg sync.WaitGroup + wg.Add(n) + var errs [n]error + for i := 0; i < n; i++ { + go func(i int) { + defer wg.Done() + _, errs[i] = c.CopyObject(context.Background(), dstOpts, srcOpts) + }(i) + } + wg.Wait() + for _, err := range errs { + if err != nil { + logError(testName, function, args, startTime, "", "CopyObject failed", err) + return + } + } + + objectsInfo = c.ListObjects(context.Background(), bucketName, minio.ListObjectsOptions{WithVersions: true, Recursive: false, Prefix: dstOpts.Object}) + infos = []minio.ObjectInfo{} + for info := range objectsInfo { + // Destination object + readerCopy, err := c.GetObject(context.Background(), bucketName, objectName+"-copy", minio.GetObjectOptions{VersionID: info.VersionID}) + if err != nil { + logError(testName, function, args, startTime, "", "GetObject failed", err) + return + } + defer readerCopy.Close() + + newestContent, err := ioutil.ReadAll(readerCopy) + if err != nil { + logError(testName, function, args, startTime, "", "Reading from GetObject reader failed", err) + return + } + + if len(newestContent) == 0 || !bytes.Equal(oldestContent, newestContent) { + logError(testName, function, args, startTime, "", "Unexpected destination object content", err) + return + } + infos = append(infos, info) + } + + if len(infos) != n { + logError(testName, function, args, startTime, "", "Unexpected number of Version elements in listing objects", nil) + return + } + + // Delete all objects and their versions as long as the bucket itself + if err = cleanupVersionedBucket(bucketName, c); err != nil { + logError(testName, function, args, startTime, "", "CleanupBucket failed", err) + return + } + + successLogger(testName, function, args, startTime).Info() +} + func testComposeObjectWithVersioning() { // initialize logging params startTime := time.Now() @@ -7548,7 +7856,7 @@ func testSSECMultipartEncryptedToSSECCopyObjectPart() { completeParts = append(completeParts, minio.CompletePart{PartNumber: part.PartNumber, ETag: part.ETag}) // Complete the multipart upload - _, err = c.CompleteMultipartUpload(context.Background(), bucketName, objectName, uploadID, completeParts) + _, err = c.CompleteMultipartUpload(context.Background(), bucketName, objectName, uploadID, completeParts, minio.PutObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "CompleteMultipartUpload call failed", err) return @@ -7606,7 +7914,7 @@ func testSSECMultipartEncryptedToSSECCopyObjectPart() { } // Complete the multipart upload - _, err = c.CompleteMultipartUpload(context.Background(), destBucketName, destObjectName, uploadID, []minio.CompletePart{fstPart, sndPart, lstPart}) + _, err = c.CompleteMultipartUpload(context.Background(), destBucketName, destObjectName, uploadID, []minio.CompletePart{fstPart, sndPart, lstPart}, minio.PutObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "CompleteMultipartUpload call failed", err) return @@ -7783,7 +8091,7 @@ func testSSECEncryptedToSSECCopyObjectPart() { } // Complete the multipart upload - _, err = c.CompleteMultipartUpload(context.Background(), destBucketName, destObjectName, uploadID, []minio.CompletePart{fstPart, sndPart, lstPart}) + _, err = c.CompleteMultipartUpload(context.Background(), destBucketName, destObjectName, uploadID, []minio.CompletePart{fstPart, sndPart, lstPart}, minio.PutObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "CompleteMultipartUpload call failed", err) return @@ -7959,7 +8267,7 @@ func testSSECEncryptedToUnencryptedCopyPart() { } // Complete the multipart upload - _, err = c.CompleteMultipartUpload(context.Background(), destBucketName, destObjectName, uploadID, []minio.CompletePart{fstPart, sndPart, lstPart}) + _, err = c.CompleteMultipartUpload(context.Background(), destBucketName, destObjectName, uploadID, []minio.CompletePart{fstPart, sndPart, lstPart}, minio.PutObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "CompleteMultipartUpload call failed", err) return @@ -8138,7 +8446,7 @@ func testSSECEncryptedToSSES3CopyObjectPart() { } // Complete the multipart upload - _, err = c.CompleteMultipartUpload(context.Background(), destBucketName, destObjectName, uploadID, []minio.CompletePart{fstPart, sndPart, lstPart}) + _, err = c.CompleteMultipartUpload(context.Background(), destBucketName, destObjectName, uploadID, []minio.CompletePart{fstPart, sndPart, lstPart}, minio.PutObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "CompleteMultipartUpload call failed", err) return @@ -8312,7 +8620,7 @@ func testUnencryptedToSSECCopyObjectPart() { } // Complete the multipart upload - _, err = c.CompleteMultipartUpload(context.Background(), destBucketName, destObjectName, uploadID, []minio.CompletePart{fstPart, sndPart, lstPart}) + _, err = c.CompleteMultipartUpload(context.Background(), destBucketName, destObjectName, uploadID, []minio.CompletePart{fstPart, sndPart, lstPart}, minio.PutObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "CompleteMultipartUpload call failed", err) return @@ -8482,7 +8790,7 @@ func testUnencryptedToUnencryptedCopyPart() { } // Complete the multipart upload - _, err = c.CompleteMultipartUpload(context.Background(), destBucketName, destObjectName, uploadID, []minio.CompletePart{fstPart, sndPart, lstPart}) + _, err = c.CompleteMultipartUpload(context.Background(), destBucketName, destObjectName, uploadID, []minio.CompletePart{fstPart, sndPart, lstPart}, minio.PutObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "CompleteMultipartUpload call failed", err) return @@ -8654,7 +8962,7 @@ func testUnencryptedToSSES3CopyObjectPart() { } // Complete the multipart upload - _, err = c.CompleteMultipartUpload(context.Background(), destBucketName, destObjectName, uploadID, []minio.CompletePart{fstPart, sndPart, lstPart}) + _, err = c.CompleteMultipartUpload(context.Background(), destBucketName, destObjectName, uploadID, []minio.CompletePart{fstPart, sndPart, lstPart}, minio.PutObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "CompleteMultipartUpload call failed", err) return @@ -8829,7 +9137,7 @@ func testSSES3EncryptedToSSECCopyObjectPart() { } // Complete the multipart upload - _, err = c.CompleteMultipartUpload(context.Background(), destBucketName, destObjectName, uploadID, []minio.CompletePart{fstPart, sndPart, lstPart}) + _, err = c.CompleteMultipartUpload(context.Background(), destBucketName, destObjectName, uploadID, []minio.CompletePart{fstPart, sndPart, lstPart}, minio.PutObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "CompleteMultipartUpload call failed", err) return @@ -9000,7 +9308,7 @@ func testSSES3EncryptedToUnencryptedCopyPart() { } // Complete the multipart upload - _, err = c.CompleteMultipartUpload(context.Background(), destBucketName, destObjectName, uploadID, []minio.CompletePart{fstPart, sndPart, lstPart}) + _, err = c.CompleteMultipartUpload(context.Background(), destBucketName, destObjectName, uploadID, []minio.CompletePart{fstPart, sndPart, lstPart}, minio.PutObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "CompleteMultipartUpload call failed", err) return @@ -9174,7 +9482,7 @@ func testSSES3EncryptedToSSES3CopyObjectPart() { } // Complete the multipart upload - _, err = c.CompleteMultipartUpload(context.Background(), destBucketName, destObjectName, uploadID, []minio.CompletePart{fstPart, sndPart, lstPart}) + _, err = c.CompleteMultipartUpload(context.Background(), destBucketName, destObjectName, uploadID, []minio.CompletePart{fstPart, sndPart, lstPart}, minio.PutObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "CompleteMultipartUpload call failed", err) return @@ -11416,6 +11724,7 @@ func main() { testFPutObjectContextV2() testFGetObjectContextV2() testPutObjectContextV2() + testPutObjectWithVersioning() testMakeBucketError() testMakeBucketRegions() testPutObjectWithMetadata() @@ -11453,6 +11762,7 @@ func main() { testStatObjectWithVersioning() testGetObjectWithVersioning() testCopyObjectWithVersioning() + testConcurrentCopyObjectWithVersioning() testComposeObjectWithVersioning() testRemoveObjectWithVersioning() testRemoveObjectsWithVersioning() diff --git a/vendor/github.com/minio/minio-go/v7/go.mod b/vendor/github.com/minio/minio-go/v7/go.mod index 5468818c1..92590e971 100644 --- a/vendor/github.com/minio/minio-go/v7/go.mod +++ b/vendor/github.com/minio/minio-go/v7/go.mod @@ -17,7 +17,7 @@ require ( github.com/sirupsen/logrus v1.8.1 github.com/smartystreets/goconvey v1.6.4 // indirect github.com/stretchr/testify v1.4.0 // indirect - golang.org/x/crypto v0.0.0-20200709230013-948cd5f35899 + golang.org/x/crypto v0.0.0-20201216223049-8b5274cf687f golang.org/x/net v0.0.0-20200707034311-ab3426394381 golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae // indirect golang.org/x/text v0.3.3 // indirect diff --git a/vendor/github.com/minio/minio-go/v7/go.sum b/vendor/github.com/minio/minio-go/v7/go.sum index 1ba98c4f1..0e8248fff 100644 --- a/vendor/github.com/minio/minio-go/v7/go.sum +++ b/vendor/github.com/minio/minio-go/v7/go.sum @@ -49,8 +49,8 @@ github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJy github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20200709230013-948cd5f35899 h1:DZhuSZLsGlFL4CmhA8BcRA0mnthyA/nZ00AqCUo7vHg= -golang.org/x/crypto v0.0.0-20200709230013-948cd5f35899/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20201216223049-8b5274cf687f h1:aZp0e2vLN4MToVqnjNEYEtrEA8RH8U8FN1CU7JgqsPU= +golang.org/x/crypto v0.0.0-20201216223049-8b5274cf687f/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20200707034311-ab3426394381 h1:VXak5I6aEWmAXeQjA+QSZzlgNrpq9mjcfDemuexIKsU= @@ -61,6 +61,7 @@ golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae h1:Ih9Yo4hSPImZOpfGuA4bR/ORKTAbhZo2AbWNRCnevdo= golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3 h1:cokOdA+Jmi5PJGXLlLllQSgYigAEfHXJAERHVMaCc2k= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= diff --git a/vendor/github.com/shirou/gopsutil/cpu/cpu_windows.go b/vendor/github.com/shirou/gopsutil/cpu/cpu_windows.go index ad1750b5c..a4ea585a5 100644 --- a/vendor/github.com/shirou/gopsutil/cpu/cpu_windows.go +++ b/vendor/github.com/shirou/gopsutil/cpu/cpu_windows.go @@ -5,6 +5,7 @@ package cpu import ( "context" "fmt" + "strings" "unsafe" "github.com/StackExchange/wmi" @@ -18,7 +19,14 @@ var ( ) type Win32_Processor struct { - LoadPercentage *uint16 + Win32_ProcessorWithoutLoadPct + LoadPercentage *uint16 +} + +// LoadPercentage takes a linearly more time as the number of sockets increases. +// For vSphere by default corespersocket = 1, meaning for a 40 vCPU VM Get Processor Info +// could take more than half a minute. +type Win32_ProcessorWithoutLoadPct struct { Family uint16 Manufacturer string Name string @@ -104,8 +112,9 @@ func Info() ([]InfoStat, error) { func InfoWithContext(ctx context.Context) ([]InfoStat, error) { var ret []InfoStat - var dst []Win32_Processor + var dst []Win32_ProcessorWithoutLoadPct q := wmi.CreateQuery(&dst, "") + q = strings.ReplaceAll(q, "Win32_ProcessorWithoutLoadPct", "Win32_Processor") if err := common.WMIQueryWithContext(ctx, q, &dst); err != nil { return ret, err } @@ -242,8 +251,9 @@ func CountsWithContext(ctx context.Context, logical bool) (int, error) { } // physical cores https://github.com/giampaolo/psutil/blob/d01a9eaa35a8aadf6c519839e987a49d8be2d891/psutil/_psutil_windows.c#L499 // for the time being, try with unreliable and slow WMI call… - var dst []Win32_Processor + var dst []Win32_ProcessorWithoutLoadPct q := wmi.CreateQuery(&dst, "") + q = strings.ReplaceAll(q, "Win32_ProcessorWithoutLoadPct", "Win32_Processor") if err := common.WMIQueryWithContext(ctx, q, &dst); err != nil { return 0, err } diff --git a/vendor/modules.txt b/vendor/modules.txt index f572c14ce..ae922175d 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -46,7 +46,7 @@ github.com/awnumar/memcall ## explicit github.com/awnumar/memguard github.com/awnumar/memguard/core -# github.com/aws/aws-sdk-go v1.38.68 +# github.com/aws/aws-sdk-go v1.38.71 ## explicit github.com/aws/aws-sdk-go/aws github.com/aws/aws-sdk-go/aws/awserr @@ -175,8 +175,6 @@ github.com/emirpasic/gods/utils # github.com/evanphx/json-patch v4.11.0+incompatible ## explicit github.com/evanphx/json-patch -# github.com/evanphx/json-patch/v5 v5.5.0 -## explicit # github.com/facebookgo/clock v0.0.0-20150410010913-600d898af40a github.com/facebookgo/clock # github.com/facebookgo/limitgroup v0.0.0-20150612190941-6abd8d71ec01 @@ -434,7 +432,7 @@ github.com/minio/md5-simd # github.com/minio/minio v0.0.0-20210507214158-ab7d5ee3d98e ## explicit github.com/minio/minio/pkg/disk -# github.com/minio/minio-go/v7 v7.0.11 +# github.com/minio/minio-go/v7 v7.0.12 ## explicit github.com/minio/minio-go/v7 github.com/minio/minio-go/v7/pkg/credentials @@ -520,7 +518,7 @@ github.com/secure-io/sio-go github.com/secure-io/sio-go/sioutil # github.com/sergi/go-diff v1.1.0 github.com/sergi/go-diff/diffmatchpatch -# github.com/shirou/gopsutil v3.21.5+incompatible +# github.com/shirou/gopsutil v3.21.6+incompatible ## explicit github.com/shirou/gopsutil/cpu github.com/shirou/gopsutil/internal/common @@ -728,7 +726,7 @@ google.golang.org/genproto/googleapis/rpc/status google.golang.org/grpc/codes google.golang.org/grpc/internal/status google.golang.org/grpc/status -# google.golang.org/protobuf v1.27.0 +# google.golang.org/protobuf v1.27.1 ## explicit google.golang.org/protobuf/encoding/protojson google.golang.org/protobuf/encoding/prototext