diff --git a/CHANGELOG.md b/CHANGELOG.md index 89e5c91979..773aaa6eae 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -86,6 +86,7 @@ * [ENHANCEMENT] OTLP: If the flag `-distributor.otel-created-timestamp-zero-ingestion-enabled` is true, OTel start timestamps are converted to Prometheus zero samples to mark series start. #9131 * [ENHANCEMENT] Querier: attach logs emitted during query consistency check to trace span for query. #9213 * [ENHANCEMENT] Query-scheduler: Experimental `-query-scheduler.prioritize-query-components` flag enables the querier-worker queue priority algorithm to take precedence over tenant rotation when dequeuing requests. #9220 +* [ENHANCEMENT] Add application credential arguments for Openstack Swift storage backend. #9181 * [BUGFIX] Ruler: add support for draining any outstanding alert notifications before shutting down. This can be enabled with the `-ruler.drain-notification-queue-on-shutdown=true` CLI flag. #8346 * [BUGFIX] Query-frontend: fix `-querier.max-query-lookback` enforcement when `-compactor.blocks-retention-period` is not set, and viceversa. #8388 * [BUGFIX] Ingester: fix sporadic `not found` error causing an internal server error if label names are queried with matchers during head compaction. #8391 diff --git a/cmd/mimir/config-descriptor.json b/cmd/mimir/config-descriptor.json index 6495f65f99..9765237a1b 100644 --- a/cmd/mimir/config-descriptor.json +++ b/cmd/mimir/config-descriptor.json @@ -7192,6 +7192,36 @@ "required": false, "desc": "", "blockEntries": [ + { + "kind": "field", + "name": "application_credential_id", + "required": false, + "desc": "OpenStack Swift application credential id", + "fieldValue": null, + "fieldDefaultValue": "", + "fieldFlag": "blocks-storage.swift.application-credential-id", + "fieldType": "string" + }, + { + "kind": "field", + "name": "application_credential_name", + "required": false, + "desc": "OpenStack Swift application credential name", + "fieldValue": null, + "fieldDefaultValue": "", + "fieldFlag": "blocks-storage.swift.application-credential-name", + "fieldType": "string" + }, + { + "kind": "field", + "name": "application_credential_secret", + "required": false, + "desc": "OpenStack Swift application credential secret", + "fieldValue": null, + "fieldDefaultValue": "", + "fieldFlag": "blocks-storage.swift.application-credential-secret", + "fieldType": "string" + }, { "kind": "field", "name": "auth_version", @@ -13207,6 +13237,36 @@ "required": false, "desc": "", "blockEntries": [ + { + "kind": "field", + "name": "application_credential_id", + "required": false, + "desc": "OpenStack Swift application credential id", + "fieldValue": null, + "fieldDefaultValue": "", + "fieldFlag": "ruler-storage.swift.application-credential-id", + "fieldType": "string" + }, + { + "kind": "field", + "name": "application_credential_name", + "required": false, + "desc": "OpenStack Swift application credential name", + "fieldValue": null, + "fieldDefaultValue": "", + "fieldFlag": "ruler-storage.swift.application-credential-name", + "fieldType": "string" + }, + { + "kind": "field", + "name": "application_credential_secret", + "required": false, + "desc": "OpenStack Swift application credential secret", + "fieldValue": null, + "fieldDefaultValue": "", + "fieldFlag": "ruler-storage.swift.application-credential-secret", + "fieldType": "string" + }, { "kind": "field", "name": "auth_version", @@ -15400,6 +15460,36 @@ "required": false, "desc": "", "blockEntries": [ + { + "kind": "field", + "name": "application_credential_id", + "required": false, + "desc": "OpenStack Swift application credential id", + "fieldValue": null, + "fieldDefaultValue": "", + "fieldFlag": "alertmanager-storage.swift.application-credential-id", + "fieldType": "string" + }, + { + "kind": "field", + "name": "application_credential_name", + "required": false, + "desc": "OpenStack Swift application credential name", + "fieldValue": null, + "fieldDefaultValue": "", + "fieldFlag": "alertmanager-storage.swift.application-credential-name", + "fieldType": "string" + }, + { + "kind": "field", + "name": "application_credential_secret", + "required": false, + "desc": "OpenStack Swift application credential secret", + "fieldValue": null, + "fieldDefaultValue": "", + "fieldFlag": "alertmanager-storage.swift.application-credential-secret", + "fieldType": "string" + }, { "kind": "field", "name": "auth_version", @@ -17815,6 +17905,36 @@ "required": false, "desc": "", "blockEntries": [ + { + "kind": "field", + "name": "application_credential_id", + "required": false, + "desc": "OpenStack Swift application credential id", + "fieldValue": null, + "fieldDefaultValue": "", + "fieldFlag": "common.storage.swift.application-credential-id", + "fieldType": "string" + }, + { + "kind": "field", + "name": "application_credential_name", + "required": false, + "desc": "OpenStack Swift application credential name", + "fieldValue": null, + "fieldDefaultValue": "", + "fieldFlag": "common.storage.swift.application-credential-name", + "fieldType": "string" + }, + { + "kind": "field", + "name": "application_credential_secret", + "required": false, + "desc": "OpenStack Swift application credential secret", + "fieldValue": null, + "fieldDefaultValue": "", + "fieldFlag": "common.storage.swift.application-credential-secret", + "fieldType": "string" + }, { "kind": "field", "name": "auth_version", diff --git a/cmd/mimir/help-all.txt.tmpl b/cmd/mimir/help-all.txt.tmpl index 9732cf15fe..a5755b29a8 100644 --- a/cmd/mimir/help-all.txt.tmpl +++ b/cmd/mimir/help-all.txt.tmpl @@ -93,6 +93,12 @@ Usage of ./cmd/mimir/mimir: When enabled, low-level S3 HTTP operation information is logged at the debug level. -alertmanager-storage.storage-prefix string Prefix for all objects stored in the backend storage. For simplicity, it may only contain digits and English alphabet letters. + -alertmanager-storage.swift.application-credential-id string + OpenStack Swift application credential id + -alertmanager-storage.swift.application-credential-name string + OpenStack Swift application credential name + -alertmanager-storage.swift.application-credential-secret string + OpenStack Swift application credential secret -alertmanager-storage.swift.auth-url string OpenStack Swift authentication URL -alertmanager-storage.swift.auth-version int @@ -761,6 +767,12 @@ Usage of ./cmd/mimir/mimir: When enabled, low-level S3 HTTP operation information is logged at the debug level. -blocks-storage.storage-prefix string Prefix for all objects stored in the backend storage. For simplicity, it may only contain digits and English alphabet letters. + -blocks-storage.swift.application-credential-id string + OpenStack Swift application credential id + -blocks-storage.swift.application-credential-name string + OpenStack Swift application credential name + -blocks-storage.swift.application-credential-secret string + OpenStack Swift application credential secret -blocks-storage.swift.auth-url string OpenStack Swift authentication URL -blocks-storage.swift.auth-version int @@ -943,6 +955,12 @@ Usage of ./cmd/mimir/mimir: Maximum time to wait for a TLS handshake. 0 means no limit. (default 10s) -common.storage.s3.trace.enabled When enabled, low-level S3 HTTP operation information is logged at the debug level. + -common.storage.swift.application-credential-id string + OpenStack Swift application credential id + -common.storage.swift.application-credential-name string + OpenStack Swift application credential name + -common.storage.swift.application-credential-secret string + OpenStack Swift application credential secret -common.storage.swift.auth-url string OpenStack Swift authentication URL -common.storage.swift.auth-version int @@ -2523,6 +2541,12 @@ Usage of ./cmd/mimir/mimir: When enabled, low-level S3 HTTP operation information is logged at the debug level. -ruler-storage.storage-prefix string Prefix for all objects stored in the backend storage. For simplicity, it may only contain digits and English alphabet letters. + -ruler-storage.swift.application-credential-id string + OpenStack Swift application credential id + -ruler-storage.swift.application-credential-name string + OpenStack Swift application credential name + -ruler-storage.swift.application-credential-secret string + OpenStack Swift application credential secret -ruler-storage.swift.auth-url string OpenStack Swift authentication URL -ruler-storage.swift.auth-version int diff --git a/cmd/mimir/help.txt.tmpl b/cmd/mimir/help.txt.tmpl index 7db5f7c671..365c2c4445 100644 --- a/cmd/mimir/help.txt.tmpl +++ b/cmd/mimir/help.txt.tmpl @@ -43,6 +43,12 @@ Usage of ./cmd/mimir/mimir: Accessing S3 resources using temporary, secure credentials provided by AWS Security Token Service. -alertmanager-storage.storage-prefix string Prefix for all objects stored in the backend storage. For simplicity, it may only contain digits and English alphabet letters. + -alertmanager-storage.swift.application-credential-id string + OpenStack Swift application credential id + -alertmanager-storage.swift.application-credential-name string + OpenStack Swift application credential name + -alertmanager-storage.swift.application-credential-secret string + OpenStack Swift application credential secret -alertmanager-storage.swift.auth-url string OpenStack Swift authentication URL -alertmanager-storage.swift.auth-version int @@ -209,6 +215,12 @@ Usage of ./cmd/mimir/mimir: Accessing S3 resources using temporary, secure credentials provided by AWS Security Token Service. -blocks-storage.storage-prefix string Prefix for all objects stored in the backend storage. For simplicity, it may only contain digits and English alphabet letters. + -blocks-storage.swift.application-credential-id string + OpenStack Swift application credential id + -blocks-storage.swift.application-credential-name string + OpenStack Swift application credential name + -blocks-storage.swift.application-credential-secret string + OpenStack Swift application credential secret -blocks-storage.swift.auth-url string OpenStack Swift authentication URL -blocks-storage.swift.auth-version int @@ -281,6 +293,12 @@ Usage of ./cmd/mimir/mimir: Enable AWS Server Side Encryption. Supported values: SSE-KMS, SSE-S3. -common.storage.s3.sts-endpoint string Accessing S3 resources using temporary, secure credentials provided by AWS Security Token Service. + -common.storage.swift.application-credential-id string + OpenStack Swift application credential id + -common.storage.swift.application-credential-name string + OpenStack Swift application credential name + -common.storage.swift.application-credential-secret string + OpenStack Swift application credential secret -common.storage.swift.auth-url string OpenStack Swift authentication URL -common.storage.swift.auth-version int @@ -665,6 +683,12 @@ Usage of ./cmd/mimir/mimir: Accessing S3 resources using temporary, secure credentials provided by AWS Security Token Service. -ruler-storage.storage-prefix string Prefix for all objects stored in the backend storage. For simplicity, it may only contain digits and English alphabet letters. + -ruler-storage.swift.application-credential-id string + OpenStack Swift application credential id + -ruler-storage.swift.application-credential-name string + OpenStack Swift application credential name + -ruler-storage.swift.application-credential-secret string + OpenStack Swift application credential secret -ruler-storage.swift.auth-url string OpenStack Swift authentication URL -ruler-storage.swift.auth-version int diff --git a/docs/sources/mimir/configure/configuration-parameters/index.md b/docs/sources/mimir/configure/configuration-parameters/index.md index 2193ce6a8f..9420968be7 100644 --- a/docs/sources/mimir/configure/configuration-parameters/index.md +++ b/docs/sources/mimir/configure/configuration-parameters/index.md @@ -5265,6 +5265,18 @@ The `swift_storage_backend` block configures the connection to OpenStack Object   ```yaml +# OpenStack Swift application credential id +# CLI flag: -.swift.application-credential-id +[application_credential_id: | default = ""] + +# OpenStack Swift application credential name +# CLI flag: -.swift.application-credential-name +[application_credential_name: | default = ""] + +# OpenStack Swift application credential secret +# CLI flag: -.swift.application-credential-secret +[application_credential_secret: | default = ""] + # OpenStack Swift authentication API version. 0 to autodetect. # CLI flag: -.swift.auth-version [auth_version: | default = 0] diff --git a/pkg/storage/bucket/swift/bucket_client.go b/pkg/storage/bucket/swift/bucket_client.go index 22e28d405a..5d13dc6399 100644 --- a/pkg/storage/bucket/swift/bucket_client.go +++ b/pkg/storage/bucket/swift/bucket_client.go @@ -16,24 +16,27 @@ import ( // NewBucketClient creates a new Swift bucket client func NewBucketClient(cfg Config, _ string, logger log.Logger) (objstore.Bucket, error) { bucketConfig := swift.Config{ - AuthVersion: cfg.AuthVersion, - AuthUrl: cfg.AuthURL, - Username: cfg.Username, - UserDomainName: cfg.UserDomainName, - UserDomainID: cfg.UserDomainID, - UserId: cfg.UserID, - Password: cfg.Password.String(), - DomainId: cfg.DomainID, - DomainName: cfg.DomainName, - ProjectID: cfg.ProjectID, - ProjectName: cfg.ProjectName, - ProjectDomainID: cfg.ProjectDomainID, - ProjectDomainName: cfg.ProjectDomainName, - RegionName: cfg.RegionName, - ContainerName: cfg.ContainerName, - Retries: cfg.MaxRetries, - ConnectTimeout: model.Duration(cfg.ConnectTimeout), - Timeout: model.Duration(cfg.RequestTimeout), + ApplicationCredentialID: cfg.ApplicationCredentialID, + ApplicationCredentialName: cfg.ApplicationCredentialName, + ApplicationCredentialSecret: cfg.ApplicationCredentialSecret.String(), + AuthVersion: cfg.AuthVersion, + AuthUrl: cfg.AuthURL, + Username: cfg.Username, + UserDomainName: cfg.UserDomainName, + UserDomainID: cfg.UserDomainID, + UserId: cfg.UserID, + Password: cfg.Password.String(), + DomainId: cfg.DomainID, + DomainName: cfg.DomainName, + ProjectID: cfg.ProjectID, + ProjectName: cfg.ProjectName, + ProjectDomainID: cfg.ProjectDomainID, + ProjectDomainName: cfg.ProjectDomainName, + RegionName: cfg.RegionName, + ContainerName: cfg.ContainerName, + Retries: cfg.MaxRetries, + ConnectTimeout: model.Duration(cfg.ConnectTimeout), + Timeout: model.Duration(cfg.RequestTimeout), // Hard-coded defaults. ChunkSize: swift.DefaultConfig.ChunkSize, diff --git a/pkg/storage/bucket/swift/config.go b/pkg/storage/bucket/swift/config.go index c766174474..bc2a8db85e 100644 --- a/pkg/storage/bucket/swift/config.go +++ b/pkg/storage/bucket/swift/config.go @@ -14,24 +14,27 @@ import ( // Config holds the config options for Swift backend type Config struct { - AuthVersion int `yaml:"auth_version"` - AuthURL string `yaml:"auth_url"` - Username string `yaml:"username"` - UserDomainName string `yaml:"user_domain_name"` - UserDomainID string `yaml:"user_domain_id"` - UserID string `yaml:"user_id"` - Password flagext.Secret `yaml:"password"` - DomainID string `yaml:"domain_id"` - DomainName string `yaml:"domain_name"` - ProjectID string `yaml:"project_id"` - ProjectName string `yaml:"project_name"` - ProjectDomainID string `yaml:"project_domain_id"` - ProjectDomainName string `yaml:"project_domain_name"` - RegionName string `yaml:"region_name"` - ContainerName string `yaml:"container_name"` - MaxRetries int `yaml:"max_retries" category:"advanced"` - ConnectTimeout time.Duration `yaml:"connect_timeout" category:"advanced"` - RequestTimeout time.Duration `yaml:"request_timeout" category:"advanced"` + ApplicationCredentialID string `yaml:"application_credential_id"` + ApplicationCredentialName string `yaml:"application_credential_name"` + ApplicationCredentialSecret flagext.Secret `yaml:"application_credential_secret"` + AuthVersion int `yaml:"auth_version"` + AuthURL string `yaml:"auth_url"` + Username string `yaml:"username"` + UserDomainName string `yaml:"user_domain_name"` + UserDomainID string `yaml:"user_domain_id"` + UserID string `yaml:"user_id"` + Password flagext.Secret `yaml:"password"` + DomainID string `yaml:"domain_id"` + DomainName string `yaml:"domain_name"` + ProjectID string `yaml:"project_id"` + ProjectName string `yaml:"project_name"` + ProjectDomainID string `yaml:"project_domain_id"` + ProjectDomainName string `yaml:"project_domain_name"` + RegionName string `yaml:"region_name"` + ContainerName string `yaml:"container_name"` + MaxRetries int `yaml:"max_retries" category:"advanced"` + ConnectTimeout time.Duration `yaml:"connect_timeout" category:"advanced"` + RequestTimeout time.Duration `yaml:"request_timeout" category:"advanced"` } // RegisterFlags registers the flags for Swift storage @@ -41,6 +44,9 @@ func (cfg *Config) RegisterFlags(f *flag.FlagSet) { // RegisterFlagsWithPrefix registers the flags for Swift storage with the provided prefix func (cfg *Config) RegisterFlagsWithPrefix(prefix string, f *flag.FlagSet) { + f.StringVar(&cfg.ApplicationCredentialID, prefix+"swift.application-credential-id", "", "OpenStack Swift application credential id") + f.StringVar(&cfg.ApplicationCredentialName, prefix+"swift.application-credential-name", "", "OpenStack Swift application credential name") + f.Var(&cfg.ApplicationCredentialSecret, prefix+"swift.application-credential-secret", "OpenStack Swift application credential secret") f.IntVar(&cfg.AuthVersion, prefix+"swift.auth-version", 0, "OpenStack Swift authentication API version. 0 to autodetect.") f.StringVar(&cfg.AuthURL, prefix+"swift.auth-url", "", "OpenStack Swift authentication URL") f.StringVar(&cfg.Username, prefix+"swift.username", "", "OpenStack Swift username.")