Add application credential arguments for openstack swift (#9181)

Update configuration options to pass application credentials
grafana · Sep 17, 2024 · bac778e · bac778e
1 parent 82f3667
commit bac778e
Show file tree

Hide file tree

Showing 7 changed files with 226 additions and 36 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -86,6 +86,7 @@
 * [ENHANCEMENT] OTLP: If the flag `-distributor.otel-created-timestamp-zero-ingestion-enabled` is true, OTel start timestamps are converted to Prometheus zero samples to mark series start. #9131
 * [ENHANCEMENT] Querier: attach logs emitted during query consistency check to trace span for query. #9213
 * [ENHANCEMENT] Query-scheduler: Experimental `-query-scheduler.prioritize-query-components` flag enables the querier-worker queue priority algorithm to take precedence over tenant rotation when dequeuing requests. #9220
+* [ENHANCEMENT] Add application credential arguments for Openstack Swift storage backend. #9181
 * [BUGFIX] Ruler: add support for draining any outstanding alert notifications before shutting down. This can be enabled with the `-ruler.drain-notification-queue-on-shutdown=true` CLI flag. #8346
 * [BUGFIX] Query-frontend: fix `-querier.max-query-lookback` enforcement when `-compactor.blocks-retention-period` is not set, and viceversa. #8388
 * [BUGFIX] Ingester: fix sporadic `not found` error causing an internal server error if label names are queried with matchers during head compaction. #8391

diff --git a/cmd/mimir/config-descriptor.json b/cmd/mimir/config-descriptor.json
@@ -7192,6 +7192,36 @@
           "required": false,
           "desc": "",
           "blockEntries": [
+            {
+              "kind": "field",
+              "name": "application_credential_id",
+              "required": false,
+              "desc": "OpenStack Swift application credential id",
+              "fieldValue": null,
+              "fieldDefaultValue": "",
+              "fieldFlag": "blocks-storage.swift.application-credential-id",
+              "fieldType": "string"
+            },
+            {
+              "kind": "field",
+              "name": "application_credential_name",
+              "required": false,
+              "desc": "OpenStack Swift application credential name",
+              "fieldValue": null,
+              "fieldDefaultValue": "",
+              "fieldFlag": "blocks-storage.swift.application-credential-name",
+              "fieldType": "string"
+            },
+            {
+              "kind": "field",
+              "name": "application_credential_secret",
+              "required": false,
+              "desc": "OpenStack Swift application credential secret",
+              "fieldValue": null,
+              "fieldDefaultValue": "",
+              "fieldFlag": "blocks-storage.swift.application-credential-secret",
+              "fieldType": "string"
+            },
             {
               "kind": "field",
               "name": "auth_version",
@@ -13207,6 +13237,36 @@
           "required": false,
           "desc": "",
           "blockEntries": [
+            {
+              "kind": "field",
+              "name": "application_credential_id",
+              "required": false,
+              "desc": "OpenStack Swift application credential id",
+              "fieldValue": null,
+              "fieldDefaultValue": "",
+              "fieldFlag": "ruler-storage.swift.application-credential-id",
+              "fieldType": "string"
+            },
+            {
+              "kind": "field",
+              "name": "application_credential_name",
+              "required": false,
+              "desc": "OpenStack Swift application credential name",
+              "fieldValue": null,
+              "fieldDefaultValue": "",
+              "fieldFlag": "ruler-storage.swift.application-credential-name",
+              "fieldType": "string"
+            },
+            {
+              "kind": "field",
+              "name": "application_credential_secret",
+              "required": false,
+              "desc": "OpenStack Swift application credential secret",
+              "fieldValue": null,
+              "fieldDefaultValue": "",
+              "fieldFlag": "ruler-storage.swift.application-credential-secret",
+              "fieldType": "string"
+            },
             {
               "kind": "field",
               "name": "auth_version",
@@ -15400,6 +15460,36 @@
           "required": false,
           "desc": "",
           "blockEntries": [
+            {
+              "kind": "field",
+              "name": "application_credential_id",
+              "required": false,
+              "desc": "OpenStack Swift application credential id",
+              "fieldValue": null,
+              "fieldDefaultValue": "",
+              "fieldFlag": "alertmanager-storage.swift.application-credential-id",
+              "fieldType": "string"
+            },
+            {
+              "kind": "field",
+              "name": "application_credential_name",
+              "required": false,
+              "desc": "OpenStack Swift application credential name",
+              "fieldValue": null,
+              "fieldDefaultValue": "",
+              "fieldFlag": "alertmanager-storage.swift.application-credential-name",
+              "fieldType": "string"
+            },
+            {
+              "kind": "field",
+              "name": "application_credential_secret",
+              "required": false,
+              "desc": "OpenStack Swift application credential secret",
+              "fieldValue": null,
+              "fieldDefaultValue": "",
+              "fieldFlag": "alertmanager-storage.swift.application-credential-secret",
+              "fieldType": "string"
+            },
             {
               "kind": "field",
               "name": "auth_version",
@@ -17815,6 +17905,36 @@
               "required": false,
               "desc": "",
               "blockEntries": [
+                {
+                  "kind": "field",
+                  "name": "application_credential_id",
+                  "required": false,
+                  "desc": "OpenStack Swift application credential id",
+                  "fieldValue": null,
+                  "fieldDefaultValue": "",
+                  "fieldFlag": "common.storage.swift.application-credential-id",
+                  "fieldType": "string"
+                },
+                {
+                  "kind": "field",
+                  "name": "application_credential_name",
+                  "required": false,
+                  "desc": "OpenStack Swift application credential name",
+                  "fieldValue": null,
+                  "fieldDefaultValue": "",
+                  "fieldFlag": "common.storage.swift.application-credential-name",
+                  "fieldType": "string"
+                },
+                {
+                  "kind": "field",
+                  "name": "application_credential_secret",
+                  "required": false,
+                  "desc": "OpenStack Swift application credential secret",
+                  "fieldValue": null,
+                  "fieldDefaultValue": "",
+                  "fieldFlag": "common.storage.swift.application-credential-secret",
+                  "fieldType": "string"
+                },
                 {
                   "kind": "field",
                   "name": "auth_version",

diff --git a/cmd/mimir/help-all.txt.tmpl b/cmd/mimir/help-all.txt.tmpl
@@ -93,6 +93,12 @@ Usage of ./cmd/mimir/mimir:
     	When enabled, low-level S3 HTTP operation information is logged at the debug level.
   -alertmanager-storage.storage-prefix string
     	Prefix for all objects stored in the backend storage. For simplicity, it may only contain digits and English alphabet letters.
+  -alertmanager-storage.swift.application-credential-id string
+    	OpenStack Swift application credential id
+  -alertmanager-storage.swift.application-credential-name string
+    	OpenStack Swift application credential name
+  -alertmanager-storage.swift.application-credential-secret string
+    	OpenStack Swift application credential secret
   -alertmanager-storage.swift.auth-url string
     	OpenStack Swift authentication URL
   -alertmanager-storage.swift.auth-version int
@@ -761,6 +767,12 @@ Usage of ./cmd/mimir/mimir:
     	When enabled, low-level S3 HTTP operation information is logged at the debug level.
   -blocks-storage.storage-prefix string
     	Prefix for all objects stored in the backend storage. For simplicity, it may only contain digits and English alphabet letters.
+  -blocks-storage.swift.application-credential-id string
+    	OpenStack Swift application credential id
+  -blocks-storage.swift.application-credential-name string
+    	OpenStack Swift application credential name
+  -blocks-storage.swift.application-credential-secret string
+    	OpenStack Swift application credential secret
   -blocks-storage.swift.auth-url string
     	OpenStack Swift authentication URL
   -blocks-storage.swift.auth-version int
@@ -943,6 +955,12 @@ Usage of ./cmd/mimir/mimir:
     	Maximum time to wait for a TLS handshake. 0 means no limit. (default 10s)
   -common.storage.s3.trace.enabled
     	When enabled, low-level S3 HTTP operation information is logged at the debug level.
+  -common.storage.swift.application-credential-id string
+    	OpenStack Swift application credential id
+  -common.storage.swift.application-credential-name string
+    	OpenStack Swift application credential name
+  -common.storage.swift.application-credential-secret string
+    	OpenStack Swift application credential secret
   -common.storage.swift.auth-url string
     	OpenStack Swift authentication URL
   -common.storage.swift.auth-version int
@@ -2523,6 +2541,12 @@ Usage of ./cmd/mimir/mimir:
     	When enabled, low-level S3 HTTP operation information is logged at the debug level.
   -ruler-storage.storage-prefix string
     	Prefix for all objects stored in the backend storage. For simplicity, it may only contain digits and English alphabet letters.
+  -ruler-storage.swift.application-credential-id string
+    	OpenStack Swift application credential id
+  -ruler-storage.swift.application-credential-name string
+    	OpenStack Swift application credential name
+  -ruler-storage.swift.application-credential-secret string
+    	OpenStack Swift application credential secret
   -ruler-storage.swift.auth-url string
     	OpenStack Swift authentication URL
   -ruler-storage.swift.auth-version int

diff --git a/cmd/mimir/help.txt.tmpl b/cmd/mimir/help.txt.tmpl
@@ -43,6 +43,12 @@ Usage of ./cmd/mimir/mimir:
     	Accessing S3 resources using temporary, secure credentials provided by AWS Security Token Service.
   -alertmanager-storage.storage-prefix string
     	Prefix for all objects stored in the backend storage. For simplicity, it may only contain digits and English alphabet letters.
+  -alertmanager-storage.swift.application-credential-id string
+    	OpenStack Swift application credential id
+  -alertmanager-storage.swift.application-credential-name string
+    	OpenStack Swift application credential name
+  -alertmanager-storage.swift.application-credential-secret string
+    	OpenStack Swift application credential secret
   -alertmanager-storage.swift.auth-url string
     	OpenStack Swift authentication URL
   -alertmanager-storage.swift.auth-version int
@@ -209,6 +215,12 @@ Usage of ./cmd/mimir/mimir:
     	Accessing S3 resources using temporary, secure credentials provided by AWS Security Token Service.
   -blocks-storage.storage-prefix string
     	Prefix for all objects stored in the backend storage. For simplicity, it may only contain digits and English alphabet letters.
+  -blocks-storage.swift.application-credential-id string
+    	OpenStack Swift application credential id
+  -blocks-storage.swift.application-credential-name string
+    	OpenStack Swift application credential name
+  -blocks-storage.swift.application-credential-secret string
+    	OpenStack Swift application credential secret
   -blocks-storage.swift.auth-url string
     	OpenStack Swift authentication URL
   -blocks-storage.swift.auth-version int
@@ -281,6 +293,12 @@ Usage of ./cmd/mimir/mimir:
     	Enable AWS Server Side Encryption. Supported values: SSE-KMS, SSE-S3.
   -common.storage.s3.sts-endpoint string
     	Accessing S3 resources using temporary, secure credentials provided by AWS Security Token Service.
+  -common.storage.swift.application-credential-id string
+    	OpenStack Swift application credential id
+  -common.storage.swift.application-credential-name string
+    	OpenStack Swift application credential name
+  -common.storage.swift.application-credential-secret string
+    	OpenStack Swift application credential secret
   -common.storage.swift.auth-url string
     	OpenStack Swift authentication URL
   -common.storage.swift.auth-version int
@@ -665,6 +683,12 @@ Usage of ./cmd/mimir/mimir:
     	Accessing S3 resources using temporary, secure credentials provided by AWS Security Token Service.
   -ruler-storage.storage-prefix string
     	Prefix for all objects stored in the backend storage. For simplicity, it may only contain digits and English alphabet letters.
+  -ruler-storage.swift.application-credential-id string
+    	OpenStack Swift application credential id
+  -ruler-storage.swift.application-credential-name string
+    	OpenStack Swift application credential name
+  -ruler-storage.swift.application-credential-secret string
+    	OpenStack Swift application credential secret
   -ruler-storage.swift.auth-url string
     	OpenStack Swift authentication URL
   -ruler-storage.swift.auth-version int

diff --git a/docs/sources/mimir/configure/configuration-parameters/index.md b/docs/sources/mimir/configure/configuration-parameters/index.md
@@ -5265,6 +5265,18 @@ The `swift_storage_backend` block configures the connection to OpenStack Object
 &nbsp;
 
 ```yaml
+# OpenStack Swift application credential id
+# CLI flag: -<prefix>.swift.application-credential-id
+[application_credential_id: <string> | default = ""]
+
+# OpenStack Swift application credential name
+# CLI flag: -<prefix>.swift.application-credential-name
+[application_credential_name: <string> | default = ""]
+
+# OpenStack Swift application credential secret
+# CLI flag: -<prefix>.swift.application-credential-secret
+[application_credential_secret: <string> | default = ""]
+
 # OpenStack Swift authentication API version. 0 to autodetect.
 # CLI flag: -<prefix>.swift.auth-version
 [auth_version: <int> | default = 0]

diff --git a/pkg/storage/bucket/swift/bucket_client.go b/pkg/storage/bucket/swift/bucket_client.go
@@ -16,24 +16,27 @@ import (
 // NewBucketClient creates a new Swift bucket client
 func NewBucketClient(cfg Config, _ string, logger log.Logger) (objstore.Bucket, error) {
 	bucketConfig := swift.Config{
-		AuthVersion:       cfg.AuthVersion,
-		AuthUrl:           cfg.AuthURL,
-		Username:          cfg.Username,
-		UserDomainName:    cfg.UserDomainName,
-		UserDomainID:      cfg.UserDomainID,
-		UserId:            cfg.UserID,
-		Password:          cfg.Password.String(),
-		DomainId:          cfg.DomainID,
-		DomainName:        cfg.DomainName,
-		ProjectID:         cfg.ProjectID,
-		ProjectName:       cfg.ProjectName,
-		ProjectDomainID:   cfg.ProjectDomainID,
-		ProjectDomainName: cfg.ProjectDomainName,
-		RegionName:        cfg.RegionName,
-		ContainerName:     cfg.ContainerName,
-		Retries:           cfg.MaxRetries,
-		ConnectTimeout:    model.Duration(cfg.ConnectTimeout),
-		Timeout:           model.Duration(cfg.RequestTimeout),
+		ApplicationCredentialID:     cfg.ApplicationCredentialID,
+		ApplicationCredentialName:   cfg.ApplicationCredentialName,
+		ApplicationCredentialSecret: cfg.ApplicationCredentialSecret.String(),
+		AuthVersion:                 cfg.AuthVersion,
+		AuthUrl:                     cfg.AuthURL,
+		Username:                    cfg.Username,
+		UserDomainName:              cfg.UserDomainName,
+		UserDomainID:                cfg.UserDomainID,
+		UserId:                      cfg.UserID,
+		Password:                    cfg.Password.String(),
+		DomainId:                    cfg.DomainID,
+		DomainName:                  cfg.DomainName,
+		ProjectID:                   cfg.ProjectID,
+		ProjectName:                 cfg.ProjectName,
+		ProjectDomainID:             cfg.ProjectDomainID,
+		ProjectDomainName:           cfg.ProjectDomainName,
+		RegionName:                  cfg.RegionName,
+		ContainerName:               cfg.ContainerName,
+		Retries:                     cfg.MaxRetries,
+		ConnectTimeout:              model.Duration(cfg.ConnectTimeout),
+		Timeout:                     model.Duration(cfg.RequestTimeout),
 
 		// Hard-coded defaults.
 		ChunkSize:              swift.DefaultConfig.ChunkSize,

diff --git a/pkg/storage/bucket/swift/config.go b/pkg/storage/bucket/swift/config.go
@@ -14,24 +14,27 @@ import (
 
 // Config holds the config options for Swift backend
 type Config struct {
-	AuthVersion       int            `yaml:"auth_version"`
-	AuthURL           string         `yaml:"auth_url"`
-	Username          string         `yaml:"username"`
-	UserDomainName    string         `yaml:"user_domain_name"`
-	UserDomainID      string         `yaml:"user_domain_id"`
-	UserID            string         `yaml:"user_id"`
-	Password          flagext.Secret `yaml:"password"`
-	DomainID          string         `yaml:"domain_id"`
-	DomainName        string         `yaml:"domain_name"`
-	ProjectID         string         `yaml:"project_id"`
-	ProjectName       string         `yaml:"project_name"`
-	ProjectDomainID   string         `yaml:"project_domain_id"`
-	ProjectDomainName string         `yaml:"project_domain_name"`
-	RegionName        string         `yaml:"region_name"`
-	ContainerName     string         `yaml:"container_name"`
-	MaxRetries        int            `yaml:"max_retries" category:"advanced"`
-	ConnectTimeout    time.Duration  `yaml:"connect_timeout" category:"advanced"`
-	RequestTimeout    time.Duration  `yaml:"request_timeout" category:"advanced"`
+	ApplicationCredentialID     string         `yaml:"application_credential_id"`
+	ApplicationCredentialName   string         `yaml:"application_credential_name"`
+	ApplicationCredentialSecret flagext.Secret `yaml:"application_credential_secret"`
+	AuthVersion                 int            `yaml:"auth_version"`
+	AuthURL                     string         `yaml:"auth_url"`
+	Username                    string         `yaml:"username"`
+	UserDomainName              string         `yaml:"user_domain_name"`
+	UserDomainID                string         `yaml:"user_domain_id"`
+	UserID                      string         `yaml:"user_id"`
+	Password                    flagext.Secret `yaml:"password"`
+	DomainID                    string         `yaml:"domain_id"`
+	DomainName                  string         `yaml:"domain_name"`
+	ProjectID                   string         `yaml:"project_id"`
+	ProjectName                 string         `yaml:"project_name"`
+	ProjectDomainID             string         `yaml:"project_domain_id"`
+	ProjectDomainName           string         `yaml:"project_domain_name"`
+	RegionName                  string         `yaml:"region_name"`
+	ContainerName               string         `yaml:"container_name"`
+	MaxRetries                  int            `yaml:"max_retries" category:"advanced"`
+	ConnectTimeout              time.Duration  `yaml:"connect_timeout" category:"advanced"`
+	RequestTimeout              time.Duration  `yaml:"request_timeout" category:"advanced"`
 }
 
 // RegisterFlags registers the flags for Swift storage
@@ -41,6 +44,9 @@ func (cfg *Config) RegisterFlags(f *flag.FlagSet) {
 
 // RegisterFlagsWithPrefix registers the flags for Swift storage with the provided prefix
 func (cfg *Config) RegisterFlagsWithPrefix(prefix string, f *flag.FlagSet) {
+	f.StringVar(&cfg.ApplicationCredentialID, prefix+"swift.application-credential-id", "", "OpenStack Swift application credential id")
+	f.StringVar(&cfg.ApplicationCredentialName, prefix+"swift.application-credential-name", "", "OpenStack Swift application credential name")
+	f.Var(&cfg.ApplicationCredentialSecret, prefix+"swift.application-credential-secret", "OpenStack Swift application credential secret")
 	f.IntVar(&cfg.AuthVersion, prefix+"swift.auth-version", 0, "OpenStack Swift authentication API version. 0 to autodetect.")
 	f.StringVar(&cfg.AuthURL, prefix+"swift.auth-url", "", "OpenStack Swift authentication URL")
 	f.StringVar(&cfg.Username, prefix+"swift.username", "", "OpenStack Swift username.")