Upgrade objstore and change Azure bucket client (cortexproject#5645)

* upgrade objstore and change Azure bucket clinet

Signed-off-by: Ben Ye <[email protected]>

* update pr number

Signed-off-by: Ben Ye <[email protected]>

* fix doc again

Signed-off-by: Ben Ye <[email protected]>

---------

Signed-off-by: Ben Ye <[email protected]>

CHANGELOG.md

@@ -1,8 +1,9 @@
 # Changelog
 
 ## master / unreleased
-* [ENHANCEMENT] Store Gateway: Added `-store-gateway.enabled-tenants` and `-store-gateway.disabled-tenants` to explicitly enable or disable store-gateway for specific tenants. #5638
+* [CHANGE] Azure Storage: Upgraded objstore dependency and support Azure Workload Identity Authentication. Added `connection_string` to support authenticating via SAS token. Marked `msi_resource` config as deprecating. #5645
 * [FEATURE] Ingester: Add per-tenant new metric `cortex_ingester_tsdb_data_replay_duration_seconds`. #5477
+* [ENHANCEMENT] Store Gateway: Added `-store-gateway.enabled-tenants` and `-store-gateway.disabled-tenants` to explicitly enable or disable store-gateway for specific tenants. #5638
 
 ## 1.16.0 in progress
 

docs/blocks-storage/querier.md

@@ -339,6 +339,13 @@ blocks_storage:
     # CLI flag: -blocks-storage.azure.account-key
     [account_key: <string> | default = ""]
 
+    # The values of `account-name` and `endpoint-suffix` values will not be
+    # ignored if `connection-string` is set. Use this method over `account-key`
+    # if you need to authenticate via a SAS token or if you use the Azurite
+    # emulator.
+    # CLI flag: -blocks-storage.azure.connection-string
+    [connection_string: <string> | default = ""]
+
     # Azure storage container name
     # CLI flag: -blocks-storage.azure.container-name
     [container_name: <string> | default = ""]
@@ -352,12 +359,14 @@ blocks_storage:
     # CLI flag: -blocks-storage.azure.max-retries
     [max_retries: <int> | default = 20]
 
-    # Azure storage MSI resource. Either this or account key must be set.
+    # Deprecated: Azure storage MSI resource. It will be set automatically by
+    # Azure SDK.
     # CLI flag: -blocks-storage.azure.msi-resource
     [msi_resource: <string> | default = ""]
 
     # Azure storage MSI resource managed identity client Id. If not supplied
-    # system assigned identity is used
+    # default Azure credential will be used. Set it to empty if you need to
+    # authenticate via Azure Workload Identity.
     # CLI flag: -blocks-storage.azure.user-assigned-id
     [user_assigned_id: <string> | default = ""]
 

docs/blocks-storage/store-gateway.md

@@ -459,6 +459,13 @@ blocks_storage:
     # CLI flag: -blocks-storage.azure.account-key
     [account_key: <string> | default = ""]
 
+    # The values of `account-name` and `endpoint-suffix` values will not be
+    # ignored if `connection-string` is set. Use this method over `account-key`
+    # if you need to authenticate via a SAS token or if you use the Azurite
+    # emulator.
+    # CLI flag: -blocks-storage.azure.connection-string
+    [connection_string: <string> | default = ""]
+
     # Azure storage container name
     # CLI flag: -blocks-storage.azure.container-name
     [container_name: <string> | default = ""]
@@ -472,12 +479,14 @@ blocks_storage:
     # CLI flag: -blocks-storage.azure.max-retries
     [max_retries: <int> | default = 20]
 
-    # Azure storage MSI resource. Either this or account key must be set.
+    # Deprecated: Azure storage MSI resource. It will be set automatically by
+    # Azure SDK.
     # CLI flag: -blocks-storage.azure.msi-resource
     [msi_resource: <string> | default = ""]
 
     # Azure storage MSI resource managed identity client Id. If not supplied
-    # system assigned identity is used
+    # default Azure credential will be used. Set it to empty if you need to
+    # authenticate via Azure Workload Identity.
     # CLI flag: -blocks-storage.azure.user-assigned-id
     [user_assigned_id: <string> | default = ""]
 

docs/configuration/config-file-reference.md

@@ -621,6 +621,12 @@ azure:
   # CLI flag: -alertmanager-storage.azure.account-key
   [account_key: <string> | default = ""]
 
+  # The values of `account-name` and `endpoint-suffix` values will not be
+  # ignored if `connection-string` is set. Use this method over `account-key` if
+  # you need to authenticate via a SAS token or if you use the Azurite emulator.
+  # CLI flag: -alertmanager-storage.azure.connection-string
+  [connection_string: <string> | default = ""]
+
   # Azure storage container name
   # CLI flag: -alertmanager-storage.azure.container-name
   [container_name: <string> | default = ""]
@@ -634,12 +640,14 @@ azure:
   # CLI flag: -alertmanager-storage.azure.max-retries
   [max_retries: <int> | default = 20]
 
-  # Azure storage MSI resource. Either this or account key must be set.
+  # Deprecated: Azure storage MSI resource. It will be set automatically by
+  # Azure SDK.
   # CLI flag: -alertmanager-storage.azure.msi-resource
   [msi_resource: <string> | default = ""]
 
   # Azure storage MSI resource managed identity client Id. If not supplied
-  # system assigned identity is used
+  # default Azure credential will be used. Set it to empty if you need to
+  # authenticate via Azure Workload Identity.
   # CLI flag: -alertmanager-storage.azure.user-assigned-id
   [user_assigned_id: <string> | default = ""]
 
@@ -886,6 +894,12 @@ azure:
   # CLI flag: -blocks-storage.azure.account-key
   [account_key: <string> | default = ""]
 
+  # The values of `account-name` and `endpoint-suffix` values will not be
+  # ignored if `connection-string` is set. Use this method over `account-key` if
+  # you need to authenticate via a SAS token or if you use the Azurite emulator.
+  # CLI flag: -blocks-storage.azure.connection-string
+  [connection_string: <string> | default = ""]
+
   # Azure storage container name
   # CLI flag: -blocks-storage.azure.container-name
   [container_name: <string> | default = ""]
@@ -899,12 +913,14 @@ azure:
   # CLI flag: -blocks-storage.azure.max-retries
   [max_retries: <int> | default = 20]
 
-  # Azure storage MSI resource. Either this or account key must be set.
+  # Deprecated: Azure storage MSI resource. It will be set automatically by
+  # Azure SDK.
   # CLI flag: -blocks-storage.azure.msi-resource
   [msi_resource: <string> | default = ""]
 
   # Azure storage MSI resource managed identity client Id. If not supplied
-  # system assigned identity is used
+  # default Azure credential will be used. Set it to empty if you need to
+  # authenticate via Azure Workload Identity.
   # CLI flag: -blocks-storage.azure.user-assigned-id
   [user_assigned_id: <string> | default = ""]
 
@@ -4142,6 +4158,12 @@ azure:
   # CLI flag: -ruler-storage.azure.account-key
   [account_key: <string> | default = ""]
 
+  # The values of `account-name` and `endpoint-suffix` values will not be
+  # ignored if `connection-string` is set. Use this method over `account-key` if
+  # you need to authenticate via a SAS token or if you use the Azurite emulator.
+  # CLI flag: -ruler-storage.azure.connection-string
+  [connection_string: <string> | default = ""]
+
   # Azure storage container name
   # CLI flag: -ruler-storage.azure.container-name
   [container_name: <string> | default = ""]
@@ -4155,12 +4177,14 @@ azure:
   # CLI flag: -ruler-storage.azure.max-retries
   [max_retries: <int> | default = 20]
 
-  # Azure storage MSI resource. Either this or account key must be set.
+  # Deprecated: Azure storage MSI resource. It will be set automatically by
+  # Azure SDK.
   # CLI flag: -ruler-storage.azure.msi-resource
   [msi_resource: <string> | default = ""]
 
   # Azure storage MSI resource managed identity client Id. If not supplied
-  # system assigned identity is used
+  # default Azure credential will be used. Set it to empty if you need to
+  # authenticate via Azure Workload Identity.
   # CLI flag: -ruler-storage.azure.user-assigned-id
   [user_assigned_id: <string> | default = ""]
 
@@ -4415,6 +4439,12 @@ azure:
   # CLI flag: -runtime-config.azure.account-key
   [account_key: <string> | default = ""]
 
+  # The values of `account-name` and `endpoint-suffix` values will not be
+  # ignored if `connection-string` is set. Use this method over `account-key` if
+  # you need to authenticate via a SAS token or if you use the Azurite emulator.
+  # CLI flag: -runtime-config.azure.connection-string
+  [connection_string: <string> | default = ""]
+
   # Azure storage container name
   # CLI flag: -runtime-config.azure.container-name
   [container_name: <string> | default = ""]
@@ -4428,12 +4458,14 @@ azure:
   # CLI flag: -runtime-config.azure.max-retries
   [max_retries: <int> | default = 20]
 
-  # Azure storage MSI resource. Either this or account key must be set.
+  # Deprecated: Azure storage MSI resource. It will be set automatically by
+  # Azure SDK.
   # CLI flag: -runtime-config.azure.msi-resource
   [msi_resource: <string> | default = ""]
 
   # Azure storage MSI resource managed identity client Id. If not supplied
-  # system assigned identity is used
+  # default Azure credential will be used. Set it to empty if you need to
+  # authenticate via Azure Workload Identity.
   # CLI flag: -runtime-config.azure.user-assigned-id
   [user_assigned_id: <string> | default = ""]
 

go.mod

@@ -51,7 +51,7 @@ require (
 	github.com/sony/gobreaker v0.5.0
 	github.com/spf13/afero v1.9.5
 	github.com/stretchr/testify v1.8.4
-	github.com/thanos-io/objstore v0.0.0-20230921130928-63a603e651ed
+	github.com/thanos-io/objstore v0.0.0-20231112185854-37752ee64d98
 	github.com/thanos-io/promql-engine v0.0.0-20231013104847-4517c0d5f591
 	github.com/thanos-io/thanos v0.32.5-0.20231103115946-463a6ce8b53c
 	github.com/uber/jaeger-client-go v2.30.0+incompatible
@@ -89,8 +89,8 @@ require (
 	cloud.google.com/go/compute/metadata v0.2.3 // indirect
 	cloud.google.com/go/iam v1.1.1 // indirect
 	cloud.google.com/go/storage v1.30.1 // indirect
-	github.com/Azure/azure-sdk-for-go/sdk/azcore v1.7.1 // indirect
-	github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.3.1 // indirect
+	github.com/Azure/azure-sdk-for-go/sdk/azcore v1.8.0 // indirect
+	github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.4.0 // indirect
 	github.com/Azure/azure-sdk-for-go/sdk/internal v1.3.0 // indirect
 	github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.0.0 // indirect
 	github.com/AzureAD/microsoft-authentication-library-for-go v1.1.1 // indirect
@@ -143,7 +143,7 @@ require (
 	github.com/google/btree v1.0.1 // indirect
 	github.com/google/pprof v0.0.0-20230705174524-200ffdc848b8 // indirect
 	github.com/google/s2a-go v0.1.4 // indirect
-	github.com/google/uuid v1.3.0 // indirect
+	github.com/google/uuid v1.3.1 // indirect
 	github.com/googleapis/enterprise-certificate-proxy v0.2.5 // indirect
 	github.com/googleapis/gax-go/v2 v2.12.0 // indirect
 	github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.0.0-rc.2.0.20201207153454-9f6bf00c00a7 // indirect

go.sum

@@ -605,10 +605,10 @@ gioui.org v0.0.0-20210308172011-57750fc8a0a6/go.mod h1:RSH6KIUZ0p2xy5zHDxgAM4zum
 git.sr.ht/~sbinet/gg v0.3.1/go.mod h1:KGYtlADtqsqANL9ueOFkWymvzUvLMQllU5Ixo+8v3pc=
 github.com/Azure/azure-sdk-for-go v65.0.0+incompatible h1:HzKLt3kIwMm4KeJYTdx9EbjRYTySD/t8i1Ee/W5EGXw=
 github.com/Azure/azure-sdk-for-go v65.0.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc=
-github.com/Azure/azure-sdk-for-go/sdk/azcore v1.7.1 h1:/iHxaJhsFr0+xVFfbMr5vxz848jyiWuIEDhYq3y5odY=
-github.com/Azure/azure-sdk-for-go/sdk/azcore v1.7.1/go.mod h1:bjGvMhVMb+EEm3VRNQawDMUyMMjo+S5ewNjflkep/0Q=
-github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.3.1 h1:LNHhpdK7hzUcx/k1LIcuh5k7k1LGIWLQfCjaneSj7Fc=
-github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.3.1/go.mod h1:uE9zaUfEQT/nbQjVi2IblCG9iaLtZsuYZ8ne+PuQ02M=
+github.com/Azure/azure-sdk-for-go/sdk/azcore v1.8.0 h1:9kDVnTz3vbfweTqAUmk/a/pH5pWFCHtvRpHYC0G/dcA=
+github.com/Azure/azure-sdk-for-go/sdk/azcore v1.8.0/go.mod h1:3Ug6Qzto9anB6mGlEdgYMDF5zHQ+wwhEaYR4s17PHMw=
+github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.4.0 h1:BMAjVKJM0U/CYF27gA0ZMmXGkOcvfFtD0oHVZ1TIPRI=
+github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.4.0/go.mod h1:1fXstnBMas5kzG+S3q8UoJcmyU6nUeunJcMDHcRYHhs=
 github.com/Azure/azure-sdk-for-go/sdk/internal v1.3.0 h1:sXr+ck84g/ZlZUOZiNELInmMgOsuGwdjjVkEIde0OtY=
 github.com/Azure/azure-sdk-for-go/sdk/internal v1.3.0/go.mod h1:okt5dMMTOFjX/aovMlrjvvXoPMBVSPzk9185BT0+eZM=
 github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.0.0 h1:u/LLAOFgsMv7HmNL4Qufg58y+qElGOt5qv0z1mURkRY=
@@ -1068,8 +1068,9 @@ github.com/google/s2a-go v0.1.4 h1:1kZ/sQM3srePvKs3tXAvQzo66XfcReoqFpIpIccE7Oc=
 github.com/google/s2a-go v0.1.4/go.mod h1:Ej+mSEMGRnqRzjc7VtF+jdBwYG5fuJfiZ8ELkjEwM0A=
 github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
-github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I=
 github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/google/uuid v1.3.1 h1:KjJaJ9iWZ3jOFZIf1Lqf4laDRCasjl0BCmnEGxkdLb4=
+github.com/google/uuid v1.3.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/googleapis/enterprise-certificate-proxy v0.0.0-20220520183353-fd19c99a87aa/go.mod h1:17drOmN3MwGY7t0e+Ei9b45FFGA3fBs3x36SsCg1hq8=
 github.com/googleapis/enterprise-certificate-proxy v0.1.0/go.mod h1:17drOmN3MwGY7t0e+Ei9b45FFGA3fBs3x36SsCg1hq8=
 github.com/googleapis/enterprise-certificate-proxy v0.2.0/go.mod h1:8C0jb7/mgJe/9KK8Lm7X9ctZC2t60YyIpYEI16jx0Qg=
@@ -1514,8 +1515,8 @@ github.com/tencentyun/cos-go-sdk-v5 v0.7.40 h1:W6vDGKCHe4wBACI1d2UgE6+50sJFhRWU4
 github.com/tencentyun/cos-go-sdk-v5 v0.7.40/go.mod h1:4dCEtLHGh8QPxHEkgq+nFaky7yZxQuYwgSJM87icDaw=
 github.com/thanos-community/galaxycache v0.0.0-20211122094458-3a32041a1f1e h1:f1Zsv7OAU9iQhZwigp50Yl38W10g/vd5NC8Rdk1Jzng=
 github.com/thanos-community/galaxycache v0.0.0-20211122094458-3a32041a1f1e/go.mod h1:jXcofnrSln/cLI6/dhlBxPQZEEQHVPCcFaH75M+nSzM=
-github.com/thanos-io/objstore v0.0.0-20230921130928-63a603e651ed h1:iWQdY3S6DpWjelVvKKSKgS7LeLkhK4VaEnQfphB9ZXA=
-github.com/thanos-io/objstore v0.0.0-20230921130928-63a603e651ed/go.mod h1:oJ82xgcBDzGJrEgUsjlTj6n01+ZWUMMUR8BlZzX5xDE=
+github.com/thanos-io/objstore v0.0.0-20231112185854-37752ee64d98 h1:gx2MTto1UQRumGoJzY3aFPQ31Ov3nOV7NaD7j6q288k=
+github.com/thanos-io/objstore v0.0.0-20231112185854-37752ee64d98/go.mod h1:JauBAcJ61tRSv9widgISVmA6akQXDeUMXBrVmWW4xog=
 github.com/thanos-io/promql-engine v0.0.0-20231013104847-4517c0d5f591 h1:6bZbFM+Mvy2kL8BeL8TJ5+5pV3sUR2PSLaZyw911rtQ=
 github.com/thanos-io/promql-engine v0.0.0-20231013104847-4517c0d5f591/go.mod h1:vfXJv1JXNdLfHnjsHsLLJl5tyI7KblF76Wo5lZ9YC4Q=
 github.com/thanos-io/thanos v0.32.5-0.20231103115946-463a6ce8b53c h1:hMpXd1ybZB/vnR3+zex93va42rQ++2E0qi2wVSf3AwY=

pkg/storage/bucket/azure/bucket_client.go

@@ -4,20 +4,21 @@ import (
 	"github.com/go-kit/log"
 	"github.com/prometheus/common/model"
 	"github.com/thanos-io/objstore"
+	"github.com/thanos-io/objstore/exthttp"
 	"github.com/thanos-io/objstore/providers/azure"
 	yaml "gopkg.in/yaml.v2"
 )
 
 func NewBucketClient(cfg Config, name string, logger log.Logger) (objstore.Bucket, error) {
 	bucketConfig := azure.Config{
-		StorageAccountName: cfg.StorageAccountName,
-		StorageAccountKey:  cfg.StorageAccountKey.Value,
-		ContainerName:      cfg.ContainerName,
-		Endpoint:           cfg.Endpoint,
-		MaxRetries:         cfg.MaxRetries,
-		MSIResource:        cfg.MSIResource,
-		UserAssignedID:     cfg.UserAssignedID,
-		HTTPConfig: azure.HTTPConfig{
+		StorageAccountName:      cfg.StorageAccountName,
+		StorageAccountKey:       cfg.StorageAccountKey.Value,
+		StorageConnectionString: cfg.StorageConnectionString.Value,
+		ContainerName:           cfg.ContainerName,
+		Endpoint:                cfg.Endpoint,
+		MaxRetries:              cfg.MaxRetries,
+		UserAssignedID:          cfg.UserAssignedID,
+		HTTPConfig: exthttp.HTTPConfig{
 			IdleConnTimeout:       model.Duration(cfg.IdleConnTimeout),
 			ResponseHeaderTimeout: model.Duration(cfg.ResponseHeaderTimeout),
 			InsecureSkipVerify:    cfg.InsecureSkipVerify,

pkg/storage/bucket/azure/config.go

@@ -9,13 +9,15 @@ import (
 
 // Config holds the config options for an Azure backend
 type Config struct {
-	StorageAccountName string         `yaml:"account_name"`
-	StorageAccountKey  flagext.Secret `yaml:"account_key"`
-	ContainerName      string         `yaml:"container_name"`
-	Endpoint           string         `yaml:"endpoint_suffix"`
-	MaxRetries         int            `yaml:"max_retries"`
-	MSIResource        string         `yaml:"msi_resource"`
-	UserAssignedID     string         `yaml:"user_assigned_id"`
+	StorageAccountName      string         `yaml:"account_name"`
+	StorageAccountKey       flagext.Secret `yaml:"account_key"`
+	StorageConnectionString flagext.Secret `yaml:"connection_string"`
+	ContainerName           string         `yaml:"container_name"`
+	Endpoint                string         `yaml:"endpoint_suffix"`
+	MaxRetries              int            `yaml:"max_retries"`
+	// Deprecated: set automatically by Azure SDK.
+	MSIResource    string `yaml:"msi_resource"`
+	UserAssignedID string `yaml:"user_assigned_id"`
 
 	http.Config `yaml:"http"`
 }
@@ -29,10 +31,11 @@ func (cfg *Config) RegisterFlags(f *flag.FlagSet) {
 func (cfg *Config) RegisterFlagsWithPrefix(prefix string, f *flag.FlagSet) {
 	f.StringVar(&cfg.StorageAccountName, prefix+"azure.account-name", "", "Azure storage account name")
 	f.Var(&cfg.StorageAccountKey, prefix+"azure.account-key", "Azure storage account key")
+	f.Var(&cfg.StorageConnectionString, prefix+"azure.connection-string", "The values of `account-name` and `endpoint-suffix` values will not be ignored if `connection-string` is set. Use this method over `account-key` if you need to authenticate via a SAS token or if you use the Azurite emulator.")
 	f.StringVar(&cfg.ContainerName, prefix+"azure.container-name", "", "Azure storage container name")
 	f.StringVar(&cfg.Endpoint, prefix+"azure.endpoint-suffix", "", "Azure storage endpoint suffix without schema. The account name will be prefixed to this value to create the FQDN")
 	f.IntVar(&cfg.MaxRetries, prefix+"azure.max-retries", 20, "Number of retries for recoverable errors")
-	f.StringVar(&cfg.MSIResource, prefix+"azure.msi-resource", "", "Azure storage MSI resource. Either this or account key must be set.")
-	f.StringVar(&cfg.UserAssignedID, prefix+"azure.user-assigned-id", "", "Azure storage MSI resource managed identity client Id. If not supplied system assigned identity is used")
+	f.StringVar(&cfg.MSIResource, prefix+"azure.msi-resource", "", "Deprecated: Azure storage MSI resource. It will be set automatically by Azure SDK.")
+	f.StringVar(&cfg.UserAssignedID, prefix+"azure.user-assigned-id", "", "Azure storage MSI resource managed identity client Id. If not supplied default Azure credential will be used. Set it to empty if you need to authenticate via Azure Workload Identity.")
 	cfg.Config.RegisterFlagsWithPrefix(prefix+"azure.", f)
 }

pkg/storage/bucket/azure/config_test.go

@@ -45,7 +45,6 @@ account_name: test-account-name
 account_key: test-account-key
 container_name: test-container-name
 endpoint_suffix: test-endpoint-suffix
-msi_resource: test-msi-resource
 user_assigned_id: test-user-assigned-id
 max_retries: 1
 http:
@@ -63,7 +62,6 @@ http:
 				StorageAccountKey:  flagext.Secret{Value: "test-account-key"},
 				ContainerName:      "test-container-name",
 				Endpoint:           "test-endpoint-suffix",
-				MSIResource:        "test-msi-resource",
 				UserAssignedID:     "test-user-assigned-id",
 				MaxRetries:         1,
 				Config: http.Config{