From 06b6bf8196895dcd071d0bf9c3bee3545fb585d6 Mon Sep 17 00:00:00 2001 From: Vasiliy Rumyantsev <4119114+xBazilio@users.noreply.github.com> Date: Thu, 21 Dec 2023 18:25:09 +0300 Subject: [PATCH 1/5] query-frontend: Added support of auto_discovery for memcached Signed-off-by: Vasiliy Rumyantsev <4119114+xBazilio@users.noreply.github.com> --- .../cortex/chunk/cache/memcached_client.go | 24 +++++++++++++++---- pkg/queryfrontend/config.go | 1 + 2 files changed, 20 insertions(+), 5 deletions(-) diff --git a/internal/cortex/chunk/cache/memcached_client.go b/internal/cortex/chunk/cache/memcached_client.go index a2f7b6b88b..0a8850eb3f 100644 --- a/internal/cortex/chunk/cache/memcached_client.go +++ b/internal/cortex/chunk/cache/memcached_client.go @@ -20,6 +20,9 @@ import ( "github.com/prometheus/client_golang/prometheus/promauto" "github.com/sony/gobreaker" "github.com/thanos-io/thanos/pkg/discovery/dns" + memcacheDiscovery "github.com/thanos-io/thanos/pkg/discovery/memcache" + "github.com/thanos-io/thanos/pkg/extprom" + "github.com/thanos-io/thanos/pkg/httpconfig" ) // MemcachedClient interface exists for mocking memcacheClient. @@ -45,7 +48,7 @@ type memcachedClient struct { service string addresses []string - provider *dns.Provider + provider httpconfig.AddressProvider cbs map[ /*address*/ string]*gobreaker.CircuitBreaker cbFailures uint @@ -68,6 +71,7 @@ type MemcachedClientConfig struct { Host string `yaml:"host"` Service string `yaml:"service"` Addresses string `yaml:"addresses"` // EXPERIMENTAL. + AutoDicovery bool `yaml:"auto_dicovery"` Timeout time.Duration `yaml:"timeout"` MaxIdleConns int `yaml:"max_idle_conns"` MaxItemSize int `yaml:"max_item_size"` @@ -107,9 +111,19 @@ func NewMemcachedClient(cfg MemcachedClientConfig, name string, r prometheus.Reg client.Timeout = cfg.Timeout client.MaxIdleConns = cfg.MaxIdleConns - dnsProviderRegisterer := prometheus.WrapRegistererWithPrefix("cortex_", prometheus.WrapRegistererWith(prometheus.Labels{ - "name": name, - }, r)) + var addressProvider httpconfig.AddressProvider + if cfg.AutoDicovery { + addressProvider = memcacheDiscovery.NewProvider( + logger, + extprom.WrapRegistererWithPrefix("thanos_memcached_", r), + cfg.Timeout, + ) + } else { + dnsProviderRegisterer := prometheus.WrapRegistererWithPrefix("cortex_", prometheus.WrapRegistererWith(prometheus.Labels{ + "name": name, + }, r)) + addressProvider = dns.NewProvider(logger, dnsProviderRegisterer, dns.GolangResolverType) + } newClient := &memcachedClient{ name: name, @@ -118,7 +132,7 @@ func NewMemcachedClient(cfg MemcachedClientConfig, name string, r prometheus.Reg hostname: cfg.Host, service: cfg.Service, logger: logger, - provider: dns.NewProvider(logger, dnsProviderRegisterer, dns.GolangResolverType), + provider: addressProvider, cbs: make(map[string]*gobreaker.CircuitBreaker), cbFailures: cfg.CBFailures, cbInterval: cfg.CBInterval, diff --git a/pkg/queryfrontend/config.go b/pkg/queryfrontend/config.go index ba45f80d9e..2371878e4c 100644 --- a/pkg/queryfrontend/config.go +++ b/pkg/queryfrontend/config.go @@ -142,6 +142,7 @@ func NewCacheConfig(logger log.Logger, confContentYaml []byte) (*cortexcache.Con Timeout: config.Memcached.Timeout, MaxIdleConns: config.Memcached.MaxIdleConnections, Addresses: strings.Join(config.Memcached.Addresses, ","), + AutoDicovery: config.Memcached.AutoDiscovery, UpdateInterval: config.Memcached.DNSProviderUpdateInterval, MaxItemSize: int(config.Memcached.MaxItemSize), }, From 7e399dbf08380e022c8bee59d9dde9f6bda13ca4 Mon Sep 17 00:00:00 2001 From: Vasiliy Rumyantsev <4119114+xBazilio@users.noreply.github.com> Date: Thu, 21 Dec 2023 18:34:25 +0300 Subject: [PATCH 2/5] adjustments to build on main branch Signed-off-by: Vasiliy Rumyantsev <4119114+xBazilio@users.noreply.github.com> --- internal/cortex/chunk/cache/memcached_client.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/internal/cortex/chunk/cache/memcached_client.go b/internal/cortex/chunk/cache/memcached_client.go index 0a8850eb3f..a1538b4078 100644 --- a/internal/cortex/chunk/cache/memcached_client.go +++ b/internal/cortex/chunk/cache/memcached_client.go @@ -19,10 +19,10 @@ import ( "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promauto" "github.com/sony/gobreaker" + "github.com/thanos-io/thanos/pkg/clientconfig" "github.com/thanos-io/thanos/pkg/discovery/dns" memcacheDiscovery "github.com/thanos-io/thanos/pkg/discovery/memcache" "github.com/thanos-io/thanos/pkg/extprom" - "github.com/thanos-io/thanos/pkg/httpconfig" ) // MemcachedClient interface exists for mocking memcacheClient. @@ -48,7 +48,7 @@ type memcachedClient struct { service string addresses []string - provider httpconfig.AddressProvider + provider clientconfig.AddressProvider cbs map[ /*address*/ string]*gobreaker.CircuitBreaker cbFailures uint @@ -111,7 +111,7 @@ func NewMemcachedClient(cfg MemcachedClientConfig, name string, r prometheus.Reg client.Timeout = cfg.Timeout client.MaxIdleConns = cfg.MaxIdleConns - var addressProvider httpconfig.AddressProvider + var addressProvider clientconfig.AddressProvider if cfg.AutoDicovery { addressProvider = memcacheDiscovery.NewProvider( logger, From 5a6bcaf0db6d7ce1afc89eb8c1260312cb731e0b Mon Sep 17 00:00:00 2001 From: Vasiliy Rumyantsev <4119114+xBazilio@users.noreply.github.com> Date: Thu, 21 Dec 2023 18:37:50 +0300 Subject: [PATCH 3/5] CHANGELOG.md Signed-off-by: Vasiliy Rumyantsev <4119114+xBazilio@users.noreply.github.com> --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index df6e93e4cd..ae9048fac0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -31,6 +31,7 @@ We use *breaking :warning:* to mark changes that are not backward compatible (re ### Fixed +- [#7004](https://github.com/thanos-io/thanos/pull/7004) Query Frontend: support documented auto discovery for memcached - [#6817](https://github.com/thanos-io/thanos/pull/6817) Store Gateway: fix `matchersToPostingGroups` label values variable got shadowed bug. ### Added From 5cb309d00e672d639c97799ed101943d6362a7ef Mon Sep 17 00:00:00 2001 From: Vasiliy Rumyantsev <4119114+xBazilio@users.noreply.github.com> Date: Sat, 23 Dec 2023 10:58:55 +0300 Subject: [PATCH 4/5] typo fixed Signed-off-by: Vasiliy Rumyantsev <4119114+xBazilio@users.noreply.github.com> --- internal/cortex/chunk/cache/memcached_client.go | 4 ++-- pkg/queryfrontend/config.go | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/internal/cortex/chunk/cache/memcached_client.go b/internal/cortex/chunk/cache/memcached_client.go index a1538b4078..c5021176a1 100644 --- a/internal/cortex/chunk/cache/memcached_client.go +++ b/internal/cortex/chunk/cache/memcached_client.go @@ -71,7 +71,7 @@ type MemcachedClientConfig struct { Host string `yaml:"host"` Service string `yaml:"service"` Addresses string `yaml:"addresses"` // EXPERIMENTAL. - AutoDicovery bool `yaml:"auto_dicovery"` + AutoDiscovery bool `yaml:"auto_discovery"` Timeout time.Duration `yaml:"timeout"` MaxIdleConns int `yaml:"max_idle_conns"` MaxItemSize int `yaml:"max_item_size"` @@ -112,7 +112,7 @@ func NewMemcachedClient(cfg MemcachedClientConfig, name string, r prometheus.Reg client.MaxIdleConns = cfg.MaxIdleConns var addressProvider clientconfig.AddressProvider - if cfg.AutoDicovery { + if cfg.AutoDiscovery { addressProvider = memcacheDiscovery.NewProvider( logger, extprom.WrapRegistererWithPrefix("thanos_memcached_", r), diff --git a/pkg/queryfrontend/config.go b/pkg/queryfrontend/config.go index 2371878e4c..4b915764c5 100644 --- a/pkg/queryfrontend/config.go +++ b/pkg/queryfrontend/config.go @@ -142,7 +142,7 @@ func NewCacheConfig(logger log.Logger, confContentYaml []byte) (*cortexcache.Con Timeout: config.Memcached.Timeout, MaxIdleConns: config.Memcached.MaxIdleConnections, Addresses: strings.Join(config.Memcached.Addresses, ","), - AutoDicovery: config.Memcached.AutoDiscovery, + AutoDiscovery: config.Memcached.AutoDiscovery, UpdateInterval: config.Memcached.DNSProviderUpdateInterval, MaxItemSize: int(config.Memcached.MaxItemSize), }, From 9445f58818dec46684309760f9a2670923612961 Mon Sep 17 00:00:00 2001 From: Vasiliy Rumyantsev <4119114+xBazilio@users.noreply.github.com> Date: Mon, 25 Dec 2023 10:23:05 +0300 Subject: [PATCH 5/5] minor fixex after review Signed-off-by: Vasiliy Rumyantsev <4119114+xBazilio@users.noreply.github.com> --- CHANGELOG.md | 2 +- internal/cortex/chunk/cache/memcached_client.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ae9048fac0..84ac476be8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -31,7 +31,6 @@ We use *breaking :warning:* to mark changes that are not backward compatible (re ### Fixed -- [#7004](https://github.com/thanos-io/thanos/pull/7004) Query Frontend: support documented auto discovery for memcached - [#6817](https://github.com/thanos-io/thanos/pull/6817) Store Gateway: fix `matchersToPostingGroups` label values variable got shadowed bug. ### Added @@ -39,6 +38,7 @@ We use *breaking :warning:* to mark changes that are not backward compatible (re - [#6605](https://github.com/thanos-io/thanos/pull/6605) Query Frontend: Support vertical sharding binary expression with metric name when no matching labels specified. - [#6308](https://github.com/thanos-io/thanos/pull/6308) Ruler: Support configuration flag that allows customizing template for alert message. - [#6760](https://github.com/thanos-io/thanos/pull/6760) Query Frontend: Added TLS support in `--query-frontend.downstream-tripper-config` and `--query-frontend.downstream-tripper-config-file` +- [#7004](https://github.com/thanos-io/thanos/pull/7004) Query Frontend: Support documented auto discovery for memcached - [#6749](https://github.com/thanos-io/thanos/pull/6749) Store Gateway: Added `thanos_store_index_cache_fetch_duration_seconds` histogram for tracking latency of fetching data from index cache. - [#6690](https://github.com/thanos-io/thanos/pull/6690) Store: *breaking :warning:* Add tenant label to relevant exported metrics. Note that this change may cause some pre-existing dashboard queries to be incorrect due to the added label. - [#6530](https://github.com/thanos-io/thanos/pull/6530) / [#6690](https://github.com/thanos-io/thanos/pull/6690) Query: Add command line arguments for configuring tenants and forward tenant information to Store Gateway. diff --git a/internal/cortex/chunk/cache/memcached_client.go b/internal/cortex/chunk/cache/memcached_client.go index c5021176a1..9455f76400 100644 --- a/internal/cortex/chunk/cache/memcached_client.go +++ b/internal/cortex/chunk/cache/memcached_client.go @@ -115,7 +115,7 @@ func NewMemcachedClient(cfg MemcachedClientConfig, name string, r prometheus.Reg if cfg.AutoDiscovery { addressProvider = memcacheDiscovery.NewProvider( logger, - extprom.WrapRegistererWithPrefix("thanos_memcached_", r), + extprom.WrapRegistererWithPrefix("cortex_", r), cfg.Timeout, ) } else {