Skip to content

Commit

Permalink
Support filtered index cache (cortexproject#5587)
Browse files Browse the repository at this point in the history 
* filtered cache support for Cortex

Signed-off-by: Ben Ye <[email protected]>

* update thanos version

Signed-off-by: Ben Ye <[email protected]>

---------

Signed-off-by: Ben Ye <[email protected]>
  • Loading branch information
@yeya24
yeya24 authored Oct 6, 2023
1 parent 7812330 commit 577045c
Show file tree
Hide file tree
Showing 21 changed files with 772 additions and 285 deletions.
15 changes: 15 additions & 0 deletions docs/blocks-storage/querier.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -536,6 +536,11 @@ blocks_storage:
# CLI flag: -blocks-storage.bucket-store.index-cache.inmemory.max-size-bytes
[max_size_bytes: <int> | default = 1073741824]
# Selectively cache index item types. Supported values are Postings,
# ExpandedPostings and Series
# CLI flag: -blocks-storage.bucket-store.index-cache.inmemory.enabled-items
[enabled_items: <list of string> | default = []]
memcached:
# Comma separated list of memcached addresses. Supported prefixes are:
# dns+ (looked up as an A/AAAA query), dnssrv+ (looked up as a SRV
Expand Down Expand Up @@ -583,6 +588,11 @@ blocks_storage:
# CLI flag: -blocks-storage.bucket-store.index-cache.memcached.auto-discovery
[auto_discovery: <boolean> | default = false]
# Selectively cache index item types. Supported values are Postings,
# ExpandedPostings and Series
# CLI flag: -blocks-storage.bucket-store.index-cache.memcached.enabled-items
[enabled_items: <list of string> | default = []]
redis:
# Comma separated list of redis addresses. Supported prefixes are: dns+
# (looked up as an A/AAAA query), dnssrv+ (looked up as a SRV query,
Expand Down Expand Up @@ -679,6 +689,11 @@ blocks_storage:
# CLI flag: -blocks-storage.bucket-store.index-cache.redis.cache-size
[cache_size: <int> | default = 0]
# Selectively cache index item types. Supported values are Postings,
# ExpandedPostings and Series
# CLI flag: -blocks-storage.bucket-store.index-cache.redis.enabled-items
[enabled_items: <list of string> | default = []]
chunks_cache:
# Backend for chunks cache, if not empty. Supported values: memcached.
# CLI flag: -blocks-storage.bucket-store.chunks-cache.backend
Expand Down
15 changes: 15 additions & 0 deletions docs/blocks-storage/store-gateway.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -643,6 +643,11 @@ blocks_storage:
# CLI flag: -blocks-storage.bucket-store.index-cache.inmemory.max-size-bytes
[max_size_bytes: <int> | default = 1073741824]
# Selectively cache index item types. Supported values are Postings,
# ExpandedPostings and Series
# CLI flag: -blocks-storage.bucket-store.index-cache.inmemory.enabled-items
[enabled_items: <list of string> | default = []]
memcached:
# Comma separated list of memcached addresses. Supported prefixes are:
# dns+ (looked up as an A/AAAA query), dnssrv+ (looked up as a SRV
Expand Down Expand Up @@ -690,6 +695,11 @@ blocks_storage:
# CLI flag: -blocks-storage.bucket-store.index-cache.memcached.auto-discovery
[auto_discovery: <boolean> | default = false]
# Selectively cache index item types. Supported values are Postings,
# ExpandedPostings and Series
# CLI flag: -blocks-storage.bucket-store.index-cache.memcached.enabled-items
[enabled_items: <list of string> | default = []]
redis:
# Comma separated list of redis addresses. Supported prefixes are: dns+
# (looked up as an A/AAAA query), dnssrv+ (looked up as a SRV query,
Expand Down Expand Up @@ -786,6 +796,11 @@ blocks_storage:
# CLI flag: -blocks-storage.bucket-store.index-cache.redis.cache-size
[cache_size: <int> | default = 0]
# Selectively cache index item types. Supported values are Postings,
# ExpandedPostings and Series
# CLI flag: -blocks-storage.bucket-store.index-cache.redis.enabled-items
[enabled_items: <list of string> | default = []]
chunks_cache:
# Backend for chunks cache, if not empty. Supported values: memcached.
# CLI flag: -blocks-storage.bucket-store.chunks-cache.backend
Expand Down
15 changes: 15 additions & 0 deletions docs/configuration/config-file-reference.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1083,6 +1083,11 @@ bucket_store:
# CLI flag: -blocks-storage.bucket-store.index-cache.inmemory.max-size-bytes
[max_size_bytes: <int> | default = 1073741824]
# Selectively cache index item types. Supported values are Postings,
# ExpandedPostings and Series
# CLI flag: -blocks-storage.bucket-store.index-cache.inmemory.enabled-items
[enabled_items: <list of string> | default = []]
memcached:
# Comma separated list of memcached addresses. Supported prefixes are:
# dns+ (looked up as an A/AAAA query), dnssrv+ (looked up as a SRV query,
Expand Down Expand Up @@ -1130,6 +1135,11 @@ bucket_store:
# CLI flag: -blocks-storage.bucket-store.index-cache.memcached.auto-discovery
[auto_discovery: <boolean> | default = false]
# Selectively cache index item types. Supported values are Postings,
# ExpandedPostings and Series
# CLI flag: -blocks-storage.bucket-store.index-cache.memcached.enabled-items
[enabled_items: <list of string> | default = []]
redis:
# Comma separated list of redis addresses. Supported prefixes are: dns+
# (looked up as an A/AAAA query), dnssrv+ (looked up as a SRV query,
Expand Down Expand Up @@ -1226,6 +1236,11 @@ bucket_store:
# CLI flag: -blocks-storage.bucket-store.index-cache.redis.cache-size
[cache_size: <int> | default = 0]
# Selectively cache index item types. Supported values are Postings,
# ExpandedPostings and Series
# CLI flag: -blocks-storage.bucket-store.index-cache.redis.enabled-items
[enabled_items: <list of string> | default = []]
chunks_cache:
# Backend for chunks cache, if not empty. Supported values: memcached.
# CLI flag: -blocks-storage.bucket-store.chunks-cache.backend
Expand Down
2 changes: 1 addition & 1 deletion go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -53,7 +53,7 @@ require (
github.com/stretchr/testify v1.8.4
github.com/thanos-io/objstore v0.0.0-20230921130928-63a603e651ed
github.com/thanos-io/promql-engine v0.0.0-20230821193351-e1ae4275b96e
github.com/thanos-io/thanos v0.32.4-0.20231001083734-531cdb1e8ec3
github.com/thanos-io/thanos v0.32.5-0.20231006043659-79bbf34b4275
github.com/uber/jaeger-client-go v2.30.0+incompatible
github.com/weaveworks/common v0.0.0-20221201103051-7c2720a9024d
go.etcd.io/etcd/api/v3 v3.5.9
Expand Down
4 changes: 2 additions & 2 deletions go.sum
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1212,8 +1212,8 @@ github.com/thanos-io/objstore v0.0.0-20230921130928-63a603e651ed h1:iWQdY3S6DpWj
github.com/thanos-io/objstore v0.0.0-20230921130928-63a603e651ed/go.mod h1:oJ82xgcBDzGJrEgUsjlTj6n01+ZWUMMUR8BlZzX5xDE=
github.com/thanos-io/promql-engine v0.0.0-20230821193351-e1ae4275b96e h1:kwsFCU8eSkZehbrAN3nXPw5RdMHi/Bok/y8l2C4M+gk=
github.com/thanos-io/promql-engine v0.0.0-20230821193351-e1ae4275b96e/go.mod h1:+T/ZYNCGybT6eTsGGvVtGb63nT1cvUmH6MjqRrcQoKw=
github.com/thanos-io/thanos v0.32.4-0.20231001083734-531cdb1e8ec3 h1:ekD3P1XF0Hlg/u7rSNqdyLhwYE4W4RGfkMDudtepRL8=
github.com/thanos-io/thanos v0.32.4-0.20231001083734-531cdb1e8ec3/go.mod h1:Px5Boq60s+2WwR+V4v4oxgmxfw9WHrwMwjRou6pkUNw=
github.com/thanos-io/thanos v0.32.5-0.20231006043659-79bbf34b4275 h1:y2YPqM1XiBw7EhLg45F6A1g8bgt4yYxkaRAeQaNLWYk=
github.com/thanos-io/thanos v0.32.5-0.20231006043659-79bbf34b4275/go.mod h1:HwiHn7u6GeES403BTACOYib/JKAJknf8dByU/uJiEr0=
github.com/themihai/gomemcache v0.0.0-20180902122335-24332e2d58ab h1:7ZR3hmisBWw77ZpO1/o86g+JV3VKlk3d48jopJxzTjU=
github.com/themihai/gomemcache v0.0.0-20180902122335-24332e2d58ab/go.mod h1:eheTFp954zcWZXCU8d0AT76ftsQOTo4DTqkN/h3k1MY=
github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk=
Expand Down
75 changes: 67 additions & 8 deletions pkg/storage/tsdb/index_cache.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ import (
storecache "github.com/thanos-io/thanos/pkg/store/cache"

"github.com/cortexproject/cortex/pkg/util"
"github.com/cortexproject/cortex/pkg/util/flagext"
)

const (
Expand Down Expand Up @@ -41,10 +42,10 @@ var (
)

type IndexCacheConfig struct {
Backend string `yaml:"backend"`
InMemory InMemoryIndexCacheConfig `yaml:"inmemory"`
Memcached MemcachedClientConfig `yaml:"memcached"`
Redis RedisClientConfig `yaml:"redis"`
Backend string `yaml:"backend"`
InMemory InMemoryIndexCacheConfig `yaml:"inmemory"`
Memcached MemcachedIndexCacheConfig `yaml:"memcached"`
Redis RedisIndexCacheConfig `yaml:"redis"`
}

func (cfg *IndexCacheConfig) RegisterFlags(f *flag.FlagSet) {
Expand Down Expand Up @@ -85,6 +86,10 @@ func (cfg *IndexCacheConfig) Validate() error {
if err := cfg.Redis.Validate(); err != nil {
return err
}
} else {
if err := cfg.InMemory.Validate(); err != nil {
return err
}
}

configuredBackends[backend] = struct{}{}
Expand All @@ -94,17 +99,63 @@ func (cfg *IndexCacheConfig) Validate() error {
}

type InMemoryIndexCacheConfig struct {
MaxSizeBytes uint64 `yaml:"max_size_bytes"`
MaxSizeBytes uint64 `yaml:"max_size_bytes"`
EnabledItems []string `yaml:"enabled_items"`
}

func (cfg *InMemoryIndexCacheConfig) Validate() error {
if err := storecache.ValidateEnabledItems(cfg.EnabledItems); err != nil {
return err
}
return nil
}

func (cfg *InMemoryIndexCacheConfig) RegisterFlagsWithPrefix(f *flag.FlagSet, prefix string) {
f.Uint64Var(&cfg.MaxSizeBytes, prefix+"max-size-bytes", uint64(1*units.Gibibyte), "Maximum size in bytes of in-memory index cache used to speed up blocks index lookups (shared between all tenants).")
f.Var((*flagext.StringSlice)(&cfg.EnabledItems), prefix+"enabled-items", "Selectively cache index item types. Supported values are Postings, ExpandedPostings and Series")
}

type MemcachedIndexCacheConfig struct {
ClientConfig MemcachedClientConfig `yaml:",inline"`
EnabledItems []string `yaml:"enabled_items"`
}

func (cfg *MemcachedIndexCacheConfig) Validate() error {
if err := cfg.ClientConfig.Validate(); err != nil {
return err
}
return storecache.ValidateEnabledItems(cfg.EnabledItems)
}

func (cfg *MemcachedIndexCacheConfig) RegisterFlagsWithPrefix(f *flag.FlagSet, prefix string) {
cfg.ClientConfig.RegisterFlagsWithPrefix(f, prefix)
f.Var((*flagext.StringSlice)(&cfg.EnabledItems), prefix+"enabled-items", "Selectively cache index item types. Supported values are Postings, ExpandedPostings and Series")
}

type RedisIndexCacheConfig struct {
ClientConfig RedisClientConfig `yaml:",inline"`
EnabledItems []string `yaml:"enabled_items"`
}

func (cfg *RedisIndexCacheConfig) RegisterFlagsWithPrefix(f *flag.FlagSet, prefix string) {
cfg.ClientConfig.RegisterFlagsWithPrefix(f, prefix)
f.Var((*flagext.StringSlice)(&cfg.EnabledItems), prefix+"enabled-items", "Selectively cache index item types. Supported values are Postings, ExpandedPostings and Series")
}

func (cfg *RedisIndexCacheConfig) Validate() error {
if err := cfg.ClientConfig.Validate(); err != nil {
return err
}
return storecache.ValidateEnabledItems(cfg.EnabledItems)
}

// NewIndexCache creates a new index cache based on the input configuration.
func NewIndexCache(cfg IndexCacheConfig, logger log.Logger, registerer prometheus.Registerer) (storecache.IndexCache, error) {
splitBackends := strings.Split(cfg.Backend, ",")
var caches []storecache.IndexCache
var (
caches []storecache.IndexCache
enabledItems []string
)

for i, backend := range splitBackends {
iReg := registerer
Expand All @@ -121,8 +172,9 @@ func NewIndexCache(cfg IndexCacheConfig, logger log.Logger, registerer prometheu
return c, err
}
caches = append(caches, c)
enabledItems = cfg.InMemory.EnabledItems
case IndexCacheBackendMemcached:
c, err := newMemcachedIndexCacheClient(cfg.Memcached, logger, registerer)
c, err := newMemcachedIndexCacheClient(cfg.Memcached.ClientConfig, logger, registerer)
if err != nil {
return nil, err
}
Expand All @@ -131,8 +183,9 @@ func NewIndexCache(cfg IndexCacheConfig, logger log.Logger, registerer prometheu
return nil, err
}
caches = append(caches, cache)
enabledItems = cfg.Memcached.EnabledItems
case IndexCacheBackendRedis:
c, err := newRedisIndexCacheClient(cfg.Redis, logger, iReg)
c, err := newRedisIndexCacheClient(cfg.Redis.ClientConfig, logger, iReg)
if err != nil {
return nil, err
}
Expand All @@ -141,9 +194,15 @@ func NewIndexCache(cfg IndexCacheConfig, logger log.Logger, registerer prometheu
return nil, err
}
caches = append(caches, cache)
enabledItems = cfg.Redis.EnabledItems
default:
return nil, errUnsupportedIndexCacheBackend
}
if len(enabledItems) > 0 {
latestCache := caches[len(caches)-1]
cache := storecache.NewFilteredIndexCache(latestCache, enabledItems)
caches[len(caches)-1] = cache
}
}

return newMultiLevelCache(caches...), nil
Expand Down
40 changes: 38 additions & 2 deletions pkg/storage/tsdb/index_cache_test.go
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
package tsdb

import (
"fmt"
"testing"

"github.com/stretchr/testify/assert"
Expand Down Expand Up @@ -35,11 +36,46 @@ func TestIndexCacheConfig_Validate(t *testing.T) {
"one memcached address should pass": {
cfg: IndexCacheConfig{
Backend: "memcached",
Memcached: MemcachedClientConfig{
Addresses: "dns+localhost:11211",
Memcached: MemcachedIndexCacheConfig{
ClientConfig: MemcachedClientConfig{
Addresses: "dns+localhost:11211",
},
},
},
},
"invalid enabled items memcached": {
cfg: IndexCacheConfig{
Backend: "memcached",
Memcached: MemcachedIndexCacheConfig{
ClientConfig: MemcachedClientConfig{
Addresses: "dns+localhost:11211",
},
EnabledItems: []string{"foo", "bar"},
},
},
expected: fmt.Errorf("unsupported item type foo"),
},
"invalid enabled items inmemory": {
cfg: IndexCacheConfig{
Backend: "inmemory",
InMemory: InMemoryIndexCacheConfig{
EnabledItems: []string{"foo", "bar"},
},
},
expected: fmt.Errorf("unsupported item type foo"),
},
"invalid enabled items redis": {
cfg: IndexCacheConfig{
Backend: "redis",
Redis: RedisIndexCacheConfig{
ClientConfig: RedisClientConfig{
Addresses: "test",
},
EnabledItems: []string{"foo", "bar"},
},
},
expected: fmt.Errorf("unsupported item type foo"),
},
}

for testName, testData := range tests {
Expand Down
Loading

0 comments on commit 577045c

Please sign in to comment.