From a900cb5c973be8583dac572f40f6cc9ed923224a Mon Sep 17 00:00:00 2001 From: Sebastian Rabenhorst <4246554+rabenhorst@users.noreply.github.com> Date: Wed, 20 Dec 2023 13:58:13 +0100 Subject: [PATCH 01/26] rule: native histogram support (#6390) * Added native histogram support for ruler Signed-off-by: Sebastian Rabenhorst Formatted imports Signed-off-by: Sebastian Rabenhorst Fixed imports Signed-off-by: Sebastian Rabenhorst Formated imports Signed-off-by: Sebastian Rabenhorst * Fixed native histogram tests Signed-off-by: Sebastian Rabenhorst Fixed receiver type Signed-off-by: Sebastian Rabenhorst * Fix for rebase Signed-off-by: Sebastian Rabenhorst * Added docs for query endpoints differences Signed-off-by: Sebastian Rabenhorst * Fixed comments and naming Signed-off-by: Sebastian Rabenhorst * made HTTPConfig optional Signed-off-by: Sebastian Rabenhorst * made HTTPConfig optional Signed-off-by: Sebastian Rabenhorst * Reverted and added check Signed-off-by: Sebastian Rabenhorst * Fixes from comments Signed-off-by: Sebastian Rabenhorst * renamed queryconfig to clientconfig Signed-off-by: Sebastian Rabenhorst * common prepareEndpointSet Signed-off-by: Sebastian Rabenhorst * fixed lint Signed-off-by: Sebastian Rabenhorst * Fixed sidecar Signed-off-by: Sebastian Rabenhorst * Fixed tests Signed-off-by: Sebastian Rabenhorst --------- Signed-off-by: Sebastian Rabenhorst --- cmd/thanos/query.go | 147 ++++++++++------- cmd/thanos/rule.go | 196 ++++++++++++++++++----- cmd/thanos/sidecar.go | 8 +- docs/components/rule.md | 11 +- pkg/alert/config.go | 20 +-- pkg/alert/config_test.go | 20 +-- pkg/clientconfig/config.go | 99 ++++++++++++ pkg/clientconfig/config_test.go | 162 +++++++++++++++++++ pkg/clientconfig/grpc.go | 8 + pkg/{httpconfig => clientconfig}/http.go | 57 ++++--- pkg/httpconfig/config.go | 75 --------- pkg/httpconfig/config_test.go | 94 ----------- pkg/promclient/promclient.go | 4 +- pkg/query/remote_engine.go | 65 +++++++- pkg/query/remote_engine_test.go | 6 +- pkg/receive/head_series_limiter.go | 9 +- pkg/receive/limiter_config.go | 11 +- pkg/rules/queryable.go | 8 +- pkg/store/prometheus.go | 4 +- pkg/store/storepb/prompb/custom.go | 12 ++ scripts/cfggen/main.go | 11 +- test/e2e/compatibility_test.go | 30 ++-- test/e2e/e2ethanos/services.go | 9 +- test/e2e/native_histograms_test.go | 98 ++++++++++-- test/e2e/rule_test.go | 74 +++++---- test/e2e/rules_api_test.go | 12 +- 26 files changed, 840 insertions(+), 410 deletions(-) create mode 100644 pkg/clientconfig/config.go create mode 100644 pkg/clientconfig/config_test.go create mode 100644 pkg/clientconfig/grpc.go rename pkg/{httpconfig => clientconfig}/http.go (87%) delete mode 100644 pkg/httpconfig/config.go delete mode 100644 pkg/httpconfig/config_test.go diff --git a/cmd/thanos/query.go b/cmd/thanos/query.go index 86b7936080..82f04f67d1 100644 --- a/cmd/thanos/query.go +++ b/cmd/thanos/query.go @@ -502,56 +502,29 @@ func runQuery( } var ( - endpoints = query.NewEndpointSet( - time.Now, + endpoints = prepareEndpointSet( + g, logger, reg, - func() (specs []*query.GRPCEndpointSpec) { - // Add strict & static nodes. - for _, addr := range strictStores { - specs = append(specs, query.NewGRPCEndpointSpec(addr, true)) - } - - for _, addr := range strictEndpoints { - specs = append(specs, query.NewGRPCEndpointSpec(addr, true)) - } - - for _, dnsProvider := range []*dns.Provider{ - dnsStoreProvider, - dnsRuleProvider, - dnsExemplarProvider, - dnsMetadataProvider, - dnsTargetProvider, - dnsEndpointProvider, - } { - var tmpSpecs []*query.GRPCEndpointSpec - - for _, addr := range dnsProvider.Addresses() { - tmpSpecs = append(tmpSpecs, query.NewGRPCEndpointSpec(addr, false)) - } - tmpSpecs = removeDuplicateEndpointSpecs(logger, duplicatedStores, tmpSpecs) - specs = append(specs, tmpSpecs...) - } - - for _, eg := range endpointGroupAddrs { - addr := fmt.Sprintf("dns:///%s", eg) - spec := query.NewGRPCEndpointSpec(addr, false, extgrpc.EndpointGroupGRPCOpts()...) - specs = append(specs, spec) - } - - for _, eg := range strictEndpointGroups { - addr := fmt.Sprintf("dns:///%s", eg) - spec := query.NewGRPCEndpointSpec(addr, true, extgrpc.EndpointGroupGRPCOpts()...) - specs = append(specs, spec) - } - - return specs + []*dns.Provider{ + dnsStoreProvider, + dnsRuleProvider, + dnsExemplarProvider, + dnsMetadataProvider, + dnsTargetProvider, + dnsEndpointProvider, }, + duplicatedStores, + strictStores, + strictEndpoints, + endpointGroupAddrs, + strictEndpointGroups, dialOpts, unhealthyStoreTimeout, endpointInfoTimeout, queryConnMetricLabels..., ) + proxy = store.NewProxyStore(logger, reg, endpoints.GetStoreClients, component.Query, selectorLset, storeResponseTimeout, store.RetrievalStrategy(grpcProxyStrategy), options...) rulesProxy = rules.NewProxy(logger, endpoints.GetRulesClients) targetsProxy = targets.NewProxy(logger, endpoints.GetTargetsClients) @@ -566,20 +539,6 @@ func runQuery( ) ) - // Periodically update the store set with the addresses we see in our cluster. - { - ctx, cancel := context.WithCancel(context.Background()) - g.Add(func() error { - return runutil.Repeat(5*time.Second, ctx.Done(), func() error { - endpoints.Update(ctx) - return nil - }) - }, func(error) { - cancel() - endpoints.Close() - }) - } - // Run File Service Discovery and update the store set when the files are modified. if fileSD != nil { var fileSDUpdates chan []*targetgroup.Group @@ -861,6 +820,82 @@ func removeDuplicateEndpointSpecs(logger log.Logger, duplicatedStores prometheus return deduplicated } +func prepareEndpointSet( + g *run.Group, + logger log.Logger, + reg *prometheus.Registry, + dnsProviders []*dns.Provider, + duplicatedStores prometheus.Counter, + strictStores []string, + strictEndpoints []string, + endpointGroupAddrs []string, + strictEndpointGroups []string, + dialOpts []grpc.DialOption, + unhealthyStoreTimeout time.Duration, + endpointInfoTimeout time.Duration, + queryConnMetricLabels ...string, +) *query.EndpointSet { + endpointSet := query.NewEndpointSet( + time.Now, + logger, + reg, + func() (specs []*query.GRPCEndpointSpec) { + // Add strict & static nodes. + for _, addr := range strictStores { + specs = append(specs, query.NewGRPCEndpointSpec(addr, true)) + } + + for _, addr := range strictEndpoints { + specs = append(specs, query.NewGRPCEndpointSpec(addr, true)) + } + + for _, dnsProvider := range dnsProviders { + var tmpSpecs []*query.GRPCEndpointSpec + + for _, addr := range dnsProvider.Addresses() { + tmpSpecs = append(tmpSpecs, query.NewGRPCEndpointSpec(addr, false)) + } + tmpSpecs = removeDuplicateEndpointSpecs(logger, duplicatedStores, tmpSpecs) + specs = append(specs, tmpSpecs...) + } + + for _, eg := range endpointGroupAddrs { + addr := fmt.Sprintf("dns:///%s", eg) + spec := query.NewGRPCEndpointSpec(addr, false, extgrpc.EndpointGroupGRPCOpts()...) + specs = append(specs, spec) + } + + for _, eg := range strictEndpointGroups { + addr := fmt.Sprintf("dns:///%s", eg) + spec := query.NewGRPCEndpointSpec(addr, true, extgrpc.EndpointGroupGRPCOpts()...) + specs = append(specs, spec) + } + + return specs + }, + dialOpts, + unhealthyStoreTimeout, + endpointInfoTimeout, + queryConnMetricLabels..., + ) + + // Periodically update the store set with the addresses we see in our cluster. + { + ctx, cancel := context.WithCancel(context.Background()) + g.Add(func() error { + return runutil.Repeat(5*time.Second, ctx.Done(), func() error { + endpointSet.Update(ctx) + return nil + }) + }, func(error) { + cancel() + endpointSet.Close() + }) + } + + return endpointSet +} + // LookbackDeltaFactory creates from 1 to 3 lookback deltas depending on // dynamicLookbackDelta and eo.LookbackDelta and returns a function // that returns appropriate lookback delta for given maxSourceResolutionMillis. diff --git a/cmd/thanos/rule.go b/cmd/thanos/rule.go index e8f867347b..982ea055dc 100644 --- a/cmd/thanos/rule.go +++ b/cmd/thanos/rule.go @@ -50,18 +50,20 @@ import ( "github.com/thanos-io/thanos/pkg/alert" v1 "github.com/thanos-io/thanos/pkg/api/rule" "github.com/thanos-io/thanos/pkg/block/metadata" + "github.com/thanos-io/thanos/pkg/clientconfig" "github.com/thanos-io/thanos/pkg/component" "github.com/thanos-io/thanos/pkg/discovery/dns" "github.com/thanos-io/thanos/pkg/errutil" + "github.com/thanos-io/thanos/pkg/extgrpc" "github.com/thanos-io/thanos/pkg/extkingpin" "github.com/thanos-io/thanos/pkg/extprom" extpromhttp "github.com/thanos-io/thanos/pkg/extprom/http" - "github.com/thanos-io/thanos/pkg/httpconfig" "github.com/thanos-io/thanos/pkg/info" "github.com/thanos-io/thanos/pkg/info/infopb" "github.com/thanos-io/thanos/pkg/logging" "github.com/thanos-io/thanos/pkg/prober" "github.com/thanos-io/thanos/pkg/promclient" + "github.com/thanos-io/thanos/pkg/query" thanosrules "github.com/thanos-io/thanos/pkg/rules" "github.com/thanos-io/thanos/pkg/runutil" grpcserver "github.com/thanos-io/thanos/pkg/server/grpc" @@ -75,14 +77,17 @@ import ( "github.com/thanos-io/thanos/pkg/ui" ) +const dnsSDResolver = "miekgdns" + type ruleConfig struct { http httpConfig grpc grpcConfig web webConfig shipper shipperConfig - query queryConfig - queryConfigYAML []byte + query queryConfig + queryConfigYAML []byte + grpcQueryEndpoints []string alertmgr alertMgrConfig alertmgrsConfigYAML []byte @@ -148,6 +153,9 @@ func registerRule(app *extkingpin.App) { cmd.Flag("restore-ignored-label", "Label names to be ignored when restoring alerts from the remote storage. This is only used in stateless mode."). StringsVar(&conf.ignoredLabelNames) + cmd.Flag("grpc-query-endpoint", "Addresses of Thanos gRPC query API servers (repeatable). The scheme may be prefixed with 'dns+' or 'dnssrv+' to detect Thanos API servers through respective DNS lookups."). + PlaceHolder("").StringsVar(&conf.grpcQueryEndpoints) + conf.rwConfig = extflag.RegisterPathOrContent(cmd, "remote-write.config", "YAML config for the remote-write configurations, that specify servers where samples should be sent to (see https://prometheus.io/docs/prometheus/latest/configuration/configuration/#remote_write). This automatically enables stateless mode for ruler and no series will be stored in the ruler's TSDB. If an empty config (or file) is provided, the flag is ignored and ruler is run with its own TSDB.", extflag.WithEnvSubstitution()) conf.objStoreConfig = extkingpin.RegisterCommonObjStoreFlags(cmd, "", false) @@ -193,11 +201,12 @@ func registerRule(app *extkingpin.App) { if err != nil { return err } - if len(conf.query.sdFiles) == 0 && len(conf.query.addrs) == 0 && len(conf.queryConfigYAML) == 0 { - return errors.New("no --query parameter was given") + + if len(conf.query.sdFiles) == 0 && len(conf.query.addrs) == 0 && len(conf.queryConfigYAML) == 0 && len(conf.grpcQueryEndpoints) == 0 { + return errors.New("no query configuration parameter was given") } - if (len(conf.query.sdFiles) != 0 || len(conf.query.addrs) != 0) && len(conf.queryConfigYAML) != 0 { - return errors.New("--query/--query.sd-files and --query.config* parameters cannot be defined at the same time") + if (len(conf.query.sdFiles) != 0 || len(conf.query.addrs) != 0 || len(conf.grpcQueryEndpoints) != 0) && len(conf.queryConfigYAML) != 0 { + return errors.New("--query/--query.sd-files/--grpc-query-endpoint and --query.config* parameters cannot be defined at the same time") } // Parse and check alerting configuration. @@ -224,7 +233,8 @@ func registerRule(app *extkingpin.App) { return errors.Wrap(err, "error while parsing config for request logging") } - return runRule(g, + return runRule( + g, logger, reg, tracer, @@ -304,35 +314,43 @@ func runRule( ) error { metrics := newRuleMetrics(reg) - var queryCfg []httpconfig.Config + var queryCfg []clientconfig.Config var err error if len(conf.queryConfigYAML) > 0 { - queryCfg, err = httpconfig.LoadConfigs(conf.queryConfigYAML) + queryCfg, err = clientconfig.LoadConfigs(conf.queryConfigYAML) if err != nil { return err } } else { - queryCfg, err = httpconfig.BuildConfig(conf.query.addrs) + queryCfg, err = clientconfig.BuildConfigFromHTTPAddresses(conf.query.addrs) if err != nil { return errors.Wrap(err, "query configuration") } // Build the query configuration from the legacy query flags. - var fileSDConfigs []httpconfig.FileSDConfig + var fileSDConfigs []clientconfig.HTTPFileSDConfig if len(conf.query.sdFiles) > 0 { - fileSDConfigs = append(fileSDConfigs, httpconfig.FileSDConfig{ + fileSDConfigs = append(fileSDConfigs, clientconfig.HTTPFileSDConfig{ Files: conf.query.sdFiles, RefreshInterval: model.Duration(conf.query.sdInterval), }) queryCfg = append(queryCfg, - httpconfig.Config{ - EndpointsConfig: httpconfig.EndpointsConfig{ - Scheme: "http", - FileSDConfigs: fileSDConfigs, + clientconfig.Config{ + HTTPConfig: clientconfig.HTTPConfig{ + EndpointsConfig: clientconfig.HTTPEndpointsConfig{ + Scheme: "http", + FileSDConfigs: fileSDConfigs, + }, }, }, ) } + + grpcQueryCfg, err := clientconfig.BuildConfigFromGRPCAddresses(conf.grpcQueryEndpoints) + if err != nil { + return errors.Wrap(err, "query configuration") + } + queryCfg = append(queryCfg, grpcQueryCfg...) } if err := validateTemplate(*conf.alertmgr.alertSourceTemplate); err != nil { @@ -345,26 +363,97 @@ func runRule( dns.ResolverType(conf.query.dnsSDResolver), ) var ( - queryClients []*httpconfig.Client - promClients []*promclient.Client + queryClients []*clientconfig.HTTPClient + promClients []*promclient.Client + grpcEndpointSet *query.EndpointSet + grpcEndpoints []string ) + queryClientMetrics := extpromhttp.NewClientMetrics(extprom.WrapRegistererWith(prometheus.Labels{"client": "query"}, reg)) + for _, cfg := range queryCfg { - cfg.HTTPClientConfig.ClientMetrics = queryClientMetrics - c, err := httpconfig.NewHTTPClient(cfg.HTTPClientConfig, "query") - if err != nil { - return err + if cfg.HTTPConfig.NotEmpty() { + cfg.HTTPConfig.HTTPClientConfig.ClientMetrics = queryClientMetrics + c, err := clientconfig.NewHTTPClient(cfg.HTTPConfig.HTTPClientConfig, "query") + if err != nil { + return err + } + c.Transport = tracing.HTTPTripperware(logger, c.Transport) + queryClient, err := clientconfig.NewClient(logger, cfg.HTTPConfig.EndpointsConfig, c, queryProvider.Clone()) + if err != nil { + return err + } + queryClients = append(queryClients, queryClient) + promClients = append(promClients, promclient.NewClient(queryClient, logger, "thanos-rule")) + // Discover and resolve query addresses. + addDiscoveryGroups(g, queryClient, conf.query.dnsSDInterval) } - c.Transport = tracing.HTTPTripperware(logger, c.Transport) - queryClient, err := httpconfig.NewClient(logger, cfg.EndpointsConfig, c, queryProvider.Clone()) + + if cfg.GRPCConfig != nil { + grpcEndpoints = append(grpcEndpoints, cfg.GRPCConfig.EndpointAddrs...) + } + } + + if len(grpcEndpoints) > 0 { + duplicatedGRPCEndpoints := promauto.With(reg).NewCounter(prometheus.CounterOpts{ + Name: "thanos_rule_grpc_endpoints_duplicated_total", + Help: "The number of times a duplicated grpc endpoint is detected from the different configs in rule", + }) + + dnsEndpointProvider := dns.NewProvider( + logger, + extprom.WrapRegistererWithPrefix("thanos_rule_grpc_endpoints_", reg), + dnsSDResolver, + ) + + dialOpts, err := extgrpc.StoreClientGRPCOpts( + logger, + reg, + tracer, + false, + false, + "", + "", + "", + "", + ) if err != nil { return err } - queryClients = append(queryClients, queryClient) - promClients = append(promClients, promclient.NewClient(queryClient, logger, "thanos-rule")) - // Discover and resolve query addresses. - addDiscoveryGroups(g, queryClient, conf.query.dnsSDInterval) + + grpcEndpointSet = prepareEndpointSet( + g, + logger, + reg, + []*dns.Provider{dnsEndpointProvider}, + duplicatedGRPCEndpoints, + nil, + nil, + nil, + nil, + dialOpts, + 5*time.Minute, + 5*time.Second, + ) + + // Periodically update the GRPC addresses from query config by resolving them using DNS SD if necessary. + { + ctx, cancel := context.WithCancel(context.Background()) + g.Add(func() error { + return runutil.Repeat(5*time.Second, ctx.Done(), func() error { + resolveCtx, resolveCancel := context.WithTimeout(ctx, 5*time.Second) + defer resolveCancel() + if err := dnsEndpointProvider.Resolve(resolveCtx, grpcEndpoints); err != nil { + level.Error(logger).Log("msg", "failed to resolve addresses passed using grpc query config", "err", err) + } + return nil + }) + }, func(error) { + cancel() + }) + } } + var ( appendable storage.Appendable queryable storage.Queryable @@ -473,13 +562,13 @@ func runRule( ) for _, cfg := range alertingCfg.Alertmanagers { cfg.HTTPClientConfig.ClientMetrics = amClientMetrics - c, err := httpconfig.NewHTTPClient(cfg.HTTPClientConfig, "alertmanager") + c, err := clientconfig.NewHTTPClient(cfg.HTTPClientConfig, "alertmanager") if err != nil { return err } c.Transport = tracing.HTTPTripperware(logger, c.Transport) // Each Alertmanager client has a different list of targets thus each needs its own DNS provider. - amClient, err := httpconfig.NewClient(logger, cfg.EndpointsConfig, c, amProvider.Clone()) + amClient, err := clientconfig.NewClient(logger, cfg.EndpointsConfig, c, amProvider.Clone()) if err != nil { return err } @@ -538,7 +627,7 @@ func runRule( OutageTolerance: conf.outageTolerance, ForGracePeriod: conf.forGracePeriod, }, - queryFuncCreator(logger, queryClients, promClients, metrics.duplicatedQuery, metrics.ruleEvalWarnings, conf.query.httpMethod, conf.query.doNotAddThanosParams), + queryFuncCreator(logger, queryClients, promClients, grpcEndpointSet, metrics.duplicatedQuery, metrics.ruleEvalWarnings, conf.query.httpMethod, conf.query.doNotAddThanosParams), conf.lset, // In our case the querying URL is the external URL because in Prometheus // --web.external-url points to it i.e. it points at something where the user @@ -816,8 +905,9 @@ func labelsTSDBToProm(lset labels.Labels) (res labels.Labels) { func queryFuncCreator( logger log.Logger, - queriers []*httpconfig.Client, + queriers []*clientconfig.HTTPClient, promClients []*promclient.Client, + grpcEndpointSet *query.EndpointSet, duplicatedQuery prometheus.Counter, ruleEvalWarnings *prometheus.CounterVec, httpMethod string, @@ -839,13 +929,13 @@ func queryFuncCreator( panic(errors.Errorf("unknown partial response strategy %v", partialResponseStrategy).Error()) } - return func(ctx context.Context, q string, t time.Time) (promql.Vector, error) { + return func(ctx context.Context, qs string, t time.Time) (promql.Vector, error) { for _, i := range rand.Perm(len(queriers)) { promClient := promClients[i] endpoints := thanosrules.RemoveDuplicateQueryEndpoints(logger, duplicatedQuery, queriers[i].Endpoints()) for _, i := range rand.Perm(len(endpoints)) { span, ctx := tracing.StartSpan(ctx, spanID) - v, warns, err := promClient.PromqlQueryInstant(ctx, endpoints[i], q, t, promclient.QueryOptions{ + v, warns, err := promClient.PromqlQueryInstant(ctx, endpoints[i], qs, t, promclient.QueryOptions{ Deduplicate: true, PartialResponseStrategy: partialResponseStrategy, Method: httpMethod, @@ -854,23 +944,53 @@ func queryFuncCreator( span.Finish() if err != nil { - level.Error(logger).Log("err", err, "query", q) + level.Error(logger).Log("err", err, "query", qs) continue } if len(warns) > 0 { ruleEvalWarnings.WithLabelValues(strings.ToLower(partialResponseStrategy.String())).Inc() // TODO(bwplotka): Propagate those to UI, probably requires changing rule manager code ): - level.Warn(logger).Log("warnings", strings.Join(warns, ", "), "query", q) + level.Warn(logger).Log("warnings", strings.Join(warns, ", "), "query", qs) } return v, nil } } + + if grpcEndpointSet != nil { + queryAPIClients := grpcEndpointSet.GetQueryAPIClients() + for _, i := range rand.Perm(len(queryAPIClients)) { + e := query.NewRemoteEngine(logger, queryAPIClients[i], query.Opts{}) + q, err := e.NewInstantQuery(ctx, nil, qs, t) + if err != nil { + level.Error(logger).Log("err", err, "query", qs) + continue + } + + result := q.Exec(ctx) + v, err := result.Vector() + if err != nil { + level.Error(logger).Log("err", err, "query", qs) + continue + } + + if len(result.Warnings) > 0 { + ruleEvalWarnings.WithLabelValues(strings.ToLower(partialResponseStrategy.String())).Inc() + warnings := make([]string, 0, len(result.Warnings)) + for _, w := range result.Warnings { + warnings = append(warnings, w.Error()) + } + level.Warn(logger).Log("warnings", strings.Join(warnings, ", "), "query", qs) + } + + return v, nil + } + } return nil, errors.Errorf("no query API server reachable") } } } -func addDiscoveryGroups(g *run.Group, c *httpconfig.Client, interval time.Duration) { +func addDiscoveryGroups(g *run.Group, c *clientconfig.HTTPClient, interval time.Duration) { ctx, cancel := context.WithCancel(context.Background()) g.Add(func() error { c.Discover(ctx) diff --git a/cmd/thanos/sidecar.go b/cmd/thanos/sidecar.go index 3b8846d146..74ab3090fa 100644 --- a/cmd/thanos/sidecar.go +++ b/cmd/thanos/sidecar.go @@ -30,11 +30,11 @@ import ( objstoretracing "github.com/thanos-io/objstore/tracing/opentracing" "github.com/thanos-io/thanos/pkg/block/metadata" + "github.com/thanos-io/thanos/pkg/clientconfig" "github.com/thanos-io/thanos/pkg/component" "github.com/thanos-io/thanos/pkg/exemplars" "github.com/thanos-io/thanos/pkg/extkingpin" "github.com/thanos-io/thanos/pkg/extprom" - "github.com/thanos-io/thanos/pkg/httpconfig" "github.com/thanos-io/thanos/pkg/info" "github.com/thanos-io/thanos/pkg/info/infopb" "github.com/thanos-io/thanos/pkg/logging" @@ -68,12 +68,12 @@ func registerSidecar(app *extkingpin.App) { if err != nil { return errors.Wrap(err, "getting http client config") } - httpClientConfig, err := httpconfig.NewClientConfigFromYAML(httpConfContentYaml) + httpClientConfig, err := clientconfig.NewHTTPClientConfigFromYAML(httpConfContentYaml) if err != nil { return errors.Wrap(err, "parsing http config YAML") } - httpClient, err := httpconfig.NewHTTPClient(*httpClientConfig, "thanos-sidecar") + httpClient, err := clientconfig.NewHTTPClient(*httpClientConfig, "thanos-sidecar") if err != nil { return errors.Wrap(err, "Improper http client config") } @@ -260,7 +260,7 @@ func runSidecar( }) } { - c := promclient.NewWithTracingClient(logger, httpClient, httpconfig.ThanosUserAgent) + c := promclient.NewWithTracingClient(logger, httpClient, clientconfig.ThanosUserAgent) promStore, err := store.NewPrometheusStore(logger, reg, c, conf.prometheus.url, component.Sidecar, m.Labels, m.Timestamps, m.Version) if err != nil { diff --git a/docs/components/rule.md b/docs/components/rule.md index 5e6a1a3d8f..e6211c9eea 100644 --- a/docs/components/rule.md +++ b/docs/components/rule.md @@ -330,6 +330,11 @@ Flags: from other components. --grpc-grace-period=2m Time to wait after an interrupt received for GRPC Server. + --grpc-query-endpoint= ... + Addresses of Thanos gRPC query API servers + (repeatable). The scheme may be prefixed + with 'dns+' or 'dnssrv+' to detect Thanos API + servers through respective DNS lookups. --grpc-server-max-connection-age=60m The grpc server max connection age. This controls how often to re-establish connections @@ -551,7 +556,9 @@ Supported values for `api_version` are `v1` or `v2`. ### Query API -The `--query.config` and `--query.config-file` flags allow specifying multiple query endpoints. Those entries are treated as a single HA group. This means that query failure is claimed only if the Ruler fails to query all instances. +The `--query.config` and `--query.config-file` flags allow specifying multiple query endpoints. Those entries are treated as a single HA group, where HTTP endpoints are given priority over gRPC Query API endpoints. This means that query failure is claimed only if the Ruler fails to query all instances. + +Rules that produce native histograms (experimental feature) are exclusively supported through the gRPC Query API. However, for all other rules, there is no difference in functionality between HTTP and gRPC. The configuration format is the following: @@ -576,4 +583,6 @@ The configuration format is the following: refresh_interval: 0s scheme: http path_prefix: "" + grpc_config: + endpoint_addresses: [] ``` diff --git a/pkg/alert/config.go b/pkg/alert/config.go index 67613870d9..f509ccc993 100644 --- a/pkg/alert/config.go +++ b/pkg/alert/config.go @@ -15,8 +15,8 @@ import ( "github.com/prometheus/prometheus/model/relabel" "gopkg.in/yaml.v2" + "github.com/thanos-io/thanos/pkg/clientconfig" "github.com/thanos-io/thanos/pkg/discovery/dns" - "github.com/thanos-io/thanos/pkg/httpconfig" ) type AlertingConfig struct { @@ -25,10 +25,10 @@ type AlertingConfig struct { // AlertmanagerConfig represents a client to a cluster of Alertmanager endpoints. type AlertmanagerConfig struct { - HTTPClientConfig httpconfig.ClientConfig `yaml:"http_config"` - EndpointsConfig httpconfig.EndpointsConfig `yaml:",inline"` - Timeout model.Duration `yaml:"timeout"` - APIVersion APIVersion `yaml:"api_version"` + HTTPClientConfig clientconfig.HTTPClientConfig `yaml:"http_config"` + EndpointsConfig clientconfig.HTTPEndpointsConfig `yaml:",inline"` + Timeout model.Duration `yaml:"timeout"` + APIVersion APIVersion `yaml:"api_version"` } // APIVersion represents the API version of the Alertmanager endpoint. @@ -61,10 +61,10 @@ func (v *APIVersion) UnmarshalYAML(unmarshal func(interface{}) error) error { func DefaultAlertmanagerConfig() AlertmanagerConfig { return AlertmanagerConfig{ - EndpointsConfig: httpconfig.EndpointsConfig{ + EndpointsConfig: clientconfig.HTTPEndpointsConfig{ Scheme: "http", StaticAddresses: []string{}, - FileSDConfigs: []httpconfig.FileSDConfig{}, + FileSDConfigs: []clientconfig.HTTPFileSDConfig{}, }, Timeout: model.Duration(time.Second * 10), APIVersion: APIv1, @@ -119,7 +119,7 @@ func BuildAlertmanagerConfig(address string, timeout time.Duration) (Alertmanage break } } - var basicAuth httpconfig.BasicAuth + var basicAuth clientconfig.BasicAuth if parsed.User != nil && parsed.User.String() != "" { basicAuth.Username = parsed.User.Username() pw, _ := parsed.User.Password() @@ -127,10 +127,10 @@ func BuildAlertmanagerConfig(address string, timeout time.Duration) (Alertmanage } return AlertmanagerConfig{ - HTTPClientConfig: httpconfig.ClientConfig{ + HTTPClientConfig: clientconfig.HTTPClientConfig{ BasicAuth: basicAuth, }, - EndpointsConfig: httpconfig.EndpointsConfig{ + EndpointsConfig: clientconfig.HTTPEndpointsConfig{ PathPrefix: parsed.Path, Scheme: scheme, StaticAddresses: []string{host}, diff --git a/pkg/alert/config_test.go b/pkg/alert/config_test.go index 73abb0d9b9..a0e259756c 100644 --- a/pkg/alert/config_test.go +++ b/pkg/alert/config_test.go @@ -10,7 +10,7 @@ import ( "gopkg.in/yaml.v2" "github.com/efficientgo/core/testutil" - "github.com/thanos-io/thanos/pkg/httpconfig" + "github.com/thanos-io/thanos/pkg/clientconfig" ) func TestUnmarshalAPIVersion(t *testing.T) { @@ -54,7 +54,7 @@ func TestBuildAlertmanagerConfiguration(t *testing.T) { { address: "http://localhost:9093", expected: AlertmanagerConfig{ - EndpointsConfig: httpconfig.EndpointsConfig{ + EndpointsConfig: clientconfig.HTTPEndpointsConfig{ StaticAddresses: []string{"localhost:9093"}, Scheme: "http", }, @@ -64,7 +64,7 @@ func TestBuildAlertmanagerConfiguration(t *testing.T) { { address: "https://am.example.com", expected: AlertmanagerConfig{ - EndpointsConfig: httpconfig.EndpointsConfig{ + EndpointsConfig: clientconfig.HTTPEndpointsConfig{ StaticAddresses: []string{"am.example.com"}, Scheme: "https", }, @@ -74,7 +74,7 @@ func TestBuildAlertmanagerConfiguration(t *testing.T) { { address: "dns+http://localhost:9093", expected: AlertmanagerConfig{ - EndpointsConfig: httpconfig.EndpointsConfig{ + EndpointsConfig: clientconfig.HTTPEndpointsConfig{ StaticAddresses: []string{"dns+localhost:9093"}, Scheme: "http", }, @@ -84,7 +84,7 @@ func TestBuildAlertmanagerConfiguration(t *testing.T) { { address: "dnssrv+http://localhost", expected: AlertmanagerConfig{ - EndpointsConfig: httpconfig.EndpointsConfig{ + EndpointsConfig: clientconfig.HTTPEndpointsConfig{ StaticAddresses: []string{"dnssrv+localhost"}, Scheme: "http", }, @@ -94,7 +94,7 @@ func TestBuildAlertmanagerConfiguration(t *testing.T) { { address: "ssh+http://localhost", expected: AlertmanagerConfig{ - EndpointsConfig: httpconfig.EndpointsConfig{ + EndpointsConfig: clientconfig.HTTPEndpointsConfig{ StaticAddresses: []string{"localhost"}, Scheme: "ssh+http", }, @@ -104,7 +104,7 @@ func TestBuildAlertmanagerConfiguration(t *testing.T) { { address: "dns+https://localhost/path/prefix/", expected: AlertmanagerConfig{ - EndpointsConfig: httpconfig.EndpointsConfig{ + EndpointsConfig: clientconfig.HTTPEndpointsConfig{ StaticAddresses: []string{"dns+localhost:9093"}, Scheme: "https", PathPrefix: "/path/prefix/", @@ -115,13 +115,13 @@ func TestBuildAlertmanagerConfiguration(t *testing.T) { { address: "http://user:pass@localhost:9093", expected: AlertmanagerConfig{ - HTTPClientConfig: httpconfig.ClientConfig{ - BasicAuth: httpconfig.BasicAuth{ + HTTPClientConfig: clientconfig.HTTPClientConfig{ + BasicAuth: clientconfig.BasicAuth{ Username: "user", Password: "pass", }, }, - EndpointsConfig: httpconfig.EndpointsConfig{ + EndpointsConfig: clientconfig.HTTPEndpointsConfig{ StaticAddresses: []string{"localhost:9093"}, Scheme: "http", }, diff --git a/pkg/clientconfig/config.go b/pkg/clientconfig/config.go new file mode 100644 index 0000000000..9de1b4f580 --- /dev/null +++ b/pkg/clientconfig/config.go @@ -0,0 +1,99 @@ +// Copyright (c) The Thanos Authors. +// Licensed under the Apache License 2.0. + +// Package clientconfig is a wrapper around github.com/prometheus/common/config with additional +// support for gRPC clients. +package clientconfig + +import ( + "fmt" + "net/url" + "strings" + + "github.com/pkg/errors" + "gopkg.in/yaml.v2" +) + +// Config is a structure that allows pointing to various HTTP and GRPC endpoints, e.g. ruler connecting to queriers. +type Config struct { + HTTPConfig HTTPConfig `yaml:",inline"` + GRPCConfig *GRPCConfig `yaml:"grpc_config"` +} + +func DefaultConfig() Config { + return Config{ + HTTPConfig: HTTPConfig{ + EndpointsConfig: HTTPEndpointsConfig{ + Scheme: "http", + StaticAddresses: []string{}, + FileSDConfigs: []HTTPFileSDConfig{}, + }, + }, + GRPCConfig: &GRPCConfig{ + EndpointAddrs: []string{}, + }, + } +} + +// UnmarshalYAML implements the yaml.Unmarshaler interface. +func (c *Config) UnmarshalYAML(unmarshal func(interface{}) error) error { + *c = DefaultConfig() + type plain Config + return unmarshal((*plain)(c)) +} + +// LoadConfigs loads a list of Config from YAML data. +func LoadConfigs(confYAML []byte) ([]Config, error) { + var clientCfg []Config + if err := yaml.UnmarshalStrict(confYAML, &clientCfg); err != nil { + return nil, err + } + return clientCfg, nil +} + +// BuildConfigFromHTTPAddresses returns a configuration from static addresses. +func BuildConfigFromHTTPAddresses(addrs []string) ([]Config, error) { + configs := make([]Config, 0, len(addrs)) + for i, addr := range addrs { + if addr == "" { + return nil, errors.Errorf("static address cannot be empty at index %d", i) + } + // If addr is missing schema, add http. + if !strings.Contains(addr, "://") { + addr = fmt.Sprintf("http://%s", addr) + } + u, err := url.Parse(addr) + if err != nil { + return nil, errors.Wrapf(err, "failed to parse addr %q", addr) + } + if u.Scheme != "http" && u.Scheme != "https" { + return nil, errors.Errorf("%q is not supported scheme for address", u.Scheme) + } + configs = append(configs, Config{ + HTTPConfig: HTTPConfig{ + EndpointsConfig: HTTPEndpointsConfig{ + Scheme: u.Scheme, + StaticAddresses: []string{u.Host}, + PathPrefix: u.Path, + }, + }, + }) + } + return configs, nil +} + +// BuildConfigFromGRPCAddresses returns a configuration from a static addresses. +func BuildConfigFromGRPCAddresses(addrs []string) ([]Config, error) { + configs := make([]Config, 0, len(addrs)) + for i, addr := range addrs { + if addr == "" { + return nil, errors.Errorf("static address cannot be empty at index %d", i) + } + configs = append(configs, Config{ + GRPCConfig: &GRPCConfig{ + EndpointAddrs: []string{addr}, + }, + }) + } + return configs, nil +} diff --git a/pkg/clientconfig/config_test.go b/pkg/clientconfig/config_test.go new file mode 100644 index 0000000000..44cdea6e72 --- /dev/null +++ b/pkg/clientconfig/config_test.go @@ -0,0 +1,162 @@ +// Copyright (c) The Thanos Authors. +// Licensed under the Apache License 2.0. + +package clientconfig + +import ( + "testing" + + "github.com/efficientgo/core/testutil" +) + +func TestBuildHTTPConfig(t *testing.T) { + for _, tc := range []struct { + desc string + addresses []string + err bool + expected []Config + }{ + { + desc: "single addr without path", + addresses: []string{"localhost:9093"}, + expected: []Config{ + { + HTTPConfig: HTTPConfig{ + EndpointsConfig: HTTPEndpointsConfig{ + StaticAddresses: []string{"localhost:9093"}, + Scheme: "http", + }, + }, + }, + }, + }, + { + desc: "1st addr without path, 2nd with", + addresses: []string{"localhost:9093", "localhost:9094/prefix"}, + expected: []Config{ + { + HTTPConfig: HTTPConfig{ + EndpointsConfig: HTTPEndpointsConfig{ + StaticAddresses: []string{"localhost:9093"}, + Scheme: "http", + }, + }, + }, + { + HTTPConfig: HTTPConfig{ + EndpointsConfig: HTTPEndpointsConfig{ + StaticAddresses: []string{"localhost:9094"}, + Scheme: "http", + PathPrefix: "/prefix", + }, + }, + }, + }, + }, + { + desc: "single addr with path and http scheme", + addresses: []string{"http://localhost:9093"}, + expected: []Config{ + { + HTTPConfig: HTTPConfig{ + EndpointsConfig: HTTPEndpointsConfig{ + StaticAddresses: []string{"localhost:9093"}, + Scheme: "http", + }, + }, + }, + }, + }, + { + desc: "single addr with path and https scheme", + addresses: []string{"https://localhost:9093"}, + expected: []Config{ + { + HTTPConfig: HTTPConfig{ + EndpointsConfig: HTTPEndpointsConfig{ + StaticAddresses: []string{"localhost:9093"}, + Scheme: "https", + }, + }, + }, + }, + }, + { + desc: "not supported scheme", + addresses: []string{"ttp://localhost:9093"}, + err: true, + }, + { + desc: "invalid addr", + addresses: []string{"this is not a valid addr"}, + err: true, + }, + { + desc: "empty addr", + addresses: []string{""}, + err: true, + }, + } { + t.Run(tc.desc, func(t *testing.T) { + cfg, err := BuildConfigFromHTTPAddresses(tc.addresses) + if tc.err { + testutil.NotOk(t, err) + return + } + + testutil.Equals(t, tc.expected, cfg) + }) + } +} + +func TestBuildGRPCConfig(t *testing.T) { + for _, tc := range []struct { + desc string + addresses []string + err bool + expected []Config + }{ + { + desc: "single addr", + addresses: []string{"localhost:9093"}, + expected: []Config{ + { + GRPCConfig: &GRPCConfig{ + EndpointAddrs: []string{"localhost:9093"}, + }, + }, + }, + }, + { + desc: "multiple addr", + addresses: []string{"localhost:9093", "localhost:9094"}, + expected: []Config{ + { + GRPCConfig: &GRPCConfig{ + EndpointAddrs: []string{"localhost:9093"}, + }, + }, + { + GRPCConfig: &GRPCConfig{ + EndpointAddrs: []string{"localhost:9094"}, + }, + }, + }, + }, + { + desc: "empty addr", + addresses: []string{""}, + err: true, + }, + } { + t.Run(tc.desc, func(t *testing.T) { + cfg, err := BuildConfigFromGRPCAddresses(tc.addresses) + if tc.err { + testutil.NotOk(t, err) + return + } + + testutil.Equals(t, tc.expected, cfg) + }) + } +} diff --git a/pkg/clientconfig/grpc.go b/pkg/clientconfig/grpc.go new file mode 100644 index 0000000000..e0987b6c8a --- /dev/null +++ b/pkg/clientconfig/grpc.go @@ -0,0 +1,8 @@ +// Copyright (c) The Thanos Authors. +// Licensed under the Apache License 2.0. + +package clientconfig + +type GRPCConfig struct { + EndpointAddrs []string `yaml:"endpoint_addresses"` +} diff --git a/pkg/httpconfig/http.go b/pkg/clientconfig/http.go similarity index 87% rename from pkg/httpconfig/http.go rename to pkg/clientconfig/http.go index 6cc54de493..b99dd9ef8f 100644 --- a/pkg/httpconfig/http.go +++ b/pkg/clientconfig/http.go @@ -1,8 +1,7 @@ // Copyright (c) The Thanos Authors. // Licensed under the Apache License 2.0. -// Package httpconfig is a wrapper around github.com/prometheus/common/config. -package httpconfig +package clientconfig import ( "context" @@ -32,8 +31,18 @@ import ( "github.com/thanos-io/thanos/pkg/discovery/cache" ) -// ClientConfig configures an HTTP client. -type ClientConfig struct { +// HTTPConfig is a structure that allows pointing to various HTTP endpoint, e.g ruler connecting to queriers. +type HTTPConfig struct { + HTTPClientConfig HTTPClientConfig `yaml:"http_config"` + EndpointsConfig HTTPEndpointsConfig `yaml:",inline"` +} + +func (c *HTTPConfig) NotEmpty() bool { + return len(c.EndpointsConfig.FileSDConfigs) > 0 || len(c.EndpointsConfig.StaticAddresses) > 0 +} + +// HTTPClientConfig configures an HTTP client. +type HTTPClientConfig struct { // The HTTP basic authentication credentials for the targets. BasicAuth BasicAuth `yaml:"basic_auth"` // The bearer token for the targets. @@ -77,7 +86,7 @@ func (b BasicAuth) IsZero() bool { return b.Username == "" && b.Password == "" && b.PasswordFile == "" } -// Transport configures client's transport properties. +// TransportConfig configures client's transport properties. type TransportConfig struct { MaxIdleConns int `yaml:"max_idle_conns"` MaxIdleConnsPerHost int `yaml:"max_idle_conns_per_host"` @@ -102,12 +111,12 @@ var defaultTransportConfig TransportConfig = TransportConfig{ DialerTimeout: int64(5 * time.Second), } -func NewDefaultClientConfig() ClientConfig { - return ClientConfig{TransportConfig: defaultTransportConfig} +func NewDefaultHTTPClientConfig() HTTPClientConfig { + return HTTPClientConfig{TransportConfig: defaultTransportConfig} } -func NewClientConfigFromYAML(cfg []byte) (*ClientConfig, error) { - conf := &ClientConfig{TransportConfig: defaultTransportConfig} +func NewHTTPClientConfigFromYAML(cfg []byte) (*HTTPClientConfig, error) { + conf := &HTTPClientConfig{TransportConfig: defaultTransportConfig} if err := yaml.Unmarshal(cfg, conf); err != nil { return nil, err } @@ -190,7 +199,7 @@ func NewRoundTripperFromConfig(cfg config_util.HTTPClientConfig, transportConfig } // NewHTTPClient returns a new HTTP client. -func NewHTTPClient(cfg ClientConfig, name string) (*http.Client, error) { +func NewHTTPClient(cfg HTTPClientConfig, name string) (*http.Client, error) { httpClientConfig := config_util.HTTPClientConfig{ BearerToken: config_util.Secret(cfg.BearerToken), BearerTokenFile: cfg.BearerTokenFile, @@ -274,13 +283,13 @@ func (u userAgentRoundTripper) RoundTrip(r *http.Request) (*http.Response, error return u.rt.RoundTrip(r) } -// EndpointsConfig configures a cluster of HTTP endpoints from static addresses and +// HTTPEndpointsConfig configures a cluster of HTTP endpoints from static addresses and // file service discovery. -type EndpointsConfig struct { +type HTTPEndpointsConfig struct { // List of addresses with DNS prefixes. StaticAddresses []string `yaml:"static_configs"` // List of file configurations (our FileSD supports different DNS lookups). - FileSDConfigs []FileSDConfig `yaml:"file_sd_configs"` + FileSDConfigs []HTTPFileSDConfig `yaml:"file_sd_configs"` // The URL scheme to use when talking to targets. Scheme string `yaml:"scheme"` @@ -289,13 +298,13 @@ type EndpointsConfig struct { PathPrefix string `yaml:"path_prefix"` } -// FileSDConfig represents a file service discovery configuration. -type FileSDConfig struct { +// HTTPFileSDConfig represents a file service discovery configuration. +type HTTPFileSDConfig struct { Files []string `yaml:"files"` RefreshInterval model.Duration `yaml:"refresh_interval"` } -func (c FileSDConfig) convert() (file.SDConfig, error) { +func (c HTTPFileSDConfig) convert() (file.SDConfig, error) { var fileSDConfig file.SDConfig b, err := yaml.Marshal(c) if err != nil { @@ -310,8 +319,8 @@ type AddressProvider interface { Addresses() []string } -// Client represents a client that can send requests to a cluster of HTTP-based endpoints. -type Client struct { +// HTTPClient represents a client that can send requests to a cluster of HTTP-based endpoints. +type HTTPClient struct { logger log.Logger httpClient *http.Client @@ -326,7 +335,7 @@ type Client struct { } // NewClient returns a new Client. -func NewClient(logger log.Logger, cfg EndpointsConfig, client *http.Client, provider AddressProvider) (*Client, error) { +func NewClient(logger log.Logger, cfg HTTPEndpointsConfig, client *http.Client, provider AddressProvider) (*HTTPClient, error) { if logger == nil { logger = log.NewNopLogger() } @@ -344,7 +353,7 @@ func NewClient(logger log.Logger, cfg EndpointsConfig, client *http.Client, prov } discoverers = append(discoverers, discovery) } - return &Client{ + return &HTTPClient{ logger: logger, httpClient: client, scheme: cfg.Scheme, @@ -357,12 +366,12 @@ func NewClient(logger log.Logger, cfg EndpointsConfig, client *http.Client, prov } // Do executes an HTTP request with the underlying HTTP client. -func (c *Client) Do(req *http.Request) (*http.Response, error) { +func (c *HTTPClient) Do(req *http.Request) (*http.Response, error) { return c.httpClient.Do(req) } // Endpoints returns the list of known endpoints. -func (c *Client) Endpoints() []*url.URL { +func (c *HTTPClient) Endpoints() []*url.URL { var urls []*url.URL for _, addr := range c.provider.Addresses() { urls = append(urls, @@ -377,7 +386,7 @@ func (c *Client) Endpoints() []*url.URL { } // Discover runs the service to discover endpoints until the given context is done. -func (c *Client) Discover(ctx context.Context) { +func (c *HTTPClient) Discover(ctx context.Context) { var wg sync.WaitGroup ch := make(chan []*targetgroup.Group) @@ -407,6 +416,6 @@ func (c *Client) Discover(ctx context.Context) { } // Resolve refreshes and resolves the list of targets. -func (c *Client) Resolve(ctx context.Context) error { +func (c *HTTPClient) Resolve(ctx context.Context) error { return c.provider.Resolve(ctx, append(c.fileSDCache.Addresses(), c.staticAddresses...)) } diff --git a/pkg/httpconfig/config.go b/pkg/httpconfig/config.go deleted file mode 100644 index 3280e33378..0000000000 --- a/pkg/httpconfig/config.go +++ /dev/null @@ -1,75 +0,0 @@ -// Copyright (c) The Thanos Authors. -// Licensed under the Apache License 2.0. - -package httpconfig - -import ( - "fmt" - "net/url" - "strings" - - "gopkg.in/yaml.v2" - - "github.com/pkg/errors" -) - -// Config is a structure that allows pointing to various HTTP endpoint, e.g ruler connecting to queriers. -type Config struct { - HTTPClientConfig ClientConfig `yaml:"http_config"` - EndpointsConfig EndpointsConfig `yaml:",inline"` -} - -func DefaultConfig() Config { - return Config{ - EndpointsConfig: EndpointsConfig{ - Scheme: "http", - StaticAddresses: []string{}, - FileSDConfigs: []FileSDConfig{}, - }, - } -} - -// UnmarshalYAML implements the yaml.Unmarshaler interface. -func (c *Config) UnmarshalYAML(unmarshal func(interface{}) error) error { - *c = DefaultConfig() - type plain Config - return unmarshal((*plain)(c)) -} - -// LoadConfigs loads a list of Config from YAML data. -func LoadConfigs(confYAML []byte) ([]Config, error) { - var queryCfg []Config - if err := yaml.UnmarshalStrict(confYAML, &queryCfg); err != nil { - return nil, err - } - return queryCfg, nil -} - -// BuildConfig returns a configuration from a static addresses. -func BuildConfig(addrs []string) ([]Config, error) { - configs := make([]Config, 0, len(addrs)) - for i, addr := range addrs { - if addr == "" { - return nil, errors.Errorf("static address cannot be empty at index %d", i) - } - // If addr is missing schema, add http. - if !strings.Contains(addr, "://") { - addr = fmt.Sprintf("http://%s", addr) - } - u, err := url.Parse(addr) - if err != nil { - return nil, errors.Wrapf(err, "failed to parse addr %q", addr) - } - if u.Scheme != "http" && u.Scheme != "https" { - return nil, errors.Errorf("%q is not supported scheme for address", u.Scheme) - } - configs = append(configs, Config{ - EndpointsConfig: EndpointsConfig{ - Scheme: u.Scheme, - StaticAddresses: []string{u.Host}, - PathPrefix: u.Path, - }, - }) - } - return configs, nil -} diff --git a/pkg/httpconfig/config_test.go b/pkg/httpconfig/config_test.go deleted file mode 100644 index 4738c56544..0000000000 --- a/pkg/httpconfig/config_test.go +++ /dev/null @@ -1,94 +0,0 @@ -// Copyright (c) The Thanos Authors. -// Licensed under the Apache License 2.0. - -package httpconfig - -import ( - "testing" - - "github.com/efficientgo/core/testutil" -) - -func TestBuildConfig(t *testing.T) { - for _, tc := range []struct { - desc string - addresses []string - err bool - expected []Config - }{ - { - desc: "single addr without path", - addresses: []string{"localhost:9093"}, - expected: []Config{{ - EndpointsConfig: EndpointsConfig{ - StaticAddresses: []string{"localhost:9093"}, - Scheme: "http", - }, - }}, - }, - { - desc: "1st addr without path, 2nd with", - addresses: []string{"localhost:9093", "localhost:9094/prefix"}, - expected: []Config{ - { - EndpointsConfig: EndpointsConfig{ - StaticAddresses: []string{"localhost:9093"}, - Scheme: "http", - }, - }, - { - EndpointsConfig: EndpointsConfig{ - StaticAddresses: []string{"localhost:9094"}, - Scheme: "http", - PathPrefix: "/prefix", - }, - }, - }, - }, - { - desc: "single addr with path and http scheme", - addresses: []string{"http://localhost:9093"}, - expected: []Config{{ - EndpointsConfig: EndpointsConfig{ - StaticAddresses: []string{"localhost:9093"}, - Scheme: "http", - }, - }}, - }, - { - desc: "single addr with path and https scheme", - addresses: []string{"https://localhost:9093"}, - expected: []Config{{ - EndpointsConfig: EndpointsConfig{ - StaticAddresses: []string{"localhost:9093"}, - Scheme: "https", - }, - }}, - }, - { - desc: "not supported scheme", - addresses: []string{"ttp://localhost:9093"}, - err: true, - }, - { - desc: "invalid addr", - addresses: []string{"this is not a valid addr"}, - err: true, - }, - { - desc: "empty addr", - addresses: []string{""}, - err: true, - }, - } { - t.Run(tc.desc, func(t *testing.T) { - cfg, err := BuildConfig(tc.addresses) - if tc.err { - testutil.NotOk(t, err) - return - } - - testutil.Equals(t, tc.expected, cfg) - }) - } -} diff --git a/pkg/promclient/promclient.go b/pkg/promclient/promclient.go index 16c292d2f8..30cf85af40 100644 --- a/pkg/promclient/promclient.go +++ b/pkg/promclient/promclient.go @@ -33,8 +33,8 @@ import ( "google.golang.org/grpc/codes" "gopkg.in/yaml.v2" + "github.com/thanos-io/thanos/pkg/clientconfig" "github.com/thanos-io/thanos/pkg/exemplars/exemplarspb" - "github.com/thanos-io/thanos/pkg/httpconfig" "github.com/thanos-io/thanos/pkg/metadata/metadatapb" "github.com/thanos-io/thanos/pkg/rules/rulespb" "github.com/thanos-io/thanos/pkg/runutil" @@ -85,7 +85,7 @@ func NewClient(c HTTPClient, logger log.Logger, userAgent string) *Client { // NewDefaultClient returns Client with tracing tripperware. func NewDefaultClient() *Client { - client, _ := httpconfig.NewHTTPClient(httpconfig.ClientConfig{}, "") + client, _ := clientconfig.NewHTTPClient(clientconfig.HTTPClientConfig{}, "") return NewWithTracingClient( log.NewNopLogger(), client, diff --git a/pkg/query/remote_engine.go b/pkg/query/remote_engine.go index de926b250e..4cb0021083 100644 --- a/pkg/query/remote_engine.go +++ b/pkg/query/remote_engine.go @@ -76,7 +76,7 @@ func (r remoteEndpoints) Engines() []api.RemoteEngine { clients := r.getClients() engines := make([]api.RemoteEngine, len(clients)) for i := range clients { - engines[i] = newRemoteEngine(r.logger, clients[i], r.opts) + engines[i] = NewRemoteEngine(r.logger, clients[i], r.opts) } return engines } @@ -95,7 +95,7 @@ type remoteEngine struct { labelSets []labels.Labels } -func newRemoteEngine(logger log.Logger, queryClient Client, opts Opts) api.RemoteEngine { +func NewRemoteEngine(logger log.Logger, queryClient Client, opts Opts) *remoteEngine { return &remoteEngine{ logger: logger, client: queryClient, @@ -194,6 +194,19 @@ func (r *remoteEngine) NewRangeQuery(_ context.Context, opts promql.QueryOpts, q }, nil } +func (r *remoteEngine) NewInstantQuery(_ context.Context, _ promql.QueryOpts, qs string, ts time.Time) (promql.Query, error) { + return &remoteQuery{ + logger: r.logger, + client: r.client, + opts: r.opts, + + qs: qs, + start: ts, + end: ts, + interval: 0, + }, nil +} + type remoteQuery struct { logger log.Logger client Client @@ -219,6 +232,54 @@ func (r *remoteQuery) Exec(ctx context.Context) *promql.Result { maxResolution = int64(r.interval.Seconds() / 5) } + // Instant query. + if r.start == r.end { + request := &querypb.QueryRequest{ + Query: r.qs, + TimeSeconds: r.start.Unix(), + TimeoutSeconds: int64(r.opts.Timeout.Seconds()), + EnablePartialResponse: r.opts.EnablePartialResponse, + // TODO (fpetkovski): Allow specifying these parameters at query time. + // This will likely require a change in the remote engine interface. + ReplicaLabels: r.opts.ReplicaLabels, + MaxResolutionSeconds: maxResolution, + EnableDedup: true, + } + + qry, err := r.client.Query(qctx, request) + if err != nil { + return &promql.Result{Err: err} + } + result := make(promql.Vector, 0) + + for { + msg, err := qry.Recv() + if err == io.EOF { + break + } + if err != nil { + return &promql.Result{Err: err} + } + + if warn := msg.GetWarnings(); warn != "" { + return &promql.Result{Err: errors.New(warn)} + } + + ts := msg.GetTimeseries() + + // Point might have a different timestamp, force it to the evaluation + // timestamp as that is when we ran the evaluation. + // See https://github.com/prometheus/prometheus/blob/b727e69b7601b069ded5c34348dca41b80988f4b/promql/engine.go#L693-L699 + if len(ts.Histograms) > 0 { + result = append(result, promql.Sample{Metric: labelpb.ZLabelsToPromLabels(ts.Labels), H: prompb.FromProtoHistogram(ts.Histograms[0]), T: r.start.UnixMilli()}) + } else { + result = append(result, promql.Sample{Metric: labelpb.ZLabelsToPromLabels(ts.Labels), F: ts.Samples[0].Value, T: r.start.UnixMilli()}) + } + } + + return &promql.Result{Value: result} + } + request := &querypb.QueryRangeRequest{ Query: r.qs, StartTimeSeconds: r.start.Unix(), diff --git a/pkg/query/remote_engine_test.go b/pkg/query/remote_engine_test.go index 7ecfdf2301..224acc6039 100644 --- a/pkg/query/remote_engine_test.go +++ b/pkg/query/remote_engine_test.go @@ -23,7 +23,7 @@ import ( func TestRemoteEngine_Warnings(t *testing.T) { client := NewClient(&queryWarnClient{}, "", nil) - engine := newRemoteEngine(log.NewNopLogger(), client, Opts{ + engine := NewRemoteEngine(log.NewNopLogger(), client, Opts{ Timeout: 1 * time.Second, }) var ( @@ -87,7 +87,7 @@ func TestRemoteEngine_LabelSets(t *testing.T) { for _, testCase := range tests { t.Run(testCase.name, func(t *testing.T) { client := NewClient(nil, "", testCase.tsdbInfos) - engine := newRemoteEngine(log.NewNopLogger(), client, Opts{ + engine := NewRemoteEngine(log.NewNopLogger(), client, Opts{ ReplicaLabels: testCase.replicaLabels, }) @@ -174,7 +174,7 @@ func TestRemoteEngine_MinT(t *testing.T) { for _, testCase := range tests { t.Run(testCase.name, func(t *testing.T) { client := NewClient(nil, "", testCase.tsdbInfos) - engine := newRemoteEngine(log.NewNopLogger(), client, Opts{ + engine := NewRemoteEngine(log.NewNopLogger(), client, Opts{ ReplicaLabels: testCase.replicaLabels, }) diff --git a/pkg/receive/head_series_limiter.go b/pkg/receive/head_series_limiter.go index 994221c2f6..492e3ecdf2 100644 --- a/pkg/receive/head_series_limiter.go +++ b/pkg/receive/head_series_limiter.go @@ -14,8 +14,9 @@ import ( "github.com/go-kit/log/level" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promauto" + + "github.com/thanos-io/thanos/pkg/clientconfig" "github.com/thanos-io/thanos/pkg/errors" - "github.com/thanos-io/thanos/pkg/httpconfig" "github.com/thanos-io/thanos/pkg/promclient" ) @@ -86,13 +87,13 @@ func NewHeadSeriesLimit(w WriteLimitsConfig, registerer prometheus.Registerer, l limit.tenantCurrentSeriesMap = map[string]float64{} // Use specified HTTPConfig (if any) to make requests to meta-monitoring. - c := httpconfig.NewDefaultClientConfig() + c := clientconfig.NewDefaultHTTPClientConfig() if w.GlobalLimits.MetaMonitoringHTTPClient != nil { c = *w.GlobalLimits.MetaMonitoringHTTPClient } var err error - limit.metaMonitoringClient, err = httpconfig.NewHTTPClient(c, "meta-mon-for-limit") + limit.metaMonitoringClient, err = clientconfig.NewHTTPClient(c, "meta-mon-for-limit") if err != nil { level.Error(logger).Log("msg", "improper http client config", "err", err.Error()) } @@ -104,7 +105,7 @@ func NewHeadSeriesLimit(w WriteLimitsConfig, registerer prometheus.Registerer, l // solution with the configured query for getting current active (head) series of all tenants. // It then populates tenantCurrentSeries map with result. func (h *headSeriesLimit) QueryMetaMonitoring(ctx context.Context) error { - c := promclient.NewWithTracingClient(h.logger, h.metaMonitoringClient, httpconfig.ThanosUserAgent) + c := promclient.NewWithTracingClient(h.logger, h.metaMonitoringClient, clientconfig.ThanosUserAgent) vectorRes, _, _, err := c.QueryInstant(ctx, h.metaMonitoringURL, h.metaMonitoringQuery, time.Now(), promclient.QueryOptions{Deduplicate: true}) if err != nil { diff --git a/pkg/receive/limiter_config.go b/pkg/receive/limiter_config.go index c3bd330b6e..cce9af3cd1 100644 --- a/pkg/receive/limiter_config.go +++ b/pkg/receive/limiter_config.go @@ -6,9 +6,10 @@ package receive import ( "net/url" - "github.com/thanos-io/thanos/pkg/errors" - "github.com/thanos-io/thanos/pkg/httpconfig" "gopkg.in/yaml.v2" + + "github.com/thanos-io/thanos/pkg/clientconfig" + "github.com/thanos-io/thanos/pkg/errors" ) // RootLimitsConfig is the root configuration for limits. @@ -64,9 +65,9 @@ type GlobalLimitsConfig struct { // MaxConcurrency represents the maximum concurrency during write operations. MaxConcurrency int64 `yaml:"max_concurrency"` // MetaMonitoring options specify the query, url and client for Query API address used in head series limiting. - MetaMonitoringURL string `yaml:"meta_monitoring_url"` - MetaMonitoringHTTPClient *httpconfig.ClientConfig `yaml:"meta_monitoring_http_client"` - MetaMonitoringLimitQuery string `yaml:"meta_monitoring_limit_query"` + MetaMonitoringURL string `yaml:"meta_monitoring_url"` + MetaMonitoringHTTPClient *clientconfig.HTTPClientConfig `yaml:"meta_monitoring_http_client"` + MetaMonitoringLimitQuery string `yaml:"meta_monitoring_limit_query"` metaMonitoringURL *url.URL } diff --git a/pkg/rules/queryable.go b/pkg/rules/queryable.go index d178df7ba1..3d2335fd5a 100644 --- a/pkg/rules/queryable.go +++ b/pkg/rules/queryable.go @@ -20,7 +20,7 @@ import ( "github.com/prometheus/prometheus/util/annotations" "github.com/thanos-io/thanos/internal/cortex/querier/series" - "github.com/thanos-io/thanos/pkg/httpconfig" + "github.com/thanos-io/thanos/pkg/clientconfig" "github.com/thanos-io/thanos/pkg/promclient" "github.com/thanos-io/thanos/pkg/store/storepb" ) @@ -31,7 +31,7 @@ type promClientsQueryable struct { logger log.Logger promClients []*promclient.Client - queryClients []*httpconfig.Client + queryClients []*clientconfig.HTTPClient ignoredLabelNames []string duplicatedQuery prometheus.Counter @@ -43,7 +43,7 @@ type promClientsQuerier struct { logger log.Logger promClients []*promclient.Client - queryClients []*httpconfig.Client + queryClients []*clientconfig.HTTPClient restoreIgnoreLabels []string // We use a dummy counter here because the duplicated @@ -52,7 +52,7 @@ type promClientsQuerier struct { } // NewPromClientsQueryable creates a queryable that queries queriers from Prometheus clients. -func NewPromClientsQueryable(logger log.Logger, queryClients []*httpconfig.Client, promClients []*promclient.Client, +func NewPromClientsQueryable(logger log.Logger, queryClients []*clientconfig.HTTPClient, promClients []*promclient.Client, httpMethod string, step time.Duration, ignoredLabelNames []string) *promClientsQueryable { return &promClientsQueryable{ logger: logger, diff --git a/pkg/store/prometheus.go b/pkg/store/prometheus.go index ea48f7e1a2..197364ca04 100644 --- a/pkg/store/prometheus.go +++ b/pkg/store/prometheus.go @@ -33,9 +33,9 @@ import ( "google.golang.org/grpc/codes" "google.golang.org/grpc/status" + "github.com/thanos-io/thanos/pkg/clientconfig" "github.com/thanos-io/thanos/pkg/component" "github.com/thanos-io/thanos/pkg/dedup" - "github.com/thanos-io/thanos/pkg/httpconfig" "github.com/thanos-io/thanos/pkg/info/infopb" "github.com/thanos-io/thanos/pkg/promclient" "github.com/thanos-io/thanos/pkg/runutil" @@ -578,7 +578,7 @@ func (p *PrometheusStore) startPromRemoteRead(ctx context.Context, q *prompb.Que preq.Header.Set("Content-Type", "application/x-stream-protobuf") preq.Header.Set("X-Prometheus-Remote-Read-Version", "0.1.0") - preq.Header.Set("User-Agent", httpconfig.ThanosUserAgent) + preq.Header.Set("User-Agent", clientconfig.ThanosUserAgent) presp, err = p.client.Do(preq.WithContext(ctx)) if err != nil { return nil, errors.Wrap(err, "send request") diff --git a/pkg/store/storepb/prompb/custom.go b/pkg/store/storepb/prompb/custom.go index fb3b395a9a..5619977da9 100644 --- a/pkg/store/storepb/prompb/custom.go +++ b/pkg/store/storepb/prompb/custom.go @@ -3,7 +3,19 @@ package prompb +import ( + "github.com/prometheus/prometheus/model/histogram" +) + func (h Histogram) IsFloatHistogram() bool { _, ok := h.GetCount().(*Histogram_CountFloat) return ok } + +func FromProtoHistogram(h Histogram) *histogram.FloatHistogram { + if h.IsFloatHistogram() { + return FloatHistogramProtoToFloatHistogram(h) + } else { + return HistogramProtoToFloatHistogram(h) + } +} diff --git a/scripts/cfggen/main.go b/scripts/cfggen/main.go index d275d29cc4..d0674e7efc 100644 --- a/scripts/cfggen/main.go +++ b/scripts/cfggen/main.go @@ -27,10 +27,9 @@ import ( "gopkg.in/alecthomas/kingpin.v2" "gopkg.in/yaml.v2" - "github.com/thanos-io/thanos/pkg/httpconfig" - "github.com/thanos-io/thanos/pkg/alert" "github.com/thanos-io/thanos/pkg/cacheutil" + "github.com/thanos-io/thanos/pkg/clientconfig" "github.com/thanos-io/thanos/pkg/logging" "github.com/thanos-io/thanos/pkg/queryfrontend" storecache "github.com/thanos-io/thanos/pkg/store/cache" @@ -82,12 +81,12 @@ func init() { configs[name(logging.RequestConfig{})] = logging.RequestConfig{} alertmgrCfg := alert.DefaultAlertmanagerConfig() - alertmgrCfg.EndpointsConfig.FileSDConfigs = []httpconfig.FileSDConfig{{}} + alertmgrCfg.EndpointsConfig.FileSDConfigs = []clientconfig.HTTPFileSDConfig{{}} configs[name(alert.AlertingConfig{})] = alert.AlertingConfig{Alertmanagers: []alert.AlertmanagerConfig{alertmgrCfg}} - queryCfg := httpconfig.DefaultConfig() - queryCfg.EndpointsConfig.FileSDConfigs = []httpconfig.FileSDConfig{{}} - configs[name(httpconfig.Config{})] = []httpconfig.Config{queryCfg} + queryCfg := clientconfig.DefaultConfig() + queryCfg.HTTPConfig.EndpointsConfig.FileSDConfigs = []clientconfig.HTTPFileSDConfig{{}} + configs[name(clientconfig.Config{})] = []clientconfig.Config{queryCfg} for typ, config := range bucketConfigs { configs[name(config)] = client.BucketConfig{Type: typ, Config: config} diff --git a/test/e2e/compatibility_test.go b/test/e2e/compatibility_test.go index d8cab523b5..c910c96f3f 100644 --- a/test/e2e/compatibility_test.go +++ b/test/e2e/compatibility_test.go @@ -27,7 +27,7 @@ import ( "github.com/efficientgo/core/testutil" "github.com/thanos-io/thanos/pkg/alert" - "github.com/thanos-io/thanos/pkg/httpconfig" + "github.com/thanos-io/thanos/pkg/clientconfig" "github.com/thanos-io/thanos/pkg/queryfrontend" "github.com/thanos-io/thanos/pkg/store" "github.com/thanos-io/thanos/test/e2e/e2ethanos" @@ -198,7 +198,7 @@ func TestAlertCompliance(t *testing.T) { rFuture := e2ethanos.NewRulerBuilder(e, "1") ruler := rFuture.WithAlertManagerConfig([]alert.AlertmanagerConfig{ { - EndpointsConfig: httpconfig.EndpointsConfig{ + EndpointsConfig: clientconfig.HTTPEndpointsConfig{ StaticAddresses: []string{compliance.InternalEndpoint("http")}, Scheme: "http", }, @@ -210,13 +210,15 @@ func TestAlertCompliance(t *testing.T) { WithResendDelay("1m"). WithEvalInterval("1m"). WithReplicaLabel(""). - InitTSDB(filepath.Join(rFuture.InternalDir(), "rules"), []httpconfig.Config{ + InitTSDB(filepath.Join(rFuture.InternalDir(), "rules"), []clientconfig.Config{ { - EndpointsConfig: httpconfig.EndpointsConfig{ - StaticAddresses: []string{ - querierBuilder.InternalEndpoint("http"), + HTTPConfig: clientconfig.HTTPConfig{ + EndpointsConfig: clientconfig.HTTPEndpointsConfig{ + StaticAddresses: []string{ + querierBuilder.InternalEndpoint("http"), + }, + Scheme: "http", }, - Scheme: "http", }, }, }) @@ -279,7 +281,7 @@ func TestAlertCompliance(t *testing.T) { ruler := rFuture.WithAlertManagerConfig([]alert.AlertmanagerConfig{ { - EndpointsConfig: httpconfig.EndpointsConfig{ + EndpointsConfig: clientconfig.HTTPEndpointsConfig{ StaticAddresses: []string{compliance.InternalEndpoint("http")}, Scheme: "http", }, @@ -292,13 +294,15 @@ func TestAlertCompliance(t *testing.T) { WithEvalInterval("1m"). WithReplicaLabel(""). WithRestoreIgnoredLabels("tenant_id"). - InitStateless(filepath.Join(rFuture.InternalDir(), "rules"), []httpconfig.Config{ + InitStateless(filepath.Join(rFuture.InternalDir(), "rules"), []clientconfig.Config{ { - EndpointsConfig: httpconfig.EndpointsConfig{ - StaticAddresses: []string{ - query.InternalEndpoint("http"), + HTTPConfig: clientconfig.HTTPConfig{ + EndpointsConfig: clientconfig.HTTPEndpointsConfig{ + StaticAddresses: []string{ + query.InternalEndpoint("http"), + }, + Scheme: "http", }, - Scheme: "http", }, }, }, []*config.RemoteWriteConfig{ diff --git a/test/e2e/e2ethanos/services.go b/test/e2e/e2ethanos/services.go index 0401868d89..73f6b68925 100644 --- a/test/e2e/e2ethanos/services.go +++ b/test/e2e/e2ethanos/services.go @@ -29,8 +29,7 @@ import ( "github.com/thanos-io/objstore/exthttp" "github.com/thanos-io/thanos/pkg/alert" - "github.com/thanos-io/thanos/pkg/httpconfig" - + "github.com/thanos-io/thanos/pkg/clientconfig" "github.com/thanos-io/thanos/pkg/queryfrontend" "github.com/thanos-io/thanos/pkg/receive" ) @@ -735,15 +734,15 @@ func (r *RulerBuilder) WithRestoreIgnoredLabels(labels ...string) *RulerBuilder return r } -func (r *RulerBuilder) InitTSDB(internalRuleDir string, queryCfg []httpconfig.Config) *e2eobs.Observable { +func (r *RulerBuilder) InitTSDB(internalRuleDir string, queryCfg []clientconfig.Config) *e2eobs.Observable { return r.initRule(internalRuleDir, queryCfg, nil) } -func (r *RulerBuilder) InitStateless(internalRuleDir string, queryCfg []httpconfig.Config, remoteWriteCfg []*config.RemoteWriteConfig) *e2eobs.Observable { +func (r *RulerBuilder) InitStateless(internalRuleDir string, queryCfg []clientconfig.Config, remoteWriteCfg []*config.RemoteWriteConfig) *e2eobs.Observable { return r.initRule(internalRuleDir, queryCfg, remoteWriteCfg) } -func (r *RulerBuilder) initRule(internalRuleDir string, queryCfg []httpconfig.Config, remoteWriteCfg []*config.RemoteWriteConfig) *e2eobs.Observable { +func (r *RulerBuilder) initRule(internalRuleDir string, queryCfg []clientconfig.Config, remoteWriteCfg []*config.RemoteWriteConfig) *e2eobs.Observable { if err := os.MkdirAll(r.f.Dir(), 0750); err != nil { return &e2eobs.Observable{Runnable: e2e.NewFailedRunnable(r.Name(), errors.Wrap(err, "create rule dir"))} } diff --git a/test/e2e/native_histograms_test.go b/test/e2e/native_histograms_test.go index dc9ce19f00..29c1532661 100644 --- a/test/e2e/native_histograms_test.go +++ b/test/e2e/native_histograms_test.go @@ -6,6 +6,8 @@ package e2e_test import ( "context" "fmt" + "os" + "path/filepath" "reflect" "testing" "time" @@ -14,19 +16,33 @@ import ( "github.com/efficientgo/e2e" e2emon "github.com/efficientgo/e2e/monitoring" "github.com/efficientgo/e2e/monitoring/matchers" + common_cfg "github.com/prometheus/common/config" "github.com/prometheus/common/model" + "github.com/prometheus/prometheus/config" "github.com/prometheus/prometheus/model/histogram" "github.com/prometheus/prometheus/prompb" "github.com/prometheus/prometheus/storage/remote" "github.com/prometheus/prometheus/tsdb/tsdbutil" + "github.com/thanos-io/thanos/pkg/clientconfig" "github.com/thanos-io/thanos/pkg/promclient" "github.com/thanos-io/thanos/pkg/queryfrontend" "github.com/thanos-io/thanos/pkg/receive" "github.com/thanos-io/thanos/test/e2e/e2ethanos" ) -const testHistogramMetricName = "fake_histogram" +const ( + testHistogramMetricName = "fake_histogram" + + testRuleRecordHistogramSum = ` +groups: +- name: fake_histogram_sum + interval: 1s + rules: + - record: fake_histogram:sum + expr: sum(fake_histogram) +` +) func TestQueryNativeHistograms(t *testing.T) { e, err := e2e.NewDockerEnvironment("nat-hist-query") @@ -73,7 +89,6 @@ func TestQueryNativeHistograms(t *testing.T) { &model.Sample{ Value: 39, Metric: model.Metric{ - "foo": "bar", "prometheus": "prom-ha", }, }, @@ -184,9 +199,8 @@ func TestQueryFrontendNativeHistograms(t *testing.T) { // doesn't need to retry when we send queries to the frontend later. queryAndAssertSeries(t, ctx, querier.Endpoint("http"), func() string { return testHistogramMetricName }, time.Now, promclient.QueryOptions{Deduplicate: true}, []model.Metric{ { - "__name__": testHistogramMetricName, - "prometheus": "prom-ha", - "foo": "bar", + model.MetricNameLabel: testHistogramMetricName, + "prometheus": "prom-ha", }, }) @@ -323,7 +337,66 @@ func TestQueryFrontendNativeHistograms(t *testing.T) { }) } -func writeHistograms(ctx context.Context, now time.Time, name string, histograms []*histogram.Histogram, floatHistograms []*histogram.FloatHistogram, rawRemoteWriteURL string) (time.Time, error) { +func TestRuleNativeHistograms(t *testing.T) { + t.Parallel() + + e, err := e2e.NewDockerEnvironment("hist-rule-rw") + testutil.Ok(t, err) + t.Cleanup(e2ethanos.CleanScenario(t, e)) + + ctx, cancel := context.WithTimeout(context.Background(), 3*time.Minute) + t.Cleanup(cancel) + + rFuture := e2ethanos.NewRulerBuilder(e, "1") + rulesSubDir := "rules" + rulesPath := filepath.Join(rFuture.Dir(), rulesSubDir) + testutil.Ok(t, os.MkdirAll(rulesPath, os.ModePerm)) + + for i, rule := range []string{testRuleRecordHistogramSum} { + createRuleFile(t, filepath.Join(rulesPath, fmt.Sprintf("rules-%d.yaml", i)), rule) + } + + receiver := e2ethanos.NewReceiveBuilder(e, "1").WithIngestionEnabled().WithNativeHistograms().Init() + testutil.Ok(t, e2e.StartAndWaitReady(receiver)) + rwURL := urlParse(t, e2ethanos.RemoteWriteEndpoint(receiver.InternalEndpoint("remote-write"))) + + receiver2 := e2ethanos.NewReceiveBuilder(e, "2").WithIngestionEnabled().WithNativeHistograms().Init() + testutil.Ok(t, e2e.StartAndWaitReady(receiver2)) + rwURL2 := urlParse(t, e2ethanos.RemoteWriteEndpoint(receiver2.InternalEndpoint("remote-write"))) + + q := e2ethanos.NewQuerierBuilder(e, "1", receiver.InternalEndpoint("grpc"), receiver2.InternalEndpoint("grpc")).WithReplicaLabels("receive", "replica").Init() + testutil.Ok(t, e2e.StartAndWaitReady(q)) + + histograms := tsdbutil.GenerateTestHistograms(4) + ts := time.Now().Add(-2 * time.Minute) + rawRemoteWriteURL1 := "http://" + receiver.Endpoint("remote-write") + "/api/v1/receive" + _, err = writeHistograms(ctx, ts, testHistogramMetricName, histograms, nil, rawRemoteWriteURL1, prompb.Label{Name: "series", Value: "one"}) + testutil.Ok(t, err) + rawRemoteWriteURL2 := "http://" + receiver2.Endpoint("remote-write") + "/api/v1/receive" + _, err = writeHistograms(ctx, ts, testHistogramMetricName, histograms, nil, rawRemoteWriteURL2, prompb.Label{Name: "series", Value: "two"}) + testutil.Ok(t, err) + + r := rFuture.InitStateless(filepath.Join(rFuture.InternalDir(), rulesSubDir), []clientconfig.Config{ + { + GRPCConfig: &clientconfig.GRPCConfig{ + EndpointAddrs: []string{q.InternalEndpoint("grpc")}, + }, + }, + }, []*config.RemoteWriteConfig{ + {URL: &common_cfg.URL{URL: rwURL}, Name: "thanos-receiver", SendNativeHistograms: true}, + {URL: &common_cfg.URL{URL: rwURL2}, Name: "thanos-receiver2", SendNativeHistograms: true}, + }) + testutil.Ok(t, e2e.StartAndWaitReady(r)) + + // Wait until remote write samples are written to receivers successfully. + testutil.Ok(t, r.WaitSumMetricsWithOptions(e2emon.GreaterOrEqual(1), []string{"prometheus_remote_storage_histograms_total"}, e2emon.WaitMissingMetrics())) + + expectedRecordedName := testHistogramMetricName + ":sum" + expectedRecordedHistogram := histograms[len(histograms)-1].ToFloat(nil).Mul(2) + queryAndAssert(t, ctx, q.Endpoint("http"), func() string { return expectedRecordedName }, time.Now, promclient.QueryOptions{Deduplicate: true}, expectedHistogramModelVector(expectedRecordedName, nil, expectedRecordedHistogram, map[string]string{"tenant_id": "default-tenant"})) +} + +func writeHistograms(ctx context.Context, now time.Time, name string, histograms []*histogram.Histogram, floatHistograms []*histogram.FloatHistogram, rawRemoteWriteURL string, labels ...prompb.Label) (time.Time, error) { startTime := now.Add(time.Duration(len(histograms)-1) * -30 * time.Second).Truncate(30 * time.Second) prompbHistograms := make([]prompb.Histogram, 0, len(histograms)) @@ -338,10 +411,9 @@ func writeHistograms(ctx context.Context, now time.Time, name string, histograms } timeSeriespb := prompb.TimeSeries{ - Labels: []prompb.Label{ - {Name: "__name__", Value: name}, - {Name: "foo", Value: "bar"}, - }, + Labels: append([]prompb.Label{ + {Name: model.MetricNameLabel, Value: name}, + }, labels...), Histograms: prompbHistograms, } @@ -352,8 +424,7 @@ func writeHistograms(ctx context.Context, now time.Time, name string, histograms func expectedHistogramModelVector(metricName string, histogram *histogram.Histogram, floatHistogram *histogram.FloatHistogram, externalLabels map[string]string) model.Vector { metrics := model.Metric{ - "__name__": model.LabelValue(metricName), - "foo": "bar", + model.MetricNameLabel: model.LabelValue(metricName), } for labelKey, labelValue := range externalLabels { metrics[model.LabelName(labelKey)] = model.LabelValue(labelValue) @@ -377,8 +448,7 @@ func expectedHistogramModelVector(metricName string, histogram *histogram.Histog func expectedHistogramModelMatrix(metricName string, histograms []*histogram.Histogram, floatHistograms []*histogram.FloatHistogram, startTime time.Time, externalLabels map[string]string) model.Matrix { metrics := model.Metric{ - "__name__": model.LabelValue(metricName), - "foo": "bar", + model.MetricNameLabel: model.LabelValue(metricName), } for labelKey, labelValue := range externalLabels { metrics[model.LabelName(labelKey)] = model.LabelValue(labelValue) diff --git a/test/e2e/rule_test.go b/test/e2e/rule_test.go index ce636eac3e..4f918a0c5f 100644 --- a/test/e2e/rule_test.go +++ b/test/e2e/rule_test.go @@ -27,7 +27,7 @@ import ( "github.com/efficientgo/core/testutil" "github.com/thanos-io/thanos/pkg/alert" - "github.com/thanos-io/thanos/pkg/httpconfig" + "github.com/thanos-io/thanos/pkg/clientconfig" "github.com/thanos-io/thanos/pkg/promclient" "github.com/thanos-io/thanos/pkg/rules/rulespb" "github.com/thanos-io/thanos/pkg/runutil" @@ -323,8 +323,8 @@ func TestRule(t *testing.T) { r := rFuture.WithAlertManagerConfig([]alert.AlertmanagerConfig{ { - EndpointsConfig: httpconfig.EndpointsConfig{ - FileSDConfigs: []httpconfig.FileSDConfig{ + EndpointsConfig: clientconfig.HTTPEndpointsConfig{ + FileSDConfigs: []clientconfig.HTTPFileSDConfig{ { // FileSD which will be used to register discover dynamically am1. Files: []string{filepath.Join(rFuture.InternalDir(), amTargetsSubDir, "*.yaml")}, @@ -339,18 +339,20 @@ func TestRule(t *testing.T) { Timeout: amTimeout, APIVersion: alert.APIv1, }, - }).InitTSDB(filepath.Join(rFuture.InternalDir(), rulesSubDir), []httpconfig.Config{ + }).InitTSDB(filepath.Join(rFuture.InternalDir(), rulesSubDir), []clientconfig.Config{ { - EndpointsConfig: httpconfig.EndpointsConfig{ - // We test Statically Addressed queries in other tests. Focus on FileSD here. - FileSDConfigs: []httpconfig.FileSDConfig{ - { - // FileSD which will be used to register discover dynamically q. - Files: []string{filepath.Join(rFuture.InternalDir(), queryTargetsSubDir, "*.yaml")}, - RefreshInterval: model.Duration(time.Second), + HTTPConfig: clientconfig.HTTPConfig{ + EndpointsConfig: clientconfig.HTTPEndpointsConfig{ + // We test Statically Addressed queries in other tests. Focus on FileSD here. + FileSDConfigs: []clientconfig.HTTPFileSDConfig{ + { + // FileSD which will be used to register discover dynamically q. + Files: []string{filepath.Join(rFuture.InternalDir(), queryTargetsSubDir, "*.yaml")}, + RefreshInterval: model.Duration(time.Second), + }, }, + Scheme: "http", }, - Scheme: "http", }, }, }) @@ -556,18 +558,20 @@ func TestRule_KeepFiringFor(t *testing.T) { testutil.Ok(t, os.MkdirAll(rulesPath, os.ModePerm)) createRuleFile(t, filepath.Join(rulesPath, "alert_keep_firing_for.yaml"), testAlertRuleKeepFiringFor) - r := rFuture.InitTSDB(filepath.Join(rFuture.InternalDir(), rulesSubDir), []httpconfig.Config{ + r := rFuture.InitTSDB(filepath.Join(rFuture.InternalDir(), rulesSubDir), []clientconfig.Config{ { - EndpointsConfig: httpconfig.EndpointsConfig{ - // We test Statically Addressed queries in other tests. Focus on FileSD here. - FileSDConfigs: []httpconfig.FileSDConfig{ - { - // FileSD which will be used to register discover dynamically q. - Files: []string{filepath.Join(rFuture.InternalDir(), queryTargetsSubDir, "*.yaml")}, - RefreshInterval: model.Duration(time.Second), + HTTPConfig: clientconfig.HTTPConfig{ + EndpointsConfig: clientconfig.HTTPEndpointsConfig{ + // We test Statically Addressed queries in other tests. Focus on FileSD here. + FileSDConfigs: []clientconfig.HTTPFileSDConfig{ + { + // FileSD which will be used to register discover dynamically q. + Files: []string{filepath.Join(rFuture.InternalDir(), queryTargetsSubDir, "*.yaml")}, + RefreshInterval: model.Duration(time.Second), + }, }, + Scheme: "http", }, - Scheme: "http", }, }, }) @@ -666,7 +670,7 @@ func TestRule_CanRemoteWriteData(t *testing.T) { r := rFuture.WithAlertManagerConfig([]alert.AlertmanagerConfig{ { - EndpointsConfig: httpconfig.EndpointsConfig{ + EndpointsConfig: clientconfig.HTTPEndpointsConfig{ StaticAddresses: []string{ am.InternalEndpoint("http"), }, @@ -675,13 +679,15 @@ func TestRule_CanRemoteWriteData(t *testing.T) { Timeout: amTimeout, APIVersion: alert.APIv1, }, - }).InitStateless(filepath.Join(rFuture.InternalDir(), rulesSubDir), []httpconfig.Config{ + }).InitStateless(filepath.Join(rFuture.InternalDir(), rulesSubDir), []clientconfig.Config{ { - EndpointsConfig: httpconfig.EndpointsConfig{ - StaticAddresses: []string{ - q.InternalEndpoint("http"), + HTTPConfig: clientconfig.HTTPConfig{ + EndpointsConfig: clientconfig.HTTPEndpointsConfig{ + StaticAddresses: []string{ + q.InternalEndpoint("http"), + }, + Scheme: "http", }, - Scheme: "http", }, }, }, []*config.RemoteWriteConfig{ @@ -747,7 +753,7 @@ func TestStatelessRulerAlertStateRestore(t *testing.T) { } r := rFuture.WithAlertManagerConfig([]alert.AlertmanagerConfig{ { - EndpointsConfig: httpconfig.EndpointsConfig{ + EndpointsConfig: clientconfig.HTTPEndpointsConfig{ StaticAddresses: []string{ am.InternalEndpoint("http"), }, @@ -758,13 +764,15 @@ func TestStatelessRulerAlertStateRestore(t *testing.T) { }, }).WithForGracePeriod("500ms"). WithRestoreIgnoredLabels("tenant_id"). - InitStateless(filepath.Join(rFuture.InternalDir(), rulesSubDir), []httpconfig.Config{ + InitStateless(filepath.Join(rFuture.InternalDir(), rulesSubDir), []clientconfig.Config{ { - EndpointsConfig: httpconfig.EndpointsConfig{ - StaticAddresses: []string{ - q.InternalEndpoint("http"), + HTTPConfig: clientconfig.HTTPConfig{ + EndpointsConfig: clientconfig.HTTPEndpointsConfig{ + StaticAddresses: []string{ + q.InternalEndpoint("http"), + }, + Scheme: "http", }, - Scheme: "http", }, }, }, []*config.RemoteWriteConfig{ diff --git a/test/e2e/rules_api_test.go b/test/e2e/rules_api_test.go index 6303ed3d7e..2b5670a598 100644 --- a/test/e2e/rules_api_test.go +++ b/test/e2e/rules_api_test.go @@ -20,7 +20,7 @@ import ( "github.com/prometheus/prometheus/rules" "github.com/efficientgo/core/testutil" - "github.com/thanos-io/thanos/pkg/httpconfig" + "github.com/thanos-io/thanos/pkg/clientconfig" "github.com/thanos-io/thanos/pkg/promclient" "github.com/thanos-io/thanos/pkg/rules/rulespb" "github.com/thanos-io/thanos/pkg/runutil" @@ -67,11 +67,13 @@ func TestRulesAPI_Fanout(t *testing.T) { ) testutil.Ok(t, e2e.StartAndWaitReady(prom1, sidecar1, prom2, sidecar2)) - queryCfg := []httpconfig.Config{ + queryCfg := []clientconfig.Config{ { - EndpointsConfig: httpconfig.EndpointsConfig{ - StaticAddresses: []string{qBuilder.InternalEndpoint("http")}, - Scheme: "http", + HTTPConfig: clientconfig.HTTPConfig{ + EndpointsConfig: clientconfig.HTTPEndpointsConfig{ + StaticAddresses: []string{qBuilder.InternalEndpoint("http")}, + Scheme: "http", + }, }, }, } From b16b71d9680d18de67b5bb40ca75d12c5926b6b4 Mon Sep 17 00:00:00 2001 From: Ben Ye Date: Wed, 20 Dec 2023 20:57:39 -0800 Subject: [PATCH 02/26] mdox ignore checking twitter urls (#7001) Signed-off-by: Ben Ye --- .mdox.validate.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.mdox.validate.yaml b/.mdox.validate.yaml index 73632ba748..94355e319e 100644 --- a/.mdox.validate.yaml +++ b/.mdox.validate.yaml @@ -40,3 +40,5 @@ validators: type: 'ignore' - regex: 'krisztianfekete\.org' type: 'ignore' + - regex: 'twitter\.com' + type: 'ignore' From d37b686ff59cce838a573c41a3272f20c2e5e84b Mon Sep 17 00:00:00 2001 From: Kartikay <120778728+kartikaysaxena@users.noreply.github.com> Date: Fri, 22 Dec 2023 03:05:14 +0530 Subject: [PATCH 03/26] Updated Grofers logo (#7006) Signed-off-by: Kartikay --- website/.hugo_build.lock | 0 website/data/adopters.yml | 6 +++--- website/static/logos/blinkit.png | Bin 0 -> 2570 bytes website/static/logos/grofers.png | Bin 18955 -> 0 bytes 4 files changed, 3 insertions(+), 3 deletions(-) create mode 100644 website/.hugo_build.lock create mode 100644 website/static/logos/blinkit.png delete mode 100644 website/static/logos/grofers.png diff --git a/website/.hugo_build.lock b/website/.hugo_build.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/website/data/adopters.yml b/website/data/adopters.yml index 5b2ad40aeb..7605b38f59 100644 --- a/website/data/adopters.yml +++ b/website/data/adopters.yml @@ -69,9 +69,9 @@ adopters: - name: Amadeus url: https://amadeus.com logo: amadeus.png -- name: Grofers - url: https://grofers.com - logo: grofers.png +- name: Blinkit + url: https://blinkit.com + logo: blinkit.png - name: Tencent url: https://github.com/tkestack/tke logo: tencent.png diff --git a/website/static/logos/blinkit.png b/website/static/logos/blinkit.png new file mode 100644 index 0000000000000000000000000000000000000000..22e2add1af68d75963a632b43af4c1b155129fd1 GIT binary patch literal 2570 zcmV+l3ib7gP)N&`0{sMgV{v01O)5wm|;PM*s>M(yTt8gENzQFaQ7-00tW6yh7o^ zL7k{H@5Dr`jWzhmJ9(Tfq=+;a865Y=EuyM5{OP^_{P+LVN&Dldjdd>%6B}feD)`P- z{`&9v(q;Yf+4|dy000vJdKrRjExDLD`PO&)-j~j)JtH9i}C`FAWgP$+2uQvj)>LCCC2>(e$K~#90 z?VW3TqBs;)j3Md**8rA|KmNj|X&~0RdCNQUv zaPo$J^HNNfLSxJser>uSSN!^#p#p*h#f)L&Lo_BEsL;?=Ly#-Fa>I!#f?RPTGaNQz zB2f)y4JC$+9m&YB!lI$Ck|0;~^M+$}8exnx&KZX4G{P7bW({4nVpNcH8EZrJ1bI`0 zhKi+`n02y=4OL4sF-~fzWECcrNK|cw0xL08FpR5(bZFeU0{QUTAVM4KvYIOT>!)xJW&xZhCaX2=`jkW5q#QXZxg zqI9AWsfH-r-wQ0CD4j}0G9|Yvr&69wZ;7%Kix?IXl}Dul)%_3zK@bE%5ClOG1VIo4 zK@bE%5ClOG1VIpFFR;y@3MVqSk=?V`XYCCJ+2X{?*X{B!$vN9TY6Cs}q9=UmzH&Vq zfD^AMGt*6DMyoxHEx&Gqq?h9|vNg8sUpzB#dT$_>Hh zRaL`&dePZsxncV#p`93RcD4UNLo`G~G(^J&8Sc^_Ry36KhwpDlh?!FLVbu&J?B7r_ zKq?r%J^m^jJQPDlJ+r%5bWM-Kabez&rifu`b~C*T0e2}ggcno3`#6O}TNE17&5_*< zDScYC;`X&%Exb)J@`mQgr-YLQV*{|sZ^OL)Y-vP|^!~(597_yY_v9qK{F0g%pA#|7>Ha+V@_%JMeihc=^3;2 zbV+W=%+6UaMqJ&6nTF@mXzK{D&OucxF>6Q%_qyEz*!qh}<%Zc!PCGLdaALYx!&SDqn94p$#Bru!NXz#y-&9^e46R4EOvYosiC(zOHb7K-Wm;im)Qzj zm(GYwSJ}{h>9z9M`hdv|=_U9$^qE)a@%L%iDwYhCuCifE@-emSZ&YeX`$l?F=etMh zH7vn?pj#PAiQ=zd$jl)9Jc5U)_xu3EmVQEi+t9xAB!)1{*c~(ON$WPWJ7@Y?kc~9m zWLp~YKH@CM?9uzIV3^G^-F4Q^8Ag2vblbhRh)Sj}uP*18Gdq)XdfZ_fLtDQ;;Wz0) z$6od#?v~v<_({(udjckyMA0pJnz!ubAr~B7Y&`fxebbE+glmh_9D6qdjxxf z?|H*ameEf=A~j@t9JTRM+>=c|OkRMZzMxy=Qac$Xc#r8lctgw?!YtE!Oa$wkAM&Ds z)o8}<(gJXa>W2K|NbW&kJIfh*Aj<^zU>shfq5cGCG_X9uPp^89fA6KYzv^uE8`&tG z&r*B41wY#Q3`IhRc;C|xxl`RRnp-8MeMzWPqr}AmNKG@B0P-+OUvnv7bxDv&HVYj;B<+g^wBWxb;X-MD6Fq`7) zH08G^EGOG9ZyBNYat+mI`E`53r zn~cu8${Gx98~k(g>^cp5nGO+-{CT$9zoET^RmA6 z5vy!f43D0-SSv3Y+IoMzKFYlQ85ngKgX*)Vs)l?*==}`RXSddI`i5cKj2FUpGi=$$ zY#^p~UUvnTJna{6?}qw`#}9)UVsGY^8h!?o{2wNtA%6a~^D0J1e|TK|wahM@@_uCB zhC$Ts=qZ?98GYG0O=S2hvyAcg-|gznQL++c2SIL`#=hHTC90ZXvYNOA`E*GoYi)Vy zBXBWez zY_)p5P+Q&5*vb&zC~6K;LqdwHraKtw{X6h^ZUpb5h1x9)=i3KfGxg1+|H1} zWm6HSQbR&N!&+>&t9FR=)X2PjNe7RH5qB1LpzAAhQx5Rl`wywwTQIYm678Vo<>P~R*0XZa5)B=c5ZzE6$K;@B4ogzcY;cX-X$kYsw9kPBoUx4aX`s zL}$zmHI+xEBTX18+~-?VAhs^pdDjC^^1sctAF-6`Rg-1AUG856~E$kcsHubQD?IC1^yZFm@8rjACp gQ{SCPdUqNB0mcsZU%Y}{OaK4?07*qoM6N<$f|}ItZvX%Q literal 0 HcmV?d00001 diff --git a/website/static/logos/grofers.png b/website/static/logos/grofers.png deleted file mode 100644 index f73d67cd1b6a4e52bedd4a74e7a4dd619ef7c299..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 18955 zcmbSz^Bhc>)N4kFL@w|{O9HrVQup1bnmh#DJBX5W6 z9)B%eSH<6bZ+-g46dAz6R-4>E%(PT%D|CgZ8g{nWqqhjhy<_AT_3~Yb^$oz;ak=rLN5?wa< zn}X!#a2kP2d;ow?fatu`6s}3yiOF&m-yV!s# zI06c=@y`#CgCs^+NX4<>M&J@QKhH@^C1KwPTl^FY*-nFcIDWkQ?2ZixxaX32X?}|Z z2slu?tJ4AiW`%Yz1^|E}y9DkfBml(d!&);E0^rhl1VvH`(DljC88LGT08!ngWhT9X zpCIyW$jGyf0s+ulc4QaeemjJQ z7p)_PpWyC0Q0IU@zcU8G4?o8pA6Tzt{;sWgi@e45NonB+!&1deVHG(_X->xRgwe}# zkC*cdm&70d;Lns;ySQJ}GMnnvXndWGJWULDVb*P+iU0s`n>cs(qX>(j9fIGG+t_wv z%-d%#4gl;?Mep)O&8D$%ObiC?=i^d^0{|=aQZruy-}=i@MzvUgh0b;1kv(ivPD>U1 z^|X&KGZ%Zai{VzeimT(A?O$Ez=LmT7HA2bZne3>3y=0p&jnv&8n50Z(JW$z`0#U*f z_Bcq9X-_j*k_^X#iV~m#06ccn9Bl2)HYOjat}4E_VE_PCHXW>uHhm2%W82J8pujL7 z0>F5-%|OmZsqSFMS*u-}N>P=B76IVc_)fUI8evas_T#(lRt_PgDQtL4N(A|(7wIYf zpcyLXn2XDY*G`N2qUQ{7#mahta)<4bVqzE!4>)?^RsQa+B=VFv%CMu50f4s}r=bUF8y(5EVkJO%ct%WS6hPH5Nez3x=88AmZ{XCc*6YqbB3~^y zI>pyudMDb9OnGs+xI~o{w2H$6pT@I?AAI7O^xKlj-t`V@*yw8v(Fh$I&_M z0D0K)VggzZc!xOA0{~q`zc@7`WgWsPJ$RR@vf+Lz5r^!0p^K|6(dM}Ieg>Ry8`$k! z3EAZPDO(O1mOUk8z&g44oC>F4DIZx^x!-6xygnZSS9BxqyuhJ`T?}EYxBvi1wX6f- zY3FsGzo0)90BB;GC>90#Ca>8K-ZOr{Hvs@hFaN-zJv~!81hf30gYUp_V8vziOuUBh zD-PS8!7)r9Z`PN7yaw48k3Nd2TO?vLfK;}Hm4E?TK$ zb@HRwDr_@=q8`yqb0>Chy<|4APTJp=vfS^}?3>Y~&I14(RoW<6UF#y@iZ&^GpL`xZhs*$Qju!J}z;{-71iPEG(4*BMAVj=X zV@E322oYeqRD#U5-lPhEFzC`3%VAca0m8O&$f{#E7!d#(W8P`f-nB@8KF2d8wjxVm zAiz|_GD4IYuzu%-=!JyPg&}GdaY+RNOsG-b#~2HAar^uZOyI!{HgPQg=LnI0K@QAe z0l(~$oqTc_r?L{*9}Czm%Te6@L-WXNkPRu{dl82c5bwK;Y%?5QEel{-$aLGAzcS|s zz_j_lR1udWX(PZ?8kNq=Mcc6e5U=@5h0>iv_=>yJL}0DAD*^zR7ndp=mWToXa%r;o zsYn1A!;Av{4BSG7;mBax1mKuDu=6e91sMteh8Y6&Gh;#pU{9zLC>AxN0U&^$Pr+#InM|y5tCc<+k zv%+mg=929^>LolxAO<&jQ0Z)rrYfBgOj#0^;@40 zgIXE})$k_SH#T_Sjlb+&w#0AJ?s!>LQgis3ebZzk5WY_YsP*a_MdB1(ek)nLNL}hP z7J@IIfU~+EpXfil;Yy-j4KKV!|^R&Rcux6O`H?NZ|tY0;}6;V02nBq;ryjH_jtf&LU6G+c5m zi_^7s^z2-@MH{r3fHVL5(eEARQ#0Q0-bN>@9vh{^8#b_iBjr(O)$aazm3u9KwKtb> z<%fT*bFBbuCE<8EF}boOGpjQ$w8B{U{e11ZcBuz7V{N{nQG%I=xc2DM^!;w)k&O@! zL8bSfxh?%qH}}XE9>2VW+EXRiFPgjvM$p2|kAJeGaZEeptn2n#9e=m;2Kww&6)1D-Z|{o{2@EJl+=j{-^Hj)$-NYT>0%7*PHFx@;Mx34 zuE0DDO*;a50|$Q!?3j-RejcG?l$E7KM}A+IplmaxleYHMR@q^#5W?S{8>`q$;ks+y zk}teikUyE%`;2jiBpCaV4p`f&H~Hg5e;ykDb(k5Sno<-&HLe!WnLJ~KNT)~XQvO;l zh|kbgYfH-7ad|?e1<|EJa(I=DUabf!f{HOVRKKThu`By$`fx^{VM-{)^*b*!F{j(% zn%k$y5<-<-nf$Mb8$&A0@us`|$%A7B>@~HylWMu}GT`D!LGQYA4-bNIoGmFQDB@&0 zqH-sHs)P?Jtk(RoveUfw?l1lc!~CrmyI`*n@f%yGYf1T&U#iGAkSRSq&la&x7kMRe zxl8*980>{8^~3o6Z6E!545?7{4|=19!yU7xO7C6rFABr6-$q1cWYJ_Y)PcO4IAbjN zC8>H6qp19!=4IhvQF1zH#HqzCYWVlSjKBu$ zZU|la&ROyni^dCGkz99$E>_dml%T|bpR!(|wde)ei6wt{p^2a8d(pfvS`lflahYc! zh*d~C-{i;@#0D=iK7ZB=e+)#~q<7RQQ+iT`V4i$U*zhBYP<-_F@L6Gx zWy*Wc9*M_R`J6Y*miJ*-H|IZ`2j$9$M}f?nB#)3g-d@GEi%luhhjjz8Dk+^g7wJN` zV|(e|=Dqp5lHrv8sXKxO52utpGm{GPE@d96_=6a0JBoq|ep83Z4fP#O;Js2+VI}U$ zQxj`*#E?VIG5P+1?cMx0^Mj(Er&9SE@M>!(tg##MXwrw^Gc|22>Icy)8dic^?T3lP2^9;>)kc~4X zea8&pSMuaHzHD6qUNaHhUdS*SAw5aRV{Izcj$8R`~-7bwQ)n zDPf*lKv#ET{Vw-@R9(ked&AuzPGCC!1Ml? zGDbDWa5^%eIGB%SGt5Z7pCg+JGp0Dwb0bu=IF*QnB>be_8wzhe0TFiBXL&1xZ&(fi4te*qy_y^k32YIa#5?BZQwu}1TIGM>gLM%HfY3JA&pmx zLX+la*vVxf*%0HC;+}q6xtu~?wjedH*;qSqxv+7ahhykF#)55PRH|xvx-?a!mkBDi ze5d5!mwxmK-*@8$bG|0x>MoETqu;PbR+)GoZ) z)i4U>!Xr;;es#{tb~=8k-@LHV7vxC0!~0*Vm7VxDB}e-6ICArs-&KED>!GwGHxpV^3cFw3v{Qs}xtTM1%t2faIKX|dc@ z6d&yu2?GB-GJiutUyu2bZK(D|ykuEi6t@e27S|SPW~xY;trz#?7vV4#nUa89Bjd-k zJ}(;9UsCU_hkF?XA+9M)Hgk1(&kFHiT#b@r7HkeB+j^Mo!ClfU##+Rq5G9mo&@TvS zg>;YGQAAx$zr$d3Z=$-5Eq;mQW2mpcRQBN}2wCm7)X^p4n)5)j9O6+nZdPQ^7va=zD(z5o%pohL;SV=qYr23X{ouYCQs(<$W$he^EM7 zM9oYq*oOj7V+e~)fvBp@kvN?PVu#tBG5h3Zz(GHumU4H#hm2@ z9sx-G_wYbB1g9e)B#A5i^p0U_9uAQ=6PxMWhIa$D5F7iK$xCmTTJOcxD^E+;Ro=JTm`{k7-^ zAo)OHjv_XhXF1bmagVuH%&)Kkq->)$9ea-Fv%lJ8CQ$D^6VxsSB{!t12Xcx1oWT1M zE6EmalCnNaaIU?<*Alm-w_x(tm$ULN0yn+%yCTljbNC&e(ir8>i-dNuZZ$&G9>60B z9a|1Q9*ICjc!WVz`0vhTyEI`%CVms^X`L<)gmgAIdXwvdR6?Ps5@m#Ax; zKE;1y@CZ(J;%jwDw1&k_u4Ri@)GC$Td8_T8`O?ZNnBWh-mrbjWWospoG+l*-Q+3@vq2eBD1Meu$1~wsCT*6 zL~+@T6#A^+-@T^}5HEL}qJpgfe|lt9m#xXl{R57K11qY?TmIpsg_KL_XT~LY{a|Jk zk~*cX91BYdDf$!7UD5Zp7wMv|M{Xh1x#H$SVF@YMpCR4PLzHe%8`w+y*4S6HNJa5) zfFx;F1=)2+rOwSRK2|(UA`w|!I{)HqqbrS`_lqGMCZZ_M5*xqqD@LbD5PZfi8QyQ)xH+ zzUB7NU&Z0LPa(T)rO$jLSY&bYX=L)?2H=kM)9h;VF=rg|hSBo`Yp|90&&~?XK%xE5 z!^JDf81*RAAEZuEpA2+9wcaX@g+~sI(2!zG7?1+}=U_2a`ff>b3ajdShf1h2-~7^2 zvoTm-JbzZO=^c!gqwxaxD^C}r&sETmEC=fNfa#pLdu>!l29ym7S@dVtj!mz11R47+ zzy%L?M#fs+GLSybWc$w(Dc0lN6l(U~Uf5Xa?-CqarK)vcH2s!=36IPLtvPvRei&18 zYChR(wrBUPk^|TQ0;6^R>ZZ-?8mRJ}Y-|-_caq*?Ikc$83L_u}9HZBL@ z$}j=!UGyhMBp;7F4I1=vVQn*KwGb@DZ>m1%!38ldeT4ylZY@p=H{#cpGMa;ur?HDC zN4376jOHP{JWT+T@E3)$hy>Q-J)%>{M_zd`MnQZyN71GJyg1S#M4A@$_EPnFlVF^H zy<(v`!$X-kGFJYkuL}L*SPE%mfd3ZU?_>H=p}soIeSs`U%tPj$^4l1r?w7^*#irN| zy;a6+gQ1>Zo=AWxE&GyYcNE4IJXXFID55Zt`IZxcRZMIRh>d{1G!sRaa3Jc+S`>3Z zrKs$6Uz;I@U4k{Sk-^?g!EblL1^ zPwWY(t2R*2eKXb&c8Ue=j!HZW7Qdc;G1n5F;M2YLXCxg`96opzom&RYF?K|#!FL&; zZ@KdZ`+is7;>n3m#HsB96ZH^(o|=;OwbS5xnVyp6jx%y?=WZ_nSDOv4Hao9RVi!R~ zuF|rdjk;+^&h%@YGY>lCH_Tx&)nw$^eRZj>&;zwtsA9^RY4wMk|KVJYZD72~@FK+N zJZ8T%^+3jV7hR*&@XqQ>nbT*=c{BZsl)j!dzhR%2ehw>NXpWlJ97~&Nr?4kuO|^7$ zIZycmB2huGB*Am&__OO{W?)y`45sFTcBBO3%+?v{P&OUER`zNTtOgVj*o{1-q!MIW z8u`N-<{wnjLH4Wo6CTsPYullg{tYhOews~oN3}<#vNe(vVZZ~7s}zF+MjAx1{jnIA zz6bvKgiSJjq#u0Mb(5CgDZHrEA~B{i0om1&Zf8W*X8#osK?#i)KF9n;7S_&FTcWmb zG|WDRX@-;K?bu0z+RXp?px;}6!b-~zH7WUtPrZ{CWp2;qG3pYtjvx>xtb`|><8^lx zdAAamk9tOz+?D&|J{saSuTkXCq)o~z`OzJ;;n_U|e*`9%2}l-n2;(w*3sGJFm{2U| zt4Ya&`eU}{1yG6#sl`sAEK1*=AAtiTqmLS&p@{Y!AIUo055)J(kMl zNv|)e&)5EvVTRt#KPiOz5?s=g;RRaW^<9<4F<9+w-_^5H#RW5Fd(wZbe0;38_ zv{{v>t?(lb^J6((c?0N6L(w=LrEMGC?c4^L6$KIvu8zs~Z8s@iQcVlfx_;H+nCYLfA(`9@T@@w%@WV)kwb%#V^Rfo1 z3zI|WFA!6zs*}O&B0UsugedwvRT{|4Z)$!qv`5|S`Cut2|IN}0;}VlirounJR|Fb< zS@7BFo)6dm{mK?{IItMC^o@_kMXWdbv^gw*-JR zoGrMeUcMERunO08mu!e1*WSe9UUH6Tp_}V%G_ZVQvvsQ$m>;Jd(lV!SMa zfkS|ve+>CBQ*wvCsdES;vXyo`mzEo}*5CIp(_*bEzV_pW>A0I(FlDX;az;M5zon4E z9Pa0XPBrqD6*kJtdX*ge_yNiUx~A#T*d(e1*xms3uowh+Xu5<+Zgn;irdJQ`ovWc! z3$GqYrz|Cmirv?JJoaUOGbWOpsg!|g$QA62-Peul@~o1a&}m0|LU_pWx4f32tFQb zEsLFvh1=$Rs7ySXc5Qx%&6Wq!^5xtAg$%X(wj0MbB3*bdzif1<%Yn z#5&g$!Bww5e<2IDtz2kGe+% z#(Mf#W`ePcU6j3gA08E|o8@o~oU1~IHfGmyVm zFK-J?F7FE%gkE`ROK%TR{}BSbgVWIq0}3Dx>Z{jE)l!$Sk~634QmF>bLLy6Q3y>{R zuhE~ziTXEDoD-%if0XpHP~vJwY`GHokkP#UaG~!;k9ZB&?jOSG?2)T7s6@c^#UMToCqg zdHS_p5H=wpA#l^6%j$^fd)HL%5F}F9l%3Zf#zNqjsGWsd3XQ-^?HqT++b)3(6El;6 z!`ka*{pA(%@5ry|o5vPS$iH_%)bW4+5{)@-9F6uY6Pe$pnMb1<6 zxfiON$jbHBGr#LEc+vPlgS7($rh@9Uh?g-RX-H-~XMS_-A=6%kIUpv?V4ykUBwI#c zy1w;#AN`Qw2F_Tg3C z9ibG8WUGz;w5K$@oU+FEclQZgEqTMJ%(%RCQx(D6LUd9k8b{*nN+qkeud9{ucMvGq zzcBaNz@(9OB+0l+%1VMZKzBoE&GbC8`F$)=7Kot8m|~|Z(OI;g zg*P7%a?cyiQ7JJZun1X%hx_07cAuODG|l~&v>&g6qltBrD+?_%@0=YOQz1TD{L-AK zV6i(yjqe)k5u06-?|*@H8=IPzvIO?PpV`irJi(cfO_uAo{cEc{%veY!fHh$&B-3E& zxEM_W1hn9WMD*w9%HrJQXmh*8(d`@OhsVG%v)?d>x1XqNn7Lk-ohW8VkkZkx z%krz^e$l(G7P*TM=aF@y!{+$|(3y=z+J_OR2P2Z&T(YK-Ett)_poLV&5ap1ugbn=q zv-2xb{FZ1o00YG|U7BbPEJbc?rEv^tBO)W8U7z>7al!`vH}L+tW7kgmEygyABQ0Zt zU}B0#1+%f0)-j}|=d>V`Gv#&ZS9Tuia*PZL1~K6` zrbYIZ!?To_^!U4gkmmnVaCOmn6CZEoLdd`f+;=m{VhUS%kkm85tjCGotmuOBAN}*9)sQ&le63FOvuOpG?MZ1y83wX6j1;US;k7Uo!@NX*;%br{V!Xuo(E#K zGo6c1Z!wS+haqHTw(hARv2wE!n-;3rI2FIuf7~hTnDU^C8$M%@n*Ps8DH({ zpqiuhejIihhY+7G+ZN$>-?dN@plb={dO)H*e}-_7;|9de;IDo7vffL8OnS5cP_Mz` z-sD<#({K|>f?i`xD>NQ@W47AQ$u>zq7D(>65x&S}%$9-4U}Nb8#N?&22G!VF3ZIBR z&rAG7WZp9(gHwvdtjlE+A2)elseFOC(L+?-9#rhe7h5Ez6}sL8v-}EG(QY<#!URE; znXbQUFTJLc1(n!wo?o#a9b#ZgQ8iV8JA^QU6QH6D3P{nc5;K@)8i&)@rG5-f?#5yN zYaEISe9}uEBsoVDMsH*-dp>p9z)#479-`BjW~wi9@UOcZl{Z%czwLeqwgRE&@cz8m z(8Bz|#4wJ@btF5nHpc`USBOOu(fn(#9N#>Fof9#)4m_X#Uk7ng^dx#1{PJOWr8atP z!UnH0z0}K~4^o}so*6CJ_2xf!YP-?i*_#S1{%km}mKFdaPo*Ne>e|0`m#ewjkj}pS zX#CfSS?XrJ=^H)#f>ECj8~l=XbyG0nDk-Me|3+}Aaf0*h9HBeIBNEqGm;-`Vw;5PN zs}jTUY9DzVLd?LLWRDgth*k?4oc=E}Vpn0P&TulrQdcmqVuwJz5Sf)spg8n{0$gpj z_sVEFl}f{s@nui&i}P|5mcQz?I-y5rz~xT*C{BmL=g6Uu z$|zFd``@7?!=^gByLRjN41DGCe5%hwr~<-wJ6a*<(1MpsyW+7!S;(&YJp6Mr^5Fjp zmSs)|ohYP;=icc1dOPsfqlH1Lg;(U&l?uOlGn2u+OkXLnTgeL_Sa|EpH(?eqF!`lQ zE}0#fVJs=VNSxBz$eVOb0Xjv{`r=HS1?-|Pf!j$}?44ENd?%pqym433ftGm^+;m*I!xsNKBl7 z+$XXQn8N1*xCNe;Z2OW={w@T22N3$ONvWxOJ%(UZX5eO<#Qt&&?y@`V2T9~@D0)Tk+Vd1&+d<{ z(fh~rGYPY zaVp+V%tQR;YLxswt4Ni<7YiC;`7ea)WF{5(J+t(tM4MIE z8b~e28e9|l<$=?Ra@66k%Z2leS|BEZ5tZ&Kb-HVyxO5qGlT<}GcY@V<)=aO`8-aTu zr!geKihNJH>C@+tH1TSXbA8-%4*C}$tux?@O{#pDpgKd)evylI_mALt-%At3 zT{YgZ{IuNjKG^ANM+lAWTH)5{G=#xn&F4+9lAK=+Xhi1lP2I|bW?(aIRlrE~TcAWW zh>miMvV1~Iqy#lcALCtFUpKQzSUyzW@ICKAqq+D`%*Z+@TfKzEkhD|pJ|P=TaG!ip zhCa}q)dv_XlG=R0Rp8ir?zhO+CgOqK$Vw?Y4Z(aL)mi~_q3e`ghSbP+MsQ$0R2$J6 zfPXtkO6`u2TPg2Tuyl2g>20-zOQLkxaV;;iYE<+q* z@A+zocu1_b;v0VrXc^xgh=*fw@)5<+kpt8F%=BHZFJBo0R$qfetO94Snvq`>$omJ^ zDN+pV^r5KtEzzo#G-J6Q&_28o>WNYdC?)hJjD#hhZAlqYLO7E_`skSmuGqg&Y+)%K zklfXz9xCZwB~%WW3(GvjfbJuBt(&7u6&W&g`3Ku?%&Jk4O%d2mAgZfQJ_@H-sSR`D z_PLd;w;`}2o3@s*w8pCA+yre>jnTosOLt>I`o2C`+_b0ijaU(=(H{Iz826*w6f1FN zbM;RBoylDhni?jFIyszNWHZzX<&vzI73V9`uK2X-dcSr?h74ykNQtzvwBcKV_>k~C z#G!4suj;Dli`^qhV9SA|Q5m$Me+AMOq!%TBgxA;WsB_DUPa=$#NM{WWIqlc2_$Gjh zXA4{5+G4V-34ONYY&%V83ecP&x00X5k`+ruZVvsPiA&lQ!!8-gg$E~i(o`QfpB4sR z0U9VaWFs27sgK}5S}gWuP^m&EYFHg_`#>Ds#!O7?*iZC~xNaq)F$>4^N_uVmduf3m2D6#7BUxiNIGr*mG zsutS3M;C%lf8wMr-&Ow__8%vAp2!Pqa?aDk=Z@xJW}3cX7N{B}mTdLiEIYrIj%*pU zrYwDE-G6Bu^FjSR^@8sG7(A&XtUl4yU^1{WY9(f=;KpEhHgbcLfO^6q#!jOfVtETZ zuiG%QP%FB2Dt6Y{<*M6b#1tyKHSt#to`^`gtxgv0Q-J0Y&=4)co$`5wEEM^3U3{9eD? zM^svqFGzCWksOE|8=dEKA{ThSQI%uWnu*@#)qOGJp zs-6>d_=8Sh_H0;n+lv%ohsJ)t{5SO;^S|~aV5KQqPC8Oz^{qFps%)b!tK~!OPIlB- zFd|b-x?|f&$sZH6tYMt;KOrh8{0WV%AwGRwfGh#8`5#gOlL4tpu;UgEIF5EF#(m)L z?S7H;Sd;{QE894M$>dxIa7EC*3Zf3zM@)8mJ710tqme?%2o)$v0*i@*@wLie12uqrUxN+Xf}eZfH`p?LL|%Rh;<8x^00zG$A6k1ja#U0k zG=UiuNPKlQfhqojxrM_pFY3v1Xm2Ri51rXBA_%A_k=pjk%8;{+BG;{GGEu)yO?O*P zcP3zhE1*3Mu|t_5`oj7Z4@2ViLUtl`HPah_)wel@%&cqBgD%2xKCn1n`1+#1*EcI$>;D)HgEnQw-tHj0`j`jh!~ABhqGoSwrk&ef z^u@$XW*Wj~4ekiqSD?+7Y@4`0V@y5u=iR~+z&+*0OTc9cjE5ubev7%ntBoNqPrU(u z>{bwo0|hFPhn*UWIwQ)dMLUJd+}6;!yhd<$M8YB*=+g(j3TzOFJzxunPDp)edZR&K zoZxazJSGs9_cyIJ5I@tdh5E0pDA^p|PsIa&vYTiOflmI>EvUu}B1s_4Rq0Xg`>-GK zUo4cOImLJgIR58%=?*_o)rxn125_KW;yn2750t=lK^v2`w+8pq1x72S)$lGkXdEYK zJ+Wb4hbfRNFkOP&do|O#102xJq%zhA;U!somulmHu&hKrH1Bol@oQQ% z;M-Xor8lMt%i9_Le=1jYw}C9QoQp$O1;av{5FJ8R=vaPZX`6IV4vj!Tibn8#hFn^{ z4mR=l{O*xsF*a~`puRH(`Y*{cP;E5Hmf#Dmg**R`Vc|$?3>*#al$t;%uNEtpOjrqj z3TW9GjT6nc|6!HoRA1TIbSbcz-J2FGFBW7fm}=V=@qP4{0vegKOSu->$L9AMZ*4a$ zu{)s`j)#so0~ILKknAM(NoOe<-p3*h>jYpv`q=eeus|$amO0~h>#NtQafU2eklxtT zw>{bLdlo6+-W_o7g)z~9YAXU|;F4$U3~Rj1B`$HDW(|jZcc7yh_y3rG?i2+&!;q+0 zou;6|CN`{r-nZdF4q&EGHQ!tW3{l-_2+dq6d;n9pzIET_!rYOC45!(x!ASzRrQL=b zLDf&_Ukzns6zmLpvR!J-2-ba2enL_bQT8#3WlQ1n@i?DulS#>@3mcz1J(favHH$U& zZ$jQ;?Qp&%!y2wY(sRE0;tTNaN#tnG-Y|fBA9)OP-eIwe`oh0Lvay`3p-b`Wj(?L~ zwijbO%%mdIQIAPow;G#Gph{A6T)j zbSda*dQfbG@Unhg`CUDOKx(L?mld9lPtwnY)T z28rsqf2KB3^vqHMwzIo`+fLTz$mT<7rzIo1L43;(^Y}MKYVhqc_S=1IJl1c7D??lw z;j<#C`My%rXSY2pR~awfK+cVAS@OMci8>@-=Y3*gH~+#s)~D5AxyGwk}F&7cgSOahjS40<2*Zr~EpolMhusUl|O~&o&6G7f{pObl;6W0SPa%YgHhK4q?`OBIs+PYjO1O3#Db3U-K7&4)2oo_pJdNdAs&eilp__; zfY@P5$B>oEH08No{*Py$ijqAo`5%9g?&=Gv*Lwkz>YmpXP|p#;*u3tjTaU$Qn#_*3l2%)P$aIS}3d#bwF*M zaxO0)vnfzj2NYaavlr*8aL_YP{NDVG_8v^q0;80gSakHuh-5EQJv<3gCRupY& z#m!_XPE-!_n<%2s&ixsf{AWlfjTqw0b69!1+Ka;5*k4%Ogii;-e+Dbk+mfv0R0wsR>{{9K0E-bEbx}!qyowkKrx0v+%~6W zad*!s?k!S{v3snNmAw$eL!d9%^HiQg=aAb9P2>!Isx800Gi5hmyMIKI!>YSTQy_R{ zpGIf!=@)qzVpF%k#fG+(Tfz7VpG{+K(Xzb-HAY(kah?}UC&rvqz2fyn8}tO^9U$tR ztS~EpyR5?WS;M~I6-B2D76Y7trwlrSJuBwfC;j?Rverh9%7PZ?C!B*TWo_x#z-tnf zyo4Y{sSwsUv@@{<2plcyw;HsMGg$9AWre0EQA2R_u;?QUL`eQHD+Nz+!i?H@_9VRc ztgaWnKcKCH2&5~Y#pE|NN!1ID>M6(cBwUpr#pbXW4hLArx4e{efL?M;nk8LDJQIXN zp|=UPV1yOp0Sfgj-61W%93uywLlK8$&$JzoH=9g8myY@@WRWLLV$9nFy>0#TKjRYb z0b<#;-5&);e@F*(yssNS+IRCm+R=M2Rm=&gQ-~&s%|f(*Ny;jDRY7_>CA$6arX}pc zvS=gk*OR6@OdqTzdZd3ke@IF3)i!cZA2+Xm5|xa4gV3Sz10usxuUXBluLU$2FiYlP zPgQSL3?TTAaGr?SRnn3bWi%_XOCh4I2H}hmDsH3PD=ZHjuPGr%bz|ZWHnD|1$G@wr8+cIxMRE)xvsHCkqt)ua z_q~xbafqLrf%KyjYB`Fi=mNJzRdxWqA@ekR!$hGXgMzroaC7!NftXmf9DAz#(DX84 zb)>oFL^WODM&_^?kW^^C>cVT@H0tu75?W`76WI;1CGv+Y>La;wblj^d3*qoP1U_RX zctNdApln3aeUATH;%z0N1$r|QrB;>&OmJ&*x!To{yzx>98JRCqtkHKsdF!ATb}k z`{*=+Nm=x_oKM}Whv>c2Kqf}51&9RGRwdryvk`V{9q!J*u8v|nl8y)-V$YFlfBQ>g z<6(kRcoM*h8}^4u!3g+m)a)ydL!RA-E7Q-O{}Dvwg^BHi1Fk*u#)2RkEJohvk!l*u z?De6p!hEKuEu+30)-P}}ZC@XJS;grpE1~M^J)7s-n(-NP-q z|1?*ID5HksnZbYo)^(=K{m5FT6(cs12B4fd6@<76(@3VRDGE3`czmuL&lo-$fnT2R zBYN;d+$HoXcGV^G>}mDsAK>gATujln03SB!ui&DPuY|7qH(L!o2V){3e&g}b3sg|P zbsC$C=8_`=IxK5~h4WD41T0{!Xh(Ma!F=b16F)0PE<$3V5HKOr>3!(WffbJ)&m9KD zu$SRnpbkUjZyZA(o&5^=fx{qoHe097_&?E~wl&uX`EXG`_pIb2lj9v$%yt?DN+)^b zqw0Y)7x;XFOepq|A=V(9C=*Mjk9ir;-zl1G4j5#pNYWWX? zM6Z`D>MUU1h+_0k=1LC|Wqe5D-(-MU3;iQ#+zc)d24Wc(Gk#2C55zH@W2cAv{OcpIRY9YGUj>+q zI5W+Pl+1qJ@cfl*XU?bByzDH~>Cil6WqYTiZl9kW58&qiy1ey1t-P5VL%!EaE5`e5 zHTq0X-lV(vtcSgRGUo5IPN$Orokn%97~Rx&K#d8JsD@_)RZnddDBu+@DgNsA82TE< zmm^7Nww_f$7XR?pbz{yEoFf8MBuQ55Z@)p!ib8F;1Q*6a`PlelYkSIm4xZ)Tf%U61 zdT(hv@IHdR3vTzGEm4bDNA7)xM6wmwNJ__RYkC)MH34x3DI1(`TE7X_np;Xxhp5p4&0Q_Pg~4~`sJol; zQ)3V3eg0pH(rrK$ZrdOB-RPUJi5p?5ai|e{SH%XcRPj;S=wG9t+LGQRY9-Z?lne1lyE{iH;4in!%!` zF@Lr2NOCJX7VJNTbgv1)_<#AHP>3|d;p|N*YrcN;T1<8{%jeYf zAp4i*SLh1Kj%p(!iOp}>O5z4`Sqgr?l1y|~U?aY7f-p|5uSWlrT<>U$8G0P_7g_SF z&AD~9fvG-SSNVfjtsw|mIoXiB$=7|s=yGQ{{{(}rr`xgEUuQW#F|V9X`a72~57vuB zN9!LoqcCSld#7`6bvn`pMFKs)d(IL5#n(uOJ`(QCIEj@d`0?W74j06cl=zNdhe5TN zFTUjumToM60{gSdg5rJNZY&PQbII#as=L)O7{bEh6+%fm#$%5!Xxur0B z+YKIR8=GT1SmDJ}ojv0a=iSI!J6ACpk_l~xzxsc|1jS@syo4*o4vuFX=JG+BmR^Nj9lW32cc>ftCliFddQE*?YfBzqjD9JC zzSC&})ok>BIyoN9dc=^F6fgE3>DT`=9ASH$qTo78UL}2Wd4o+n-ucB zC%gb(^WurtGFKdHX10N9e!Gy4-8&PPy}m8FgFKYRF?V8DS1sV``y^Yi^2@Q;30nSK zr}%{l$tC5PZa;E5Yt_D-stDk%uNz!rTP4hYIGrA*dcU?gP9vIr`29~K9$<32S^QNr zvp+Y-4bT?;TU#@FA&iEYk&IV*VeSo>2a89_&Qx}457ui>?h;klZP%anqFUu%uD)MU zyvR3gg#WXF!9$IU@su5K7Ig*9rpVe}#Ogi2`7598Ff^l&+yKDu*=6(XgKLHt8I@06 z7>`wC9uGfUXVTl>PKD(Re*h;_RAN3-Pgt|OLYhB@|DY09pk^xJpU#@8zx#ikocT8t zS{%T?7<+cvM%E{uqBohBCA~KpYDTuP<{`$2smKx$rNNP+vIg>#kBd z&gl%6Cj=EtW|lE8J^RFFdTeB;PG9%Uoe0o0&Q9mece%dpA1IQqc16u<7Oiuh^0YF& ziErCKy?VS{5RjXH5Fk3f@`&4qdk-fCdxmQ?9f-KT`c1Wn1&#M1u`PzXj;E83Ms2Np zNzF&KJ?GtBE@>O~39fYSqeLen2%8byQ{?Xcy~4jYXKDwxFg87~X^6Nb7ZKGHTh7}{ zH}%uq*7wwZOELVY$Cc}{Q9^Tk!IIZudK*q!O;T|a*Y9O{qhB>nyjDYLuff~#N|`f# z56qAhq>bU3D#n?nGn456%s6l9)81w(a=`dKBnfvL@$lwLahKH>h&zzBK<`gYfXx<-8F_<`aKKp8 z$fT6zR!<7U0_Z!@tROuKsCOTRJCtyUToOK=X z>*DMK5tUxZcM?RS*(}&_(=foHjT=qqb$AOyRF$7-NLigEJl@sQ)LA{{DGXKoO6ir# z1ZN;*W{!;(&f$dMNXd?NNqw?41clBkF2C=xLil#ruw3HgbUWxIgs&K^u<8`$z#u|{ ztEK_zgQeOj;b&tu2|?=a5pn<7TuCt6=@FYeF_<0)M$B8-u4z+6aJe$n$|KxuF_#kq zJ1gZDg9lb<11>2s%KNvGfcIpigT`l z-_Ih@CRa~xc36(W0wlW1wi0J-BCCWdRPg%f%2a=Oya}O#e zgjEfT0mExJeEy}+;*Jesz`K!NSKNNnLtjD$n!A=}biXSd^3C!X@C(>b*C*4o@vWYsv;8xr4rrX+gg|;lo zpF8Z$L`IO$rO(RuDRsANcT%(VRKJ$62HrlEZ?rkm)y=0;{4=tu$l6_Vrau9+!`Ux> z+{LCDzpLZNsLx&)u*~c$4pPmTKGM`%N}Z`X_O|CQM700ESgsHkN7oc$$q2+j!(=->I}HYN565v0{|bS4A`{4cF$zZd~3^0R@1A{$*+jRZipbFwYB2}u4I Dq%^FC From bd7accb97db351f64c8304dbca0fb1a4c0811a3e Mon Sep 17 00:00:00 2001 From: Vanshika <102902652+Vanshikav123@users.noreply.github.com> Date: Fri, 22 Dec 2023 18:47:44 +0530 Subject: [PATCH 04/26] TraceID : Fetching TraceID (#6973) --- pkg/ui/react-app/src/pages/graph/Panel.tsx | 16 +++++++++++++--- .../src/pages/graph/QueryStatsView.test.tsx | 13 ++++++++----- .../react-app/src/pages/graph/QueryStatsView.tsx | 9 +++++---- pkg/ui/react-app/src/types/types.ts | 1 + 4 files changed, 27 insertions(+), 12 deletions(-) diff --git a/pkg/ui/react-app/src/pages/graph/Panel.tsx b/pkg/ui/react-app/src/pages/graph/Panel.tsx index 354831d548..2ba02d1131 100644 --- a/pkg/ui/react-app/src/pages/graph/Panel.tsx +++ b/pkg/ui/react-app/src/pages/graph/Panel.tsx @@ -231,6 +231,7 @@ class Panel extends Component { method: 'GET', headers: { 'Content-Type': 'application/json', + 'X-Thanos-Force-Tracing': 'true', // Conditionally add the header if the checkbox is enabled ...(this.props.options.forceTracing ? { 'X-Thanos-Force-Tracing': 'true' } : {}), }, @@ -238,8 +239,15 @@ class Panel extends Component { credentials: 'same-origin', signal: abortController.signal, }) - .then((resp) => resp.json()) - .then((json) => { + .then((resp) => { + return resp.json().then((json) => { + return { + json, + headers: resp.headers, + }; + }); + }) + .then(({ json, headers }) => { if (json.status !== 'success') { throw new Error(json.error || 'invalid response JSON'); } @@ -254,7 +262,7 @@ class Panel extends Component { } analysis = json.data.analysis; } - + const traceID = headers.get('X-Thanos-Trace-ID'); this.setState({ error: null, data: json.data, @@ -262,12 +270,14 @@ class Panel extends Component { startTime, endTime, resolution, + traceID: traceID ? traceID : '', }, warnings: json.warnings, stats: { loadTime: Date.now() - queryStart, resolution, resultSeries, + traceID, }, loading: false, analysis: analysis, diff --git a/pkg/ui/react-app/src/pages/graph/QueryStatsView.test.tsx b/pkg/ui/react-app/src/pages/graph/QueryStatsView.test.tsx index e04c914e1a..4df5fc37c3 100755 --- a/pkg/ui/react-app/src/pages/graph/QueryStatsView.test.tsx +++ b/pkg/ui/react-app/src/pages/graph/QueryStatsView.test.tsx @@ -1,5 +1,5 @@ import * as React from 'react'; -import { shallow } from 'enzyme'; +import { mount } from 'enzyme'; import QueryStatsView from './QueryStatsView'; describe('QueryStatsView', () => { @@ -8,10 +8,13 @@ describe('QueryStatsView', () => { loadTime: 100, resolution: 5, resultSeries: 10000, + traceID: 'e575f9d4eab63a90cdc3dc4ef1b8dda0', }; - const queryStatsView = shallow(); - expect(queryStatsView.prop('className')).toEqual('query-stats'); - expect(queryStatsView.children().prop('className')).toEqual('float-right'); - expect(queryStatsView.children().text()).toEqual('Load time: 100ms   Resolution: 5s   Result series: 10000'); + const queryStatsView = mount(); + expect(queryStatsView.find('.query-stats').prop('className')).toEqual('query-stats'); + expect(queryStatsView.find('.float-right').prop('className')).toEqual('float-right'); + expect(queryStatsView.find('.float-right').html()).toEqual( + `Load time: ${queryStatsProps.loadTime}ms   Resolution: ${queryStatsProps.resolution}s   Result series: ${queryStatsProps.resultSeries}   Trace ID: ${queryStatsProps.traceID}` + ); }); }); diff --git a/pkg/ui/react-app/src/pages/graph/QueryStatsView.tsx b/pkg/ui/react-app/src/pages/graph/QueryStatsView.tsx index 8bfd91c74b..d3e9a7a5c4 100644 --- a/pkg/ui/react-app/src/pages/graph/QueryStatsView.tsx +++ b/pkg/ui/react-app/src/pages/graph/QueryStatsView.tsx @@ -4,16 +4,17 @@ export interface QueryStats { loadTime: number; resolution: number; resultSeries: number; + traceID: string | null; } const QueryStatsView: FC = (props) => { - const { loadTime, resolution, resultSeries } = props; + const { loadTime, resolution, resultSeries, traceID } = props; + const prev = `Load time: ${loadTime}ms   Resolution: ${resolution}s   Result series: ${resultSeries}`; + const str = traceID ? prev + `   Trace ID: ${traceID}` : prev; return (
- - Load time: {loadTime}ms   Resolution: {resolution}s   Result series: {resultSeries} - +
); }; diff --git a/pkg/ui/react-app/src/types/types.ts b/pkg/ui/react-app/src/types/types.ts index ca31bc6cc4..08054fa1ac 100644 --- a/pkg/ui/react-app/src/types/types.ts +++ b/pkg/ui/react-app/src/types/types.ts @@ -17,6 +17,7 @@ export interface QueryParams { startTime: number; endTime: number; resolution: number; + traceID: string; } export type Rule = { From d27365c5bdc339c699c8da5b1227c926cdfce2eb Mon Sep 17 00:00:00 2001 From: Michael Hoffmann Date: Sun, 24 Dec 2023 10:31:01 +0100 Subject: [PATCH 05/26] docs: add promcon 2023 thanos talks Signed-off-by: Michael Hoffmann --- docs/getting-started.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/getting-started.md b/docs/getting-started.md index 9e0a7a8ed0..da6b414472 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -88,6 +88,10 @@ See up to date [jsonnet mixins](https://github.com/thanos-io/thanos/tree/main/mi ## Talks +* 2023 + * [Planetscale monitoring: Handling billions of active series with Prometheus and Thanos](https://www.youtube.com/watch?v=Or8r46fSaOg) + * [Taming the Tsunami: low latency ingestion of push-based metrics in Prometheus](https://www.youtube.com/watch?v=W81x1j765hc) + * 2022 * [Story of Correlation: Integrating Thanos Metrics with Observability Signals](https://www.youtube.com/watch?v=rWFb01GW0mQ) * [Running the Observability As a Service For Your Teams With Thanos](https://www.youtube.com/watch?v=I4Mfyfd_4M8) From 2d6acc5c1cb378d646fc90a919f1c4024b98f107 Mon Sep 17 00:00:00 2001 From: Kartikay <120778728+kartikaysaxena@users.noreply.github.com> Date: Mon, 25 Dec 2023 03:25:17 +0530 Subject: [PATCH 06/26] =?UTF-8?q?Added=20website=20page=20for=20companies?= =?UTF-8?q?=20who=20offer=20consultancy=20and=20enterprise=E2=80=A6=20(#70?= =?UTF-8?q?00)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Added website page for companies who offer consultancy and enterprise support for Thanos Signed-off-by: Kartikay * adopters.yml revert Signed-off-by: Kartikay * retrigger checks Signed-off-by: Kartikay * added a new line in welcome.md Signed-off-by: Kartikay * retrigger checks Signed-off-by: Kartikay --------- Signed-off-by: Kartikay --- .mdox.yaml | 4 ++ docs/support/welcome.md | 6 +++ website/data/adopters.yml | 2 +- website/layouts/_default/baseof.html | 3 ++ .../partials/versioning/version-picker.html | 1 + website/layouts/support/list.html | 28 +++++++++++++ website/layouts/support/single.html | 14 +++++++ website/static/cloudraft.png | Bin 0 -> 1883 bytes website/static/o11y.svg | 37 ++++++++++++++++++ 9 files changed, 94 insertions(+), 1 deletion(-) create mode 100644 docs/support/welcome.md create mode 100644 website/layouts/support/list.html create mode 100644 website/layouts/support/single.html create mode 100644 website/static/cloudraft.png create mode 100644 website/static/o11y.svg diff --git a/.mdox.yaml b/.mdox.yaml index 4f8a0be006..1c9c4dd589 100644 --- a/.mdox.yaml +++ b/.mdox.yaml @@ -72,6 +72,10 @@ transformations: backMatter: *docBackMatter # Non-versioned element: Blog. + + - glob: "support/*" + path: /../support/* + - glob: "blog/*" path: /../blog/* diff --git a/docs/support/welcome.md b/docs/support/welcome.md new file mode 100644 index 0000000000..c4e48c5582 --- /dev/null +++ b/docs/support/welcome.md @@ -0,0 +1,6 @@ +--- +title: Welcome to Support and Training! +author: Thanos Team +--- + +Anyone who has developed a Thanos training program or offers related services can add themselves to this page by opening a pull request against it. diff --git a/website/data/adopters.yml b/website/data/adopters.yml index 7605b38f59..1ae6e5b264 100644 --- a/website/data/adopters.yml +++ b/website/data/adopters.yml @@ -236,4 +236,4 @@ adopters: logo: grupo-olx.png - name: TrueLayer url: https://truelayer.com/ - logo: truelayer.png + logo: truelayer.png \ No newline at end of file diff --git a/website/layouts/_default/baseof.html b/website/layouts/_default/baseof.html index 4fd3803768..83b539295b 100644 --- a/website/layouts/_default/baseof.html +++ b/website/layouts/_default/baseof.html @@ -46,6 +46,9 @@ +
  • + Support and Training +
  • Releases
    • diff --git a/website/layouts/partials/versioning/version-picker.html b/website/layouts/partials/versioning/version-picker.html index 732afd453d..f2f4fa9c1e 100644 --- a/website/layouts/partials/versioning/version-picker.html +++ b/website/layouts/partials/versioning/version-picker.html @@ -4,6 +4,7 @@ {{- range .Site.Sections.Reverse }} {{- $version := .Section }} {{- if eq $version "blog" }}{{continue}}{{end}} + {{- if eq $version "support" }}{{continue}}{{end}} {{ $version }} diff --git a/website/layouts/support/list.html b/website/layouts/support/list.html new file mode 100644 index 0000000000..84057b7ef6 --- /dev/null +++ b/website/layouts/support/list.html @@ -0,0 +1,28 @@ +{{ define "main" }} +
    +
    +
    + {{ range .Paginator.Pages }} +
    +

    Support and Training

    +

    Firms that offer consultancy and enterprise support.

    +
    + + + + + + +
    +
    + {{ .Summary }} +
    +
    + Add your company +
    + {{ end}} + {{ template "_internal/pagination.html" . }} +
    +
    +
    + {{ end }} \ No newline at end of file diff --git a/website/layouts/support/single.html b/website/layouts/support/single.html new file mode 100644 index 0000000000..f0a955227e --- /dev/null +++ b/website/layouts/support/single.html @@ -0,0 +1,14 @@ +{{ define "main" }} +
    +
    +
    +
    +

    {{ .Title }}

    +

    +
    + {{ .Content }} +
    +
    +
    +
    + {{ end }} \ No newline at end of file diff --git a/website/static/cloudraft.png b/website/static/cloudraft.png new file mode 100644 index 0000000000000000000000000000000000000000..c76ca7987f5cd70f35a42410515cca131540e5b9 GIT binary patch literal 1883 zcmV-h2c-CkP)5<7Wvxj<7d zDbPI{?CwcZ<|-l`-`5awu6(&M3|a0Wtk1tfK3xAX^(m>)Uw&*m^-=aEFm<@A!_PXr zcZ6*n3hXPyAyz&;U3Y`=v2}Px_x->7G#2UHqN$6N#xZE>xrn@m%hZIIIy@k*tqx%m z%H;YjFf0!~Z>qN;6oUI@O0XUI-eu$)hKFN?tP%^_qF)g$+6_4t264)Is8S5{$g(W- zSaH3Jye36H8E&D+WtMq|VK70y4tW^l$#vb4_j6|6GQ4~`a`VsF@3>_h-q4tCY)V{- zwDgx6Qhf0zf=7d9lui_v_4+e;YUZ>O?bj!-$Wn< z>@z*A&y3PPMx-5crZs5GK&F#Jw+sgRGsqUY8CQ;`7J4l5W?CD0jX~2^DYKtL+U(|n zGA;55`)O{T2IKb2E*vr}23Zjlx6l!@uCN%lazYtm9VUcDd;PAtzqu$anT5_=*4f0LaKI8zJs$2$5JC{Ds@8b62q=`JXoYKEgA&s@Te&gf3 zL53iOwOBBuwBhcS67$50_;=(ZO}U))*SQm#tOnyDXYvwy3N-gu2EWP4<_i`KJ*Sll zo3bVG9N=w{jK=wNj8eo6mJ4q)H2!!j)6!ty=b&ZXaeaxMEIiX%q)$Ma*}BBUkkT4) zd}{A+Dq%+(LXKZS{wvJ4e)^Dd84NS%Ni5pl=@dt&L8D0j9J!txi0n8dQ@=zSG?B$V z_MySUVM0D)c~lsNP(hdM+z1D#SjY_B_5^y4$~)|+98%bO=#gIh&5<-RUz(MtXtT+e zPM@$m42B_W@CyGADn;6GS<(m1N+}9USpT=e@r@b0RrT86SCLp=g*d+mmp3V-w9*tZ zm?wv3#avU?bmYG@E|5kA-79IdUGfj;_c|A7+M{nROeQjW!Rd)Ys?s`~DWtTOI5wE) z+AVaZSmcj@p@VLz`;ePbx+t;#T4xNo&s=$ZyBJt$Y1EUO zu5U0uN;IZEHB!!=VR`K<;*?0v>?zv;*Fl+6U~qU_;8?r`W%og83LmeqdpRIB$o=DU z?9Wj+v|o2hbx!nQG2Sas1=_Ehr6|uW;=f2Bp@BlaCgu>2EOW}&0bkI5jWGs?^jGsX z(A8@p(ihNifp&lH!(gWG)5=C9K?~@JQFM!OD@S%@3Yt+iR9NlW{wAU9HPQwNB2{T= zfO!bYjUA*N(mU8abV|4t=?(N~>c?QFN^7wHMF48tz6P~u6#h9P?UxcsJvRbt3Fa?YFzGf6@hsHrsr{U&` z3cK|HeKLj>weW#{qqP>VOMXMMwM=$yi+CvQR*|08jeXx1qP026KoxKNxxxqDTg@{3 z?(e|_d6d|_44>jj{d}bC63v=B^oVkqIcqO^272s%`LT{U@Y|Hz@TmP*0m;Ec1LLuE z^jgHfA&nC4$3Il(Q#$!3$3g!|NlI^f^1V>z83qHL44S-YNic-df;!1YQLc7|Wgelg zKPjum6lWYVOgXeoz{g7W!^^yks6xLqFs(-66yodb@~DEtg!zd2H4!33q}NDN*TEBfZ@J~zjU~tom*Zf + + + + + + + + + + + From 665e64370a2cdfb30a7059c26ba7bdbae9a26309 Mon Sep 17 00:00:00 2001 From: Ben Ye Date: Sun, 24 Dec 2023 13:56:00 -0800 Subject: [PATCH 07/26] Lazy downloaded index header (#6984) * lazy downloaded index header Signed-off-by: Ben Ye * update tests Signed-off-by: Ben Ye * address comments Signed-off-by: Ben Ye * address comments Signed-off-by: Ben Ye * changelog Signed-off-by: Ben Ye --------- Signed-off-by: Ben Ye --- CHANGELOG.md | 1 + cmd/thanos/store.go | 10 + docs/components/store.md | 6 + pkg/block/indexheader/header_test.go | 2 +- pkg/block/indexheader/lazy_binary_reader.go | 7 +- .../indexheader/lazy_binary_reader_test.go | 336 ++++++++++-------- pkg/block/indexheader/reader_pool.go | 46 ++- pkg/block/indexheader/reader_pool_test.go | 13 +- pkg/store/bucket.go | 44 ++- pkg/store/bucket_test.go | 2 +- 10 files changed, 294 insertions(+), 173 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index df6e93e4cd..64b849f9b1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,6 +22,7 @@ We use *breaking :warning:* to mark changes that are not backward compatible (re - [#6954](https://github.com/thanos-io/thanos/pull/6954) Index Cache: Support tracing for fetch APIs. - [#6943](https://github.com/thanos-io/thanos/pull/6943) Ruler: Added `keep_firing_for` field in alerting rule. - [#6972](https://github.com/thanos-io/thanos/pull/6972) Store Gateway: Apply series limit when streaming series for series actually matched if lazy postings is enabled. +- [#6984](https://github.com/thanos-io/thanos/pull/6984) Store Gateway: Added `--store.index-header-lazy-download-strategy` to specify how to lazily download index headers when lazy mmap is enabled. ### Changed diff --git a/cmd/thanos/store.go b/cmd/thanos/store.go index 9191b77ce8..7d80687ec3 100644 --- a/cmd/thanos/store.go +++ b/cmd/thanos/store.go @@ -28,6 +28,7 @@ import ( blocksAPI "github.com/thanos-io/thanos/pkg/api/blocks" "github.com/thanos-io/thanos/pkg/block" + "github.com/thanos-io/thanos/pkg/block/indexheader" "github.com/thanos-io/thanos/pkg/block/metadata" "github.com/thanos-io/thanos/pkg/component" hidden "github.com/thanos-io/thanos/pkg/extflag" @@ -89,6 +90,8 @@ type storeConfig struct { lazyIndexReaderEnabled bool lazyIndexReaderIdleTimeout time.Duration lazyExpandedPostingsEnabled bool + + indexHeaderLazyDownloadStrategy string } func (sc *storeConfig) registerFlag(cmd extkingpin.FlagClause) { @@ -186,6 +189,10 @@ func (sc *storeConfig) registerFlag(cmd extkingpin.FlagClause) { cmd.Flag("store.enable-lazy-expanded-postings", "If true, Store Gateway will estimate postings size and try to lazily expand postings if it downloads less data than expanding all postings."). Default("false").BoolVar(&sc.lazyExpandedPostingsEnabled) + cmd.Flag("store.index-header-lazy-download-strategy", "Strategy of how to download index headers lazily. Supported values: eager, lazy. If eager, always download index header during initial load. If lazy, download index header during query time."). + Default(string(indexheader.EagerDownloadStrategy)). + EnumVar(&sc.indexHeaderLazyDownloadStrategy, string(indexheader.EagerDownloadStrategy), string(indexheader.LazyDownloadStrategy)) + cmd.Flag("web.disable", "Disable Block Viewer UI.").Default("false").BoolVar(&sc.disableWeb) cmd.Flag("web.external-prefix", "Static prefix for all HTML links and redirect URLs in the bucket web UI interface. Actual endpoints are still served on / or the web.route-prefix. This allows thanos bucket web UI to be served behind a reverse proxy that strips a URL sub-path."). @@ -388,6 +395,9 @@ func runStore( return conf.estimatedMaxChunkSize }), store.WithLazyExpandedPostings(conf.lazyExpandedPostingsEnabled), + store.WithIndexHeaderLazyDownloadStrategy( + indexheader.IndexHeaderLazyDownloadStrategy(conf.indexHeaderLazyDownloadStrategy).StrategyToDownloadFunc(), + ), } if conf.debugLogging { diff --git a/docs/components/store.md b/docs/components/store.md index 3ba59a2d9e..85fd4ce688 100644 --- a/docs/components/store.md +++ b/docs/components/store.md @@ -193,6 +193,12 @@ Flags: DEPRECATED: use store.limits.request-samples. --store.grpc.touched-series-limit=0 DEPRECATED: use store.limits.request-series. + --store.index-header-lazy-download-strategy=eager + Strategy of how to download index headers + lazily. Supported values: eager, lazy. + If eager, always download index header during + initial load. If lazy, download index header + during query time. --store.limits.request-samples=0 The maximum samples allowed for a single Series request, The Series call fails if diff --git a/pkg/block/indexheader/header_test.go b/pkg/block/indexheader/header_test.go index 56dabc33f7..4130157a96 100644 --- a/pkg/block/indexheader/header_test.go +++ b/pkg/block/indexheader/header_test.go @@ -206,7 +206,7 @@ func TestReaders(t *testing.T) { _, err := WriteBinary(ctx, bkt, id, fn) testutil.Ok(t, err) - br, err := NewLazyBinaryReader(ctx, log.NewNopLogger(), nil, tmpDir, id, 3, NewLazyBinaryReaderMetrics(nil), NewBinaryReaderMetrics(nil), nil) + br, err := NewLazyBinaryReader(ctx, log.NewNopLogger(), nil, tmpDir, id, 3, NewLazyBinaryReaderMetrics(nil), NewBinaryReaderMetrics(nil), nil, false) testutil.Ok(t, err) defer func() { testutil.Ok(t, br.Close()) }() diff --git a/pkg/block/indexheader/lazy_binary_reader.go b/pkg/block/indexheader/lazy_binary_reader.go index d7e589c724..2b36bf8025 100644 --- a/pkg/block/indexheader/lazy_binary_reader.go +++ b/pkg/block/indexheader/lazy_binary_reader.go @@ -83,6 +83,9 @@ type LazyBinaryReader struct { // Keep track of the last time it was used. usedAt *atomic.Int64 + + // If true, index header will be downloaded at query time rather than initialization time. + lazyDownload bool } // NewLazyBinaryReader makes a new LazyBinaryReader. If the index-header does not exist @@ -99,8 +102,9 @@ func NewLazyBinaryReader( metrics *LazyBinaryReaderMetrics, binaryReaderMetrics *BinaryReaderMetrics, onClosed func(*LazyBinaryReader), + lazyDownload bool, ) (*LazyBinaryReader, error) { - if dir != "" { + if dir != "" && !lazyDownload { indexHeaderFile := filepath.Join(dir, id.String(), block.IndexHeaderFilename) // If the index-header doesn't exist we should download it. if _, err := os.Stat(indexHeaderFile); err != nil { @@ -131,6 +135,7 @@ func NewLazyBinaryReader( binaryReaderMetrics: binaryReaderMetrics, usedAt: atomic.NewInt64(time.Now().UnixNano()), onClosed: onClosed, + lazyDownload: lazyDownload, }, nil } diff --git a/pkg/block/indexheader/lazy_binary_reader_test.go b/pkg/block/indexheader/lazy_binary_reader_test.go index 150f2d649b..d740da99ab 100644 --- a/pkg/block/indexheader/lazy_binary_reader_test.go +++ b/pkg/block/indexheader/lazy_binary_reader_test.go @@ -5,6 +5,7 @@ package indexheader import ( "context" + "fmt" "os" "path/filepath" "sync" @@ -31,11 +32,11 @@ func TestNewLazyBinaryReader_ShouldFailIfUnableToBuildIndexHeader(t *testing.T) bkt, err := filesystem.NewBucket(filepath.Join(tmpDir, "bkt")) testutil.Ok(t, err) defer func() { testutil.Ok(t, bkt.Close()) }() - _, err = NewLazyBinaryReader(ctx, log.NewNopLogger(), bkt, tmpDir, ulid.MustNew(0, nil), 3, NewLazyBinaryReaderMetrics(nil), NewBinaryReaderMetrics(nil), nil) + _, err = NewLazyBinaryReader(ctx, log.NewNopLogger(), bkt, tmpDir, ulid.MustNew(0, nil), 3, NewLazyBinaryReaderMetrics(nil), NewBinaryReaderMetrics(nil), nil, false) testutil.NotOk(t, err) } -func TestNewLazyBinaryReader_ShouldBuildIndexHeaderFromBucket(t *testing.T) { +func TestNewLazyBinaryReader_ShouldNotFailIfUnableToBuildIndexHeaderWhenLazyDownload(t *testing.T) { ctx := context.Background() tmpDir := t.TempDir() @@ -43,36 +44,61 @@ func TestNewLazyBinaryReader_ShouldBuildIndexHeaderFromBucket(t *testing.T) { bkt, err := filesystem.NewBucket(filepath.Join(tmpDir, "bkt")) testutil.Ok(t, err) defer func() { testutil.Ok(t, bkt.Close()) }() - - // Create block. - blockID, err := e2eutil.CreateBlock(ctx, tmpDir, []labels.Labels{ - {{Name: "a", Value: "1"}}, - {{Name: "a", Value: "2"}}, - }, 100, 0, 1000, labels.Labels{{Name: "ext1", Value: "1"}}, 124, metadata.NoneFunc) + _, err = NewLazyBinaryReader(ctx, log.NewNopLogger(), bkt, tmpDir, ulid.MustNew(0, nil), 3, NewLazyBinaryReaderMetrics(nil), NewBinaryReaderMetrics(nil), nil, true) testutil.Ok(t, err) - testutil.Ok(t, block.Upload(ctx, log.NewNopLogger(), bkt, filepath.Join(tmpDir, blockID.String()), metadata.NoneFunc)) +} - m := NewLazyBinaryReaderMetrics(nil) - bm := NewBinaryReaderMetrics(nil) - r, err := NewLazyBinaryReader(ctx, log.NewNopLogger(), bkt, tmpDir, blockID, 3, m, bm, nil) - testutil.Ok(t, err) - testutil.Assert(t, r.reader == nil) - testutil.Equals(t, float64(0), promtestutil.ToFloat64(m.loadCount)) - testutil.Equals(t, float64(0), promtestutil.ToFloat64(m.unloadCount)) +func TestNewLazyBinaryReader_ShouldBuildIndexHeaderFromBucket(t *testing.T) { + ctx := context.Background() - // Should lazy load the index upon first usage. - v, err := r.IndexVersion() - testutil.Ok(t, err) - testutil.Equals(t, 2, v) - testutil.Assert(t, r.reader != nil) - testutil.Equals(t, float64(1), promtestutil.ToFloat64(m.loadCount)) - testutil.Equals(t, float64(0), promtestutil.ToFloat64(m.unloadCount)) + tmpDir := t.TempDir() - labelNames, err := r.LabelNames() + bkt, err := filesystem.NewBucket(filepath.Join(tmpDir, "bkt")) testutil.Ok(t, err) - testutil.Equals(t, []string{"a"}, labelNames) - testutil.Equals(t, float64(1), promtestutil.ToFloat64(m.loadCount)) - testutil.Equals(t, float64(0), promtestutil.ToFloat64(m.unloadCount)) + defer func() { testutil.Ok(t, bkt.Close()) }() + + for _, lazyDownload := range []bool{false, true} { + t.Run(fmt.Sprintf("lazyDownload=%v", lazyDownload), func(t *testing.T) { + // Create block. + blockID, err := e2eutil.CreateBlock(ctx, tmpDir, []labels.Labels{ + {{Name: "a", Value: "1"}}, + {{Name: "a", Value: "2"}}, + }, 100, 0, 1000, labels.Labels{{Name: "ext1", Value: "1"}}, 124, metadata.NoneFunc) + testutil.Ok(t, err) + testutil.Ok(t, block.Upload(ctx, log.NewNopLogger(), bkt, filepath.Join(tmpDir, blockID.String()), metadata.NoneFunc)) + + m := NewLazyBinaryReaderMetrics(nil) + bm := NewBinaryReaderMetrics(nil) + r, err := NewLazyBinaryReader(ctx, log.NewNopLogger(), bkt, tmpDir, blockID, 3, m, bm, nil, lazyDownload) + testutil.Ok(t, err) + testutil.Assert(t, r.reader == nil) + testutil.Equals(t, float64(0), promtestutil.ToFloat64(m.loadCount)) + testutil.Equals(t, float64(0), promtestutil.ToFloat64(m.unloadCount)) + + _, err = os.Stat(filepath.Join(r.dir, blockID.String(), block.IndexHeaderFilename)) + // Index file shouldn't exist. + if lazyDownload { + testutil.Equals(t, true, os.IsNotExist(err)) + } + // Should lazy load the index upon first usage. + v, err := r.IndexVersion() + testutil.Ok(t, err) + if lazyDownload { + _, err = os.Stat(filepath.Join(r.dir, blockID.String(), block.IndexHeaderFilename)) + testutil.Ok(t, err) + } + testutil.Equals(t, 2, v) + testutil.Assert(t, r.reader != nil) + testutil.Equals(t, float64(1), promtestutil.ToFloat64(m.loadCount)) + testutil.Equals(t, float64(0), promtestutil.ToFloat64(m.unloadCount)) + + labelNames, err := r.LabelNames() + testutil.Ok(t, err) + testutil.Equals(t, []string{"a"}, labelNames) + testutil.Equals(t, float64(1), promtestutil.ToFloat64(m.loadCount)) + testutil.Equals(t, float64(0), promtestutil.ToFloat64(m.unloadCount)) + }) + } } func TestNewLazyBinaryReader_ShouldRebuildCorruptedIndexHeader(t *testing.T) { @@ -96,22 +122,26 @@ func TestNewLazyBinaryReader_ShouldRebuildCorruptedIndexHeader(t *testing.T) { headerFilename := filepath.Join(tmpDir, blockID.String(), block.IndexHeaderFilename) testutil.Ok(t, os.WriteFile(headerFilename, []byte("xxx"), os.ModePerm)) - m := NewLazyBinaryReaderMetrics(nil) - bm := NewBinaryReaderMetrics(nil) - r, err := NewLazyBinaryReader(ctx, log.NewNopLogger(), bkt, tmpDir, blockID, 3, m, bm, nil) - testutil.Ok(t, err) - testutil.Assert(t, r.reader == nil) - testutil.Equals(t, float64(0), promtestutil.ToFloat64(m.loadCount)) - testutil.Equals(t, float64(0), promtestutil.ToFloat64(m.loadFailedCount)) - testutil.Equals(t, float64(0), promtestutil.ToFloat64(m.unloadCount)) - - // Ensure it can read data. - labelNames, err := r.LabelNames() - testutil.Ok(t, err) - testutil.Equals(t, []string{"a"}, labelNames) - testutil.Equals(t, float64(1), promtestutil.ToFloat64(m.loadCount)) - testutil.Equals(t, float64(0), promtestutil.ToFloat64(m.loadFailedCount)) - testutil.Equals(t, float64(0), promtestutil.ToFloat64(m.unloadCount)) + for _, lazyDownload := range []bool{false, true} { + t.Run(fmt.Sprintf("lazyDownload=%v", lazyDownload), func(t *testing.T) { + m := NewLazyBinaryReaderMetrics(nil) + bm := NewBinaryReaderMetrics(nil) + r, err := NewLazyBinaryReader(ctx, log.NewNopLogger(), bkt, tmpDir, blockID, 3, m, bm, nil, lazyDownload) + testutil.Ok(t, err) + testutil.Assert(t, r.reader == nil) + testutil.Equals(t, float64(0), promtestutil.ToFloat64(m.loadCount)) + testutil.Equals(t, float64(0), promtestutil.ToFloat64(m.loadFailedCount)) + testutil.Equals(t, float64(0), promtestutil.ToFloat64(m.unloadCount)) + + // Ensure it can read data. + labelNames, err := r.LabelNames() + testutil.Ok(t, err) + testutil.Equals(t, []string{"a"}, labelNames) + testutil.Equals(t, float64(1), promtestutil.ToFloat64(m.loadCount)) + testutil.Equals(t, float64(0), promtestutil.ToFloat64(m.loadFailedCount)) + testutil.Equals(t, float64(0), promtestutil.ToFloat64(m.unloadCount)) + }) + } } func TestLazyBinaryReader_ShouldReopenOnUsageAfterClose(t *testing.T) { @@ -131,37 +161,41 @@ func TestLazyBinaryReader_ShouldReopenOnUsageAfterClose(t *testing.T) { testutil.Ok(t, err) testutil.Ok(t, block.Upload(ctx, log.NewNopLogger(), bkt, filepath.Join(tmpDir, blockID.String()), metadata.NoneFunc)) - m := NewLazyBinaryReaderMetrics(nil) - bm := NewBinaryReaderMetrics(nil) - r, err := NewLazyBinaryReader(ctx, log.NewNopLogger(), bkt, tmpDir, blockID, 3, m, bm, nil) - testutil.Ok(t, err) - testutil.Assert(t, r.reader == nil) - - // Should lazy load the index upon first usage. - labelNames, err := r.LabelNames() - testutil.Ok(t, err) - testutil.Equals(t, []string{"a"}, labelNames) - testutil.Equals(t, float64(1), promtestutil.ToFloat64(m.loadCount)) - testutil.Equals(t, float64(0), promtestutil.ToFloat64(m.loadFailedCount)) - - // Close it. - testutil.Ok(t, r.Close()) - testutil.Assert(t, r.reader == nil) - testutil.Equals(t, float64(1), promtestutil.ToFloat64(m.unloadCount)) - testutil.Equals(t, float64(0), promtestutil.ToFloat64(m.unloadFailedCount)) - - // Should lazy load again upon next usage. - labelNames, err = r.LabelNames() - testutil.Ok(t, err) - testutil.Equals(t, []string{"a"}, labelNames) - testutil.Equals(t, float64(2), promtestutil.ToFloat64(m.loadCount)) - testutil.Equals(t, float64(0), promtestutil.ToFloat64(m.loadFailedCount)) - - // Closing an already closed lazy reader should be a no-op. - for i := 0; i < 2; i++ { - testutil.Ok(t, r.Close()) - testutil.Equals(t, float64(2), promtestutil.ToFloat64(m.unloadCount)) - testutil.Equals(t, float64(0), promtestutil.ToFloat64(m.unloadFailedCount)) + for _, lazyDownload := range []bool{false, true} { + t.Run(fmt.Sprintf("lazyDownload=%v", lazyDownload), func(t *testing.T) { + m := NewLazyBinaryReaderMetrics(nil) + bm := NewBinaryReaderMetrics(nil) + r, err := NewLazyBinaryReader(ctx, log.NewNopLogger(), bkt, tmpDir, blockID, 3, m, bm, nil, lazyDownload) + testutil.Ok(t, err) + testutil.Assert(t, r.reader == nil) + + // Should lazy load the index upon first usage. + labelNames, err := r.LabelNames() + testutil.Ok(t, err) + testutil.Equals(t, []string{"a"}, labelNames) + testutil.Equals(t, float64(1), promtestutil.ToFloat64(m.loadCount)) + testutil.Equals(t, float64(0), promtestutil.ToFloat64(m.loadFailedCount)) + + // Close it. + testutil.Ok(t, r.Close()) + testutil.Assert(t, r.reader == nil) + testutil.Equals(t, float64(1), promtestutil.ToFloat64(m.unloadCount)) + testutil.Equals(t, float64(0), promtestutil.ToFloat64(m.unloadFailedCount)) + + // Should lazy load again upon next usage. + labelNames, err = r.LabelNames() + testutil.Ok(t, err) + testutil.Equals(t, []string{"a"}, labelNames) + testutil.Equals(t, float64(2), promtestutil.ToFloat64(m.loadCount)) + testutil.Equals(t, float64(0), promtestutil.ToFloat64(m.loadFailedCount)) + + // Closing an already closed lazy reader should be a no-op. + for i := 0; i < 2; i++ { + testutil.Ok(t, r.Close()) + testutil.Equals(t, float64(2), promtestutil.ToFloat64(m.unloadCount)) + testutil.Equals(t, float64(0), promtestutil.ToFloat64(m.unloadFailedCount)) + } + }) } } @@ -182,34 +216,38 @@ func TestLazyBinaryReader_unload_ShouldReturnErrorIfNotIdle(t *testing.T) { testutil.Ok(t, err) testutil.Ok(t, block.Upload(ctx, log.NewNopLogger(), bkt, filepath.Join(tmpDir, blockID.String()), metadata.NoneFunc)) - m := NewLazyBinaryReaderMetrics(nil) - bm := NewBinaryReaderMetrics(nil) - r, err := NewLazyBinaryReader(ctx, log.NewNopLogger(), bkt, tmpDir, blockID, 3, m, bm, nil) - testutil.Ok(t, err) - testutil.Assert(t, r.reader == nil) - - // Should lazy load the index upon first usage. - labelNames, err := r.LabelNames() - testutil.Ok(t, err) - testutil.Equals(t, []string{"a"}, labelNames) - testutil.Equals(t, float64(1), promtestutil.ToFloat64(m.loadCount)) - testutil.Equals(t, float64(0), promtestutil.ToFloat64(m.loadFailedCount)) - testutil.Equals(t, float64(0), promtestutil.ToFloat64(m.unloadCount)) - testutil.Equals(t, float64(0), promtestutil.ToFloat64(m.unloadFailedCount)) - - // Try to unload but not idle since enough time. - testutil.Equals(t, errNotIdle, r.unloadIfIdleSince(time.Now().Add(-time.Minute).UnixNano())) - testutil.Equals(t, float64(1), promtestutil.ToFloat64(m.loadCount)) - testutil.Equals(t, float64(0), promtestutil.ToFloat64(m.loadFailedCount)) - testutil.Equals(t, float64(0), promtestutil.ToFloat64(m.unloadCount)) - testutil.Equals(t, float64(0), promtestutil.ToFloat64(m.unloadFailedCount)) - - // Try to unload and idle since enough time. - testutil.Ok(t, r.unloadIfIdleSince(time.Now().UnixNano())) - testutil.Equals(t, float64(1), promtestutil.ToFloat64(m.loadCount)) - testutil.Equals(t, float64(0), promtestutil.ToFloat64(m.loadFailedCount)) - testutil.Equals(t, float64(1), promtestutil.ToFloat64(m.unloadCount)) - testutil.Equals(t, float64(0), promtestutil.ToFloat64(m.unloadFailedCount)) + for _, lazyDownload := range []bool{false, true} { + t.Run(fmt.Sprintf("lazyDownload=%v", lazyDownload), func(t *testing.T) { + m := NewLazyBinaryReaderMetrics(nil) + bm := NewBinaryReaderMetrics(nil) + r, err := NewLazyBinaryReader(ctx, log.NewNopLogger(), bkt, tmpDir, blockID, 3, m, bm, nil, lazyDownload) + testutil.Ok(t, err) + testutil.Assert(t, r.reader == nil) + + // Should lazy load the index upon first usage. + labelNames, err := r.LabelNames() + testutil.Ok(t, err) + testutil.Equals(t, []string{"a"}, labelNames) + testutil.Equals(t, float64(1), promtestutil.ToFloat64(m.loadCount)) + testutil.Equals(t, float64(0), promtestutil.ToFloat64(m.loadFailedCount)) + testutil.Equals(t, float64(0), promtestutil.ToFloat64(m.unloadCount)) + testutil.Equals(t, float64(0), promtestutil.ToFloat64(m.unloadFailedCount)) + + // Try to unload but not idle since enough time. + testutil.Equals(t, errNotIdle, r.unloadIfIdleSince(time.Now().Add(-time.Minute).UnixNano())) + testutil.Equals(t, float64(1), promtestutil.ToFloat64(m.loadCount)) + testutil.Equals(t, float64(0), promtestutil.ToFloat64(m.loadFailedCount)) + testutil.Equals(t, float64(0), promtestutil.ToFloat64(m.unloadCount)) + testutil.Equals(t, float64(0), promtestutil.ToFloat64(m.unloadFailedCount)) + + // Try to unload and idle since enough time. + testutil.Ok(t, r.unloadIfIdleSince(time.Now().UnixNano())) + testutil.Equals(t, float64(1), promtestutil.ToFloat64(m.loadCount)) + testutil.Equals(t, float64(0), promtestutil.ToFloat64(m.loadFailedCount)) + testutil.Equals(t, float64(1), promtestutil.ToFloat64(m.unloadCount)) + testutil.Equals(t, float64(0), promtestutil.ToFloat64(m.unloadFailedCount)) + }) + } } func TestLazyBinaryReader_LoadUnloadRaceCondition(t *testing.T) { @@ -232,49 +270,53 @@ func TestLazyBinaryReader_LoadUnloadRaceCondition(t *testing.T) { testutil.Ok(t, err) testutil.Ok(t, block.Upload(ctx, log.NewNopLogger(), bkt, filepath.Join(tmpDir, blockID.String()), metadata.NoneFunc)) - m := NewLazyBinaryReaderMetrics(nil) - bm := NewBinaryReaderMetrics(nil) - r, err := NewLazyBinaryReader(ctx, log.NewNopLogger(), bkt, tmpDir, blockID, 3, m, bm, nil) - testutil.Ok(t, err) - testutil.Assert(t, r.reader == nil) - t.Cleanup(func() { - testutil.Ok(t, r.Close()) - }) - - done := make(chan struct{}) - time.AfterFunc(runDuration, func() { close(done) }) - wg := sync.WaitGroup{} - wg.Add(2) - - // Start a goroutine which continuously try to unload the reader. - go func() { - defer wg.Done() - - for { - select { - case <-done: - return - default: - testutil.Ok(t, r.unloadIfIdleSince(0)) - } - } - }() - - // Try to read multiple times, while the other goroutine continuously try to unload it. - go func() { - defer wg.Done() - - for { - select { - case <-done: - return - default: - _, err := r.PostingsOffset("a", "1") - testutil.Assert(t, err == nil || err == errUnloadedWhileLoading) - } - } - }() - - // Wait until both goroutines have done. - wg.Wait() + for _, lazyDownload := range []bool{false, true} { + t.Run(fmt.Sprintf("lazyDownload=%v", lazyDownload), func(t *testing.T) { + m := NewLazyBinaryReaderMetrics(nil) + bm := NewBinaryReaderMetrics(nil) + r, err := NewLazyBinaryReader(ctx, log.NewNopLogger(), bkt, tmpDir, blockID, 3, m, bm, nil, lazyDownload) + testutil.Ok(t, err) + testutil.Assert(t, r.reader == nil) + t.Cleanup(func() { + testutil.Ok(t, r.Close()) + }) + + done := make(chan struct{}) + time.AfterFunc(runDuration, func() { close(done) }) + wg := sync.WaitGroup{} + wg.Add(2) + + // Start a goroutine which continuously try to unload the reader. + go func() { + defer wg.Done() + + for { + select { + case <-done: + return + default: + testutil.Ok(t, r.unloadIfIdleSince(0)) + } + } + }() + + // Try to read multiple times, while the other goroutine continuously try to unload it. + go func() { + defer wg.Done() + + for { + select { + case <-done: + return + default: + _, err := r.PostingsOffset("a", "1") + testutil.Assert(t, err == nil || err == errUnloadedWhileLoading) + } + } + }() + + // Wait until both goroutines have done. + wg.Wait() + }) + } } diff --git a/pkg/block/indexheader/reader_pool.go b/pkg/block/indexheader/reader_pool.go index fc8cb26813..e9fe5eb7dc 100644 --- a/pkg/block/indexheader/reader_pool.go +++ b/pkg/block/indexheader/reader_pool.go @@ -14,6 +14,8 @@ import ( "github.com/pkg/errors" "github.com/prometheus/client_golang/prometheus" "github.com/thanos-io/objstore" + + "github.com/thanos-io/thanos/pkg/block/metadata" ) // ReaderPoolMetrics holds metrics tracked by ReaderPool. @@ -46,10 +48,47 @@ type ReaderPool struct { // Keep track of all readers managed by the pool. lazyReadersMx sync.Mutex lazyReaders map[*LazyBinaryReader]struct{} + + lazyDownloadFunc LazyDownloadIndexHeaderFunc +} + +// IndexHeaderLazyDownloadStrategy specifies how to download index headers +// lazily. Only used when lazy mmap is enabled. +type IndexHeaderLazyDownloadStrategy string + +const ( + // EagerDownloadStrategy always disables lazy downloading index headers. + EagerDownloadStrategy IndexHeaderLazyDownloadStrategy = "eager" + // LazyDownloadStrategy always lazily download index headers. + LazyDownloadStrategy IndexHeaderLazyDownloadStrategy = "lazy" +) + +func (s IndexHeaderLazyDownloadStrategy) StrategyToDownloadFunc() LazyDownloadIndexHeaderFunc { + switch s { + case LazyDownloadStrategy: + return AlwaysLazyDownloadIndexHeader + default: + // Always fallback to eager download index header. + return AlwaysEagerDownloadIndexHeader + } +} + +// LazyDownloadIndexHeaderFunc is used to determinte whether to download the index header lazily +// or not by checking its block metadata. Usecase can be by time or by index file size. +type LazyDownloadIndexHeaderFunc func(meta *metadata.Meta) bool + +// AlwaysEagerDownloadIndexHeader always eagerly download index header. +func AlwaysEagerDownloadIndexHeader(meta *metadata.Meta) bool { + return false +} + +// AlwaysLazyDownloadIndexHeader always lazily download index header. +func AlwaysLazyDownloadIndexHeader(meta *metadata.Meta) bool { + return true } // NewReaderPool makes a new ReaderPool. -func NewReaderPool(logger log.Logger, lazyReaderEnabled bool, lazyReaderIdleTimeout time.Duration, metrics *ReaderPoolMetrics) *ReaderPool { +func NewReaderPool(logger log.Logger, lazyReaderEnabled bool, lazyReaderIdleTimeout time.Duration, metrics *ReaderPoolMetrics, lazyDownloadFunc LazyDownloadIndexHeaderFunc) *ReaderPool { p := &ReaderPool{ logger: logger, metrics: metrics, @@ -57,6 +96,7 @@ func NewReaderPool(logger log.Logger, lazyReaderEnabled bool, lazyReaderIdleTime lazyReaderIdleTimeout: lazyReaderIdleTimeout, lazyReaders: make(map[*LazyBinaryReader]struct{}), close: make(chan struct{}), + lazyDownloadFunc: lazyDownloadFunc, } // Start a goroutine to close idle readers (only if required). @@ -81,12 +121,12 @@ func NewReaderPool(logger log.Logger, lazyReaderEnabled bool, lazyReaderIdleTime // NewBinaryReader creates and returns a new binary reader. If the pool has been configured // with lazy reader enabled, this function will return a lazy reader. The returned lazy reader // is tracked by the pool and automatically closed once the idle timeout expires. -func (p *ReaderPool) NewBinaryReader(ctx context.Context, logger log.Logger, bkt objstore.BucketReader, dir string, id ulid.ULID, postingOffsetsInMemSampling int) (Reader, error) { +func (p *ReaderPool) NewBinaryReader(ctx context.Context, logger log.Logger, bkt objstore.BucketReader, dir string, id ulid.ULID, postingOffsetsInMemSampling int, meta *metadata.Meta) (Reader, error) { var reader Reader var err error if p.lazyReaderEnabled { - reader, err = NewLazyBinaryReader(ctx, logger, bkt, dir, id, postingOffsetsInMemSampling, p.metrics.lazyReader, p.metrics.binaryReader, p.onLazyReaderClosed) + reader, err = NewLazyBinaryReader(ctx, logger, bkt, dir, id, postingOffsetsInMemSampling, p.metrics.lazyReader, p.metrics.binaryReader, p.onLazyReaderClosed, p.lazyDownloadFunc(meta)) } else { reader, err = NewBinaryReader(ctx, logger, bkt, dir, id, postingOffsetsInMemSampling, p.metrics.binaryReader) } diff --git a/pkg/block/indexheader/reader_pool_test.go b/pkg/block/indexheader/reader_pool_test.go index 4ed60ea8fb..a7445f0fed 100644 --- a/pkg/block/indexheader/reader_pool_test.go +++ b/pkg/block/indexheader/reader_pool_test.go @@ -54,12 +54,15 @@ func TestReaderPool_NewBinaryReader(t *testing.T) { testutil.Ok(t, err) testutil.Ok(t, block.Upload(ctx, log.NewNopLogger(), bkt, filepath.Join(tmpDir, blockID.String()), metadata.NoneFunc)) + meta, err := metadata.ReadFromDir(filepath.Join(tmpDir, blockID.String())) + testutil.Ok(t, err) + for testName, testData := range tests { t.Run(testName, func(t *testing.T) { - pool := NewReaderPool(log.NewNopLogger(), testData.lazyReaderEnabled, testData.lazyReaderIdleTimeout, NewReaderPoolMetrics(nil)) + pool := NewReaderPool(log.NewNopLogger(), testData.lazyReaderEnabled, testData.lazyReaderIdleTimeout, NewReaderPoolMetrics(nil), AlwaysEagerDownloadIndexHeader) defer pool.Close() - r, err := pool.NewBinaryReader(ctx, log.NewNopLogger(), bkt, tmpDir, blockID, 3) + r, err := pool.NewBinaryReader(ctx, log.NewNopLogger(), bkt, tmpDir, blockID, 3, meta) testutil.Ok(t, err) defer func() { testutil.Ok(t, r.Close()) }() @@ -89,12 +92,14 @@ func TestReaderPool_ShouldCloseIdleLazyReaders(t *testing.T) { }, 100, 0, 1000, labels.Labels{{Name: "ext1", Value: "1"}}, 124, metadata.NoneFunc) testutil.Ok(t, err) testutil.Ok(t, block.Upload(ctx, log.NewNopLogger(), bkt, filepath.Join(tmpDir, blockID.String()), metadata.NoneFunc)) + meta, err := metadata.ReadFromDir(filepath.Join(tmpDir, blockID.String())) + testutil.Ok(t, err) metrics := NewReaderPoolMetrics(nil) - pool := NewReaderPool(log.NewNopLogger(), true, idleTimeout, metrics) + pool := NewReaderPool(log.NewNopLogger(), true, idleTimeout, metrics, AlwaysEagerDownloadIndexHeader) defer pool.Close() - r, err := pool.NewBinaryReader(ctx, log.NewNopLogger(), bkt, tmpDir, blockID, 3) + r, err := pool.NewBinaryReader(ctx, log.NewNopLogger(), bkt, tmpDir, blockID, 3, meta) testutil.Ok(t, err) defer func() { testutil.Ok(t, r.Close()) }() diff --git a/pkg/store/bucket.go b/pkg/store/bucket.go index 3ebd6f06a4..fd4fb7392c 100644 --- a/pkg/store/bucket.go +++ b/pkg/store/bucket.go @@ -413,6 +413,8 @@ type BucketStore struct { blockEstimatedMaxSeriesFunc BlockEstimator blockEstimatedMaxChunkFunc BlockEstimator + + indexHeaderLazyDownloadStrategy indexheader.LazyDownloadIndexHeaderFunc } func (s *BucketStore) validate() error { @@ -531,6 +533,14 @@ func WithDontResort(true bool) BucketStoreOption { } } +// WithIndexHeaderLazyDownloadStrategy specifies what block to lazy download its index header. +// Only used when lazy mmap is enabled at the same time. +func WithIndexHeaderLazyDownloadStrategy(strategy indexheader.LazyDownloadIndexHeaderFunc) BucketStoreOption { + return func(s *BucketStore) { + s.indexHeaderLazyDownloadStrategy = strategy + } +} + // NewBucketStore creates a new bucket backed store that implements the store API against // an object store bucket. It is optimized to work against high latency backends. func NewBucketStore( @@ -559,21 +569,22 @@ func NewBucketStore( b := make([]byte, 0, initialBufSize) return &b }}, - chunkPool: pool.NoopBytes{}, - blocks: map[ulid.ULID]*bucketBlock{}, - blockSets: map[uint64]*bucketBlockSet{}, - blockSyncConcurrency: blockSyncConcurrency, - queryGate: gate.NewNoop(), - chunksLimiterFactory: chunksLimiterFactory, - seriesLimiterFactory: seriesLimiterFactory, - bytesLimiterFactory: bytesLimiterFactory, - partitioner: partitioner, - enableCompatibilityLabel: enableCompatibilityLabel, - postingOffsetsInMemSampling: postingOffsetsInMemSampling, - enableSeriesResponseHints: enableSeriesResponseHints, - enableChunkHashCalculation: enableChunkHashCalculation, - seriesBatchSize: SeriesBatchSize, - sortingStrategy: sortingStrategyStore, + chunkPool: pool.NoopBytes{}, + blocks: map[ulid.ULID]*bucketBlock{}, + blockSets: map[uint64]*bucketBlockSet{}, + blockSyncConcurrency: blockSyncConcurrency, + queryGate: gate.NewNoop(), + chunksLimiterFactory: chunksLimiterFactory, + seriesLimiterFactory: seriesLimiterFactory, + bytesLimiterFactory: bytesLimiterFactory, + partitioner: partitioner, + enableCompatibilityLabel: enableCompatibilityLabel, + postingOffsetsInMemSampling: postingOffsetsInMemSampling, + enableSeriesResponseHints: enableSeriesResponseHints, + enableChunkHashCalculation: enableChunkHashCalculation, + seriesBatchSize: SeriesBatchSize, + sortingStrategy: sortingStrategyStore, + indexHeaderLazyDownloadStrategy: indexheader.AlwaysEagerDownloadIndexHeader, } for _, option := range options { @@ -582,7 +593,7 @@ func NewBucketStore( // Depend on the options indexReaderPoolMetrics := indexheader.NewReaderPoolMetrics(extprom.WrapRegistererWithPrefix("thanos_bucket_store_", s.reg)) - s.indexReaderPool = indexheader.NewReaderPool(s.logger, lazyIndexReaderEnabled, lazyIndexReaderIdleTimeout, indexReaderPoolMetrics) + s.indexReaderPool = indexheader.NewReaderPool(s.logger, lazyIndexReaderEnabled, lazyIndexReaderIdleTimeout, indexReaderPoolMetrics, s.indexHeaderLazyDownloadStrategy) s.metrics = newBucketStoreMetrics(s.reg) // TODO(metalmatze): Might be possible via Option too if err := s.validate(); err != nil { @@ -759,6 +770,7 @@ func (s *BucketStore) addBlock(ctx context.Context, meta *metadata.Meta) (err er s.dir, meta.ULID, s.postingOffsetsInMemSampling, + meta, ) if err != nil { return errors.Wrap(err, "create index header reader") diff --git a/pkg/store/bucket_test.go b/pkg/store/bucket_test.go index 87659f5450..67223a9467 100644 --- a/pkg/store/bucket_test.go +++ b/pkg/store/bucket_test.go @@ -1658,7 +1658,7 @@ func TestBucketSeries_OneBlock_InMemIndexCacheSegfault(t *testing.T) { bkt: objstore.WithNoopInstr(bkt), logger: logger, indexCache: indexCache, - indexReaderPool: indexheader.NewReaderPool(log.NewNopLogger(), false, 0, indexheader.NewReaderPoolMetrics(nil)), + indexReaderPool: indexheader.NewReaderPool(log.NewNopLogger(), false, 0, indexheader.NewReaderPoolMetrics(nil), indexheader.AlwaysEagerDownloadIndexHeader), metrics: newBucketStoreMetrics(nil), blockSets: map[uint64]*bucketBlockSet{ labels.Labels{{Name: "ext1", Value: "1"}}.Hash(): {blocks: [][]*bucketBlock{{b1, b2}}}, From a59a3ef4f2ed1435c8144d69ae36efcfce0cb83b Mon Sep 17 00:00:00 2001 From: Pranav <101933072+pawarpranav83@users.noreply.github.com> Date: Mon, 25 Dec 2023 14:22:06 +0530 Subject: [PATCH 08/26] tests: use remote write in query frontend tests (#6998) --- test/e2e/query_frontend_test.go | 116 +++++++++++++++++++++----------- test/e2e/query_test.go | 34 ++++++++++ 2 files changed, 112 insertions(+), 38 deletions(-) diff --git a/test/e2e/query_frontend_test.go b/test/e2e/query_frontend_test.go index f93219cae6..5fda32c0f7 100644 --- a/test/e2e/query_frontend_test.go +++ b/test/e2e/query_frontend_test.go @@ -24,6 +24,7 @@ import ( "github.com/prometheus/common/model" "github.com/prometheus/prometheus/model/labels" "github.com/prometheus/prometheus/model/timestamp" + "github.com/prometheus/prometheus/prompb" "github.com/thanos-io/thanos/pkg/block/metadata" "github.com/thanos-io/thanos/pkg/cacheutil" @@ -41,12 +42,13 @@ func TestQueryFrontend(t *testing.T) { testutil.Ok(t, err) t.Cleanup(e2ethanos.CleanScenario(t, e)) - now := time.Now() + // Predefined Timestamp + predefTimestamp := time.Date(2023, time.December, 22, 12, 0, 0, 0, time.UTC) - prom, sidecar := e2ethanos.NewPrometheusWithSidecar(e, "1", e2ethanos.DefaultPromConfig("test", 0, "", "", e2ethanos.LocalPrometheusTarget), "", e2ethanos.DefaultPrometheusImage(), "") - testutil.Ok(t, e2e.StartAndWaitReady(prom, sidecar)) + i := e2ethanos.NewReceiveBuilder(e, "ingestor-rw").WithIngestionEnabled().Init() + testutil.Ok(t, e2e.StartAndWaitReady(i)) - q := e2ethanos.NewQuerierBuilder(e, "1", sidecar.InternalEndpoint("grpc")).Init() + q := e2ethanos.NewQuerierBuilder(e, "1", i.InternalEndpoint("grpc")).Init() testutil.Ok(t, e2e.StartAndWaitReady(q)) inMemoryCacheConfig := queryfrontend.CacheProviderConfig{ @@ -64,17 +66,34 @@ func TestQueryFrontend(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), time.Minute) t.Cleanup(cancel) + // Writing a custom Timeseries into the receiver + testutil.Ok(t, remoteWrite(ctx, []prompb.TimeSeries{{ + Labels: []prompb.Label{ + {Name: "__name__", Value: "up"}, + {Name: "instance", Value: "localhost:9090"}, + {Name: "job", Value: "myself"}, + {Name: "prometheus", Value: "test"}, + {Name: "replica", Value: "0"}, + }, + Samples: []prompb.Sample{ + {Value: float64(0), Timestamp: timestamp.FromTime(predefTimestamp)}, + }}}, + i.Endpoint("remote-write"), + )) + testutil.Ok(t, q.WaitSumMetricsWithOptions(e2emon.Equals(1), []string{"thanos_store_nodes_grpc_connections"}, e2emon.WaitMissingMetrics())) // Ensure we can get the result from Querier first so that it // doesn't need to retry when we send queries to the frontend later. - queryAndAssertSeries(t, ctx, q.Endpoint("http"), e2ethanos.QueryUpWithoutInstance, time.Now, promclient.QueryOptions{ + queryAndAssertSeries(t, ctx, q.Endpoint("http"), e2ethanos.QueryUpWithoutInstance, func() time.Time { return predefTimestamp }, promclient.QueryOptions{ Deduplicate: false, }, []model.Metric{ { "job": "myself", "prometheus": "test", + "receive": "receive-ingestor-rw", "replica": "0", + "tenant_id": "default-tenant", }, }) @@ -86,13 +105,15 @@ func TestQueryFrontend(t *testing.T) { queryTimes := vals[0] t.Run("query frontend works for instant query", func(t *testing.T) { - queryAndAssertSeries(t, ctx, queryFrontend.Endpoint("http"), e2ethanos.QueryUpWithoutInstance, time.Now, promclient.QueryOptions{ + queryAndAssertSeries(t, ctx, queryFrontend.Endpoint("http"), e2ethanos.QueryUpWithoutInstance, func() time.Time { return predefTimestamp }, promclient.QueryOptions{ Deduplicate: false, }, []model.Metric{ { "job": "myself", "prometheus": "test", + "receive": "receive-ingestor-rw", "replica": "0", + "tenant_id": "default-tenant", }, }) @@ -115,8 +136,8 @@ func TestQueryFrontend(t *testing.T) { ctx, queryFrontend.Endpoint("http"), e2ethanos.QueryUpWithoutInstance, - timestamp.FromTime(now.Add(-time.Hour)), - timestamp.FromTime(now.Add(time.Hour)), + timestamp.FromTime(predefTimestamp.Add(-time.Hour)), + timestamp.FromTime(predefTimestamp.Add(time.Hour)), 14, promclient.QueryOptions{ Deduplicate: true, @@ -159,8 +180,8 @@ func TestQueryFrontend(t *testing.T) { ctx, queryFrontend.Endpoint("http"), e2ethanos.QueryUpWithoutInstance, - timestamp.FromTime(now.Add(-time.Hour)), - timestamp.FromTime(now.Add(time.Hour)), + timestamp.FromTime(predefTimestamp.Add(-time.Hour)), + timestamp.FromTime(predefTimestamp.Add(time.Hour)), 14, promclient.QueryOptions{ Deduplicate: true, @@ -181,7 +202,7 @@ func TestQueryFrontend(t *testing.T) { testutil.Ok(t, queryFrontend.WaitSumMetrics(e2emon.Equals(2), "cortex_cache_fetched_keys_total")) testutil.Ok(t, queryFrontend.WaitSumMetrics(e2emon.Equals(1), "cortex_cache_hits_total")) testutil.Ok(t, queryFrontend.WaitSumMetrics(e2emon.Equals(1), "querier_cache_added_new_total")) - testutil.Ok(t, queryFrontend.WaitSumMetrics(e2emon.Equals(2), "querier_cache_added_total")) + testutil.Ok(t, queryFrontend.WaitSumMetrics(e2emon.Equals(1), "querier_cache_added_total")) testutil.Ok(t, queryFrontend.WaitSumMetrics(e2emon.Equals(1), "querier_cache_entries")) testutil.Ok(t, queryFrontend.WaitSumMetrics(e2emon.Equals(2), "querier_cache_gets_total")) testutil.Ok(t, queryFrontend.WaitSumMetrics(e2emon.Equals(1), "querier_cache_misses_total")) @@ -192,9 +213,8 @@ func TestQueryFrontend(t *testing.T) { e2emon.WithLabelMatchers(matchers.MustNewMatcher(matchers.MatchEqual, "tripperware", "query_range"))), ) - // One more request is needed in order to satisfy the req range. testutil.Ok(t, q.WaitSumMetricsWithOptions( - e2emon.Equals(2), + e2emon.Equals(1), []string{"http_requests_total"}, e2emon.WithLabelMatchers(matchers.MustNewMatcher(matchers.MatchEqual, "handler", "query_range"))), ) @@ -206,8 +226,8 @@ func TestQueryFrontend(t *testing.T) { ctx, queryFrontend.Endpoint("http"), e2ethanos.QueryUpWithoutInstance, - timestamp.FromTime(now.Add(-time.Hour)), - timestamp.FromTime(now.Add(24*time.Hour)), + timestamp.FromTime(predefTimestamp.Add(-time.Hour)), + timestamp.FromTime(predefTimestamp.Add(24*time.Hour)), 14, promclient.QueryOptions{ Deduplicate: true, @@ -225,13 +245,13 @@ func TestQueryFrontend(t *testing.T) { []string{"thanos_query_frontend_queries_total"}, e2emon.WithLabelMatchers(matchers.MustNewMatcher(matchers.MatchEqual, "op", "query_range"))), ) - testutil.Ok(t, queryFrontend.WaitSumMetrics(e2emon.Equals(3), "cortex_cache_fetched_keys_total")) + testutil.Ok(t, queryFrontend.WaitSumMetrics(e2emon.Equals(4), "cortex_cache_fetched_keys_total")) testutil.Ok(t, queryFrontend.WaitSumMetrics(e2emon.Equals(2), "cortex_cache_hits_total")) - testutil.Ok(t, queryFrontend.WaitSumMetrics(e2emon.Equals(1), "querier_cache_added_new_total")) + testutil.Ok(t, queryFrontend.WaitSumMetrics(e2emon.Equals(2), "querier_cache_added_new_total")) testutil.Ok(t, queryFrontend.WaitSumMetrics(e2emon.Equals(3), "querier_cache_added_total")) - testutil.Ok(t, queryFrontend.WaitSumMetrics(e2emon.Equals(1), "querier_cache_entries")) - testutil.Ok(t, queryFrontend.WaitSumMetrics(e2emon.Equals(3), "querier_cache_gets_total")) - testutil.Ok(t, queryFrontend.WaitSumMetrics(e2emon.Equals(1), "querier_cache_misses_total")) + testutil.Ok(t, queryFrontend.WaitSumMetrics(e2emon.Equals(2), "querier_cache_entries")) + testutil.Ok(t, queryFrontend.WaitSumMetrics(e2emon.Equals(4), "querier_cache_gets_total")) + testutil.Ok(t, queryFrontend.WaitSumMetrics(e2emon.Equals(2), "querier_cache_misses_total")) // Query is 25h so it will be split to 2 requests. testutil.Ok(t, queryFrontend.WaitSumMetricsWithOptions( @@ -240,7 +260,7 @@ func TestQueryFrontend(t *testing.T) { ) testutil.Ok(t, q.WaitSumMetricsWithOptions( - e2emon.Equals(4), + e2emon.Equals(3), []string{"http_requests_total"}, e2emon.WithLabelMatchers(matchers.MustNewMatcher(matchers.MatchEqual, "handler", "query_range"))), ) @@ -248,7 +268,7 @@ func TestQueryFrontend(t *testing.T) { t.Run("query frontend splitting works for labels names API", func(t *testing.T) { // LabelNames and LabelValues API should still work via query frontend. - labelNames(t, ctx, queryFrontend.Endpoint("http"), nil, timestamp.FromTime(now.Add(-time.Hour)), timestamp.FromTime(now.Add(time.Hour)), func(res []string) bool { + labelNames(t, ctx, queryFrontend.Endpoint("http"), nil, timestamp.FromTime(predefTimestamp.Add(-time.Hour)), timestamp.FromTime(predefTimestamp.Add(time.Hour)), func(res []string) bool { return len(res) > 0 }) testutil.Ok(t, q.WaitSumMetricsWithOptions( @@ -267,7 +287,7 @@ func TestQueryFrontend(t *testing.T) { e2emon.WithLabelMatchers(matchers.MustNewMatcher(matchers.MatchEqual, "tripperware", "labels"))), ) - labelNames(t, ctx, queryFrontend.Endpoint("http"), nil, timestamp.FromTime(now.Add(-24*time.Hour)), timestamp.FromTime(now.Add(time.Hour)), func(res []string) bool { + labelNames(t, ctx, queryFrontend.Endpoint("http"), nil, timestamp.FromTime(predefTimestamp.Add(-24*time.Hour)), timestamp.FromTime(predefTimestamp.Add(time.Hour)), func(res []string) bool { return len(res) > 0 }) testutil.Ok(t, q.WaitSumMetricsWithOptions( @@ -288,7 +308,7 @@ func TestQueryFrontend(t *testing.T) { }) t.Run("query frontend splitting works for labels values API", func(t *testing.T) { - labelValues(t, ctx, queryFrontend.Endpoint("http"), "instance", nil, timestamp.FromTime(now.Add(-time.Hour)), timestamp.FromTime(now.Add(time.Hour)), func(res []string) bool { + labelValues(t, ctx, queryFrontend.Endpoint("http"), "instance", nil, timestamp.FromTime(predefTimestamp.Add(-time.Hour)), timestamp.FromTime(predefTimestamp.Add(time.Hour)), func(res []string) bool { return len(res) == 1 && res[0] == "localhost:9090" }) testutil.Ok(t, q.WaitSumMetricsWithOptions( @@ -307,7 +327,7 @@ func TestQueryFrontend(t *testing.T) { e2emon.WithLabelMatchers(matchers.MustNewMatcher(matchers.MatchEqual, "tripperware", "labels"))), ) - labelValues(t, ctx, queryFrontend.Endpoint("http"), "instance", nil, timestamp.FromTime(now.Add(-24*time.Hour)), timestamp.FromTime(now.Add(time.Hour)), func(res []string) bool { + labelValues(t, ctx, queryFrontend.Endpoint("http"), "instance", nil, timestamp.FromTime(predefTimestamp.Add(-24*time.Hour)), timestamp.FromTime(predefTimestamp.Add(time.Hour)), func(res []string) bool { return len(res) == 1 && res[0] == "localhost:9090" }) testutil.Ok(t, q.WaitSumMetricsWithOptions( @@ -333,8 +353,8 @@ func TestQueryFrontend(t *testing.T) { ctx, queryFrontend.Endpoint("http"), []*labels.Matcher{labels.MustNewMatcher(labels.MatchEqual, "__name__", "up")}, - timestamp.FromTime(now.Add(-time.Hour)), - timestamp.FromTime(now.Add(time.Hour)), + timestamp.FromTime(predefTimestamp.Add(-time.Hour)), + timestamp.FromTime(predefTimestamp.Add(time.Hour)), func(res []map[string]string) bool { if len(res) != 1 { return false @@ -345,6 +365,8 @@ func TestQueryFrontend(t *testing.T) { "instance": "localhost:9090", "job": "myself", "prometheus": "test", + "receive": "receive-ingestor-rw", + "tenant_id": "default-tenant", }) }, ) @@ -369,8 +391,8 @@ func TestQueryFrontend(t *testing.T) { ctx, queryFrontend.Endpoint("http"), []*labels.Matcher{labels.MustNewMatcher(labels.MatchEqual, "__name__", "up")}, - timestamp.FromTime(now.Add(-24*time.Hour)), - timestamp.FromTime(now.Add(time.Hour)), + timestamp.FromTime(predefTimestamp.Add(-24*time.Hour)), + timestamp.FromTime(predefTimestamp.Add(time.Hour)), func(res []map[string]string) bool { if len(res) != 1 { return false @@ -381,6 +403,8 @@ func TestQueryFrontend(t *testing.T) { "instance": "localhost:9090", "job": "myself", "prometheus": "test", + "receive": "receive-ingestor-rw", + "tenant_id": "default-tenant", }) }, ) @@ -409,12 +433,13 @@ func TestQueryFrontendMemcachedCache(t *testing.T) { testutil.Ok(t, err) t.Cleanup(e2ethanos.CleanScenario(t, e)) - now := time.Now() + // Predefined timestamp + predefTimestamp := time.Date(2023, time.December, 22, 12, 0, 0, 0, time.UTC) - prom, sidecar := e2ethanos.NewPrometheusWithSidecar(e, "1", e2ethanos.DefaultPromConfig("test", 0, "", "", e2ethanos.LocalPrometheusTarget), "", e2ethanos.DefaultPrometheusImage(), "") - testutil.Ok(t, e2e.StartAndWaitReady(prom, sidecar)) + i := e2ethanos.NewReceiveBuilder(e, "ingestor-rw").WithIngestionEnabled().Init() + testutil.Ok(t, e2e.StartAndWaitReady(i)) - q := e2ethanos.NewQuerierBuilder(e, "1", sidecar.InternalEndpoint("grpc")).Init() + q := e2ethanos.NewQuerierBuilder(e, "1", i.InternalEndpoint("grpc")).Init() testutil.Ok(t, e2e.StartAndWaitReady(q)) memcached := e2ethanos.NewMemcached(e, "1") @@ -443,19 +468,34 @@ func TestQueryFrontendMemcachedCache(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), time.Minute) t.Cleanup(cancel) + testutil.Ok(t, remoteWrite(ctx, []prompb.TimeSeries{{ + Labels: []prompb.Label{ + {Name: "__name__", Value: "up"}, + {Name: "instance", Value: "localhost:9090"}, + {Name: "job", Value: "myself"}, + {Name: "prometheus", Value: "test"}, + {Name: "replica", Value: "0"}, + }, + Samples: []prompb.Sample{ + {Value: float64(0), Timestamp: timestamp.FromTime(predefTimestamp)}, + }}}, + i.Endpoint("remote-write"))) + testutil.Ok(t, q.WaitSumMetricsWithOptions(e2emon.Equals(1), []string{"thanos_store_nodes_grpc_connections"}, e2emon.WaitMissingMetrics())) testutil.Ok(t, queryFrontend.WaitSumMetrics(e2emon.Equals(1), "cortex_memcache_client_servers")) // Ensure we can get the result from Querier first so that it // doesn't need to retry when we send queries to the frontend later. - queryAndAssertSeries(t, ctx, q.Endpoint("http"), e2ethanos.QueryUpWithoutInstance, time.Now, promclient.QueryOptions{ + queryAndAssertSeries(t, ctx, q.Endpoint("http"), e2ethanos.QueryUpWithoutInstance, func() time.Time { return predefTimestamp }, promclient.QueryOptions{ Deduplicate: false, }, []model.Metric{ { "job": "myself", "prometheus": "test", + "receive": "receive-ingestor-rw", "replica": "0", + "tenant_id": "default-tenant", }, }) @@ -469,8 +509,8 @@ func TestQueryFrontendMemcachedCache(t *testing.T) { ctx, queryFrontend.Endpoint("http"), e2ethanos.QueryUpWithoutInstance, - timestamp.FromTime(now.Add(-time.Hour)), - timestamp.FromTime(now.Add(time.Hour)), + timestamp.FromTime(predefTimestamp.Add(-time.Hour)), + timestamp.FromTime(predefTimestamp.Add(time.Hour)), 14, promclient.QueryOptions{ Deduplicate: true, @@ -501,8 +541,8 @@ func TestQueryFrontendMemcachedCache(t *testing.T) { ctx, queryFrontend.Endpoint("http"), e2ethanos.QueryUpWithoutInstance, - timestamp.FromTime(now.Add(-time.Hour)), - timestamp.FromTime(now.Add(time.Hour)), + timestamp.FromTime(predefTimestamp.Add(-time.Hour)), + timestamp.FromTime(predefTimestamp.Add(time.Hour)), 14, promclient.QueryOptions{ Deduplicate: true, diff --git a/test/e2e/query_test.go b/test/e2e/query_test.go index 6584c7b842..5b9a120b90 100644 --- a/test/e2e/query_test.go +++ b/test/e2e/query_test.go @@ -4,6 +4,7 @@ package e2e_test import ( + "bytes" "context" "fmt" "io" @@ -1720,6 +1721,39 @@ func rangeQuery(t *testing.T, ctx context.Context, addr string, q func() string, return retExplanation } +// Performs a remote write at the receiver external endpoint. +func remoteWrite(ctx context.Context, timeseries []prompb.TimeSeries, addr string) error { + // Create write request + data, err := proto.Marshal(&prompb.WriteRequest{Timeseries: timeseries}) + if err != nil { + return err + } + + // Create HTTP request + compressed := snappy.Encode(nil, data) + req, err := http.NewRequest("POST", fmt.Sprintf("http://%s/api/v1/receive", addr), bytes.NewReader(compressed)) + if err != nil { + return err + } + + req.Header.Add("Content-Encoding", "snappy") + req.Header.Set("Content-Type", "application/x-protobuf") + req.Header.Set("X-Prometheus-Remote-Write-Version", "0.1.0") + + // Execute HTTP request + res, err := promclient.NewDefaultClient().HTTPClient.Do(req.WithContext(ctx)) + if err != nil { + return err + } + defer runutil.ExhaustCloseWithErrCapture(&err, res.Body, "%s: close body", req.URL.String()) + + if res.StatusCode/100 != 2 { + return errors.Errorf("request failed with code %s", res.Status) + } + + return nil +} + func queryExemplars(t *testing.T, ctx context.Context, addr, q string, start, end int64, check func(data []*exemplarspb.ExemplarData) error) { t.Helper() From ed8a317d8a87d8112bb0467120db6ee8d256d1ac Mon Sep 17 00:00:00 2001 From: Vasiliy Rumyantsev <4119114+xBazilio@users.noreply.github.com> Date: Wed, 27 Dec 2023 09:59:58 +0300 Subject: [PATCH 09/26] query-frontend: Added support of auto_discovery for memcached (#7004) * query-frontend: Added support of auto_discovery for memcached Signed-off-by: Vasiliy Rumyantsev <4119114+xBazilio@users.noreply.github.com> * adjustments to build on main branch Signed-off-by: Vasiliy Rumyantsev <4119114+xBazilio@users.noreply.github.com> * CHANGELOG.md Signed-off-by: Vasiliy Rumyantsev <4119114+xBazilio@users.noreply.github.com> * typo fixed Signed-off-by: Vasiliy Rumyantsev <4119114+xBazilio@users.noreply.github.com> * minor fixex after review Signed-off-by: Vasiliy Rumyantsev <4119114+xBazilio@users.noreply.github.com> --------- Signed-off-by: Vasiliy Rumyantsev <4119114+xBazilio@users.noreply.github.com> --- CHANGELOG.md | 1 + .../cortex/chunk/cache/memcached_client.go | 24 +++++++++++++++---- pkg/queryfrontend/config.go | 1 + 3 files changed, 21 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 64b849f9b1..5cc472138c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -39,6 +39,7 @@ We use *breaking :warning:* to mark changes that are not backward compatible (re - [#6605](https://github.com/thanos-io/thanos/pull/6605) Query Frontend: Support vertical sharding binary expression with metric name when no matching labels specified. - [#6308](https://github.com/thanos-io/thanos/pull/6308) Ruler: Support configuration flag that allows customizing template for alert message. - [#6760](https://github.com/thanos-io/thanos/pull/6760) Query Frontend: Added TLS support in `--query-frontend.downstream-tripper-config` and `--query-frontend.downstream-tripper-config-file` +- [#7004](https://github.com/thanos-io/thanos/pull/7004) Query Frontend: Support documented auto discovery for memcached - [#6749](https://github.com/thanos-io/thanos/pull/6749) Store Gateway: Added `thanos_store_index_cache_fetch_duration_seconds` histogram for tracking latency of fetching data from index cache. - [#6690](https://github.com/thanos-io/thanos/pull/6690) Store: *breaking :warning:* Add tenant label to relevant exported metrics. Note that this change may cause some pre-existing dashboard queries to be incorrect due to the added label. - [#6530](https://github.com/thanos-io/thanos/pull/6530) / [#6690](https://github.com/thanos-io/thanos/pull/6690) Query: Add command line arguments for configuring tenants and forward tenant information to Store Gateway. diff --git a/internal/cortex/chunk/cache/memcached_client.go b/internal/cortex/chunk/cache/memcached_client.go index a2f7b6b88b..9455f76400 100644 --- a/internal/cortex/chunk/cache/memcached_client.go +++ b/internal/cortex/chunk/cache/memcached_client.go @@ -19,7 +19,10 @@ import ( "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promauto" "github.com/sony/gobreaker" + "github.com/thanos-io/thanos/pkg/clientconfig" "github.com/thanos-io/thanos/pkg/discovery/dns" + memcacheDiscovery "github.com/thanos-io/thanos/pkg/discovery/memcache" + "github.com/thanos-io/thanos/pkg/extprom" ) // MemcachedClient interface exists for mocking memcacheClient. @@ -45,7 +48,7 @@ type memcachedClient struct { service string addresses []string - provider *dns.Provider + provider clientconfig.AddressProvider cbs map[ /*address*/ string]*gobreaker.CircuitBreaker cbFailures uint @@ -68,6 +71,7 @@ type MemcachedClientConfig struct { Host string `yaml:"host"` Service string `yaml:"service"` Addresses string `yaml:"addresses"` // EXPERIMENTAL. + AutoDiscovery bool `yaml:"auto_discovery"` Timeout time.Duration `yaml:"timeout"` MaxIdleConns int `yaml:"max_idle_conns"` MaxItemSize int `yaml:"max_item_size"` @@ -107,9 +111,19 @@ func NewMemcachedClient(cfg MemcachedClientConfig, name string, r prometheus.Reg client.Timeout = cfg.Timeout client.MaxIdleConns = cfg.MaxIdleConns - dnsProviderRegisterer := prometheus.WrapRegistererWithPrefix("cortex_", prometheus.WrapRegistererWith(prometheus.Labels{ - "name": name, - }, r)) + var addressProvider clientconfig.AddressProvider + if cfg.AutoDiscovery { + addressProvider = memcacheDiscovery.NewProvider( + logger, + extprom.WrapRegistererWithPrefix("cortex_", r), + cfg.Timeout, + ) + } else { + dnsProviderRegisterer := prometheus.WrapRegistererWithPrefix("cortex_", prometheus.WrapRegistererWith(prometheus.Labels{ + "name": name, + }, r)) + addressProvider = dns.NewProvider(logger, dnsProviderRegisterer, dns.GolangResolverType) + } newClient := &memcachedClient{ name: name, @@ -118,7 +132,7 @@ func NewMemcachedClient(cfg MemcachedClientConfig, name string, r prometheus.Reg hostname: cfg.Host, service: cfg.Service, logger: logger, - provider: dns.NewProvider(logger, dnsProviderRegisterer, dns.GolangResolverType), + provider: addressProvider, cbs: make(map[string]*gobreaker.CircuitBreaker), cbFailures: cfg.CBFailures, cbInterval: cfg.CBInterval, diff --git a/pkg/queryfrontend/config.go b/pkg/queryfrontend/config.go index ba45f80d9e..4b915764c5 100644 --- a/pkg/queryfrontend/config.go +++ b/pkg/queryfrontend/config.go @@ -142,6 +142,7 @@ func NewCacheConfig(logger log.Logger, confContentYaml []byte) (*cortexcache.Con Timeout: config.Memcached.Timeout, MaxIdleConns: config.Memcached.MaxIdleConnections, Addresses: strings.Join(config.Memcached.Addresses, ","), + AutoDiscovery: config.Memcached.AutoDiscovery, UpdateInterval: config.Memcached.DNSProviderUpdateInterval, MaxItemSize: int(config.Memcached.MaxItemSize), }, From e77caa8e583f6e6f4cae1fb7ae6876ed5265db1f Mon Sep 17 00:00:00 2001 From: Vanshika Date: Wed, 13 Dec 2023 16:55:05 +0530 Subject: [PATCH 10/26] ui: enable partial response strategy by default Rebuild Signed-off-by: Vanshika --- .../react-app/src/pages/graph/Panel.test.tsx | 4 ++++ pkg/ui/react-app/src/pages/graph/Panel.tsx | 23 +++++++++++++++++-- .../react-app/src/pages/graph/PanelList.tsx | 9 ++++++++ 3 files changed, 34 insertions(+), 2 deletions(-) diff --git a/pkg/ui/react-app/src/pages/graph/Panel.test.tsx b/pkg/ui/react-app/src/pages/graph/Panel.test.tsx index 993c60b1a4..0837513997 100644 --- a/pkg/ui/react-app/src/pages/graph/Panel.test.tsx +++ b/pkg/ui/react-app/src/pages/graph/Panel.test.tsx @@ -26,6 +26,9 @@ const defaultProps: PanelProps = { analyze: false, disableAnalyzeCheckbox: false, }, + onUsePartialResponseChange: (): void => { + // Do nothing. + }, onOptionsChanged: (): void => { // Do nothing. }, @@ -47,6 +50,7 @@ const defaultProps: PanelProps = { enableHighlighting: true, enableLinter: true, defaultEngine: 'prometheus', + usePartialResponse: true, }; describe('Panel', () => { diff --git a/pkg/ui/react-app/src/pages/graph/Panel.tsx b/pkg/ui/react-app/src/pages/graph/Panel.tsx index 2ba02d1131..915d5d1813 100644 --- a/pkg/ui/react-app/src/pages/graph/Panel.tsx +++ b/pkg/ui/react-app/src/pages/graph/Panel.tsx @@ -43,9 +43,11 @@ export interface PanelProps { stores: Store[]; enableAutocomplete: boolean; enableHighlighting: boolean; + usePartialResponse: boolean; enableLinter: boolean; defaultStep: string; defaultEngine: string; + onUsePartialResponseChange: (value: boolean) => void; } interface PanelState { @@ -93,7 +95,7 @@ export const PanelDefaultOptions: PanelOptions = { maxSourceResolution: '0s', useDeduplication: true, forceTracing: false, - usePartialResponse: false, + usePartialResponse: true, storeMatches: [], engine: '', analyze: false, @@ -166,6 +168,13 @@ class Panel extends Component { componentDidMount(): void { this.executeQuery(); + const storedValue = localStorage.getItem('usePartialResponse'); + if (storedValue !== null) { + // Set the default value in state and local storage + this.setOptions({ usePartialResponse: true }); + this.props.onUsePartialResponseChange(true); + localStorage.setItem('usePartialResponse', JSON.stringify(true)); + } } executeQuery = (): void => { @@ -346,7 +355,17 @@ class Panel extends Component { }; handleChangePartialResponse = (event: React.ChangeEvent): void => { - this.setOptions({ usePartialResponse: event.target.checked }); + let newValue = event.target.checked; + + const storedValue = localStorage.getItem('usePartialResponse'); + + if (storedValue === 'true') { + newValue = true; + } + this.setOptions({ usePartialResponse: newValue }); + this.props.onUsePartialResponseChange(newValue); + + localStorage.setItem('usePartialResponse', JSON.stringify(event.target.checked)); }; handleStoreMatchChange = (selectedStores: any): void => { diff --git a/pkg/ui/react-app/src/pages/graph/PanelList.tsx b/pkg/ui/react-app/src/pages/graph/PanelList.tsx index 1cc66ddbdd..19fb9e1319 100644 --- a/pkg/ui/react-app/src/pages/graph/PanelList.tsx +++ b/pkg/ui/react-app/src/pages/graph/PanelList.tsx @@ -31,6 +31,7 @@ interface PanelListProps extends PathPrefixProps, RouteComponentProps { enableLinter: boolean; defaultStep: string; defaultEngine: string; + usePartialResponse: boolean; } export const PanelListContent: FC = ({ @@ -44,6 +45,7 @@ export const PanelListContent: FC = ({ enableLinter, defaultStep, defaultEngine, + usePartialResponse, ...rest }) => { const [panels, setPanels] = useState(rest.panels); @@ -95,6 +97,9 @@ export const PanelListContent: FC = ({ }, ]); }; + const handleUsePartialResponseChange = (value: boolean): void => { + localStorage.setItem('usePartialResponse', JSON.stringify(value)); + }; return ( <> @@ -128,6 +133,8 @@ export const PanelListContent: FC = ({ defaultEngine={defaultEngine} enableLinter={enableLinter} defaultStep={defaultStep} + usePartialResponse={usePartialResponse} + onUsePartialResponseChange={handleUsePartialResponseChange} /> ))}