Merge branch 'thanos-io:main' into frontend_warnings

.mdox.validate.yaml

@@ -40,3 +40,5 @@ validators:
     type: 'ignore'
   - regex: 'krisztianfekete\.org'
     type: 'ignore'
+  - regex: 'twitter\.com'
+    type: 'ignore'

.mdox.yaml

@@ -72,6 +72,10 @@ transformations:
     backMatter: *docBackMatter
 
   # Non-versioned element: Blog.
+
+  - glob: "support/*"
+    path: /../support/*
+
   - glob: "blog/*"
     path: /../blog/*
 

CHANGELOG.md

@@ -12,20 +12,28 @@ We use *breaking :warning:* to mark changes that are not backward compatible (re
 
 ### Fixed
 
+- [#7011](https://github.com/thanos-io/thanos/pull/7011) Query Frontend: queries with negative offset should check whether it is cacheable or not.
 - [#6874](https://github.com/thanos-io/thanos/pull/6874) Sidecar: fix labels returned by 'api/v1/series' in presence of conflicting external and inner labels.
 - [#7009](https://github.com/thanos-io/thanos/pull/7009) Rule: Fix spacing error in URL.
 
 ### Added
+
+- [#6756](https://github.com/thanos-io/thanos/pull/6756) Query: Add `query.enable-tenancy` & `query.tenant-label-name` options to allow enforcement of tenancy on the query path, by injecting labels into queries (uses prom-label-proxy internally).
 - [#6944](https://github.com/thanos-io/thanos/pull/6944) Receive: Added a new flag for maximum retention bytes.
 - [#6891](https://github.com/thanos-io/thanos/pull/6891) Objstore: Bump `objstore` which adds support for Azure Workload Identity.
 - [#6453](https://github.com/thanos-io/thanos/pull/6453) Sidecar: Added `--reloader.method` to support configuration reloads via SIHUP signal.
 - [#6925](https://github.com/thanos-io/thanos/pull/6925) Store Gateway: Support float native histogram.
 - [#6954](https://github.com/thanos-io/thanos/pull/6954) Index Cache: Support tracing for fetch APIs.
 - [#6943](https://github.com/thanos-io/thanos/pull/6943) Ruler: Added `keep_firing_for` field in alerting rule.
 - [#6972](https://github.com/thanos-io/thanos/pull/6972) Store Gateway: Apply series limit when streaming series for series actually matched if lazy postings is enabled.
+- [#6984](https://github.com/thanos-io/thanos/pull/6984) Store Gateway: Added `--store.index-header-lazy-download-strategy` to specify how to lazily download index headers when lazy mmap is enabled.
+- [#6887](https://github.com/thanos-io/thanos/pull/6887) Query Frontend: *breaking :warning:* Add tenant label to relevant exported metrics. Note that this change may cause some pre-existing custom dashboard queries to be incorrect due to the added label.
+- [#7028](https://github.com/thanos-io/thanos/pull/7028) Query|Query Frontend: Add new `--query-frontend.enable-x-functions` flag to enable experimental extended functions.
 
 ### Changed
 
+- [#6539](https://github.com/thanos-io/thanos/pull/6539) Store: *breaking :warning:* Changed `--sync-block-duration` default 3m to 15m.
+
 ### Removed
 
 ## [v0.33.0](https://github.com/thanos-io/thanos/tree/release-0.33) - 18.12.2023
@@ -39,6 +47,7 @@ We use *breaking :warning:* to mark changes that are not backward compatible (re
 - [#6605](https://github.com/thanos-io/thanos/pull/6605) Query Frontend: Support vertical sharding binary expression with metric name when no matching labels specified.
 - [#6308](https://github.com/thanos-io/thanos/pull/6308) Ruler: Support configuration flag that allows customizing template for alert message.
 - [#6760](https://github.com/thanos-io/thanos/pull/6760) Query Frontend: Added TLS support in `--query-frontend.downstream-tripper-config` and `--query-frontend.downstream-tripper-config-file`
+- [#7004](https://github.com/thanos-io/thanos/pull/7004) Query Frontend: Support documented auto discovery for memcached
 - [#6749](https://github.com/thanos-io/thanos/pull/6749) Store Gateway: Added `thanos_store_index_cache_fetch_duration_seconds` histogram for tracking latency of fetching data from index cache.
 - [#6690](https://github.com/thanos-io/thanos/pull/6690) Store: *breaking :warning:* Add tenant label to relevant exported metrics. Note that this change may cause some pre-existing dashboard queries to be incorrect due to the added label.
 - [#6530](https://github.com/thanos-io/thanos/pull/6530) / [#6690](https://github.com/thanos-io/thanos/pull/6690) Query: Add command line arguments for configuring tenants and forward tenant information to Store Gateway.

MAINTAINERS.md

@@ -2,7 +2,7 @@
 
 | Name                  | Email                     | Slack                    | GitHub                                             | Company             |
 |-----------------------|---------------------------|--------------------------|----------------------------------------------------|---------------------|
-| Bartłomiej Płotka     | [email protected]        | `@bwplotka`              | [@bwplotka](https://github.com/bwplotka)           | Red Hat             |
+| Bartłomiej Płotka     | [email protected]        | `@bwplotka`              | [@bwplotka](https://github.com/bwplotka)           | Google              |
 | Frederic Branczyk     | [email protected]       | `@brancz`                | [@brancz](https://github.com/brancz)               | Polar Signals       |
 | Giedrius Statkevičius | [email protected]    | `@Giedrius Statkevičius` | [@GiedriusS](https://github.com/GiedriusS)         | Vinted              |
 | Kemal Akkoyun         | [email protected]        | `@kakkoyun`              | [@kakkoyun](https://github.com/kakkoyun)           | Polar Signals       |

cmd/thanos/query.go

@@ -102,7 +102,7 @@ func registerQuery(app *extkingpin.App) {
 
 	defaultEngine := cmd.Flag("query.promql-engine", "Default PromQL engine to use.").Default(string(apiv1.PromqlEnginePrometheus)).
 		Enum(string(apiv1.PromqlEnginePrometheus), string(apiv1.PromqlEngineThanos))
-
+	extendedFunctionsEnabled := cmd.Flag("query.enable-x-functions", "Whether to enable extended rate functions (xrate, xincrease and xdelta). Only has effect when used with Thanos engine.").Default("false").Bool()
 	promqlQueryMode := cmd.Flag("query.mode", "PromQL query mode. One of: local, distributed.").
 		Hidden().
 		Default(string(queryModeLocal)).
@@ -220,6 +220,8 @@ func registerQuery(app *extkingpin.App) {
 	tenantHeader := cmd.Flag("query.tenant-header", "HTTP header to determine tenant.").Default(tenancy.DefaultTenantHeader).String()
 	defaultTenant := cmd.Flag("query.default-tenant-id", "Default tenant ID to use if tenant header is not present").Default(tenancy.DefaultTenant).String()
 	tenantCertField := cmd.Flag("query.tenant-certificate-field", "Use TLS client's certificate field to determine tenant for write requests. Must be one of "+tenancy.CertificateFieldOrganization+", "+tenancy.CertificateFieldOrganizationalUnit+" or "+tenancy.CertificateFieldCommonName+". This setting will cause the query.tenant-header flag value to be ignored.").Default("").Enum("", tenancy.CertificateFieldOrganization, tenancy.CertificateFieldOrganizationalUnit, tenancy.CertificateFieldCommonName)
+	enforceTenancy := cmd.Flag("query.enforce-tenancy", "Enforce tenancy on Query APIs. Responses are returned only if the label value of the configured tenant-label-name and the value of the tenant header matches.").Default("false").Bool()
+	tenantLabel := cmd.Flag("query.tenant-label-name", "Label name to use when enforcing tenancy (if --query.enforce-tenancy is enabled).").Default(tenancy.DefaultTenantLabel).String()
 
 	var storeRateLimits store.SeriesSelectLimits
 	storeRateLimits.RegisterFlags(cmd)
@@ -342,10 +344,13 @@ func registerQuery(app *extkingpin.App) {
 			*queryTelemetrySeriesQuantiles,
 			*defaultEngine,
 			storeRateLimits,
+			*extendedFunctionsEnabled,
 			queryMode(*promqlQueryMode),
 			*tenantHeader,
 			*defaultTenant,
 			*tenantCertField,
+			*enforceTenancy,
+			*tenantLabel,
 		)
 	})
 }
@@ -421,10 +426,13 @@ func runQuery(
 	queryTelemetrySeriesQuantiles []float64,
 	defaultEngine string,
 	storeRateLimits store.SeriesSelectLimits,
+	extendedFunctionsEnabled bool,
 	queryMode queryMode,
 	tenantHeader string,
 	defaultTenant string,
 	tenantCertField string,
+	enforceTenancy bool,
+	tenantLabel string,
 ) error {
 	if alertQueryURL == "" {
 		lastColon := strings.LastIndex(httpBindAddr, ":")
@@ -502,56 +510,29 @@ func runQuery(
 	}
 
 	var (
-		endpoints = query.NewEndpointSet(
-			time.Now,
+		endpoints = prepareEndpointSet(
+			g,
 			logger,
 			reg,
-			func() (specs []*query.GRPCEndpointSpec) {
-				// Add strict & static nodes.
-				for _, addr := range strictStores {
-					specs = append(specs, query.NewGRPCEndpointSpec(addr, true))
-				}
-
-				for _, addr := range strictEndpoints {
-					specs = append(specs, query.NewGRPCEndpointSpec(addr, true))
-				}
-
-				for _, dnsProvider := range []*dns.Provider{
-					dnsStoreProvider,
-					dnsRuleProvider,
-					dnsExemplarProvider,
-					dnsMetadataProvider,
-					dnsTargetProvider,
-					dnsEndpointProvider,
-				} {
-					var tmpSpecs []*query.GRPCEndpointSpec
-
-					for _, addr := range dnsProvider.Addresses() {
-						tmpSpecs = append(tmpSpecs, query.NewGRPCEndpointSpec(addr, false))
-					}
-					tmpSpecs = removeDuplicateEndpointSpecs(logger, duplicatedStores, tmpSpecs)
-					specs = append(specs, tmpSpecs...)
-				}
-
-				for _, eg := range endpointGroupAddrs {
-					addr := fmt.Sprintf("dns:///%s", eg)
-					spec := query.NewGRPCEndpointSpec(addr, false, extgrpc.EndpointGroupGRPCOpts()...)
-					specs = append(specs, spec)
-				}
-
-				for _, eg := range strictEndpointGroups {
-					addr := fmt.Sprintf("dns:///%s", eg)
-					spec := query.NewGRPCEndpointSpec(addr, true, extgrpc.EndpointGroupGRPCOpts()...)
-					specs = append(specs, spec)
-				}
-
-				return specs
+			[]*dns.Provider{
+				dnsStoreProvider,
+				dnsRuleProvider,
+				dnsExemplarProvider,
+				dnsMetadataProvider,
+				dnsTargetProvider,
+				dnsEndpointProvider,
 			},
+			duplicatedStores,
+			strictStores,
+			strictEndpoints,
+			endpointGroupAddrs,
+			strictEndpointGroups,
 			dialOpts,
 			unhealthyStoreTimeout,
 			endpointInfoTimeout,
 			queryConnMetricLabels...,
 		)
+
 		proxy            = store.NewProxyStore(logger, reg, endpoints.GetStoreClients, component.Query, selectorLset, storeResponseTimeout, store.RetrievalStrategy(grpcProxyStrategy), options...)
 		rulesProxy       = rules.NewProxy(logger, endpoints.GetRulesClients)
 		targetsProxy     = targets.NewProxy(logger, endpoints.GetTargetsClients)
@@ -566,20 +547,6 @@ func runQuery(
 		)
 	)
 
-	// Periodically update the store set with the addresses we see in our cluster.
-	{
-		ctx, cancel := context.WithCancel(context.Background())
-		g.Add(func() error {
-			return runutil.Repeat(5*time.Second, ctx.Done(), func() error {
-				endpoints.Update(ctx)
-				return nil
-			})
-		}, func(error) {
-			cancel()
-			endpoints.Close()
-		})
-	}
-
 	// Run File Service Discovery and update the store set when the files are modified.
 	if fileSD != nil {
 		var fileSDUpdates chan []*targetgroup.Group
@@ -693,6 +660,7 @@ func runQuery(
 	engineFactory := apiv1.NewQueryEngineFactory(
 		engineOpts,
 		remoteEngineEndpoints,
+		extendedFunctionsEnabled,
 	)
 
 	lookbackDeltaCreator := LookbackDeltaFactory(engineOpts, dynamicLookbackDelta)
@@ -762,6 +730,8 @@ func runQuery(
 			tenantHeader,
 			defaultTenant,
 			tenantCertField,
+			enforceTenancy,
+			tenantLabel,
 		)
 
 		api.Register(router.WithPrefix("/api/v1"), tracer, logger, ins, logMiddleware)
@@ -861,6 +831,82 @@ func removeDuplicateEndpointSpecs(logger log.Logger, duplicatedStores prometheus
 	return deduplicated
 }
 
+func prepareEndpointSet(
+	g *run.Group,
+	logger log.Logger,
+	reg *prometheus.Registry,
+	dnsProviders []*dns.Provider,
+	duplicatedStores prometheus.Counter,
+	strictStores []string,
+	strictEndpoints []string,
+	endpointGroupAddrs []string,
+	strictEndpointGroups []string,
+	dialOpts []grpc.DialOption,
+	unhealthyStoreTimeout time.Duration,
+	endpointInfoTimeout time.Duration,
+	queryConnMetricLabels ...string,
+) *query.EndpointSet {
+	endpointSet := query.NewEndpointSet(
+		time.Now,
+		logger,
+		reg,
+		func() (specs []*query.GRPCEndpointSpec) {
+			// Add strict & static nodes.
+			for _, addr := range strictStores {
+				specs = append(specs, query.NewGRPCEndpointSpec(addr, true))
+			}
+
+			for _, addr := range strictEndpoints {
+				specs = append(specs, query.NewGRPCEndpointSpec(addr, true))
+			}
+
+			for _, dnsProvider := range dnsProviders {
+				var tmpSpecs []*query.GRPCEndpointSpec
+
+				for _, addr := range dnsProvider.Addresses() {
+					tmpSpecs = append(tmpSpecs, query.NewGRPCEndpointSpec(addr, false))
+				}
+				tmpSpecs = removeDuplicateEndpointSpecs(logger, duplicatedStores, tmpSpecs)
+				specs = append(specs, tmpSpecs...)
+			}
+
+			for _, eg := range endpointGroupAddrs {
+				addr := fmt.Sprintf("dns:///%s", eg)
+				spec := query.NewGRPCEndpointSpec(addr, false, extgrpc.EndpointGroupGRPCOpts()...)
+				specs = append(specs, spec)
+			}
+
+			for _, eg := range strictEndpointGroups {
+				addr := fmt.Sprintf("dns:///%s", eg)
+				spec := query.NewGRPCEndpointSpec(addr, true, extgrpc.EndpointGroupGRPCOpts()...)
+				specs = append(specs, spec)
+			}
+
+			return specs
+		},
+		dialOpts,
+		unhealthyStoreTimeout,
+		endpointInfoTimeout,
+		queryConnMetricLabels...,
+	)
+
+	// Periodically update the store set with the addresses we see in our cluster.
+	{
+		ctx, cancel := context.WithCancel(context.Background())
+		g.Add(func() error {
+			return runutil.Repeat(5*time.Second, ctx.Done(), func() error {
+				endpointSet.Update(ctx)
+				return nil
+			})
+		}, func(error) {
+			cancel()
+			endpointSet.Close()
+		})
+	}
+
+	return endpointSet
+}
+
 // LookbackDeltaFactory creates from 1 to 3 lookback deltas depending on
 // dynamicLookbackDelta and eo.LookbackDelta and returns a function
 // that returns appropriate lookback delta for given maxSourceResolutionMillis.

cmd/thanos/query_frontend.go

@@ -16,6 +16,8 @@ import (
 	"github.com/opentracing/opentracing-go"
 	"github.com/pkg/errors"
 	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/prometheus/promql/parser"
+	"github.com/thanos-io/promql-engine/execution/parse"
 	"github.com/weaveworks/common/user"
 	"gopkg.in/yaml.v2"
 
@@ -92,6 +94,9 @@ func registerQueryFrontend(app *extkingpin.App) {
 	cmd.Flag("query-range.max-retries-per-request", "Maximum number of retries for a single query range request; beyond this, the downstream error is returned.").
 		Default("5").IntVar(&cfg.QueryRangeConfig.MaxRetries)
 
+	cmd.Flag("query-frontend.enable-x-functions", "Enable experimental x- functions in query-frontend. --no-query-frontend.enable-x-functions for disabling.").
+		Default("false").BoolVar(&cfg.EnableXFunctions)
+
 	cmd.Flag("query-range.max-query-length", "Limit the query time range (end - start time) in the query-frontend, 0 disables it.").
 		Default("0").DurationVar((*time.Duration)(&cfg.QueryRangeConfig.Limits.MaxQueryLength))
 
@@ -285,6 +290,12 @@ func runQueryFrontend(
 		return errors.Wrap(err, "error validating the config")
 	}
 
+	if cfg.EnableXFunctions {
+		for fname, v := range parse.XFunctions {
+			parser.Functions[fname] = v
+		}
+	}
+
 	tripperWare, err := queryfrontend.NewTripperware(cfg.Config, reg, logger)
 	if err != nil {
 		return errors.Wrap(err, "setup tripperwares")
@@ -322,7 +333,7 @@ func runQueryFrontend(
 
 	// Configure Request Logging for HTTP calls.
 	logMiddleware := logging.NewHTTPServerMiddleware(logger, httpLogOpts...)
-	ins := extpromhttp.NewInstrumentationMiddleware(reg, nil)
+	ins := extpromhttp.NewTenantInstrumentationMiddleware(cfg.TenantHeader, cfg.DefaultTenant, reg, nil)
 
 	// Start metrics HTTP server.
 	{