diff --git a/go.mod b/go.mod index 5cfabb7ef9..4937e158ea 100644 --- a/go.mod +++ b/go.mod @@ -27,7 +27,7 @@ require ( github.com/gorilla/mux v1.8.0 github.com/grafana/regexp v0.0.0-20221122212121-6b5c0a4cb7fd github.com/grpc-ecosystem/go-grpc-middleware v1.4.0 - github.com/hashicorp/consul/api v1.22.0 + github.com/hashicorp/consul/api v1.25.1 github.com/hashicorp/go-cleanhttp v0.5.2 github.com/hashicorp/go-sockaddr v1.0.2 github.com/hashicorp/memberlist v0.5.0 @@ -46,14 +46,14 @@ require ( github.com/prometheus/client_model v0.4.0 github.com/prometheus/common v0.44.0 // Prometheus maps version 2.x.y to tags v0.x.y. - github.com/prometheus/prometheus v0.46.1-0.20230818184859-4d8e380269da + github.com/prometheus/prometheus v0.47.2-0.20231009162353-f6d9c84fde6b github.com/segmentio/fasthash v1.0.3 github.com/sony/gobreaker v0.5.0 github.com/spf13/afero v1.9.5 github.com/stretchr/testify v1.8.4 github.com/thanos-io/objstore v0.0.0-20230921130928-63a603e651ed - github.com/thanos-io/promql-engine v0.0.0-20230821193351-e1ae4275b96e - github.com/thanos-io/thanos v0.32.5-0.20231006043659-79bbf34b4275 + github.com/thanos-io/promql-engine v0.0.0-20231003153358-8605b6afba51 + github.com/thanos-io/thanos v0.32.5-0.20231010190130-dfe0bbff507b github.com/uber/jaeger-client-go v2.30.0+incompatible github.com/weaveworks/common v0.0.0-20221201103051-7c2720a9024d go.etcd.io/etcd/api/v3 v3.5.9 @@ -198,6 +198,7 @@ require ( github.com/vimeo/galaxycache v0.0.0-20210323154928-b7e5d71c067a // indirect github.com/weaveworks/promrus v1.2.0 // indirect github.com/yuin/gopher-lua v1.1.0 // indirect + github.com/zhangyunhao116/umap v0.0.0-20221211160557-cb7705fafa39 // indirect go.mongodb.org/mongo-driver v1.12.0 // indirect go.opencensus.io v0.24.0 // indirect go.opentelemetry.io/collector/pdata v1.0.0-rcv0014 // indirect @@ -216,7 +217,7 @@ require ( go4.org/intern v0.0.0-20230525184215-6c62f75575cb // indirect go4.org/unsafe/assume-no-moving-gc v0.0.0-20230525183740-e7c30c78aeb2 // indirect golang.org/x/crypto v0.12.0 // indirect - golang.org/x/exp v0.0.0-20230713183714-613f0c0eb8a1 // indirect + golang.org/x/exp v0.0.0-20230801115018-d63ba01acd4b // indirect golang.org/x/mod v0.12.0 // indirect golang.org/x/oauth2 v0.11.0 // indirect golang.org/x/sys v0.12.0 // indirect diff --git a/go.sum b/go.sum index dc95c58bfd..7d408c1db8 100644 --- a/go.sum +++ b/go.sum @@ -762,6 +762,7 @@ github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Z github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.1 h1:gK4Kx5IaGY9CD5sPJ36FHiBJ6ZXl0kilRiiCj+jdYp4= github.com/google/btree v1.0.1/go.mod h1:xXMiIv4Fb/0kKde4SpL7qlzvu5cMJDRkFDxJfI9uaxA= +github.com/google/gnostic-models v0.6.8 h1:yo/ABAfM5IMRsS1VnXjTBvUb61tFIHozhlYvRgGre9I= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= @@ -829,7 +830,6 @@ github.com/googleapis/gax-go/v2 v2.6.0/go.mod h1:1mjbznJAPHFpesgE5ucqfYEscaz5kMd github.com/googleapis/gax-go/v2 v2.7.0/go.mod h1:TEop28CZZQ2y+c0VxMUmu1lV+fQx57QpBWsYpwqHJx8= github.com/googleapis/gax-go/v2 v2.12.0 h1:A+gCJKdRfqXkr+BIRGtZLibNXf0m1f9E4HG56etFpas= github.com/googleapis/gax-go/v2 v2.12.0/go.mod h1:y+aIqrI5eb1YGMVJfuV3185Ts/D7qKpsEkdD5+I6QGU= -github.com/googleapis/gnostic v0.6.9 h1:hNeVzFMdppk7EuvFnJjiowGFBmSau2llc2rseO0+eNw= github.com/googleapis/go-type-adapters v1.0.0/go.mod h1:zHW75FOG2aur7gAO2B+MLby+cLsWGBF62rFAi7WjWO4= github.com/googleapis/google-cloud-go-testing v0.0.0-20200911160855-bcd43fbb19e8/go.mod h1:dvDLG8qkwmyD9a/MJJN3XJcT3xFxOKAvTZGvuZmac9g= github.com/gophercloud/gophercloud v1.5.0 h1:cDN6XFCLKiiqvYpjQLq9AiM7RDRbIC9450WpPH+yvXo= @@ -854,10 +854,10 @@ github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.0 h1:YBftPWNWd4WwGqtY2yeZL2ef8rH github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.0/go.mod h1:YN5jB8ie0yfIUg6VvR9Kz84aCaG7AsGZnLjhHbUqwPg= github.com/grpc-ecosystem/grpc-opentracing v0.0.0-20180507213350-8e809c8a8645/go.mod h1:6iZfnjpejD4L/4DwD7NryNaJyCQdzwWwH2MWhCA90Kw= github.com/hashicorp/consul/api v1.12.0/go.mod h1:6pVBMo0ebnYdt2S3H87XhekM/HHrUoTD2XXb/VrZVy0= -github.com/hashicorp/consul/api v1.22.0 h1:ydEvDooB/A0c/xpsBd8GSt7P2/zYPBui4KrNip0xGjE= -github.com/hashicorp/consul/api v1.22.0/go.mod h1:zHpYgZ7TeYqS6zaszjwSt128OwESRpnhU9aGa6ue3Eg= +github.com/hashicorp/consul/api v1.25.1 h1:CqrdhYzc8XZuPnhIYZWH45toM0LB9ZeYr/gvpLVI3PE= +github.com/hashicorp/consul/api v1.25.1/go.mod h1:iiLVwR/htV7mas/sy0O+XSuEnrdBUUydemjxcUrAt4g= github.com/hashicorp/consul/sdk v0.8.0/go.mod h1:GBvyrGALthsZObzUGsfgHZQDXjg4lOjagTIwIR1vPms= -github.com/hashicorp/consul/sdk v0.14.0 h1:Hly+BMNMssVzoWddbBnBFi3W+Fzytvm0haSkihhj3GU= +github.com/hashicorp/consul/sdk v0.14.1 h1:ZiwE2bKb+zro68sWzZ1SgHF3kRMBZ94TwOCFRF4ylPs= github.com/hashicorp/cronexpr v1.1.2 h1:wG/ZYIKT+RT3QkOdgYc+xsKWVRgnxJ1OJtjjy84fJ9A= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I= @@ -1134,8 +1134,8 @@ github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1 github.com/prometheus/procfs v0.8.0/go.mod h1:z7EfXMXOkbkqb9IINtpCn86r/to3BnA0uaxHdg830/4= github.com/prometheus/procfs v0.11.1 h1:xRC8Iq1yyca5ypa9n1EZnWZkt7dwcoRPQwX/5gwaUuI= github.com/prometheus/procfs v0.11.1/go.mod h1:eesXgaPo1q7lBpVMoMy0ZOFTth9hBn4W/y0/p/ScXhY= -github.com/prometheus/prometheus v0.46.1-0.20230818184859-4d8e380269da h1:D5uk+FEdNjQs9ly/wkb/pXkoWc60GcV9RVsMUpg/BIE= -github.com/prometheus/prometheus v0.46.1-0.20230818184859-4d8e380269da/go.mod h1:uvQsz/zwlfb8TRuWjK7L7ofV5ycAYq8dorvNf2iOBN4= +github.com/prometheus/prometheus v0.47.2-0.20231009162353-f6d9c84fde6b h1:oiCf/rFBXXaDLyiK1MnMKYlSA7Xm2+SQePvXnl8bNUI= +github.com/prometheus/prometheus v0.47.2-0.20231009162353-f6d9c84fde6b/go.mod h1:UC0TwJiF90m2T3iYPQBKnGu8gv3s55dF/EgpTq8gyvo= github.com/redis/rueidis v1.0.14-go1.18 h1:dGir5z8w8X1ex7JWO/Zx2FMBrZgQ8Yjm+lw9fPLSNGw= github.com/redis/rueidis v1.0.14-go1.18/go.mod h1:HGekzV3HbmzFmRK6j0xic8Z9119+ECoGMjeN1TV1NYU= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= @@ -1210,10 +1210,10 @@ github.com/thanos-community/galaxycache v0.0.0-20211122094458-3a32041a1f1e h1:f1 github.com/thanos-community/galaxycache v0.0.0-20211122094458-3a32041a1f1e/go.mod h1:jXcofnrSln/cLI6/dhlBxPQZEEQHVPCcFaH75M+nSzM= github.com/thanos-io/objstore v0.0.0-20230921130928-63a603e651ed h1:iWQdY3S6DpWjelVvKKSKgS7LeLkhK4VaEnQfphB9ZXA= github.com/thanos-io/objstore v0.0.0-20230921130928-63a603e651ed/go.mod h1:oJ82xgcBDzGJrEgUsjlTj6n01+ZWUMMUR8BlZzX5xDE= -github.com/thanos-io/promql-engine v0.0.0-20230821193351-e1ae4275b96e h1:kwsFCU8eSkZehbrAN3nXPw5RdMHi/Bok/y8l2C4M+gk= -github.com/thanos-io/promql-engine v0.0.0-20230821193351-e1ae4275b96e/go.mod h1:+T/ZYNCGybT6eTsGGvVtGb63nT1cvUmH6MjqRrcQoKw= -github.com/thanos-io/thanos v0.32.5-0.20231006043659-79bbf34b4275 h1:y2YPqM1XiBw7EhLg45F6A1g8bgt4yYxkaRAeQaNLWYk= -github.com/thanos-io/thanos v0.32.5-0.20231006043659-79bbf34b4275/go.mod h1:HwiHn7u6GeES403BTACOYib/JKAJknf8dByU/uJiEr0= +github.com/thanos-io/promql-engine v0.0.0-20231003153358-8605b6afba51 h1:Av62ac0O9wRbLI6xvtm51BBZnxHyEgLXV/YmiJpdogc= +github.com/thanos-io/promql-engine v0.0.0-20231003153358-8605b6afba51/go.mod h1:vfXJv1JXNdLfHnjsHsLLJl5tyI7KblF76Wo5lZ9YC4Q= +github.com/thanos-io/thanos v0.32.5-0.20231010190130-dfe0bbff507b h1:7eH6FRIQ/d0wlklAHe8dFpMAxG81C6uE7LTEj5jafss= +github.com/thanos-io/thanos v0.32.5-0.20231010190130-dfe0bbff507b/go.mod h1:tqT2FQHiOF16empgE3vvZrA++fN9Cx0lwmxlMmBaVzA= github.com/themihai/gomemcache v0.0.0-20180902122335-24332e2d58ab h1:7ZR3hmisBWw77ZpO1/o86g+JV3VKlk3d48jopJxzTjU= github.com/themihai/gomemcache v0.0.0-20180902122335-24332e2d58ab/go.mod h1:eheTFp954zcWZXCU8d0AT76ftsQOTo4DTqkN/h3k1MY= github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= @@ -1245,6 +1245,8 @@ github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1 github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= github.com/yuin/gopher-lua v1.1.0 h1:BojcDhfyDWgU2f2TOzYK/g5p2gxMrku8oupLDqlnSqE= github.com/yuin/gopher-lua v1.1.0/go.mod h1:GBR0iDaNXjAgGg9zfCvksxSRnQx76gclCIb7kdAd1Pw= +github.com/zhangyunhao116/umap v0.0.0-20221211160557-cb7705fafa39 h1:D3ltj0b2c2FgUacKrB1pWGgwrUyCESY9W8XYYQ5sqY8= +github.com/zhangyunhao116/umap v0.0.0-20221211160557-cb7705fafa39/go.mod h1:r86X1CnsDRrOeLtJlqRWdELPWpkcf933GTlojQlifQw= go.etcd.io/etcd/api/v3 v3.5.4/go.mod h1:5GB2vv4A4AOn3yk7MftYGHkUfGtDHnEraIjym4dYz5A= go.etcd.io/etcd/api/v3 v3.5.9 h1:4wSsluwyTbGGmyjJktOf3wFQoTBIURXHnq9n/G/JQHs= go.etcd.io/etcd/api/v3 v3.5.9/go.mod h1:uyAal843mC8uUVSLWz6eHa/d971iDGnCRpmKd2Z+X8k= @@ -1360,8 +1362,8 @@ golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u0 golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= -golang.org/x/exp v0.0.0-20230713183714-613f0c0eb8a1 h1:MGwJjxBy0HJshjDNfLsYO8xppfqWlA5ZT9OhtUUhTNw= -golang.org/x/exp v0.0.0-20230713183714-613f0c0eb8a1/go.mod h1:FXUEEKJgO7OQYeo8N01OfiKP8RXMtf6e8aTskBGqWdc= +golang.org/x/exp v0.0.0-20230801115018-d63ba01acd4b h1:r+vk0EmXNmekl0S0BascoeeoHk/L7wmaW2QF90K+kYI= +golang.org/x/exp v0.0.0-20230801115018-d63ba01acd4b/go.mod h1:FXUEEKJgO7OQYeo8N01OfiKP8RXMtf6e8aTskBGqWdc= golang.org/x/image v0.0.0-20180708004352-c73c2afc3b81/go.mod h1:ux5Hcp/YLpHSI86hEcLt0YII63i6oz57MZXIpbrjZUs= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= @@ -1998,12 +2000,12 @@ honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWh honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= -k8s.io/api v0.27.3 h1:yR6oQXXnUEBWEWcvPWS0jQL575KoAboQPfJAuKNrw5Y= -k8s.io/apimachinery v0.27.3 h1:Ubye8oBufD04l9QnNtW05idcOe9Z3GQN8+7PqmuVcUM= -k8s.io/client-go v0.27.3 h1:7dnEGHZEJld3lYwxvLl7WoehK6lAq7GvgjxpA3nv1E8= +k8s.io/api v0.28.1 h1:i+0O8k2NPBCPYaMB+uCkseEbawEt/eFaiRqUx8aB108= +k8s.io/apimachinery v0.28.1 h1:EJD40og3GizBSV3mkIoXQBsws32okPOy+MkRyzh6nPY= +k8s.io/client-go v0.28.1 h1:pRhMzB8HyLfVwpngWKE8hDcXRqifh1ga2Z/PU9SXVK8= k8s.io/klog v1.0.0 h1:Pt+yjF5aB1xDSVbau4VsWe+dQNzA0qv1LlXdC2dF6Q8= k8s.io/klog/v2 v2.100.1 h1:7WCHKK6K8fNhTqfBhISHQ97KrnJNFZMcQvKp7gP/tmg= -k8s.io/kube-openapi v0.0.0-20230525220651-2546d827e515 h1:OmK1d0WrkD3IPfkskvroRykOulHVHf0s0ZIFRjyt+UI= +k8s.io/kube-openapi v0.0.0-20230717233707-2695361300d9 h1:LyMgNKD2P8Wn1iAwQU5OhxCKlKJy0sHc+PcDwFB24dQ= k8s.io/utils v0.0.0-20230711102312-30195339c3c7 h1:ZgnF1KZsYxWIifwSNZFZgNtWE89WI5yiP5WwlfDoIyc= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4= diff --git a/pkg/compactor/compactor_test.go b/pkg/compactor/compactor_test.go index 2afbd8d5ee..36ae05bda9 100644 --- a/pkg/compactor/compactor_test.go +++ b/pkg/compactor/compactor_test.go @@ -1467,7 +1467,7 @@ func createTSDBBlock(t *testing.T, bkt objstore.Bucket, userID string, minT, max require.NoError(t, err) } - require.NoError(t, db.Compact()) + require.NoError(t, db.Compact(context.Background())) require.NoError(t, db.Snapshot(snapshotDir, true)) // Look for the created block (we expect one). diff --git a/pkg/cortex/modules_test.go b/pkg/cortex/modules_test.go index 2eafc6ac60..7316e07274 100644 --- a/pkg/cortex/modules_test.go +++ b/pkg/cortex/modules_test.go @@ -166,7 +166,7 @@ func (p *myPusher) Push(ctx context.Context, req *cortexpb.WriteRequest) (*corte type myQueryable struct{} -func (q *myQueryable) Querier(ctx context.Context, mint, maxt int64) (prom_storage.Querier, error) { +func (q *myQueryable) Querier(mint, maxt int64) (prom_storage.Querier, error) { return prom_storage.NoopQuerier(), nil } diff --git a/pkg/ingester/ingester.go b/pkg/ingester/ingester.go index f3bad33946..96d8df52f7 100644 --- a/pkg/ingester/ingester.go +++ b/pkg/ingester/ingester.go @@ -280,12 +280,12 @@ func (u *userTSDB) Appender(ctx context.Context) storage.Appender { return u.db.Appender(ctx) } -func (u *userTSDB) Querier(ctx context.Context, mint, maxt int64) (storage.Querier, error) { - return u.db.Querier(ctx, mint, maxt) +func (u *userTSDB) Querier(mint, maxt int64) (storage.Querier, error) { + return u.db.Querier(mint, maxt) } -func (u *userTSDB) ChunkQuerier(ctx context.Context, mint, maxt int64) (storage.ChunkQuerier, error) { - return u.db.ChunkQuerier(ctx, mint, maxt) +func (u *userTSDB) ChunkQuerier(mint, maxt int64) (storage.ChunkQuerier, error) { + return u.db.ChunkQuerier(mint, maxt) } func (u *userTSDB) ExemplarQuerier(ctx context.Context) (storage.ExemplarQuerier, error) { @@ -304,8 +304,8 @@ func (u *userTSDB) Close() error { return u.db.Close() } -func (u *userTSDB) Compact() error { - return u.db.Compact() +func (u *userTSDB) Compact(ctx context.Context) error { + return u.db.Compact(ctx) } func (u *userTSDB) StartTime() (int64, error) { @@ -1273,14 +1273,14 @@ func (i *Ingester) Query(ctx context.Context, req *client.QueryRequest) (*client return &client.QueryResponse{}, nil } - q, err := db.Querier(ctx, int64(from), int64(through)) + q, err := db.Querier(int64(from), int64(through)) if err != nil { return nil, err } defer q.Close() // It's not required to return sorted series because series are sorted by the Cortex querier. - ss := q.Select(false, nil, matchers...) + ss := q.Select(ctx, false, nil, matchers...) if ss.Err() != nil { return nil, ss.Err() } @@ -1429,7 +1429,7 @@ func (i *Ingester) labelsValuesCommon(ctx context.Context, req *client.LabelValu return nil, cleanup, err } - q, err := db.Querier(ctx, mint, maxt) + q, err := db.Querier(mint, maxt) if err != nil { return nil, cleanup, err } @@ -1438,7 +1438,7 @@ func (i *Ingester) labelsValuesCommon(ctx context.Context, req *client.LabelValu q.Close() } - vals, _, err := q.LabelValues(labelName, matchers...) + vals, _, err := q.LabelValues(ctx, labelName, matchers...) if err != nil { return nil, cleanup, err } @@ -1505,7 +1505,7 @@ func (i *Ingester) labelNamesCommon(ctx context.Context, req *client.LabelNamesR return nil, cleanup, err } - q, err := db.Querier(ctx, mint, maxt) + q, err := db.Querier(mint, maxt) if err != nil { return nil, cleanup, err } @@ -1514,7 +1514,7 @@ func (i *Ingester) labelNamesCommon(ctx context.Context, req *client.LabelNamesR q.Close() } - names, _, err := q.LabelNames() + names, _, err := q.LabelNames(ctx) if err != nil { return nil, cleanup, err } @@ -1585,7 +1585,7 @@ func (i *Ingester) metricsForLabelMatchersCommon(ctx context.Context, req *clien return nil, cleanup, err } - q, err := db.Querier(ctx, mint, maxt) + q, err := db.Querier(mint, maxt) if err != nil { return nil, cleanup, err } @@ -1612,12 +1612,12 @@ func (i *Ingester) metricsForLabelMatchersCommon(ctx context.Context, req *clien return nil, cleanup, ctx.Err() } - seriesSet := q.Select(true, hints, matchers...) + seriesSet := q.Select(ctx, true, hints, matchers...) sets = append(sets, seriesSet) } mergedSet = storage.NewMergeSeriesSet(sets, storage.ChainedSeriesMerge) } else { - mergedSet = q.Select(false, hints, matchersSet[0]...) + mergedSet = q.Select(ctx, false, hints, matchersSet[0]...) } // Generate the response merging all series sets. @@ -1783,14 +1783,14 @@ func (i *Ingester) QueryStream(req *client.QueryRequest, stream client.Ingester_ // queryStreamChunks streams metrics from a TSDB. This implements the client.IngesterServer interface func (i *Ingester) queryStreamChunks(ctx context.Context, db *userTSDB, from, through int64, matchers []*labels.Matcher, sm *storepb.ShardMatcher, stream client.Ingester_QueryStreamServer) (numSeries, numSamples int, _ error) { - q, err := db.ChunkQuerier(ctx, from, through) + q, err := db.ChunkQuerier(from, through) if err != nil { return 0, 0, err } defer q.Close() // It's not required to return sorted series because series are sorted by the Cortex querier. - ss := q.Select(false, nil, matchers...) + ss := q.Select(ctx, false, nil, matchers...) if ss.Err() != nil { return 0, 0, ss.Err() } @@ -2002,7 +2002,7 @@ func (i *Ingester) createTSDB(userID string) (*userTSDB, error) { // this will actually create the blocks. If there is no data (empty TSDB), this is a no-op, although // local blocks compaction may still take place if configured. level.Info(userLogger).Log("msg", "Running compaction after WAL replay") - err = db.Compact() + err = db.Compact(context.TODO()) if err != nil { return nil, errors.Wrapf(err, "failed to compact TSDB: %s", udir) } @@ -2400,7 +2400,7 @@ func (i *Ingester) compactBlocks(ctx context.Context, force bool, allowed *util. default: reason = "regular" - err = userDB.Compact() + err = userDB.Compact(ctx) } if err != nil { diff --git a/pkg/ingester/ingester_test.go b/pkg/ingester/ingester_test.go index fce1c60b1c..132003f4ff 100644 --- a/pkg/ingester/ingester_test.go +++ b/pkg/ingester/ingester_test.go @@ -3776,7 +3776,7 @@ func TestIngesterNotDeleteUnshippedBlocks(t *testing.T) { db := i.getTSDB(userID) require.NotNil(t, db) - require.Nil(t, db.Compact()) + require.Nil(t, db.Compact(ctx)) oldBlocks := db.Blocks() require.Equal(t, 3, len(oldBlocks)) @@ -3800,7 +3800,7 @@ func TestIngesterNotDeleteUnshippedBlocks(t *testing.T) { _, err := i.Push(ctx, req) require.NoError(t, err) } - require.Nil(t, db.Compact()) + require.Nil(t, db.Compact(ctx)) // Only the second block should be gone along with a new block. newBlocks := db.Blocks() @@ -3828,7 +3828,7 @@ func TestIngesterNotDeleteUnshippedBlocks(t *testing.T) { _, err := i.Push(ctx, req) require.NoError(t, err) } - require.Nil(t, db.Compact()) + require.Nil(t, db.Compact(ctx)) // All blocks from the old blocks should be gone now. newBlocks2 := db.Blocks() diff --git a/pkg/querier/block.go b/pkg/querier/block.go index c0ce54f4b7..936879dac9 100644 --- a/pkg/querier/block.go +++ b/pkg/querier/block.go @@ -4,15 +4,16 @@ import ( "math" "sort" - "github.com/cortexproject/cortex/pkg/querier/iterators" - "github.com/cortexproject/cortex/pkg/querier/series" - "github.com/pkg/errors" "github.com/prometheus/prometheus/model/labels" "github.com/prometheus/prometheus/storage" "github.com/prometheus/prometheus/tsdb/chunkenc" + "github.com/prometheus/prometheus/util/annotations" "github.com/thanos-io/thanos/pkg/store/labelpb" "github.com/thanos-io/thanos/pkg/store/storepb" + + "github.com/cortexproject/cortex/pkg/querier/iterators" + "github.com/cortexproject/cortex/pkg/querier/series" ) func convertMatchersToLabelMatcher(matchers []*labels.Matcher) []storepb.LabelMatcher { @@ -42,7 +43,7 @@ func convertMatchersToLabelMatcher(matchers []*labels.Matcher) []storepb.LabelMa // Implementation of storage.SeriesSet, based on individual responses from store client. type blockQuerierSeriesSet struct { series []*storepb.Series - warnings storage.Warnings + warnings annotations.Annotations // next response to process next int @@ -82,7 +83,7 @@ func (bqss *blockQuerierSeriesSet) Err() error { return nil } -func (bqss *blockQuerierSeriesSet) Warnings() storage.Warnings { +func (bqss *blockQuerierSeriesSet) Warnings() annotations.Annotations { return bqss.warnings } diff --git a/pkg/querier/blocks_store_queryable.go b/pkg/querier/blocks_store_queryable.go index f7dac096c7..2ab1698e43 100644 --- a/pkg/querier/blocks_store_queryable.go +++ b/pkg/querier/blocks_store_queryable.go @@ -21,6 +21,7 @@ import ( "github.com/prometheus/client_golang/prometheus/promauto" "github.com/prometheus/prometheus/model/labels" "github.com/prometheus/prometheus/storage" + "github.com/prometheus/prometheus/util/annotations" "github.com/thanos-io/thanos/pkg/block" "github.com/thanos-io/thanos/pkg/extprom" "github.com/thanos-io/thanos/pkg/pool" @@ -283,21 +284,14 @@ func (q *BlocksStoreQueryable) stopping(_ error) error { } // Querier returns a new Querier on the storage. -func (q *BlocksStoreQueryable) Querier(ctx context.Context, mint, maxt int64) (storage.Querier, error) { +func (q *BlocksStoreQueryable) Querier(mint, maxt int64) (storage.Querier, error) { if s := q.State(); s != services.Running { return nil, errors.Errorf("BlocksStoreQueryable is not running: %v", s) } - userID, err := tenant.TenantID(ctx) - if err != nil { - return nil, err - } - return &blocksStoreQuerier{ - ctx: ctx, minT: mint, maxT: maxt, - userID: userID, finder: q.finder, stores: q.stores, metrics: q.metrics, @@ -309,9 +303,7 @@ func (q *BlocksStoreQueryable) Querier(ctx context.Context, mint, maxt int64) (s } type blocksStoreQuerier struct { - ctx context.Context minT, maxT int64 - userID string finder BlocksFinder stores BlocksStoreSet metrics *blocksStoreQueryableMetrics @@ -326,12 +318,17 @@ type blocksStoreQuerier struct { // Select implements storage.Querier interface. // The bool passed is ignored because the series is always sorted. -func (q *blocksStoreQuerier) Select(_ bool, sp *storage.SelectHints, matchers ...*labels.Matcher) storage.SeriesSet { - return q.selectSorted(sp, matchers...) +func (q *blocksStoreQuerier) Select(ctx context.Context, _ bool, sp *storage.SelectHints, matchers ...*labels.Matcher) storage.SeriesSet { + return q.selectSorted(ctx, sp, matchers...) } -func (q *blocksStoreQuerier) LabelNames(matchers ...*labels.Matcher) ([]string, storage.Warnings, error) { - spanLog, spanCtx := spanlogger.New(q.ctx, "blocksStoreQuerier.LabelNames") +func (q *blocksStoreQuerier) LabelNames(ctx context.Context, matchers ...*labels.Matcher) ([]string, annotations.Annotations, error) { + userID, err := tenant.TenantID(ctx) + if err != nil { + return nil, nil, err + } + + spanLog, spanCtx := spanlogger.New(ctx, "blocksStoreQuerier.LabelNames") defer spanLog.Span.Finish() minT, maxT := q.minT, q.maxT @@ -339,61 +336,64 @@ func (q *blocksStoreQuerier) LabelNames(matchers ...*labels.Matcher) ([]string, var ( resMtx sync.Mutex resNameSets = [][]string{} - resWarnings = storage.Warnings(nil) + resWarnings = annotations.Annotations(nil) convertedMatchers = convertMatchersToLabelMatcher(matchers) ) queryFunc := func(clients map[BlocksStoreClient][]ulid.ULID, minT, maxT int64) ([]ulid.ULID, error, error) { - nameSets, warnings, queriedBlocks, err, retryableError := q.fetchLabelNamesFromStore(spanCtx, clients, minT, maxT, convertedMatchers) + nameSets, warnings, queriedBlocks, err, retryableError := q.fetchLabelNamesFromStore(spanCtx, userID, clients, minT, maxT, convertedMatchers) if err != nil { return nil, err, retryableError } resMtx.Lock() resNameSets = append(resNameSets, nameSets...) - resWarnings = append(resWarnings, warnings...) + resWarnings.Merge(warnings) resMtx.Unlock() return queriedBlocks, nil, retryableError } - err := q.queryWithConsistencyCheck(spanCtx, spanLog, minT, maxT, queryFunc) - if err != nil { + if err := q.queryWithConsistencyCheck(spanCtx, spanLog, minT, maxT, userID, queryFunc); err != nil { return nil, nil, err } return strutil.MergeSlices(resNameSets...), resWarnings, nil } -func (q *blocksStoreQuerier) LabelValues(name string, matchers ...*labels.Matcher) ([]string, storage.Warnings, error) { - spanLog, spanCtx := spanlogger.New(q.ctx, "blocksStoreQuerier.LabelValues") +func (q *blocksStoreQuerier) LabelValues(ctx context.Context, name string, matchers ...*labels.Matcher) ([]string, annotations.Annotations, error) { + userID, err := tenant.TenantID(ctx) + if err != nil { + return nil, nil, err + } + + spanLog, spanCtx := spanlogger.New(ctx, "blocksStoreQuerier.LabelValues") defer spanLog.Span.Finish() minT, maxT := q.minT, q.maxT var ( resValueSets = [][]string{} - resWarnings = storage.Warnings(nil) + resWarnings = annotations.Annotations(nil) resultMtx sync.Mutex ) queryFunc := func(clients map[BlocksStoreClient][]ulid.ULID, minT, maxT int64) ([]ulid.ULID, error, error) { - valueSets, warnings, queriedBlocks, err, retryableError := q.fetchLabelValuesFromStore(spanCtx, name, clients, minT, maxT, matchers...) + valueSets, warnings, queriedBlocks, err, retryableError := q.fetchLabelValuesFromStore(spanCtx, userID, name, clients, minT, maxT, matchers...) if err != nil { return nil, err, retryableError } resultMtx.Lock() resValueSets = append(resValueSets, valueSets...) - resWarnings = append(resWarnings, warnings...) + resWarnings.Merge(warnings) resultMtx.Unlock() return queriedBlocks, nil, retryableError } - err := q.queryWithConsistencyCheck(spanCtx, spanLog, minT, maxT, queryFunc) - if err != nil { + if err := q.queryWithConsistencyCheck(spanCtx, spanLog, minT, maxT, userID, queryFunc); err != nil { return nil, nil, err } @@ -404,8 +404,13 @@ func (q *blocksStoreQuerier) Close() error { return nil } -func (q *blocksStoreQuerier) selectSorted(sp *storage.SelectHints, matchers ...*labels.Matcher) storage.SeriesSet { - spanLog, spanCtx := spanlogger.New(q.ctx, "blocksStoreQuerier.selectSorted") +func (q *blocksStoreQuerier) selectSorted(ctx context.Context, sp *storage.SelectHints, matchers ...*labels.Matcher) storage.SeriesSet { + userID, err := tenant.TenantID(ctx) + if err != nil { + return storage.ErrSeriesSet(err) + } + + spanLog, spanCtx := spanlogger.New(ctx, "blocksStoreQuerier.selectSorted") defer spanLog.Span.Finish() minT, maxT := q.minT, q.maxT @@ -415,16 +420,16 @@ func (q *blocksStoreQuerier) selectSorted(sp *storage.SelectHints, matchers ...* var ( resSeriesSets = []storage.SeriesSet(nil) - resWarnings = storage.Warnings(nil) + resWarnings = annotations.Annotations(nil) - maxChunksLimit = q.limits.MaxChunksPerQueryFromStore(q.userID) + maxChunksLimit = q.limits.MaxChunksPerQueryFromStore(userID) leftChunksLimit = maxChunksLimit resultMtx sync.Mutex ) queryFunc := func(clients map[BlocksStoreClient][]ulid.ULID, minT, maxT int64) ([]ulid.ULID, error, error) { - seriesSets, queriedBlocks, warnings, numChunks, err, retryableError := q.fetchSeriesFromStores(spanCtx, sp, clients, minT, maxT, matchers, maxChunksLimit, leftChunksLimit) + seriesSets, queriedBlocks, warnings, numChunks, err, retryableError := q.fetchSeriesFromStores(spanCtx, sp, userID, clients, minT, maxT, matchers, maxChunksLimit, leftChunksLimit) if err != nil { return nil, err, retryableError } @@ -432,7 +437,7 @@ func (q *blocksStoreQuerier) selectSorted(sp *storage.SelectHints, matchers ...* resultMtx.Lock() resSeriesSets = append(resSeriesSets, seriesSets...) - resWarnings = append(resWarnings, warnings...) + resWarnings.Merge(warnings) // Given a single block is guaranteed to not be queried twice, we can safely decrease the number of // chunks we can still read before hitting the limit (max == 0 means disabled). @@ -444,8 +449,7 @@ func (q *blocksStoreQuerier) selectSorted(sp *storage.SelectHints, matchers ...* return queriedBlocks, nil, retryableError } - err := q.queryWithConsistencyCheck(spanCtx, spanLog, minT, maxT, queryFunc) - if err != nil { + if err := q.queryWithConsistencyCheck(spanCtx, spanLog, minT, maxT, userID, queryFunc); err != nil { return storage.ErrSeriesSet(err) } @@ -458,7 +462,7 @@ func (q *blocksStoreQuerier) selectSorted(sp *storage.SelectHints, matchers ...* resWarnings) } -func (q *blocksStoreQuerier) queryWithConsistencyCheck(ctx context.Context, logger log.Logger, minT, maxT int64, +func (q *blocksStoreQuerier) queryWithConsistencyCheck(ctx context.Context, logger log.Logger, minT, maxT int64, userID string, queryFunc func(clients map[BlocksStoreClient][]ulid.ULID, minT, maxT int64) ([]ulid.ULID, error, error)) error { // If queryStoreAfter is enabled, we do manipulate the query maxt to query samples up until // now - queryStoreAfter, because the most recent time range is covered by ingesters. This @@ -481,7 +485,7 @@ func (q *blocksStoreQuerier) queryWithConsistencyCheck(ctx context.Context, logg } // Find the list of blocks we need to query given the time range. - knownBlocks, knownDeletionMarks, err := q.finder.GetBlocks(ctx, q.userID, minT, maxT) + knownBlocks, knownDeletionMarks, err := q.finder.GetBlocks(ctx, userID, minT, maxT) if err != nil { return err } @@ -510,7 +514,7 @@ func (q *blocksStoreQuerier) queryWithConsistencyCheck(ctx context.Context, logg for attempt := 1; attempt <= maxFetchSeriesAttempts; attempt++ { // Find the set of store-gateway instances having the blocks. The exclude parameter is the // map of blocks queried so far, with the list of store-gateway addresses for each block. - clients, err := q.stores.GetClientsFor(q.userID, remainingBlocks, attemptedBlocks, attemptedBlocksZones) + clients, err := q.stores.GetClientsFor(userID, remainingBlocks, attemptedBlocks, attemptedBlocksZones) if err != nil { // If it's a retry and we get an error, it means there are no more store-gateways left // from which running another attempt, so we're just stopping retrying. @@ -571,19 +575,20 @@ func (q *blocksStoreQuerier) queryWithConsistencyCheck(ctx context.Context, logg func (q *blocksStoreQuerier) fetchSeriesFromStores( ctx context.Context, sp *storage.SelectHints, + userID string, clients map[BlocksStoreClient][]ulid.ULID, minT int64, maxT int64, matchers []*labels.Matcher, maxChunksLimit int, leftChunksLimit int, -) ([]storage.SeriesSet, []ulid.ULID, storage.Warnings, int, error, error) { +) ([]storage.SeriesSet, []ulid.ULID, annotations.Annotations, int, error, error) { var ( - reqCtx = grpc_metadata.AppendToOutgoingContext(ctx, cortex_tsdb.TenantIDExternalLabel, q.userID) + reqCtx = grpc_metadata.AppendToOutgoingContext(ctx, cortex_tsdb.TenantIDExternalLabel, userID) g, gCtx = errgroup.WithContext(reqCtx) mtx = sync.Mutex{} seriesSets = []storage.SeriesSet(nil) - warnings = storage.Warnings(nil) + warnings = annotations.Annotations(nil) queriedBlocks = []ulid.ULID(nil) numChunks = atomic.NewInt32(0) spanLog = spanlogger.FromContext(ctx) @@ -635,7 +640,7 @@ func (q *blocksStoreQuerier) fetchSeriesFromStores( } mySeries := []*storepb.Series(nil) - myWarnings := storage.Warnings(nil) + myWarnings := annotations.Annotations(nil) myQueriedBlocks := []ulid.ULID(nil) for { @@ -707,7 +712,7 @@ func (q *blocksStoreQuerier) fetchSeriesFromStores( } if w := resp.GetWarning(); w != "" { - myWarnings = append(myWarnings, errors.New(w)) + myWarnings.Add(errors.New(w)) } if h := resp.GetHints(); h != nil { @@ -781,7 +786,7 @@ func (q *blocksStoreQuerier) fetchSeriesFromStores( // Store the result. mtx.Lock() seriesSets = append(seriesSets, &blockQuerierSeriesSet{series: mySeries}) - warnings = append(warnings, myWarnings...) + warnings.Merge(myWarnings) queriedBlocks = append(queriedBlocks, myQueriedBlocks...) mtx.Unlock() @@ -799,17 +804,18 @@ func (q *blocksStoreQuerier) fetchSeriesFromStores( func (q *blocksStoreQuerier) fetchLabelNamesFromStore( ctx context.Context, + userID string, clients map[BlocksStoreClient][]ulid.ULID, minT int64, maxT int64, matchers []storepb.LabelMatcher, -) ([][]string, storage.Warnings, []ulid.ULID, error, error) { +) ([][]string, annotations.Annotations, []ulid.ULID, error, error) { var ( - reqCtx = grpc_metadata.AppendToOutgoingContext(ctx, cortex_tsdb.TenantIDExternalLabel, q.userID) + reqCtx = grpc_metadata.AppendToOutgoingContext(ctx, cortex_tsdb.TenantIDExternalLabel, userID) g, gCtx = errgroup.WithContext(reqCtx) mtx = sync.Mutex{} nameSets = [][]string{} - warnings = storage.Warnings(nil) + warnings = annotations.Annotations(nil) queriedBlocks = []ulid.ULID(nil) spanLog = spanlogger.FromContext(ctx) merrMtx = sync.Mutex{} @@ -880,7 +886,7 @@ func (q *blocksStoreQuerier) fetchLabelNamesFromStore( mtx.Lock() nameSets = append(nameSets, namesResp.Names) for _, w := range namesResp.Warnings { - warnings = append(warnings, errors.New(w)) + warnings.Add(errors.New(w)) } queriedBlocks = append(queriedBlocks, myQueriedBlocks...) mtx.Unlock() @@ -899,18 +905,19 @@ func (q *blocksStoreQuerier) fetchLabelNamesFromStore( func (q *blocksStoreQuerier) fetchLabelValuesFromStore( ctx context.Context, + userID string, name string, clients map[BlocksStoreClient][]ulid.ULID, minT int64, maxT int64, matchers ...*labels.Matcher, -) ([][]string, storage.Warnings, []ulid.ULID, error, error) { +) ([][]string, annotations.Annotations, []ulid.ULID, error, error) { var ( - reqCtx = grpc_metadata.AppendToOutgoingContext(ctx, cortex_tsdb.TenantIDExternalLabel, q.userID) + reqCtx = grpc_metadata.AppendToOutgoingContext(ctx, cortex_tsdb.TenantIDExternalLabel, userID) g, gCtx = errgroup.WithContext(reqCtx) mtx = sync.Mutex{} valueSets = [][]string{} - warnings = storage.Warnings(nil) + warnings = annotations.Annotations(nil) queriedBlocks = []ulid.ULID(nil) spanLog = spanlogger.FromContext(ctx) merrMtx = sync.Mutex{} @@ -984,7 +991,7 @@ func (q *blocksStoreQuerier) fetchLabelValuesFromStore( mtx.Lock() valueSets = append(valueSets, valuesResp.Values) for _, w := range valuesResp.Warnings { - warnings = append(warnings, errors.New(w)) + warnings.Add(errors.New(w)) } queriedBlocks = append(queriedBlocks, myQueriedBlocks...) mtx.Unlock() diff --git a/pkg/querier/blocks_store_queryable_test.go b/pkg/querier/blocks_store_queryable_test.go index 23114f2663..9b05c4f94b 100644 --- a/pkg/querier/blocks_store_queryable_test.go +++ b/pkg/querier/blocks_store_queryable_test.go @@ -825,17 +825,16 @@ func TestBlocksStoreQuerier_Select(t *testing.T) { t.Run(testName, func(t *testing.T) { t.Parallel() - ctx := limiter.AddQueryLimiterToContext(context.Background(), testData.queryLimiter) + ctx := user.InjectOrgID(context.Background(), "user-1") + ctx = limiter.AddQueryLimiterToContext(ctx, testData.queryLimiter) reg := prometheus.NewPedanticRegistry() stores := &blocksStoreSetMock{mockedResponses: testData.storeSetResponses} finder := &blocksFinderMock{} finder.On("GetBlocks", mock.Anything, "user-1", minT, maxT).Return(testData.finderResult, map[ulid.ULID]*bucketindex.BlockDeletionMark(nil), testData.finderErr) q := &blocksStoreQuerier{ - ctx: ctx, minT: minT, maxT: maxT, - userID: "user-1", finder: finder, stores: stores, consistency: NewBlocksConsistencyChecker(0, 0, log.NewNopLogger(), nil), @@ -848,7 +847,7 @@ func TestBlocksStoreQuerier_Select(t *testing.T) { labels.MustNewMatcher(labels.MatchEqual, labels.MetricName, metricName), } - set := q.Select(true, nil, matchers...) + set := q.Select(ctx, true, nil, matchers...) if testData.expectedErr != nil { assert.EqualError(t, set.Err(), testData.expectedErr.Error()) assert.IsType(t, set.Err(), testData.expectedErr) @@ -1344,17 +1343,15 @@ func TestBlocksStoreQuerier_Labels(t *testing.T) { // Splitting it because we need a new registry for names and values. // And also the initial expectedErr checking needs to be done for both. for _, testFunc := range []string{"LabelNames", "LabelValues"} { - ctx := context.Background() + ctx := user.InjectOrgID(context.Background(), "user-1") reg := prometheus.NewPedanticRegistry() stores := &blocksStoreSetMock{mockedResponses: testData.storeSetResponses} finder := &blocksFinderMock{} finder.On("GetBlocks", mock.Anything, "user-1", minT, maxT).Return(testData.finderResult, map[ulid.ULID]*bucketindex.BlockDeletionMark(nil), testData.finderErr) q := &blocksStoreQuerier{ - ctx: ctx, minT: minT, maxT: maxT, - userID: "user-1", finder: finder, stores: stores, consistency: NewBlocksConsistencyChecker(0, 0, log.NewNopLogger(), nil), @@ -1364,7 +1361,7 @@ func TestBlocksStoreQuerier_Labels(t *testing.T) { } if testFunc == "LabelNames" { - names, warnings, err := q.LabelNames() + names, warnings, err := q.LabelNames(ctx) if testData.expectedErr != "" { require.Equal(t, testData.expectedErr, err.Error()) continue @@ -1381,7 +1378,7 @@ func TestBlocksStoreQuerier_Labels(t *testing.T) { } if testFunc == "LabelValues" { - values, warnings, err := q.LabelValues(labels.MetricName) + values, warnings, err := q.LabelValues(ctx, labels.MetricName) if testData.expectedErr != "" { require.Equal(t, testData.expectedErr, err.Error()) continue @@ -1447,14 +1444,13 @@ func TestBlocksStoreQuerier_SelectSortedShouldHonorQueryStoreAfter(t *testing.T) t.Run(testName, func(t *testing.T) { t.Parallel() + ctx := user.InjectOrgID(context.Background(), "user-1") finder := &blocksFinderMock{} finder.On("GetBlocks", mock.Anything, "user-1", mock.Anything, mock.Anything).Return(bucketindex.Blocks(nil), map[ulid.ULID]*bucketindex.BlockDeletionMark(nil), error(nil)) q := &blocksStoreQuerier{ - ctx: context.Background(), minT: testData.queryMinT, maxT: testData.queryMaxT, - userID: "user-1", finder: finder, stores: &blocksStoreSetMock{}, consistency: NewBlocksConsistencyChecker(0, 0, log.NewNopLogger(), nil), @@ -1469,7 +1465,7 @@ func TestBlocksStoreQuerier_SelectSortedShouldHonorQueryStoreAfter(t *testing.T) End: testData.queryMaxT, } - set := q.selectSorted(sp) + set := q.selectSorted(ctx, sp) require.NoError(t, set.Err()) if testData.expectedMinT == 0 && testData.expectedMaxT == 0 { diff --git a/pkg/querier/distributor_queryable.go b/pkg/querier/distributor_queryable.go index ae3468cc65..10e4db4326 100644 --- a/pkg/querier/distributor_queryable.go +++ b/pkg/querier/distributor_queryable.go @@ -11,6 +11,7 @@ import ( "github.com/prometheus/prometheus/model/labels" "github.com/prometheus/prometheus/scrape" "github.com/prometheus/prometheus/storage" + "github.com/prometheus/prometheus/util/annotations" "github.com/cortexproject/cortex/pkg/cortexpb" "github.com/cortexproject/cortex/pkg/ingester/client" @@ -57,10 +58,9 @@ type distributorQueryable struct { queryStoreForLabels bool } -func (d distributorQueryable) Querier(ctx context.Context, mint, maxt int64) (storage.Querier, error) { +func (d distributorQueryable) Querier(mint, maxt int64) (storage.Querier, error) { return &distributorQuerier{ distributor: d.distributor, - ctx: ctx, mint: mint, maxt: maxt, streaming: d.streaming, @@ -78,7 +78,6 @@ func (d distributorQueryable) UseQueryable(now time.Time, _, queryMaxT int64) bo type distributorQuerier struct { distributor Distributor - ctx context.Context mint, maxt int64 streaming bool streamingMetadata bool @@ -89,8 +88,8 @@ type distributorQuerier struct { // Select implements storage.Querier interface. // The bool passed is ignored because the series is always sorted. -func (q *distributorQuerier) Select(sortSeries bool, sp *storage.SelectHints, matchers ...*labels.Matcher) storage.SeriesSet { - log, ctx := spanlogger.New(q.ctx, "distributorQuerier.Select") +func (q *distributorQuerier) Select(ctx context.Context, sortSeries bool, sp *storage.SelectHints, matchers ...*labels.Matcher) storage.SeriesSet { + log, ctx := spanlogger.New(ctx, "distributorQuerier.Select") defer log.Span.Finish() minT, maxT := q.mint, q.maxt @@ -208,27 +207,27 @@ func (q *distributorQuerier) streamingSelect(ctx context.Context, sortSeries boo return storage.NewMergeSeriesSet(sets, storage.ChainedSeriesMerge) } -func (q *distributorQuerier) LabelValues(name string, matchers ...*labels.Matcher) ([]string, storage.Warnings, error) { +func (q *distributorQuerier) LabelValues(ctx context.Context, name string, matchers ...*labels.Matcher) ([]string, annotations.Annotations, error) { var ( lvs []string err error ) if q.streamingMetadata { - lvs, err = q.distributor.LabelValuesForLabelNameStream(q.ctx, model.Time(q.mint), model.Time(q.maxt), model.LabelName(name), matchers...) + lvs, err = q.distributor.LabelValuesForLabelNameStream(ctx, model.Time(q.mint), model.Time(q.maxt), model.LabelName(name), matchers...) } else { - lvs, err = q.distributor.LabelValuesForLabelName(q.ctx, model.Time(q.mint), model.Time(q.maxt), model.LabelName(name), matchers...) + lvs, err = q.distributor.LabelValuesForLabelName(ctx, model.Time(q.mint), model.Time(q.maxt), model.LabelName(name), matchers...) } return lvs, nil, err } -func (q *distributorQuerier) LabelNames(matchers ...*labels.Matcher) ([]string, storage.Warnings, error) { +func (q *distributorQuerier) LabelNames(ctx context.Context, matchers ...*labels.Matcher) ([]string, annotations.Annotations, error) { if len(matchers) > 0 { - return q.labelNamesWithMatchers(matchers...) + return q.labelNamesWithMatchers(ctx, matchers...) } - log, ctx := spanlogger.New(q.ctx, "distributorQuerier.LabelNames") + log, ctx := spanlogger.New(ctx, "distributorQuerier.LabelNames") defer log.Span.Finish() var ( @@ -246,8 +245,8 @@ func (q *distributorQuerier) LabelNames(matchers ...*labels.Matcher) ([]string, } // labelNamesWithMatchers performs the LabelNames call by calling ingester's MetricsForLabelMatchers method -func (q *distributorQuerier) labelNamesWithMatchers(matchers ...*labels.Matcher) ([]string, storage.Warnings, error) { - log, ctx := spanlogger.New(q.ctx, "distributorQuerier.labelNamesWithMatchers") +func (q *distributorQuerier) labelNamesWithMatchers(ctx context.Context, matchers ...*labels.Matcher) ([]string, annotations.Annotations, error) { + log, ctx := spanlogger.New(ctx, "distributorQuerier.labelNamesWithMatchers") defer log.Span.Finish() var ( diff --git a/pkg/querier/distributor_queryable_test.go b/pkg/querier/distributor_queryable_test.go index d3520adaaa..451502167e 100644 --- a/pkg/querier/distributor_queryable_test.go +++ b/pkg/querier/distributor_queryable_test.go @@ -50,10 +50,11 @@ func TestDistributorQuerier(t *testing.T) { nil) queryable := newDistributorQueryable(d, false, false, nil, 0, false) - querier, err := queryable.Querier(context.Background(), mint, maxt) + querier, err := queryable.Querier(mint, maxt) require.NoError(t, err) - seriesSet := querier.Select(true, &storage.SelectHints{Start: mint, End: maxt}) + ctx := context.Background() + seriesSet := querier.Select(ctx, true, &storage.SelectHints{Start: mint, End: maxt}) require.NoError(t, seriesSet.Err()) require.True(t, seriesSet.Next()) @@ -142,7 +143,7 @@ func TestDistributorQuerier_SelectShouldHonorQueryIngestersWithin(t *testing.T) ctx := user.InjectOrgID(context.Background(), "test") queryable := newDistributorQueryable(distributor, streamingEnabled, streamingEnabled, nil, testData.queryIngestersWithin, testData.queryStoreForLabels) - querier, err := queryable.Querier(ctx, testData.queryMinT, testData.queryMaxT) + querier, err := queryable.Querier(testData.queryMinT, testData.queryMaxT) require.NoError(t, err) limits := DefaultLimitsConfig() @@ -161,7 +162,7 @@ func TestDistributorQuerier_SelectShouldHonorQueryIngestersWithin(t *testing.T) } } - seriesSet := querier.Select(true, hints) + seriesSet := querier.Select(ctx, true, hints) require.NoError(t, seriesSet.Err()) if testData.expectedMinT == 0 && testData.expectedMaxT == 0 { @@ -231,10 +232,10 @@ func TestIngesterStreaming(t *testing.T) { ctx := user.InjectOrgID(context.Background(), "0") queryable := newDistributorQueryable(d, true, true, mergeChunks, 0, true) - querier, err := queryable.Querier(ctx, mint, maxt) + querier, err := queryable.Querier(mint, maxt) require.NoError(t, err) - seriesSet := querier.Select(true, &storage.SelectHints{Start: mint, End: maxt}) + seriesSet := querier.Select(ctx, true, &storage.SelectHints{Start: mint, End: maxt}) require.NoError(t, seriesSet.Err()) require.True(t, seriesSet.Next()) @@ -309,10 +310,10 @@ func TestIngesterStreamingMixedResults(t *testing.T) { ctx := user.InjectOrgID(context.Background(), "0") queryable := newDistributorQueryable(d, true, true, mergeChunks, 0, true) - querier, err := queryable.Querier(ctx, mint, maxt) + querier, err := queryable.Querier(mint, maxt) require.NoError(t, err) - seriesSet := querier.Select(true, &storage.SelectHints{Start: mint, End: maxt}, labels.MustNewMatcher(labels.MatchRegexp, labels.MetricName, ".*")) + seriesSet := querier.Select(ctx, true, &storage.SelectHints{Start: mint, End: maxt}, labels.MustNewMatcher(labels.MatchRegexp, labels.MetricName, ".*")) require.NoError(t, seriesSet.Err()) require.True(t, seriesSet.Next()) @@ -365,10 +366,11 @@ func TestDistributorQuerier_LabelNames(t *testing.T) { Return(metrics, nil) queryable := newDistributorQueryable(d, false, streamingEnabled, nil, 0, true) - querier, err := queryable.Querier(context.Background(), mint, maxt) + querier, err := queryable.Querier(mint, maxt) require.NoError(t, err) - names, warnings, err := querier.LabelNames(someMatchers...) + ctx := context.Background() + names, warnings, err := querier.LabelNames(ctx, someMatchers...) require.NoError(t, err) assert.Empty(t, warnings) assert.Equal(t, labelNames, names) diff --git a/pkg/querier/duplicates_test.go b/pkg/querier/duplicates_test.go index a319c4d657..629b2a4fa0 100644 --- a/pkg/querier/duplicates_test.go +++ b/pkg/querier/duplicates_test.go @@ -11,6 +11,7 @@ import ( "github.com/prometheus/prometheus/model/labels" "github.com/prometheus/prometheus/promql" "github.com/prometheus/prometheus/storage" + "github.com/prometheus/prometheus/util/annotations" "github.com/stretchr/testify/require" "github.com/cortexproject/cortex/pkg/cortexpb" @@ -115,7 +116,7 @@ type testQueryable struct { ts storage.SeriesSet } -func (t *testQueryable) Querier(_ context.Context, _, _ int64) (storage.Querier, error) { +func (t *testQueryable) Querier(_, _ int64) (storage.Querier, error) { return testQuerier{ts: t.ts}, nil } @@ -123,15 +124,15 @@ type testQuerier struct { ts storage.SeriesSet } -func (m testQuerier) Select(_ bool, _ *storage.SelectHints, _ ...*labels.Matcher) storage.SeriesSet { +func (m testQuerier) Select(ctx context.Context, _ bool, _ *storage.SelectHints, _ ...*labels.Matcher) storage.SeriesSet { return m.ts } -func (m testQuerier) LabelValues(name string, matchers ...*labels.Matcher) ([]string, storage.Warnings, error) { +func (m testQuerier) LabelValues(ctx context.Context, name string, matchers ...*labels.Matcher) ([]string, annotations.Annotations, error) { return nil, nil, nil } -func (m testQuerier) LabelNames(matchers ...*labels.Matcher) ([]string, storage.Warnings, error) { +func (m testQuerier) LabelNames(ctx context.Context, matchers ...*labels.Matcher) ([]string, annotations.Annotations, error) { return nil, nil, nil } diff --git a/pkg/querier/error_translate_queryable.go b/pkg/querier/error_translate_queryable.go index b08e18554e..ccf0cee8c7 100644 --- a/pkg/querier/error_translate_queryable.go +++ b/pkg/querier/error_translate_queryable.go @@ -8,6 +8,7 @@ import ( "github.com/prometheus/prometheus/model/labels" "github.com/prometheus/prometheus/promql" "github.com/prometheus/prometheus/storage" + "github.com/prometheus/prometheus/util/annotations" "github.com/cortexproject/cortex/pkg/util/validation" ) @@ -97,8 +98,8 @@ type errorTranslateQueryable struct { fn ErrTranslateFn } -func (e errorTranslateQueryable) Querier(ctx context.Context, mint, maxt int64) (storage.Querier, error) { - q, err := e.q.Querier(ctx, mint, maxt) +func (e errorTranslateQueryable) Querier(mint, maxt int64) (storage.Querier, error) { + q, err := e.q.Querier(mint, maxt) return errorTranslateQuerier{q: q, fn: e.fn}, e.fn(err) } @@ -107,13 +108,13 @@ type errorTranslateSampleAndChunkQueryable struct { fn ErrTranslateFn } -func (e errorTranslateSampleAndChunkQueryable) Querier(ctx context.Context, mint, maxt int64) (storage.Querier, error) { - q, err := e.q.Querier(ctx, mint, maxt) +func (e errorTranslateSampleAndChunkQueryable) Querier(mint, maxt int64) (storage.Querier, error) { + q, err := e.q.Querier(mint, maxt) return errorTranslateQuerier{q: q, fn: e.fn}, e.fn(err) } -func (e errorTranslateSampleAndChunkQueryable) ChunkQuerier(ctx context.Context, mint, maxt int64) (storage.ChunkQuerier, error) { - q, err := e.q.ChunkQuerier(ctx, mint, maxt) +func (e errorTranslateSampleAndChunkQueryable) ChunkQuerier(mint, maxt int64) (storage.ChunkQuerier, error) { + q, err := e.q.ChunkQuerier(mint, maxt) return errorTranslateChunkQuerier{q: q, fn: e.fn}, e.fn(err) } @@ -122,13 +123,13 @@ type errorTranslateQuerier struct { fn ErrTranslateFn } -func (e errorTranslateQuerier) LabelValues(name string, matchers ...*labels.Matcher) ([]string, storage.Warnings, error) { - values, warnings, err := e.q.LabelValues(name, matchers...) +func (e errorTranslateQuerier) LabelValues(ctx context.Context, name string, matchers ...*labels.Matcher) ([]string, annotations.Annotations, error) { + values, warnings, err := e.q.LabelValues(ctx, name, matchers...) return values, warnings, e.fn(err) } -func (e errorTranslateQuerier) LabelNames(matchers ...*labels.Matcher) ([]string, storage.Warnings, error) { - values, warnings, err := e.q.LabelNames(matchers...) +func (e errorTranslateQuerier) LabelNames(ctx context.Context, matchers ...*labels.Matcher) ([]string, annotations.Annotations, error) { + values, warnings, err := e.q.LabelNames(ctx, matchers...) return values, warnings, e.fn(err) } @@ -136,8 +137,8 @@ func (e errorTranslateQuerier) Close() error { return e.fn(e.q.Close()) } -func (e errorTranslateQuerier) Select(sortSeries bool, hints *storage.SelectHints, matchers ...*labels.Matcher) storage.SeriesSet { - s := e.q.Select(sortSeries, hints, matchers...) +func (e errorTranslateQuerier) Select(ctx context.Context, sortSeries bool, hints *storage.SelectHints, matchers ...*labels.Matcher) storage.SeriesSet { + s := e.q.Select(ctx, sortSeries, hints, matchers...) return errorTranslateSeriesSet{s: s, fn: e.fn} } @@ -146,13 +147,13 @@ type errorTranslateChunkQuerier struct { fn ErrTranslateFn } -func (e errorTranslateChunkQuerier) LabelValues(name string, matchers ...*labels.Matcher) ([]string, storage.Warnings, error) { - values, warnings, err := e.q.LabelValues(name, matchers...) +func (e errorTranslateChunkQuerier) LabelValues(ctx context.Context, name string, matchers ...*labels.Matcher) ([]string, annotations.Annotations, error) { + values, warnings, err := e.q.LabelValues(ctx, name, matchers...) return values, warnings, e.fn(err) } -func (e errorTranslateChunkQuerier) LabelNames(matchers ...*labels.Matcher) ([]string, storage.Warnings, error) { - values, warnings, err := e.q.LabelNames(matchers...) +func (e errorTranslateChunkQuerier) LabelNames(ctx context.Context, matchers ...*labels.Matcher) ([]string, annotations.Annotations, error) { + values, warnings, err := e.q.LabelNames(ctx, matchers...) return values, warnings, e.fn(err) } @@ -160,8 +161,8 @@ func (e errorTranslateChunkQuerier) Close() error { return e.fn(e.q.Close()) } -func (e errorTranslateChunkQuerier) Select(sortSeries bool, hints *storage.SelectHints, matchers ...*labels.Matcher) storage.ChunkSeriesSet { - s := e.q.Select(sortSeries, hints, matchers...) +func (e errorTranslateChunkQuerier) Select(ctx context.Context, sortSeries bool, hints *storage.SelectHints, matchers ...*labels.Matcher) storage.ChunkSeriesSet { + s := e.q.Select(ctx, sortSeries, hints, matchers...) return errorTranslateChunkSeriesSet{s: s, fn: e.fn} } @@ -182,7 +183,7 @@ func (e errorTranslateSeriesSet) Err() error { return e.fn(e.s.Err()) } -func (e errorTranslateSeriesSet) Warnings() storage.Warnings { +func (e errorTranslateSeriesSet) Warnings() annotations.Annotations { return e.s.Warnings() } @@ -203,6 +204,6 @@ func (e errorTranslateChunkSeriesSet) Err() error { return e.fn(e.s.Err()) } -func (e errorTranslateChunkSeriesSet) Warnings() storage.Warnings { +func (e errorTranslateChunkSeriesSet) Warnings() annotations.Annotations { return e.s.Warnings() } diff --git a/pkg/querier/error_translate_queryable_test.go b/pkg/querier/error_translate_queryable_test.go index b5293cc2a1..43c560d4c1 100644 --- a/pkg/querier/error_translate_queryable_test.go +++ b/pkg/querier/error_translate_queryable_test.go @@ -17,6 +17,7 @@ import ( "github.com/prometheus/prometheus/model/labels" "github.com/prometheus/prometheus/promql" "github.com/prometheus/prometheus/storage" + "github.com/prometheus/prometheus/util/annotations" v1 "github.com/prometheus/prometheus/web/api/v1" "github.com/stretchr/testify/require" "github.com/weaveworks/common/httpgrpc" @@ -176,11 +177,11 @@ type errorTestQueryable struct { err error } -func (t errorTestQueryable) ChunkQuerier(ctx context.Context, mint, maxt int64) (storage.ChunkQuerier, error) { +func (t errorTestQueryable) ChunkQuerier(mint, maxt int64) (storage.ChunkQuerier, error) { return nil, t.err } -func (t errorTestQueryable) Querier(ctx context.Context, mint, maxt int64) (storage.Querier, error) { +func (t errorTestQueryable) Querier(mint, maxt int64) (storage.Querier, error) { if t.q != nil { return t.q, nil } @@ -192,11 +193,11 @@ type errorTestQuerier struct { err error } -func (t errorTestQuerier) LabelValues(name string, matchers ...*labels.Matcher) ([]string, storage.Warnings, error) { +func (t errorTestQuerier) LabelValues(ctx context.Context, name string, matchers ...*labels.Matcher) ([]string, annotations.Annotations, error) { return nil, nil, t.err } -func (t errorTestQuerier) LabelNames(matchers ...*labels.Matcher) ([]string, storage.Warnings, error) { +func (t errorTestQuerier) LabelNames(ctx context.Context, matchers ...*labels.Matcher) ([]string, annotations.Annotations, error) { return nil, nil, t.err } @@ -204,7 +205,7 @@ func (t errorTestQuerier) Close() error { return nil } -func (t errorTestQuerier) Select(sortSeries bool, hints *storage.SelectHints, matchers ...*labels.Matcher) storage.SeriesSet { +func (t errorTestQuerier) Select(ctx context.Context, sortSeries bool, hints *storage.SelectHints, matchers ...*labels.Matcher) storage.SeriesSet { if t.s != nil { return t.s } @@ -227,6 +228,6 @@ func (t errorTestSeriesSet) Err() error { return t.err } -func (t errorTestSeriesSet) Warnings() storage.Warnings { +func (t errorTestSeriesSet) Warnings() annotations.Annotations { return nil } diff --git a/pkg/querier/lazyquery/lazyquery.go b/pkg/querier/lazyquery/lazyquery.go index 361baa0213..93634db957 100644 --- a/pkg/querier/lazyquery/lazyquery.go +++ b/pkg/querier/lazyquery/lazyquery.go @@ -1,8 +1,11 @@ package lazyquery import ( + "context" + "github.com/prometheus/prometheus/model/labels" "github.com/prometheus/prometheus/storage" + "github.com/prometheus/prometheus/util/annotations" ) // LazyQuerier is a lazy-loaded adapter for a storage.Querier @@ -17,12 +20,12 @@ func NewLazyQuerier(next storage.Querier) storage.Querier { } // Select implements Storage.Querier -func (l LazyQuerier) Select(selectSorted bool, params *storage.SelectHints, matchers ...*labels.Matcher) storage.SeriesSet { +func (l LazyQuerier) Select(ctx context.Context, selectSorted bool, params *storage.SelectHints, matchers ...*labels.Matcher) storage.SeriesSet { // make sure there is space in the buffer, to unblock the goroutine and let it die even if nobody is // waiting for the result yet (or anymore). future := make(chan storage.SeriesSet, 1) go func() { - future <- l.next.Select(selectSorted, params, matchers...) + future <- l.next.Select(ctx, selectSorted, params, matchers...) }() return &lazySeriesSet{ @@ -31,13 +34,13 @@ func (l LazyQuerier) Select(selectSorted bool, params *storage.SelectHints, matc } // LabelValues implements Storage.Querier -func (l LazyQuerier) LabelValues(name string, matchers ...*labels.Matcher) ([]string, storage.Warnings, error) { - return l.next.LabelValues(name, matchers...) +func (l LazyQuerier) LabelValues(ctx context.Context, name string, matchers ...*labels.Matcher) ([]string, annotations.Annotations, error) { + return l.next.LabelValues(ctx, name, matchers...) } // LabelNames implements Storage.Querier -func (l LazyQuerier) LabelNames(matchers ...*labels.Matcher) ([]string, storage.Warnings, error) { - return l.next.LabelNames(matchers...) +func (l LazyQuerier) LabelNames(ctx context.Context, matchers ...*labels.Matcher) ([]string, annotations.Annotations, error) { + return l.next.LabelNames(ctx, matchers...) } // Close implements Storage.Querier @@ -75,6 +78,6 @@ func (s *lazySeriesSet) Err() error { } // Warnings implements storage.SeriesSet. -func (s *lazySeriesSet) Warnings() storage.Warnings { +func (s *lazySeriesSet) Warnings() annotations.Annotations { return nil } diff --git a/pkg/querier/querier.go b/pkg/querier/querier.go index a85fb1d092..1455dbe37e 100644 --- a/pkg/querier/querier.go +++ b/pkg/querier/querier.go @@ -17,6 +17,7 @@ import ( "github.com/prometheus/prometheus/model/labels" "github.com/prometheus/prometheus/promql" "github.com/prometheus/prometheus/storage" + "github.com/prometheus/prometheus/util/annotations" v1 "github.com/prometheus/prometheus/web/api/v1" "github.com/thanos-io/promql-engine/engine" "github.com/thanos-io/promql-engine/logicalplan" @@ -166,8 +167,8 @@ func New(cfg Config, limits *validation.Overrides, distributor Distributor, stor queryable := NewQueryable(distributorQueryable, ns, iteratorFunc, cfg, limits, tombstonesLoader) exemplarQueryable := newDistributorExemplarQueryable(distributor) - lazyQueryable := storage.QueryableFunc(func(ctx context.Context, mint int64, maxt int64) (storage.Querier, error) { - querier, err := queryable.Querier(ctx, mint, maxt) + lazyQueryable := storage.QueryableFunc(func(mint int64, maxt int64) (storage.Querier, error) { + querier, err := queryable.Querier(mint, maxt) if err != nil { return nil, err } @@ -218,7 +219,7 @@ type sampleAndChunkQueryable struct { storage.Queryable } -func (q *sampleAndChunkQueryable) ChunkQuerier(ctx context.Context, mint, maxt int64) (storage.ChunkQuerier, error) { +func (q *sampleAndChunkQueryable) ChunkQuerier(mint, maxt int64) (storage.ChunkQuerier, error) { return nil, errors.New("ChunkQuerier not implemented") } @@ -243,25 +244,9 @@ type QueryableWithFilter interface { // NewQueryable creates a new Queryable for cortex. func NewQueryable(distributor QueryableWithFilter, stores []QueryableWithFilter, chunkIterFn chunkIteratorFunc, cfg Config, limits *validation.Overrides, tombstonesLoader purger.TombstonesLoader) storage.Queryable { - return storage.QueryableFunc(func(ctx context.Context, mint, maxt int64) (storage.Querier, error) { - now := time.Now() - - userID, err := tenant.TenantID(ctx) - if err != nil { - return nil, err - } - - ctx = limiter.AddQueryLimiterToContext(ctx, limiter.NewQueryLimiter(limits.MaxFetchedSeriesPerQuery(userID), limits.MaxFetchedChunkBytesPerQuery(userID), limits.MaxChunksPerQuery(userID), limits.MaxFetchedDataBytesPerQuery(userID))) - - mint, maxt, err = validateQueryTimeRange(ctx, userID, mint, maxt, limits, cfg.MaxQueryIntoFuture) - if err == errEmptyTimeRange { - return storage.NoopQuerier(), nil - } else if err != nil { - return nil, err - } - + return storage.QueryableFunc(func(mint, maxt int64) (storage.Querier, error) { q := querier{ - ctx: ctx, + now: time.Now(), mint: mint, maxt: maxt, chunkIterFn: chunkIterFn, @@ -269,30 +254,8 @@ func NewQueryable(distributor QueryableWithFilter, stores []QueryableWithFilter, limits: limits, maxQueryIntoFuture: cfg.MaxQueryIntoFuture, queryStoreForLabels: cfg.QueryStoreForLabels, - } - - dqr, err := distributor.Querier(ctx, mint, maxt) - if err != nil { - return nil, err - } - - q.metadataQuerier = dqr - - if distributor.UseQueryable(now, mint, maxt) { - q.queriers = append(q.queriers, dqr) - } - - for _, s := range stores { - if !s.UseQueryable(now, mint, maxt) { - continue - } - - cqr, err := s.Querier(ctx, mint, maxt) - if err != nil { - return nil, err - } - - q.queriers = append(q.queriers, cqr) + distributor: distributor, + stores: stores, } return q, nil @@ -300,26 +263,68 @@ func NewQueryable(distributor QueryableWithFilter, stores []QueryableWithFilter, } type querier struct { - // used for labels and metadata queries - metadataQuerier storage.Querier - - // used for selecting series - queriers []storage.Querier - chunkIterFn chunkIteratorFunc - ctx context.Context + now time.Time mint, maxt int64 tombstonesLoader purger.TombstonesLoader limits *validation.Overrides maxQueryIntoFuture time.Duration queryStoreForLabels bool + distributor QueryableWithFilter + stores []QueryableWithFilter +} + +func (q querier) setupFromCtx(ctx context.Context) (context.Context, string, int64, int64, storage.Querier, []storage.Querier, error) { + userID, err := tenant.TenantID(ctx) + if err != nil { + return ctx, userID, 0, 0, nil, nil, err + } + + ctx = limiter.AddQueryLimiterToContext(ctx, limiter.NewQueryLimiter(q.limits.MaxFetchedSeriesPerQuery(userID), q.limits.MaxFetchedChunkBytesPerQuery(userID), q.limits.MaxChunksPerQuery(userID), q.limits.MaxFetchedDataBytesPerQuery(userID))) + + mint, maxt, err := validateQueryTimeRange(ctx, userID, q.mint, q.maxt, q.limits, q.maxQueryIntoFuture) + if err != nil { + return ctx, userID, 0, 0, nil, nil, err + } + + dqr, err := q.distributor.Querier(mint, maxt) + if err != nil { + return ctx, userID, 0, 0, nil, nil, err + } + metadataQuerier := dqr + + queriers := make([]storage.Querier, 0) + if q.distributor.UseQueryable(q.now, mint, maxt) { + queriers = append(queriers, dqr) + } + + for _, s := range q.stores { + if !s.UseQueryable(q.now, mint, maxt) { + continue + } + + cqr, err := s.Querier(mint, maxt) + if err != nil { + return ctx, userID, 0, 0, nil, nil, err + } + + queriers = append(queriers, cqr) + } + return ctx, userID, mint, maxt, metadataQuerier, queriers, nil } // Select implements storage.Querier interface. // The bool passed is ignored because the series is always sorted. -func (q querier) Select(sortSeries bool, sp *storage.SelectHints, matchers ...*labels.Matcher) storage.SeriesSet { - log, ctx := spanlogger.New(q.ctx, "querier.Select") +func (q querier) Select(ctx context.Context, sortSeries bool, sp *storage.SelectHints, matchers ...*labels.Matcher) storage.SeriesSet { + ctx, userID, mint, maxt, metadataQuerier, queriers, err := q.setupFromCtx(ctx) + if err == errEmptyTimeRange { + return storage.EmptySeriesSet() + } else if err != nil { + return storage.ErrSeriesSet(err) + } + + log, ctx := spanlogger.New(ctx, "querier.Select") defer log.Span.Finish() if sp != nil { @@ -327,8 +332,14 @@ func (q querier) Select(sortSeries bool, sp *storage.SelectHints, matchers ...*l } if sp == nil { + mint, maxt, err = validateQueryTimeRange(ctx, userID, mint, maxt, q.limits, q.maxQueryIntoFuture) + if err == errEmptyTimeRange { + return storage.EmptySeriesSet() + } else if err != nil { + return storage.ErrSeriesSet(err) + } // if SelectHints is null, rely on minT, maxT of querier to scope in range for Select stmt - sp = &storage.SelectHints{Start: q.mint, End: q.maxt} + sp = &storage.SelectHints{Start: mint, End: maxt} } else if sp.Func == "series" && !q.queryStoreForLabels { // Else if the querier receives a 'series' query, it means only metadata is needed. // Here we expect that metadataQuerier querier will handle that. @@ -337,12 +348,7 @@ func (q querier) Select(sortSeries bool, sp *storage.SelectHints, matchers ...*l // In this case, the query time range has already been validated when the querier has been // created. - return q.metadataQuerier.Select(true, sp, matchers...) - } - - userID, err := tenant.TenantID(ctx) - if err != nil { - return storage.ErrSeriesSet(err) + return metadataQuerier.Select(ctx, true, sp, matchers...) } // Validate query time range. Even if the time range has already been validated when we created @@ -363,7 +369,7 @@ func (q querier) Select(sortSeries bool, sp *storage.SelectHints, matchers ...*l // For series queries without specifying the start time, we prefer to // only query ingesters and not to query maxQueryLength to avoid OOM kill. if sp.Func == "series" && startMs == 0 { - return q.metadataQuerier.Select(true, sp, matchers...) + return metadataQuerier.Select(ctx, true, sp, matchers...) } startTime := model.Time(startMs) @@ -382,8 +388,8 @@ func (q querier) Select(sortSeries bool, sp *storage.SelectHints, matchers ...*l return storage.ErrSeriesSet(err) } - if len(q.queriers) == 1 { - seriesSet := q.queriers[0].Select(sortSeries, sp, matchers...) + if len(queriers) == 1 { + seriesSet := queriers[0].Select(ctx, sortSeries, sp, matchers...) if tombstones.Len() != 0 { seriesSet = series.NewDeletedSeriesSet(seriesSet, tombstones, model.Interval{Start: startTime, End: endTime}) @@ -392,16 +398,16 @@ func (q querier) Select(sortSeries bool, sp *storage.SelectHints, matchers ...*l return seriesSet } - sets := make(chan storage.SeriesSet, len(q.queriers)) - for _, querier := range q.queriers { + sets := make(chan storage.SeriesSet, len(queriers)) + for _, querier := range queriers { go func(querier storage.Querier) { // We should always select sorted here as we will need to merge the series - sets <- querier.Select(true, sp, matchers...) + sets <- querier.Select(ctx, true, sp, matchers...) }(querier) } var result []storage.SeriesSet - for range q.queriers { + for range queriers { select { case set := <-sets: result = append(result, set) @@ -422,88 +428,99 @@ func (q querier) Select(sortSeries bool, sp *storage.SelectHints, matchers ...*l } // LabelValues implements storage.Querier. -func (q querier) LabelValues(name string, matchers ...*labels.Matcher) ([]string, storage.Warnings, error) { +func (q querier) LabelValues(ctx context.Context, name string, matchers ...*labels.Matcher) ([]string, annotations.Annotations, error) { + ctx, _, _, _, metadataQuerier, queriers, err := q.setupFromCtx(ctx) + if err == errEmptyTimeRange { + return nil, nil, nil + } else if err != nil { + return nil, nil, err + } if !q.queryStoreForLabels { - return q.metadataQuerier.LabelValues(name, matchers...) + return metadataQuerier.LabelValues(ctx, name, matchers...) } - if len(q.queriers) == 1 { - return q.queriers[0].LabelValues(name, matchers...) + if len(queriers) == 1 { + return queriers[0].LabelValues(ctx, name, matchers...) } var ( - g, _ = errgroup.WithContext(q.ctx) + g, _ = errgroup.WithContext(ctx) sets = [][]string{} - warnings = storage.Warnings(nil) + warnings = annotations.Annotations(nil) resMtx sync.Mutex ) - for _, querier := range q.queriers { + for _, querier := range queriers { // Need to reassign as the original variable will change and can't be relied on in a goroutine. querier := querier g.Go(func() error { // NB: Values are sorted in Cortex already. - myValues, myWarnings, err := querier.LabelValues(name, matchers...) + myValues, myWarnings, err := querier.LabelValues(ctx, name, matchers...) if err != nil { return err } resMtx.Lock() sets = append(sets, myValues) - warnings = append(warnings, myWarnings...) + warnings.Merge(myWarnings) resMtx.Unlock() return nil }) } - err := g.Wait() - if err != nil { + if err := g.Wait(); err != nil { return nil, nil, err } return strutil.MergeSlices(sets...), warnings, nil } -func (q querier) LabelNames(matchers ...*labels.Matcher) ([]string, storage.Warnings, error) { +func (q querier) LabelNames(ctx context.Context, matchers ...*labels.Matcher) ([]string, annotations.Annotations, error) { + ctx, _, _, _, metadataQuerier, queriers, err := q.setupFromCtx(ctx) + if err == errEmptyTimeRange { + return nil, nil, nil + } else if err != nil { + return nil, nil, err + } + if !q.queryStoreForLabels { - return q.metadataQuerier.LabelNames(matchers...) + return metadataQuerier.LabelNames(ctx, matchers...) } - if len(q.queriers) == 1 { - return q.queriers[0].LabelNames(matchers...) + if len(queriers) == 1 { + return queriers[0].LabelNames(ctx, matchers...) } var ( - g, _ = errgroup.WithContext(q.ctx) + g, _ = errgroup.WithContext(ctx) sets = [][]string{} - warnings = storage.Warnings(nil) + warnings = annotations.Annotations(nil) resMtx sync.Mutex ) - for _, querier := range q.queriers { + for _, querier := range queriers { // Need to reassign as the original variable will change and can't be relied on in a goroutine. querier := querier g.Go(func() error { // NB: Names are sorted in Cortex already. - myNames, myWarnings, err := querier.LabelNames(matchers...) + myNames, myWarnings, err := querier.LabelNames(ctx, matchers...) if err != nil { return err } resMtx.Lock() sets = append(sets, myNames) - warnings = append(warnings, myWarnings...) + warnings.Merge(myWarnings) resMtx.Unlock() return nil }) } - err := g.Wait() - if err != nil { + if err := g.Wait(); err != nil { return nil, nil, err } @@ -578,7 +595,7 @@ func (s *sliceSeriesSet) Err() error { return nil } -func (s *sliceSeriesSet) Warnings() storage.Warnings { +func (s *sliceSeriesSet) Warnings() annotations.Annotations { return nil } diff --git a/pkg/querier/querier_test.go b/pkg/querier/querier_test.go index d2da0cc3fa..9d1882257a 100644 --- a/pkg/querier/querier_test.go +++ b/pkg/querier/querier_test.go @@ -17,6 +17,7 @@ import ( "github.com/prometheus/prometheus/scrape" "github.com/prometheus/prometheus/storage" "github.com/prometheus/prometheus/tsdb" + "github.com/prometheus/prometheus/util/annotations" v1 "github.com/prometheus/prometheus/web/api/v1" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" @@ -55,9 +56,9 @@ type wrappedQuerier struct { selectCallsArgs [][]interface{} } -func (q *wrappedQuerier) Select(sortSeries bool, hints *storage.SelectHints, matchers ...*labels.Matcher) storage.SeriesSet { +func (q *wrappedQuerier) Select(ctx context.Context, sortSeries bool, hints *storage.SelectHints, matchers ...*labels.Matcher) storage.SeriesSet { q.selectCallsArgs = append(q.selectCallsArgs, []interface{}{sortSeries, hints, matchers}) - return q.Querier.Select(sortSeries, hints, matchers...) + return q.Querier.Select(ctx, sortSeries, hints, matchers...) } type wrappedSampleAndChunkQueryable struct { @@ -65,8 +66,8 @@ type wrappedSampleAndChunkQueryable struct { queriers []*wrappedQuerier } -func (q *wrappedSampleAndChunkQueryable) Querier(ctx context.Context, mint, maxt int64) (storage.Querier, error) { - querier, err := q.QueryableWithFilter.Querier(ctx, mint, maxt) +func (q *wrappedSampleAndChunkQueryable) Querier(mint, maxt int64) (storage.Querier, error) { + querier, err := q.QueryableWithFilter.Querier(mint, maxt) wQuerier := &wrappedQuerier{Querier: querier} q.queriers = append(q.queriers, wQuerier) return wQuerier, err @@ -424,7 +425,7 @@ func mockTSDB(t *testing.T, labels []labels.Labels, mint model.Time, samples int require.NoError(t, app.Commit()) - return storage.QueryableFunc(func(ctx context.Context, mint, maxt int64) (storage.Querier, error) { + return storage.QueryableFunc(func(mint, maxt int64) (storage.Querier, error) { return tsdb.NewBlockQuerier(head, mint, maxt) }), rSamples } @@ -865,7 +866,7 @@ func TestQuerier_ValidateQueryTimeRange_MaxQueryLookback(t *testing.T) { distributor.On("MetricsForLabelMatchersStream", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return([]metric.Metric{}, nil) queryable, _, _ := New(cfg, overrides, distributor, queryables, purger.NewNoopTombstonesLoader(), nil, log.NewNopLogger()) - q, err := queryable.Querier(ctx, util.TimeToMillis(testData.queryStartTime), util.TimeToMillis(testData.queryEndTime)) + q, err := queryable.Querier(util.TimeToMillis(testData.queryStartTime), util.TimeToMillis(testData.queryEndTime)) require.NoError(t, err) // We apply the validation here again since when initializing querier we change the start/end time, @@ -883,7 +884,7 @@ func TestQuerier_ValidateQueryTimeRange_MaxQueryLookback(t *testing.T) { } matcher := labels.MustNewMatcher(labels.MatchEqual, labels.MetricName, "test") - set := q.Select(false, hints, matcher) + set := q.Select(ctx, false, hints, matcher) require.False(t, set.Next()) // Expected to be empty. require.NoError(t, set.Err()) @@ -906,10 +907,10 @@ func TestQuerier_ValidateQueryTimeRange_MaxQueryLookback(t *testing.T) { distributor.On("LabelNamesStream", mock.Anything, mock.Anything, mock.Anything).Return([]string{}, nil) queryable, _, _ := New(cfg, overrides, distributor, queryables, purger.NewNoopTombstonesLoader(), nil, log.NewNopLogger()) - q, err := queryable.Querier(ctx, util.TimeToMillis(testData.queryStartTime), util.TimeToMillis(testData.queryEndTime)) + q, err := queryable.Querier(util.TimeToMillis(testData.queryStartTime), util.TimeToMillis(testData.queryEndTime)) require.NoError(t, err) - _, _, err = q.LabelNames() + _, _, err = q.LabelNames(ctx) require.NoError(t, err) if !testData.expectedSkipped { @@ -934,10 +935,10 @@ func TestQuerier_ValidateQueryTimeRange_MaxQueryLookback(t *testing.T) { distributor.On("MetricsForLabelMatchersStream", mock.Anything, mock.Anything, mock.Anything, matchers).Return([]metric.Metric{}, nil) queryable, _, _ := New(cfg, overrides, distributor, queryables, purger.NewNoopTombstonesLoader(), nil, log.NewNopLogger()) - q, err := queryable.Querier(ctx, util.TimeToMillis(testData.queryStartTime), util.TimeToMillis(testData.queryEndTime)) + q, err := queryable.Querier(util.TimeToMillis(testData.queryStartTime), util.TimeToMillis(testData.queryEndTime)) require.NoError(t, err) - _, _, err = q.LabelNames(matchers...) + _, _, err = q.LabelNames(ctx, matchers...) require.NoError(t, err) if !testData.expectedSkipped { @@ -961,10 +962,10 @@ func TestQuerier_ValidateQueryTimeRange_MaxQueryLookback(t *testing.T) { distributor.On("LabelValuesForLabelNameStream", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return([]string{}, nil) queryable, _, _ := New(cfg, overrides, distributor, queryables, purger.NewNoopTombstonesLoader(), nil, log.NewNopLogger()) - q, err := queryable.Querier(ctx, util.TimeToMillis(testData.queryStartTime), util.TimeToMillis(testData.queryEndTime)) + q, err := queryable.Querier(util.TimeToMillis(testData.queryStartTime), util.TimeToMillis(testData.queryEndTime)) require.NoError(t, err) - _, _, err = q.LabelValues(labels.MetricName) + _, _, err = q.LabelValues(ctx, labels.MetricName) require.NoError(t, err) if !testData.expectedSkipped { @@ -1192,11 +1193,10 @@ func NewMockStoreQueryable(cfg Config, store mockStore) storage.Queryable { } func newMockStoreQueryable(store mockStore, chunkIteratorFunc chunkIteratorFunc) storage.Queryable { - return storage.QueryableFunc(func(ctx context.Context, mint, maxt int64) (storage.Querier, error) { + return storage.QueryableFunc(func(mint, maxt int64) (storage.Querier, error) { return &mockStoreQuerier{ store: store, chunkIteratorFunc: chunkIteratorFunc, - ctx: ctx, mint: mint, maxt: maxt, }, nil @@ -1206,14 +1206,13 @@ func newMockStoreQueryable(store mockStore, chunkIteratorFunc chunkIteratorFunc) type mockStoreQuerier struct { store mockStore chunkIteratorFunc chunkIteratorFunc - ctx context.Context mint, maxt int64 } // Select implements storage.Querier interface. // The bool passed is ignored because the series is always sorted. -func (q *mockStoreQuerier) Select(_ bool, sp *storage.SelectHints, matchers ...*labels.Matcher) storage.SeriesSet { - userID, err := tenant.TenantID(q.ctx) +func (q *mockStoreQuerier) Select(ctx context.Context, _ bool, sp *storage.SelectHints, matchers ...*labels.Matcher) storage.SeriesSet { + userID, err := tenant.TenantID(ctx) if err != nil { return storage.ErrSeriesSet(err) } @@ -1230,7 +1229,7 @@ func (q *mockStoreQuerier) Select(_ bool, sp *storage.SelectHints, matchers ...* return storage.EmptySeriesSet() } - chunks, err := q.store.Get(q.ctx, userID, model.Time(minT), model.Time(maxT), matchers...) + chunks, err := q.store.Get(ctx, userID, model.Time(minT), model.Time(maxT), matchers...) if err != nil { return storage.ErrSeriesSet(err) } @@ -1238,11 +1237,11 @@ func (q *mockStoreQuerier) Select(_ bool, sp *storage.SelectHints, matchers ...* return partitionChunks(chunks, q.mint, q.maxt, q.chunkIteratorFunc) } -func (q *mockStoreQuerier) LabelValues(name string, labels ...*labels.Matcher) ([]string, storage.Warnings, error) { +func (q *mockStoreQuerier) LabelValues(ctx context.Context, name string, labels ...*labels.Matcher) ([]string, annotations.Annotations, error) { return nil, nil, nil } -func (q *mockStoreQuerier) LabelNames(matchers ...*labels.Matcher) ([]string, storage.Warnings, error) { +func (q *mockStoreQuerier) LabelNames(ctx context.Context, matchers ...*labels.Matcher) ([]string, annotations.Annotations, error) { return nil, nil, nil } @@ -1426,7 +1425,7 @@ type mockQueryableWithFilter struct { useQueryableCalled bool } -func (m *mockQueryableWithFilter) Querier(_ context.Context, _, _ int64) (storage.Querier, error) { +func (m *mockQueryableWithFilter) Querier(_, _ int64) (storage.Querier, error) { return nil, nil } diff --git a/pkg/querier/remote_read.go b/pkg/querier/remote_read.go index 12a1251b44..aea82b8c54 100644 --- a/pkg/querier/remote_read.go +++ b/pkg/querier/remote_read.go @@ -42,7 +42,7 @@ func RemoteReadHandler(q storage.Queryable, logger log.Logger) http.Handler { return } - querier, err := q.Querier(ctx, int64(from), int64(to)) + querier, err := q.Querier(int64(from), int64(to)) if err != nil { errors <- err return @@ -52,7 +52,7 @@ func RemoteReadHandler(q storage.Queryable, logger log.Logger) http.Handler { Start: int64(from), End: int64(to), } - seriesSet := querier.Select(false, params, matchers...) + seriesSet := querier.Select(ctx, false, params, matchers...) resp.Results[i], err = seriesSetToQueryResponse(seriesSet) errors <- err }(i, qr) diff --git a/pkg/querier/remote_read_test.go b/pkg/querier/remote_read_test.go index c044c86ec4..13e8881a2b 100644 --- a/pkg/querier/remote_read_test.go +++ b/pkg/querier/remote_read_test.go @@ -15,6 +15,7 @@ import ( "github.com/prometheus/common/model" "github.com/prometheus/prometheus/model/labels" "github.com/prometheus/prometheus/storage" + "github.com/prometheus/prometheus/util/annotations" "github.com/stretchr/testify/require" "github.com/cortexproject/cortex/pkg/cortexpb" @@ -24,7 +25,7 @@ import ( func TestRemoteReadHandler(t *testing.T) { t.Parallel() - q := storage.QueryableFunc(func(ctx context.Context, mint, maxt int64) (storage.Querier, error) { + q := storage.QueryableFunc(func(mint, maxt int64) (storage.Querier, error) { return mockQuerier{ matrix: model.Matrix{ { @@ -91,18 +92,18 @@ type mockQuerier struct { matrix model.Matrix } -func (m mockQuerier) Select(sortSeries bool, sp *storage.SelectHints, matchers ...*labels.Matcher) storage.SeriesSet { +func (m mockQuerier) Select(ctx context.Context, sortSeries bool, sp *storage.SelectHints, matchers ...*labels.Matcher) storage.SeriesSet { if sp == nil { panic(fmt.Errorf("select params must be set")) } return series.MatrixToSeriesSet(sortSeries, m.matrix) } -func (m mockQuerier) LabelValues(name string, matchers ...*labels.Matcher) ([]string, storage.Warnings, error) { +func (m mockQuerier) LabelValues(ctx context.Context, name string, matchers ...*labels.Matcher) ([]string, annotations.Annotations, error) { return nil, nil, nil } -func (m mockQuerier) LabelNames(matchers ...*labels.Matcher) ([]string, storage.Warnings, error) { +func (m mockQuerier) LabelNames(ctx context.Context, matchers ...*labels.Matcher) ([]string, annotations.Annotations, error) { return nil, nil, nil } diff --git a/pkg/querier/series/series_set.go b/pkg/querier/series/series_set.go index d2c9270fc2..90e2b83f62 100644 --- a/pkg/querier/series/series_set.go +++ b/pkg/querier/series/series_set.go @@ -19,14 +19,15 @@ package series import ( "sort" - "github.com/cortexproject/cortex/pkg/prom1/storage/metric" - "github.com/cortexproject/cortex/pkg/purger" - "github.com/cortexproject/cortex/pkg/querier/iterators" - "github.com/prometheus/common/model" "github.com/prometheus/prometheus/model/labels" "github.com/prometheus/prometheus/storage" "github.com/prometheus/prometheus/tsdb/chunkenc" + "github.com/prometheus/prometheus/util/annotations" + + "github.com/cortexproject/cortex/pkg/prom1/storage/metric" + "github.com/cortexproject/cortex/pkg/purger" + "github.com/cortexproject/cortex/pkg/querier/iterators" ) // ConcreteSeriesSet implements storage.SeriesSet. @@ -64,7 +65,7 @@ func (c *ConcreteSeriesSet) Err() error { } // Warnings implements storage.SeriesSet. -func (c *ConcreteSeriesSet) Warnings() storage.Warnings { +func (c *ConcreteSeriesSet) Warnings() annotations.Annotations { return nil } @@ -232,7 +233,7 @@ func (d DeletedSeriesSet) Err() error { return d.seriesSet.Err() } -func (d DeletedSeriesSet) Warnings() storage.Warnings { +func (d DeletedSeriesSet) Warnings() annotations.Annotations { return nil } @@ -361,10 +362,10 @@ func (emptySeriesIterator) Err() error { type seriesSetWithWarnings struct { wrapped storage.SeriesSet - warnings storage.Warnings + warnings annotations.Annotations } -func NewSeriesSetWithWarnings(wrapped storage.SeriesSet, warnings storage.Warnings) storage.SeriesSet { +func NewSeriesSetWithWarnings(wrapped storage.SeriesSet, warnings annotations.Annotations) storage.SeriesSet { return seriesSetWithWarnings{ wrapped: wrapped, warnings: warnings, @@ -383,6 +384,7 @@ func (s seriesSetWithWarnings) Err() error { return s.wrapped.Err() } -func (s seriesSetWithWarnings) Warnings() storage.Warnings { - return append(s.wrapped.Warnings(), s.warnings...) +func (s seriesSetWithWarnings) Warnings() annotations.Annotations { + w := s.wrapped.Warnings() + return w.Merge(s.warnings) } diff --git a/pkg/querier/tenantfederation/merge_queryable.go b/pkg/querier/tenantfederation/merge_queryable.go index bb13e7ace4..c765f9f2d3 100644 --- a/pkg/querier/tenantfederation/merge_queryable.go +++ b/pkg/querier/tenantfederation/merge_queryable.go @@ -10,7 +10,7 @@ import ( "github.com/prometheus/prometheus/model/labels" "github.com/prometheus/prometheus/storage" "github.com/prometheus/prometheus/tsdb/chunkenc" - tsdb_errors "github.com/prometheus/prometheus/tsdb/errors" + "github.com/prometheus/prometheus/util/annotations" "github.com/weaveworks/common/user" "github.com/cortexproject/cortex/pkg/tenant" @@ -48,9 +48,8 @@ func tenantQuerierCallback(queryable storage.Queryable) MergeQuerierCallback { } var queriers = make([]storage.Querier, len(tenantIDs)) - for pos, tenantID := range tenantIDs { + for pos := range tenantIDs { q, err := queryable.Querier( - user.InjectOrgID(ctx, tenantID), mint, maxt, ) @@ -96,25 +95,13 @@ type mergeQueryable struct { // Querier returns a new mergeQuerier, which aggregates results from multiple // underlying queriers into a single result. -func (m *mergeQueryable) Querier(ctx context.Context, mint int64, maxt int64) (storage.Querier, error) { - // TODO: it's necessary to think how to override context inside querier - // to mark spans created inside querier as child of a span created inside - // methods of merged querier. - ids, queriers, err := m.callback(ctx, mint, maxt) - if err != nil { - return nil, err - } - - // by pass when only single querier is returned - if m.byPassWithSingleQuerier && len(queriers) == 1 { - return queriers[0], nil - } - +func (m *mergeQueryable) Querier(mint int64, maxt int64) (storage.Querier, error) { return &mergeQuerier{ - ctx: ctx, - idLabelName: m.idLabelName, - queriers: queriers, - ids: ids, + idLabelName: m.idLabelName, + mint: mint, + maxt: maxt, + byPassWithSingleQuerier: m.byPassWithSingleQuerier, + callback: m.callback, }, nil } @@ -125,10 +112,11 @@ func (m *mergeQueryable) Querier(ctx context.Context, mint int64, maxt int64) (s // the previous value is exposed through a new label prefixed with "original_". // This behaviour is not implemented recursively type mergeQuerier struct { - ctx context.Context - queriers []storage.Querier idLabelName string - ids []string + mint, maxt int64 + callback MergeQuerierCallback + + byPassWithSingleQuerier bool } // LabelValues returns all potential values for a label name. It is not safe @@ -136,15 +124,24 @@ type mergeQuerier struct { // For the label `idLabelName` it will return all the underlying ids available. // For the label "original_" + `idLabelName it will return all the values // of the underlying queriers for `idLabelName`. -func (m *mergeQuerier) LabelValues(name string, matchers ...*labels.Matcher) ([]string, storage.Warnings, error) { - log, _ := spanlogger.New(m.ctx, "mergeQuerier.LabelValues") +func (m *mergeQuerier) LabelValues(ctx context.Context, name string, matchers ...*labels.Matcher) ([]string, annotations.Annotations, error) { + ids, queriers, err := m.callback(ctx, m.mint, m.maxt) + if err != nil { + return nil, nil, err + } + + // by pass when only single querier is returned + if m.byPassWithSingleQuerier && len(queriers) == 1 { + return queriers[0].LabelValues(ctx, name, matchers...) + } + log, _ := spanlogger.New(ctx, "mergeQuerier.LabelValues") defer log.Span.Finish() - matchedTenants, filteredMatchers := filterValuesByMatchers(m.idLabelName, m.ids, matchers...) + matchedTenants, filteredMatchers := filterValuesByMatchers(m.idLabelName, ids, matchers...) if name == m.idLabelName { var labelValues = make([]string, 0, len(matchedTenants)) - for _, id := range m.ids { + for _, id := range ids { if _, matched := matchedTenants[id]; matched { labelValues = append(labelValues, id) } @@ -158,23 +155,32 @@ func (m *mergeQuerier) LabelValues(name string, matchers ...*labels.Matcher) ([] name = m.idLabelName } - return m.mergeDistinctStringSliceWithTenants(func(ctx context.Context, q storage.Querier) ([]string, storage.Warnings, error) { - return q.LabelValues(name, filteredMatchers...) - }, matchedTenants) + return m.mergeDistinctStringSliceWithTenants(ctx, func(ctx context.Context, q storage.Querier) ([]string, annotations.Annotations, error) { + return q.LabelValues(ctx, name, filteredMatchers...) + }, matchedTenants, ids, queriers) } // LabelNames returns all the unique label names present in the underlying // queriers. It also adds the `idLabelName` and if present in the original // results the original `idLabelName`. -func (m *mergeQuerier) LabelNames(matchers ...*labels.Matcher) ([]string, storage.Warnings, error) { - log, _ := spanlogger.New(m.ctx, "mergeQuerier.LabelNames") +func (m *mergeQuerier) LabelNames(ctx context.Context, matchers ...*labels.Matcher) ([]string, annotations.Annotations, error) { + ids, queriers, err := m.callback(ctx, m.mint, m.maxt) + if err != nil { + return nil, nil, err + } + + // by pass when only single querier is returned + if m.byPassWithSingleQuerier && len(queriers) == 1 { + return queriers[0].LabelNames(ctx, matchers...) + } + log, _ := spanlogger.New(ctx, "mergeQuerier.LabelNames") defer log.Span.Finish() - matchedTenants, filteredMatchers := filterValuesByMatchers(m.idLabelName, m.ids, matchers...) + matchedTenants, filteredMatchers := filterValuesByMatchers(m.idLabelName, ids, matchers...) - labelNames, warnings, err := m.mergeDistinctStringSliceWithTenants(func(ctx context.Context, q storage.Querier) ([]string, storage.Warnings, error) { - return q.LabelNames(filteredMatchers...) - }, matchedTenants) + labelNames, warnings, err := m.mergeDistinctStringSliceWithTenants(ctx, func(ctx context.Context, q storage.Querier) ([]string, annotations.Annotations, error) { + return q.LabelNames(ctx, filteredMatchers...) + }, matchedTenants, ids, queriers) if err != nil { return nil, nil, err } @@ -203,13 +209,13 @@ func (m *mergeQuerier) LabelNames(matchers ...*labels.Matcher) ([]string, storag return labelNames, warnings, nil } -type stringSliceFunc func(context.Context, storage.Querier) ([]string, storage.Warnings, error) +type stringSliceFunc func(context.Context, storage.Querier) ([]string, annotations.Annotations, error) type stringSliceFuncJob struct { querier storage.Querier id string result []string - warnings storage.Warnings + warnings annotations.Annotations } // mergeDistinctStringSliceWithTenants aggregates stringSliceFunc call @@ -217,10 +223,10 @@ type stringSliceFuncJob struct { // provided, all queriers are used. It removes duplicates and sorts the result. // It doesn't require the output of the stringSliceFunc to be sorted, as results // of LabelValues are not sorted. -func (m *mergeQuerier) mergeDistinctStringSliceWithTenants(f stringSliceFunc, tenants map[string]struct{}) ([]string, storage.Warnings, error) { +func (m *mergeQuerier) mergeDistinctStringSliceWithTenants(ctx context.Context, f stringSliceFunc, tenants map[string]struct{}, ids []string, queriers []storage.Querier) ([]string, annotations.Annotations, error) { var jobs []interface{} - for pos, id := range m.ids { + for pos, id := range ids { if tenants != nil { if _, matched := tenants[id]; !matched { continue @@ -228,11 +234,12 @@ func (m *mergeQuerier) mergeDistinctStringSliceWithTenants(f stringSliceFunc, te } jobs = append(jobs, &stringSliceFuncJob{ - querier: m.queriers[pos], - id: m.ids[pos], + querier: queriers[pos], + id: ids[pos], }) } + parentCtx := ctx run := func(ctx context.Context, jobIntf interface{}) error { job, ok := jobIntf.(*stringSliceFuncJob) if !ok { @@ -240,7 +247,9 @@ func (m *mergeQuerier) mergeDistinctStringSliceWithTenants(f stringSliceFunc, te } var err error - job.result, job.warnings, err = f(ctx, job.querier) + // Based on parent ctx here as we are using lazy querier. + newCtx := user.InjectOrgID(parentCtx, job.id) + job.result, job.warnings, err = f(newCtx, job.querier) if err != nil { return errors.Wrapf(err, "error querying %s %s", rewriteLabelName(m.idLabelName), job.id) } @@ -248,13 +257,13 @@ func (m *mergeQuerier) mergeDistinctStringSliceWithTenants(f stringSliceFunc, te return nil } - err := concurrency.ForEach(m.ctx, jobs, maxConcurrency, run) + err := concurrency.ForEach(ctx, jobs, maxConcurrency, run) if err != nil { return nil, nil, err } // aggregate warnings and deduplicate string results - var warnings storage.Warnings + var warnings annotations.Annotations resultMap := make(map[string]struct{}) for _, jobIntf := range jobs { job, ok := jobIntf.(*stringSliceFuncJob) @@ -267,7 +276,7 @@ func (m *mergeQuerier) mergeDistinctStringSliceWithTenants(f stringSliceFunc, te } for _, w := range job.warnings { - warnings = append(warnings, errors.Wrapf(w, "warning querying %s %s", rewriteLabelName(m.idLabelName), job.id)) + warnings.Add(errors.Wrapf(w, "warning querying %s %s", rewriteLabelName(m.idLabelName), job.id)) } } @@ -281,11 +290,7 @@ func (m *mergeQuerier) mergeDistinctStringSliceWithTenants(f stringSliceFunc, te // Close releases the resources of the Querier. func (m *mergeQuerier) Close() error { - errs := tsdb_errors.NewMulti() - for pos, id := range m.ids { - errs.Add(errors.Wrapf(m.queriers[pos].Close(), "failed to close querier for %s %s", rewriteLabelName(m.idLabelName), id)) - } - return errs.Err() + return nil } type selectJob struct { @@ -298,32 +303,45 @@ type selectJob struct { // `idLabelName` is matched on, it only considers those queriers // matching. The forwarded labelSelector is not containing those that operate // on `idLabelName`. -func (m *mergeQuerier) Select(sortSeries bool, hints *storage.SelectHints, matchers ...*labels.Matcher) storage.SeriesSet { - log, ctx := spanlogger.New(m.ctx, "mergeQuerier.Select") +func (m *mergeQuerier) Select(ctx context.Context, sortSeries bool, hints *storage.SelectHints, matchers ...*labels.Matcher) storage.SeriesSet { + ids, queriers, err := m.callback(ctx, m.mint, m.maxt) + if err != nil { + return storage.ErrSeriesSet(err) + } + + // by pass when only single querier is returned + if m.byPassWithSingleQuerier && len(queriers) == 1 { + return queriers[0].Select(ctx, sortSeries, hints, matchers...) + } + + log, ctx := spanlogger.New(ctx, "mergeQuerier.Select") defer log.Span.Finish() - matchedValues, filteredMatchers := filterValuesByMatchers(m.idLabelName, m.ids, matchers...) + matchedValues, filteredMatchers := filterValuesByMatchers(m.idLabelName, ids, matchers...) var jobs = make([]interface{}, len(matchedValues)) var seriesSets = make([]storage.SeriesSet, len(matchedValues)) var jobPos int - for labelPos := range m.ids { - if _, matched := matchedValues[m.ids[labelPos]]; !matched { + for labelPos := range ids { + if _, matched := matchedValues[ids[labelPos]]; !matched { continue } jobs[jobPos] = &selectJob{ pos: jobPos, - querier: m.queriers[labelPos], - id: m.ids[labelPos], + querier: queriers[labelPos], + id: ids[labelPos], } jobPos++ } + parentCtx := ctx run := func(ctx context.Context, jobIntf interface{}) error { job, ok := jobIntf.(*selectJob) if !ok { return fmt.Errorf("unexpected type %T", jobIntf) } + // Based on parent ctx here as we are using lazy querier. + newCtx := user.InjectOrgID(parentCtx, ids[job.pos]) seriesSets[job.pos] = &addLabelsSeriesSet{ - upstream: job.querier.Select(sortSeries, hints, filteredMatchers...), + upstream: job.querier.Select(newCtx, sortSeries, hints, filteredMatchers...), labels: labels.Labels{ { Name: m.idLabelName, @@ -334,8 +352,7 @@ func (m *mergeQuerier) Select(sortSeries bool, hints *storage.SelectHints, match return nil } - err := concurrency.ForEach(ctx, jobs, maxConcurrency, run) - if err != nil { + if err := concurrency.ForEach(ctx, jobs, maxConcurrency, run); err != nil { return storage.ErrSeriesSet(err) } @@ -417,9 +434,9 @@ func (m *addLabelsSeriesSet) Err() error { // A collection of warnings for the whole set. // Warnings could be return even iteration has not failed with error. -func (m *addLabelsSeriesSet) Warnings() storage.Warnings { +func (m *addLabelsSeriesSet) Warnings() annotations.Annotations { upstream := m.upstream.Warnings() - warnings := make(storage.Warnings, len(upstream)) + warnings := make(annotations.Annotations, len(upstream)) for pos := range upstream { warnings[pos] = errors.Wrapf(upstream[pos], "warning querying %s", labelsToString(m.labels)) } diff --git a/pkg/querier/tenantfederation/merge_queryable_test.go b/pkg/querier/tenantfederation/merge_queryable_test.go index 7beb5a20e6..e8aa04ea28 100644 --- a/pkg/querier/tenantfederation/merge_queryable_test.go +++ b/pkg/querier/tenantfederation/merge_queryable_test.go @@ -15,6 +15,7 @@ import ( "github.com/prometheus/common/model" "github.com/prometheus/prometheus/model/labels" "github.com/prometheus/prometheus/storage" + "github.com/prometheus/prometheus/util/annotations" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/weaveworks/common/user" @@ -42,36 +43,18 @@ type mockTenantQueryableWithFilter struct { // extraLabels are labels added to all series for all tenants. extraLabels []string // warningsByTenant are warnings that will be returned for queries of that tenant. - warningsByTenant map[string]storage.Warnings + warningsByTenant map[string]annotations.Annotations // queryErrByTenant is an error that will be returne for queries of that tenant. queryErrByTenant map[string]error } // Querier implements the storage.Queryable interface. -func (m *mockTenantQueryableWithFilter) Querier(ctx context.Context, _, _ int64) (storage.Querier, error) { - tenantIDs, err := tenant.TenantIDs(ctx) - if err != nil { - return nil, err - } - +func (m *mockTenantQueryableWithFilter) Querier(_, _ int64) (storage.Querier, error) { q := mockTenantQuerier{ - tenant: tenantIDs[0], - extraLabels: m.extraLabels, - ctx: ctx, - } - - // set warning if exists - if m.warningsByTenant != nil { - if w, ok := m.warningsByTenant[q.tenant]; ok { - q.warnings = append([]error(nil), w...) - } - } - - // set queryErr if exists - if m.queryErrByTenant != nil { - if err, ok := m.queryErrByTenant[q.tenant]; ok { - q.queryErr = err - } + extraLabels: m.extraLabels, + warnings: annotations.Annotations(nil), + warningsByTenant: m.warningsByTenant, + queryErrByTenant: m.queryErrByTenant, } return q, nil @@ -84,25 +67,28 @@ func (m *mockTenantQueryableWithFilter) UseQueryable(_ time.Time, _, _ int64) bo } type mockTenantQuerier struct { - tenant string extraLabels []string - warnings storage.Warnings + warnings annotations.Annotations queryErr error - ctx context.Context + + // warningsByTenant are warnings that will be returned for queries of that tenant. + warningsByTenant map[string]annotations.Annotations + // queryErrByTenant is an error that will be returne for queries of that tenant. + queryErrByTenant map[string]error } -func (m mockTenantQuerier) matrix() model.Matrix { +func (m mockTenantQuerier) matrix(tenant string) model.Matrix { matrix := model.Matrix{ &model.SampleStream{ Metric: model.Metric{ - "instance": "host1", - "tenant-" + model.LabelName(m.tenant): "static", + "instance": "host1", + "tenant-" + model.LabelName(tenant): "static", }, }, &model.SampleStream{ Metric: model.Metric{ - "instance": "host2." + model.LabelValue(m.tenant), + "instance": "host2." + model.LabelValue(tenant), }, }, } @@ -134,7 +120,7 @@ func metricMatches(m model.Metric, selector labels.Selector) bool { type mockSeriesSet struct { upstream storage.SeriesSet - warnings storage.Warnings + warnings annotations.Annotations queryErr error } @@ -155,17 +141,36 @@ func (m *mockSeriesSet) Err() error { // Warnings implements the storage.SeriesSet interface. It returns a collection of warnings for the whole set. // Warnings could be returned even if iteration has not failed with error. -func (m *mockSeriesSet) Warnings() storage.Warnings { +func (m *mockSeriesSet) Warnings() annotations.Annotations { return m.warnings } // Select implements the storage.Querier interface. -func (m mockTenantQuerier) Select(_ bool, sp *storage.SelectHints, matchers ...*labels.Matcher) storage.SeriesSet { - log, _ := spanlogger.New(m.ctx, "mockTenantQuerier.select") +func (m mockTenantQuerier) Select(ctx context.Context, _ bool, sp *storage.SelectHints, matchers ...*labels.Matcher) storage.SeriesSet { + tenantIDs, err := tenant.TenantIDs(ctx) + if err != nil { + return storage.ErrSeriesSet(err) + } + + // set warning if exists + if m.warningsByTenant != nil { + if w, ok := m.warningsByTenant[tenantIDs[0]]; ok { + m.warnings.Merge(w) + } + } + + // set queryErr if exists + if m.queryErrByTenant != nil { + if err, ok := m.queryErrByTenant[tenantIDs[0]]; ok { + m.queryErr = err + } + } + + log, _ := spanlogger.New(ctx, "mockTenantQuerier.select") defer log.Span.Finish() var matrix model.Matrix - for _, s := range m.matrix() { + for _, s := range m.matrix(tenantIDs[0]) { if metricMatches(s.Metric, matchers) { matrix = append(matrix, s) } @@ -180,9 +185,28 @@ func (m mockTenantQuerier) Select(_ bool, sp *storage.SelectHints, matchers ...* // LabelValues implements the storage.LabelQuerier interface. // The mockTenantQuerier returns all a sorted slice of all label values and does not support reducing the result set with matchers. -func (m mockTenantQuerier) LabelValues(name string, matchers ...*labels.Matcher) ([]string, storage.Warnings, error) { +func (m mockTenantQuerier) LabelValues(ctx context.Context, name string, matchers ...*labels.Matcher) ([]string, annotations.Annotations, error) { + tenantIDs, err := tenant.TenantIDs(ctx) + if err != nil { + return nil, nil, err + } + + // set warning if exists + if m.warningsByTenant != nil { + if w, ok := m.warningsByTenant[tenantIDs[0]]; ok { + m.warnings.Merge(w) + } + } + + // set queryErr if exists + if m.queryErrByTenant != nil { + if err, ok := m.queryErrByTenant[tenantIDs[0]]; ok { + m.queryErr = err + } + } + if len(matchers) > 0 { - m.warnings = append(m.warnings, errors.New(mockMatchersNotImplemented)) + m.warnings.Add(errors.New(mockMatchersNotImplemented)) } if m.queryErr != nil { @@ -190,7 +214,7 @@ func (m mockTenantQuerier) LabelValues(name string, matchers ...*labels.Matcher) } labelValues := make(map[string]struct{}) - for _, s := range m.matrix() { + for _, s := range m.matrix(tenantIDs[0]) { for k, v := range s.Metric { if k == model.LabelName(name) { labelValues[string(v)] = struct{}{} @@ -209,7 +233,25 @@ func (m mockTenantQuerier) LabelValues(name string, matchers ...*labels.Matcher) // It returns a sorted slice of all label names in the querier. // If only one matcher is provided with label Name=seriesWithLabelNames then the resulting set will have the values of that matchers pipe-split appended. // I.e. querying for {seriesWithLabelNames="foo|bar|baz"} will have as result [bar, baz, foo, ] -func (m mockTenantQuerier) LabelNames(matchers ...*labels.Matcher) ([]string, storage.Warnings, error) { +func (m mockTenantQuerier) LabelNames(ctx context.Context, matchers ...*labels.Matcher) ([]string, annotations.Annotations, error) { + tenantIDs, err := tenant.TenantIDs(ctx) + if err != nil { + return nil, nil, err + } + // set warning if exists + if m.warningsByTenant != nil { + if w, ok := m.warningsByTenant[tenantIDs[0]]; ok { + m.warnings.Merge(w) + } + } + + // set queryErr if exists + if m.queryErrByTenant != nil { + if err, ok := m.queryErrByTenant[tenantIDs[0]]; ok { + m.queryErr = err + } + } + var results []string if len(matchers) == 1 && matchers[0].Name == seriesWithLabelNames { @@ -218,7 +260,7 @@ func (m mockTenantQuerier) LabelNames(matchers ...*labels.Matcher) ([]string, st } results = strings.Split(matchers[0].Value, "|") } else if len(matchers) > 1 { - m.warnings = append(m.warnings, errors.New(mockMatchersNotImplemented)) + m.warnings.Add(errors.New(mockMatchersNotImplemented)) } if m.queryErr != nil { @@ -226,7 +268,7 @@ func (m mockTenantQuerier) LabelNames(matchers ...*labels.Matcher) ([]string, st } labelValues := make(map[string]struct{}) - for _, s := range m.matrix() { + for _, s := range m.matrix(tenantIDs[0]) { for k := range s.Metric { labelValues[string(k)] = struct{}{} } @@ -259,14 +301,8 @@ func (s *mergeQueryableScenario) init() (storage.Querier, error) { // initialize with default tenant label q := NewQueryable(&s.queryable, !s.doNotByPassSingleQuerier) - // inject tenants into context - ctx := context.Background() - if len(s.tenants) > 0 { - ctx = user.InjectOrgID(ctx, strings.Join(s.tenants, "|")) - } - // retrieve querier - return q.Querier(ctx, mint, maxt) + return q.Querier(mint, maxt) } // selectTestCase is the inputs and expected outputs of a call to Select. @@ -279,7 +315,7 @@ type selectTestCase struct { expectedSeriesCount int // expectedLabels is the expected label sets returned by a Select filtered by the Matchers in selector. expectedLabels []labels.Labels - // expectedWarnings is a slice of storage.Warnings messages expected when querying. + // expectedWarnings is a slice of annotations.Annotations messages expected when querying. expectedWarnings []string // expectedQueryErr is the error expected when querying. expectedQueryErr error @@ -299,7 +335,7 @@ type labelNamesTestCase struct { matchers []*labels.Matcher // expectedLabelNames are the expected label names returned from the queryable. expectedLabelNames []string - // expectedWarnings is a slice of storage.Warnings messages expected when querying. + // expectedWarnings is a slice of annotations.Annotations messages expected when querying. expectedWarnings []string // expectedQueryErr is the error expected when querying. expectedQueryErr error @@ -321,7 +357,7 @@ type labelValuesTestCase struct { matchers []*labels.Matcher // expectedLabelValues are the expected label values returned from the queryable. expectedLabelValues []string - // expectedWarnings is a slice of storage.Warnings messages expected when querying. + // expectedWarnings is a slice of annotations.Annotations messages expected when querying. expectedWarnings []string // expectedQueryErr is the error expected when querying. expectedQueryErr error @@ -338,10 +374,11 @@ func TestMergeQueryable_Querier(t *testing.T) { t.Parallel() queryable := &mockTenantQueryableWithFilter{} q := NewQueryable(queryable, false /* byPassWithSingleQuerier */) - // Create a context with no tenant specified. - ctx := context.Background() - _, err := q.Querier(ctx, mint, maxt) + querier, err := q.Querier(mint, maxt) + require.NoError(t, err) + + _, _, err = querier.LabelValues(context.Background(), "test") require.EqualError(t, err, user.ErrNoOrgID.Error()) }) } @@ -375,9 +412,9 @@ var ( name: "three tenants, two with warnings", tenants: []string{"team-a", "team-b", "team-c"}, queryable: mockTenantQueryableWithFilter{ - warningsByTenant: map[string]storage.Warnings{ - "team-b": storage.Warnings([]error{errors.New("don't like them")}), - "team-c": storage.Warnings([]error{errors.New("out of office")}), + warningsByTenant: map[string]annotations.Annotations{ + "team-b": annotations.New().Add(errors.New("don't like them")), + "team-c": annotations.New().Add(errors.New("out of office")), }, }, } @@ -522,11 +559,17 @@ func TestMergeQueryable_Select(t *testing.T) { querier, err := scenario.init() require.NoError(t, err) + // inject tenants into context + ctx := context.Background() + if len(scenario.tenants) > 0 { + ctx = user.InjectOrgID(ctx, strings.Join(scenario.tenants, "|")) + } + for _, tc := range scenario.selectTestCases { tc := tc t.Run(tc.name, func(t *testing.T) { t.Parallel() - seriesSet := querier.Select(true, &storage.SelectHints{Start: mint, End: maxt}, tc.matchers...) + seriesSet := querier.Select(ctx, true, &storage.SelectHints{Start: mint, End: maxt}, tc.matchers...) if tc.expectedQueryErr != nil { require.EqualError(t, seriesSet.Err(), tc.expectedQueryErr.Error()) @@ -675,9 +718,15 @@ func TestMergeQueryable_LabelNames(t *testing.T) { querier, err := scenario.init() require.NoError(t, err) + // inject tenants into context + ctx := context.Background() + if len(scenario.tenants) > 0 { + ctx = user.InjectOrgID(ctx, strings.Join(scenario.tenants, "|")) + } + t.Run(scenario.labelNamesTestCase.name, func(t *testing.T) { t.Parallel() - labelNames, warnings, err := querier.LabelNames(scenario.labelNamesTestCase.matchers...) + labelNames, warnings, err := querier.LabelNames(ctx, scenario.labelNamesTestCase.matchers...) if scenario.labelNamesTestCase.expectedQueryErr != nil { require.EqualError(t, err, scenario.labelNamesTestCase.expectedQueryErr.Error()) } else { @@ -854,11 +903,17 @@ func TestMergeQueryable_LabelValues(t *testing.T) { querier, err := scenario.init() require.NoError(t, err) + // inject tenants into context + ctx := context.Background() + if len(scenario.tenants) > 0 { + ctx = user.InjectOrgID(ctx, strings.Join(scenario.tenants, "|")) + } + for _, tc := range scenario.labelValuesTestCases { tc := tc t.Run(tc.name, func(t *testing.T) { t.Parallel() - actLabelValues, warnings, err := querier.LabelValues(tc.labelName, tc.matchers...) + actLabelValues, warnings, err := querier.LabelValues(ctx, tc.labelName, tc.matchers...) if tc.expectedQueryErr != nil { require.EqualError(t, err, tc.expectedQueryErr.Error()) } else { @@ -873,13 +928,14 @@ func TestMergeQueryable_LabelValues(t *testing.T) { } // assertEqualWarnings asserts that all the expected warning messages are present. -func assertEqualWarnings(t *testing.T, exp []string, act storage.Warnings) { +func assertEqualWarnings(t *testing.T, exp []string, act annotations.Annotations) { if len(exp) == 0 && len(act) == 0 { return } var actStrings = make([]string, len(act)) - for pos := range act { - actStrings[pos] = act[pos].Error() + warnings := act.AsErrors() + for pos := range warnings { + actStrings[pos] = warnings[pos].Error() } assert.ElementsMatch(t, exp, actStrings) } @@ -932,10 +988,10 @@ func TestTracingMergeQueryable(t *testing.T) { filter := mockTenantQueryableWithFilter{} q := NewQueryable(&filter, false) // retrieve querier if set - querier, err := q.Querier(ctx, mint, maxt) + querier, err := q.Querier(mint, maxt) require.NoError(t, err) - seriesSet := querier.Select(true, &storage.SelectHints{Start: mint, + seriesSet := querier.Select(ctx, true, &storage.SelectHints{Start: mint, End: maxt}) require.NoError(t, seriesSet.Err()) diff --git a/pkg/querier/timeseries_series_set.go b/pkg/querier/timeseries_series_set.go index 3001ea2096..4c7af48377 100644 --- a/pkg/querier/timeseries_series_set.go +++ b/pkg/querier/timeseries_series_set.go @@ -3,12 +3,13 @@ package querier import ( "sort" - "github.com/cortexproject/cortex/pkg/cortexpb" - "github.com/cortexproject/cortex/pkg/querier/iterators" - "github.com/prometheus/prometheus/model/labels" "github.com/prometheus/prometheus/storage" "github.com/prometheus/prometheus/tsdb/chunkenc" + "github.com/prometheus/prometheus/util/annotations" + + "github.com/cortexproject/cortex/pkg/cortexpb" + "github.com/cortexproject/cortex/pkg/querier/iterators" ) // timeSeriesSeriesSet is a wrapper around a cortexpb.TimeSeries slice to implement to SeriesSet interface @@ -43,7 +44,7 @@ func (t *timeSeriesSeriesSet) At() storage.Series { func (t *timeSeriesSeriesSet) Err() error { return nil } // Warnings implements storage.SeriesSet interface. -func (t *timeSeriesSeriesSet) Warnings() storage.Warnings { return nil } +func (t *timeSeriesSeriesSet) Warnings() annotations.Annotations { return nil } // timeseries is a type wrapper that implements the storage.Series interface type timeseries struct { diff --git a/pkg/ruler/ruler_test.go b/pkg/ruler/ruler_test.go index 32734cf751..92f92094c9 100644 --- a/pkg/ruler/ruler_test.go +++ b/pkg/ruler/ruler_test.go @@ -3,7 +3,7 @@ package ruler import ( "context" "fmt" - io "io" + "io" "math/rand" "net/http" "net/http/httptest" @@ -31,6 +31,7 @@ import ( "github.com/prometheus/prometheus/promql" promRules "github.com/prometheus/prometheus/rules" "github.com/prometheus/prometheus/storage" + "github.com/prometheus/prometheus/util/annotations" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" @@ -110,7 +111,7 @@ func (r ruleLimits) DisabledRuleGroups(userID string) validation.DisabledRuleGro } func newEmptyQueryable() storage.Queryable { - return storage.QueryableFunc(func(ctx context.Context, mint, maxt int64) (storage.Querier, error) { + return storage.QueryableFunc(func(mint, maxt int64) (storage.Querier, error) { return emptyQuerier{}, nil }) } @@ -118,11 +119,11 @@ func newEmptyQueryable() storage.Queryable { type emptyQuerier struct { } -func (e emptyQuerier) LabelValues(name string, matchers ...*labels.Matcher) ([]string, storage.Warnings, error) { +func (e emptyQuerier) LabelValues(ctx context.Context, name string, matchers ...*labels.Matcher) ([]string, annotations.Annotations, error) { return nil, nil, nil } -func (e emptyQuerier) LabelNames(matchers ...*labels.Matcher) ([]string, storage.Warnings, error) { +func (e emptyQuerier) LabelNames(ctx context.Context, matchers ...*labels.Matcher) ([]string, annotations.Annotations, error) { return nil, nil, nil } @@ -130,7 +131,7 @@ func (e emptyQuerier) Close() error { return nil } -func (e emptyQuerier) Select(sortSeries bool, hints *storage.SelectHints, matchers ...*labels.Matcher) storage.SeriesSet { +func (e emptyQuerier) Select(ctx context.Context, sortSeries bool, hints *storage.SelectHints, matchers ...*labels.Matcher) storage.SeriesSet { return storage.EmptySeriesSet() } @@ -141,12 +142,12 @@ func testQueryableFunc(querierTestConfig *querier.TestConfig, reg prometheus.Reg overrides, _ := validation.NewOverrides(querier.DefaultLimitsConfig(), nil) q, _, _ := querier.New(querierTestConfig.Cfg, overrides, querierTestConfig.Distributor, querierTestConfig.Stores, purger.NewNoopTombstonesLoader(), reg, logger) - return func(ctx context.Context, mint, maxt int64) (storage.Querier, error) { - return q.Querier(ctx, mint, maxt) + return func(mint, maxt int64) (storage.Querier, error) { + return q.Querier(mint, maxt) } } - return func(ctx context.Context, mint, maxt int64) (storage.Querier, error) { + return func(mint, maxt int64) (storage.Querier, error) { return storage.NoopQuerier(), nil } } diff --git a/pkg/storage/tsdb/index_cache.go b/pkg/storage/tsdb/index_cache.go index 796f5a291a..46947ebc95 100644 --- a/pkg/storage/tsdb/index_cache.go +++ b/pkg/storage/tsdb/index_cache.go @@ -4,6 +4,7 @@ import ( "flag" "fmt" "strings" + "time" "github.com/alecthomas/units" "github.com/go-kit/log" @@ -31,6 +32,8 @@ const ( IndexCacheBackendDefault = IndexCacheBackendInMemory defaultMaxItemSize = model.Bytes(128 * units.MiB) + + defaultTTL = 24 * time.Hour ) var ( @@ -178,7 +181,8 @@ func NewIndexCache(cfg IndexCacheConfig, logger log.Logger, registerer prometheu if err != nil { return nil, err } - cache, err := storecache.NewRemoteIndexCache(logger, c, nil, iReg) + // TODO(yeya24): expose TTL + cache, err := storecache.NewRemoteIndexCache(logger, c, nil, iReg, defaultTTL) if err != nil { return nil, err } @@ -189,7 +193,8 @@ func NewIndexCache(cfg IndexCacheConfig, logger log.Logger, registerer prometheu if err != nil { return nil, err } - cache, err := storecache.NewRemoteIndexCache(logger, c, nil, iReg) + // TODO(yeya24): expose TTL + cache, err := storecache.NewRemoteIndexCache(logger, c, nil, iReg, defaultTTL) if err != nil { return nil, err } diff --git a/pkg/storegateway/bucket_store_inmemory_server.go b/pkg/storegateway/bucket_store_inmemory_server.go index 195e191efa..d3902d5b23 100644 --- a/pkg/storegateway/bucket_store_inmemory_server.go +++ b/pkg/storegateway/bucket_store_inmemory_server.go @@ -5,7 +5,7 @@ import ( "github.com/gogo/protobuf/types" "github.com/pkg/errors" - "github.com/prometheus/prometheus/storage" + "github.com/prometheus/prometheus/util/annotations" "github.com/thanos-io/thanos/pkg/store/hintspb" "github.com/thanos-io/thanos/pkg/store/storepb" ) @@ -20,7 +20,7 @@ type bucketStoreSeriesServer struct { ctx context.Context SeriesSet []*storepb.Series - Warnings storage.Warnings + Warnings annotations.Annotations Hints hintspb.SeriesResponseHints } @@ -30,7 +30,7 @@ func newBucketStoreSeriesServer(ctx context.Context) *bucketStoreSeriesServer { func (s *bucketStoreSeriesServer) Send(r *storepb.SeriesResponse) error { if r.GetWarning() != "" { - s.Warnings = append(s.Warnings, errors.New(r.GetWarning())) + s.Warnings.Add(errors.New(r.GetWarning())) } if rawHints := r.GetHints(); rawHints != nil { diff --git a/pkg/storegateway/bucket_stores.go b/pkg/storegateway/bucket_stores.go index e5630a2c1a..228099e1af 100644 --- a/pkg/storegateway/bucket_stores.go +++ b/pkg/storegateway/bucket_stores.go @@ -588,6 +588,7 @@ func (u *BucketStores) getOrCreateStore(userID string) (*store.BucketStore, erro return u.cfg.BucketStore.EstimatedMaxSeriesSizeBytes }), store.WithLazyExpandedPostings(u.cfg.BucketStore.LazyExpandedPostingsEnabled), + store.WithDontResort(true), // Cortex doesn't need to resort series in store gateway. } if u.logLevel.String() == "debug" { bucketStoreOpts = append(bucketStoreOpts, store.WithDebugLogging()) diff --git a/pkg/storegateway/bucket_stores_test.go b/pkg/storegateway/bucket_stores_test.go index 1b9b488768..9d57d52d42 100644 --- a/pkg/storegateway/bucket_stores_test.go +++ b/pkg/storegateway/bucket_stores_test.go @@ -21,8 +21,8 @@ import ( "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/testutil" "github.com/prometheus/prometheus/model/labels" - "github.com/prometheus/prometheus/storage" "github.com/prometheus/prometheus/tsdb" + "github.com/prometheus/prometheus/util/annotations" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" @@ -37,13 +37,11 @@ import ( "google.golang.org/grpc/codes" "google.golang.org/grpc/metadata" - "github.com/cortexproject/cortex/pkg/storage/tsdb/bucketindex" - - cortex_testutil "github.com/cortexproject/cortex/pkg/storage/tsdb/testutil" - "github.com/cortexproject/cortex/pkg/storage/bucket" "github.com/cortexproject/cortex/pkg/storage/bucket/filesystem" cortex_tsdb "github.com/cortexproject/cortex/pkg/storage/tsdb" + "github.com/cortexproject/cortex/pkg/storage/tsdb/bucketindex" + cortex_testutil "github.com/cortexproject/cortex/pkg/storage/tsdb/testutil" "github.com/cortexproject/cortex/pkg/util" "github.com/cortexproject/cortex/pkg/util/flagext" ) @@ -594,7 +592,7 @@ func generateStorageBlock(t *testing.T, storageDir, userID string, metricName st require.NoError(t, db.Snapshot(userDir, true)) } -func querySeries(stores *BucketStores, userID, metricName string, minT, maxT int64) ([]*storepb.Series, storage.Warnings, error) { +func querySeries(stores *BucketStores, userID, metricName string, minT, maxT int64) ([]*storepb.Series, annotations.Annotations, error) { req := &storepb.SeriesRequest{ MinTime: minT, MaxTime: maxT, diff --git a/pkg/storegateway/gateway_test.go b/pkg/storegateway/gateway_test.go index f186063d63..b499f72aa3 100644 --- a/pkg/storegateway/gateway_test.go +++ b/pkg/storegateway/gateway_test.go @@ -1208,14 +1208,15 @@ func mockTSDB(t *testing.T, dir string, numSeries, numBlocks int, minT, maxT int db.DisableCompactions() step := (maxT - minT) / int64(numSeries) + ctx := context.Background() addSample := func(i int) { lbls := labels.Labels{labels.Label{Name: "series_id", Value: strconv.Itoa(i)}} - app := db.Appender(context.Background()) + app := db.Appender(ctx) _, err := app.Append(0, lbls, minT+(step*int64(i)), float64(i)) require.NoError(t, err) require.NoError(t, app.Commit()) - require.NoError(t, db.Compact()) + require.NoError(t, db.Compact(ctx)) } if numBlocks > 0 { i := 0 diff --git a/pkg/util/concurrency/runner.go b/pkg/util/concurrency/runner.go index 5f5078cc86..5151093b93 100644 --- a/pkg/util/concurrency/runner.go +++ b/pkg/util/concurrency/runner.go @@ -96,7 +96,7 @@ func ForEach(ctx context.Context, jobs []interface{}, concurrency int, jobFunc f return g.Wait() } -// CreateJobsFromStrings is an utility to create jobs from an slice of strings. +// CreateJobsFromStrings is a utility to create jobs from an slice of strings. func CreateJobsFromStrings(values []string) []interface{} { jobs := make([]interface{}, len(values)) for i := 0; i < len(values); i++ { diff --git a/vendor/github.com/hashicorp/consul/api/acl.go b/vendor/github.com/hashicorp/consul/api/acl.go index 93087cd31c..48d2e66ee9 100644 --- a/vendor/github.com/hashicorp/consul/api/acl.go +++ b/vendor/github.com/hashicorp/consul/api/acl.go @@ -272,6 +272,13 @@ type ACLAuthMethod struct { Partition string `json:",omitempty"` } +type ACLTokenFilterOptions struct { + AuthMethod string `json:",omitempty"` + Policy string `json:",omitempty"` + Role string `json:",omitempty"` + ServiceName string `json:",omitempty"` +} + func (m *ACLAuthMethod) MarshalJSON() ([]byte, error) { type Alias ACLAuthMethod exported := &struct { @@ -895,6 +902,44 @@ func (a *ACL) TokenList(q *QueryOptions) ([]*ACLTokenListEntry, *QueryMeta, erro return entries, qm, nil } +// TokenListFiltered lists all tokens that match the given filter options. +// The listing does not contain any SecretIDs as those may only be retrieved by a call to TokenRead. +func (a *ACL) TokenListFiltered(t ACLTokenFilterOptions, q *QueryOptions) ([]*ACLTokenListEntry, *QueryMeta, error) { + r := a.c.newRequest("GET", "/v1/acl/tokens") + r.setQueryOptions(q) + + if t.AuthMethod != "" { + r.params.Set("authmethod", t.AuthMethod) + } + if t.Policy != "" { + r.params.Set("policy", t.Policy) + } + if t.Role != "" { + r.params.Set("role", t.Role) + } + if t.ServiceName != "" { + r.params.Set("servicename", t.ServiceName) + } + + rtt, resp, err := a.c.doRequest(r) + if err != nil { + return nil, nil, err + } + defer closeResponseBody(resp) + if err := requireOK(resp); err != nil { + return nil, nil, err + } + qm := &QueryMeta{} + parseQueryMeta(resp, qm) + qm.RequestTime = rtt + + var entries []*ACLTokenListEntry + if err := decodeBody(resp, &entries); err != nil { + return nil, nil, err + } + return entries, qm, nil +} + // PolicyCreate will create a new policy. It is not allowed for the policy parameters // ID field to be set as this will be generated by Consul while processing the request. func (a *ACL) PolicyCreate(policy *ACLPolicy, q *WriteOptions) (*ACLPolicy, *WriteMeta, error) { diff --git a/vendor/github.com/hashicorp/consul/api/agent.go b/vendor/github.com/hashicorp/consul/api/agent.go index f45929cb5b..6775edf425 100644 --- a/vendor/github.com/hashicorp/consul/api/agent.go +++ b/vendor/github.com/hashicorp/consul/api/agent.go @@ -274,6 +274,8 @@ type MembersOpts struct { // Segment is the LAN segment to show members for. Setting this to the // AllSegments value above will show members in all segments. Segment string + + Filter string } // AgentServiceRegistration is used to register a new service @@ -343,6 +345,7 @@ type AgentServiceCheck struct { Method string `json:",omitempty"` Body string `json:",omitempty"` TCP string `json:",omitempty"` + TCPUseTLS bool `json:",omitempty"` UDP string `json:",omitempty"` Status string `json:",omitempty"` Notes string `json:",omitempty"` @@ -790,6 +793,10 @@ func (a *Agent) MembersOpts(opts MembersOpts) ([]*AgentMember, error) { r.params.Set("wan", "1") } + if opts.Filter != "" { + r.params.Set("filter", opts.Filter) + } + _, resp, err := a.c.doRequest(r) if err != nil { return nil, err diff --git a/vendor/github.com/hashicorp/consul/api/api.go b/vendor/github.com/hashicorp/consul/api/api.go index 1fe0c71b61..f62c0c5a1b 100644 --- a/vendor/github.com/hashicorp/consul/api/api.go +++ b/vendor/github.com/hashicorp/consul/api/api.go @@ -1000,6 +1000,19 @@ func (r *request) toHTTP() (*http.Request, error) { return nil, err } + // validate that socket communications that do not use the host, detect + // slashes in the host name and replace it with local host. + // this is required since go started validating req.host in 1.20.6 and 1.19.11. + // prior to that they would strip out the slashes for you. They removed that + // behavior and added more strict validation as part of a CVE. + // This issue is being tracked by the Go team: + // https://github.com/golang/go/issues/61431 + // If there is a resolution in this issue, we will remove this code. + // In the time being, this is the accepted workaround. + if strings.HasPrefix(r.url.Host, "/") { + r.url.Host = "localhost" + } + req.URL.Host = r.url.Host req.URL.Scheme = r.url.Scheme req.Host = r.url.Host diff --git a/vendor/github.com/hashicorp/consul/api/config_entry.go b/vendor/github.com/hashicorp/consul/api/config_entry.go index 125619b55d..405e92ef27 100644 --- a/vendor/github.com/hashicorp/consul/api/config_entry.go +++ b/vendor/github.com/hashicorp/consul/api/config_entry.go @@ -42,7 +42,6 @@ const ( BuiltinAWSLambdaExtension string = "builtin/aws/lambda" BuiltinExtAuthzExtension string = "builtin/ext-authz" BuiltinLuaExtension string = "builtin/lua" - BuiltinLocalRatelimitExtension string = "builtin/http/localratelimit" BuiltinPropertyOverrideExtension string = "builtin/property-override" BuiltinWasmExtension string = "builtin/wasm" // BuiltinValidateExtension should not be exposed directly or accepted as a valid configured diff --git a/vendor/github.com/hashicorp/consul/api/config_entry_jwt_provider.go b/vendor/github.com/hashicorp/consul/api/config_entry_jwt_provider.go index e27974af3e..270f0d5641 100644 --- a/vendor/github.com/hashicorp/consul/api/config_entry_jwt_provider.go +++ b/vendor/github.com/hashicorp/consul/api/config_entry_jwt_provider.go @@ -7,6 +7,14 @@ import ( "time" ) +const ( + DiscoveryTypeStrictDNS ClusterDiscoveryType = "STRICT_DNS" + DiscoveryTypeStatic ClusterDiscoveryType = "STATIC" + DiscoveryTypeLogicalDNS ClusterDiscoveryType = "LOGICAL_DNS" + DiscoveryTypeEDS ClusterDiscoveryType = "EDS" + DiscoveryTypeOriginalDST ClusterDiscoveryType = "ORIGINAL_DST" +) + type JWTProviderConfigEntry struct { // Kind is the kind of configuration entry and must be "jwt-provider". Kind string `json:",omitempty"` @@ -188,6 +196,71 @@ type RemoteJWKS struct { // // There is no retry by default. RetryPolicy *JWKSRetryPolicy `json:",omitempty" alias:"retry_policy"` + + // JWKSCluster defines how the specified Remote JWKS URI is to be fetched. + JWKSCluster *JWKSCluster `json:",omitempty" alias:"jwks_cluster"` +} + +type JWKSCluster struct { + // DiscoveryType refers to the service discovery type to use for resolving the cluster. + // + // This defaults to STRICT_DNS. + // Other options include STATIC, LOGICAL_DNS, EDS or ORIGINAL_DST. + DiscoveryType ClusterDiscoveryType `json:",omitempty" alias:"discovery_type"` + + // TLSCertificates refers to the data containing certificate authority certificates to use + // in verifying a presented peer certificate. + // If not specified and a peer certificate is presented it will not be verified. + // + // Must be either CaCertificateProviderInstance or TrustedCA. + TLSCertificates *JWKSTLSCertificate `json:",omitempty" alias:"tls_certificates"` + + // The timeout for new network connections to hosts in the cluster. + // If not set, a default value of 5s will be used. + ConnectTimeout time.Duration `json:",omitempty" alias:"connect_timeout"` +} + +type ClusterDiscoveryType string + +// JWKSTLSCertificate refers to the data containing certificate authority certificates to use +// in verifying a presented peer certificate. +// If not specified and a peer certificate is presented it will not be verified. +// +// Must be either CaCertificateProviderInstance or TrustedCA. +type JWKSTLSCertificate struct { + // CaCertificateProviderInstance Certificate provider instance for fetching TLS certificates. + CaCertificateProviderInstance *JWKSTLSCertProviderInstance `json:",omitempty" alias:"ca_certificate_provider_instance"` + + // TrustedCA defines TLS certificate data containing certificate authority certificates + // to use in verifying a presented peer certificate. + // + // Exactly one of Filename, EnvironmentVariable, InlineString or InlineBytes must be specified. + TrustedCA *JWKSTLSCertTrustedCA `json:",omitempty" alias:"trusted_ca"` +} + +// JWKSTLSCertTrustedCA defines TLS certificate data containing certificate authority certificates +// to use in verifying a presented peer certificate. +// +// Exactly one of Filename, EnvironmentVariable, InlineString or InlineBytes must be specified. +type JWKSTLSCertTrustedCA struct { + Filename string `json:",omitempty" alias:"filename"` + EnvironmentVariable string `json:",omitempty" alias:"environment_variable"` + InlineString string `json:",omitempty" alias:"inline_string"` + InlineBytes []byte `json:",omitempty" alias:"inline_bytes"` +} + +type JWKSTLSCertProviderInstance struct { + // InstanceName refers to the certificate provider instance name + // + // The default value is "default". + InstanceName string `json:",omitempty" alias:"instance_name"` + + // CertificateName is used to specify certificate instances or types. For example, "ROOTCA" to specify + // a root-certificate (validation context) or "example.com" to specify a certificate for a + // particular domain. + // + // The default value is the empty string. + CertificateName string `json:",omitempty" alias:"certificate_name"` } type JWKSRetryPolicy struct { diff --git a/vendor/github.com/hashicorp/consul/api/health.go b/vendor/github.com/hashicorp/consul/api/health.go index 932317fdb0..a023002046 100644 --- a/vendor/github.com/hashicorp/consul/api/health.go +++ b/vendor/github.com/hashicorp/consul/api/health.go @@ -67,6 +67,7 @@ type HealthCheckDefinition struct { TLSServerName string TLSSkipVerify bool TCP string + TCPUseTLS bool UDP string GRPC string OSService string diff --git a/vendor/github.com/hashicorp/consul/api/operator_audit.go b/vendor/github.com/hashicorp/consul/api/operator_audit.go new file mode 100644 index 0000000000..5240d38a70 --- /dev/null +++ b/vendor/github.com/hashicorp/consul/api/operator_audit.go @@ -0,0 +1,40 @@ +// Copyright (c) HashiCorp, Inc. +// SPDX-License-Identifier: MPL-2.0 + +// The /v1/operator/audit-hash endpoint is available only in Consul Enterprise and +// interact with its audit logging subsystem. + +package api + +type AuditHashRequest struct { + Input string +} + +type AuditHashResponse struct { + Hash string +} + +func (op *Operator) AuditHash(a *AuditHashRequest, q *QueryOptions) (*AuditHashResponse, error) { + r := op.c.newRequest("POST", "/v1/operator/audit-hash") + r.setQueryOptions(q) + r.obj = a + + rtt, resp, err := op.c.doRequest(r) + if err != nil { + return nil, err + } + defer closeResponseBody(resp) + if err := requireOK(resp); err != nil { + return nil, err + } + + wm := &WriteMeta{} + wm.RequestTime = rtt + + var out AuditHashResponse + if err := decodeBody(resp, &out); err != nil { + return nil, err + } + + return &out, nil +} diff --git a/vendor/github.com/hashicorp/consul/api/operator_raft.go b/vendor/github.com/hashicorp/consul/api/operator_raft.go index 393d6fb3c5..d72c00c97b 100644 --- a/vendor/github.com/hashicorp/consul/api/operator_raft.go +++ b/vendor/github.com/hashicorp/consul/api/operator_raft.go @@ -28,6 +28,9 @@ type RaftServer struct { // it's a non-voting server, which will be added in a future release of // Consul. Voter bool + + // LastIndex is the last log index this server has a record of in its Raft log. + LastIndex uint64 } // RaftConfiguration is returned when querying for the current Raft configuration. diff --git a/vendor/github.com/hashicorp/consul/api/operator_usage.go b/vendor/github.com/hashicorp/consul/api/operator_usage.go index e47d4b53e0..8977449ddd 100644 --- a/vendor/github.com/hashicorp/consul/api/operator_usage.go +++ b/vendor/github.com/hashicorp/consul/api/operator_usage.go @@ -10,6 +10,7 @@ type Usage struct { // ServiceUsage contains information about the number of services and service instances for a datacenter. type ServiceUsage struct { + Nodes int Services int ServiceInstances int ConnectServiceInstances map[string]int diff --git a/vendor/github.com/hashicorp/consul/api/prepared_query.go b/vendor/github.com/hashicorp/consul/api/prepared_query.go index bb40e6a7fd..8ebc852f3a 100644 --- a/vendor/github.com/hashicorp/consul/api/prepared_query.go +++ b/vendor/github.com/hashicorp/consul/api/prepared_query.go @@ -32,11 +32,11 @@ type QueryFailoverTarget struct { // Partition specifies a partition to try during failover // Note: Partition are available only in Consul Enterprise - Partition string + Partition string `json:",omitempty"` // Namespace specifies a namespace to try during failover // Note: Namespaces are available only in Consul Enterprise - Namespace string + Namespace string `json:",omitempty"` } // QueryDNSOptions controls settings when query results are served over DNS. diff --git a/vendor/github.com/prometheus/prometheus/config/config.go b/vendor/github.com/prometheus/prometheus/config/config.go index 7f7595dcdf..7824780c34 100644 --- a/vendor/github.com/prometheus/prometheus/config/config.go +++ b/vendor/github.com/prometheus/prometheus/config/config.go @@ -819,6 +819,7 @@ type AlertmanagerConfig struct { ServiceDiscoveryConfigs discovery.Configs `yaml:"-"` HTTPClientConfig config.HTTPClientConfig `yaml:",inline"` + SigV4Config *sigv4.SigV4Config `yaml:"sigv4,omitempty"` // The URL scheme to use when talking to Alertmanagers. Scheme string `yaml:"scheme,omitempty"` @@ -854,6 +855,13 @@ func (c *AlertmanagerConfig) UnmarshalYAML(unmarshal func(interface{}) error) er return err } + httpClientConfigAuthEnabled := c.HTTPClientConfig.BasicAuth != nil || + c.HTTPClientConfig.Authorization != nil || c.HTTPClientConfig.OAuth2 != nil + + if httpClientConfigAuthEnabled && c.SigV4Config != nil { + return fmt.Errorf("at most one of basic_auth, authorization, oauth2, & sigv4 must be configured") + } + // Check for users putting URLs in target groups. if len(c.RelabelConfigs) == 0 { if err := checkStaticTargets(c.ServiceDiscoveryConfigs); err != nil { diff --git a/vendor/github.com/prometheus/prometheus/discovery/manager.go b/vendor/github.com/prometheus/prometheus/discovery/manager.go index 8b304a0faf..4d6027691f 100644 --- a/vendor/github.com/prometheus/prometheus/discovery/manager.go +++ b/vendor/github.com/prometheus/prometheus/discovery/manager.go @@ -180,11 +180,9 @@ func (m *Manager) Providers() []*Provider { // Run starts the background processing. func (m *Manager) Run() error { go m.sender() - for range m.ctx.Done() { - m.cancelDiscoverers() - return m.ctx.Err() - } - return nil + <-m.ctx.Done() + m.cancelDiscoverers() + return m.ctx.Err() } // SyncCh returns a read only channel used by all the clients to receive target updates. diff --git a/vendor/github.com/prometheus/prometheus/model/histogram/float_histogram.go b/vendor/github.com/prometheus/prometheus/model/histogram/float_histogram.go index d3f013935c..41873278cb 100644 --- a/vendor/github.com/prometheus/prometheus/model/histogram/float_histogram.go +++ b/vendor/github.com/prometheus/prometheus/model/histogram/float_histogram.go @@ -15,6 +15,7 @@ package histogram import ( "fmt" + "math" "strings" ) @@ -130,6 +131,55 @@ func (h *FloatHistogram) String() string { return sb.String() } +// TestExpression returns the string representation of this histogram as it is used in the internal PromQL testing +// framework as well as in promtool rules unit tests. +// The syntax is described in https://prometheus.io/docs/prometheus/latest/configuration/unit_testing_rules/#series +func (h *FloatHistogram) TestExpression() string { + var res []string + m := h.Copy() + + m.Compact(math.MaxInt) // Compact to reduce the number of positive and negative spans to 1. + + if m.Schema != 0 { + res = append(res, fmt.Sprintf("schema:%d", m.Schema)) + } + if m.Count != 0 { + res = append(res, fmt.Sprintf("count:%g", m.Count)) + } + if m.Sum != 0 { + res = append(res, fmt.Sprintf("sum:%g", m.Sum)) + } + if m.ZeroCount != 0 { + res = append(res, fmt.Sprintf("z_bucket:%g", m.ZeroCount)) + } + if m.ZeroThreshold != 0 { + res = append(res, fmt.Sprintf("z_bucket_w:%g", m.ZeroThreshold)) + } + + addBuckets := func(kind, bucketsKey, offsetKey string, buckets []float64, spans []Span) []string { + if len(spans) > 1 { + panic(fmt.Sprintf("histogram with multiple %s spans not supported", kind)) + } + for _, span := range spans { + if span.Offset != 0 { + res = append(res, fmt.Sprintf("%s:%d", offsetKey, span.Offset)) + } + } + + var bucketStr []string + for _, bucket := range buckets { + bucketStr = append(bucketStr, fmt.Sprintf("%g", bucket)) + } + if len(bucketStr) > 0 { + res = append(res, fmt.Sprintf("%s:[%s]", bucketsKey, strings.Join(bucketStr, " "))) + } + return res + } + res = addBuckets("positive", "buckets", "offset", m.PositiveBuckets, m.PositiveSpans) + res = addBuckets("negative", "n_buckets", "n_offset", m.NegativeBuckets, m.NegativeSpans) + return "{{" + strings.Join(res, " ") + "}}" +} + // ZeroBucket returns the zero bucket. func (h *FloatHistogram) ZeroBucket() Bucket[float64] { return Bucket[float64]{ @@ -159,7 +209,7 @@ func (h *FloatHistogram) Mul(factor float64) *FloatHistogram { return h } -// Div works like Scale but divides instead of multiplies. +// Div works like Mul but divides instead of multiplies. // When dividing by 0, everything will be set to Inf. func (h *FloatHistogram) Div(scalar float64) *FloatHistogram { h.ZeroCount /= scalar @@ -218,23 +268,17 @@ func (h *FloatHistogram) Add(other *FloatHistogram) *FloatHistogram { h.Count += other.Count h.Sum += other.Sum - // TODO(beorn7): If needed, this can be optimized by inspecting the - // spans in other and create missing buckets in h in batches. - var iInSpan, index int32 - for iSpan, iBucket, it := -1, -1, other.floatBucketIterator(true, h.ZeroThreshold, h.Schema); it.Next(); { - b := it.At() - h.PositiveSpans, h.PositiveBuckets, iSpan, iBucket, iInSpan = addBucket( - b, h.PositiveSpans, h.PositiveBuckets, iSpan, iBucket, iInSpan, index, - ) - index = b.Index - } - for iSpan, iBucket, it := -1, -1, other.floatBucketIterator(false, h.ZeroThreshold, h.Schema); it.Next(); { - b := it.At() - h.NegativeSpans, h.NegativeBuckets, iSpan, iBucket, iInSpan = addBucket( - b, h.NegativeSpans, h.NegativeBuckets, iSpan, iBucket, iInSpan, index, - ) - index = b.Index + otherPositiveSpans := other.PositiveSpans + otherPositiveBuckets := other.PositiveBuckets + otherNegativeSpans := other.NegativeSpans + otherNegativeBuckets := other.NegativeBuckets + if other.Schema != h.Schema { + otherPositiveSpans, otherPositiveBuckets = mergeToSchema(other.PositiveSpans, other.PositiveBuckets, other.Schema, h.Schema) + otherNegativeSpans, otherNegativeBuckets = mergeToSchema(other.NegativeSpans, other.NegativeBuckets, other.Schema, h.Schema) } + + h.PositiveSpans, h.PositiveBuckets = addBuckets(h.Schema, h.ZeroThreshold, false, h.PositiveSpans, h.PositiveBuckets, otherPositiveSpans, otherPositiveBuckets) + h.NegativeSpans, h.NegativeBuckets = addBuckets(h.Schema, h.ZeroThreshold, false, h.NegativeSpans, h.NegativeBuckets, otherNegativeSpans, otherNegativeBuckets) return h } @@ -245,25 +289,17 @@ func (h *FloatHistogram) Sub(other *FloatHistogram) *FloatHistogram { h.Count -= other.Count h.Sum -= other.Sum - // TODO(beorn7): If needed, this can be optimized by inspecting the - // spans in other and create missing buckets in h in batches. - var iInSpan, index int32 - for iSpan, iBucket, it := -1, -1, other.floatBucketIterator(true, h.ZeroThreshold, h.Schema); it.Next(); { - b := it.At() - b.Count *= -1 - h.PositiveSpans, h.PositiveBuckets, iSpan, iBucket, iInSpan = addBucket( - b, h.PositiveSpans, h.PositiveBuckets, iSpan, iBucket, iInSpan, index, - ) - index = b.Index - } - for iSpan, iBucket, it := -1, -1, other.floatBucketIterator(false, h.ZeroThreshold, h.Schema); it.Next(); { - b := it.At() - b.Count *= -1 - h.NegativeSpans, h.NegativeBuckets, iSpan, iBucket, iInSpan = addBucket( - b, h.NegativeSpans, h.NegativeBuckets, iSpan, iBucket, iInSpan, index, - ) - index = b.Index + otherPositiveSpans := other.PositiveSpans + otherPositiveBuckets := other.PositiveBuckets + otherNegativeSpans := other.NegativeSpans + otherNegativeBuckets := other.NegativeBuckets + if other.Schema != h.Schema { + otherPositiveSpans, otherPositiveBuckets = mergeToSchema(other.PositiveSpans, other.PositiveBuckets, other.Schema, h.Schema) + otherNegativeSpans, otherNegativeBuckets = mergeToSchema(other.NegativeSpans, other.NegativeBuckets, other.Schema, h.Schema) } + + h.PositiveSpans, h.PositiveBuckets = addBuckets(h.Schema, h.ZeroThreshold, true, h.PositiveSpans, h.PositiveBuckets, otherPositiveSpans, otherPositiveBuckets) + h.NegativeSpans, h.NegativeBuckets = addBuckets(h.Schema, h.ZeroThreshold, true, h.NegativeSpans, h.NegativeBuckets, otherNegativeSpans, otherNegativeBuckets) return h } @@ -298,103 +334,6 @@ func (h *FloatHistogram) Equals(h2 *FloatHistogram) bool { return true } -// addBucket takes the "coordinates" of the last bucket that was handled and -// adds the provided bucket after it. If a corresponding bucket exists, the -// count is added. If not, the bucket is inserted. The updated slices and the -// coordinates of the inserted or added-to bucket are returned. -func addBucket( - b Bucket[float64], - spans []Span, buckets []float64, - iSpan, iBucket int, - iInSpan, index int32, -) ( - newSpans []Span, newBuckets []float64, - newISpan, newIBucket int, newIInSpan int32, -) { - if iSpan == -1 { - // First add, check if it is before all spans. - if len(spans) == 0 || spans[0].Offset > b.Index { - // Add bucket before all others. - buckets = append(buckets, 0) - copy(buckets[1:], buckets) - buckets[0] = b.Count - if len(spans) > 0 && spans[0].Offset == b.Index+1 { - spans[0].Length++ - spans[0].Offset-- - return spans, buckets, 0, 0, 0 - } - spans = append(spans, Span{}) - copy(spans[1:], spans) - spans[0] = Span{Offset: b.Index, Length: 1} - if len(spans) > 1 { - // Convert the absolute offset in the formerly - // first span to a relative offset. - spans[1].Offset -= b.Index + 1 - } - return spans, buckets, 0, 0, 0 - } - if spans[0].Offset == b.Index { - // Just add to first bucket. - buckets[0] += b.Count - return spans, buckets, 0, 0, 0 - } - // We are behind the first bucket, so set everything to the - // first bucket and continue normally. - iSpan, iBucket, iInSpan = 0, 0, 0 - index = spans[0].Offset - } - deltaIndex := b.Index - index - for { - remainingInSpan := int32(spans[iSpan].Length) - iInSpan - if deltaIndex < remainingInSpan { - // Bucket is in current span. - iBucket += int(deltaIndex) - iInSpan += deltaIndex - buckets[iBucket] += b.Count - return spans, buckets, iSpan, iBucket, iInSpan - } - deltaIndex -= remainingInSpan - iBucket += int(remainingInSpan) - iSpan++ - if iSpan == len(spans) || deltaIndex < spans[iSpan].Offset { - // Bucket is in gap behind previous span (or there are no further spans). - buckets = append(buckets, 0) - copy(buckets[iBucket+1:], buckets[iBucket:]) - buckets[iBucket] = b.Count - if deltaIndex == 0 { - // Directly after previous span, extend previous span. - if iSpan < len(spans) { - spans[iSpan].Offset-- - } - iSpan-- - iInSpan = int32(spans[iSpan].Length) - spans[iSpan].Length++ - return spans, buckets, iSpan, iBucket, iInSpan - } - if iSpan < len(spans) && deltaIndex == spans[iSpan].Offset-1 { - // Directly before next span, extend next span. - iInSpan = 0 - spans[iSpan].Offset-- - spans[iSpan].Length++ - return spans, buckets, iSpan, iBucket, iInSpan - } - // No next span, or next span is not directly adjacent to new bucket. - // Add new span. - iInSpan = 0 - if iSpan < len(spans) { - spans[iSpan].Offset -= deltaIndex + 1 - } - spans = append(spans, Span{}) - copy(spans[iSpan+1:], spans[iSpan:]) - spans[iSpan] = Span{Length: 1, Offset: deltaIndex} - return spans, buckets, iSpan, iBucket, iInSpan - } - // Try start of next span. - deltaIndex -= spans[iSpan].Offset - iInSpan = 0 - } -} - // Compact eliminates empty buckets at the beginning and end of each span, then // merges spans that are consecutive or at most maxEmptyBuckets apart, and // finally splits spans that contain more consecutive empty buckets than @@ -1033,3 +972,133 @@ func mergeToSchema(originSpans []Span, originBuckets []float64, originSchema, ta return targetSpans, targetBuckets } + +// addBuckets adds the buckets described by spansB/bucketsB to the buckets described by spansA/bucketsA, +// creating missing buckets in spansA/bucketsA as needed. +// It returns the resulting spans/buckets (which must be used instead of the original spansA/bucketsA, +// although spansA/bucketsA might get modified by this function). +// All buckets must use the same provided schema. +// Buckets in spansB/bucketsB with an absolute upper limit ≤ threshold are ignored. +// If negative is true, the buckets in spansB/bucketsB are subtracted rather than added. +func addBuckets( + schema int32, threshold float64, negative bool, + spansA []Span, bucketsA []float64, + spansB []Span, bucketsB []float64, +) ([]Span, []float64) { + var ( + iSpan int = -1 + iBucket int = -1 + iInSpan int32 + indexA int32 + indexB int32 + bIdxB int + bucketB float64 + deltaIndex int32 + lowerThanThreshold = true + ) + + for _, spanB := range spansB { + indexB += spanB.Offset + for j := 0; j < int(spanB.Length); j++ { + if lowerThanThreshold && getBound(indexB, schema) <= threshold { + goto nextLoop + } + lowerThanThreshold = false + + bucketB = bucketsB[bIdxB] + if negative { + bucketB *= -1 + } + + if iSpan == -1 { + if len(spansA) == 0 || spansA[0].Offset > indexB { + // Add bucket before all others. + bucketsA = append(bucketsA, 0) + copy(bucketsA[1:], bucketsA) + bucketsA[0] = bucketB + if len(spansA) > 0 && spansA[0].Offset == indexB+1 { + spansA[0].Length++ + spansA[0].Offset-- + goto nextLoop + } else { + spansA = append(spansA, Span{}) + copy(spansA[1:], spansA) + spansA[0] = Span{Offset: indexB, Length: 1} + if len(spansA) > 1 { + // Convert the absolute offset in the formerly + // first span to a relative offset. + spansA[1].Offset -= indexB + 1 + } + goto nextLoop + } + } else if spansA[0].Offset == indexB { + // Just add to first bucket. + bucketsA[0] += bucketB + goto nextLoop + } + iSpan, iBucket, iInSpan = 0, 0, 0 + indexA = spansA[0].Offset + } + deltaIndex = indexB - indexA + for { + remainingInSpan := int32(spansA[iSpan].Length) - iInSpan + if deltaIndex < remainingInSpan { + // Bucket is in current span. + iBucket += int(deltaIndex) + iInSpan += deltaIndex + bucketsA[iBucket] += bucketB + break + } else { + deltaIndex -= remainingInSpan + iBucket += int(remainingInSpan) + iSpan++ + if iSpan == len(spansA) || deltaIndex < spansA[iSpan].Offset { + // Bucket is in gap behind previous span (or there are no further spans). + bucketsA = append(bucketsA, 0) + copy(bucketsA[iBucket+1:], bucketsA[iBucket:]) + bucketsA[iBucket] = bucketB + switch { + case deltaIndex == 0: + // Directly after previous span, extend previous span. + if iSpan < len(spansA) { + spansA[iSpan].Offset-- + } + iSpan-- + iInSpan = int32(spansA[iSpan].Length) + spansA[iSpan].Length++ + goto nextLoop + case iSpan < len(spansA) && deltaIndex == spansA[iSpan].Offset-1: + // Directly before next span, extend next span. + iInSpan = 0 + spansA[iSpan].Offset-- + spansA[iSpan].Length++ + goto nextLoop + default: + // No next span, or next span is not directly adjacent to new bucket. + // Add new span. + iInSpan = 0 + if iSpan < len(spansA) { + spansA[iSpan].Offset -= deltaIndex + 1 + } + spansA = append(spansA, Span{}) + copy(spansA[iSpan+1:], spansA[iSpan:]) + spansA[iSpan] = Span{Length: 1, Offset: deltaIndex} + goto nextLoop + } + } else { + // Try start of next span. + deltaIndex -= spansA[iSpan].Offset + iInSpan = 0 + } + } + } + + nextLoop: + indexA = indexB + indexB++ + bIdxB++ + } + } + + return spansA, bucketsA +} diff --git a/vendor/github.com/prometheus/prometheus/model/labels/labels.go b/vendor/github.com/prometheus/prometheus/model/labels/labels.go index 0c27e15c72..3dc3049b1c 100644 --- a/vendor/github.com/prometheus/prometheus/model/labels/labels.go +++ b/vendor/github.com/prometheus/prometheus/model/labels/labels.go @@ -19,6 +19,7 @@ import ( "bytes" "encoding/json" "strconv" + "strings" "github.com/cespare/xxhash/v2" "github.com/prometheus/common/model" @@ -362,7 +363,7 @@ func EmptyLabels() Labels { func New(ls ...Label) Labels { set := make(Labels, 0, len(ls)) set = append(set, ls...) - slices.SortFunc(set, func(a, b Label) bool { return a.Name < b.Name }) + slices.SortFunc(set, func(a, b Label) int { return strings.Compare(a.Name, b.Name) }) return set } @@ -386,7 +387,7 @@ func FromStrings(ss ...string) Labels { res = append(res, Label{Name: ss[i], Value: ss[i+1]}) } - slices.SortFunc(res, func(a, b Label) bool { return a.Name < b.Name }) + slices.SortFunc(res, func(a, b Label) int { return strings.Compare(a.Name, b.Name) }) return res } @@ -591,7 +592,7 @@ func (b *Builder) Labels() Labels { } if len(b.add) > 0 { // Base is already in order, so we only need to sort if we add to it. res = append(res, b.add...) - slices.SortFunc(res, func(a, b Label) bool { return a.Name < b.Name }) + slices.SortFunc(res, func(a, b Label) int { return strings.Compare(a.Name, b.Name) }) } return res } @@ -618,7 +619,7 @@ func (b *ScratchBuilder) Add(name, value string) { // Sort the labels added so far by name. func (b *ScratchBuilder) Sort() { - slices.SortFunc(b.add, func(a, b Label) bool { return a.Name < b.Name }) + slices.SortFunc(b.add, func(a, b Label) int { return strings.Compare(a.Name, b.Name) }) } // Assign is for when you already have a Labels which you want this ScratchBuilder to return. diff --git a/vendor/github.com/prometheus/prometheus/model/labels/labels_stringlabels.go b/vendor/github.com/prometheus/prometheus/model/labels/labels_stringlabels.go index a87545a26b..cc6bfcc700 100644 --- a/vendor/github.com/prometheus/prometheus/model/labels/labels_stringlabels.go +++ b/vendor/github.com/prometheus/prometheus/model/labels/labels_stringlabels.go @@ -20,6 +20,7 @@ import ( "encoding/json" "reflect" "strconv" + "strings" "unsafe" "github.com/cespare/xxhash/v2" @@ -412,7 +413,7 @@ func yoloBytes(s string) (b []byte) { // New returns a sorted Labels from the given labels. // The caller has to guarantee that all label names are unique. func New(ls ...Label) Labels { - slices.SortFunc(ls, func(a, b Label) bool { return a.Name < b.Name }) + slices.SortFunc(ls, func(a, b Label) int { return strings.Compare(a.Name, b.Name) }) size := labelsSize(ls) buf := make([]byte, size) marshalLabelsToSizedBuffer(ls, buf) @@ -671,7 +672,7 @@ func (b *Builder) Labels() Labels { return b.base } - slices.SortFunc(b.add, func(a, b Label) bool { return a.Name < b.Name }) + slices.SortFunc(b.add, func(a, b Label) int { return strings.Compare(a.Name, b.Name) }) slices.Sort(b.del) a, d := 0, 0 @@ -830,7 +831,7 @@ func (b *ScratchBuilder) Add(name, value string) { // Sort the labels added so far by name. func (b *ScratchBuilder) Sort() { - slices.SortFunc(b.add, func(a, b Label) bool { return a.Name < b.Name }) + slices.SortFunc(b.add, func(a, b Label) int { return strings.Compare(a.Name, b.Name) }) } // Assign is for when you already have a Labels which you want this ScratchBuilder to return. diff --git a/vendor/github.com/prometheus/prometheus/model/rulefmt/rulefmt.go b/vendor/github.com/prometheus/prometheus/model/rulefmt/rulefmt.go index 30b3face0d..03cbd8849c 100644 --- a/vendor/github.com/prometheus/prometheus/model/rulefmt/rulefmt.go +++ b/vendor/github.com/prometheus/prometheus/model/rulefmt/rulefmt.go @@ -173,17 +173,11 @@ func (r *RuleNode) Validate() (nodes []WrappedError) { }) } if r.Record.Value == "" && r.Alert.Value == "" { - if r.Record.Value == "0" { - nodes = append(nodes, WrappedError{ - err: fmt.Errorf("one of 'record' or 'alert' must be set"), - node: &r.Alert, - }) - } else { - nodes = append(nodes, WrappedError{ - err: fmt.Errorf("one of 'record' or 'alert' must be set"), - node: &r.Record, - }) - } + nodes = append(nodes, WrappedError{ + err: fmt.Errorf("one of 'record' or 'alert' must be set"), + node: &r.Record, + nodeAlt: &r.Alert, + }) } if r.Expr.Value == "" { diff --git a/vendor/github.com/prometheus/prometheus/model/textparse/openmetricsparse.go b/vendor/github.com/prometheus/prometheus/model/textparse/openmetricsparse.go index e0833636f1..5623e6833f 100644 --- a/vendor/github.com/prometheus/prometheus/model/textparse/openmetricsparse.go +++ b/vendor/github.com/prometheus/prometheus/model/textparse/openmetricsparse.go @@ -338,7 +338,7 @@ func (p *OpenMetricsParser) Next() (Entry, error) { var ts float64 // A float is enough to hold what we need for millisecond resolution. if ts, err = parseFloat(yoloString(p.l.buf()[1:])); err != nil { - return EntryInvalid, fmt.Errorf("%v while parsing: %q", err, p.l.b[p.start:p.l.i]) + return EntryInvalid, fmt.Errorf("%w while parsing: %q", err, p.l.b[p.start:p.l.i]) } if math.IsNaN(ts) || math.IsInf(ts, 0) { return EntryInvalid, fmt.Errorf("invalid timestamp %f", ts) @@ -391,7 +391,7 @@ func (p *OpenMetricsParser) parseComment() error { var ts float64 // A float is enough to hold what we need for millisecond resolution. if ts, err = parseFloat(yoloString(p.l.buf()[1:])); err != nil { - return fmt.Errorf("%v while parsing: %q", err, p.l.b[p.start:p.l.i]) + return fmt.Errorf("%w while parsing: %q", err, p.l.b[p.start:p.l.i]) } if math.IsNaN(ts) || math.IsInf(ts, 0) { return fmt.Errorf("invalid exemplar timestamp %f", ts) @@ -461,7 +461,7 @@ func (p *OpenMetricsParser) getFloatValue(t token, after string) (float64, error } val, err := parseFloat(yoloString(p.l.buf()[1:])) if err != nil { - return 0, fmt.Errorf("%v while parsing: %q", err, p.l.b[p.start:p.l.i]) + return 0, fmt.Errorf("%w while parsing: %q", err, p.l.b[p.start:p.l.i]) } // Ensure canonical NaN value. if math.IsNaN(p.exemplarVal) { diff --git a/vendor/github.com/prometheus/prometheus/model/textparse/promparse.go b/vendor/github.com/prometheus/prometheus/model/textparse/promparse.go index 94338a6660..04c295dd00 100644 --- a/vendor/github.com/prometheus/prometheus/model/textparse/promparse.go +++ b/vendor/github.com/prometheus/prometheus/model/textparse/promparse.go @@ -348,7 +348,7 @@ func (p *PromParser) Next() (Entry, error) { return EntryInvalid, p.parseError("expected value after metric", t2) } if p.val, err = parseFloat(yoloString(p.l.buf())); err != nil { - return EntryInvalid, fmt.Errorf("%v while parsing: %q", err, p.l.b[p.start:p.l.i]) + return EntryInvalid, fmt.Errorf("%w while parsing: %q", err, p.l.b[p.start:p.l.i]) } // Ensure canonical NaN value. if math.IsNaN(p.val) { @@ -361,7 +361,7 @@ func (p *PromParser) Next() (Entry, error) { case tTimestamp: p.hasTS = true if p.ts, err = strconv.ParseInt(yoloString(p.l.buf()), 10, 64); err != nil { - return EntryInvalid, fmt.Errorf("%v while parsing: %q", err, p.l.b[p.start:p.l.i]) + return EntryInvalid, fmt.Errorf("%w while parsing: %q", err, p.l.b[p.start:p.l.i]) } if t2 := p.nextToken(); t2 != tLinebreak { return EntryInvalid, p.parseError("expected next entry after timestamp", t2) diff --git a/vendor/github.com/prometheus/prometheus/model/textparse/protobufparse.go b/vendor/github.com/prometheus/prometheus/model/textparse/protobufparse.go index c111bb0657..fbb84a2bd3 100644 --- a/vendor/github.com/prometheus/prometheus/model/textparse/protobufparse.go +++ b/vendor/github.com/prometheus/prometheus/model/textparse/protobufparse.go @@ -56,6 +56,10 @@ type ProtobufParser struct { fieldsDone bool // true if no more fields of a Summary or (legacy) Histogram to be processed. redoClassic bool // true after parsing a native histogram if we need to parse it again as a classic histogram. + // exemplarReturned is set to true each time an exemplar has been + // returned, and set back to false upon each Next() call. + exemplarReturned bool + // state is marked by the entry we are processing. EntryInvalid implies // that we have to decode the next MetricFamily. state Entry @@ -293,8 +297,12 @@ func (p *ProtobufParser) Metric(l *labels.Labels) string { // Exemplar writes the exemplar of the current sample into the passed // exemplar. It returns if an exemplar exists or not. In case of a native // histogram, the legacy bucket section is still used for exemplars. To ingest -// all examplars, call the Exemplar method repeatedly until it returns false. +// all exemplars, call the Exemplar method repeatedly until it returns false. func (p *ProtobufParser) Exemplar(ex *exemplar.Exemplar) bool { + if p.exemplarReturned && p.state == EntrySeries { + // We only ever return one exemplar per (non-native-histogram) series. + return false + } m := p.mf.GetMetric()[p.metricPos] var exProto *dto.Exemplar switch p.mf.GetType() { @@ -335,6 +343,7 @@ func (p *ProtobufParser) Exemplar(ex *exemplar.Exemplar) bool { } p.builder.Sort() ex.Labels = p.builder.Labels() + p.exemplarReturned = true return true } @@ -342,6 +351,7 @@ func (p *ProtobufParser) Exemplar(ex *exemplar.Exemplar) bool { // text format parser). It returns (EntryInvalid, io.EOF) if no samples were // read. func (p *ProtobufParser) Next() (Entry, error) { + p.exemplarReturned = false switch p.state { case EntryInvalid: p.metricPos = 0 diff --git a/vendor/github.com/prometheus/prometheus/notifier/notifier.go b/vendor/github.com/prometheus/prometheus/notifier/notifier.go index 891372c43e..af55799337 100644 --- a/vendor/github.com/prometheus/prometheus/notifier/notifier.go +++ b/vendor/github.com/prometheus/prometheus/notifier/notifier.go @@ -32,6 +32,7 @@ import ( "github.com/prometheus/client_golang/prometheus" config_util "github.com/prometheus/common/config" "github.com/prometheus/common/model" + "github.com/prometheus/common/sigv4" "github.com/prometheus/common/version" "go.uber.org/atomic" @@ -640,6 +641,17 @@ func newAlertmanagerSet(cfg *config.AlertmanagerConfig, logger log.Logger, metri if err != nil { return nil, err } + t := client.Transport + + if cfg.SigV4Config != nil { + t, err = sigv4.NewSigV4RoundTripper(cfg.SigV4Config, client.Transport) + if err != nil { + return nil, err + } + } + + client.Transport = t + s := &alertmanagerSet{ client: client, cfg: cfg, diff --git a/vendor/github.com/prometheus/prometheus/promql/engine.go b/vendor/github.com/prometheus/prometheus/promql/engine.go index 816f20721e..161aa85acb 100644 --- a/vendor/github.com/prometheus/prometheus/promql/engine.go +++ b/vendor/github.com/prometheus/prometheus/promql/engine.go @@ -44,6 +44,7 @@ import ( "github.com/prometheus/prometheus/promql/parser" "github.com/prometheus/prometheus/storage" "github.com/prometheus/prometheus/tsdb/chunkenc" + "github.com/prometheus/prometheus/util/annotations" "github.com/prometheus/prometheus/util/stats" "github.com/prometheus/prometheus/util/zeropool" ) @@ -59,6 +60,11 @@ const ( maxInt64 = 9223372036854774784 // The smallest SampleValue that can be converted to an int64 without underflow. minInt64 = -9223372036854775808 + + // Max initial size for the pooled points slices. + // The getHPointSlice and getFPointSlice functions are called with an estimated size which often can be + // over-estimated. + maxPointsSliceSize = 5000 ) type engineMetrics struct { @@ -573,7 +579,7 @@ func (ng *Engine) newTestQuery(f func(context.Context) error) Query { // // At this point per query only one EvalStmt is evaluated. Alert and record // statements are not handled by the Engine. -func (ng *Engine) exec(ctx context.Context, q *query) (v parser.Value, ws storage.Warnings, err error) { +func (ng *Engine) exec(ctx context.Context, q *query) (v parser.Value, ws annotations.Annotations, err error) { ng.metrics.currentQueries.Inc() defer func() { ng.metrics.currentQueries.Dec() @@ -666,17 +672,17 @@ func durationMilliseconds(d time.Duration) int64 { } // execEvalStmt evaluates the expression of an evaluation statement for the given time range. -func (ng *Engine) execEvalStmt(ctx context.Context, query *query, s *parser.EvalStmt) (parser.Value, storage.Warnings, error) { +func (ng *Engine) execEvalStmt(ctx context.Context, query *query, s *parser.EvalStmt) (parser.Value, annotations.Annotations, error) { prepareSpanTimer, ctxPrepare := query.stats.GetSpanTimer(ctx, stats.QueryPreparationTime, ng.metrics.queryPrepareTime) mint, maxt := ng.findMinMaxTime(s) - querier, err := query.queryable.Querier(ctxPrepare, mint, maxt) + querier, err := query.queryable.Querier(mint, maxt) if err != nil { prepareSpanTimer.Finish() return nil, nil, err } defer querier.Close() - ng.populateSeries(querier, s) + ng.populateSeries(ctxPrepare, querier, s) prepareSpanTimer.Finish() // Modify the offset of vector and matrix selectors for the @ modifier @@ -890,7 +896,7 @@ func (ng *Engine) getLastSubqueryInterval(path []parser.Node) time.Duration { return interval } -func (ng *Engine) populateSeries(querier storage.Querier, s *parser.EvalStmt) { +func (ng *Engine) populateSeries(ctx context.Context, querier storage.Querier, s *parser.EvalStmt) { // Whenever a MatrixSelector is evaluated, evalRange is set to the corresponding range. // The evaluation of the VectorSelector inside then evaluates the given range and unsets // the variable. @@ -913,7 +919,7 @@ func (ng *Engine) populateSeries(querier storage.Querier, s *parser.EvalStmt) { } evalRange = 0 hints.By, hints.Grouping = extractGroupsFromPath(path) - n.UnexpandedSeriesSet = querier.Select(false, hints, n.LabelMatchers...) + n.UnexpandedSeriesSet = querier.Select(ctx, false, hints, n.LabelMatchers...) case *parser.MatrixSelector: evalRange = n.Range @@ -952,7 +958,7 @@ func extractGroupsFromPath(p []parser.Node) (bool, []string) { return false, nil } -func checkAndExpandSeriesSet(ctx context.Context, expr parser.Expr) (storage.Warnings, error) { +func checkAndExpandSeriesSet(ctx context.Context, expr parser.Expr) (annotations.Annotations, error) { switch e := expr.(type) { case *parser.MatrixSelector: return checkAndExpandSeriesSet(ctx, e.VectorSelector) @@ -967,7 +973,7 @@ func checkAndExpandSeriesSet(ctx context.Context, expr parser.Expr) (storage.War return nil, nil } -func expandSeriesSet(ctx context.Context, it storage.SeriesSet) (res []storage.Series, ws storage.Warnings, err error) { +func expandSeriesSet(ctx context.Context, it storage.SeriesSet) (res []storage.Series, ws annotations.Annotations, err error) { for it.Next() { select { case <-ctx.Done(): @@ -981,7 +987,7 @@ func expandSeriesSet(ctx context.Context, it storage.SeriesSet) (res []storage.S type errWithWarnings struct { err error - warnings storage.Warnings + warnings annotations.Annotations } func (e errWithWarnings) Error() string { return e.err.Error() } @@ -1016,7 +1022,7 @@ func (ev *evaluator) error(err error) { } // recover is the handler that turns panics into returns from the top level of evaluation. -func (ev *evaluator) recover(expr parser.Expr, ws *storage.Warnings, errp *error) { +func (ev *evaluator) recover(expr parser.Expr, ws *annotations.Annotations, errp *error) { e := recover() if e == nil { return @@ -1032,7 +1038,7 @@ func (ev *evaluator) recover(expr parser.Expr, ws *storage.Warnings, errp *error *errp = fmt.Errorf("unexpected error: %w", err) case errWithWarnings: *errp = err.err - *ws = append(*ws, err.warnings...) + ws.Merge(err.warnings) case error: *errp = err default: @@ -1040,7 +1046,7 @@ func (ev *evaluator) recover(expr parser.Expr, ws *storage.Warnings, errp *error } } -func (ev *evaluator) Eval(expr parser.Expr) (v parser.Value, ws storage.Warnings, err error) { +func (ev *evaluator) Eval(expr parser.Expr) (v parser.Value, ws annotations.Annotations, err error) { defer ev.recover(expr, &ws, &err) v, ws = ev.eval(expr) @@ -1109,19 +1115,19 @@ func (enh *EvalNodeHelper) DropMetricName(l labels.Labels) labels.Labels { // function call results. // The prepSeries function (if provided) can be used to prepare the helper // for each series, then passed to each call funcCall. -func (ev *evaluator) rangeEval(prepSeries func(labels.Labels, *EvalSeriesHelper), funcCall func([]parser.Value, [][]EvalSeriesHelper, *EvalNodeHelper) (Vector, storage.Warnings), exprs ...parser.Expr) (Matrix, storage.Warnings) { +func (ev *evaluator) rangeEval(prepSeries func(labels.Labels, *EvalSeriesHelper), funcCall func([]parser.Value, [][]EvalSeriesHelper, *EvalNodeHelper) (Vector, annotations.Annotations), exprs ...parser.Expr) (Matrix, annotations.Annotations) { numSteps := int((ev.endTimestamp-ev.startTimestamp)/ev.interval) + 1 matrixes := make([]Matrix, len(exprs)) origMatrixes := make([]Matrix, len(exprs)) originalNumSamples := ev.currentSamples - var warnings storage.Warnings + var warnings annotations.Annotations for i, e := range exprs { // Functions will take string arguments from the expressions, not the values. if e != nil && e.Type() != parser.ValueTypeString { // ev.currentSamples will be updated to the correct value within the ev.eval call. val, ws := ev.eval(e) - warnings = append(warnings, ws...) + warnings.Merge(ws) matrixes[i] = val.(Matrix) // Keep a copy of the original point slices so that they @@ -1188,41 +1194,24 @@ func (ev *evaluator) rangeEval(prepSeries func(labels.Labels, *EvalSeriesHelper) } for si, series := range matrixes[i] { - for _, point := range series.Floats { - if point.T == ts { - if ev.currentSamples < ev.maxSamples { - vectors[i] = append(vectors[i], Sample{Metric: series.Metric, F: point.F, T: ts}) - if prepSeries != nil { - bufHelpers[i] = append(bufHelpers[i], seriesHelpers[i][si]) - } - - // Move input vectors forward so we don't have to re-scan the same - // past points at the next step. - matrixes[i][si].Floats = series.Floats[1:] - ev.currentSamples++ - } else { - ev.error(ErrTooManySamples(env)) - } - } - break + switch { + case len(series.Floats) > 0 && series.Floats[0].T == ts: + vectors[i] = append(vectors[i], Sample{Metric: series.Metric, F: series.Floats[0].F, T: ts}) + // Move input vectors forward so we don't have to re-scan the same + // past points at the next step. + matrixes[i][si].Floats = series.Floats[1:] + case len(series.Histograms) > 0 && series.Histograms[0].T == ts: + vectors[i] = append(vectors[i], Sample{Metric: series.Metric, H: series.Histograms[0].H, T: ts}) + matrixes[i][si].Histograms = series.Histograms[1:] + default: + continue } - for _, point := range series.Histograms { - if point.T == ts { - if ev.currentSamples < ev.maxSamples { - vectors[i] = append(vectors[i], Sample{Metric: series.Metric, H: point.H, T: ts}) - if prepSeries != nil { - bufHelpers[i] = append(bufHelpers[i], seriesHelpers[i][si]) - } - - // Move input vectors forward so we don't have to re-scan the same - // past points at the next step. - matrixes[i][si].Histograms = series.Histograms[1:] - ev.currentSamples++ - } else { - ev.error(ErrTooManySamples(env)) - } - } - break + if prepSeries != nil { + bufHelpers[i] = append(bufHelpers[i], seriesHelpers[i][si]) + } + ev.currentSamples++ + if ev.currentSamples > ev.maxSamples { + ev.error(ErrTooManySamples(env)) } } args[i] = vectors[i] @@ -1233,7 +1222,7 @@ func (ev *evaluator) rangeEval(prepSeries func(labels.Labels, *EvalSeriesHelper) enh.Ts = ts result, ws := funcCall(args, bufHelpers, enh) enh.Out = result[:0] // Reuse result vector. - warnings = append(warnings, ws...) + warnings.Merge(ws) ev.currentSamples += len(result) // When we reset currentSamples to tempNumSamples during the next iteration of the loop it also @@ -1310,7 +1299,7 @@ func (ev *evaluator) rangeEval(prepSeries func(labels.Labels, *EvalSeriesHelper) // evalSubquery evaluates given SubqueryExpr and returns an equivalent // evaluated MatrixSelector in its place. Note that the Name and LabelMatchers are not set. -func (ev *evaluator) evalSubquery(subq *parser.SubqueryExpr) (*parser.MatrixSelector, int, storage.Warnings) { +func (ev *evaluator) evalSubquery(subq *parser.SubqueryExpr) (*parser.MatrixSelector, int, annotations.Annotations) { samplesStats := ev.samplesStats // Avoid double counting samples when running a subquery, those samples will be counted in later stage. ev.samplesStats = ev.samplesStats.NewChild() @@ -1343,7 +1332,7 @@ func (ev *evaluator) evalSubquery(subq *parser.SubqueryExpr) (*parser.MatrixSele } // eval evaluates the given expression as the given AST expression node requires. -func (ev *evaluator) eval(expr parser.Expr) (parser.Value, storage.Warnings) { +func (ev *evaluator) eval(expr parser.Expr) (parser.Value, annotations.Annotations) { // This is the top-level evaluation method. // Thus, we check for timeout/cancellation here. if err := contextDone(ev.ctx, "expression evaluation"); err != nil { @@ -1372,17 +1361,17 @@ func (ev *evaluator) eval(expr parser.Expr) (parser.Value, storage.Warnings) { param := unwrapStepInvariantExpr(e.Param) unwrapParenExpr(¶m) if s, ok := param.(*parser.StringLiteral); ok { - return ev.rangeEval(initSeries, func(v []parser.Value, sh [][]EvalSeriesHelper, enh *EvalNodeHelper) (Vector, storage.Warnings) { - return ev.aggregation(e.Op, sortedGrouping, e.Without, s.Val, v[0].(Vector), sh[0], enh), nil + return ev.rangeEval(initSeries, func(v []parser.Value, sh [][]EvalSeriesHelper, enh *EvalNodeHelper) (Vector, annotations.Annotations) { + return ev.aggregation(e, sortedGrouping, s.Val, v[0].(Vector), sh[0], enh) }, e.Expr) } - return ev.rangeEval(initSeries, func(v []parser.Value, sh [][]EvalSeriesHelper, enh *EvalNodeHelper) (Vector, storage.Warnings) { + return ev.rangeEval(initSeries, func(v []parser.Value, sh [][]EvalSeriesHelper, enh *EvalNodeHelper) (Vector, annotations.Annotations) { var param float64 if e.Param != nil { param = v[0].(Vector)[0].F } - return ev.aggregation(e.Op, sortedGrouping, e.Without, param, v[1].(Vector), sh[1], enh), nil + return ev.aggregation(e, sortedGrouping, param, v[1].(Vector), sh[1], enh) }, e.Param, e.Expr) case *parser.Call: @@ -1404,7 +1393,7 @@ func (ev *evaluator) eval(expr parser.Expr) (parser.Value, storage.Warnings) { var ( matrixArgIndex int matrixArg bool - warnings storage.Warnings + warnings annotations.Annotations ) for i := range e.Args { unwrapParenExpr(&e.Args[i]) @@ -1422,7 +1411,7 @@ func (ev *evaluator) eval(expr parser.Expr) (parser.Value, storage.Warnings) { // Replacing parser.SubqueryExpr with parser.MatrixSelector. val, totalSamples, ws := ev.evalSubquery(subq) e.Args[i] = val - warnings = append(warnings, ws...) + warnings.Merge(ws) defer func() { // subquery result takes space in the memory. Get rid of that at the end. val.VectorSelector.(*parser.VectorSelector).Series = nil @@ -1433,8 +1422,9 @@ func (ev *evaluator) eval(expr parser.Expr) (parser.Value, storage.Warnings) { } if !matrixArg { // Does not have a matrix argument. - return ev.rangeEval(nil, func(v []parser.Value, _ [][]EvalSeriesHelper, enh *EvalNodeHelper) (Vector, storage.Warnings) { - return call(v, e.Args, enh), warnings + return ev.rangeEval(nil, func(v []parser.Value, _ [][]EvalSeriesHelper, enh *EvalNodeHelper) (Vector, annotations.Annotations) { + vec, annos := call(v, e.Args, enh) + return vec, warnings.Merge(annos) }, e.Args...) } @@ -1448,7 +1438,7 @@ func (ev *evaluator) eval(expr parser.Expr) (parser.Value, storage.Warnings) { otherArgs[i] = val.(Matrix) otherInArgs[i] = Vector{Sample{}} inArgs[i] = otherInArgs[i] - warnings = append(warnings, ws...) + warnings.Merge(ws) } } @@ -1459,7 +1449,7 @@ func (ev *evaluator) eval(expr parser.Expr) (parser.Value, storage.Warnings) { selVS := sel.VectorSelector.(*parser.VectorSelector) ws, err := checkAndExpandSeriesSet(ev.ctx, sel) - warnings = append(warnings, ws...) + warnings.Merge(ws) if err != nil { ev.error(errWithWarnings{fmt.Errorf("expanding series: %w", err), warnings}) } @@ -1522,8 +1512,10 @@ func (ev *evaluator) eval(expr parser.Expr) (parser.Value, storage.Warnings) { inMatrix[0].Histograms = histograms enh.Ts = ts // Make the function call. - outVec := call(inArgs, e.Args, enh) + outVec, annos := call(inArgs, e.Args, enh) + warnings.Merge(annos) ev.samplesStats.IncrementSamplesAtStep(step, int64(len(floats)+len(histograms))) + enh.Out = outVec[:0] if len(outVec) > 0 { if outVec[0].H == nil { @@ -1626,7 +1618,7 @@ func (ev *evaluator) eval(expr parser.Expr) (parser.Value, storage.Warnings) { case *parser.BinaryExpr: switch lt, rt := e.LHS.Type(), e.RHS.Type(); { case lt == parser.ValueTypeScalar && rt == parser.ValueTypeScalar: - return ev.rangeEval(nil, func(v []parser.Value, _ [][]EvalSeriesHelper, enh *EvalNodeHelper) (Vector, storage.Warnings) { + return ev.rangeEval(nil, func(v []parser.Value, _ [][]EvalSeriesHelper, enh *EvalNodeHelper) (Vector, annotations.Annotations) { val := scalarBinop(e.Op, v[0].(Vector)[0].F, v[1].(Vector)[0].F) return append(enh.Out, Sample{F: val}), nil }, e.LHS, e.RHS) @@ -1639,36 +1631,36 @@ func (ev *evaluator) eval(expr parser.Expr) (parser.Value, storage.Warnings) { } switch e.Op { case parser.LAND: - return ev.rangeEval(initSignatures, func(v []parser.Value, sh [][]EvalSeriesHelper, enh *EvalNodeHelper) (Vector, storage.Warnings) { + return ev.rangeEval(initSignatures, func(v []parser.Value, sh [][]EvalSeriesHelper, enh *EvalNodeHelper) (Vector, annotations.Annotations) { return ev.VectorAnd(v[0].(Vector), v[1].(Vector), e.VectorMatching, sh[0], sh[1], enh), nil }, e.LHS, e.RHS) case parser.LOR: - return ev.rangeEval(initSignatures, func(v []parser.Value, sh [][]EvalSeriesHelper, enh *EvalNodeHelper) (Vector, storage.Warnings) { + return ev.rangeEval(initSignatures, func(v []parser.Value, sh [][]EvalSeriesHelper, enh *EvalNodeHelper) (Vector, annotations.Annotations) { return ev.VectorOr(v[0].(Vector), v[1].(Vector), e.VectorMatching, sh[0], sh[1], enh), nil }, e.LHS, e.RHS) case parser.LUNLESS: - return ev.rangeEval(initSignatures, func(v []parser.Value, sh [][]EvalSeriesHelper, enh *EvalNodeHelper) (Vector, storage.Warnings) { + return ev.rangeEval(initSignatures, func(v []parser.Value, sh [][]EvalSeriesHelper, enh *EvalNodeHelper) (Vector, annotations.Annotations) { return ev.VectorUnless(v[0].(Vector), v[1].(Vector), e.VectorMatching, sh[0], sh[1], enh), nil }, e.LHS, e.RHS) default: - return ev.rangeEval(initSignatures, func(v []parser.Value, sh [][]EvalSeriesHelper, enh *EvalNodeHelper) (Vector, storage.Warnings) { + return ev.rangeEval(initSignatures, func(v []parser.Value, sh [][]EvalSeriesHelper, enh *EvalNodeHelper) (Vector, annotations.Annotations) { return ev.VectorBinop(e.Op, v[0].(Vector), v[1].(Vector), e.VectorMatching, e.ReturnBool, sh[0], sh[1], enh), nil }, e.LHS, e.RHS) } case lt == parser.ValueTypeVector && rt == parser.ValueTypeScalar: - return ev.rangeEval(nil, func(v []parser.Value, _ [][]EvalSeriesHelper, enh *EvalNodeHelper) (Vector, storage.Warnings) { + return ev.rangeEval(nil, func(v []parser.Value, _ [][]EvalSeriesHelper, enh *EvalNodeHelper) (Vector, annotations.Annotations) { return ev.VectorscalarBinop(e.Op, v[0].(Vector), Scalar{V: v[1].(Vector)[0].F}, false, e.ReturnBool, enh), nil }, e.LHS, e.RHS) case lt == parser.ValueTypeScalar && rt == parser.ValueTypeVector: - return ev.rangeEval(nil, func(v []parser.Value, _ [][]EvalSeriesHelper, enh *EvalNodeHelper) (Vector, storage.Warnings) { + return ev.rangeEval(nil, func(v []parser.Value, _ [][]EvalSeriesHelper, enh *EvalNodeHelper) (Vector, annotations.Annotations) { return ev.VectorscalarBinop(e.Op, v[1].(Vector), Scalar{V: v[0].(Vector)[0].F}, true, e.ReturnBool, enh), nil }, e.LHS, e.RHS) } case *parser.NumberLiteral: - return ev.rangeEval(nil, func(v []parser.Value, _ [][]EvalSeriesHelper, enh *EvalNodeHelper) (Vector, storage.Warnings) { + return ev.rangeEval(nil, func(v []parser.Value, _ [][]EvalSeriesHelper, enh *EvalNodeHelper) (Vector, annotations.Annotations) { return append(enh.Out, Sample{F: e.Val, Metric: labels.EmptyLabels()}), nil }) @@ -1834,7 +1826,7 @@ func (ev *evaluator) eval(expr parser.Expr) (parser.Value, storage.Warnings) { panic(fmt.Errorf("unhandled expression of type: %T", expr)) } -func (ev *evaluator) rangeEvalTimestampFunctionOverVectorSelector(vs *parser.VectorSelector, call FunctionCall, e *parser.Call) (parser.Value, storage.Warnings) { +func (ev *evaluator) rangeEvalTimestampFunctionOverVectorSelector(vs *parser.VectorSelector, call FunctionCall, e *parser.Call) (parser.Value, annotations.Annotations) { ws, err := checkAndExpandSeriesSet(ev.ctx, vs) if err != nil { ev.error(errWithWarnings{fmt.Errorf("expanding series: %w", err), ws}) @@ -1846,7 +1838,7 @@ func (ev *evaluator) rangeEvalTimestampFunctionOverVectorSelector(vs *parser.Vec seriesIterators[i] = storage.NewMemoizedIterator(it, durationMilliseconds(ev.lookbackDelta)) } - return ev.rangeEval(nil, func(v []parser.Value, _ [][]EvalSeriesHelper, enh *EvalNodeHelper) (Vector, storage.Warnings) { + return ev.rangeEval(nil, func(v []parser.Value, _ [][]EvalSeriesHelper, enh *EvalNodeHelper) (Vector, annotations.Annotations) { if vs.Timestamp != nil { // This is a special case for "timestamp()" when the @ modifier is used, to ensure that // we return a point for each time step in this case. @@ -1874,7 +1866,8 @@ func (ev *evaluator) rangeEvalTimestampFunctionOverVectorSelector(vs *parser.Vec } } ev.samplesStats.UpdatePeak(ev.currentSamples) - return call([]parser.Value{vec}, e.Args, enh), ws + vec, annos := call([]parser.Value{vec}, e.Args, enh) + return vec, ws.Merge(annos) }) } @@ -1922,19 +1915,33 @@ func getFPointSlice(sz int) []FPoint { if p := fPointPool.Get(); p != nil { return p } + + if sz > maxPointsSliceSize { + sz = maxPointsSliceSize + } + return make([]FPoint, 0, sz) } +// putFPointSlice will return a FPoint slice of size max(maxPointsSliceSize, sz). +// This function is called with an estimated size which often can be over-estimated. func putFPointSlice(p []FPoint) { if p != nil { fPointPool.Put(p[:0]) } } +// getHPointSlice will return a HPoint slice of size max(maxPointsSliceSize, sz). +// This function is called with an estimated size which often can be over-estimated. func getHPointSlice(sz int) []HPoint { if p := hPointPool.Get(); p != nil { return p } + + if sz > maxPointsSliceSize { + sz = maxPointsSliceSize + } + return make([]HPoint, 0, sz) } @@ -1945,7 +1952,7 @@ func putHPointSlice(p []HPoint) { } // matrixSelector evaluates a *parser.MatrixSelector expression. -func (ev *evaluator) matrixSelector(node *parser.MatrixSelector) (Matrix, storage.Warnings) { +func (ev *evaluator) matrixSelector(node *parser.MatrixSelector) (Matrix, annotations.Annotations) { var ( vs = node.VectorSelector.(*parser.VectorSelector) @@ -2525,7 +2532,10 @@ type groupedAggregation struct { // aggregation evaluates an aggregation operation on a Vector. The provided grouping labels // must be sorted. -func (ev *evaluator) aggregation(op parser.ItemType, grouping []string, without bool, param interface{}, vec Vector, seriesHelper []EvalSeriesHelper, enh *EvalNodeHelper) Vector { +func (ev *evaluator) aggregation(e *parser.AggregateExpr, grouping []string, param interface{}, vec Vector, seriesHelper []EvalSeriesHelper, enh *EvalNodeHelper) (Vector, annotations.Annotations) { + op := e.Op + without := e.Without + annos := annotations.Annotations{} result := map[uint64]*groupedAggregation{} orderedResult := []*groupedAggregation{} var k int64 @@ -2536,7 +2546,7 @@ func (ev *evaluator) aggregation(op parser.ItemType, grouping []string, without } k = int64(f) if k < 1 { - return Vector{} + return Vector{}, annos } } var q float64 @@ -2789,7 +2799,8 @@ func (ev *evaluator) aggregation(op parser.ItemType, grouping []string, without case parser.AVG: if aggr.hasFloat && aggr.hasHistogram { // We cannot aggregate histogram sample with a float64 sample. - // TODO(zenador): Issue warning when plumbing is in place. + metricName := aggr.labels.Get(labels.MetricName) + annos.Add(annotations.NewMixedFloatsHistogramsWarning(metricName, e.Expr.PositionRange())) continue } if aggr.hasHistogram { @@ -2834,12 +2845,16 @@ func (ev *evaluator) aggregation(op parser.ItemType, grouping []string, without continue // Bypass default append. case parser.QUANTILE: + if math.IsNaN(q) || q < 0 || q > 1 { + annos.Add(annotations.NewInvalidQuantileWarning(q, e.Param.PositionRange())) + } aggr.floatValue = quantile(q, aggr.heap) case parser.SUM: if aggr.hasFloat && aggr.hasHistogram { // We cannot aggregate histogram sample with a float64 sample. - // TODO(zenador): Issue warning when plumbing is in place. + metricName := aggr.labels.Get(labels.MetricName) + annos.Add(annotations.NewMixedFloatsHistogramsWarning(metricName, e.Expr.PositionRange())) continue } if aggr.hasHistogram { @@ -2855,7 +2870,7 @@ func (ev *evaluator) aggregation(op parser.ItemType, grouping []string, without H: aggr.histogramValue, }) } - return enh.Out + return enh.Out, annos } // groupingKey builds and returns the grouping key for the given metric and diff --git a/vendor/github.com/prometheus/prometheus/promql/functions.go b/vendor/github.com/prometheus/prometheus/promql/functions.go index 96bffab96d..8eb0cad8ad 100644 --- a/vendor/github.com/prometheus/prometheus/promql/functions.go +++ b/vendor/github.com/prometheus/prometheus/promql/functions.go @@ -28,6 +28,8 @@ import ( "github.com/prometheus/prometheus/model/histogram" "github.com/prometheus/prometheus/model/labels" "github.com/prometheus/prometheus/promql/parser" + "github.com/prometheus/prometheus/promql/parser/posrange" + "github.com/prometheus/prometheus/util/annotations" ) // FunctionCall is the type of a PromQL function implementation @@ -51,20 +53,20 @@ import ( // metrics, the timestamp are not needed. // // Scalar results should be returned as the value of a sample in a Vector. -type FunctionCall func(vals []parser.Value, args parser.Expressions, enh *EvalNodeHelper) Vector +type FunctionCall func(vals []parser.Value, args parser.Expressions, enh *EvalNodeHelper) (Vector, annotations.Annotations) // === time() float64 === -func funcTime(vals []parser.Value, args parser.Expressions, enh *EvalNodeHelper) Vector { +func funcTime(vals []parser.Value, args parser.Expressions, enh *EvalNodeHelper) (Vector, annotations.Annotations) { return Vector{Sample{ F: float64(enh.Ts) / 1000, - }} + }}, nil } // extrapolatedRate is a utility function for rate/increase/delta. // It calculates the rate (allowing for counter resets if isCounter is true), // extrapolates if the first/last sample is close to the boundary, and returns // the result as either per-second (if isRate is true) or overall. -func extrapolatedRate(vals []parser.Value, args parser.Expressions, enh *EvalNodeHelper, isCounter, isRate bool) Vector { +func extrapolatedRate(vals []parser.Value, args parser.Expressions, enh *EvalNodeHelper, isCounter, isRate bool) (Vector, annotations.Annotations) { ms := args[0].(*parser.MatrixSelector) vs := ms.VectorSelector.(*parser.VectorSelector) var ( @@ -75,14 +77,19 @@ func extrapolatedRate(vals []parser.Value, args parser.Expressions, enh *EvalNod resultHistogram *histogram.FloatHistogram firstT, lastT int64 numSamplesMinusOne int + annos = annotations.Annotations{} ) // We need either at least two Histograms and no Floats, or at least two // Floats and no Histograms to calculate a rate. Otherwise, drop this // Vector element. + metricName := samples.Metric.Get(labels.MetricName) if len(samples.Histograms) > 0 && len(samples.Floats) > 0 { - // Mix of histograms and floats. TODO(beorn7): Communicate this failure reason. - return enh.Out + return enh.Out, annos.Add(annotations.NewMixedFloatsHistogramsWarning(metricName, args[0].PositionRange())) + } + + if isCounter && !strings.HasSuffix(metricName, "_total") && !strings.HasSuffix(metricName, "_sum") && !strings.HasSuffix(metricName, "_count") { + annos.Add(annotations.NewPossibleNonCounterInfo(metricName, args[0].PositionRange())) } switch { @@ -90,11 +97,11 @@ func extrapolatedRate(vals []parser.Value, args parser.Expressions, enh *EvalNod numSamplesMinusOne = len(samples.Histograms) - 1 firstT = samples.Histograms[0].T lastT = samples.Histograms[numSamplesMinusOne].T - resultHistogram = histogramRate(samples.Histograms, isCounter) + var newAnnos annotations.Annotations + resultHistogram, newAnnos = histogramRate(samples.Histograms, isCounter, metricName, args[0].PositionRange()) if resultHistogram == nil { // The histograms are not compatible with each other. - // TODO(beorn7): Communicate this failure reason. - return enh.Out + return enh.Out, annos.Merge(newAnnos) } case len(samples.Floats) > 1: numSamplesMinusOne = len(samples.Floats) - 1 @@ -113,8 +120,8 @@ func extrapolatedRate(vals []parser.Value, args parser.Expressions, enh *EvalNod prevValue = currPoint.F } default: - // Not enough samples. TODO(beorn7): Communicate this failure reason. - return enh.Out + // TODO: add RangeTooShortWarning + return enh.Out, annos } // Duration between first/last samples and boundary of range. @@ -165,17 +172,18 @@ func extrapolatedRate(vals []parser.Value, args parser.Expressions, enh *EvalNod resultHistogram.Mul(factor) } - return append(enh.Out, Sample{F: resultFloat, H: resultHistogram}) + return append(enh.Out, Sample{F: resultFloat, H: resultHistogram}), annos } // histogramRate is a helper function for extrapolatedRate. It requires // points[0] to be a histogram. It returns nil if any other Point in points is -// not a histogram. -func histogramRate(points []HPoint, isCounter bool) *histogram.FloatHistogram { +// not a histogram, and a warning wrapped in an annotation in that case. +// Otherwise, it returns the calculated histogram and an empty annotation. +func histogramRate(points []HPoint, isCounter bool, metricName string, pos posrange.PositionRange) (*histogram.FloatHistogram, annotations.Annotations) { prev := points[0].H last := points[len(points)-1].H if last == nil { - return nil // Range contains a mix of histograms and floats. + return nil, annotations.New().Add(annotations.NewMixedFloatsHistogramsWarning(metricName, pos)) } minSchema := prev.Schema if last.Schema < minSchema { @@ -190,7 +198,7 @@ func histogramRate(points []HPoint, isCounter bool) *histogram.FloatHistogram { for _, currPoint := range points[1 : len(points)-1] { curr := currPoint.H if curr == nil { - return nil // Range contains a mix of histograms and floats. + return nil, annotations.New().Add(annotations.NewMixedFloatsHistogramsWarning(metricName, pos)) } // TODO(trevorwhitney): Check if isCounter is consistent with curr.CounterResetHint. if !isCounter { @@ -216,40 +224,41 @@ func histogramRate(points []HPoint, isCounter bool) *histogram.FloatHistogram { } h.CounterResetHint = histogram.GaugeType - return h.Compact(0) + return h.Compact(0), nil } -// === delta(Matrix parser.ValueTypeMatrix) Vector === -func funcDelta(vals []parser.Value, args parser.Expressions, enh *EvalNodeHelper) Vector { +// === delta(Matrix parser.ValueTypeMatrix) (Vector, Annotations) === +func funcDelta(vals []parser.Value, args parser.Expressions, enh *EvalNodeHelper) (Vector, annotations.Annotations) { return extrapolatedRate(vals, args, enh, false, false) } -// === rate(node parser.ValueTypeMatrix) Vector === -func funcRate(vals []parser.Value, args parser.Expressions, enh *EvalNodeHelper) Vector { +// === rate(node parser.ValueTypeMatrix) (Vector, Annotations) === +func funcRate(vals []parser.Value, args parser.Expressions, enh *EvalNodeHelper) (Vector, annotations.Annotations) { return extrapolatedRate(vals, args, enh, true, true) } -// === increase(node parser.ValueTypeMatrix) Vector === -func funcIncrease(vals []parser.Value, args parser.Expressions, enh *EvalNodeHelper) Vector { +// === increase(node parser.ValueTypeMatrix) (Vector, Annotations) === +func funcIncrease(vals []parser.Value, args parser.Expressions, enh *EvalNodeHelper) (Vector, annotations.Annotations) { return extrapolatedRate(vals, args, enh, true, false) } -// === irate(node parser.ValueTypeMatrix) Vector === -func funcIrate(vals []parser.Value, args parser.Expressions, enh *EvalNodeHelper) Vector { +// === irate(node parser.ValueTypeMatrix) (Vector, Annotations) === +func funcIrate(vals []parser.Value, args parser.Expressions, enh *EvalNodeHelper) (Vector, annotations.Annotations) { return instantValue(vals, enh.Out, true) } -// === idelta(node model.ValMatrix) Vector === -func funcIdelta(vals []parser.Value, args parser.Expressions, enh *EvalNodeHelper) Vector { +// === idelta(node model.ValMatrix) (Vector, Annotations) === +func funcIdelta(vals []parser.Value, args parser.Expressions, enh *EvalNodeHelper) (Vector, annotations.Annotations) { return instantValue(vals, enh.Out, false) } -func instantValue(vals []parser.Value, out Vector, isRate bool) Vector { +func instantValue(vals []parser.Value, out Vector, isRate bool) (Vector, annotations.Annotations) { samples := vals[0].(Matrix)[0] // No sense in trying to compute a rate without at least two points. Drop // this Vector element. + // TODO: add RangeTooShortWarning if len(samples.Floats) < 2 { - return out + return out, nil } lastSample := samples.Floats[len(samples.Floats)-1] @@ -266,7 +275,7 @@ func instantValue(vals []parser.Value, out Vector, isRate bool) Vector { sampledInterval := lastSample.T - previousSample.T if sampledInterval == 0 { // Avoid dividing by 0. - return out + return out, nil } if isRate { @@ -274,7 +283,7 @@ func instantValue(vals []parser.Value, out Vector, isRate bool) Vector { resultValue /= float64(sampledInterval) / 1000 } - return append(out, Sample{F: resultValue}) + return append(out, Sample{F: resultValue}), nil } // Calculate the trend value at the given index i in raw data d. @@ -299,7 +308,7 @@ func calcTrendValue(i int, tf, s0, s1, b float64) float64 { // data. A lower smoothing factor increases the influence of historical data. The trend factor (0 < tf < 1) affects // how trends in historical data will affect the current data. A higher trend factor increases the influence. // of trends. Algorithm taken from https://en.wikipedia.org/wiki/Exponential_smoothing titled: "Double exponential smoothing". -func funcHoltWinters(vals []parser.Value, args parser.Expressions, enh *EvalNodeHelper) Vector { +func funcHoltWinters(vals []parser.Value, args parser.Expressions, enh *EvalNodeHelper) (Vector, annotations.Annotations) { samples := vals[0].(Matrix)[0] // The smoothing factor argument. @@ -320,7 +329,7 @@ func funcHoltWinters(vals []parser.Value, args parser.Expressions, enh *EvalNode // Can't do the smoothing operation with less than two points. if l < 2 { - return enh.Out + return enh.Out, nil } var s0, s1, b float64 @@ -342,34 +351,34 @@ func funcHoltWinters(vals []parser.Value, args parser.Expressions, enh *EvalNode s0, s1 = s1, x+y } - return append(enh.Out, Sample{F: s1}) + return append(enh.Out, Sample{F: s1}), nil } -// === sort(node parser.ValueTypeVector) Vector === -func funcSort(vals []parser.Value, args parser.Expressions, enh *EvalNodeHelper) Vector { +// === sort(node parser.ValueTypeVector) (Vector, Annotations) === +func funcSort(vals []parser.Value, args parser.Expressions, enh *EvalNodeHelper) (Vector, annotations.Annotations) { // NaN should sort to the bottom, so take descending sort with NaN first and // reverse it. byValueSorter := vectorByReverseValueHeap(vals[0].(Vector)) sort.Sort(sort.Reverse(byValueSorter)) - return Vector(byValueSorter) + return Vector(byValueSorter), nil } -// === sortDesc(node parser.ValueTypeVector) Vector === -func funcSortDesc(vals []parser.Value, args parser.Expressions, enh *EvalNodeHelper) Vector { +// === sortDesc(node parser.ValueTypeVector) (Vector, Annotations) === +func funcSortDesc(vals []parser.Value, args parser.Expressions, enh *EvalNodeHelper) (Vector, annotations.Annotations) { // NaN should sort to the bottom, so take ascending sort with NaN first and // reverse it. byValueSorter := vectorByValueHeap(vals[0].(Vector)) sort.Sort(sort.Reverse(byValueSorter)) - return Vector(byValueSorter) + return Vector(byValueSorter), nil } -// === clamp(Vector parser.ValueTypeVector, min, max Scalar) Vector === -func funcClamp(vals []parser.Value, args parser.Expressions, enh *EvalNodeHelper) Vector { +// === clamp(Vector parser.ValueTypeVector, min, max Scalar) (Vector, Annotations) === +func funcClamp(vals []parser.Value, args parser.Expressions, enh *EvalNodeHelper) (Vector, annotations.Annotations) { vec := vals[0].(Vector) min := vals[1].(Vector)[0].F max := vals[2].(Vector)[0].F if max < min { - return enh.Out + return enh.Out, nil } for _, el := range vec { enh.Out = append(enh.Out, Sample{ @@ -377,11 +386,11 @@ func funcClamp(vals []parser.Value, args parser.Expressions, enh *EvalNodeHelper F: math.Max(min, math.Min(max, el.F)), }) } - return enh.Out + return enh.Out, nil } -// === clamp_max(Vector parser.ValueTypeVector, max Scalar) Vector === -func funcClampMax(vals []parser.Value, args parser.Expressions, enh *EvalNodeHelper) Vector { +// === clamp_max(Vector parser.ValueTypeVector, max Scalar) (Vector, Annotations) === +func funcClampMax(vals []parser.Value, args parser.Expressions, enh *EvalNodeHelper) (Vector, annotations.Annotations) { vec := vals[0].(Vector) max := vals[1].(Vector)[0].F for _, el := range vec { @@ -390,11 +399,11 @@ func funcClampMax(vals []parser.Value, args parser.Expressions, enh *EvalNodeHel F: math.Min(max, el.F), }) } - return enh.Out + return enh.Out, nil } -// === clamp_min(Vector parser.ValueTypeVector, min Scalar) Vector === -func funcClampMin(vals []parser.Value, args parser.Expressions, enh *EvalNodeHelper) Vector { +// === clamp_min(Vector parser.ValueTypeVector, min Scalar) (Vector, Annotations) === +func funcClampMin(vals []parser.Value, args parser.Expressions, enh *EvalNodeHelper) (Vector, annotations.Annotations) { vec := vals[0].(Vector) min := vals[1].(Vector)[0].F for _, el := range vec { @@ -403,11 +412,11 @@ func funcClampMin(vals []parser.Value, args parser.Expressions, enh *EvalNodeHel F: math.Max(min, el.F), }) } - return enh.Out + return enh.Out, nil } -// === round(Vector parser.ValueTypeVector, toNearest=1 Scalar) Vector === -func funcRound(vals []parser.Value, args parser.Expressions, enh *EvalNodeHelper) Vector { +// === round(Vector parser.ValueTypeVector, toNearest=1 Scalar) (Vector, Annotations) === +func funcRound(vals []parser.Value, args parser.Expressions, enh *EvalNodeHelper) (Vector, annotations.Annotations) { vec := vals[0].(Vector) // round returns a number rounded to toNearest. // Ties are solved by rounding up. @@ -425,16 +434,16 @@ func funcRound(vals []parser.Value, args parser.Expressions, enh *EvalNodeHelper F: f, }) } - return enh.Out + return enh.Out, nil } // === Scalar(node parser.ValueTypeVector) Scalar === -func funcScalar(vals []parser.Value, args parser.Expressions, enh *EvalNodeHelper) Vector { +func funcScalar(vals []parser.Value, args parser.Expressions, enh *EvalNodeHelper) (Vector, annotations.Annotations) { v := vals[0].(Vector) if len(v) != 1 { - return append(enh.Out, Sample{F: math.NaN()}) + return append(enh.Out, Sample{F: math.NaN()}), nil } - return append(enh.Out, Sample{F: v[0].F}) + return append(enh.Out, Sample{F: v[0].F}), nil } func aggrOverTime(vals []parser.Value, enh *EvalNodeHelper, aggrFn func(Series) float64) Vector { @@ -449,13 +458,14 @@ func aggrHistOverTime(vals []parser.Value, enh *EvalNodeHelper, aggrFn func(Seri return append(enh.Out, Sample{H: aggrFn(el)}) } -// === avg_over_time(Matrix parser.ValueTypeMatrix) Vector === -func funcAvgOverTime(vals []parser.Value, args parser.Expressions, enh *EvalNodeHelper) Vector { - if len(vals[0].(Matrix)[0].Floats) > 0 && len(vals[0].(Matrix)[0].Histograms) > 0 { - // TODO(zenador): Add warning for mixed floats and histograms. - return enh.Out +// === avg_over_time(Matrix parser.ValueTypeMatrix) (Vector, Annotations) === +func funcAvgOverTime(vals []parser.Value, args parser.Expressions, enh *EvalNodeHelper) (Vector, annotations.Annotations) { + firstSeries := vals[0].(Matrix)[0] + if len(firstSeries.Floats) > 0 && len(firstSeries.Histograms) > 0 { + metricName := firstSeries.Metric.Get(labels.MetricName) + return enh.Out, annotations.New().Add(annotations.NewMixedFloatsHistogramsWarning(metricName, args[0].PositionRange())) } - if len(vals[0].(Matrix)[0].Floats) == 0 { + if len(firstSeries.Floats) == 0 { // The passed values only contain histograms. return aggrHistOverTime(vals, enh, func(s Series) *histogram.FloatHistogram { count := 1 @@ -475,7 +485,7 @@ func funcAvgOverTime(vals []parser.Value, args parser.Expressions, enh *EvalNode } } return mean - }) + }), nil } return aggrOverTime(vals, enh, func(s Series) float64 { var mean, count, c float64 @@ -505,18 +515,18 @@ func funcAvgOverTime(vals []parser.Value, args parser.Expressions, enh *EvalNode return mean } return mean + c - }) + }), nil } -// === count_over_time(Matrix parser.ValueTypeMatrix) Vector === -func funcCountOverTime(vals []parser.Value, args parser.Expressions, enh *EvalNodeHelper) Vector { +// === count_over_time(Matrix parser.ValueTypeMatrix) (Vector, Notes) === +func funcCountOverTime(vals []parser.Value, args parser.Expressions, enh *EvalNodeHelper) (Vector, annotations.Annotations) { return aggrOverTime(vals, enh, func(s Series) float64 { return float64(len(s.Floats) + len(s.Histograms)) - }) + }), nil } -// === last_over_time(Matrix parser.ValueTypeMatrix) Vector === -func funcLastOverTime(vals []parser.Value, args parser.Expressions, enh *EvalNodeHelper) Vector { +// === last_over_time(Matrix parser.ValueTypeMatrix) (Vector, Notes) === +func funcLastOverTime(vals []parser.Value, args parser.Expressions, enh *EvalNodeHelper) (Vector, annotations.Annotations) { el := vals[0].(Matrix)[0] var f FPoint @@ -533,22 +543,22 @@ func funcLastOverTime(vals []parser.Value, args parser.Expressions, enh *EvalNod return append(enh.Out, Sample{ Metric: el.Metric, F: f.F, - }) + }), nil } return append(enh.Out, Sample{ Metric: el.Metric, H: h.H, - }) + }), nil } -// === max_over_time(Matrix parser.ValueTypeMatrix) Vector === -func funcMaxOverTime(vals []parser.Value, args parser.Expressions, enh *EvalNodeHelper) Vector { +// === max_over_time(Matrix parser.ValueTypeMatrix) (Vector, Annotations) === +func funcMaxOverTime(vals []parser.Value, args parser.Expressions, enh *EvalNodeHelper) (Vector, annotations.Annotations) { if len(vals[0].(Matrix)[0].Floats) == 0 { // TODO(beorn7): The passed values only contain // histograms. max_over_time ignores histograms for now. If // there are only histograms, we have to return without adding // anything to enh.Out. - return enh.Out + return enh.Out, nil } return aggrOverTime(vals, enh, func(s Series) float64 { max := s.Floats[0].F @@ -558,17 +568,17 @@ func funcMaxOverTime(vals []parser.Value, args parser.Expressions, enh *EvalNode } } return max - }) + }), nil } -// === min_over_time(Matrix parser.ValueTypeMatrix) Vector === -func funcMinOverTime(vals []parser.Value, args parser.Expressions, enh *EvalNodeHelper) Vector { +// === min_over_time(Matrix parser.ValueTypeMatrix) (Vector, Annotations) === +func funcMinOverTime(vals []parser.Value, args parser.Expressions, enh *EvalNodeHelper) (Vector, annotations.Annotations) { if len(vals[0].(Matrix)[0].Floats) == 0 { // TODO(beorn7): The passed values only contain // histograms. min_over_time ignores histograms for now. If // there are only histograms, we have to return without adding // anything to enh.Out. - return enh.Out + return enh.Out, nil } return aggrOverTime(vals, enh, func(s Series) float64 { min := s.Floats[0].F @@ -578,16 +588,17 @@ func funcMinOverTime(vals []parser.Value, args parser.Expressions, enh *EvalNode } } return min - }) + }), nil } -// === sum_over_time(Matrix parser.ValueTypeMatrix) Vector === -func funcSumOverTime(vals []parser.Value, args parser.Expressions, enh *EvalNodeHelper) Vector { - if len(vals[0].(Matrix)[0].Floats) > 0 && len(vals[0].(Matrix)[0].Histograms) > 0 { - // TODO(zenador): Add warning for mixed floats and histograms. - return enh.Out +// === sum_over_time(Matrix parser.ValueTypeMatrix) (Vector, Annotations) === +func funcSumOverTime(vals []parser.Value, args parser.Expressions, enh *EvalNodeHelper) (Vector, annotations.Annotations) { + firstSeries := vals[0].(Matrix)[0] + if len(firstSeries.Floats) > 0 && len(firstSeries.Histograms) > 0 { + metricName := firstSeries.Metric.Get(labels.MetricName) + return enh.Out, annotations.New().Add(annotations.NewMixedFloatsHistogramsWarning(metricName, args[0].PositionRange())) } - if len(vals[0].(Matrix)[0].Floats) == 0 { + if len(firstSeries.Floats) == 0 { // The passed values only contain histograms. return aggrHistOverTime(vals, enh, func(s Series) *histogram.FloatHistogram { sum := s.Histograms[0].H.Copy() @@ -601,7 +612,7 @@ func funcSumOverTime(vals []parser.Value, args parser.Expressions, enh *EvalNode } } return sum - }) + }), nil } return aggrOverTime(vals, enh, func(s Series) float64 { var sum, c float64 @@ -612,11 +623,11 @@ func funcSumOverTime(vals []parser.Value, args parser.Expressions, enh *EvalNode return sum } return sum + c - }) + }), nil } -// === quantile_over_time(Matrix parser.ValueTypeMatrix) Vector === -func funcQuantileOverTime(vals []parser.Value, args parser.Expressions, enh *EvalNodeHelper) Vector { +// === quantile_over_time(Matrix parser.ValueTypeMatrix) (Vector, Annotations) === +func funcQuantileOverTime(vals []parser.Value, args parser.Expressions, enh *EvalNodeHelper) (Vector, annotations.Annotations) { q := vals[0].(Vector)[0].F el := vals[1].(Matrix)[0] if len(el.Floats) == 0 { @@ -624,24 +635,29 @@ func funcQuantileOverTime(vals []parser.Value, args parser.Expressions, enh *Eva // histograms. quantile_over_time ignores histograms for now. If // there are only histograms, we have to return without adding // anything to enh.Out. - return enh.Out + return enh.Out, nil + } + + annos := annotations.Annotations{} + if math.IsNaN(q) || q < 0 || q > 1 { + annos.Add(annotations.NewInvalidQuantileWarning(q, args[0].PositionRange())) } values := make(vectorByValueHeap, 0, len(el.Floats)) for _, f := range el.Floats { values = append(values, Sample{F: f.F}) } - return append(enh.Out, Sample{F: quantile(q, values)}) + return append(enh.Out, Sample{F: quantile(q, values)}), annos } -// === stddev_over_time(Matrix parser.ValueTypeMatrix) Vector === -func funcStddevOverTime(vals []parser.Value, args parser.Expressions, enh *EvalNodeHelper) Vector { +// === stddev_over_time(Matrix parser.ValueTypeMatrix) (Vector, Annotations) === +func funcStddevOverTime(vals []parser.Value, args parser.Expressions, enh *EvalNodeHelper) (Vector, annotations.Annotations) { if len(vals[0].(Matrix)[0].Floats) == 0 { // TODO(beorn7): The passed values only contain // histograms. stddev_over_time ignores histograms for now. If // there are only histograms, we have to return without adding // anything to enh.Out. - return enh.Out + return enh.Out, nil } return aggrOverTime(vals, enh, func(s Series) float64 { var count float64 @@ -654,17 +670,17 @@ func funcStddevOverTime(vals []parser.Value, args parser.Expressions, enh *EvalN aux, cAux = kahanSumInc(delta*(f.F-(mean+cMean)), aux, cAux) } return math.Sqrt((aux + cAux) / count) - }) + }), nil } -// === stdvar_over_time(Matrix parser.ValueTypeMatrix) Vector === -func funcStdvarOverTime(vals []parser.Value, args parser.Expressions, enh *EvalNodeHelper) Vector { +// === stdvar_over_time(Matrix parser.ValueTypeMatrix) (Vector, Annotations) === +func funcStdvarOverTime(vals []parser.Value, args parser.Expressions, enh *EvalNodeHelper) (Vector, annotations.Annotations) { if len(vals[0].(Matrix)[0].Floats) == 0 { // TODO(beorn7): The passed values only contain // histograms. stdvar_over_time ignores histograms for now. If // there are only histograms, we have to return without adding // anything to enh.Out. - return enh.Out + return enh.Out, nil } return aggrOverTime(vals, enh, func(s Series) float64 { var count float64 @@ -677,35 +693,35 @@ func funcStdvarOverTime(vals []parser.Value, args parser.Expressions, enh *EvalN aux, cAux = kahanSumInc(delta*(f.F-(mean+cMean)), aux, cAux) } return (aux + cAux) / count - }) + }), nil } -// === absent(Vector parser.ValueTypeVector) Vector === -func funcAbsent(vals []parser.Value, args parser.Expressions, enh *EvalNodeHelper) Vector { +// === absent(Vector parser.ValueTypeVector) (Vector, Annotations) === +func funcAbsent(vals []parser.Value, args parser.Expressions, enh *EvalNodeHelper) (Vector, annotations.Annotations) { if len(vals[0].(Vector)) > 0 { - return enh.Out + return enh.Out, nil } return append(enh.Out, Sample{ Metric: createLabelsForAbsentFunction(args[0]), F: 1, - }) + }), nil } -// === absent_over_time(Vector parser.ValueTypeMatrix) Vector === +// === absent_over_time(Vector parser.ValueTypeMatrix) (Vector, Annotations) === // As this function has a matrix as argument, it does not get all the Series. // This function will return 1 if the matrix has at least one element. // Due to engine optimization, this function is only called when this condition is true. // Then, the engine post-processes the results to get the expected output. -func funcAbsentOverTime(vals []parser.Value, args parser.Expressions, enh *EvalNodeHelper) Vector { - return append(enh.Out, Sample{F: 1}) +func funcAbsentOverTime(vals []parser.Value, args parser.Expressions, enh *EvalNodeHelper) (Vector, annotations.Annotations) { + return append(enh.Out, Sample{F: 1}), nil } -// === present_over_time(Vector parser.ValueTypeMatrix) Vector === -func funcPresentOverTime(vals []parser.Value, args parser.Expressions, enh *EvalNodeHelper) Vector { +// === present_over_time(Vector parser.ValueTypeMatrix) (Vector, Annotations) === +func funcPresentOverTime(vals []parser.Value, args parser.Expressions, enh *EvalNodeHelper) (Vector, annotations.Annotations) { return aggrOverTime(vals, enh, func(s Series) float64 { return 1 - }) + }), nil } func simpleFunc(vals []parser.Value, enh *EvalNodeHelper, f func(float64) float64) Vector { @@ -720,127 +736,127 @@ func simpleFunc(vals []parser.Value, enh *EvalNodeHelper, f func(float64) float6 return enh.Out } -// === abs(Vector parser.ValueTypeVector) Vector === -func funcAbs(vals []parser.Value, args parser.Expressions, enh *EvalNodeHelper) Vector { - return simpleFunc(vals, enh, math.Abs) +// === abs(Vector parser.ValueTypeVector) (Vector, Annotations) === +func funcAbs(vals []parser.Value, args parser.Expressions, enh *EvalNodeHelper) (Vector, annotations.Annotations) { + return simpleFunc(vals, enh, math.Abs), nil } -// === ceil(Vector parser.ValueTypeVector) Vector === -func funcCeil(vals []parser.Value, args parser.Expressions, enh *EvalNodeHelper) Vector { - return simpleFunc(vals, enh, math.Ceil) +// === ceil(Vector parser.ValueTypeVector) (Vector, Annotations) === +func funcCeil(vals []parser.Value, args parser.Expressions, enh *EvalNodeHelper) (Vector, annotations.Annotations) { + return simpleFunc(vals, enh, math.Ceil), nil } -// === floor(Vector parser.ValueTypeVector) Vector === -func funcFloor(vals []parser.Value, args parser.Expressions, enh *EvalNodeHelper) Vector { - return simpleFunc(vals, enh, math.Floor) +// === floor(Vector parser.ValueTypeVector) (Vector, Annotations) === +func funcFloor(vals []parser.Value, args parser.Expressions, enh *EvalNodeHelper) (Vector, annotations.Annotations) { + return simpleFunc(vals, enh, math.Floor), nil } -// === exp(Vector parser.ValueTypeVector) Vector === -func funcExp(vals []parser.Value, args parser.Expressions, enh *EvalNodeHelper) Vector { - return simpleFunc(vals, enh, math.Exp) +// === exp(Vector parser.ValueTypeVector) (Vector, Annotations) === +func funcExp(vals []parser.Value, args parser.Expressions, enh *EvalNodeHelper) (Vector, annotations.Annotations) { + return simpleFunc(vals, enh, math.Exp), nil } -// === sqrt(Vector VectorNode) Vector === -func funcSqrt(vals []parser.Value, args parser.Expressions, enh *EvalNodeHelper) Vector { - return simpleFunc(vals, enh, math.Sqrt) +// === sqrt(Vector VectorNode) (Vector, Annotations) === +func funcSqrt(vals []parser.Value, args parser.Expressions, enh *EvalNodeHelper) (Vector, annotations.Annotations) { + return simpleFunc(vals, enh, math.Sqrt), nil } -// === ln(Vector parser.ValueTypeVector) Vector === -func funcLn(vals []parser.Value, args parser.Expressions, enh *EvalNodeHelper) Vector { - return simpleFunc(vals, enh, math.Log) +// === ln(Vector parser.ValueTypeVector) (Vector, Annotations) === +func funcLn(vals []parser.Value, args parser.Expressions, enh *EvalNodeHelper) (Vector, annotations.Annotations) { + return simpleFunc(vals, enh, math.Log), nil } -// === log2(Vector parser.ValueTypeVector) Vector === -func funcLog2(vals []parser.Value, args parser.Expressions, enh *EvalNodeHelper) Vector { - return simpleFunc(vals, enh, math.Log2) +// === log2(Vector parser.ValueTypeVector) (Vector, Annotations) === +func funcLog2(vals []parser.Value, args parser.Expressions, enh *EvalNodeHelper) (Vector, annotations.Annotations) { + return simpleFunc(vals, enh, math.Log2), nil } -// === log10(Vector parser.ValueTypeVector) Vector === -func funcLog10(vals []parser.Value, args parser.Expressions, enh *EvalNodeHelper) Vector { - return simpleFunc(vals, enh, math.Log10) +// === log10(Vector parser.ValueTypeVector) (Vector, Annotations) === +func funcLog10(vals []parser.Value, args parser.Expressions, enh *EvalNodeHelper) (Vector, annotations.Annotations) { + return simpleFunc(vals, enh, math.Log10), nil } -// === sin(Vector parser.ValueTypeVector) Vector === -func funcSin(vals []parser.Value, args parser.Expressions, enh *EvalNodeHelper) Vector { - return simpleFunc(vals, enh, math.Sin) +// === sin(Vector parser.ValueTypeVector) (Vector, Annotations) === +func funcSin(vals []parser.Value, args parser.Expressions, enh *EvalNodeHelper) (Vector, annotations.Annotations) { + return simpleFunc(vals, enh, math.Sin), nil } -// === cos(Vector parser.ValueTypeVector) Vector === -func funcCos(vals []parser.Value, args parser.Expressions, enh *EvalNodeHelper) Vector { - return simpleFunc(vals, enh, math.Cos) +// === cos(Vector parser.ValueTypeVector) (Vector, Annotations) === +func funcCos(vals []parser.Value, args parser.Expressions, enh *EvalNodeHelper) (Vector, annotations.Annotations) { + return simpleFunc(vals, enh, math.Cos), nil } -// === tan(Vector parser.ValueTypeVector) Vector === -func funcTan(vals []parser.Value, args parser.Expressions, enh *EvalNodeHelper) Vector { - return simpleFunc(vals, enh, math.Tan) +// === tan(Vector parser.ValueTypeVector) (Vector, Annotations) === +func funcTan(vals []parser.Value, args parser.Expressions, enh *EvalNodeHelper) (Vector, annotations.Annotations) { + return simpleFunc(vals, enh, math.Tan), nil } -// == asin(Vector parser.ValueTypeVector) Vector === -func funcAsin(vals []parser.Value, args parser.Expressions, enh *EvalNodeHelper) Vector { - return simpleFunc(vals, enh, math.Asin) +// == asin(Vector parser.ValueTypeVector) (Vector, Annotations) === +func funcAsin(vals []parser.Value, args parser.Expressions, enh *EvalNodeHelper) (Vector, annotations.Annotations) { + return simpleFunc(vals, enh, math.Asin), nil } -// == acos(Vector parser.ValueTypeVector) Vector === -func funcAcos(vals []parser.Value, args parser.Expressions, enh *EvalNodeHelper) Vector { - return simpleFunc(vals, enh, math.Acos) +// == acos(Vector parser.ValueTypeVector) (Vector, Annotations) === +func funcAcos(vals []parser.Value, args parser.Expressions, enh *EvalNodeHelper) (Vector, annotations.Annotations) { + return simpleFunc(vals, enh, math.Acos), nil } -// == atan(Vector parser.ValueTypeVector) Vector === -func funcAtan(vals []parser.Value, args parser.Expressions, enh *EvalNodeHelper) Vector { - return simpleFunc(vals, enh, math.Atan) +// == atan(Vector parser.ValueTypeVector) (Vector, Annotations) === +func funcAtan(vals []parser.Value, args parser.Expressions, enh *EvalNodeHelper) (Vector, annotations.Annotations) { + return simpleFunc(vals, enh, math.Atan), nil } -// == sinh(Vector parser.ValueTypeVector) Vector === -func funcSinh(vals []parser.Value, args parser.Expressions, enh *EvalNodeHelper) Vector { - return simpleFunc(vals, enh, math.Sinh) +// == sinh(Vector parser.ValueTypeVector) (Vector, Annotations) === +func funcSinh(vals []parser.Value, args parser.Expressions, enh *EvalNodeHelper) (Vector, annotations.Annotations) { + return simpleFunc(vals, enh, math.Sinh), nil } -// == cosh(Vector parser.ValueTypeVector) Vector === -func funcCosh(vals []parser.Value, args parser.Expressions, enh *EvalNodeHelper) Vector { - return simpleFunc(vals, enh, math.Cosh) +// == cosh(Vector parser.ValueTypeVector) (Vector, Annotations) === +func funcCosh(vals []parser.Value, args parser.Expressions, enh *EvalNodeHelper) (Vector, annotations.Annotations) { + return simpleFunc(vals, enh, math.Cosh), nil } -// == tanh(Vector parser.ValueTypeVector) Vector === -func funcTanh(vals []parser.Value, args parser.Expressions, enh *EvalNodeHelper) Vector { - return simpleFunc(vals, enh, math.Tanh) +// == tanh(Vector parser.ValueTypeVector) (Vector, Annotations) === +func funcTanh(vals []parser.Value, args parser.Expressions, enh *EvalNodeHelper) (Vector, annotations.Annotations) { + return simpleFunc(vals, enh, math.Tanh), nil } -// == asinh(Vector parser.ValueTypeVector) Vector === -func funcAsinh(vals []parser.Value, args parser.Expressions, enh *EvalNodeHelper) Vector { - return simpleFunc(vals, enh, math.Asinh) +// == asinh(Vector parser.ValueTypeVector) (Vector, Annotations) === +func funcAsinh(vals []parser.Value, args parser.Expressions, enh *EvalNodeHelper) (Vector, annotations.Annotations) { + return simpleFunc(vals, enh, math.Asinh), nil } -// == acosh(Vector parser.ValueTypeVector) Vector === -func funcAcosh(vals []parser.Value, args parser.Expressions, enh *EvalNodeHelper) Vector { - return simpleFunc(vals, enh, math.Acosh) +// == acosh(Vector parser.ValueTypeVector) (Vector, Annotations) === +func funcAcosh(vals []parser.Value, args parser.Expressions, enh *EvalNodeHelper) (Vector, annotations.Annotations) { + return simpleFunc(vals, enh, math.Acosh), nil } -// == atanh(Vector parser.ValueTypeVector) Vector === -func funcAtanh(vals []parser.Value, args parser.Expressions, enh *EvalNodeHelper) Vector { - return simpleFunc(vals, enh, math.Atanh) +// == atanh(Vector parser.ValueTypeVector) (Vector, Annotations) === +func funcAtanh(vals []parser.Value, args parser.Expressions, enh *EvalNodeHelper) (Vector, annotations.Annotations) { + return simpleFunc(vals, enh, math.Atanh), nil } -// === rad(Vector parser.ValueTypeVector) Vector === -func funcRad(vals []parser.Value, args parser.Expressions, enh *EvalNodeHelper) Vector { +// === rad(Vector parser.ValueTypeVector) (Vector, Annotations) === +func funcRad(vals []parser.Value, args parser.Expressions, enh *EvalNodeHelper) (Vector, annotations.Annotations) { return simpleFunc(vals, enh, func(v float64) float64 { return v * math.Pi / 180 - }) + }), nil } -// === deg(Vector parser.ValueTypeVector) Vector === -func funcDeg(vals []parser.Value, args parser.Expressions, enh *EvalNodeHelper) Vector { +// === deg(Vector parser.ValueTypeVector) (Vector, Annotations) === +func funcDeg(vals []parser.Value, args parser.Expressions, enh *EvalNodeHelper) (Vector, annotations.Annotations) { return simpleFunc(vals, enh, func(v float64) float64 { return v * 180 / math.Pi - }) + }), nil } // === pi() Scalar === -func funcPi(vals []parser.Value, args parser.Expressions, enh *EvalNodeHelper) Vector { - return Vector{Sample{F: math.Pi}} +func funcPi(vals []parser.Value, args parser.Expressions, enh *EvalNodeHelper) (Vector, annotations.Annotations) { + return Vector{Sample{F: math.Pi}}, nil } -// === sgn(Vector parser.ValueTypeVector) Vector === -func funcSgn(vals []parser.Value, args parser.Expressions, enh *EvalNodeHelper) Vector { +// === sgn(Vector parser.ValueTypeVector) (Vector, Annotations) === +func funcSgn(vals []parser.Value, args parser.Expressions, enh *EvalNodeHelper) (Vector, annotations.Annotations) { return simpleFunc(vals, enh, func(v float64) float64 { switch { case v < 0: @@ -850,11 +866,11 @@ func funcSgn(vals []parser.Value, args parser.Expressions, enh *EvalNodeHelper) default: return v } - }) + }), nil } -// === timestamp(Vector parser.ValueTypeVector) Vector === -func funcTimestamp(vals []parser.Value, args parser.Expressions, enh *EvalNodeHelper) Vector { +// === timestamp(Vector parser.ValueTypeVector) (Vector, Annotations) === +func funcTimestamp(vals []parser.Value, args parser.Expressions, enh *EvalNodeHelper) (Vector, annotations.Annotations) { vec := vals[0].(Vector) for _, el := range vec { enh.Out = append(enh.Out, Sample{ @@ -862,7 +878,7 @@ func funcTimestamp(vals []parser.Value, args parser.Expressions, enh *EvalNodeHe F: float64(el.T) / 1000, }) } - return enh.Out + return enh.Out, nil } func kahanSum(samples []float64) float64 { @@ -931,39 +947,39 @@ func linearRegression(samples []FPoint, interceptTime int64) (slope, intercept f return slope, intercept } -// === deriv(node parser.ValueTypeMatrix) Vector === -func funcDeriv(vals []parser.Value, args parser.Expressions, enh *EvalNodeHelper) Vector { +// === deriv(node parser.ValueTypeMatrix) (Vector, Annotations) === +func funcDeriv(vals []parser.Value, args parser.Expressions, enh *EvalNodeHelper) (Vector, annotations.Annotations) { samples := vals[0].(Matrix)[0] // No sense in trying to compute a derivative without at least two points. // Drop this Vector element. if len(samples.Floats) < 2 { - return enh.Out + return enh.Out, nil } // We pass in an arbitrary timestamp that is near the values in use // to avoid floating point accuracy issues, see // https://github.com/prometheus/prometheus/issues/2674 slope, _ := linearRegression(samples.Floats, samples.Floats[0].T) - return append(enh.Out, Sample{F: slope}) + return append(enh.Out, Sample{F: slope}), nil } -// === predict_linear(node parser.ValueTypeMatrix, k parser.ValueTypeScalar) Vector === -func funcPredictLinear(vals []parser.Value, args parser.Expressions, enh *EvalNodeHelper) Vector { +// === predict_linear(node parser.ValueTypeMatrix, k parser.ValueTypeScalar) (Vector, Annotations) === +func funcPredictLinear(vals []parser.Value, args parser.Expressions, enh *EvalNodeHelper) (Vector, annotations.Annotations) { samples := vals[0].(Matrix)[0] duration := vals[1].(Vector)[0].F // No sense in trying to predict anything without at least two points. // Drop this Vector element. if len(samples.Floats) < 2 { - return enh.Out + return enh.Out, nil } slope, intercept := linearRegression(samples.Floats, enh.Ts) - return append(enh.Out, Sample{F: slope*duration + intercept}) + return append(enh.Out, Sample{F: slope*duration + intercept}), nil } -// === histogram_count(Vector parser.ValueTypeVector) Vector === -func funcHistogramCount(vals []parser.Value, args parser.Expressions, enh *EvalNodeHelper) Vector { +// === histogram_count(Vector parser.ValueTypeVector) (Vector, Annotations) === +func funcHistogramCount(vals []parser.Value, args parser.Expressions, enh *EvalNodeHelper) (Vector, annotations.Annotations) { inVec := vals[0].(Vector) for _, sample := range inVec { @@ -976,11 +992,11 @@ func funcHistogramCount(vals []parser.Value, args parser.Expressions, enh *EvalN F: sample.H.Count, }) } - return enh.Out + return enh.Out, nil } -// === histogram_sum(Vector parser.ValueTypeVector) Vector === -func funcHistogramSum(vals []parser.Value, args parser.Expressions, enh *EvalNodeHelper) Vector { +// === histogram_sum(Vector parser.ValueTypeVector) (Vector, Annotations) === +func funcHistogramSum(vals []parser.Value, args parser.Expressions, enh *EvalNodeHelper) (Vector, annotations.Annotations) { inVec := vals[0].(Vector) for _, sample := range inVec { @@ -993,11 +1009,77 @@ func funcHistogramSum(vals []parser.Value, args parser.Expressions, enh *EvalNod F: sample.H.Sum, }) } - return enh.Out + return enh.Out, nil } -// === histogram_fraction(lower, upper parser.ValueTypeScalar, Vector parser.ValueTypeVector) Vector === -func funcHistogramFraction(vals []parser.Value, args parser.Expressions, enh *EvalNodeHelper) Vector { +// === histogram_stddev(Vector parser.ValueTypeVector) (Vector, Annotations) === +func funcHistogramStdDev(vals []parser.Value, args parser.Expressions, enh *EvalNodeHelper) (Vector, annotations.Annotations) { + inVec := vals[0].(Vector) + + for _, sample := range inVec { + // Skip non-histogram samples. + if sample.H == nil { + continue + } + mean := sample.H.Sum / sample.H.Count + var variance, cVariance float64 + it := sample.H.AllBucketIterator() + for it.Next() { + bucket := it.At() + var val float64 + if bucket.Lower <= 0 && 0 <= bucket.Upper { + val = 0 + } else { + val = math.Sqrt(bucket.Upper * bucket.Lower) + } + delta := val - mean + variance, cVariance = kahanSumInc(bucket.Count*delta*delta, variance, cVariance) + } + variance += cVariance + variance /= sample.H.Count + enh.Out = append(enh.Out, Sample{ + Metric: enh.DropMetricName(sample.Metric), + F: math.Sqrt(variance), + }) + } + return enh.Out, nil +} + +// === histogram_stdvar(Vector parser.ValueTypeVector) (Vector, Annotations) === +func funcHistogramStdVar(vals []parser.Value, args parser.Expressions, enh *EvalNodeHelper) (Vector, annotations.Annotations) { + inVec := vals[0].(Vector) + + for _, sample := range inVec { + // Skip non-histogram samples. + if sample.H == nil { + continue + } + mean := sample.H.Sum / sample.H.Count + var variance, cVariance float64 + it := sample.H.AllBucketIterator() + for it.Next() { + bucket := it.At() + var val float64 + if bucket.Lower <= 0 && 0 <= bucket.Upper { + val = 0 + } else { + val = math.Sqrt(bucket.Upper * bucket.Lower) + } + delta := val - mean + variance, cVariance = kahanSumInc(bucket.Count*delta*delta, variance, cVariance) + } + variance += cVariance + variance /= sample.H.Count + enh.Out = append(enh.Out, Sample{ + Metric: enh.DropMetricName(sample.Metric), + F: variance, + }) + } + return enh.Out, nil +} + +// === histogram_fraction(lower, upper parser.ValueTypeScalar, Vector parser.ValueTypeVector) (Vector, Annotations) === +func funcHistogramFraction(vals []parser.Value, args parser.Expressions, enh *EvalNodeHelper) (Vector, annotations.Annotations) { lower := vals[0].(Vector)[0].F upper := vals[1].(Vector)[0].F inVec := vals[2].(Vector) @@ -1012,13 +1094,18 @@ func funcHistogramFraction(vals []parser.Value, args parser.Expressions, enh *Ev F: histogramFraction(lower, upper, sample.H), }) } - return enh.Out + return enh.Out, nil } -// === histogram_quantile(k parser.ValueTypeScalar, Vector parser.ValueTypeVector) Vector === -func funcHistogramQuantile(vals []parser.Value, args parser.Expressions, enh *EvalNodeHelper) Vector { +// === histogram_quantile(k parser.ValueTypeScalar, Vector parser.ValueTypeVector) (Vector, Annotations) === +func funcHistogramQuantile(vals []parser.Value, args parser.Expressions, enh *EvalNodeHelper) (Vector, annotations.Annotations) { q := vals[0].(Vector)[0].F inVec := vals[1].(Vector) + annos := annotations.Annotations{} + + if math.IsNaN(q) || q < 0 || q > 1 { + annos.Add(annotations.NewInvalidQuantileWarning(q, args[0].PositionRange())) + } if enh.signatureToMetricWithBuckets == nil { enh.signatureToMetricWithBuckets = map[string]*metricWithBuckets{} @@ -1042,8 +1129,7 @@ func funcHistogramQuantile(vals []parser.Value, args parser.Expressions, enh *Ev sample.Metric.Get(model.BucketLabel), 64, ) if err != nil { - // Oops, no bucket label or malformed label value. Skip. - // TODO(beorn7): Issue a warning somehow. + annos.Add(annotations.NewBadBucketLabelWarning(sample.Metric.Get(labels.MetricName), sample.Metric.Get(model.BucketLabel), args[1].PositionRange())) continue } enh.lblBuf = sample.Metric.BytesWithoutLabels(enh.lblBuf, labels.BucketLabel) @@ -1069,7 +1155,7 @@ func funcHistogramQuantile(vals []parser.Value, args parser.Expressions, enh *Ev // At this data point, we have conventional histogram // buckets and a native histogram with the same name and // labels. Do not evaluate anything. - // TODO(beorn7): Issue a warning somehow. + annos.Add(annotations.NewMixedClassicNativeHistogramsWarning(sample.Metric.Get(labels.MetricName), args[1].PositionRange())) delete(enh.signatureToMetricWithBuckets, string(enh.lblBuf)) continue } @@ -1082,18 +1168,22 @@ func funcHistogramQuantile(vals []parser.Value, args parser.Expressions, enh *Ev for _, mb := range enh.signatureToMetricWithBuckets { if len(mb.buckets) > 0 { + res, forcedMonotonicity := bucketQuantile(q, mb.buckets) enh.Out = append(enh.Out, Sample{ Metric: mb.metric, - F: bucketQuantile(q, mb.buckets), + F: res, }) + if forcedMonotonicity { + annos.Add(annotations.NewHistogramQuantileForcedMonotonicityInfo(mb.metric.Get(labels.MetricName), args[1].PositionRange())) + } } } - return enh.Out + return enh.Out, annos } -// === resets(Matrix parser.ValueTypeMatrix) Vector === -func funcResets(vals []parser.Value, args parser.Expressions, enh *EvalNodeHelper) Vector { +// === resets(Matrix parser.ValueTypeMatrix) (Vector, Annotations) === +func funcResets(vals []parser.Value, args parser.Expressions, enh *EvalNodeHelper) (Vector, annotations.Annotations) { floats := vals[0].(Matrix)[0].Floats histograms := vals[0].(Matrix)[0].Histograms resets := 0 @@ -1120,17 +1210,17 @@ func funcResets(vals []parser.Value, args parser.Expressions, enh *EvalNodeHelpe } } - return append(enh.Out, Sample{F: float64(resets)}) + return append(enh.Out, Sample{F: float64(resets)}), nil } -// === changes(Matrix parser.ValueTypeMatrix) Vector === -func funcChanges(vals []parser.Value, args parser.Expressions, enh *EvalNodeHelper) Vector { +// === changes(Matrix parser.ValueTypeMatrix) (Vector, Annotations) === +func funcChanges(vals []parser.Value, args parser.Expressions, enh *EvalNodeHelper) (Vector, annotations.Annotations) { floats := vals[0].(Matrix)[0].Floats changes := 0 if len(floats) == 0 { // TODO(beorn7): Only histogram values, still need to add support. - return enh.Out + return enh.Out, nil } prev := floats[0].F @@ -1142,11 +1232,11 @@ func funcChanges(vals []parser.Value, args parser.Expressions, enh *EvalNodeHelp prev = current } - return append(enh.Out, Sample{F: float64(changes)}) + return append(enh.Out, Sample{F: float64(changes)}), nil } -// === label_replace(Vector parser.ValueTypeVector, dst_label, replacement, src_labelname, regex parser.ValueTypeString) Vector === -func funcLabelReplace(vals []parser.Value, args parser.Expressions, enh *EvalNodeHelper) Vector { +// === label_replace(Vector parser.ValueTypeVector, dst_label, replacement, src_labelname, regex parser.ValueTypeString) (Vector, Annotations) === +func funcLabelReplace(vals []parser.Value, args parser.Expressions, enh *EvalNodeHelper) (Vector, annotations.Annotations) { var ( vector = vals[0].(Vector) dst = stringFromArg(args[1]) @@ -1197,20 +1287,20 @@ func funcLabelReplace(vals []parser.Value, args parser.Expressions, enh *EvalNod H: el.H, }) } - return enh.Out + return enh.Out, nil } -// === Vector(s Scalar) Vector === -func funcVector(vals []parser.Value, args parser.Expressions, enh *EvalNodeHelper) Vector { +// === Vector(s Scalar) (Vector, Annotations) === +func funcVector(vals []parser.Value, args parser.Expressions, enh *EvalNodeHelper) (Vector, annotations.Annotations) { return append(enh.Out, Sample{ Metric: labels.Labels{}, F: vals[0].(Vector)[0].F, - }) + }), nil } -// === label_join(vector model.ValVector, dest_labelname, separator, src_labelname...) Vector === -func funcLabelJoin(vals []parser.Value, args parser.Expressions, enh *EvalNodeHelper) Vector { +// === label_join(vector model.ValVector, dest_labelname, separator, src_labelname...) (Vector, Annotations) === +func funcLabelJoin(vals []parser.Value, args parser.Expressions, enh *EvalNodeHelper) (Vector, annotations.Annotations) { var ( vector = vals[0].(Vector) dst = stringFromArg(args[1]) @@ -1265,7 +1355,7 @@ func funcLabelJoin(vals []parser.Value, args parser.Expressions, enh *EvalNodeHe H: el.H, }) } - return enh.Out + return enh.Out, nil } // Common code for date related functions. @@ -1289,59 +1379,59 @@ func dateWrapper(vals []parser.Value, enh *EvalNodeHelper, f func(time.Time) flo } // === days_in_month(v Vector) Scalar === -func funcDaysInMonth(vals []parser.Value, args parser.Expressions, enh *EvalNodeHelper) Vector { +func funcDaysInMonth(vals []parser.Value, args parser.Expressions, enh *EvalNodeHelper) (Vector, annotations.Annotations) { return dateWrapper(vals, enh, func(t time.Time) float64 { return float64(32 - time.Date(t.Year(), t.Month(), 32, 0, 0, 0, 0, time.UTC).Day()) - }) + }), nil } // === day_of_month(v Vector) Scalar === -func funcDayOfMonth(vals []parser.Value, args parser.Expressions, enh *EvalNodeHelper) Vector { +func funcDayOfMonth(vals []parser.Value, args parser.Expressions, enh *EvalNodeHelper) (Vector, annotations.Annotations) { return dateWrapper(vals, enh, func(t time.Time) float64 { return float64(t.Day()) - }) + }), nil } // === day_of_week(v Vector) Scalar === -func funcDayOfWeek(vals []parser.Value, args parser.Expressions, enh *EvalNodeHelper) Vector { +func funcDayOfWeek(vals []parser.Value, args parser.Expressions, enh *EvalNodeHelper) (Vector, annotations.Annotations) { return dateWrapper(vals, enh, func(t time.Time) float64 { return float64(t.Weekday()) - }) + }), nil } // === day_of_year(v Vector) Scalar === -func funcDayOfYear(vals []parser.Value, args parser.Expressions, enh *EvalNodeHelper) Vector { +func funcDayOfYear(vals []parser.Value, args parser.Expressions, enh *EvalNodeHelper) (Vector, annotations.Annotations) { return dateWrapper(vals, enh, func(t time.Time) float64 { return float64(t.YearDay()) - }) + }), nil } // === hour(v Vector) Scalar === -func funcHour(vals []parser.Value, args parser.Expressions, enh *EvalNodeHelper) Vector { +func funcHour(vals []parser.Value, args parser.Expressions, enh *EvalNodeHelper) (Vector, annotations.Annotations) { return dateWrapper(vals, enh, func(t time.Time) float64 { return float64(t.Hour()) - }) + }), nil } // === minute(v Vector) Scalar === -func funcMinute(vals []parser.Value, args parser.Expressions, enh *EvalNodeHelper) Vector { +func funcMinute(vals []parser.Value, args parser.Expressions, enh *EvalNodeHelper) (Vector, annotations.Annotations) { return dateWrapper(vals, enh, func(t time.Time) float64 { return float64(t.Minute()) - }) + }), nil } // === month(v Vector) Scalar === -func funcMonth(vals []parser.Value, args parser.Expressions, enh *EvalNodeHelper) Vector { +func funcMonth(vals []parser.Value, args parser.Expressions, enh *EvalNodeHelper) (Vector, annotations.Annotations) { return dateWrapper(vals, enh, func(t time.Time) float64 { return float64(t.Month()) - }) + }), nil } // === year(v Vector) Scalar === -func funcYear(vals []parser.Value, args parser.Expressions, enh *EvalNodeHelper) Vector { +func funcYear(vals []parser.Value, args parser.Expressions, enh *EvalNodeHelper) (Vector, annotations.Annotations) { return dateWrapper(vals, enh, func(t time.Time) float64 { return float64(t.Year()) - }) + }), nil } // FunctionCalls is a list of all functions supported by PromQL, including their types. @@ -1377,6 +1467,8 @@ var FunctionCalls = map[string]FunctionCall{ "histogram_fraction": funcHistogramFraction, "histogram_quantile": funcHistogramQuantile, "histogram_sum": funcHistogramSum, + "histogram_stddev": funcHistogramStdDev, + "histogram_stdvar": funcHistogramStdVar, "holt_winters": funcHoltWinters, "hour": funcHour, "idelta": funcIdelta, diff --git a/vendor/github.com/prometheus/prometheus/promql/parser/ast.go b/vendor/github.com/prometheus/prometheus/promql/parser/ast.go index 86f1394998..58136266fd 100644 --- a/vendor/github.com/prometheus/prometheus/promql/parser/ast.go +++ b/vendor/github.com/prometheus/prometheus/promql/parser/ast.go @@ -20,6 +20,8 @@ import ( "github.com/prometheus/prometheus/model/labels" "github.com/prometheus/prometheus/storage" + + "github.com/prometheus/prometheus/promql/parser/posrange" ) // Node is a generic interface for all nodes in an AST. @@ -45,7 +47,7 @@ type Node interface { Pretty(level int) string // PositionRange returns the position of the AST Node in the query string. - PositionRange() PositionRange + PositionRange() posrange.PositionRange } // Statement is a generic interface for all statements. @@ -94,7 +96,7 @@ type AggregateExpr struct { Param Expr // Parameter used by some aggregators. Grouping []string // The labels by which to group the Vector. Without bool // Whether to drop the given labels rather than keep them. - PosRange PositionRange + PosRange posrange.PositionRange } // BinaryExpr represents a binary expression between two child expressions. @@ -115,7 +117,7 @@ type Call struct { Func *Function // The function that was called. Args Expressions // Arguments used in the call. - PosRange PositionRange + PosRange posrange.PositionRange } // MatrixSelector represents a Matrix selection. @@ -125,7 +127,7 @@ type MatrixSelector struct { VectorSelector Expr Range time.Duration - EndPos Pos + EndPos posrange.Pos } // SubqueryExpr represents a subquery. @@ -143,27 +145,27 @@ type SubqueryExpr struct { StartOrEnd ItemType // Set when @ is used with start() or end() Step time.Duration - EndPos Pos + EndPos posrange.Pos } // NumberLiteral represents a number. type NumberLiteral struct { Val float64 - PosRange PositionRange + PosRange posrange.PositionRange } // ParenExpr wraps an expression so it cannot be disassembled as a consequence // of operator precedence. type ParenExpr struct { Expr Expr - PosRange PositionRange + PosRange posrange.PositionRange } // StringLiteral represents a string. type StringLiteral struct { Val string - PosRange PositionRange + PosRange posrange.PositionRange } // UnaryExpr represents a unary operation on another expression. @@ -172,7 +174,7 @@ type UnaryExpr struct { Op ItemType Expr Expr - StartPos Pos + StartPos posrange.Pos } // StepInvariantExpr represents a query which evaluates to the same result @@ -184,7 +186,9 @@ type StepInvariantExpr struct { func (e *StepInvariantExpr) String() string { return e.Expr.String() } -func (e *StepInvariantExpr) PositionRange() PositionRange { return e.Expr.PositionRange() } +func (e *StepInvariantExpr) PositionRange() posrange.PositionRange { + return e.Expr.PositionRange() +} // VectorSelector represents a Vector selection. type VectorSelector struct { @@ -204,7 +208,7 @@ type VectorSelector struct { UnexpandedSeriesSet storage.SeriesSet Series []storage.Series - PosRange PositionRange + PosRange posrange.PositionRange } // TestStmt is an internal helper statement that allows execution @@ -215,8 +219,8 @@ func (TestStmt) String() string { return "test statement" } func (TestStmt) PromQLStmt() {} func (t TestStmt) Pretty(int) string { return t.String() } -func (TestStmt) PositionRange() PositionRange { - return PositionRange{ +func (TestStmt) PositionRange() posrange.PositionRange { + return posrange.PositionRange{ Start: -1, End: -1, } @@ -405,17 +409,11 @@ func Children(node Node) []Node { } } -// PositionRange describes a position in the input string of the parser. -type PositionRange struct { - Start Pos - End Pos -} - // mergeRanges is a helper function to merge the PositionRanges of two Nodes. // Note that the arguments must be in the same order as they // occur in the input string. -func mergeRanges(first, last Node) PositionRange { - return PositionRange{ +func mergeRanges(first, last Node) posrange.PositionRange { + return posrange.PositionRange{ Start: first.PositionRange().Start, End: last.PositionRange().End, } @@ -423,33 +421,33 @@ func mergeRanges(first, last Node) PositionRange { // Item implements the Node interface. // This makes it possible to call mergeRanges on them. -func (i *Item) PositionRange() PositionRange { - return PositionRange{ +func (i *Item) PositionRange() posrange.PositionRange { + return posrange.PositionRange{ Start: i.Pos, - End: i.Pos + Pos(len(i.Val)), + End: i.Pos + posrange.Pos(len(i.Val)), } } -func (e *AggregateExpr) PositionRange() PositionRange { +func (e *AggregateExpr) PositionRange() posrange.PositionRange { return e.PosRange } -func (e *BinaryExpr) PositionRange() PositionRange { +func (e *BinaryExpr) PositionRange() posrange.PositionRange { return mergeRanges(e.LHS, e.RHS) } -func (e *Call) PositionRange() PositionRange { +func (e *Call) PositionRange() posrange.PositionRange { return e.PosRange } -func (e *EvalStmt) PositionRange() PositionRange { +func (e *EvalStmt) PositionRange() posrange.PositionRange { return e.Expr.PositionRange() } -func (e Expressions) PositionRange() PositionRange { +func (e Expressions) PositionRange() posrange.PositionRange { if len(e) == 0 { // Position undefined. - return PositionRange{ + return posrange.PositionRange{ Start: -1, End: -1, } @@ -457,39 +455,39 @@ func (e Expressions) PositionRange() PositionRange { return mergeRanges(e[0], e[len(e)-1]) } -func (e *MatrixSelector) PositionRange() PositionRange { - return PositionRange{ +func (e *MatrixSelector) PositionRange() posrange.PositionRange { + return posrange.PositionRange{ Start: e.VectorSelector.PositionRange().Start, End: e.EndPos, } } -func (e *SubqueryExpr) PositionRange() PositionRange { - return PositionRange{ +func (e *SubqueryExpr) PositionRange() posrange.PositionRange { + return posrange.PositionRange{ Start: e.Expr.PositionRange().Start, End: e.EndPos, } } -func (e *NumberLiteral) PositionRange() PositionRange { +func (e *NumberLiteral) PositionRange() posrange.PositionRange { return e.PosRange } -func (e *ParenExpr) PositionRange() PositionRange { +func (e *ParenExpr) PositionRange() posrange.PositionRange { return e.PosRange } -func (e *StringLiteral) PositionRange() PositionRange { +func (e *StringLiteral) PositionRange() posrange.PositionRange { return e.PosRange } -func (e *UnaryExpr) PositionRange() PositionRange { - return PositionRange{ +func (e *UnaryExpr) PositionRange() posrange.PositionRange { + return posrange.PositionRange{ Start: e.StartPos, End: e.Expr.PositionRange().End, } } -func (e *VectorSelector) PositionRange() PositionRange { +func (e *VectorSelector) PositionRange() posrange.PositionRange { return e.PosRange } diff --git a/vendor/github.com/prometheus/prometheus/promql/parser/functions.go b/vendor/github.com/prometheus/prometheus/promql/parser/functions.go index 479c7f635d..45a30219e6 100644 --- a/vendor/github.com/prometheus/prometheus/promql/parser/functions.go +++ b/vendor/github.com/prometheus/prometheus/promql/parser/functions.go @@ -173,6 +173,16 @@ var Functions = map[string]*Function{ ArgTypes: []ValueType{ValueTypeVector}, ReturnType: ValueTypeVector, }, + "histogram_stddev": { + Name: "histogram_stddev", + ArgTypes: []ValueType{ValueTypeVector}, + ReturnType: ValueTypeVector, + }, + "histogram_stdvar": { + Name: "histogram_stdvar", + ArgTypes: []ValueType{ValueTypeVector}, + ReturnType: ValueTypeVector, + }, "histogram_fraction": { Name: "histogram_fraction", ArgTypes: []ValueType{ValueTypeScalar, ValueTypeScalar, ValueTypeVector}, diff --git a/vendor/github.com/prometheus/prometheus/promql/parser/generated_parser.y b/vendor/github.com/prometheus/prometheus/promql/parser/generated_parser.y index b28e9d544c..676fd9fb5b 100644 --- a/vendor/github.com/prometheus/prometheus/promql/parser/generated_parser.y +++ b/vendor/github.com/prometheus/prometheus/promql/parser/generated_parser.y @@ -21,23 +21,29 @@ import ( "github.com/prometheus/prometheus/model/labels" "github.com/prometheus/prometheus/model/value" + "github.com/prometheus/prometheus/model/histogram" + "github.com/prometheus/prometheus/promql/parser/posrange" ) %} %union { - node Node - item Item - matchers []*labels.Matcher - matcher *labels.Matcher - label labels.Label - labels labels.Labels - lblList []labels.Label - strings []string - series []SequenceValue - uint uint64 - float float64 - duration time.Duration + node Node + item Item + matchers []*labels.Matcher + matcher *labels.Matcher + label labels.Label + labels labels.Labels + lblList []labels.Label + strings []string + series []SequenceValue + histogram *histogram.FloatHistogram + descriptors map[string]interface{} + bucket_set []float64 + int int64 + uint uint64 + float float64 + duration time.Duration } @@ -54,6 +60,8 @@ IDENTIFIER LEFT_BRACE LEFT_BRACKET LEFT_PAREN +OPEN_HIST +CLOSE_HIST METRIC_IDENTIFIER NUMBER RIGHT_BRACE @@ -64,6 +72,20 @@ SPACE STRING TIMES +// Histogram Descriptors. +%token histogramDescStart +%token +SUM_DESC +COUNT_DESC +SCHEMA_DESC +OFFSET_DESC +NEGATIVE_OFFSET_DESC +BUCKETS_DESC +NEGATIVE_BUCKETS_DESC +ZERO_BUCKET_DESC +ZERO_BUCKET_WIDTH_DESC +%token histogramDescEnd + // Operators. %token operatorsStart %token @@ -145,6 +167,10 @@ START_METRIC_SELECTOR %type