From 874ea9c79f8ba7bf0250a55b94a9678b1b966af2 Mon Sep 17 00:00:00 2001 From: Charles Korn Date: Fri, 1 Dec 2023 14:39:17 +1100 Subject: [PATCH] Fix issue where failed remote rule evaluation errors are logged without context such as trace IDs (#6789) * Fix issue where failed remote rule evaluation errors are logged without context such as trace IDs. * Add changelog entry. --- CHANGELOG.md | 1 + pkg/ruler/remotequerier.go | 6 +++--- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ce94f2ac3f..2861491f44 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,7 @@ * [BUGFIX] Fix issue where concatenatingChunkIterator can obscure errors #6766 * [BUGFIX] Fix panic during tsdb Commit #6766 * [BUGFIX] tsdb/head: wlog exemplars after samples #6766 +* [BUGFIX] Ruler: fix issue where "failed to remotely evaluate query expression, will retry" messages are logged without context such as the trace ID and do not appear in trace events. #6789 ### Mixin diff --git a/pkg/ruler/remotequerier.go b/pkg/ruler/remotequerier.go index 8d4e59fbb2..8a3268e091 100644 --- a/pkg/ruler/remotequerier.go +++ b/pkg/ruler/remotequerier.go @@ -224,7 +224,7 @@ func (q *RemoteQuerier) query(ctx context.Context, query string, ts time.Time, l ctx, cancel := context.WithTimeout(ctx, q.timeout) defer cancel() - resp, err := q.sendRequest(ctx, &req) + resp, err := q.sendRequest(ctx, &req, logger) if err != nil { level.Warn(logger).Log("msg", "failed to remotely evaluate query expression", "err", err, "qs", query, "tm", ts) return promql.Vector{}, err @@ -282,7 +282,7 @@ func (q *RemoteQuerier) createRequest(ctx context.Context, query string, ts time return req, nil } -func (q *RemoteQuerier) sendRequest(ctx context.Context, req *httpgrpc.HTTPRequest) (*httpgrpc.HTTPResponse, error) { +func (q *RemoteQuerier) sendRequest(ctx context.Context, req *httpgrpc.HTTPRequest, logger log.Logger) (*httpgrpc.HTTPResponse, error) { // Ongoing request may be cancelled during evaluation due to some transient error or server shutdown, // so we'll keep retrying until we get a successful response or backoff is terminated. retryConfig := backoff.Config{ @@ -300,7 +300,7 @@ func (q *RemoteQuerier) sendRequest(ctx context.Context, req *httpgrpc.HTTPReque if !retry.Ongoing() { return nil, err } - level.Warn(q.logger).Log("msg", "failed to remotely evaluate query expression, will retry", "err", err) + level.Warn(logger).Log("msg", "failed to remotely evaluate query expression, will retry", "err", err) retry.Wait() // Avoid masking last known error if context was cancelled while waiting.