From 52d520b5f5a9388e538f224a646c5d8daac94ef7 Mon Sep 17 00:00:00 2001 From: Justin Jung Date: Mon, 16 Oct 2023 13:24:59 -0700 Subject: [PATCH 01/49] Separate user queue channel into two - normalQueue and highPriorityQueue Signed-off-by: Justin Jung --- pkg/frontend/v1/frontend.go | 5 +++ pkg/scheduler/queue/queue.go | 19 ++++++--- pkg/scheduler/queue/queue_test.go | 54 +++++++++++++++++++++++-- pkg/scheduler/queue/user_queues.go | 28 +++++++++---- pkg/scheduler/queue/user_queues_test.go | 29 ++++++++----- pkg/scheduler/scheduler.go | 5 +++ 6 files changed, 112 insertions(+), 28 deletions(-) diff --git a/pkg/frontend/v1/frontend.go b/pkg/frontend/v1/frontend.go index ac5074dd1c..b36a62162e 100644 --- a/pkg/frontend/v1/frontend.go +++ b/pkg/frontend/v1/frontend.go @@ -93,6 +93,11 @@ type request struct { response chan *httpgrpc.HTTPResponse } +func (r request) IsHighPriority() bool { + //TODO implement me + return true +} + // New creates a new frontend. Frontend implements service, and must be started and stopped. func New(cfg Config, limits Limits, log log.Logger, registerer prometheus.Registerer, retry *transport.Retry) (*Frontend, error) { f := &Frontend{ diff --git a/pkg/scheduler/queue/queue.go b/pkg/scheduler/queue/queue.go index 42b8238b56..93095d58f7 100644 --- a/pkg/scheduler/queue/queue.go +++ b/pkg/scheduler/queue/queue.go @@ -44,7 +44,9 @@ func FirstUser() UserIndex { } // Request stored into the queue. -type Request interface{} +type Request interface { + IsHighPriority() bool +} // RequestQueue holds incoming requests in per-user queues. It also assigns each user specified number of queriers, // and when querier asks for next request to handle (using GetNextRequestForQuerier), it returns requests @@ -96,13 +98,21 @@ func (q *RequestQueue) EnqueueRequest(userID string, req Request, maxQueriers fl } shardSize := util.DynamicShardSize(maxQueriers, len(q.queues.queriers)) - queue := q.queues.getOrAddQueue(userID, shardSize) + queue := q.queues.getOrAddQueue(userID, shardSize, req.IsHighPriority()) + maxOutstandingRequests := q.queues.limits.MaxOutstandingPerTenant(userID) + if queue == nil { // This can only happen if userID is "". return errors.New("no queue found") } q.totalRequests.WithLabelValues(userID).Inc() + + if q.queues.getTotalQueueSize(userID) >= maxOutstandingRequests { + q.discardedRequests.WithLabelValues(userID).Inc() + return ErrTooManyRequests + } + select { case queue <- req: q.queueLength.WithLabelValues(userID).Inc() @@ -112,9 +122,6 @@ func (q *RequestQueue) EnqueueRequest(userID string, req Request, maxQueriers fl successFn() } return nil - default: - q.discardedRequests.WithLabelValues(userID).Inc() - return ErrTooManyRequests } } @@ -152,7 +159,7 @@ FindQueue: // Pick next request from the queue. for { request := <-queue - if len(queue) == 0 { + if q.queues.getTotalQueueSize(userID) == 0 { q.queues.deleteQueue(userID) } diff --git a/pkg/scheduler/queue/queue_test.go b/pkg/scheduler/queue/queue_test.go index 8e8f0a94d0..18edb80989 100644 --- a/pkg/scheduler/queue/queue_test.go +++ b/pkg/scheduler/queue/queue_test.go @@ -39,7 +39,7 @@ func BenchmarkGetNextRequest(b *testing.B) { for j := 0; j < numTenants; j++ { userID := strconv.Itoa(j) - err := queue.EnqueueRequest(userID, "request", 0, nil) + err := queue.EnqueueRequest(userID, MockRequest{}, 0, nil) if err != nil { b.Fatal(err) } @@ -79,7 +79,7 @@ func BenchmarkQueueRequest(b *testing.B) { queues := make([]*RequestQueue, 0, b.N) users := make([]string, 0, numTenants) - requests := make([]string, 0, numTenants) + requests := make([]MockRequest, 0, numTenants) for n := 0; n < b.N; n++ { q := NewRequestQueue(maxOutstandingPerTenant, 0, @@ -96,7 +96,7 @@ func BenchmarkQueueRequest(b *testing.B) { queues = append(queues, q) for j := 0; j < numTenants; j++ { - requests = append(requests, fmt.Sprintf("%d-%d", n, j)) + requests = append(requests, MockRequest{id: fmt.Sprintf("%d-%d", n, j)}) users = append(users, strconv.Itoa(j)) } } @@ -149,7 +149,7 @@ func TestRequestQueue_GetNextRequestForQuerier_ShouldGetRequestAfterReshardingBe // Enqueue a request from an user which would be assigned to querier-1. // NOTE: "user-1" hash falls in the querier-1 shard. - require.NoError(t, queue.EnqueueRequest("user-1", "request", 1, nil)) + require.NoError(t, queue.EnqueueRequest("user-1", MockRequest{}, 1, nil)) startTime := time.Now() querier2wg.Wait() @@ -158,3 +158,49 @@ func TestRequestQueue_GetNextRequestForQuerier_ShouldGetRequestAfterReshardingBe // We expect that querier-2 got the request only after querier-1 forget delay is passed. assert.GreaterOrEqual(t, waitTime.Milliseconds(), forgetDelay.Milliseconds()) } + +func TestRequestQueue_HighPriority(t *testing.T) { + queue := NewRequestQueue(3, 0, + prometheus.NewGaugeVec(prometheus.GaugeOpts{}, []string{"user"}), + prometheus.NewCounterVec(prometheus.CounterOpts{}, []string{"user"}), + MockLimits{MaxOutstanding: 3}, + nil, + ) + ctx := context.Background() + queue.RegisterQuerierConnection("querier-1") + + normalRequest1 := MockRequest{ + id: "normal query 1", + isHighPriority: false, + } + normalRequest2 := MockRequest{ + id: "normal query 1", + isHighPriority: false, + } + highPriorityRequest := MockRequest{ + id: "high priority query", + isHighPriority: true, + } + + queue.EnqueueRequest("userID", normalRequest1, 1, func() {}) + queue.EnqueueRequest("userID", normalRequest2, 1, func() {}) + queue.EnqueueRequest("userID", highPriorityRequest, 1, func() {}) + + assert.Error(t, queue.EnqueueRequest("userID", highPriorityRequest, 1, func() {})) // should fail due to maxOutstandingPerTenant = 3 + assert.Equal(t, 3, queue.queues.getTotalQueueSize("userID")) + nextRequest, _, _ := queue.GetNextRequestForQuerier(ctx, FirstUser(), "querier-1") + assert.Equal(t, highPriorityRequest, nextRequest) // high priority request returned, although it was enqueued the last + nextRequest, _, _ = queue.GetNextRequestForQuerier(ctx, FirstUser(), "querier-1") + assert.Equal(t, normalRequest1, nextRequest) + nextRequest, _, _ = queue.GetNextRequestForQuerier(ctx, FirstUser(), "querier-1") + assert.Equal(t, normalRequest2, nextRequest) +} + +type MockRequest struct { + id string + isHighPriority bool +} + +func (r MockRequest) IsHighPriority() bool { + return r.isHighPriority +} diff --git a/pkg/scheduler/queue/user_queues.go b/pkg/scheduler/queue/user_queues.go index 0f9cd1a081..9ab34f9820 100644 --- a/pkg/scheduler/queue/user_queues.go +++ b/pkg/scheduler/queue/user_queues.go @@ -53,7 +53,8 @@ type queues struct { } type userQueue struct { - ch chan Request + normalQueue chan Request + highPriorityQueue chan Request // If not nil, only these queriers can handle user requests. If nil, all queriers can. // We set this to nil if number of available queriers <= maxQueriers. @@ -103,7 +104,7 @@ func (q *queues) deleteQueue(userID string) { // MaxQueriers is used to compute which queriers should handle requests for this user. // If maxQueriers is <= 0, all queriers can handle this user's requests. // If maxQueriers has changed since the last call, queriers for this are recomputed. -func (q *queues) getOrAddQueue(userID string, maxQueriers int) chan Request { +func (q *queues) getOrAddQueue(userID string, maxQueriers int, isHighPriority bool) chan Request { // Empty user is not allowed, as that would break our users list ("" is used for free spot). if userID == "" { return nil @@ -123,9 +124,10 @@ func (q *queues) getOrAddQueue(userID string, maxQueriers int) chan Request { queueSize = q.maxUserQueueSize } uq = &userQueue{ - ch: make(chan Request, queueSize), - seed: util.ShuffleShardSeed(userID, ""), - index: -1, + normalQueue: make(chan Request, queueSize), + highPriorityQueue: make(chan Request, queueSize), + seed: util.ShuffleShardSeed(userID, ""), + index: -1, } q.userQueues[userID] = uq @@ -150,7 +152,15 @@ func (q *queues) getOrAddQueue(userID string, maxQueriers int) chan Request { uq.queriers = shuffleQueriersForUser(uq.seed, maxQueriers, q.sortedQueriers, nil) } - return uq.ch + if isHighPriority { + return uq.highPriorityQueue + } + + return uq.normalQueue +} + +func (q *queues) getTotalQueueSize(userID string) int { + return len(q.userQueues[userID].normalQueue) + len(q.userQueues[userID].highPriorityQueue) } // Finds next queue for the querier. To support fair scheduling between users, client is expected @@ -182,7 +192,11 @@ func (q *queues) getNextQueueForQuerier(lastUserIndex int, querierID string) (ch } } - return q.ch, u, uid + if len(q.highPriorityQueue) > 0 { + return q.highPriorityQueue, u, uid + } + + return q.normalQueue, u, uid } return nil, "", uid } diff --git a/pkg/scheduler/queue/user_queues_test.go b/pkg/scheduler/queue/user_queues_test.go index cc986cac41..56f05c6f48 100644 --- a/pkg/scheduler/queue/user_queues_test.go +++ b/pkg/scheduler/queue/user_queues_test.go @@ -5,6 +5,7 @@ import ( "math" "math/rand" "sort" + "sync" "testing" "time" @@ -22,11 +23,11 @@ func TestQueues(t *testing.T) { assert.Equal(t, "", u) // Add queues: [one] - qOne := getOrAdd(t, uq, "one", 0) + qOne := getOrAdd(t, uq, "one", 0, false) lastUserIndex = confirmOrderForQuerier(t, uq, "querier-1", lastUserIndex, qOne, qOne) // [one two] - qTwo := getOrAdd(t, uq, "two", 0) + qTwo := getOrAdd(t, uq, "two", 0, false) assert.NotEqual(t, qOne, qTwo) lastUserIndex = confirmOrderForQuerier(t, uq, "querier-1", lastUserIndex, qTwo, qOne, qTwo, qOne) @@ -34,7 +35,7 @@ func TestQueues(t *testing.T) { // [one two three] // confirm fifo by adding a third queue and iterating to it - qThree := getOrAdd(t, uq, "three", 0) + qThree := getOrAdd(t, uq, "three", 0, false) lastUserIndex = confirmOrderForQuerier(t, uq, "querier-1", lastUserIndex, qTwo, qThree, qOne) @@ -45,7 +46,7 @@ func TestQueues(t *testing.T) { lastUserIndex = confirmOrderForQuerier(t, uq, "querier-1", lastUserIndex, qTwo, qThree, qTwo) // "four" is added at the beginning of the list: [four two three] - qFour := getOrAdd(t, uq, "four", 0) + qFour := getOrAdd(t, uq, "four", 0, false) lastUserIndex = confirmOrderForQuerier(t, uq, "querier-1", lastUserIndex, qThree, qFour, qTwo, qThree) @@ -92,7 +93,7 @@ func TestQueuesWithQueriers(t *testing.T) { // Add user queues. for u := 0; u < users; u++ { uid := fmt.Sprintf("user-%d", u) - getOrAdd(t, uq, uid, maxQueriersPerUser) + getOrAdd(t, uq, uid, maxQueriersPerUser, false) // Verify it has maxQueriersPerUser queriers assigned now. qs := uq.userQueues[uid].queriers @@ -156,9 +157,10 @@ func TestQueuesConsistency(t *testing.T) { conns := map[string]int{} for i := 0; i < 10000; i++ { + queue := uq.getOrAddQueue(generateTenant(r), 3, false) switch r.Int() % 6 { case 0: - assert.NotNil(t, uq.getOrAddQueue(generateTenant(r), 3)) + assert.NotNil(t, queue) case 1: qid := generateQuerier(r) _, _, luid := uq.getNextQueueForQuerier(lastUserIndexes[qid], qid) @@ -207,7 +209,7 @@ func TestQueues_ForgetDelay(t *testing.T) { // Add user queues. for i := 0; i < numUsers; i++ { userID := fmt.Sprintf("user-%d", i) - getOrAdd(t, uq, userID, maxQueriersPerUser) + getOrAdd(t, uq, userID, maxQueriersPerUser, false) } // We expect querier-1 to have some users. @@ -299,7 +301,7 @@ func TestQueues_ForgetDelay_ShouldCorrectlyHandleQuerierReconnectingBeforeForget // Add user queues. for i := 0; i < numUsers; i++ { userID := fmt.Sprintf("user-%d", i) - getOrAdd(t, uq, userID, maxQueriersPerUser) + getOrAdd(t, uq, userID, maxQueriersPerUser, false) } // We expect querier-1 to have some users. @@ -359,11 +361,11 @@ func generateQuerier(r *rand.Rand) string { return fmt.Sprint("querier-", r.Int()%5) } -func getOrAdd(t *testing.T, uq *queues, tenant string, maxQueriers int) chan Request { - q := uq.getOrAddQueue(tenant, maxQueriers) +func getOrAdd(t *testing.T, uq *queues, tenant string, maxQueriers int, isHighPriority bool) chan Request { + q := uq.getOrAddQueue(tenant, maxQueriers, isHighPriority) assert.NotNil(t, q) assert.NoError(t, isConsistent(uq)) - assert.Equal(t, q, uq.getOrAddQueue(tenant, maxQueriers)) + assert.Equal(t, q, uq.getOrAddQueue(tenant, maxQueriers, isHighPriority)) return q } @@ -437,6 +439,11 @@ func getUsersByQuerier(queues *queues, querierID string) []string { return userIDs } +func enqueueRequest(queue chan Request, request Request, wg *sync.WaitGroup) { + defer wg.Done() + queue <- request +} + func TestShuffleQueriers(t *testing.T) { allQueriers := []string{"a", "b", "c", "d", "e"} diff --git a/pkg/scheduler/scheduler.go b/pkg/scheduler/scheduler.go index f03def5ea3..602bc0d296 100644 --- a/pkg/scheduler/scheduler.go +++ b/pkg/scheduler/scheduler.go @@ -165,6 +165,11 @@ type schedulerRequest struct { parentSpanContext opentracing.SpanContext } +func (s schedulerRequest) IsHighPriority() bool { + //TODO implement me + return true +} + // FrontendLoop handles connection from frontend. func (s *Scheduler) FrontendLoop(frontend schedulerpb.SchedulerForFrontend_FrontendLoopServer) error { frontendAddress, frontendCtx, err := s.frontendConnected(frontend) From 646853a49579c63b3fb2bf3a30cd5b85d6f06981 Mon Sep 17 00:00:00 2001 From: Justin Jung Date: Tue, 17 Oct 2023 10:12:27 -0700 Subject: [PATCH 02/49] Add ReservedHighPriorityQueriers Signed-off-by: Justin Jung --- pkg/scheduler/queue/queue.go | 2 +- pkg/scheduler/queue/queue_test.go | 39 +++++++++++- pkg/scheduler/queue/user_queues.go | 75 +++++++++++++++++----- pkg/scheduler/queue/user_queues_test.go | 84 +++++++++++++++++++++---- pkg/util/validation/limits.go | 9 ++- 5 files changed, 179 insertions(+), 30 deletions(-) diff --git a/pkg/scheduler/queue/queue.go b/pkg/scheduler/queue/queue.go index 93095d58f7..0597f2f76e 100644 --- a/pkg/scheduler/queue/queue.go +++ b/pkg/scheduler/queue/queue.go @@ -152,7 +152,7 @@ FindQueue: for { queue, userID, idx := q.queues.getNextQueueForQuerier(last.last, querierID) last.last = idx - if queue == nil { + if queue == nil || len(queue) < 1 { break } diff --git a/pkg/scheduler/queue/queue_test.go b/pkg/scheduler/queue/queue_test.go index 18edb80989..bb5c86a22c 100644 --- a/pkg/scheduler/queue/queue_test.go +++ b/pkg/scheduler/queue/queue_test.go @@ -159,7 +159,7 @@ func TestRequestQueue_GetNextRequestForQuerier_ShouldGetRequestAfterReshardingBe assert.GreaterOrEqual(t, waitTime.Milliseconds(), forgetDelay.Milliseconds()) } -func TestRequestQueue_HighPriority(t *testing.T) { +func TestRequestQueue_QueriersShouldGetHighPriorityQueryFirst(t *testing.T) { queue := NewRequestQueue(3, 0, prometheus.NewGaugeVec(prometheus.GaugeOpts{}, []string{"user"}), prometheus.NewCounterVec(prometheus.CounterOpts{}, []string{"user"}), @@ -196,6 +196,43 @@ func TestRequestQueue_HighPriority(t *testing.T) { assert.Equal(t, normalRequest2, nextRequest) } +func TestRequestQueue_ReservedQueriersShouldOnlyGetHighPriorityQueries(t *testing.T) { + queue := NewRequestQueue(3, 0, + prometheus.NewGaugeVec(prometheus.GaugeOpts{}, []string{"user"}), + prometheus.NewCounterVec(prometheus.CounterOpts{}, []string{"user"}), + MockLimits{MaxOutstanding: 3, ReservedQueriers: 1}, + nil, + ) + ctx := context.Background() + + queue.RegisterQuerierConnection("querier-1") + + normalRequest := MockRequest{ + id: "normal query", + isHighPriority: false, + } + highPriorityRequest := MockRequest{ + id: "high priority query", + isHighPriority: true, + } + + queue.EnqueueRequest("userID", normalRequest, 1, func() {}) + queue.EnqueueRequest("userID", highPriorityRequest, 1, func() {}) + + nextRequest, _, _ := queue.GetNextRequestForQuerier(ctx, FirstUser(), "querier-1") + assert.Equal(t, highPriorityRequest, nextRequest) + + ctxTimeout, cancel := context.WithTimeout(ctx, 1*time.Second) + defer cancel() + + time.AfterFunc(2*time.Second, func() { + queue.cond.Broadcast() + }) + nextRequest, _, _ = queue.GetNextRequestForQuerier(ctxTimeout, FirstUser(), "querier-1") + assert.Nil(t, nextRequest) + assert.Equal(t, 1, queue.queues.getTotalQueueSize("userID")) +} + type MockRequest struct { id string isHighPriority bool diff --git a/pkg/scheduler/queue/user_queues.go b/pkg/scheduler/queue/user_queues.go index 9ab34f9820..fa6ac4296d 100644 --- a/pkg/scheduler/queue/user_queues.go +++ b/pkg/scheduler/queue/user_queues.go @@ -1,6 +1,7 @@ package queue import ( + "math" "math/rand" "sort" "time" @@ -13,6 +14,13 @@ type Limits interface { // MaxOutstandingPerTenant returns the limit to the maximum number // of outstanding requests per tenant per request queue. MaxOutstandingPerTenant(user string) int + + // ReservedHighPriorityQueriers returns the minimum number of queriers dedicated for high priority + // queue per tenant. All queriers still handle priority queue first, but this provides extra protection on + // high priority queries from slow normal queries exhausting all queriers. + // If ReservedHighPriorityQueriers is capped by MaxQueriersPerUser. + // If less than 1, it will be applied as a percentage of MaxQueriersPerUser. + ReservedHighPriorityQueriers(user string) float64 } // querier holds information about a querier registered in the queue. @@ -58,8 +66,9 @@ type userQueue struct { // If not nil, only these queriers can handle user requests. If nil, all queriers can. // We set this to nil if number of available queriers <= maxQueriers. - queriers map[string]struct{} - maxQueriers int + queriers map[string]struct{} + reservedQueriers map[string]struct{} + maxQueriers int // Seed for shuffle sharding of queriers. This seed is based on userID only and is therefore consistent // between different frontends. @@ -149,7 +158,7 @@ func (q *queues) getOrAddQueue(userID string, maxQueriers int, isHighPriority bo if uq.maxQueriers != maxQueriers { uq.maxQueriers = maxQueriers - uq.queriers = shuffleQueriersForUser(uq.seed, maxQueriers, q.sortedQueriers, nil) + uq.queriers, uq.reservedQueriers = shuffleQueriersForUser(uq.seed, maxQueriers, q.sortedQueriers, q.limits.ReservedHighPriorityQueriers(userID), nil) } if isHighPriority { @@ -183,20 +192,21 @@ func (q *queues) getNextQueueForQuerier(lastUserIndex int, querierID string) (ch continue } - q := q.userQueues[u] + uq := q.userQueues[u] - if q.queriers != nil { - if _, ok := q.queriers[querierID]; !ok { + if uq.queriers != nil { + if _, ok := uq.queriers[querierID]; !ok { // This querier is not handling the user. continue } } - if len(q.highPriorityQueue) > 0 { - return q.highPriorityQueue, u, uid + _, isReserved := uq.reservedQueriers[querierID] + if isReserved || len(uq.highPriorityQueue) > 0 { + return uq.highPriorityQueue, u, uid } - return q.normalQueue, u, uid + return uq.normalQueue, u, uid } return nil, "", uid } @@ -304,19 +314,26 @@ func (q *queues) recomputeUserQueriers() { scratchpad := make([]string, 0, len(q.sortedQueriers)) for _, uq := range q.userQueues { - uq.queriers = shuffleQueriersForUser(uq.seed, uq.maxQueriers, q.sortedQueriers, scratchpad) + userID := q.users[uq.index] + uq.queriers, uq.reservedQueriers = shuffleQueriersForUser(uq.seed, uq.maxQueriers, q.sortedQueriers, q.limits.ReservedHighPriorityQueriers(userID), scratchpad) } } // shuffleQueriersForUser returns nil if queriersToSelect is 0 or there are not enough queriers to select from. // In that case *all* queriers should be used. // Scratchpad is used for shuffling, to avoid new allocations. If nil, new slice is allocated. -func shuffleQueriersForUser(userSeed int64, queriersToSelect int, allSortedQueriers []string, scratchpad []string) map[string]struct{} { +func shuffleQueriersForUser(userSeed int64, queriersToSelect int, allSortedQueriers []string, numOfReservedQueriersFloat float64, scratchpad []string) (map[string]struct{}, map[string]struct{}) { + numOfReservedQueriers := getNumOfReservedQueriers(queriersToSelect, len(allSortedQueriers), numOfReservedQueriersFloat) + reservedQueriers := make(map[string]struct{}, numOfReservedQueriers) + if queriersToSelect == 0 || len(allSortedQueriers) <= queriersToSelect { - return nil + for i := 0; i < numOfReservedQueriers; i++ { + reservedQueriers[allSortedQueriers[i]] = struct{}{} + } + return nil, reservedQueriers } - result := make(map[string]struct{}, queriersToSelect) + queriers := make(map[string]struct{}, queriersToSelect) rnd := rand.New(rand.NewSource(userSeed)) scratchpad = scratchpad[:0] @@ -325,20 +342,46 @@ func shuffleQueriersForUser(userSeed int64, queriersToSelect int, allSortedQueri last := len(scratchpad) - 1 for i := 0; i < queriersToSelect; i++ { r := rnd.Intn(last + 1) - result[scratchpad[r]] = struct{}{} + queriers[scratchpad[r]] = struct{}{} + if i < numOfReservedQueriers { + reservedQueriers[scratchpad[r]] = struct{}{} + } // move selected item to the end, it won't be selected anymore. scratchpad[r], scratchpad[last] = scratchpad[last], scratchpad[r] last-- } - return result + return queriers, reservedQueriers +} + +func getNumOfReservedQueriers(queriersToSelect int, totalNumOfQueriers int, reservedQueriers float64) int { + numOfReservedQueriers := int(reservedQueriers) + + if reservedQueriers < 1 && reservedQueriers > 0 { + if queriersToSelect == 0 || queriersToSelect > totalNumOfQueriers { + queriersToSelect = totalNumOfQueriers + } + + numOfReservedQueriers = int(math.Ceil(float64(queriersToSelect) * reservedQueriers)) + } + + if numOfReservedQueriers > totalNumOfQueriers { + return totalNumOfQueriers + } + + return numOfReservedQueriers } // MockLimits implements the Limits interface. Used in tests only. type MockLimits struct { - MaxOutstanding int + MaxOutstanding int + ReservedQueriers float64 } func (l MockLimits) MaxOutstandingPerTenant(_ string) int { return l.MaxOutstanding } + +func (l MockLimits) ReservedHighPriorityQueriers(_ string) float64 { + return l.ReservedQueriers +} diff --git a/pkg/scheduler/queue/user_queues_test.go b/pkg/scheduler/queue/user_queues_test.go index 56f05c6f48..2d47dd6c48 100644 --- a/pkg/scheduler/queue/user_queues_test.go +++ b/pkg/scheduler/queue/user_queues_test.go @@ -5,7 +5,6 @@ import ( "math" "math/rand" "sort" - "sync" "testing" "time" @@ -439,22 +438,20 @@ func getUsersByQuerier(queues *queues, querierID string) []string { return userIDs } -func enqueueRequest(queue chan Request, request Request, wg *sync.WaitGroup) { - defer wg.Done() - queue <- request -} - func TestShuffleQueriers(t *testing.T) { allQueriers := []string{"a", "b", "c", "d", "e"} - require.Nil(t, shuffleQueriersForUser(12345, 10, allQueriers, nil)) - require.Nil(t, shuffleQueriersForUser(12345, len(allQueriers), allQueriers, nil)) + queriers, _ := shuffleQueriersForUser(12345, 10, allQueriers, 0, nil) + require.Nil(t, queriers) + + queriers, _ = shuffleQueriersForUser(12345, len(allQueriers), allQueriers, 0, nil) + require.Nil(t, queriers) - r1 := shuffleQueriersForUser(12345, 3, allQueriers, nil) + r1, _ := shuffleQueriersForUser(12345, 3, allQueriers, 0, nil) require.Equal(t, 3, len(r1)) // Same input produces same output. - r2 := shuffleQueriersForUser(12345, 3, allQueriers, nil) + r2, _ := shuffleQueriersForUser(12345, 3, allQueriers, 0, nil) require.Equal(t, 3, len(r2)) require.Equal(t, r1, r2) } @@ -476,7 +473,7 @@ func TestShuffleQueriersCorrectness(t *testing.T) { toSelect = 3 } - selected := shuffleQueriersForUser(r.Int63(), toSelect, allSortedQueriers, nil) + selected, _ := shuffleQueriersForUser(r.Int63(), toSelect, allSortedQueriers, 0, nil) require.Equal(t, toSelect, len(selected)) @@ -491,3 +488,68 @@ func TestShuffleQueriersCorrectness(t *testing.T) { } } } + +func TestShuffleQueriers_WithReservedQueriers(t *testing.T) { + allQueriers := []string{"a", "b", "c", "d", "e"} + + queriers, reservedQueriers := shuffleQueriersForUser(12345, 0, allQueriers, 0, nil) + require.Nil(t, queriers) + require.Equal(t, 0, len(reservedQueriers)) + + queriers, reservedQueriers = shuffleQueriersForUser(12345, 0, allQueriers, 0.5, nil) + require.Nil(t, queriers) + require.Equal(t, 3, len(reservedQueriers)) + + queriers, reservedQueriers = shuffleQueriersForUser(12345, 0, allQueriers, 1, nil) + require.Nil(t, queriers) + require.Equal(t, 1, len(reservedQueriers)) + + queriers, reservedQueriers = shuffleQueriersForUser(12345, 0, allQueriers, 100, nil) + require.Nil(t, queriers) + require.Equal(t, 5, len(reservedQueriers)) + + queriers, reservedQueriers = shuffleQueriersForUser(12345, 3, allQueriers, 0, nil) + require.Equal(t, 3, len(queriers)) + require.Equal(t, 0, len(reservedQueriers)) + + queriers, reservedQueriers = shuffleQueriersForUser(12345, 3, allQueriers, 0.5, nil) + require.Equal(t, 3, len(queriers)) + require.Equal(t, 2, len(reservedQueriers)) + + queriers, reservedQueriers = shuffleQueriersForUser(12345, 3, allQueriers, 1, nil) + require.Equal(t, 3, len(queriers)) + require.Equal(t, 1, len(reservedQueriers)) + + queriers, reservedQueriers = shuffleQueriersForUser(12345, 3, allQueriers, 100, nil) + require.Equal(t, 3, len(queriers)) + require.Equal(t, 3, len(reservedQueriers)) + + queriers, reservedQueriers = shuffleQueriersForUser(12345, 100, allQueriers, 0, nil) + require.Nil(t, queriers) + require.Equal(t, 0, len(reservedQueriers)) + + queriers, reservedQueriers = shuffleQueriersForUser(12345, 100, allQueriers, 0.5, nil) + require.Nil(t, queriers) + require.Equal(t, 3, len(reservedQueriers)) + + queriers, reservedQueriers = shuffleQueriersForUser(12345, 100, allQueriers, 1, nil) + require.Nil(t, queriers) + require.Equal(t, 1, len(reservedQueriers)) + + queriers, reservedQueriers = shuffleQueriersForUser(12345, 100, allQueriers, 100, nil) + require.Nil(t, queriers) + require.Equal(t, 5, len(reservedQueriers)) +} + +func TestShuffleQueriers_WithReservedQueriers_Correctness(t *testing.T) { + allQueriers := []string{"a", "b", "c", "d", "e", "f", "g", "h", "i", "j", "k", "l", "m", "n"} + + prevQueriers, prevReservedQueriers := shuffleQueriersForUser(12345, 10, allQueriers, 5, nil) + for i := 0; i < 100; i++ { + queriers, reservedQueriers := shuffleQueriersForUser(12345, 10, allQueriers, 5, nil) + require.Equal(t, prevQueriers, queriers) + require.Equal(t, prevReservedQueriers, reservedQueriers) + prevQueriers = queriers + prevReservedQueriers = reservedQueriers + } +} diff --git a/pkg/util/validation/limits.go b/pkg/util/validation/limits.go index d60b2ca8e6..20e89f15df 100644 --- a/pkg/util/validation/limits.go +++ b/pkg/util/validation/limits.go @@ -101,7 +101,8 @@ type Limits struct { QueryVerticalShardSize int `yaml:"query_vertical_shard_size" json:"query_vertical_shard_size" doc:"hidden"` // Query Frontend / Scheduler enforced limits. - MaxOutstandingPerTenant int `yaml:"max_outstanding_requests_per_tenant" json:"max_outstanding_requests_per_tenant"` + MaxOutstandingPerTenant int `yaml:"max_outstanding_requests_per_tenant" json:"max_outstanding_requests_per_tenant"` + ReservedHighPriorityQueriers float64 `yaml:"reserved_high_priority_queriers" json:"reserved_high_priority_queriers"` // Ruler defaults and limits. RulerEvaluationDelay model.Duration `yaml:"ruler_evaluation_delay_duration" json:"ruler_evaluation_delay_duration"` @@ -188,6 +189,7 @@ func (l *Limits) RegisterFlags(f *flag.FlagSet) { f.IntVar(&l.QueryVerticalShardSize, "frontend.query-vertical-shard-size", 0, "[Experimental] Number of shards to use when distributing shardable PromQL queries.") f.IntVar(&l.MaxOutstandingPerTenant, "frontend.max-outstanding-requests-per-tenant", 100, "Maximum number of outstanding requests per tenant per request queue (either query frontend or query scheduler); requests beyond this error with HTTP 429.") + f.Float64Var(&l.ReservedHighPriorityQueriers, "frontend.reserved-high-priority-queriers", 0, "Number of reserved queriers to only handle high priority queue (either query frontend or query scheduler).") f.Var(&l.RulerEvaluationDelay, "ruler.evaluation-delay-duration", "Duration to delay the evaluation of rules to ensure the underlying metrics have been pushed to Cortex.") f.IntVar(&l.RulerTenantShardSize, "ruler.tenant-shard-size", 0, "The default tenant's shard size when the shuffle-sharding strategy is used by ruler. When this setting is specified in the per-tenant overrides, a value of 0 disables shuffle sharding for the tenant.") @@ -492,6 +494,11 @@ func (o *Overrides) MaxOutstandingPerTenant(userID string) int { return o.GetOverridesForUser(userID).MaxOutstandingPerTenant } +// ReservedHighPriorityQueriers returns the number of reserved queriers that only handle high priority queue for the tenant. +func (o *Overrides) ReservedHighPriorityQueriers(userID string) float64 { + return o.GetOverridesForUser(userID).ReservedHighPriorityQueriers +} + // EnforceMetricName whether to enforce the presence of a metric name. func (o *Overrides) EnforceMetricName(userID string) bool { return o.GetOverridesForUser(userID).EnforceMetricName From 39639c402fd1703c3d396f68acfebb3a4b5e4d0c Mon Sep 17 00:00:00 2001 From: Justin Jung Date: Tue, 17 Oct 2023 10:16:42 -0700 Subject: [PATCH 03/49] Change default priority for all requests to low Signed-off-by: Justin Jung --- pkg/frontend/v1/frontend.go | 2 +- pkg/scheduler/scheduler.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/frontend/v1/frontend.go b/pkg/frontend/v1/frontend.go index b36a62162e..99e5d8eff1 100644 --- a/pkg/frontend/v1/frontend.go +++ b/pkg/frontend/v1/frontend.go @@ -95,7 +95,7 @@ type request struct { func (r request) IsHighPriority() bool { //TODO implement me - return true + return false } // New creates a new frontend. Frontend implements service, and must be started and stopped. diff --git a/pkg/scheduler/scheduler.go b/pkg/scheduler/scheduler.go index 602bc0d296..42a7b7413a 100644 --- a/pkg/scheduler/scheduler.go +++ b/pkg/scheduler/scheduler.go @@ -167,7 +167,7 @@ type schedulerRequest struct { func (s schedulerRequest) IsHighPriority() bool { //TODO implement me - return true + return false } // FrontendLoop handles connection from frontend. From 38945ac1deedfed55e93713ba046be13f812f221 Mon Sep 17 00:00:00 2001 From: Justin Jung Date: Tue, 17 Oct 2023 11:23:17 -0700 Subject: [PATCH 04/49] Update config description Signed-off-by: Justin Jung --- docs/configuration/config-file-reference.md | 6 ++++++ pkg/util/validation/limits.go | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/docs/configuration/config-file-reference.md b/docs/configuration/config-file-reference.md index c2c07da5e4..3defa44f4c 100644 --- a/docs/configuration/config-file-reference.md +++ b/docs/configuration/config-file-reference.md @@ -3041,6 +3041,12 @@ The `limits_config` configures default and per-tenant limits imposed by Cortex s # CLI flag: -frontend.max-outstanding-requests-per-tenant [max_outstanding_requests_per_tenant: | default = 100] +# Number of reserved queriers to only handle high priority queue (either query +# frontend or query scheduler). If the value is between 0 and 1, it will be used +# as a percentage of per-tenant queriers. +# CLI flag: -frontend.reserved-high-priority-queriers +[reserved_high_priority_queriers: | default = 0] + # Duration to delay the evaluation of rules to ensure the underlying metrics # have been pushed to Cortex. # CLI flag: -ruler.evaluation-delay-duration diff --git a/pkg/util/validation/limits.go b/pkg/util/validation/limits.go index 20e89f15df..94db111b77 100644 --- a/pkg/util/validation/limits.go +++ b/pkg/util/validation/limits.go @@ -189,7 +189,7 @@ func (l *Limits) RegisterFlags(f *flag.FlagSet) { f.IntVar(&l.QueryVerticalShardSize, "frontend.query-vertical-shard-size", 0, "[Experimental] Number of shards to use when distributing shardable PromQL queries.") f.IntVar(&l.MaxOutstandingPerTenant, "frontend.max-outstanding-requests-per-tenant", 100, "Maximum number of outstanding requests per tenant per request queue (either query frontend or query scheduler); requests beyond this error with HTTP 429.") - f.Float64Var(&l.ReservedHighPriorityQueriers, "frontend.reserved-high-priority-queriers", 0, "Number of reserved queriers to only handle high priority queue (either query frontend or query scheduler).") + f.Float64Var(&l.ReservedHighPriorityQueriers, "frontend.reserved-high-priority-queriers", 0, "Number of reserved queriers to only handle high priority queue (either query frontend or query scheduler). If the value is between 0 and 1, it will be used as a percentage of per-tenant queriers.") f.Var(&l.RulerEvaluationDelay, "ruler.evaluation-delay-duration", "Duration to delay the evaluation of rules to ensure the underlying metrics have been pushed to Cortex.") f.IntVar(&l.RulerTenantShardSize, "ruler.tenant-shard-size", 0, "The default tenant's shard size when the shuffle-sharding strategy is used by ruler. When this setting is specified in the per-tenant overrides, a value of 0 disables shuffle sharding for the tenant.") From d0f7e741d923eb3805e3973a923729ac5e524b99 Mon Sep 17 00:00:00 2001 From: Justin Jung Date: Tue, 17 Oct 2023 11:27:33 -0700 Subject: [PATCH 05/49] Lint Signed-off-by: Justin Jung --- pkg/scheduler/queue/queue.go | 16 +++++++--------- pkg/scheduler/queue/queue_test.go | 10 +++++----- 2 files changed, 12 insertions(+), 14 deletions(-) diff --git a/pkg/scheduler/queue/queue.go b/pkg/scheduler/queue/queue.go index 0597f2f76e..0931b25f3b 100644 --- a/pkg/scheduler/queue/queue.go +++ b/pkg/scheduler/queue/queue.go @@ -113,16 +113,14 @@ func (q *RequestQueue) EnqueueRequest(userID string, req Request, maxQueriers fl return ErrTooManyRequests } - select { - case queue <- req: - q.queueLength.WithLabelValues(userID).Inc() - q.cond.Broadcast() - // Call this function while holding a lock. This guarantees that no querier can fetch the request before function returns. - if successFn != nil { - successFn() - } - return nil + queue <- req + q.queueLength.WithLabelValues(userID).Inc() + q.cond.Broadcast() + // Call this function while holding a lock. This guarantees that no querier can fetch the request before function returns. + if successFn != nil { + successFn() } + return nil } // GetNextRequestForQuerier find next user queue and takes the next request off of it. Will block if there are no requests. diff --git a/pkg/scheduler/queue/queue_test.go b/pkg/scheduler/queue/queue_test.go index bb5c86a22c..426af2d520 100644 --- a/pkg/scheduler/queue/queue_test.go +++ b/pkg/scheduler/queue/queue_test.go @@ -182,9 +182,9 @@ func TestRequestQueue_QueriersShouldGetHighPriorityQueryFirst(t *testing.T) { isHighPriority: true, } - queue.EnqueueRequest("userID", normalRequest1, 1, func() {}) - queue.EnqueueRequest("userID", normalRequest2, 1, func() {}) - queue.EnqueueRequest("userID", highPriorityRequest, 1, func() {}) + assert.NotNil(t, queue.EnqueueRequest("userID", normalRequest1, 1, func() {})) + assert.NotNil(t, queue.EnqueueRequest("userID", normalRequest2, 1, func() {})) + assert.NotNil(t, queue.EnqueueRequest("userID", highPriorityRequest, 1, func() {})) assert.Error(t, queue.EnqueueRequest("userID", highPriorityRequest, 1, func() {})) // should fail due to maxOutstandingPerTenant = 3 assert.Equal(t, 3, queue.queues.getTotalQueueSize("userID")) @@ -216,8 +216,8 @@ func TestRequestQueue_ReservedQueriersShouldOnlyGetHighPriorityQueries(t *testin isHighPriority: true, } - queue.EnqueueRequest("userID", normalRequest, 1, func() {}) - queue.EnqueueRequest("userID", highPriorityRequest, 1, func() {}) + assert.NotNil(t, queue.EnqueueRequest("userID", normalRequest, 1, func() {})) + assert.NotNil(t, queue.EnqueueRequest("userID", highPriorityRequest, 1, func() {})) nextRequest, _, _ := queue.GetNextRequestForQuerier(ctx, FirstUser(), "querier-1") assert.Equal(t, highPriorityRequest, nextRequest) From fec4a902fed2d8a193f4d9b124a5c3e13096c3bb Mon Sep 17 00:00:00 2001 From: Justin Jung Date: Tue, 17 Oct 2023 13:10:58 -0700 Subject: [PATCH 06/49] Fix test Signed-off-by: Justin Jung --- pkg/scheduler/queue/queue.go | 2 ++ pkg/scheduler/queue/queue_test.go | 10 +++++----- pkg/scheduler/scheduler_test.go | 2 +- 3 files changed, 8 insertions(+), 6 deletions(-) diff --git a/pkg/scheduler/queue/queue.go b/pkg/scheduler/queue/queue.go index 0931b25f3b..27cc6f540c 100644 --- a/pkg/scheduler/queue/queue.go +++ b/pkg/scheduler/queue/queue.go @@ -2,6 +2,7 @@ package queue import ( "context" + "fmt" "sync" "time" @@ -108,6 +109,7 @@ func (q *RequestQueue) EnqueueRequest(userID string, req Request, maxQueriers fl q.totalRequests.WithLabelValues(userID).Inc() + fmt.Println("EnqueueRequest", q.queues.getTotalQueueSize(userID), maxOutstandingRequests) if q.queues.getTotalQueueSize(userID) >= maxOutstandingRequests { q.discardedRequests.WithLabelValues(userID).Inc() return ErrTooManyRequests diff --git a/pkg/scheduler/queue/queue_test.go b/pkg/scheduler/queue/queue_test.go index 426af2d520..fca29e24df 100644 --- a/pkg/scheduler/queue/queue_test.go +++ b/pkg/scheduler/queue/queue_test.go @@ -160,7 +160,7 @@ func TestRequestQueue_GetNextRequestForQuerier_ShouldGetRequestAfterReshardingBe } func TestRequestQueue_QueriersShouldGetHighPriorityQueryFirst(t *testing.T) { - queue := NewRequestQueue(3, 0, + queue := NewRequestQueue(100, 0, prometheus.NewGaugeVec(prometheus.GaugeOpts{}, []string{"user"}), prometheus.NewCounterVec(prometheus.CounterOpts{}, []string{"user"}), MockLimits{MaxOutstanding: 3}, @@ -182,9 +182,9 @@ func TestRequestQueue_QueriersShouldGetHighPriorityQueryFirst(t *testing.T) { isHighPriority: true, } - assert.NotNil(t, queue.EnqueueRequest("userID", normalRequest1, 1, func() {})) - assert.NotNil(t, queue.EnqueueRequest("userID", normalRequest2, 1, func() {})) - assert.NotNil(t, queue.EnqueueRequest("userID", highPriorityRequest, 1, func() {})) + assert.Nil(t, queue.EnqueueRequest("userID", normalRequest1, 1, func() {})) + assert.Nil(t, queue.EnqueueRequest("userID", normalRequest2, 1, func() {})) + assert.Nil(t, queue.EnqueueRequest("userID", highPriorityRequest, 1, func() {})) assert.Error(t, queue.EnqueueRequest("userID", highPriorityRequest, 1, func() {})) // should fail due to maxOutstandingPerTenant = 3 assert.Equal(t, 3, queue.queues.getTotalQueueSize("userID")) @@ -197,7 +197,7 @@ func TestRequestQueue_QueriersShouldGetHighPriorityQueryFirst(t *testing.T) { } func TestRequestQueue_ReservedQueriersShouldOnlyGetHighPriorityQueries(t *testing.T) { - queue := NewRequestQueue(3, 0, + queue := NewRequestQueue(100, 0, prometheus.NewGaugeVec(prometheus.GaugeOpts{}, []string{"user"}), prometheus.NewCounterVec(prometheus.CounterOpts{}, []string{"user"}), MockLimits{MaxOutstanding: 3, ReservedQueriers: 1}, diff --git a/pkg/scheduler/scheduler_test.go b/pkg/scheduler/scheduler_test.go index b2ef66cc65..c309ff195e 100644 --- a/pkg/scheduler/scheduler_test.go +++ b/pkg/scheduler/scheduler_test.go @@ -37,7 +37,7 @@ func setupScheduler(t *testing.T, reg prometheus.Registerer) (*Scheduler, schedu flagext.DefaultValues(&cfg) cfg.MaxOutstandingPerTenant = testMaxOutstandingPerTenant - s, err := NewScheduler(cfg, frontendv1.MockLimits{Queriers: 2, MockLimits: queue.MockLimits{MaxOutstanding: 100}}, log.NewNopLogger(), reg) + s, err := NewScheduler(cfg, frontendv1.MockLimits{Queriers: 2, MockLimits: queue.MockLimits{MaxOutstanding: testMaxOutstandingPerTenant}}, log.NewNopLogger(), reg) require.NoError(t, err) server := grpc.NewServer() From ad046d0b6f623b6fcf860158398a912d5b80ba2f Mon Sep 17 00:00:00 2001 From: Justin Jung Date: Tue, 17 Oct 2023 13:40:36 -0700 Subject: [PATCH 07/49] More test fix Signed-off-by: Justin Jung --- pkg/scheduler/queue/queue.go | 2 -- pkg/scheduler/queue/queue_test.go | 14 +++++++------- 2 files changed, 7 insertions(+), 9 deletions(-) diff --git a/pkg/scheduler/queue/queue.go b/pkg/scheduler/queue/queue.go index 27cc6f540c..0931b25f3b 100644 --- a/pkg/scheduler/queue/queue.go +++ b/pkg/scheduler/queue/queue.go @@ -2,7 +2,6 @@ package queue import ( "context" - "fmt" "sync" "time" @@ -109,7 +108,6 @@ func (q *RequestQueue) EnqueueRequest(userID string, req Request, maxQueriers fl q.totalRequests.WithLabelValues(userID).Inc() - fmt.Println("EnqueueRequest", q.queues.getTotalQueueSize(userID), maxOutstandingRequests) if q.queues.getTotalQueueSize(userID) >= maxOutstandingRequests { q.discardedRequests.WithLabelValues(userID).Inc() return ErrTooManyRequests diff --git a/pkg/scheduler/queue/queue_test.go b/pkg/scheduler/queue/queue_test.go index fca29e24df..5e260323ec 100644 --- a/pkg/scheduler/queue/queue_test.go +++ b/pkg/scheduler/queue/queue_test.go @@ -160,7 +160,7 @@ func TestRequestQueue_GetNextRequestForQuerier_ShouldGetRequestAfterReshardingBe } func TestRequestQueue_QueriersShouldGetHighPriorityQueryFirst(t *testing.T) { - queue := NewRequestQueue(100, 0, + queue := NewRequestQueue(0, 0, prometheus.NewGaugeVec(prometheus.GaugeOpts{}, []string{"user"}), prometheus.NewCounterVec(prometheus.CounterOpts{}, []string{"user"}), MockLimits{MaxOutstanding: 3}, @@ -182,9 +182,9 @@ func TestRequestQueue_QueriersShouldGetHighPriorityQueryFirst(t *testing.T) { isHighPriority: true, } - assert.Nil(t, queue.EnqueueRequest("userID", normalRequest1, 1, func() {})) - assert.Nil(t, queue.EnqueueRequest("userID", normalRequest2, 1, func() {})) - assert.Nil(t, queue.EnqueueRequest("userID", highPriorityRequest, 1, func() {})) + assert.NoError(t, queue.EnqueueRequest("userID", normalRequest1, 1, func() {})) + assert.NoError(t, queue.EnqueueRequest("userID", normalRequest2, 1, func() {})) + assert.NoError(t, queue.EnqueueRequest("userID", highPriorityRequest, 1, func() {})) assert.Error(t, queue.EnqueueRequest("userID", highPriorityRequest, 1, func() {})) // should fail due to maxOutstandingPerTenant = 3 assert.Equal(t, 3, queue.queues.getTotalQueueSize("userID")) @@ -197,7 +197,7 @@ func TestRequestQueue_QueriersShouldGetHighPriorityQueryFirst(t *testing.T) { } func TestRequestQueue_ReservedQueriersShouldOnlyGetHighPriorityQueries(t *testing.T) { - queue := NewRequestQueue(100, 0, + queue := NewRequestQueue(0, 0, prometheus.NewGaugeVec(prometheus.GaugeOpts{}, []string{"user"}), prometheus.NewCounterVec(prometheus.CounterOpts{}, []string{"user"}), MockLimits{MaxOutstanding: 3, ReservedQueriers: 1}, @@ -216,8 +216,8 @@ func TestRequestQueue_ReservedQueriersShouldOnlyGetHighPriorityQueries(t *testin isHighPriority: true, } - assert.NotNil(t, queue.EnqueueRequest("userID", normalRequest, 1, func() {})) - assert.NotNil(t, queue.EnqueueRequest("userID", highPriorityRequest, 1, func() {})) + assert.NoError(t, queue.EnqueueRequest("userID", normalRequest, 1, func() {})) + assert.NoError(t, queue.EnqueueRequest("userID", highPriorityRequest, 1, func() {})) nextRequest, _, _ := queue.GetNextRequestForQuerier(ctx, FirstUser(), "querier-1") assert.Equal(t, highPriorityRequest, nextRequest) From a88ad90c78372dc35fb604b9b6819243d1a88453 Mon Sep 17 00:00:00 2001 From: Justin Jung Date: Tue, 17 Oct 2023 15:15:12 -0700 Subject: [PATCH 08/49] Add HighPriorityQueries confing + placeholder for IsHighPriorityQuery func Signed-off-by: Justin Jung --- docs/configuration/config-file-reference.md | 19 +++ pkg/frontend/v1/frontend.go | 11 +- pkg/frontend/v2/frontend.go | 19 +-- pkg/frontend/v2/frontend_scheduler_worker.go | 1 + pkg/scheduler/scheduler.go | 5 +- pkg/scheduler/schedulerpb/scheduler.pb.go | 139 +++++++++++++------ pkg/scheduler/schedulerpb/scheduler.proto | 1 + pkg/util/query_priority.go | 9 ++ pkg/util/validation/limits.go | 16 ++- tools/doc-generator/parser.go | 10 +- 10 files changed, 166 insertions(+), 64 deletions(-) create mode 100644 pkg/util/query_priority.go diff --git a/docs/configuration/config-file-reference.md b/docs/configuration/config-file-reference.md index 3defa44f4c..78adee77b4 100644 --- a/docs/configuration/config-file-reference.md +++ b/docs/configuration/config-file-reference.md @@ -3047,6 +3047,9 @@ The `limits_config` configures default and per-tenant limits imposed by Cortex s # CLI flag: -frontend.reserved-high-priority-queriers [reserved_high_priority_queriers: | default = 0] +# List of query definitions to be handled with a high priority. +[high_priority_queries: | default = []] + # Duration to delay the evaluation of rules to ensure the underlying metrics # have been pushed to Cortex. # CLI flag: -ruler.evaluation-delay-duration @@ -5038,6 +5041,22 @@ otel: [tls_insecure_skip_verify: | default = false] ``` +### `HighPriorityQuery` + +```yaml +# Query string regex. If evaluated true (on top of meeting all other criteria), +# query is handled with a high priority. +[regex: | default = ""] + +# If query range falls between the start_time and end_time (on top of meeting +# all other criteria), query is handled with a high priority. +[start_time: | default = 1h] + +# If query range falls between the start_time and end_time (on top of meeting +# all other criteria), query is handled with a high priority. +[end_time: | default = 0s] +``` + ### `DisabledRuleGroup` ```yaml diff --git a/pkg/frontend/v1/frontend.go b/pkg/frontend/v1/frontend.go index 99e5d8eff1..5a0ad4e52e 100644 --- a/pkg/frontend/v1/frontend.go +++ b/pkg/frontend/v1/frontend.go @@ -88,14 +88,14 @@ type request struct { queueSpan opentracing.Span originalCtx context.Context - request *httpgrpc.HTTPRequest - err chan error - response chan *httpgrpc.HTTPResponse + request *httpgrpc.HTTPRequest + err chan error + response chan *httpgrpc.HTTPResponse + isHighPriority bool } func (r request) IsHighPriority() bool { - //TODO implement me - return false + return r.isHighPriority } // New creates a new frontend. Frontend implements service, and must be started and stopped. @@ -346,6 +346,7 @@ func (f *Frontend) queueRequest(ctx context.Context, req *request) error { now := time.Now() req.enqueueTime = now req.queueSpan, _ = opentracing.StartSpanFromContext(ctx, "queued") + req.isHighPriority = util.IsHighPriorityQuery() // aggregate the max queriers limit in the case of a multi tenant query maxQueriers := validation.SmallestPositiveNonZeroFloat64PerTenant(tenantIDs, f.limits.MaxQueriersPerUser) diff --git a/pkg/frontend/v2/frontend.go b/pkg/frontend/v2/frontend.go index dea15faeaf..dfed325fc4 100644 --- a/pkg/frontend/v2/frontend.go +++ b/pkg/frontend/v2/frontend.go @@ -4,6 +4,7 @@ import ( "context" "flag" "fmt" + "github.com/cortexproject/cortex/pkg/util" "math/rand" "net/http" "sync" @@ -82,10 +83,11 @@ type Frontend struct { } type frontendRequest struct { - queryID uint64 - request *httpgrpc.HTTPRequest - userID string - statsEnabled bool + queryID uint64 + request *httpgrpc.HTTPRequest + userID string + statsEnabled bool + isHighPriority bool cancel context.CancelFunc @@ -190,10 +192,11 @@ func (f *Frontend) RoundTripGRPC(ctx context.Context, req *httpgrpc.HTTPRequest) return f.retry.Do(ctx, func() (*httpgrpc.HTTPResponse, error) { freq := &frontendRequest{ - queryID: f.lastQueryID.Inc(), - request: req, - userID: userID, - statsEnabled: stats.IsEnabled(ctx), + queryID: f.lastQueryID.Inc(), + request: req, + userID: userID, + statsEnabled: stats.IsEnabled(ctx), + isHighPriority: util.IsHighPriorityQuery(), cancel: cancel, diff --git a/pkg/frontend/v2/frontend_scheduler_worker.go b/pkg/frontend/v2/frontend_scheduler_worker.go index bbe1e2ed74..5e5b22eb13 100644 --- a/pkg/frontend/v2/frontend_scheduler_worker.go +++ b/pkg/frontend/v2/frontend_scheduler_worker.go @@ -263,6 +263,7 @@ func (w *frontendSchedulerWorker) schedulerLoop(loop schedulerpb.SchedulerForFro HttpRequest: req.request, FrontendAddress: w.frontendAddr, StatsEnabled: req.statsEnabled, + IsHighPriority: req.isHighPriority, }) if err != nil { diff --git a/pkg/scheduler/scheduler.go b/pkg/scheduler/scheduler.go index 42a7b7413a..12b5575197 100644 --- a/pkg/scheduler/scheduler.go +++ b/pkg/scheduler/scheduler.go @@ -154,6 +154,7 @@ type schedulerRequest struct { queryID uint64 request *httpgrpc.HTTPRequest statsEnabled bool + isHighPriority bool enqueueTime time.Time @@ -166,8 +167,7 @@ type schedulerRequest struct { } func (s schedulerRequest) IsHighPriority() bool { - //TODO implement me - return false + return s.isHighPriority } // FrontendLoop handles connection from frontend. @@ -298,6 +298,7 @@ func (s *Scheduler) enqueueRequest(frontendContext context.Context, frontendAddr queryID: msg.QueryID, request: msg.HttpRequest, statsEnabled: msg.StatsEnabled, + isHighPriority: msg.IsHighPriority, } now := time.Now() diff --git a/pkg/scheduler/schedulerpb/scheduler.pb.go b/pkg/scheduler/schedulerpb/scheduler.pb.go index d3288f95b3..0d5c95d68e 100644 --- a/pkg/scheduler/schedulerpb/scheduler.pb.go +++ b/pkg/scheduler/schedulerpb/scheduler.pb.go @@ -216,9 +216,10 @@ type FrontendToScheduler struct { // Each frontend manages its own queryIDs. Different frontends may use same set of query IDs. QueryID uint64 `protobuf:"varint,3,opt,name=queryID,proto3" json:"queryID,omitempty"` // Following are used by ENQUEUE only. - UserID string `protobuf:"bytes,4,opt,name=userID,proto3" json:"userID,omitempty"` - HttpRequest *httpgrpc.HTTPRequest `protobuf:"bytes,5,opt,name=httpRequest,proto3" json:"httpRequest,omitempty"` - StatsEnabled bool `protobuf:"varint,6,opt,name=statsEnabled,proto3" json:"statsEnabled,omitempty"` + UserID string `protobuf:"bytes,4,opt,name=userID,proto3" json:"userID,omitempty"` + HttpRequest *httpgrpc.HTTPRequest `protobuf:"bytes,5,opt,name=httpRequest,proto3" json:"httpRequest,omitempty"` + StatsEnabled bool `protobuf:"varint,6,opt,name=statsEnabled,proto3" json:"statsEnabled,omitempty"` + IsHighPriority bool `protobuf:"varint,7,opt,name=isHighPriority,proto3" json:"isHighPriority,omitempty"` } func (m *FrontendToScheduler) Reset() { *m = FrontendToScheduler{} } @@ -295,6 +296,13 @@ func (m *FrontendToScheduler) GetStatsEnabled() bool { return false } +func (m *FrontendToScheduler) GetIsHighPriority() bool { + if m != nil { + return m.IsHighPriority + } + return false +} + type SchedulerToFrontend struct { Status SchedulerToFrontendStatus `protobuf:"varint,1,opt,name=status,proto3,enum=schedulerpb.SchedulerToFrontendStatus" json:"status,omitempty"` Error string `protobuf:"bytes,2,opt,name=error,proto3" json:"error,omitempty"` @@ -438,48 +446,49 @@ func init() { func init() { proto.RegisterFile("scheduler.proto", fileDescriptor_2b3fc28395a6d9c5) } var fileDescriptor_2b3fc28395a6d9c5 = []byte{ - // 644 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x8c, 0x54, 0x4f, 0x4f, 0xdb, 0x4e, - 0x10, 0xf5, 0x86, 0x24, 0xc0, 0x84, 0xdf, 0x0f, 0x77, 0x81, 0x36, 0x8d, 0xe8, 0x12, 0x45, 0x55, - 0x95, 0x72, 0x48, 0xaa, 0xb4, 0x52, 0x7b, 0x40, 0x95, 0x52, 0x30, 0x25, 0x2a, 0x75, 0x60, 0xb3, - 0x51, 0xff, 0x5c, 0x22, 0x92, 0x2c, 0x09, 0x02, 0xbc, 0x66, 0x6d, 0x17, 0xe5, 0xd6, 0x63, 0x8f, - 0xfd, 0x18, 0xfd, 0x28, 0xbd, 0x54, 0xe2, 0xc8, 0xa1, 0x87, 0x62, 0x2e, 0x3d, 0xf2, 0x11, 0xaa, - 0x38, 0x76, 0xea, 0xa4, 0x0e, 0x70, 0x9b, 0x1d, 0xbf, 0xe7, 0x9d, 0xf7, 0x66, 0x66, 0x61, 0xde, - 0x6a, 0x75, 0x79, 0xdb, 0x39, 0xe2, 0xb2, 0x60, 0x4a, 0x61, 0x0b, 0x9c, 0x1a, 0x26, 0xcc, 0x66, - 0x66, 0xb1, 0x23, 0x3a, 0xc2, 0xcb, 0x17, 0xfb, 0xd1, 0x00, 0x92, 0x79, 0xd6, 0x39, 0xb0, 0xbb, - 0x4e, 0xb3, 0xd0, 0x12, 0xc7, 0xc5, 0x53, 0xbe, 0xf7, 0x89, 0x9f, 0x0a, 0x79, 0x68, 0x15, 0x5b, - 0xe2, 0xf8, 0x58, 0x18, 0xc5, 0xae, 0x6d, 0x9b, 0x1d, 0x69, 0xb6, 0x86, 0xc1, 0x80, 0x95, 0x2b, - 0x01, 0xde, 0x75, 0xb8, 0x3c, 0xe0, 0x92, 0x89, 0x5a, 0x70, 0x07, 0x5e, 0x86, 0xd9, 0x93, 0x41, - 0xb6, 0xb2, 0x91, 0x46, 0x59, 0x94, 0x9f, 0xa5, 0x7f, 0x13, 0xb9, 0x1f, 0x08, 0xf0, 0x10, 0xcb, - 0x84, 0xcf, 0xc7, 0x69, 0x98, 0xee, 0x63, 0x7a, 0x3e, 0x25, 0x4e, 0x83, 0x23, 0x7e, 0x0e, 0xa9, - 0xfe, 0xb5, 0x94, 0x9f, 0x38, 0xdc, 0xb2, 0xd3, 0xb1, 0x2c, 0xca, 0xa7, 0x4a, 0x4b, 0x85, 0x61, - 0x29, 0x5b, 0x8c, 0xed, 0xf8, 0x1f, 0x69, 0x18, 0x89, 0xf3, 0x30, 0xbf, 0x2f, 0x85, 0x61, 0x73, - 0xa3, 0x5d, 0x6e, 0xb7, 0x25, 0xb7, 0xac, 0xf4, 0x94, 0x57, 0xcd, 0x78, 0x1a, 0xdf, 0x85, 0xa4, - 0x63, 0x79, 0xe5, 0xc6, 0x3d, 0x80, 0x7f, 0xc2, 0x39, 0x98, 0xb3, 0xec, 0x3d, 0xdb, 0xd2, 0x8c, - 0xbd, 0xe6, 0x11, 0x6f, 0xa7, 0x13, 0x59, 0x94, 0x9f, 0xa1, 0x23, 0xb9, 0xdc, 0x97, 0x18, 0x2c, - 0x6c, 0xfa, 0xff, 0x0b, 0xbb, 0xf0, 0x02, 0xe2, 0x76, 0xcf, 0xe4, 0x9e, 0x9a, 0xff, 0x4b, 0x0f, - 0x0b, 0xa1, 0x1e, 0x14, 0x22, 0xf0, 0xac, 0x67, 0x72, 0xea, 0x31, 0xa2, 0xea, 0x8e, 0x45, 0xd7, - 0x1d, 0x32, 0x6d, 0x6a, 0xd4, 0xb4, 0x49, 0x8a, 0xc6, 0xcc, 0x4c, 0xdc, 0xda, 0xcc, 0x71, 0x2b, - 0x92, 0x11, 0x56, 0x1c, 0xc2, 0x42, 0xa8, 0xb3, 0x81, 0x48, 0xfc, 0x12, 0x92, 0x7d, 0x98, 0x63, - 0xf9, 0x5e, 0x3c, 0x1a, 0xf1, 0x22, 0x82, 0x51, 0xf3, 0xd0, 0xd4, 0x67, 0xe1, 0x45, 0x48, 0x70, - 0x29, 0x85, 0xf4, 0x5d, 0x18, 0x1c, 0x72, 0x6b, 0xb0, 0xac, 0x0b, 0xfb, 0x60, 0xbf, 0xe7, 0x4f, - 0x50, 0xad, 0xeb, 0xd8, 0x6d, 0x71, 0x6a, 0x04, 0x05, 0x5f, 0x3f, 0x85, 0x2b, 0xf0, 0x60, 0x02, - 0xdb, 0x32, 0x85, 0x61, 0xf1, 0xd5, 0x35, 0xb8, 0x37, 0xa1, 0x4b, 0x78, 0x06, 0xe2, 0x15, 0xbd, - 0xc2, 0x54, 0x05, 0xa7, 0x60, 0x5a, 0xd3, 0x77, 0xeb, 0x5a, 0x5d, 0x53, 0x11, 0x06, 0x48, 0xae, - 0x97, 0xf5, 0x75, 0x6d, 0x5b, 0x8d, 0xad, 0xb6, 0xe0, 0xfe, 0x44, 0x5d, 0x38, 0x09, 0xb1, 0xea, - 0x1b, 0x55, 0xc1, 0x59, 0x58, 0x66, 0xd5, 0x6a, 0xe3, 0x6d, 0x59, 0xff, 0xd0, 0xa0, 0xda, 0x6e, - 0x5d, 0xab, 0xb1, 0x5a, 0x63, 0x47, 0xa3, 0x0d, 0xa6, 0xe9, 0x65, 0x9d, 0xa9, 0x08, 0xcf, 0x42, - 0x42, 0xa3, 0xb4, 0x4a, 0xd5, 0x18, 0xbe, 0x03, 0xff, 0xd5, 0xb6, 0xea, 0x8c, 0x55, 0xf4, 0xd7, - 0x8d, 0x8d, 0xea, 0x3b, 0x5d, 0x9d, 0x2a, 0xfd, 0x44, 0x21, 0xbf, 0x37, 0x85, 0x0c, 0x56, 0xa9, - 0x0e, 0x29, 0x3f, 0xdc, 0x16, 0xc2, 0xc4, 0x2b, 0x23, 0x76, 0xff, 0xbb, 0xaf, 0x99, 0x95, 0x49, - 0xfd, 0xf0, 0xb1, 0x39, 0x25, 0x8f, 0x9e, 0x20, 0x6c, 0xc0, 0x52, 0xa4, 0x65, 0xf8, 0xf1, 0x08, - 0xff, 0xba, 0xa6, 0x64, 0x56, 0x6f, 0x03, 0x1d, 0x74, 0xa0, 0x64, 0xc2, 0x62, 0x58, 0xdd, 0x70, - 0x9c, 0xde, 0xc3, 0x5c, 0x10, 0x7b, 0xfa, 0xb2, 0x37, 0xad, 0x56, 0x26, 0x7b, 0xd3, 0xc0, 0x0d, - 0x14, 0xbe, 0x2a, 0x9f, 0x5d, 0x10, 0xe5, 0xfc, 0x82, 0x28, 0x57, 0x17, 0x04, 0x7d, 0x76, 0x09, - 0xfa, 0xe6, 0x12, 0xf4, 0xdd, 0x25, 0xe8, 0xcc, 0x25, 0xe8, 0x97, 0x4b, 0xd0, 0x6f, 0x97, 0x28, - 0x57, 0x2e, 0x41, 0x5f, 0x2f, 0x89, 0x72, 0x76, 0x49, 0x94, 0xf3, 0x4b, 0xa2, 0x7c, 0x0c, 0xbf, - 0xae, 0xcd, 0xa4, 0xf7, 0x30, 0x3e, 0xfd, 0x13, 0x00, 0x00, 0xff, 0xff, 0x88, 0x0c, 0xfe, 0x56, - 0x84, 0x05, 0x00, 0x00, + // 666 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x8c, 0x94, 0xcf, 0x4e, 0xdb, 0x40, + 0x10, 0xc6, 0xbd, 0x21, 0x09, 0x30, 0xa1, 0xe0, 0x2e, 0xd0, 0xa6, 0x11, 0x5d, 0x22, 0xab, 0x42, + 0x29, 0x87, 0xa4, 0x4a, 0x2b, 0xb5, 0x07, 0x54, 0x29, 0x05, 0x53, 0xa2, 0x52, 0x07, 0x1c, 0x47, + 0xfd, 0x73, 0x89, 0x48, 0xb2, 0x24, 0x16, 0xe0, 0x35, 0xeb, 0x75, 0x51, 0x6e, 0x7d, 0x84, 0x3e, + 0x46, 0x8f, 0x7d, 0x8c, 0x5e, 0x2a, 0x71, 0xe4, 0xd0, 0x43, 0x71, 0x2e, 0x3d, 0xf2, 0x08, 0x55, + 0x1c, 0x27, 0x75, 0x42, 0x02, 0xdc, 0x66, 0xc7, 0xdf, 0x67, 0xcf, 0xfc, 0x66, 0xd6, 0xb0, 0xe0, + 0xd4, 0x5b, 0xb4, 0xe1, 0x1e, 0x53, 0x9e, 0xb5, 0x39, 0x13, 0x0c, 0x27, 0x06, 0x09, 0xbb, 0x96, + 0x5a, 0x6a, 0xb2, 0x26, 0xf3, 0xf3, 0xb9, 0x6e, 0xd4, 0x93, 0xa4, 0x5e, 0x34, 0x4d, 0xd1, 0x72, + 0x6b, 0xd9, 0x3a, 0x3b, 0xc9, 0x9d, 0xd1, 0x83, 0x2f, 0xf4, 0x8c, 0xf1, 0x23, 0x27, 0x57, 0x67, + 0x27, 0x27, 0xcc, 0xca, 0xb5, 0x84, 0xb0, 0x9b, 0xdc, 0xae, 0x0f, 0x82, 0x9e, 0x4b, 0xc9, 0x03, + 0xde, 0x77, 0x29, 0x37, 0x29, 0x37, 0x58, 0xb9, 0xff, 0x0d, 0xbc, 0x02, 0xb3, 0xa7, 0xbd, 0x6c, + 0x71, 0x2b, 0x89, 0xd2, 0x28, 0x33, 0xab, 0xff, 0x4f, 0x28, 0xbf, 0x10, 0xe0, 0x81, 0xd6, 0x60, + 0x81, 0x1f, 0x27, 0x61, 0xba, 0xab, 0x69, 0x07, 0x96, 0xa8, 0xde, 0x3f, 0xe2, 0x97, 0x90, 0xe8, + 0x7e, 0x56, 0xa7, 0xa7, 0x2e, 0x75, 0x44, 0x32, 0x92, 0x46, 0x99, 0x44, 0x7e, 0x39, 0x3b, 0x28, + 0x65, 0xc7, 0x30, 0xf6, 0x82, 0x87, 0x7a, 0x58, 0x89, 0x33, 0xb0, 0x70, 0xc8, 0x99, 0x25, 0xa8, + 0xd5, 0x28, 0x34, 0x1a, 0x9c, 0x3a, 0x4e, 0x72, 0xca, 0xaf, 0x66, 0x34, 0x8d, 0x1f, 0x40, 0xdc, + 0x75, 0xfc, 0x72, 0xa3, 0xbe, 0x20, 0x38, 0x61, 0x05, 0xe6, 0x1c, 0x71, 0x20, 0x1c, 0xd5, 0x3a, + 0xa8, 0x1d, 0xd3, 0x46, 0x32, 0x96, 0x46, 0x99, 0x19, 0x7d, 0x28, 0xa7, 0xfc, 0x88, 0xc0, 0xe2, + 0x76, 0xf0, 0xbe, 0x30, 0x85, 0x57, 0x10, 0x15, 0x6d, 0x9b, 0xfa, 0xdd, 0xcc, 0xe7, 0x9f, 0x64, + 0x43, 0x33, 0xc8, 0x8e, 0xd1, 0x1b, 0x6d, 0x9b, 0xea, 0xbe, 0x63, 0x5c, 0xdd, 0x91, 0xf1, 0x75, + 0x87, 0xa0, 0x4d, 0x0d, 0x43, 0x9b, 0xd4, 0xd1, 0x08, 0xcc, 0xd8, 0x9d, 0x61, 0x8e, 0xa2, 0x88, + 0x5f, 0x47, 0x81, 0xd7, 0x60, 0xde, 0x74, 0x76, 0xcc, 0x66, 0x6b, 0x8f, 0x9b, 0x8c, 0x9b, 0xa2, + 0x9d, 0x9c, 0xf6, 0x55, 0x23, 0x59, 0xe5, 0x08, 0x16, 0x43, 0x1b, 0xd0, 0x87, 0x81, 0x5f, 0x43, + 0xbc, 0xfb, 0x3a, 0xd7, 0x09, 0x98, 0xad, 0x0d, 0x31, 0x1b, 0xe3, 0x28, 0xfb, 0x6a, 0x3d, 0x70, + 0xe1, 0x25, 0x88, 0x51, 0xce, 0x19, 0x0f, 0x68, 0xf5, 0x0e, 0xca, 0x06, 0xac, 0x68, 0x4c, 0x98, + 0x87, 0xed, 0x60, 0xd3, 0xca, 0x2d, 0x57, 0x34, 0xd8, 0x99, 0xd5, 0x6f, 0xec, 0xe6, 0x6d, 0x5d, + 0x85, 0xc7, 0x13, 0xdc, 0x8e, 0xcd, 0x2c, 0x87, 0xae, 0x6f, 0xc0, 0xc3, 0x09, 0xd3, 0xc4, 0x33, + 0x10, 0x2d, 0x6a, 0x45, 0x43, 0x96, 0x70, 0x02, 0xa6, 0x55, 0x6d, 0xbf, 0xa2, 0x56, 0x54, 0x19, + 0x61, 0x80, 0xf8, 0x66, 0x41, 0xdb, 0x54, 0x77, 0xe5, 0xc8, 0x7a, 0x1d, 0x1e, 0x4d, 0xec, 0x0b, + 0xc7, 0x21, 0x52, 0x7a, 0x27, 0x4b, 0x38, 0x0d, 0x2b, 0x46, 0xa9, 0x54, 0x7d, 0x5f, 0xd0, 0x3e, + 0x55, 0x75, 0x75, 0xbf, 0xa2, 0x96, 0x8d, 0x72, 0x75, 0x4f, 0xd5, 0xab, 0x86, 0xaa, 0x15, 0x34, + 0x43, 0x46, 0x78, 0x16, 0x62, 0xaa, 0xae, 0x97, 0x74, 0x39, 0x82, 0xef, 0xc3, 0xbd, 0xf2, 0x4e, + 0xc5, 0x30, 0x8a, 0xda, 0xdb, 0xea, 0x56, 0xe9, 0x83, 0x26, 0x4f, 0xe5, 0x7f, 0xa3, 0x10, 0xef, + 0x6d, 0xc6, 0xfb, 0x57, 0xae, 0x02, 0x89, 0x20, 0xdc, 0x65, 0xcc, 0xc6, 0xab, 0x43, 0xb8, 0xaf, + 0xdf, 0xeb, 0xd4, 0xea, 0xa4, 0x79, 0x04, 0x5a, 0x45, 0xca, 0xa0, 0x67, 0x08, 0x5b, 0xb0, 0x3c, + 0x16, 0x19, 0x7e, 0x3a, 0xe4, 0xbf, 0x69, 0x28, 0xa9, 0xf5, 0xbb, 0x48, 0x7b, 0x13, 0xc8, 0xdb, + 0xb0, 0x14, 0xee, 0x6e, 0xb0, 0x4e, 0x1f, 0x61, 0xae, 0x1f, 0xfb, 0xfd, 0xa5, 0x6f, 0xbb, 0x82, + 0xa9, 0xf4, 0x6d, 0x0b, 0xd7, 0xeb, 0xf0, 0x4d, 0xe1, 0xfc, 0x92, 0x48, 0x17, 0x97, 0x44, 0xba, + 0xba, 0x24, 0xe8, 0xab, 0x47, 0xd0, 0x77, 0x8f, 0xa0, 0x9f, 0x1e, 0x41, 0xe7, 0x1e, 0x41, 0x7f, + 0x3c, 0x82, 0xfe, 0x7a, 0x44, 0xba, 0xf2, 0x08, 0xfa, 0xd6, 0x21, 0xd2, 0x79, 0x87, 0x48, 0x17, + 0x1d, 0x22, 0x7d, 0x0e, 0xff, 0x85, 0x6b, 0x71, 0xff, 0x07, 0xfa, 0xfc, 0x5f, 0x00, 0x00, 0x00, + 0xff, 0xff, 0x13, 0x54, 0x85, 0x09, 0xac, 0x05, 0x00, 0x00, } func (x FrontendToSchedulerType) String() string { @@ -593,6 +602,9 @@ func (this *FrontendToScheduler) Equal(that interface{}) bool { if this.StatsEnabled != that1.StatsEnabled { return false } + if this.IsHighPriority != that1.IsHighPriority { + return false + } return true } func (this *SchedulerToFrontend) Equal(that interface{}) bool { @@ -697,7 +709,7 @@ func (this *FrontendToScheduler) GoString() string { if this == nil { return "nil" } - s := make([]string, 0, 10) + s := make([]string, 0, 11) s = append(s, "&schedulerpb.FrontendToScheduler{") s = append(s, "Type: "+fmt.Sprintf("%#v", this.Type)+",\n") s = append(s, "FrontendAddress: "+fmt.Sprintf("%#v", this.FrontendAddress)+",\n") @@ -707,6 +719,7 @@ func (this *FrontendToScheduler) GoString() string { s = append(s, "HttpRequest: "+fmt.Sprintf("%#v", this.HttpRequest)+",\n") } s = append(s, "StatsEnabled: "+fmt.Sprintf("%#v", this.StatsEnabled)+",\n") + s = append(s, "IsHighPriority: "+fmt.Sprintf("%#v", this.IsHighPriority)+",\n") s = append(s, "}") return strings.Join(s, "") } @@ -1142,6 +1155,16 @@ func (m *FrontendToScheduler) MarshalToSizedBuffer(dAtA []byte) (int, error) { _ = i var l int _ = l + if m.IsHighPriority { + i-- + if m.IsHighPriority { + dAtA[i] = 1 + } else { + dAtA[i] = 0 + } + i-- + dAtA[i] = 0x38 + } if m.StatsEnabled { i-- if m.StatsEnabled { @@ -1357,6 +1380,9 @@ func (m *FrontendToScheduler) Size() (n int) { if m.StatsEnabled { n += 2 } + if m.IsHighPriority { + n += 2 + } return n } @@ -1439,6 +1465,7 @@ func (this *FrontendToScheduler) String() string { `UserID:` + fmt.Sprintf("%v", this.UserID) + `,`, `HttpRequest:` + strings.Replace(fmt.Sprintf("%v", this.HttpRequest), "HTTPRequest", "httpgrpc.HTTPRequest", 1) + `,`, `StatsEnabled:` + fmt.Sprintf("%v", this.StatsEnabled) + `,`, + `IsHighPriority:` + fmt.Sprintf("%v", this.IsHighPriority) + `,`, `}`, }, "") return s @@ -1945,6 +1972,26 @@ func (m *FrontendToScheduler) Unmarshal(dAtA []byte) error { } } m.StatsEnabled = bool(v != 0) + case 7: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field IsHighPriority", wireType) + } + var v int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowScheduler + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + v |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + m.IsHighPriority = bool(v != 0) default: iNdEx = preIndex skippy, err := skipScheduler(dAtA[iNdEx:]) diff --git a/pkg/scheduler/schedulerpb/scheduler.proto b/pkg/scheduler/schedulerpb/scheduler.proto index eea28717b8..5e71946008 100644 --- a/pkg/scheduler/schedulerpb/scheduler.proto +++ b/pkg/scheduler/schedulerpb/scheduler.proto @@ -78,6 +78,7 @@ message FrontendToScheduler { string userID = 4; httpgrpc.HTTPRequest httpRequest = 5; bool statsEnabled = 6; + bool isHighPriority = 7; } enum SchedulerToFrontendStatus { diff --git a/pkg/util/query_priority.go b/pkg/util/query_priority.go new file mode 100644 index 0000000000..035bccb0df --- /dev/null +++ b/pkg/util/query_priority.go @@ -0,0 +1,9 @@ +package util + +func IsHighPriorityQuery() bool { + // TODO: Implement + // query string <> regex + // query param <> start_time, end_time + + return false +} diff --git a/pkg/util/validation/limits.go b/pkg/util/validation/limits.go index 94db111b77..6e1298249a 100644 --- a/pkg/util/validation/limits.go +++ b/pkg/util/validation/limits.go @@ -46,6 +46,12 @@ type DisabledRuleGroup struct { type DisabledRuleGroups []DisabledRuleGroup +type HighPriorityQuery struct { + Regex string `yaml:"regex" doc:"nocli|description=Query string regex. If evaluated true (on top of meeting all other criteria), query is handled with a high priority."` + StartTime time.Duration `yaml:"start_time" doc:"nocli|description=If query range falls between the start_time and end_time (on top of meeting all other criteria), query is handled with a high priority.|default=1h"` + EndTime time.Duration `yaml:"end_time" doc:"nocli|description=If query range falls between the start_time and end_time (on top of meeting all other criteria), query is handled with a high priority.|default=0s"` +} + // Limits describe all the limits for users; can be used to describe global default // limits via flags, or per-user limits via yaml config. type Limits struct { @@ -101,8 +107,9 @@ type Limits struct { QueryVerticalShardSize int `yaml:"query_vertical_shard_size" json:"query_vertical_shard_size" doc:"hidden"` // Query Frontend / Scheduler enforced limits. - MaxOutstandingPerTenant int `yaml:"max_outstanding_requests_per_tenant" json:"max_outstanding_requests_per_tenant"` - ReservedHighPriorityQueriers float64 `yaml:"reserved_high_priority_queriers" json:"reserved_high_priority_queriers"` + MaxOutstandingPerTenant int `yaml:"max_outstanding_requests_per_tenant" json:"max_outstanding_requests_per_tenant"` + ReservedHighPriorityQueriers float64 `yaml:"reserved_high_priority_queriers" json:"reserved_high_priority_queriers"` + HighPriorityQueries []HighPriorityQuery `yaml:"high_priority_queries" json:"high_priority_queries" doc:"nocli|description=List of query definitions to be handled with a high priority."` // Ruler defaults and limits. RulerEvaluationDelay model.Duration `yaml:"ruler_evaluation_delay_duration" json:"ruler_evaluation_delay_duration"` @@ -499,6 +506,11 @@ func (o *Overrides) ReservedHighPriorityQueriers(userID string) float64 { return o.GetOverridesForUser(userID).ReservedHighPriorityQueriers } +// HighPriorityQueries returns list of definitions for high priority query. +func (o *Overrides) HighPriorityQueries(userID string) []HighPriorityQuery { + return o.GetOverridesForUser(userID).HighPriorityQueries +} + // EnforceMetricName whether to enforce the presence of a metric name. func (o *Overrides) EnforceMetricName(userID string) bool { return o.GetOverridesForUser(userID).EnforceMetricName diff --git a/tools/doc-generator/parser.go b/tools/doc-generator/parser.go index 3ebb313368..7a6dbfebfb 100644 --- a/tools/doc-generator/parser.go +++ b/tools/doc-generator/parser.go @@ -227,7 +227,7 @@ func parseConfig(block *configBlock, cfg interface{}, flags map[uintptr]*flag.Fl required: isFieldRequired(field), fieldDesc: getFieldDescription(field, ""), fieldType: fieldType, - fieldDefault: fieldDefault, + fieldDefault: getFieldDefault(field, fieldDefault), }) continue } @@ -470,6 +470,14 @@ func getFieldDescription(f reflect.StructField, fallback string) string { return fallback } +func getFieldDefault(f reflect.StructField, fieldDefault string) string { + if defaultValue := getDocTagValue(f, "default"); defaultValue != "" { + return defaultValue + } + + return fieldDefault +} + func isRootBlock(t reflect.Type) (string, string, bool) { for _, rootBlock := range rootBlocks { if t == rootBlock.structType { From b050ac1eaefda92eadbac6395ee1c0986a90e26e Mon Sep 17 00:00:00 2001 From: Justin Jung Date: Tue, 17 Oct 2023 15:38:59 -0700 Subject: [PATCH 09/49] Nit Signed-off-by: Justin Jung --- docs/configuration/config-file-reference.md | 8 ++++---- pkg/frontend/v1/frontend.go | 3 ++- pkg/frontend/v2/frontend.go | 4 ++-- pkg/util/{ => query}/query_priority.go | 4 ++-- pkg/util/validation/limits.go | 8 ++++---- 5 files changed, 14 insertions(+), 13 deletions(-) rename pkg/util/{ => query}/query_priority.go (68%) diff --git a/docs/configuration/config-file-reference.md b/docs/configuration/config-file-reference.md index 78adee77b4..d8cffa118f 100644 --- a/docs/configuration/config-file-reference.md +++ b/docs/configuration/config-file-reference.md @@ -3047,7 +3047,7 @@ The `limits_config` configures default and per-tenant limits imposed by Cortex s # CLI flag: -frontend.reserved-high-priority-queriers [reserved_high_priority_queriers: | default = 0] -# List of query definitions to be handled with a high priority. +# List of query definitions to be treated as a high priority. [high_priority_queries: | default = []] # Duration to delay the evaluation of rules to ensure the underlying metrics @@ -5045,15 +5045,15 @@ otel: ```yaml # Query string regex. If evaluated true (on top of meeting all other criteria), -# query is handled with a high priority. +# query is treated as a high priority. [regex: | default = ""] # If query range falls between the start_time and end_time (on top of meeting -# all other criteria), query is handled with a high priority. +# all other criteria), query is treated as a high priority. [start_time: | default = 1h] # If query range falls between the start_time and end_time (on top of meeting -# all other criteria), query is handled with a high priority. +# all other criteria), query is treated as a high priority. [end_time: | default = 0s] ``` diff --git a/pkg/frontend/v1/frontend.go b/pkg/frontend/v1/frontend.go index 5a0ad4e52e..6bd37274c5 100644 --- a/pkg/frontend/v1/frontend.go +++ b/pkg/frontend/v1/frontend.go @@ -22,6 +22,7 @@ import ( "github.com/cortexproject/cortex/pkg/tenant" "github.com/cortexproject/cortex/pkg/util" "github.com/cortexproject/cortex/pkg/util/httpgrpcutil" + "github.com/cortexproject/cortex/pkg/util/query" "github.com/cortexproject/cortex/pkg/util/services" "github.com/cortexproject/cortex/pkg/util/validation" ) @@ -346,7 +347,7 @@ func (f *Frontend) queueRequest(ctx context.Context, req *request) error { now := time.Now() req.enqueueTime = now req.queueSpan, _ = opentracing.StartSpanFromContext(ctx, "queued") - req.isHighPriority = util.IsHighPriorityQuery() + req.isHighPriority = query.IsHighPriority() // aggregate the max queriers limit in the case of a multi tenant query maxQueriers := validation.SmallestPositiveNonZeroFloat64PerTenant(tenantIDs, f.limits.MaxQueriersPerUser) diff --git a/pkg/frontend/v2/frontend.go b/pkg/frontend/v2/frontend.go index dfed325fc4..202c2d3042 100644 --- a/pkg/frontend/v2/frontend.go +++ b/pkg/frontend/v2/frontend.go @@ -4,7 +4,6 @@ import ( "context" "flag" "fmt" - "github.com/cortexproject/cortex/pkg/util" "math/rand" "net/http" "sync" @@ -27,6 +26,7 @@ import ( "github.com/cortexproject/cortex/pkg/util/grpcclient" "github.com/cortexproject/cortex/pkg/util/httpgrpcutil" util_log "github.com/cortexproject/cortex/pkg/util/log" + "github.com/cortexproject/cortex/pkg/util/query" "github.com/cortexproject/cortex/pkg/util/services" ) @@ -196,7 +196,7 @@ func (f *Frontend) RoundTripGRPC(ctx context.Context, req *httpgrpc.HTTPRequest) request: req, userID: userID, statsEnabled: stats.IsEnabled(ctx), - isHighPriority: util.IsHighPriorityQuery(), + isHighPriority: query.IsHighPriority(), cancel: cancel, diff --git a/pkg/util/query_priority.go b/pkg/util/query/query_priority.go similarity index 68% rename from pkg/util/query_priority.go rename to pkg/util/query/query_priority.go index 035bccb0df..6e18fe61de 100644 --- a/pkg/util/query_priority.go +++ b/pkg/util/query/query_priority.go @@ -1,6 +1,6 @@ -package util +package query -func IsHighPriorityQuery() bool { +func IsHighPriority() bool { // TODO: Implement // query string <> regex // query param <> start_time, end_time diff --git a/pkg/util/validation/limits.go b/pkg/util/validation/limits.go index 6e1298249a..145ad317ae 100644 --- a/pkg/util/validation/limits.go +++ b/pkg/util/validation/limits.go @@ -47,9 +47,9 @@ type DisabledRuleGroup struct { type DisabledRuleGroups []DisabledRuleGroup type HighPriorityQuery struct { - Regex string `yaml:"regex" doc:"nocli|description=Query string regex. If evaluated true (on top of meeting all other criteria), query is handled with a high priority."` - StartTime time.Duration `yaml:"start_time" doc:"nocli|description=If query range falls between the start_time and end_time (on top of meeting all other criteria), query is handled with a high priority.|default=1h"` - EndTime time.Duration `yaml:"end_time" doc:"nocli|description=If query range falls between the start_time and end_time (on top of meeting all other criteria), query is handled with a high priority.|default=0s"` + Regex string `yaml:"regex" doc:"nocli|description=Query string regex. If evaluated true (on top of meeting all other criteria), query is treated as a high priority."` + StartTime time.Duration `yaml:"start_time" doc:"nocli|description=If query range falls between the start_time and end_time (on top of meeting all other criteria), query is treated as a high priority.|default=1h"` + EndTime time.Duration `yaml:"end_time" doc:"nocli|description=If query range falls between the start_time and end_time (on top of meeting all other criteria), query is treated as a high priority.|default=0s"` } // Limits describe all the limits for users; can be used to describe global default @@ -109,7 +109,7 @@ type Limits struct { // Query Frontend / Scheduler enforced limits. MaxOutstandingPerTenant int `yaml:"max_outstanding_requests_per_tenant" json:"max_outstanding_requests_per_tenant"` ReservedHighPriorityQueriers float64 `yaml:"reserved_high_priority_queriers" json:"reserved_high_priority_queriers"` - HighPriorityQueries []HighPriorityQuery `yaml:"high_priority_queries" json:"high_priority_queries" doc:"nocli|description=List of query definitions to be handled with a high priority."` + HighPriorityQueries []HighPriorityQuery `yaml:"high_priority_queries" json:"high_priority_queries" doc:"nocli|description=List of query definitions to be treated as a high priority."` // Ruler defaults and limits. RulerEvaluationDelay model.Duration `yaml:"ruler_evaluation_delay_duration" json:"ruler_evaluation_delay_duration"` From f76761490bb12c544a9b95dbeeb39f00b5043580 Mon Sep 17 00:00:00 2001 From: Justin Jung Date: Tue, 17 Oct 2023 15:48:26 -0700 Subject: [PATCH 10/49] Lint Signed-off-by: Justin Jung --- pkg/frontend/v1/frontend.go | 4 ++-- pkg/frontend/v2/frontend.go | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/pkg/frontend/v1/frontend.go b/pkg/frontend/v1/frontend.go index 6bd37274c5..9eae59255b 100644 --- a/pkg/frontend/v1/frontend.go +++ b/pkg/frontend/v1/frontend.go @@ -22,7 +22,7 @@ import ( "github.com/cortexproject/cortex/pkg/tenant" "github.com/cortexproject/cortex/pkg/util" "github.com/cortexproject/cortex/pkg/util/httpgrpcutil" - "github.com/cortexproject/cortex/pkg/util/query" + util_query "github.com/cortexproject/cortex/pkg/util/query" "github.com/cortexproject/cortex/pkg/util/services" "github.com/cortexproject/cortex/pkg/util/validation" ) @@ -347,7 +347,7 @@ func (f *Frontend) queueRequest(ctx context.Context, req *request) error { now := time.Now() req.enqueueTime = now req.queueSpan, _ = opentracing.StartSpanFromContext(ctx, "queued") - req.isHighPriority = query.IsHighPriority() + req.isHighPriority = util_query.IsHighPriority() // aggregate the max queriers limit in the case of a multi tenant query maxQueriers := validation.SmallestPositiveNonZeroFloat64PerTenant(tenantIDs, f.limits.MaxQueriersPerUser) diff --git a/pkg/frontend/v2/frontend.go b/pkg/frontend/v2/frontend.go index 202c2d3042..e01b9562e8 100644 --- a/pkg/frontend/v2/frontend.go +++ b/pkg/frontend/v2/frontend.go @@ -26,7 +26,7 @@ import ( "github.com/cortexproject/cortex/pkg/util/grpcclient" "github.com/cortexproject/cortex/pkg/util/httpgrpcutil" util_log "github.com/cortexproject/cortex/pkg/util/log" - "github.com/cortexproject/cortex/pkg/util/query" + util_query "github.com/cortexproject/cortex/pkg/util/query" "github.com/cortexproject/cortex/pkg/util/services" ) @@ -196,7 +196,7 @@ func (f *Frontend) RoundTripGRPC(ctx context.Context, req *httpgrpc.HTTPRequest) request: req, userID: userID, statsEnabled: stats.IsEnabled(ctx), - isHighPriority: query.IsHighPriority(), + isHighPriority: util_query.IsHighPriority(), cancel: cancel, From de06e37fa7d3579bbc61bad801aad6cecb871949 Mon Sep 17 00:00:00 2001 From: Justin Jung Date: Wed, 18 Oct 2023 15:33:23 -0700 Subject: [PATCH 11/49] Implement IsPriorityQuery Signed-off-by: Justin Jung --- docs/configuration/config-file-reference.md | 2 +- pkg/frontend/config.go | 2 +- pkg/frontend/transport/roundtripper.go | 5 +- pkg/frontend/v1/frontend.go | 20 ++- pkg/frontend/v2/frontend.go | 18 ++- pkg/frontend/v2/frontend_test.go | 17 ++- pkg/scheduler/queue/queue_test.go | 2 +- pkg/scheduler/queue/user_queues.go | 21 ++- pkg/util/query/priority.go | 49 +++++++ pkg/util/query/priority_test.go | 154 ++++++++++++++++++++ pkg/util/query/query_priority.go | 9 -- pkg/util/validation/limits.go | 2 +- 12 files changed, 265 insertions(+), 36 deletions(-) create mode 100644 pkg/util/query/priority.go create mode 100644 pkg/util/query/priority_test.go delete mode 100644 pkg/util/query/query_priority.go diff --git a/docs/configuration/config-file-reference.md b/docs/configuration/config-file-reference.md index d8cffa118f..9ae1ddb8d5 100644 --- a/docs/configuration/config-file-reference.md +++ b/docs/configuration/config-file-reference.md @@ -5050,7 +5050,7 @@ otel: # If query range falls between the start_time and end_time (on top of meeting # all other criteria), query is treated as a high priority. -[start_time: | default = 1h] +[start_time: | default = 0s] # If query range falls between the start_time and end_time (on top of meeting # all other criteria), query is treated as a high priority. diff --git a/pkg/frontend/config.go b/pkg/frontend/config.go index 8ef8fa3603..03dff13980 100644 --- a/pkg/frontend/config.go +++ b/pkg/frontend/config.go @@ -59,7 +59,7 @@ func InitFrontend(cfg CombinedFrontendConfig, limits v1.Limits, grpcListenPort i cfg.FrontendV2.Port = grpcListenPort } - fr, err := v2.NewFrontend(cfg.FrontendV2, log, reg, retry) + fr, err := v2.NewFrontend(cfg.FrontendV2, limits, log, reg, retry) return transport.AdaptGrpcRoundTripperToHTTPRoundTripper(fr), nil, fr, err default: diff --git a/pkg/frontend/transport/roundtripper.go b/pkg/frontend/transport/roundtripper.go index 583fc22d04..e996d8abb3 100644 --- a/pkg/frontend/transport/roundtripper.go +++ b/pkg/frontend/transport/roundtripper.go @@ -5,6 +5,7 @@ import ( "context" "io" "net/http" + "net/url" "github.com/weaveworks/common/httpgrpc" "github.com/weaveworks/common/httpgrpc/server" @@ -12,7 +13,7 @@ import ( // GrpcRoundTripper is similar to http.RoundTripper, but works with HTTP requests converted to protobuf messages. type GrpcRoundTripper interface { - RoundTripGRPC(context.Context, *httpgrpc.HTTPRequest) (*httpgrpc.HTTPResponse, error) + RoundTripGRPC(context.Context, url.Values, *httpgrpc.HTTPRequest) (*httpgrpc.HTTPResponse, error) } func AdaptGrpcRoundTripperToHTTPRoundTripper(r GrpcRoundTripper) http.RoundTripper { @@ -39,7 +40,7 @@ func (a *grpcRoundTripperAdapter) RoundTrip(r *http.Request) (*http.Response, er return nil, err } - resp, err := a.roundTripper.RoundTripGRPC(r.Context(), req) + resp, err := a.roundTripper.RoundTripGRPC(r.Context(), r.Form, req) if err != nil { return nil, err } diff --git a/pkg/frontend/v1/frontend.go b/pkg/frontend/v1/frontend.go index 9eae59255b..64e19f35cb 100644 --- a/pkg/frontend/v1/frontend.go +++ b/pkg/frontend/v1/frontend.go @@ -5,6 +5,7 @@ import ( "flag" "fmt" "net/http" + "net/url" "time" "github.com/go-kit/log" @@ -53,6 +54,7 @@ type Limits interface { // MockLimits implements the Limits interface. Used in tests only. type MockLimits struct { Queriers float64 + Queries []validation.HighPriorityQuery queue.MockLimits } @@ -60,6 +62,10 @@ func (l MockLimits) MaxQueriersPerUser(_ string) float64 { return l.Queriers } +func (l MockLimits) HighPriorityQueries(_ string) []validation.HighPriorityQuery { + return l.Queries +} + // Frontend queues HTTP requests, dispatches them to backends, and handles retries // for requests which failed. type Frontend struct { @@ -171,7 +177,7 @@ func (f *Frontend) cleanupInactiveUserMetrics(user string) { } // RoundTripGRPC round trips a proto (instead of a HTTP request). -func (f *Frontend) RoundTripGRPC(ctx context.Context, req *httpgrpc.HTTPRequest) (*httpgrpc.HTTPResponse, error) { +func (f *Frontend) RoundTripGRPC(ctx context.Context, requestParams url.Values, req *httpgrpc.HTTPRequest) (*httpgrpc.HTTPResponse, error) { // Propagate trace context in gRPC too - this will be ignored if using HTTP. tracer, span := opentracing.GlobalTracer(), opentracing.SpanFromContext(ctx) if tracer != nil && span != nil { @@ -182,10 +188,17 @@ func (f *Frontend) RoundTripGRPC(ctx context.Context, req *httpgrpc.HTTPRequest) } } + tenantIDs, err := tenant.TenantIDs(ctx) + if err != nil { + return nil, err + } + userID := tenant.JoinTenantIDs(tenantIDs) + return f.retry.Do(ctx, func() (*httpgrpc.HTTPResponse, error) { request := request{ - request: req, - originalCtx: ctx, + request: req, + originalCtx: ctx, + isHighPriority: util_query.IsHighPriority(requestParams, f.limits.HighPriorityQueries(userID)), // Buffer of 1 to ensure response can be written by the server side // of the Process stream, even if this goroutine goes away due to @@ -347,7 +360,6 @@ func (f *Frontend) queueRequest(ctx context.Context, req *request) error { now := time.Now() req.enqueueTime = now req.queueSpan, _ = opentracing.StartSpanFromContext(ctx, "queued") - req.isHighPriority = util_query.IsHighPriority() // aggregate the max queriers limit in the case of a multi tenant query maxQueriers := validation.SmallestPositiveNonZeroFloat64PerTenant(tenantIDs, f.limits.MaxQueriersPerUser) diff --git a/pkg/frontend/v2/frontend.go b/pkg/frontend/v2/frontend.go index e01b9562e8..3f0b8d3372 100644 --- a/pkg/frontend/v2/frontend.go +++ b/pkg/frontend/v2/frontend.go @@ -6,9 +6,12 @@ import ( "fmt" "math/rand" "net/http" + "net/url" "sync" "time" + "github.com/cortexproject/cortex/pkg/scheduler" + "github.com/go-kit/log" "github.com/go-kit/log/level" "github.com/opentracing/opentracing-go" @@ -65,10 +68,10 @@ func (cfg *Config) RegisterFlags(f *flag.FlagSet) { type Frontend struct { services.Service - cfg Config - log log.Logger - - retry *transport.Retry + cfg Config + log log.Logger + limits scheduler.Limits + retry *transport.Retry lastQueryID atomic.Uint64 @@ -114,7 +117,7 @@ type enqueueResult struct { } // NewFrontend creates a new frontend. -func NewFrontend(cfg Config, log log.Logger, reg prometheus.Registerer, retry *transport.Retry) (*Frontend, error) { +func NewFrontend(cfg Config, limits scheduler.Limits, log log.Logger, reg prometheus.Registerer, retry *transport.Retry) (*Frontend, error) { requestsCh := make(chan *frontendRequest) schedulerWorkers, err := newFrontendSchedulerWorkers(cfg, fmt.Sprintf("%s:%d", cfg.Addr, cfg.Port), requestsCh, log) @@ -124,6 +127,7 @@ func NewFrontend(cfg Config, log log.Logger, reg prometheus.Registerer, retry *t f := &Frontend{ cfg: cfg, + limits: limits, log: log, requestsCh: requestsCh, schedulerWorkers: schedulerWorkers, @@ -167,7 +171,7 @@ func (f *Frontend) stopping(_ error) error { } // RoundTripGRPC round trips a proto (instead of a HTTP request). -func (f *Frontend) RoundTripGRPC(ctx context.Context, req *httpgrpc.HTTPRequest) (*httpgrpc.HTTPResponse, error) { +func (f *Frontend) RoundTripGRPC(ctx context.Context, requestParams url.Values, req *httpgrpc.HTTPRequest) (*httpgrpc.HTTPResponse, error) { if s := f.State(); s != services.Running { return nil, fmt.Errorf("frontend not running: %v", s) } @@ -196,7 +200,7 @@ func (f *Frontend) RoundTripGRPC(ctx context.Context, req *httpgrpc.HTTPRequest) request: req, userID: userID, statsEnabled: stats.IsEnabled(ctx), - isHighPriority: util_query.IsHighPriority(), + isHighPriority: util_query.IsHighPriority(requestParams, f.limits.HighPriorityQueries(userID)), cancel: cancel, diff --git a/pkg/frontend/v2/frontend_test.go b/pkg/frontend/v2/frontend_test.go index 59729e1757..68023f9f57 100644 --- a/pkg/frontend/v2/frontend_test.go +++ b/pkg/frontend/v2/frontend_test.go @@ -3,12 +3,15 @@ package v2 import ( "context" "net" + "net/url" "strconv" "strings" "sync" "testing" "time" + "github.com/cortexproject/cortex/pkg/scheduler/queue" + "github.com/go-kit/log" "github.com/stretchr/testify/require" "github.com/weaveworks/common/httpgrpc" @@ -48,7 +51,7 @@ func setupFrontend(t *testing.T, schedulerReplyFunc func(f *Frontend, msg *sched //logger := log.NewLogfmtLogger(os.Stdout) logger := log.NewNopLogger() - f, err := NewFrontend(cfg, logger, nil, transport.NewRetry(maxRetries, nil)) + f, err := NewFrontend(cfg, queue.MockLimits{}, logger, nil, transport.NewRetry(maxRetries, nil)) require.NoError(t, err) frontendv2pb.RegisterFrontendForQuerierServer(server, f) @@ -110,7 +113,7 @@ func TestFrontendBasicWorkflow(t *testing.T) { return &schedulerpb.SchedulerToFrontend{Status: schedulerpb.OK} }, 0) - resp, err := f.RoundTripGRPC(user.InjectOrgID(context.Background(), userID), &httpgrpc.HTTPRequest{}) + resp, err := f.RoundTripGRPC(user.InjectOrgID(context.Background(), userID), url.Values{}, &httpgrpc.HTTPRequest{}) require.NoError(t, err) require.Equal(t, int32(200), resp.Code) require.Equal(t, []byte(body), resp.Body) @@ -140,7 +143,7 @@ func TestFrontendRetryRequest(t *testing.T) { return &schedulerpb.SchedulerToFrontend{Status: schedulerpb.OK} }, 3) - res, err := f.RoundTripGRPC(user.InjectOrgID(context.Background(), userID), &httpgrpc.HTTPRequest{}) + res, err := f.RoundTripGRPC(user.InjectOrgID(context.Background(), userID), url.Values{}, &httpgrpc.HTTPRequest{}) require.NoError(t, err) require.Equal(t, int32(200), res.Code) } @@ -167,7 +170,7 @@ func TestFrontendRetryEnqueue(t *testing.T) { return &schedulerpb.SchedulerToFrontend{Status: schedulerpb.OK} }, 0) - _, err := f.RoundTripGRPC(user.InjectOrgID(context.Background(), userID), &httpgrpc.HTTPRequest{}) + _, err := f.RoundTripGRPC(user.InjectOrgID(context.Background(), userID), url.Values{}, &httpgrpc.HTTPRequest{}) require.NoError(t, err) } @@ -176,7 +179,7 @@ func TestFrontendEnqueueFailure(t *testing.T) { return &schedulerpb.SchedulerToFrontend{Status: schedulerpb.SHUTTING_DOWN} }, 0) - _, err := f.RoundTripGRPC(user.InjectOrgID(context.Background(), "test"), &httpgrpc.HTTPRequest{}) + _, err := f.RoundTripGRPC(user.InjectOrgID(context.Background(), "test"), url.Values{}, &httpgrpc.HTTPRequest{}) require.Error(t, err) require.True(t, strings.Contains(err.Error(), "failed to enqueue request")) } @@ -187,7 +190,7 @@ func TestFrontendCancellation(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), 200*time.Millisecond) defer cancel() - resp, err := f.RoundTripGRPC(user.InjectOrgID(ctx, "test"), &httpgrpc.HTTPRequest{}) + resp, err := f.RoundTripGRPC(user.InjectOrgID(ctx, "test"), url.Values{}, &httpgrpc.HTTPRequest{}) require.EqualError(t, err, context.DeadlineExceeded.Error()) require.Nil(t, resp) @@ -236,7 +239,7 @@ func TestFrontendFailedCancellation(t *testing.T) { }() // send request - resp, err := f.RoundTripGRPC(user.InjectOrgID(ctx, "test"), &httpgrpc.HTTPRequest{}) + resp, err := f.RoundTripGRPC(user.InjectOrgID(ctx, "test"), url.Values{}, &httpgrpc.HTTPRequest{}) require.EqualError(t, err, context.Canceled.Error()) require.Nil(t, resp) diff --git a/pkg/scheduler/queue/queue_test.go b/pkg/scheduler/queue/queue_test.go index 5e260323ec..283eb9df4f 100644 --- a/pkg/scheduler/queue/queue_test.go +++ b/pkg/scheduler/queue/queue_test.go @@ -200,7 +200,7 @@ func TestRequestQueue_ReservedQueriersShouldOnlyGetHighPriorityQueries(t *testin queue := NewRequestQueue(0, 0, prometheus.NewGaugeVec(prometheus.GaugeOpts{}, []string{"user"}), prometheus.NewCounterVec(prometheus.CounterOpts{}, []string{"user"}), - MockLimits{MaxOutstanding: 3, ReservedQueriers: 1}, + MockLimits{MaxOutstanding: 3, reservedHighPriorityQueriers: 1}, nil, ) ctx := context.Background() diff --git a/pkg/scheduler/queue/user_queues.go b/pkg/scheduler/queue/user_queues.go index fa6ac4296d..29194b613b 100644 --- a/pkg/scheduler/queue/user_queues.go +++ b/pkg/scheduler/queue/user_queues.go @@ -6,6 +6,8 @@ import ( "sort" "time" + "github.com/cortexproject/cortex/pkg/util/validation" + "github.com/cortexproject/cortex/pkg/util" ) @@ -21,6 +23,9 @@ type Limits interface { // If ReservedHighPriorityQueriers is capped by MaxQueriersPerUser. // If less than 1, it will be applied as a percentage of MaxQueriersPerUser. ReservedHighPriorityQueriers(user string) float64 + + // HighPriorityQueries returns list of definitions for high priority query. + HighPriorityQueries(user string) []validation.HighPriorityQuery } // querier holds information about a querier registered in the queue. @@ -374,8 +379,14 @@ func getNumOfReservedQueriers(queriersToSelect int, totalNumOfQueriers int, rese // MockLimits implements the Limits interface. Used in tests only. type MockLimits struct { - MaxOutstanding int - ReservedQueriers float64 + MaxOutstanding int + maxQueriersPerUser float64 + reservedHighPriorityQueriers float64 + highPriorityQueries []validation.HighPriorityQuery +} + +func (l MockLimits) MaxQueriersPerUser(user string) float64 { + return l.maxQueriersPerUser } func (l MockLimits) MaxOutstandingPerTenant(_ string) int { @@ -383,5 +394,9 @@ func (l MockLimits) MaxOutstandingPerTenant(_ string) int { } func (l MockLimits) ReservedHighPriorityQueriers(_ string) float64 { - return l.ReservedQueriers + return l.reservedHighPriorityQueriers +} + +func (l MockLimits) HighPriorityQueries(_ string) []validation.HighPriorityQuery { + return l.highPriorityQueries } diff --git a/pkg/util/query/priority.go b/pkg/util/query/priority.go new file mode 100644 index 0000000000..7347ce04d1 --- /dev/null +++ b/pkg/util/query/priority.go @@ -0,0 +1,49 @@ +package query + +import ( + "net/url" + "regexp" + "strconv" + "time" + + "github.com/cortexproject/cortex/pkg/util/validation" +) + +func IsHighPriority(requestParams url.Values, highPriorityQueries []validation.HighPriorityQuery) bool { + queryParam := requestParams.Get("query") + timeParam := requestParams.Get("time") + startParam := requestParams.Get("start") + endParam := requestParams.Get("end") + + for _, highPriorityQuery := range highPriorityQueries { + regex := highPriorityQuery.Regex + + if match, err := regexp.MatchString(regex, queryParam); !match || err != nil { + continue + } + + now := time.Now() + startTimeThreshold := now.Add(-1 * highPriorityQuery.StartTime).UnixMilli() + endTimeThreshold := now.Add(-1 * highPriorityQuery.EndTime).UnixMilli() + + if time, err := strconv.ParseInt(timeParam, 10, 64); err == nil { + if isBetweenThresholds(time, time, startTimeThreshold, endTimeThreshold) { + return true + } + } + + if startTime, err := strconv.ParseInt(startParam, 10, 64); err == nil { + if endTime, err := strconv.ParseInt(endParam, 10, 64); err == nil { + if isBetweenThresholds(startTime, endTime, startTimeThreshold, endTimeThreshold) { + return true + } + } + } + } + + return false +} + +func isBetweenThresholds(start, end, startThreshold, endThreshold int64) bool { + return start >= startThreshold && end <= endThreshold +} diff --git a/pkg/util/query/priority_test.go b/pkg/util/query/priority_test.go new file mode 100644 index 0000000000..2e7a414487 --- /dev/null +++ b/pkg/util/query/priority_test.go @@ -0,0 +1,154 @@ +package query + +import ( + "net/url" + "strconv" + "testing" + "time" + + "github.com/stretchr/testify/assert" + + "github.com/cortexproject/cortex/pkg/util/validation" +) + +func Test_IsHighPriority_DefaultValues(t *testing.T) { + now := time.Now() + config := []validation.HighPriorityQuery{ + {}, // By default, it should match all queries happened at "now" + } + + assert.True(t, IsHighPriority(url.Values{ + "query": []string{"count(up)"}, + "time": []string{strconv.FormatInt(now.UnixMilli(), 10)}, + }, config)) + assert.False(t, IsHighPriority(url.Values{ + "query": []string{"count(up)"}, + "time": []string{strconv.FormatInt(now.Add(-1*time.Second).UnixMilli(), 10)}, + }, config)) +} + +func Test_IsHighPriority_ShouldMatchRegex(t *testing.T) { + now := time.Now() + config := []validation.HighPriorityQuery{ + { + Regex: "sum", + }, + } + + assert.True(t, IsHighPriority(url.Values{ + "query": []string{"sum(up)"}, + "time": []string{strconv.FormatInt(now.UnixMilli(), 10)}, + }, config)) + assert.False(t, IsHighPriority(url.Values{ + "query": []string{"count(up)"}, + "time": []string{strconv.FormatInt(now.UnixMilli(), 10)}, + }, config)) + + config = []validation.HighPriorityQuery{ + { + Regex: "up", + }, + } + + assert.True(t, IsHighPriority(url.Values{ + "query": []string{"sum(up)"}, + "time": []string{strconv.FormatInt(now.UnixMilli(), 10)}, + }, config)) + assert.True(t, IsHighPriority(url.Values{ + "query": []string{"count(up)"}, + "time": []string{strconv.FormatInt(now.UnixMilli(), 10)}, + }, config)) + + config = []validation.HighPriorityQuery{ + { + Regex: "sum", + }, + { + Regex: "c(.+)t", + }, + } + + assert.True(t, IsHighPriority(url.Values{ + "query": []string{"sum(up)"}, + "time": []string{strconv.FormatInt(now.UnixMilli(), 10)}, + }, config)) + assert.True(t, IsHighPriority(url.Values{ + "query": []string{"count(up)"}, + "time": []string{strconv.FormatInt(now.UnixMilli(), 10)}, + }, config)) + + config = []validation.HighPriorityQuery{ + { + Regex: "doesnotexist", + }, + { + Regex: "^sum$", + }, + } + + assert.False(t, IsHighPriority(url.Values{ + "query": []string{"sum(up)"}, + "time": []string{strconv.FormatInt(now.UnixMilli(), 10)}, + }, config)) + assert.False(t, IsHighPriority(url.Values{ + "query": []string{"count(up)"}, + "time": []string{strconv.FormatInt(now.UnixMilli(), 10)}, + }, config)) + + config = []validation.HighPriorityQuery{ + { + Regex: ".*", + }, + } + + assert.True(t, IsHighPriority(url.Values{ + "query": []string{"sum(up)"}, + "time": []string{strconv.FormatInt(now.UnixMilli(), 10)}, + }, config)) + assert.True(t, IsHighPriority(url.Values{ + "query": []string{"count(up)"}, + "time": []string{strconv.FormatInt(now.UnixMilli(), 10)}, + }, config)) +} + +func Test_IsHighPriority_ShouldBeBetweenStartAndEndTime(t *testing.T) { + now := time.Now() + config := []validation.HighPriorityQuery{ + { + StartTime: 1 * time.Hour, + EndTime: 30 * time.Minute, + }, + } + + assert.False(t, IsHighPriority(url.Values{ + "query": []string{"sum(up)"}, + "time": []string{strconv.FormatInt(now.Add(-2*time.Hour).UnixMilli(), 10)}, + }, config)) + assert.True(t, IsHighPriority(url.Values{ + "query": []string{"sum(up)"}, + "time": []string{strconv.FormatInt(now.Add(-1*time.Hour).UnixMilli(), 10)}, + }, config)) + assert.True(t, IsHighPriority(url.Values{ + "query": []string{"sum(up)"}, + "time": []string{strconv.FormatInt(now.Add(-30*time.Minute).UnixMilli(), 10)}, + }, config)) + assert.False(t, IsHighPriority(url.Values{ + "query": []string{"sum(up)"}, + "time": []string{strconv.FormatInt(now.Add(-1*time.Minute).UnixMilli(), 10)}, + }, config)) + assert.False(t, IsHighPriority(url.Values{ + "query": []string{"sum(up)"}, + "start": []string{strconv.FormatInt(now.Add(-2*time.Hour).UnixMilli(), 10)}, + "end": []string{strconv.FormatInt(now.Add(-30*time.Minute).UnixMilli(), 10)}, + }, config)) + assert.True(t, IsHighPriority(url.Values{ + "query": []string{"sum(up)"}, + "start": []string{strconv.FormatInt(now.Add(-1*time.Hour).UnixMilli(), 10)}, + "end": []string{strconv.FormatInt(now.Add(-30*time.Minute).UnixMilli(), 10)}, + }, config)) + assert.False(t, IsHighPriority(url.Values{ + "query": []string{"sum(up)"}, + "start": []string{strconv.FormatInt(now.Add(-1*time.Hour).UnixMilli(), 10)}, + "end": []string{strconv.FormatInt(now.UnixMilli(), 10)}, + }, config)) +} diff --git a/pkg/util/query/query_priority.go b/pkg/util/query/query_priority.go deleted file mode 100644 index 6e18fe61de..0000000000 --- a/pkg/util/query/query_priority.go +++ /dev/null @@ -1,9 +0,0 @@ -package query - -func IsHighPriority() bool { - // TODO: Implement - // query string <> regex - // query param <> start_time, end_time - - return false -} diff --git a/pkg/util/validation/limits.go b/pkg/util/validation/limits.go index 145ad317ae..301e945ea8 100644 --- a/pkg/util/validation/limits.go +++ b/pkg/util/validation/limits.go @@ -48,7 +48,7 @@ type DisabledRuleGroups []DisabledRuleGroup type HighPriorityQuery struct { Regex string `yaml:"regex" doc:"nocli|description=Query string regex. If evaluated true (on top of meeting all other criteria), query is treated as a high priority."` - StartTime time.Duration `yaml:"start_time" doc:"nocli|description=If query range falls between the start_time and end_time (on top of meeting all other criteria), query is treated as a high priority.|default=1h"` + StartTime time.Duration `yaml:"start_time" doc:"nocli|description=If query range falls between the start_time and end_time (on top of meeting all other criteria), query is treated as a high priority.|default=0s"` EndTime time.Duration `yaml:"end_time" doc:"nocli|description=If query range falls between the start_time and end_time (on top of meeting all other criteria), query is treated as a high priority.|default=0s"` } From 8b31b1cb25143b8fe8baf38de05687a1c60a0f8e Mon Sep 17 00:00:00 2001 From: Justin Jung Date: Wed, 18 Oct 2023 15:37:51 -0700 Subject: [PATCH 12/49] Add changelog Signed-off-by: Justin Jung --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index efcf471713..7a8a00c5d0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,7 @@ * [CHANGE] Store Gateway: Add a new fastcache based inmemory index cache. #5619 * [CHANGE] Index Cache: Multi level cache backfilling operation becomes async. Added `-blocks-storage.bucket-store.index-cache.multilevel.max-async-concurrency` and `-blocks-storage.bucket-store.index-cache.multilevel.max-async-buffer-size` configs and metric `cortex_store_multilevel_index_cache_backfill_dropped_items_total` for number of dropped items. #5661 * [FEATURE] Ingester: Add per-tenant new metric `cortex_ingester_tsdb_data_replay_duration_seconds`. #5477 +* [FEATURE] Query Frontend/Scheduler: Add query priority support. #5605 * [ENHANCEMENT] Store Gateway: Added `-store-gateway.enabled-tenants` and `-store-gateway.disabled-tenants` to explicitly enable or disable store-gateway for specific tenants. #5638 ## 1.16.0 2023-11-20 From d28aa335dcc66e535fa4afa5918601ad3559662e Mon Sep 17 00:00:00 2001 From: Justin Jung Date: Wed, 18 Oct 2023 16:37:00 -0700 Subject: [PATCH 13/49] Pass timestamp as param Signed-off-by: Justin Jung --- pkg/frontend/transport/roundtripper.go | 5 +- pkg/frontend/v1/frontend.go | 4 +- pkg/frontend/v2/frontend.go | 4 +- pkg/frontend/v2/frontend_test.go | 12 ++--- pkg/util/query/priority.go | 11 ++--- pkg/util/query/priority_test.go | 65 +++++++++++++------------- 6 files changed, 50 insertions(+), 51 deletions(-) diff --git a/pkg/frontend/transport/roundtripper.go b/pkg/frontend/transport/roundtripper.go index e996d8abb3..ccbd8d4cc5 100644 --- a/pkg/frontend/transport/roundtripper.go +++ b/pkg/frontend/transport/roundtripper.go @@ -6,6 +6,7 @@ import ( "io" "net/http" "net/url" + "time" "github.com/weaveworks/common/httpgrpc" "github.com/weaveworks/common/httpgrpc/server" @@ -13,7 +14,7 @@ import ( // GrpcRoundTripper is similar to http.RoundTripper, but works with HTTP requests converted to protobuf messages. type GrpcRoundTripper interface { - RoundTripGRPC(context.Context, url.Values, *httpgrpc.HTTPRequest) (*httpgrpc.HTTPResponse, error) + RoundTripGRPC(context.Context, url.Values, time.Time, *httpgrpc.HTTPRequest) (*httpgrpc.HTTPResponse, error) } func AdaptGrpcRoundTripperToHTTPRoundTripper(r GrpcRoundTripper) http.RoundTripper { @@ -40,7 +41,7 @@ func (a *grpcRoundTripperAdapter) RoundTrip(r *http.Request) (*http.Response, er return nil, err } - resp, err := a.roundTripper.RoundTripGRPC(r.Context(), r.Form, req) + resp, err := a.roundTripper.RoundTripGRPC(r.Context(), r.Form, time.Now(), req) if err != nil { return nil, err } diff --git a/pkg/frontend/v1/frontend.go b/pkg/frontend/v1/frontend.go index 64e19f35cb..499790f3af 100644 --- a/pkg/frontend/v1/frontend.go +++ b/pkg/frontend/v1/frontend.go @@ -177,7 +177,7 @@ func (f *Frontend) cleanupInactiveUserMetrics(user string) { } // RoundTripGRPC round trips a proto (instead of a HTTP request). -func (f *Frontend) RoundTripGRPC(ctx context.Context, requestParams url.Values, req *httpgrpc.HTTPRequest) (*httpgrpc.HTTPResponse, error) { +func (f *Frontend) RoundTripGRPC(ctx context.Context, requestParams url.Values, timestamp time.Time, req *httpgrpc.HTTPRequest) (*httpgrpc.HTTPResponse, error) { // Propagate trace context in gRPC too - this will be ignored if using HTTP. tracer, span := opentracing.GlobalTracer(), opentracing.SpanFromContext(ctx) if tracer != nil && span != nil { @@ -198,7 +198,7 @@ func (f *Frontend) RoundTripGRPC(ctx context.Context, requestParams url.Values, request := request{ request: req, originalCtx: ctx, - isHighPriority: util_query.IsHighPriority(requestParams, f.limits.HighPriorityQueries(userID)), + isHighPriority: util_query.IsHighPriority(requestParams, timestamp, f.limits.HighPriorityQueries(userID)), // Buffer of 1 to ensure response can be written by the server side // of the Process stream, even if this goroutine goes away due to diff --git a/pkg/frontend/v2/frontend.go b/pkg/frontend/v2/frontend.go index 3f0b8d3372..447f797a43 100644 --- a/pkg/frontend/v2/frontend.go +++ b/pkg/frontend/v2/frontend.go @@ -171,7 +171,7 @@ func (f *Frontend) stopping(_ error) error { } // RoundTripGRPC round trips a proto (instead of a HTTP request). -func (f *Frontend) RoundTripGRPC(ctx context.Context, requestParams url.Values, req *httpgrpc.HTTPRequest) (*httpgrpc.HTTPResponse, error) { +func (f *Frontend) RoundTripGRPC(ctx context.Context, requestParams url.Values, timestamp time.Time, req *httpgrpc.HTTPRequest) (*httpgrpc.HTTPResponse, error) { if s := f.State(); s != services.Running { return nil, fmt.Errorf("frontend not running: %v", s) } @@ -200,7 +200,7 @@ func (f *Frontend) RoundTripGRPC(ctx context.Context, requestParams url.Values, request: req, userID: userID, statsEnabled: stats.IsEnabled(ctx), - isHighPriority: util_query.IsHighPriority(requestParams, f.limits.HighPriorityQueries(userID)), + isHighPriority: util_query.IsHighPriority(requestParams, timestamp, f.limits.HighPriorityQueries(userID)), cancel: cancel, diff --git a/pkg/frontend/v2/frontend_test.go b/pkg/frontend/v2/frontend_test.go index 68023f9f57..3b7b9337eb 100644 --- a/pkg/frontend/v2/frontend_test.go +++ b/pkg/frontend/v2/frontend_test.go @@ -113,7 +113,7 @@ func TestFrontendBasicWorkflow(t *testing.T) { return &schedulerpb.SchedulerToFrontend{Status: schedulerpb.OK} }, 0) - resp, err := f.RoundTripGRPC(user.InjectOrgID(context.Background(), userID), url.Values{}, &httpgrpc.HTTPRequest{}) + resp, err := f.RoundTripGRPC(user.InjectOrgID(context.Background(), userID), url.Values{}, time.Now(), &httpgrpc.HTTPRequest{}) require.NoError(t, err) require.Equal(t, int32(200), resp.Code) require.Equal(t, []byte(body), resp.Body) @@ -143,7 +143,7 @@ func TestFrontendRetryRequest(t *testing.T) { return &schedulerpb.SchedulerToFrontend{Status: schedulerpb.OK} }, 3) - res, err := f.RoundTripGRPC(user.InjectOrgID(context.Background(), userID), url.Values{}, &httpgrpc.HTTPRequest{}) + res, err := f.RoundTripGRPC(user.InjectOrgID(context.Background(), userID), url.Values{}, time.Now(), &httpgrpc.HTTPRequest{}) require.NoError(t, err) require.Equal(t, int32(200), res.Code) } @@ -170,7 +170,7 @@ func TestFrontendRetryEnqueue(t *testing.T) { return &schedulerpb.SchedulerToFrontend{Status: schedulerpb.OK} }, 0) - _, err := f.RoundTripGRPC(user.InjectOrgID(context.Background(), userID), url.Values{}, &httpgrpc.HTTPRequest{}) + _, err := f.RoundTripGRPC(user.InjectOrgID(context.Background(), userID), url.Values{}, time.Now(), &httpgrpc.HTTPRequest{}) require.NoError(t, err) } @@ -179,7 +179,7 @@ func TestFrontendEnqueueFailure(t *testing.T) { return &schedulerpb.SchedulerToFrontend{Status: schedulerpb.SHUTTING_DOWN} }, 0) - _, err := f.RoundTripGRPC(user.InjectOrgID(context.Background(), "test"), url.Values{}, &httpgrpc.HTTPRequest{}) + _, err := f.RoundTripGRPC(user.InjectOrgID(context.Background(), "test"), url.Values{}, time.Now(), &httpgrpc.HTTPRequest{}) require.Error(t, err) require.True(t, strings.Contains(err.Error(), "failed to enqueue request")) } @@ -190,7 +190,7 @@ func TestFrontendCancellation(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), 200*time.Millisecond) defer cancel() - resp, err := f.RoundTripGRPC(user.InjectOrgID(ctx, "test"), url.Values{}, &httpgrpc.HTTPRequest{}) + resp, err := f.RoundTripGRPC(user.InjectOrgID(ctx, "test"), url.Values{}, time.Now(), &httpgrpc.HTTPRequest{}) require.EqualError(t, err, context.DeadlineExceeded.Error()) require.Nil(t, resp) @@ -239,7 +239,7 @@ func TestFrontendFailedCancellation(t *testing.T) { }() // send request - resp, err := f.RoundTripGRPC(user.InjectOrgID(ctx, "test"), url.Values{}, &httpgrpc.HTTPRequest{}) + resp, err := f.RoundTripGRPC(user.InjectOrgID(ctx, "test"), url.Values{}, time.Now(), &httpgrpc.HTTPRequest{}) require.EqualError(t, err, context.Canceled.Error()) require.Nil(t, resp) diff --git a/pkg/util/query/priority.go b/pkg/util/query/priority.go index 7347ce04d1..45d36f4270 100644 --- a/pkg/util/query/priority.go +++ b/pkg/util/query/priority.go @@ -9,7 +9,7 @@ import ( "github.com/cortexproject/cortex/pkg/util/validation" ) -func IsHighPriority(requestParams url.Values, highPriorityQueries []validation.HighPriorityQuery) bool { +func IsHighPriority(requestParams url.Values, timestamp time.Time, highPriorityQueries []validation.HighPriorityQuery) bool { queryParam := requestParams.Get("query") timeParam := requestParams.Get("time") startParam := requestParams.Get("start") @@ -22,12 +22,11 @@ func IsHighPriority(requestParams url.Values, highPriorityQueries []validation.H continue } - now := time.Now() - startTimeThreshold := now.Add(-1 * highPriorityQuery.StartTime).UnixMilli() - endTimeThreshold := now.Add(-1 * highPriorityQuery.EndTime).UnixMilli() + startTimeThreshold := timestamp.Add(-1 * highPriorityQuery.StartTime.Abs()).UnixMilli() + endTimeThreshold := timestamp.Add(-1 * highPriorityQuery.EndTime.Abs()).UnixMilli() - if time, err := strconv.ParseInt(timeParam, 10, 64); err == nil { - if isBetweenThresholds(time, time, startTimeThreshold, endTimeThreshold) { + if instantTime, err := strconv.ParseInt(timeParam, 10, 64); err == nil { + if isBetweenThresholds(instantTime, instantTime, startTimeThreshold, endTimeThreshold) { return true } } diff --git a/pkg/util/query/priority_test.go b/pkg/util/query/priority_test.go index 2e7a414487..dd1ac9c78b 100644 --- a/pkg/util/query/priority_test.go +++ b/pkg/util/query/priority_test.go @@ -11,22 +11,6 @@ import ( "github.com/cortexproject/cortex/pkg/util/validation" ) -func Test_IsHighPriority_DefaultValues(t *testing.T) { - now := time.Now() - config := []validation.HighPriorityQuery{ - {}, // By default, it should match all queries happened at "now" - } - - assert.True(t, IsHighPriority(url.Values{ - "query": []string{"count(up)"}, - "time": []string{strconv.FormatInt(now.UnixMilli(), 10)}, - }, config)) - assert.False(t, IsHighPriority(url.Values{ - "query": []string{"count(up)"}, - "time": []string{strconv.FormatInt(now.Add(-1*time.Second).UnixMilli(), 10)}, - }, config)) -} - func Test_IsHighPriority_ShouldMatchRegex(t *testing.T) { now := time.Now() config := []validation.HighPriorityQuery{ @@ -38,11 +22,11 @@ func Test_IsHighPriority_ShouldMatchRegex(t *testing.T) { assert.True(t, IsHighPriority(url.Values{ "query": []string{"sum(up)"}, "time": []string{strconv.FormatInt(now.UnixMilli(), 10)}, - }, config)) + }, now, config)) assert.False(t, IsHighPriority(url.Values{ "query": []string{"count(up)"}, "time": []string{strconv.FormatInt(now.UnixMilli(), 10)}, - }, config)) + }, now, config)) config = []validation.HighPriorityQuery{ { @@ -53,11 +37,11 @@ func Test_IsHighPriority_ShouldMatchRegex(t *testing.T) { assert.True(t, IsHighPriority(url.Values{ "query": []string{"sum(up)"}, "time": []string{strconv.FormatInt(now.UnixMilli(), 10)}, - }, config)) + }, now, config)) assert.True(t, IsHighPriority(url.Values{ "query": []string{"count(up)"}, "time": []string{strconv.FormatInt(now.UnixMilli(), 10)}, - }, config)) + }, now, config)) config = []validation.HighPriorityQuery{ { @@ -71,11 +55,11 @@ func Test_IsHighPriority_ShouldMatchRegex(t *testing.T) { assert.True(t, IsHighPriority(url.Values{ "query": []string{"sum(up)"}, "time": []string{strconv.FormatInt(now.UnixMilli(), 10)}, - }, config)) + }, now, config)) assert.True(t, IsHighPriority(url.Values{ "query": []string{"count(up)"}, "time": []string{strconv.FormatInt(now.UnixMilli(), 10)}, - }, config)) + }, now, config)) config = []validation.HighPriorityQuery{ { @@ -89,11 +73,11 @@ func Test_IsHighPriority_ShouldMatchRegex(t *testing.T) { assert.False(t, IsHighPriority(url.Values{ "query": []string{"sum(up)"}, "time": []string{strconv.FormatInt(now.UnixMilli(), 10)}, - }, config)) + }, now, config)) assert.False(t, IsHighPriority(url.Values{ "query": []string{"count(up)"}, "time": []string{strconv.FormatInt(now.UnixMilli(), 10)}, - }, config)) + }, now, config)) config = []validation.HighPriorityQuery{ { @@ -104,11 +88,26 @@ func Test_IsHighPriority_ShouldMatchRegex(t *testing.T) { assert.True(t, IsHighPriority(url.Values{ "query": []string{"sum(up)"}, "time": []string{strconv.FormatInt(now.UnixMilli(), 10)}, - }, config)) + }, now, config)) + assert.True(t, IsHighPriority(url.Values{ + "query": []string{"count(up)"}, + "time": []string{strconv.FormatInt(now.UnixMilli(), 10)}, + }, now, config)) + + config = []validation.HighPriorityQuery{ + { + Regex: "", + }, + } + + assert.True(t, IsHighPriority(url.Values{ + "query": []string{"sum(up)"}, + "time": []string{strconv.FormatInt(now.UnixMilli(), 10)}, + }, now, config)) assert.True(t, IsHighPriority(url.Values{ "query": []string{"count(up)"}, "time": []string{strconv.FormatInt(now.UnixMilli(), 10)}, - }, config)) + }, now, config)) } func Test_IsHighPriority_ShouldBeBetweenStartAndEndTime(t *testing.T) { @@ -123,32 +122,32 @@ func Test_IsHighPriority_ShouldBeBetweenStartAndEndTime(t *testing.T) { assert.False(t, IsHighPriority(url.Values{ "query": []string{"sum(up)"}, "time": []string{strconv.FormatInt(now.Add(-2*time.Hour).UnixMilli(), 10)}, - }, config)) + }, now, config)) assert.True(t, IsHighPriority(url.Values{ "query": []string{"sum(up)"}, "time": []string{strconv.FormatInt(now.Add(-1*time.Hour).UnixMilli(), 10)}, - }, config)) + }, now, config)) assert.True(t, IsHighPriority(url.Values{ "query": []string{"sum(up)"}, "time": []string{strconv.FormatInt(now.Add(-30*time.Minute).UnixMilli(), 10)}, - }, config)) + }, now, config)) assert.False(t, IsHighPriority(url.Values{ "query": []string{"sum(up)"}, "time": []string{strconv.FormatInt(now.Add(-1*time.Minute).UnixMilli(), 10)}, - }, config)) + }, now, config)) assert.False(t, IsHighPriority(url.Values{ "query": []string{"sum(up)"}, "start": []string{strconv.FormatInt(now.Add(-2*time.Hour).UnixMilli(), 10)}, "end": []string{strconv.FormatInt(now.Add(-30*time.Minute).UnixMilli(), 10)}, - }, config)) + }, now, config)) assert.True(t, IsHighPriority(url.Values{ "query": []string{"sum(up)"}, "start": []string{strconv.FormatInt(now.Add(-1*time.Hour).UnixMilli(), 10)}, "end": []string{strconv.FormatInt(now.Add(-30*time.Minute).UnixMilli(), 10)}, - }, config)) + }, now, config)) assert.False(t, IsHighPriority(url.Values{ "query": []string{"sum(up)"}, "start": []string{strconv.FormatInt(now.Add(-1*time.Hour).UnixMilli(), 10)}, "end": []string{strconv.FormatInt(now.UnixMilli(), 10)}, - }, config)) + }, now, config)) } From daf269bfbb6ea44fd49de66e49589cfd7907f647 Mon Sep 17 00:00:00 2001 From: Justin Jung Date: Wed, 25 Oct 2023 14:11:18 -0700 Subject: [PATCH 14/49] Parse form so that range query parameters are passed to the roundtrip Signed-off-by: Justin Jung --- pkg/frontend/transport/roundtripper.go | 4 +++ pkg/util/query/priority.go | 38 ++++++++++++++++++++------ 2 files changed, 34 insertions(+), 8 deletions(-) diff --git a/pkg/frontend/transport/roundtripper.go b/pkg/frontend/transport/roundtripper.go index ccbd8d4cc5..cc653de3b6 100644 --- a/pkg/frontend/transport/roundtripper.go +++ b/pkg/frontend/transport/roundtripper.go @@ -41,6 +41,10 @@ func (a *grpcRoundTripperAdapter) RoundTrip(r *http.Request) (*http.Response, er return nil, err } + if err = r.ParseForm(); err != nil { + return nil, err + } + resp, err := a.roundTripper.RoundTripGRPC(r.Context(), r.Form, time.Now(), req) if err != nil { return nil, err diff --git a/pkg/util/query/priority.go b/pkg/util/query/priority.go index 45d36f4270..84c7dfa262 100644 --- a/pkg/util/query/priority.go +++ b/pkg/util/query/priority.go @@ -1,20 +1,27 @@ package query import ( + "math" "net/url" "regexp" "strconv" "time" + "github.com/pkg/errors" + "github.com/cortexproject/cortex/pkg/util/validation" ) -func IsHighPriority(requestParams url.Values, timestamp time.Time, highPriorityQueries []validation.HighPriorityQuery) bool { +func IsHighPriority(requestParams url.Values, now time.Time, highPriorityQueries []validation.HighPriorityQuery) bool { queryParam := requestParams.Get("query") timeParam := requestParams.Get("time") startParam := requestParams.Get("start") endParam := requestParams.Get("end") + if queryParam == "" { + return false + } + for _, highPriorityQuery := range highPriorityQueries { regex := highPriorityQuery.Regex @@ -22,17 +29,17 @@ func IsHighPriority(requestParams url.Values, timestamp time.Time, highPriorityQ continue } - startTimeThreshold := timestamp.Add(-1 * highPriorityQuery.StartTime.Abs()).UnixMilli() - endTimeThreshold := timestamp.Add(-1 * highPriorityQuery.EndTime.Abs()).UnixMilli() + startTimeThreshold := now.Add(-1 * highPriorityQuery.StartTime.Abs()) + endTimeThreshold := now.Add(-1 * highPriorityQuery.EndTime.Abs()) - if instantTime, err := strconv.ParseInt(timeParam, 10, 64); err == nil { + if instantTime, err := parseTime(timeParam); err == nil { if isBetweenThresholds(instantTime, instantTime, startTimeThreshold, endTimeThreshold) { return true } } - if startTime, err := strconv.ParseInt(startParam, 10, 64); err == nil { - if endTime, err := strconv.ParseInt(endParam, 10, 64); err == nil { + if startTime, err := parseTime(startParam); err == nil { + if endTime, err := parseTime(endParam); err == nil { if isBetweenThresholds(startTime, endTime, startTimeThreshold, endTimeThreshold) { return true } @@ -43,6 +50,21 @@ func IsHighPriority(requestParams url.Values, timestamp time.Time, highPriorityQ return false } -func isBetweenThresholds(start, end, startThreshold, endThreshold int64) bool { - return start >= startThreshold && end <= endThreshold +func parseTime(s string) (time.Time, error) { + if s != "" { + if t, err := strconv.ParseFloat(s, 64); err == nil { + s, ns := math.Modf(t) + ns = math.Round(ns*1000) / 1000 + return time.Unix(int64(s), int64(ns*float64(time.Second))).UTC(), nil + } + if t, err := time.Parse(time.RFC3339Nano, s); err == nil { + return t, nil + } + } + + return time.Time{}, errors.Errorf("cannot parse %q to a valid timestamp", s) +} + +func isBetweenThresholds(start, end, startThreshold, endThreshold time.Time) bool { + return start.After(startThreshold) && end.Before(endThreshold) } From 3b603ba8d15b26b96cd54a4e0cf6d1baba27c081 Mon Sep 17 00:00:00 2001 From: Justin Jung Date: Sun, 29 Oct 2023 14:33:12 -0700 Subject: [PATCH 15/49] Address comments Signed-off-by: Justin Jung --- pkg/frontend/transport/roundtripper.go | 22 +++- pkg/frontend/v1/frontend.go | 21 ++-- pkg/frontend/v2/frontend.go | 28 ++--- pkg/frontend/v2/frontend_scheduler_worker.go | 2 +- pkg/scheduler/queue/queue_test.go | 2 +- pkg/scheduler/queue/user_queues.go | 6 +- pkg/scheduler/scheduler.go | 6 +- pkg/scheduler/schedulerpb/scheduler.pb.go | 114 +++++++++---------- pkg/scheduler/schedulerpb/scheduler.proto | 2 +- pkg/util/query/priority_test.go | 4 +- pkg/util/validation/limits.go | 8 +- 11 files changed, 118 insertions(+), 97 deletions(-) diff --git a/pkg/frontend/transport/roundtripper.go b/pkg/frontend/transport/roundtripper.go index cc653de3b6..aaa6f306d1 100644 --- a/pkg/frontend/transport/roundtripper.go +++ b/pkg/frontend/transport/roundtripper.go @@ -3,9 +3,12 @@ package transport import ( "bytes" "context" + "fmt" "io" "net/http" "net/url" + "regexp" + "strings" "time" "github.com/weaveworks/common/httpgrpc" @@ -14,7 +17,7 @@ import ( // GrpcRoundTripper is similar to http.RoundTripper, but works with HTTP requests converted to protobuf messages. type GrpcRoundTripper interface { - RoundTripGRPC(context.Context, url.Values, time.Time, *httpgrpc.HTTPRequest) (*httpgrpc.HTTPResponse, error) + RoundTripGRPC(context.Context, *httpgrpc.HTTPRequest, url.Values, time.Time) (*httpgrpc.HTTPResponse, error) } func AdaptGrpcRoundTripperToHTTPRoundTripper(r GrpcRoundTripper) http.RoundTripper { @@ -36,16 +39,27 @@ func (b *buffer) Bytes() []byte { } func (a *grpcRoundTripperAdapter) RoundTrip(r *http.Request) (*http.Response, error) { + regexp.MustCompile("str") req, err := server.HTTPRequest(r) if err != nil { return nil, err } + fmt.Println(r.URL.Path) - if err = r.ParseForm(); err != nil { - return nil, err + var ( + resp *httpgrpc.HTTPResponse + reqValues url.Values + ts time.Time + ) + + if strings.HasSuffix(r.URL.Path, "/query") || strings.HasSuffix(r.URL.Path, "/query_range") { + if err = r.ParseForm(); err == nil { + reqValues = r.Form + ts = time.Now() + } } - resp, err := a.roundTripper.RoundTripGRPC(r.Context(), r.Form, time.Now(), req) + resp, err = a.roundTripper.RoundTripGRPC(r.Context(), req, reqValues, ts) if err != nil { return nil, err } diff --git a/pkg/frontend/v1/frontend.go b/pkg/frontend/v1/frontend.go index 499790f3af..8f2fae830f 100644 --- a/pkg/frontend/v1/frontend.go +++ b/pkg/frontend/v1/frontend.go @@ -95,14 +95,14 @@ type request struct { queueSpan opentracing.Span originalCtx context.Context - request *httpgrpc.HTTPRequest - err chan error - response chan *httpgrpc.HTTPResponse - isHighPriority bool + request *httpgrpc.HTTPRequest + err chan error + response chan *httpgrpc.HTTPResponse + highPriority bool } func (r request) IsHighPriority() bool { - return r.isHighPriority + return r.highPriority } // New creates a new frontend. Frontend implements service, and must be started and stopped. @@ -177,7 +177,7 @@ func (f *Frontend) cleanupInactiveUserMetrics(user string) { } // RoundTripGRPC round trips a proto (instead of a HTTP request). -func (f *Frontend) RoundTripGRPC(ctx context.Context, requestParams url.Values, timestamp time.Time, req *httpgrpc.HTTPRequest) (*httpgrpc.HTTPResponse, error) { +func (f *Frontend) RoundTripGRPC(ctx context.Context, req *httpgrpc.HTTPRequest, reqParams url.Values, ts time.Time) (*httpgrpc.HTTPResponse, error) { // Propagate trace context in gRPC too - this will be ignored if using HTTP. tracer, span := opentracing.GlobalTracer(), opentracing.SpanFromContext(ctx) if tracer != nil && span != nil { @@ -196,9 +196,8 @@ func (f *Frontend) RoundTripGRPC(ctx context.Context, requestParams url.Values, return f.retry.Do(ctx, func() (*httpgrpc.HTTPResponse, error) { request := request{ - request: req, - originalCtx: ctx, - isHighPriority: util_query.IsHighPriority(requestParams, timestamp, f.limits.HighPriorityQueries(userID)), + request: req, + originalCtx: ctx, // Buffer of 1 to ensure response can be written by the server side // of the Process stream, even if this goroutine goes away due to @@ -207,6 +206,10 @@ func (f *Frontend) RoundTripGRPC(ctx context.Context, requestParams url.Values, response: make(chan *httpgrpc.HTTPResponse, 1), } + if reqParams != nil { + request.highPriority = util_query.IsHighPriority(reqParams, ts, f.limits.HighPriorityQueries(userID)) + } + if err := f.queueRequest(ctx, &request); err != nil { return nil, err } diff --git a/pkg/frontend/v2/frontend.go b/pkg/frontend/v2/frontend.go index 447f797a43..3cb00bdbca 100644 --- a/pkg/frontend/v2/frontend.go +++ b/pkg/frontend/v2/frontend.go @@ -10,8 +10,6 @@ import ( "sync" "time" - "github.com/cortexproject/cortex/pkg/scheduler" - "github.com/go-kit/log" "github.com/go-kit/log/level" "github.com/opentracing/opentracing-go" @@ -24,6 +22,7 @@ import ( "github.com/cortexproject/cortex/pkg/frontend/transport" "github.com/cortexproject/cortex/pkg/frontend/v2/frontendv2pb" "github.com/cortexproject/cortex/pkg/querier/stats" + "github.com/cortexproject/cortex/pkg/scheduler" "github.com/cortexproject/cortex/pkg/tenant" "github.com/cortexproject/cortex/pkg/util/flagext" "github.com/cortexproject/cortex/pkg/util/grpcclient" @@ -86,11 +85,11 @@ type Frontend struct { } type frontendRequest struct { - queryID uint64 - request *httpgrpc.HTTPRequest - userID string - statsEnabled bool - isHighPriority bool + queryID uint64 + request *httpgrpc.HTTPRequest + userID string + statsEnabled bool + highPriority bool cancel context.CancelFunc @@ -171,7 +170,7 @@ func (f *Frontend) stopping(_ error) error { } // RoundTripGRPC round trips a proto (instead of a HTTP request). -func (f *Frontend) RoundTripGRPC(ctx context.Context, requestParams url.Values, timestamp time.Time, req *httpgrpc.HTTPRequest) (*httpgrpc.HTTPResponse, error) { +func (f *Frontend) RoundTripGRPC(ctx context.Context, req *httpgrpc.HTTPRequest, reqParams url.Values, ts time.Time) (*httpgrpc.HTTPResponse, error) { if s := f.State(); s != services.Running { return nil, fmt.Errorf("frontend not running: %v", s) } @@ -196,11 +195,10 @@ func (f *Frontend) RoundTripGRPC(ctx context.Context, requestParams url.Values, return f.retry.Do(ctx, func() (*httpgrpc.HTTPResponse, error) { freq := &frontendRequest{ - queryID: f.lastQueryID.Inc(), - request: req, - userID: userID, - statsEnabled: stats.IsEnabled(ctx), - isHighPriority: util_query.IsHighPriority(requestParams, timestamp, f.limits.HighPriorityQueries(userID)), + queryID: f.lastQueryID.Inc(), + request: req, + userID: userID, + statsEnabled: stats.IsEnabled(ctx), cancel: cancel, @@ -212,6 +210,10 @@ func (f *Frontend) RoundTripGRPC(ctx context.Context, requestParams url.Values, retryOnTooManyOutstandingRequests: f.cfg.RetryOnTooManyOutstandingRequests && f.schedulerWorkers.getWorkersCount() > 1, } + if reqParams != nil { + freq.highPriority = util_query.IsHighPriority(reqParams, ts, f.limits.HighPriorityQueries(userID)) + } + f.requests.put(freq) defer f.requests.delete(freq.queryID) diff --git a/pkg/frontend/v2/frontend_scheduler_worker.go b/pkg/frontend/v2/frontend_scheduler_worker.go index 5e5b22eb13..557457fdc0 100644 --- a/pkg/frontend/v2/frontend_scheduler_worker.go +++ b/pkg/frontend/v2/frontend_scheduler_worker.go @@ -263,7 +263,7 @@ func (w *frontendSchedulerWorker) schedulerLoop(loop schedulerpb.SchedulerForFro HttpRequest: req.request, FrontendAddress: w.frontendAddr, StatsEnabled: req.statsEnabled, - IsHighPriority: req.isHighPriority, + HighPriority: req.highPriority, }) if err != nil { diff --git a/pkg/scheduler/queue/queue_test.go b/pkg/scheduler/queue/queue_test.go index 283eb9df4f..9dd1ee78eb 100644 --- a/pkg/scheduler/queue/queue_test.go +++ b/pkg/scheduler/queue/queue_test.go @@ -174,7 +174,7 @@ func TestRequestQueue_QueriersShouldGetHighPriorityQueryFirst(t *testing.T) { isHighPriority: false, } normalRequest2 := MockRequest{ - id: "normal query 1", + id: "normal query 2", isHighPriority: false, } highPriorityRequest := MockRequest{ diff --git a/pkg/scheduler/queue/user_queues.go b/pkg/scheduler/queue/user_queues.go index 29194b613b..a437afce83 100644 --- a/pkg/scheduler/queue/user_queues.go +++ b/pkg/scheduler/queue/user_queues.go @@ -20,7 +20,7 @@ type Limits interface { // ReservedHighPriorityQueriers returns the minimum number of queriers dedicated for high priority // queue per tenant. All queriers still handle priority queue first, but this provides extra protection on // high priority queries from slow normal queries exhausting all queriers. - // If ReservedHighPriorityQueriers is capped by MaxQueriersPerUser. + // ReservedHighPriorityQueriers is capped by MaxQueriersPerUser. // If less than 1, it will be applied as a percentage of MaxQueriersPerUser. ReservedHighPriorityQueriers(user string) float64 @@ -118,7 +118,7 @@ func (q *queues) deleteQueue(userID string) { // MaxQueriers is used to compute which queriers should handle requests for this user. // If maxQueriers is <= 0, all queriers can handle this user's requests. // If maxQueriers has changed since the last call, queriers for this are recomputed. -func (q *queues) getOrAddQueue(userID string, maxQueriers int, isHighPriority bool) chan Request { +func (q *queues) getOrAddQueue(userID string, maxQueriers int, highPriority bool) chan Request { // Empty user is not allowed, as that would break our users list ("" is used for free spot). if userID == "" { return nil @@ -166,7 +166,7 @@ func (q *queues) getOrAddQueue(userID string, maxQueriers int, isHighPriority bo uq.queriers, uq.reservedQueriers = shuffleQueriersForUser(uq.seed, maxQueriers, q.sortedQueriers, q.limits.ReservedHighPriorityQueriers(userID), nil) } - if isHighPriority { + if highPriority { return uq.highPriorityQueue } diff --git a/pkg/scheduler/scheduler.go b/pkg/scheduler/scheduler.go index 12b5575197..cbd673badf 100644 --- a/pkg/scheduler/scheduler.go +++ b/pkg/scheduler/scheduler.go @@ -154,7 +154,7 @@ type schedulerRequest struct { queryID uint64 request *httpgrpc.HTTPRequest statsEnabled bool - isHighPriority bool + highPriority bool enqueueTime time.Time @@ -167,7 +167,7 @@ type schedulerRequest struct { } func (s schedulerRequest) IsHighPriority() bool { - return s.isHighPriority + return s.highPriority } // FrontendLoop handles connection from frontend. @@ -298,7 +298,7 @@ func (s *Scheduler) enqueueRequest(frontendContext context.Context, frontendAddr queryID: msg.QueryID, request: msg.HttpRequest, statsEnabled: msg.StatsEnabled, - isHighPriority: msg.IsHighPriority, + highPriority: msg.HighPriority, } now := time.Now() diff --git a/pkg/scheduler/schedulerpb/scheduler.pb.go b/pkg/scheduler/schedulerpb/scheduler.pb.go index 0d5c95d68e..5933f21c77 100644 --- a/pkg/scheduler/schedulerpb/scheduler.pb.go +++ b/pkg/scheduler/schedulerpb/scheduler.pb.go @@ -216,10 +216,10 @@ type FrontendToScheduler struct { // Each frontend manages its own queryIDs. Different frontends may use same set of query IDs. QueryID uint64 `protobuf:"varint,3,opt,name=queryID,proto3" json:"queryID,omitempty"` // Following are used by ENQUEUE only. - UserID string `protobuf:"bytes,4,opt,name=userID,proto3" json:"userID,omitempty"` - HttpRequest *httpgrpc.HTTPRequest `protobuf:"bytes,5,opt,name=httpRequest,proto3" json:"httpRequest,omitempty"` - StatsEnabled bool `protobuf:"varint,6,opt,name=statsEnabled,proto3" json:"statsEnabled,omitempty"` - IsHighPriority bool `protobuf:"varint,7,opt,name=isHighPriority,proto3" json:"isHighPriority,omitempty"` + UserID string `protobuf:"bytes,4,opt,name=userID,proto3" json:"userID,omitempty"` + HttpRequest *httpgrpc.HTTPRequest `protobuf:"bytes,5,opt,name=httpRequest,proto3" json:"httpRequest,omitempty"` + StatsEnabled bool `protobuf:"varint,6,opt,name=statsEnabled,proto3" json:"statsEnabled,omitempty"` + HighPriority bool `protobuf:"varint,7,opt,name=highPriority,proto3" json:"highPriority,omitempty"` } func (m *FrontendToScheduler) Reset() { *m = FrontendToScheduler{} } @@ -296,9 +296,9 @@ func (m *FrontendToScheduler) GetStatsEnabled() bool { return false } -func (m *FrontendToScheduler) GetIsHighPriority() bool { +func (m *FrontendToScheduler) GetHighPriority() bool { if m != nil { - return m.IsHighPriority + return m.HighPriority } return false } @@ -446,49 +446,49 @@ func init() { func init() { proto.RegisterFile("scheduler.proto", fileDescriptor_2b3fc28395a6d9c5) } var fileDescriptor_2b3fc28395a6d9c5 = []byte{ - // 666 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x8c, 0x94, 0xcf, 0x4e, 0xdb, 0x40, - 0x10, 0xc6, 0xbd, 0x21, 0x09, 0x30, 0xa1, 0xe0, 0x2e, 0xd0, 0xa6, 0x11, 0x5d, 0x22, 0xab, 0x42, - 0x29, 0x87, 0xa4, 0x4a, 0x2b, 0xb5, 0x07, 0x54, 0x29, 0x05, 0x53, 0xa2, 0x52, 0x07, 0x1c, 0x47, - 0xfd, 0x73, 0x89, 0x48, 0xb2, 0x24, 0x16, 0xe0, 0x35, 0xeb, 0x75, 0x51, 0x6e, 0x7d, 0x84, 0x3e, - 0x46, 0x8f, 0x7d, 0x8c, 0x5e, 0x2a, 0x71, 0xe4, 0xd0, 0x43, 0x71, 0x2e, 0x3d, 0xf2, 0x08, 0x55, - 0x1c, 0x27, 0x75, 0x42, 0x02, 0xdc, 0x66, 0xc7, 0xdf, 0x67, 0xcf, 0xfc, 0x66, 0xd6, 0xb0, 0xe0, - 0xd4, 0x5b, 0xb4, 0xe1, 0x1e, 0x53, 0x9e, 0xb5, 0x39, 0x13, 0x0c, 0x27, 0x06, 0x09, 0xbb, 0x96, - 0x5a, 0x6a, 0xb2, 0x26, 0xf3, 0xf3, 0xb9, 0x6e, 0xd4, 0x93, 0xa4, 0x5e, 0x34, 0x4d, 0xd1, 0x72, - 0x6b, 0xd9, 0x3a, 0x3b, 0xc9, 0x9d, 0xd1, 0x83, 0x2f, 0xf4, 0x8c, 0xf1, 0x23, 0x27, 0x57, 0x67, - 0x27, 0x27, 0xcc, 0xca, 0xb5, 0x84, 0xb0, 0x9b, 0xdc, 0xae, 0x0f, 0x82, 0x9e, 0x4b, 0xc9, 0x03, - 0xde, 0x77, 0x29, 0x37, 0x29, 0x37, 0x58, 0xb9, 0xff, 0x0d, 0xbc, 0x02, 0xb3, 0xa7, 0xbd, 0x6c, - 0x71, 0x2b, 0x89, 0xd2, 0x28, 0x33, 0xab, 0xff, 0x4f, 0x28, 0xbf, 0x10, 0xe0, 0x81, 0xd6, 0x60, - 0x81, 0x1f, 0x27, 0x61, 0xba, 0xab, 0x69, 0x07, 0x96, 0xa8, 0xde, 0x3f, 0xe2, 0x97, 0x90, 0xe8, - 0x7e, 0x56, 0xa7, 0xa7, 0x2e, 0x75, 0x44, 0x32, 0x92, 0x46, 0x99, 0x44, 0x7e, 0x39, 0x3b, 0x28, - 0x65, 0xc7, 0x30, 0xf6, 0x82, 0x87, 0x7a, 0x58, 0x89, 0x33, 0xb0, 0x70, 0xc8, 0x99, 0x25, 0xa8, - 0xd5, 0x28, 0x34, 0x1a, 0x9c, 0x3a, 0x4e, 0x72, 0xca, 0xaf, 0x66, 0x34, 0x8d, 0x1f, 0x40, 0xdc, - 0x75, 0xfc, 0x72, 0xa3, 0xbe, 0x20, 0x38, 0x61, 0x05, 0xe6, 0x1c, 0x71, 0x20, 0x1c, 0xd5, 0x3a, - 0xa8, 0x1d, 0xd3, 0x46, 0x32, 0x96, 0x46, 0x99, 0x19, 0x7d, 0x28, 0xa7, 0xfc, 0x88, 0xc0, 0xe2, - 0x76, 0xf0, 0xbe, 0x30, 0x85, 0x57, 0x10, 0x15, 0x6d, 0x9b, 0xfa, 0xdd, 0xcc, 0xe7, 0x9f, 0x64, - 0x43, 0x33, 0xc8, 0x8e, 0xd1, 0x1b, 0x6d, 0x9b, 0xea, 0xbe, 0x63, 0x5c, 0xdd, 0x91, 0xf1, 0x75, - 0x87, 0xa0, 0x4d, 0x0d, 0x43, 0x9b, 0xd4, 0xd1, 0x08, 0xcc, 0xd8, 0x9d, 0x61, 0x8e, 0xa2, 0x88, - 0x5f, 0x47, 0x81, 0xd7, 0x60, 0xde, 0x74, 0x76, 0xcc, 0x66, 0x6b, 0x8f, 0x9b, 0x8c, 0x9b, 0xa2, - 0x9d, 0x9c, 0xf6, 0x55, 0x23, 0x59, 0xe5, 0x08, 0x16, 0x43, 0x1b, 0xd0, 0x87, 0x81, 0x5f, 0x43, - 0xbc, 0xfb, 0x3a, 0xd7, 0x09, 0x98, 0xad, 0x0d, 0x31, 0x1b, 0xe3, 0x28, 0xfb, 0x6a, 0x3d, 0x70, - 0xe1, 0x25, 0x88, 0x51, 0xce, 0x19, 0x0f, 0x68, 0xf5, 0x0e, 0xca, 0x06, 0xac, 0x68, 0x4c, 0x98, - 0x87, 0xed, 0x60, 0xd3, 0xca, 0x2d, 0x57, 0x34, 0xd8, 0x99, 0xd5, 0x6f, 0xec, 0xe6, 0x6d, 0x5d, - 0x85, 0xc7, 0x13, 0xdc, 0x8e, 0xcd, 0x2c, 0x87, 0xae, 0x6f, 0xc0, 0xc3, 0x09, 0xd3, 0xc4, 0x33, - 0x10, 0x2d, 0x6a, 0x45, 0x43, 0x96, 0x70, 0x02, 0xa6, 0x55, 0x6d, 0xbf, 0xa2, 0x56, 0x54, 0x19, - 0x61, 0x80, 0xf8, 0x66, 0x41, 0xdb, 0x54, 0x77, 0xe5, 0xc8, 0x7a, 0x1d, 0x1e, 0x4d, 0xec, 0x0b, - 0xc7, 0x21, 0x52, 0x7a, 0x27, 0x4b, 0x38, 0x0d, 0x2b, 0x46, 0xa9, 0x54, 0x7d, 0x5f, 0xd0, 0x3e, - 0x55, 0x75, 0x75, 0xbf, 0xa2, 0x96, 0x8d, 0x72, 0x75, 0x4f, 0xd5, 0xab, 0x86, 0xaa, 0x15, 0x34, - 0x43, 0x46, 0x78, 0x16, 0x62, 0xaa, 0xae, 0x97, 0x74, 0x39, 0x82, 0xef, 0xc3, 0xbd, 0xf2, 0x4e, - 0xc5, 0x30, 0x8a, 0xda, 0xdb, 0xea, 0x56, 0xe9, 0x83, 0x26, 0x4f, 0xe5, 0x7f, 0xa3, 0x10, 0xef, - 0x6d, 0xc6, 0xfb, 0x57, 0xae, 0x02, 0x89, 0x20, 0xdc, 0x65, 0xcc, 0xc6, 0xab, 0x43, 0xb8, 0xaf, - 0xdf, 0xeb, 0xd4, 0xea, 0xa4, 0x79, 0x04, 0x5a, 0x45, 0xca, 0xa0, 0x67, 0x08, 0x5b, 0xb0, 0x3c, - 0x16, 0x19, 0x7e, 0x3a, 0xe4, 0xbf, 0x69, 0x28, 0xa9, 0xf5, 0xbb, 0x48, 0x7b, 0x13, 0xc8, 0xdb, - 0xb0, 0x14, 0xee, 0x6e, 0xb0, 0x4e, 0x1f, 0x61, 0xae, 0x1f, 0xfb, 0xfd, 0xa5, 0x6f, 0xbb, 0x82, - 0xa9, 0xf4, 0x6d, 0x0b, 0xd7, 0xeb, 0xf0, 0x4d, 0xe1, 0xfc, 0x92, 0x48, 0x17, 0x97, 0x44, 0xba, - 0xba, 0x24, 0xe8, 0xab, 0x47, 0xd0, 0x77, 0x8f, 0xa0, 0x9f, 0x1e, 0x41, 0xe7, 0x1e, 0x41, 0x7f, - 0x3c, 0x82, 0xfe, 0x7a, 0x44, 0xba, 0xf2, 0x08, 0xfa, 0xd6, 0x21, 0xd2, 0x79, 0x87, 0x48, 0x17, - 0x1d, 0x22, 0x7d, 0x0e, 0xff, 0x85, 0x6b, 0x71, 0xff, 0x07, 0xfa, 0xfc, 0x5f, 0x00, 0x00, 0x00, - 0xff, 0xff, 0x13, 0x54, 0x85, 0x09, 0xac, 0x05, 0x00, 0x00, + // 662 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x8c, 0x94, 0xcd, 0x4e, 0xdb, 0x40, + 0x10, 0xc7, 0xbd, 0x21, 0x09, 0x30, 0xa1, 0xc5, 0x5d, 0xa0, 0x4d, 0x23, 0xba, 0x44, 0x56, 0x55, + 0xa5, 0x1c, 0x92, 0x2a, 0xad, 0xd4, 0x1e, 0x50, 0xa5, 0x14, 0x4c, 0x89, 0x4a, 0x1d, 0x70, 0x1c, + 0xf5, 0xe3, 0x12, 0x91, 0x64, 0x49, 0x22, 0xc0, 0x6b, 0xd6, 0xeb, 0xa2, 0xdc, 0xfa, 0x08, 0x7d, + 0x8c, 0x1e, 0xfa, 0x20, 0xbd, 0x54, 0xe2, 0xc8, 0xa1, 0x87, 0x62, 0x2e, 0x3d, 0xf2, 0x08, 0x55, + 0xfc, 0x91, 0x3a, 0x21, 0x01, 0x6e, 0xb3, 0xe3, 0xff, 0xdf, 0x9e, 0xf9, 0xcd, 0xac, 0x61, 0xde, + 0x6e, 0x76, 0x68, 0xcb, 0x39, 0xa4, 0x3c, 0x6f, 0x71, 0x26, 0x18, 0x4e, 0x0d, 0x12, 0x56, 0x23, + 0xb3, 0xd8, 0x66, 0x6d, 0xe6, 0xe5, 0x0b, 0xfd, 0xc8, 0x97, 0x64, 0x5e, 0xb4, 0xbb, 0xa2, 0xe3, + 0x34, 0xf2, 0x4d, 0x76, 0x54, 0x38, 0xa1, 0x7b, 0x5f, 0xe8, 0x09, 0xe3, 0x07, 0x76, 0xa1, 0xc9, + 0x8e, 0x8e, 0x98, 0x59, 0xe8, 0x08, 0x61, 0xb5, 0xb9, 0xd5, 0x1c, 0x04, 0xbe, 0x4b, 0x29, 0x02, + 0xde, 0x75, 0x28, 0xef, 0x52, 0x6e, 0xb0, 0x6a, 0xf8, 0x0d, 0xbc, 0x0c, 0xb3, 0xc7, 0x7e, 0xb6, + 0xbc, 0x91, 0x46, 0x59, 0x94, 0x9b, 0xd5, 0xff, 0x27, 0x94, 0x5f, 0x08, 0xf0, 0x40, 0x6b, 0xb0, + 0xc0, 0x8f, 0xd3, 0x30, 0xdd, 0xd7, 0xf4, 0x02, 0x4b, 0x5c, 0x0f, 0x8f, 0xf8, 0x25, 0xa4, 0xfa, + 0x9f, 0xd5, 0xe9, 0xb1, 0x43, 0x6d, 0x91, 0x8e, 0x65, 0x51, 0x2e, 0x55, 0x5c, 0xca, 0x0f, 0x4a, + 0xd9, 0x32, 0x8c, 0x9d, 0xe0, 0xa1, 0x1e, 0x55, 0xe2, 0x1c, 0xcc, 0xef, 0x73, 0x66, 0x0a, 0x6a, + 0xb6, 0x4a, 0xad, 0x16, 0xa7, 0xb6, 0x9d, 0x9e, 0xf2, 0xaa, 0x19, 0x4d, 0xe3, 0xfb, 0x90, 0x74, + 0x6c, 0xaf, 0xdc, 0xb8, 0x27, 0x08, 0x4e, 0x58, 0x81, 0x39, 0x5b, 0xec, 0x09, 0x5b, 0x35, 0xf7, + 0x1a, 0x87, 0xb4, 0x95, 0x4e, 0x64, 0x51, 0x6e, 0x46, 0x1f, 0xca, 0x29, 0x3f, 0x62, 0xb0, 0xb0, + 0x19, 0xbc, 0x2f, 0x4a, 0xe1, 0x15, 0xc4, 0x45, 0xcf, 0xa2, 0x5e, 0x37, 0x77, 0x8b, 0x8f, 0xf3, + 0x91, 0x19, 0xe4, 0xc7, 0xe8, 0x8d, 0x9e, 0x45, 0x75, 0xcf, 0x31, 0xae, 0xee, 0xd8, 0xf8, 0xba, + 0x23, 0xd0, 0xa6, 0x86, 0xa1, 0x4d, 0xea, 0x68, 0x04, 0x66, 0xe2, 0xd6, 0x30, 0x47, 0x51, 0x24, + 0xaf, 0xa2, 0xe8, 0x6b, 0x3a, 0xdd, 0x76, 0x67, 0x87, 0x77, 0x19, 0xef, 0x8a, 0x5e, 0x7a, 0xda, + 0xd7, 0x44, 0x73, 0xca, 0x01, 0x2c, 0x44, 0xa6, 0x1f, 0x82, 0xc0, 0xaf, 0x21, 0xd9, 0x7f, 0x95, + 0x63, 0x07, 0xbc, 0x9e, 0x0c, 0xf1, 0x1a, 0xe3, 0xa8, 0x7a, 0x6a, 0x3d, 0x70, 0xe1, 0x45, 0x48, + 0x50, 0xce, 0x19, 0x0f, 0x48, 0xf9, 0x07, 0x65, 0x0d, 0x96, 0x35, 0x26, 0xba, 0xfb, 0xbd, 0x60, + 0xcb, 0xaa, 0x1d, 0x47, 0xb4, 0xd8, 0x89, 0x19, 0x36, 0x75, 0xfd, 0xa6, 0xae, 0xc0, 0xa3, 0x09, + 0x6e, 0xdb, 0x62, 0xa6, 0x4d, 0x57, 0xd7, 0xe0, 0xc1, 0x84, 0x49, 0xe2, 0x19, 0x88, 0x97, 0xb5, + 0xb2, 0x21, 0x4b, 0x38, 0x05, 0xd3, 0xaa, 0xb6, 0x5b, 0x53, 0x6b, 0xaa, 0x8c, 0x30, 0x40, 0x72, + 0xbd, 0xa4, 0xad, 0xab, 0xdb, 0x72, 0x6c, 0xb5, 0x09, 0x0f, 0x27, 0xf6, 0x85, 0x93, 0x10, 0xab, + 0xbc, 0x93, 0x25, 0x9c, 0x85, 0x65, 0xa3, 0x52, 0xa9, 0xbf, 0x2f, 0x69, 0x9f, 0xea, 0xba, 0xba, + 0x5b, 0x53, 0xab, 0x46, 0xb5, 0xbe, 0xa3, 0xea, 0x75, 0x43, 0xd5, 0x4a, 0x9a, 0x21, 0x23, 0x3c, + 0x0b, 0x09, 0x55, 0xd7, 0x2b, 0xba, 0x1c, 0xc3, 0xf7, 0xe0, 0x4e, 0x75, 0xab, 0x66, 0x18, 0x65, + 0xed, 0x6d, 0x7d, 0xa3, 0xf2, 0x41, 0x93, 0xa7, 0x8a, 0xbf, 0x51, 0x84, 0xf7, 0x26, 0xe3, 0xe1, + 0x75, 0xab, 0x41, 0x2a, 0x08, 0xb7, 0x19, 0xb3, 0xf0, 0xca, 0x10, 0xee, 0xab, 0x77, 0x3a, 0xb3, + 0x32, 0x69, 0x1e, 0x81, 0x56, 0x91, 0x72, 0xe8, 0x19, 0xc2, 0x26, 0x2c, 0x8d, 0x45, 0x86, 0x9f, + 0x0e, 0xf9, 0xaf, 0x1b, 0x4a, 0x66, 0xf5, 0x36, 0x52, 0x7f, 0x02, 0x45, 0x0b, 0x16, 0xa3, 0xdd, + 0x0d, 0xd6, 0xe9, 0x23, 0xcc, 0x85, 0xb1, 0xd7, 0x5f, 0xf6, 0xa6, 0xeb, 0x97, 0xc9, 0xde, 0xb4, + 0x70, 0x7e, 0x87, 0x6f, 0x4a, 0xa7, 0xe7, 0x44, 0x3a, 0x3b, 0x27, 0xd2, 0xe5, 0x39, 0x41, 0x5f, + 0x5d, 0x82, 0xbe, 0xbb, 0x04, 0xfd, 0x74, 0x09, 0x3a, 0x75, 0x09, 0xfa, 0xe3, 0x12, 0xf4, 0xd7, + 0x25, 0xd2, 0xa5, 0x4b, 0xd0, 0xb7, 0x0b, 0x22, 0x9d, 0x5e, 0x10, 0xe9, 0xec, 0x82, 0x48, 0x9f, + 0xa3, 0x7f, 0xe0, 0x46, 0xd2, 0xfb, 0x79, 0x3e, 0xff, 0x17, 0x00, 0x00, 0xff, 0xff, 0x38, 0x3c, + 0xc5, 0x8f, 0xa8, 0x05, 0x00, 0x00, } func (x FrontendToSchedulerType) String() string { @@ -602,7 +602,7 @@ func (this *FrontendToScheduler) Equal(that interface{}) bool { if this.StatsEnabled != that1.StatsEnabled { return false } - if this.IsHighPriority != that1.IsHighPriority { + if this.HighPriority != that1.HighPriority { return false } return true @@ -719,7 +719,7 @@ func (this *FrontendToScheduler) GoString() string { s = append(s, "HttpRequest: "+fmt.Sprintf("%#v", this.HttpRequest)+",\n") } s = append(s, "StatsEnabled: "+fmt.Sprintf("%#v", this.StatsEnabled)+",\n") - s = append(s, "IsHighPriority: "+fmt.Sprintf("%#v", this.IsHighPriority)+",\n") + s = append(s, "HighPriority: "+fmt.Sprintf("%#v", this.HighPriority)+",\n") s = append(s, "}") return strings.Join(s, "") } @@ -1155,9 +1155,9 @@ func (m *FrontendToScheduler) MarshalToSizedBuffer(dAtA []byte) (int, error) { _ = i var l int _ = l - if m.IsHighPriority { + if m.HighPriority { i-- - if m.IsHighPriority { + if m.HighPriority { dAtA[i] = 1 } else { dAtA[i] = 0 @@ -1380,7 +1380,7 @@ func (m *FrontendToScheduler) Size() (n int) { if m.StatsEnabled { n += 2 } - if m.IsHighPriority { + if m.HighPriority { n += 2 } return n @@ -1465,7 +1465,7 @@ func (this *FrontendToScheduler) String() string { `UserID:` + fmt.Sprintf("%v", this.UserID) + `,`, `HttpRequest:` + strings.Replace(fmt.Sprintf("%v", this.HttpRequest), "HTTPRequest", "httpgrpc.HTTPRequest", 1) + `,`, `StatsEnabled:` + fmt.Sprintf("%v", this.StatsEnabled) + `,`, - `IsHighPriority:` + fmt.Sprintf("%v", this.IsHighPriority) + `,`, + `HighPriority:` + fmt.Sprintf("%v", this.HighPriority) + `,`, `}`, }, "") return s @@ -1974,7 +1974,7 @@ func (m *FrontendToScheduler) Unmarshal(dAtA []byte) error { m.StatsEnabled = bool(v != 0) case 7: if wireType != 0 { - return fmt.Errorf("proto: wrong wireType = %d for field IsHighPriority", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field HighPriority", wireType) } var v int for shift := uint(0); ; shift += 7 { @@ -1991,7 +1991,7 @@ func (m *FrontendToScheduler) Unmarshal(dAtA []byte) error { break } } - m.IsHighPriority = bool(v != 0) + m.HighPriority = bool(v != 0) default: iNdEx = preIndex skippy, err := skipScheduler(dAtA[iNdEx:]) diff --git a/pkg/scheduler/schedulerpb/scheduler.proto b/pkg/scheduler/schedulerpb/scheduler.proto index 5e71946008..1d246f73a2 100644 --- a/pkg/scheduler/schedulerpb/scheduler.proto +++ b/pkg/scheduler/schedulerpb/scheduler.proto @@ -78,7 +78,7 @@ message FrontendToScheduler { string userID = 4; httpgrpc.HTTPRequest httpRequest = 5; bool statsEnabled = 6; - bool isHighPriority = 7; + bool highPriority = 7; } enum SchedulerToFrontendStatus { diff --git a/pkg/util/query/priority_test.go b/pkg/util/query/priority_test.go index dd1ac9c78b..5085eebf60 100644 --- a/pkg/util/query/priority_test.go +++ b/pkg/util/query/priority_test.go @@ -11,7 +11,7 @@ import ( "github.com/cortexproject/cortex/pkg/util/validation" ) -func Test_IsHighPriority_ShouldMatchRegex(t *testing.T) { +func Test_IsHighPriorityShouldMatchRegex(t *testing.T) { now := time.Now() config := []validation.HighPriorityQuery{ { @@ -110,7 +110,7 @@ func Test_IsHighPriority_ShouldMatchRegex(t *testing.T) { }, now, config)) } -func Test_IsHighPriority_ShouldBeBetweenStartAndEndTime(t *testing.T) { +func Test_IsHighPriorityShouldBeBetweenStartAndEndTime(t *testing.T) { now := time.Now() config := []validation.HighPriorityQuery{ { diff --git a/pkg/util/validation/limits.go b/pkg/util/validation/limits.go index 301e945ea8..94907a1905 100644 --- a/pkg/util/validation/limits.go +++ b/pkg/util/validation/limits.go @@ -6,6 +6,7 @@ import ( "errors" "flag" "math" + "regexp" "strings" "time" @@ -47,9 +48,10 @@ type DisabledRuleGroup struct { type DisabledRuleGroups []DisabledRuleGroup type HighPriorityQuery struct { - Regex string `yaml:"regex" doc:"nocli|description=Query string regex. If evaluated true (on top of meeting all other criteria), query is treated as a high priority."` - StartTime time.Duration `yaml:"start_time" doc:"nocli|description=If query range falls between the start_time and end_time (on top of meeting all other criteria), query is treated as a high priority.|default=0s"` - EndTime time.Duration `yaml:"end_time" doc:"nocli|description=If query range falls between the start_time and end_time (on top of meeting all other criteria), query is treated as a high priority.|default=0s"` + Regex string `yaml:"regex" doc:"nocli|description=Query string regex. If evaluated true (on top of meeting all other criteria), query is treated as a high priority."` + CompiledRegex *regexp.Regexp `yaml:"-" doc:"nocli"` + StartTime time.Duration `yaml:"start_time" doc:"nocli|description=If query range falls between the start_time and end_time (on top of meeting all other criteria), query is treated as a high priority.|default=0s"` + EndTime time.Duration `yaml:"end_time" doc:"nocli|description=If query range falls between the start_time and end_time (on top of meeting all other criteria), query is treated as a high priority.|default=0s"` } // Limits describe all the limits for users; can be used to describe global default From 393d9f00a99405ede730f942945b402a291c43f5 Mon Sep 17 00:00:00 2001 From: Justin Jung Date: Mon, 30 Oct 2023 22:19:06 -0700 Subject: [PATCH 16/49] Add CompiledRegex to HighPriorityQuery Signed-off-by: Justin Jung --- pkg/util/query/priority.go | 5 ++--- pkg/util/validation/limits.go | 7 +++++++ 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/pkg/util/query/priority.go b/pkg/util/query/priority.go index 84c7dfa262..ea345074df 100644 --- a/pkg/util/query/priority.go +++ b/pkg/util/query/priority.go @@ -3,7 +3,6 @@ package query import ( "math" "net/url" - "regexp" "strconv" "time" @@ -23,9 +22,9 @@ func IsHighPriority(requestParams url.Values, now time.Time, highPriorityQueries } for _, highPriorityQuery := range highPriorityQueries { - regex := highPriorityQuery.Regex + compiledRegex := highPriorityQuery.CompiledRegex - if match, err := regexp.MatchString(regex, queryParam); !match || err != nil { + if compiledRegex == nil || !compiledRegex.MatchString(queryParam) { continue } diff --git a/pkg/util/validation/limits.go b/pkg/util/validation/limits.go index 94907a1905..45f3cceb8e 100644 --- a/pkg/util/validation/limits.go +++ b/pkg/util/validation/limits.go @@ -510,6 +510,13 @@ func (o *Overrides) ReservedHighPriorityQueriers(userID string) float64 { // HighPriorityQueries returns list of definitions for high priority query. func (o *Overrides) HighPriorityQueries(userID string) []HighPriorityQuery { + highPriorityQueries := o.GetOverridesForUser(userID).HighPriorityQueries + for index, query := range highPriorityQueries { + if query.CompiledRegex == nil { + // no need to handle error, as we will use the CompiledRegex only if it's not nil + o.GetOverridesForUser(userID).HighPriorityQueries[index].CompiledRegex, _ = regexp.Compile(query.Regex) + } + } return o.GetOverridesForUser(userID).HighPriorityQueries } From ebc2c1e9acade23eb0dc0bdbcb2254e56c6083b7 Mon Sep 17 00:00:00 2001 From: Justin Jung Date: Wed, 1 Nov 2023 08:40:24 -0700 Subject: [PATCH 17/49] Introduce numbered priority + change config structure Signed-off-by: Justin Jung --- pkg/frontend/v1/frontend.go | 22 ++-- pkg/frontend/v2/frontend.go | 4 +- pkg/frontend/v2/frontend_scheduler_worker.go | 2 +- pkg/scheduler/queue/queue.go | 4 +- pkg/scheduler/queue/user_queues.go | 67 +++++------ pkg/scheduler/scheduler.go | 8 +- pkg/scheduler/schedulerpb/scheduler.pb.go | 120 +++++++++---------- pkg/scheduler/schedulerpb/scheduler.proto | 2 +- pkg/util/query/priority.go | 40 ++++--- pkg/util/validation/limits.go | 45 ++++--- 10 files changed, 154 insertions(+), 160 deletions(-) diff --git a/pkg/frontend/v1/frontend.go b/pkg/frontend/v1/frontend.go index 8f2fae830f..5b874fa865 100644 --- a/pkg/frontend/v1/frontend.go +++ b/pkg/frontend/v1/frontend.go @@ -53,8 +53,8 @@ type Limits interface { // MockLimits implements the Limits interface. Used in tests only. type MockLimits struct { - Queriers float64 - Queries []validation.HighPriorityQuery + Queriers float64 + QueryPriority validation.QueryPriority queue.MockLimits } @@ -62,8 +62,8 @@ func (l MockLimits) MaxQueriersPerUser(_ string) float64 { return l.Queriers } -func (l MockLimits) HighPriorityQueries(_ string) []validation.HighPriorityQuery { - return l.Queries +func (l MockLimits) HighPriorityQueries(_ string) validation.QueryPriority { + return l.QueryPriority } // Frontend queues HTTP requests, dispatches them to backends, and handles retries @@ -95,14 +95,14 @@ type request struct { queueSpan opentracing.Span originalCtx context.Context - request *httpgrpc.HTTPRequest - err chan error - response chan *httpgrpc.HTTPResponse - highPriority bool + request *httpgrpc.HTTPRequest + err chan error + response chan *httpgrpc.HTTPResponse + priority int64 } -func (r request) IsHighPriority() bool { - return r.highPriority +func (r request) GetPriority() int64 { + return r.priority } // New creates a new frontend. Frontend implements service, and must be started and stopped. @@ -207,7 +207,7 @@ func (f *Frontend) RoundTripGRPC(ctx context.Context, req *httpgrpc.HTTPRequest, } if reqParams != nil { - request.highPriority = util_query.IsHighPriority(reqParams, ts, f.limits.HighPriorityQueries(userID)) + request.priority = util_query.GetPriority(reqParams, ts, f.limits.QueryPriority(userID)) } if err := f.queueRequest(ctx, &request); err != nil { diff --git a/pkg/frontend/v2/frontend.go b/pkg/frontend/v2/frontend.go index 3cb00bdbca..81d0e8c802 100644 --- a/pkg/frontend/v2/frontend.go +++ b/pkg/frontend/v2/frontend.go @@ -89,7 +89,7 @@ type frontendRequest struct { request *httpgrpc.HTTPRequest userID string statsEnabled bool - highPriority bool + priority int64 cancel context.CancelFunc @@ -211,7 +211,7 @@ func (f *Frontend) RoundTripGRPC(ctx context.Context, req *httpgrpc.HTTPRequest, } if reqParams != nil { - freq.highPriority = util_query.IsHighPriority(reqParams, ts, f.limits.HighPriorityQueries(userID)) + freq.priority = util_query.GetPriority(reqParams, ts, f.limits.QueryPriority(userID)) } f.requests.put(freq) diff --git a/pkg/frontend/v2/frontend_scheduler_worker.go b/pkg/frontend/v2/frontend_scheduler_worker.go index 557457fdc0..2abff24bf6 100644 --- a/pkg/frontend/v2/frontend_scheduler_worker.go +++ b/pkg/frontend/v2/frontend_scheduler_worker.go @@ -263,7 +263,7 @@ func (w *frontendSchedulerWorker) schedulerLoop(loop schedulerpb.SchedulerForFro HttpRequest: req.request, FrontendAddress: w.frontendAddr, StatsEnabled: req.statsEnabled, - HighPriority: req.highPriority, + Priority: req.priority, }) if err != nil { diff --git a/pkg/scheduler/queue/queue.go b/pkg/scheduler/queue/queue.go index 0931b25f3b..cf20a3573a 100644 --- a/pkg/scheduler/queue/queue.go +++ b/pkg/scheduler/queue/queue.go @@ -45,7 +45,7 @@ func FirstUser() UserIndex { // Request stored into the queue. type Request interface { - IsHighPriority() bool + GetPriority() int64 } // RequestQueue holds incoming requests in per-user queues. It also assigns each user specified number of queriers, @@ -98,7 +98,7 @@ func (q *RequestQueue) EnqueueRequest(userID string, req Request, maxQueriers fl } shardSize := util.DynamicShardSize(maxQueriers, len(q.queues.queriers)) - queue := q.queues.getOrAddQueue(userID, shardSize, req.IsHighPriority()) + queue := q.queues.getOrAddQueue(userID, shardSize, req.GetPriority()) maxOutstandingRequests := q.queues.limits.MaxOutstandingPerTenant(userID) if queue == nil { diff --git a/pkg/scheduler/queue/user_queues.go b/pkg/scheduler/queue/user_queues.go index a437afce83..dafe0b6fd0 100644 --- a/pkg/scheduler/queue/user_queues.go +++ b/pkg/scheduler/queue/user_queues.go @@ -17,15 +17,9 @@ type Limits interface { // of outstanding requests per tenant per request queue. MaxOutstandingPerTenant(user string) int - // ReservedHighPriorityQueriers returns the minimum number of queriers dedicated for high priority - // queue per tenant. All queriers still handle priority queue first, but this provides extra protection on - // high priority queries from slow normal queries exhausting all queriers. - // ReservedHighPriorityQueriers is capped by MaxQueriersPerUser. - // If less than 1, it will be applied as a percentage of MaxQueriersPerUser. - ReservedHighPriorityQueriers(user string) float64 - - // HighPriorityQueries returns list of definitions for high priority query. - HighPriorityQueries(user string) []validation.HighPriorityQuery + // QueryPriority returns query priority config for the tenant, including different priorities, + // their attributes, and how many reserved queriers each priority has. + QueryPriority(user string) validation.QueryPriority } // querier holds information about a querier registered in the queue. @@ -118,7 +112,7 @@ func (q *queues) deleteQueue(userID string) { // MaxQueriers is used to compute which queriers should handle requests for this user. // If maxQueriers is <= 0, all queriers can handle this user's requests. // If maxQueriers has changed since the last call, queriers for this are recomputed. -func (q *queues) getOrAddQueue(userID string, maxQueriers int, highPriority bool) chan Request { +func (q *queues) getOrAddQueue(userID string, maxQueriers int, priority int64) chan Request { // Empty user is not allowed, as that would break our users list ("" is used for free spot). if userID == "" { return nil @@ -163,12 +157,12 @@ func (q *queues) getOrAddQueue(userID string, maxQueriers int, highPriority bool if uq.maxQueriers != maxQueriers { uq.maxQueriers = maxQueriers - uq.queriers, uq.reservedQueriers = shuffleQueriersForUser(uq.seed, maxQueriers, q.sortedQueriers, q.limits.ReservedHighPriorityQueriers(userID), nil) + uq.queriers = shuffleQueriersForUser(uq.seed, maxQueriers, q.sortedQueriers, nil) } - if highPriority { - return uq.highPriorityQueue - } + //if highPriority { + // return uq.highPriorityQueue + //} return uq.normalQueue } @@ -319,24 +313,24 @@ func (q *queues) recomputeUserQueriers() { scratchpad := make([]string, 0, len(q.sortedQueriers)) for _, uq := range q.userQueues { - userID := q.users[uq.index] - uq.queriers, uq.reservedQueriers = shuffleQueriersForUser(uq.seed, uq.maxQueriers, q.sortedQueriers, q.limits.ReservedHighPriorityQueriers(userID), scratchpad) + //userID := q.users[uq.index] + uq.queriers = shuffleQueriersForUser(uq.seed, uq.maxQueriers, q.sortedQueriers, scratchpad) } } // shuffleQueriersForUser returns nil if queriersToSelect is 0 or there are not enough queriers to select from. // In that case *all* queriers should be used. // Scratchpad is used for shuffling, to avoid new allocations. If nil, new slice is allocated. -func shuffleQueriersForUser(userSeed int64, queriersToSelect int, allSortedQueriers []string, numOfReservedQueriersFloat float64, scratchpad []string) (map[string]struct{}, map[string]struct{}) { - numOfReservedQueriers := getNumOfReservedQueriers(queriersToSelect, len(allSortedQueriers), numOfReservedQueriersFloat) - reservedQueriers := make(map[string]struct{}, numOfReservedQueriers) +func shuffleQueriersForUser(userSeed int64, queriersToSelect int, allSortedQueriers []string, scratchpad []string) map[string]struct{} { + //numOfReservedQueriers := getNumOfReservedQueriers(queriersToSelect, len(allSortedQueriers), numOfReservedQueriersFloat) + //reservedQueriers := make(map[string]struct{}, numOfReservedQueriers) - if queriersToSelect == 0 || len(allSortedQueriers) <= queriersToSelect { - for i := 0; i < numOfReservedQueriers; i++ { - reservedQueriers[allSortedQueriers[i]] = struct{}{} - } - return nil, reservedQueriers - } + //if queriersToSelect == 0 || len(allSortedQueriers) <= queriersToSelect { + // for i := 0; i < numOfReservedQueriers; i++ { + // reservedQueriers[allSortedQueriers[i]] = struct{}{} + // } + // return nil, reservedQueriers + //} queriers := make(map[string]struct{}, queriersToSelect) rnd := rand.New(rand.NewSource(userSeed)) @@ -348,15 +342,15 @@ func shuffleQueriersForUser(userSeed int64, queriersToSelect int, allSortedQueri for i := 0; i < queriersToSelect; i++ { r := rnd.Intn(last + 1) queriers[scratchpad[r]] = struct{}{} - if i < numOfReservedQueriers { - reservedQueriers[scratchpad[r]] = struct{}{} - } + //if i < numOfReservedQueriers { + // reservedQueriers[scratchpad[r]] = struct{}{} + //} // move selected item to the end, it won't be selected anymore. scratchpad[r], scratchpad[last] = scratchpad[last], scratchpad[r] last-- } - return queriers, reservedQueriers + return queriers } func getNumOfReservedQueriers(queriersToSelect int, totalNumOfQueriers int, reservedQueriers float64) int { @@ -379,10 +373,9 @@ func getNumOfReservedQueriers(queriersToSelect int, totalNumOfQueriers int, rese // MockLimits implements the Limits interface. Used in tests only. type MockLimits struct { - MaxOutstanding int - maxQueriersPerUser float64 - reservedHighPriorityQueriers float64 - highPriorityQueries []validation.HighPriorityQuery + MaxOutstanding int + maxQueriersPerUser float64 + queryPriority validation.QueryPriority } func (l MockLimits) MaxQueriersPerUser(user string) float64 { @@ -393,10 +386,6 @@ func (l MockLimits) MaxOutstandingPerTenant(_ string) int { return l.MaxOutstanding } -func (l MockLimits) ReservedHighPriorityQueriers(_ string) float64 { - return l.reservedHighPriorityQueriers -} - -func (l MockLimits) HighPriorityQueries(_ string) []validation.HighPriorityQuery { - return l.highPriorityQueries +func (l MockLimits) QueryPriority(_ string) validation.QueryPriority { + return l.queryPriority } diff --git a/pkg/scheduler/scheduler.go b/pkg/scheduler/scheduler.go index cbd673badf..d4f7ac8541 100644 --- a/pkg/scheduler/scheduler.go +++ b/pkg/scheduler/scheduler.go @@ -154,7 +154,7 @@ type schedulerRequest struct { queryID uint64 request *httpgrpc.HTTPRequest statsEnabled bool - highPriority bool + priority int64 enqueueTime time.Time @@ -166,8 +166,8 @@ type schedulerRequest struct { parentSpanContext opentracing.SpanContext } -func (s schedulerRequest) IsHighPriority() bool { - return s.highPriority +func (s schedulerRequest) GetPriority() int64 { + return s.priority } // FrontendLoop handles connection from frontend. @@ -298,7 +298,7 @@ func (s *Scheduler) enqueueRequest(frontendContext context.Context, frontendAddr queryID: msg.QueryID, request: msg.HttpRequest, statsEnabled: msg.StatsEnabled, - highPriority: msg.HighPriority, + priority: msg.Priority, } now := time.Now() diff --git a/pkg/scheduler/schedulerpb/scheduler.pb.go b/pkg/scheduler/schedulerpb/scheduler.pb.go index 5933f21c77..2f352a8138 100644 --- a/pkg/scheduler/schedulerpb/scheduler.pb.go +++ b/pkg/scheduler/schedulerpb/scheduler.pb.go @@ -219,7 +219,7 @@ type FrontendToScheduler struct { UserID string `protobuf:"bytes,4,opt,name=userID,proto3" json:"userID,omitempty"` HttpRequest *httpgrpc.HTTPRequest `protobuf:"bytes,5,opt,name=httpRequest,proto3" json:"httpRequest,omitempty"` StatsEnabled bool `protobuf:"varint,6,opt,name=statsEnabled,proto3" json:"statsEnabled,omitempty"` - HighPriority bool `protobuf:"varint,7,opt,name=highPriority,proto3" json:"highPriority,omitempty"` + Priority int64 `protobuf:"varint,7,opt,name=priority,proto3" json:"priority,omitempty"` } func (m *FrontendToScheduler) Reset() { *m = FrontendToScheduler{} } @@ -296,11 +296,11 @@ func (m *FrontendToScheduler) GetStatsEnabled() bool { return false } -func (m *FrontendToScheduler) GetHighPriority() bool { +func (m *FrontendToScheduler) GetPriority() int64 { if m != nil { - return m.HighPriority + return m.Priority } - return false + return 0 } type SchedulerToFrontend struct { @@ -446,49 +446,49 @@ func init() { func init() { proto.RegisterFile("scheduler.proto", fileDescriptor_2b3fc28395a6d9c5) } var fileDescriptor_2b3fc28395a6d9c5 = []byte{ - // 662 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x8c, 0x94, 0xcd, 0x4e, 0xdb, 0x40, - 0x10, 0xc7, 0xbd, 0x21, 0x09, 0x30, 0xa1, 0xc5, 0x5d, 0xa0, 0x4d, 0x23, 0xba, 0x44, 0x56, 0x55, - 0xa5, 0x1c, 0x92, 0x2a, 0xad, 0xd4, 0x1e, 0x50, 0xa5, 0x14, 0x4c, 0x89, 0x4a, 0x1d, 0x70, 0x1c, - 0xf5, 0xe3, 0x12, 0x91, 0x64, 0x49, 0x22, 0xc0, 0x6b, 0xd6, 0xeb, 0xa2, 0xdc, 0xfa, 0x08, 0x7d, - 0x8c, 0x1e, 0xfa, 0x20, 0xbd, 0x54, 0xe2, 0xc8, 0xa1, 0x87, 0x62, 0x2e, 0x3d, 0xf2, 0x08, 0x55, - 0xfc, 0x91, 0x3a, 0x21, 0x01, 0x6e, 0xb3, 0xe3, 0xff, 0xdf, 0x9e, 0xf9, 0xcd, 0xac, 0x61, 0xde, - 0x6e, 0x76, 0x68, 0xcb, 0x39, 0xa4, 0x3c, 0x6f, 0x71, 0x26, 0x18, 0x4e, 0x0d, 0x12, 0x56, 0x23, - 0xb3, 0xd8, 0x66, 0x6d, 0xe6, 0xe5, 0x0b, 0xfd, 0xc8, 0x97, 0x64, 0x5e, 0xb4, 0xbb, 0xa2, 0xe3, - 0x34, 0xf2, 0x4d, 0x76, 0x54, 0x38, 0xa1, 0x7b, 0x5f, 0xe8, 0x09, 0xe3, 0x07, 0x76, 0xa1, 0xc9, - 0x8e, 0x8e, 0x98, 0x59, 0xe8, 0x08, 0x61, 0xb5, 0xb9, 0xd5, 0x1c, 0x04, 0xbe, 0x4b, 0x29, 0x02, - 0xde, 0x75, 0x28, 0xef, 0x52, 0x6e, 0xb0, 0x6a, 0xf8, 0x0d, 0xbc, 0x0c, 0xb3, 0xc7, 0x7e, 0xb6, - 0xbc, 0x91, 0x46, 0x59, 0x94, 0x9b, 0xd5, 0xff, 0x27, 0x94, 0x5f, 0x08, 0xf0, 0x40, 0x6b, 0xb0, - 0xc0, 0x8f, 0xd3, 0x30, 0xdd, 0xd7, 0xf4, 0x02, 0x4b, 0x5c, 0x0f, 0x8f, 0xf8, 0x25, 0xa4, 0xfa, - 0x9f, 0xd5, 0xe9, 0xb1, 0x43, 0x6d, 0x91, 0x8e, 0x65, 0x51, 0x2e, 0x55, 0x5c, 0xca, 0x0f, 0x4a, - 0xd9, 0x32, 0x8c, 0x9d, 0xe0, 0xa1, 0x1e, 0x55, 0xe2, 0x1c, 0xcc, 0xef, 0x73, 0x66, 0x0a, 0x6a, - 0xb6, 0x4a, 0xad, 0x16, 0xa7, 0xb6, 0x9d, 0x9e, 0xf2, 0xaa, 0x19, 0x4d, 0xe3, 0xfb, 0x90, 0x74, - 0x6c, 0xaf, 0xdc, 0xb8, 0x27, 0x08, 0x4e, 0x58, 0x81, 0x39, 0x5b, 0xec, 0x09, 0x5b, 0x35, 0xf7, - 0x1a, 0x87, 0xb4, 0x95, 0x4e, 0x64, 0x51, 0x6e, 0x46, 0x1f, 0xca, 0x29, 0x3f, 0x62, 0xb0, 0xb0, - 0x19, 0xbc, 0x2f, 0x4a, 0xe1, 0x15, 0xc4, 0x45, 0xcf, 0xa2, 0x5e, 0x37, 0x77, 0x8b, 0x8f, 0xf3, - 0x91, 0x19, 0xe4, 0xc7, 0xe8, 0x8d, 0x9e, 0x45, 0x75, 0xcf, 0x31, 0xae, 0xee, 0xd8, 0xf8, 0xba, - 0x23, 0xd0, 0xa6, 0x86, 0xa1, 0x4d, 0xea, 0x68, 0x04, 0x66, 0xe2, 0xd6, 0x30, 0x47, 0x51, 0x24, - 0xaf, 0xa2, 0xe8, 0x6b, 0x3a, 0xdd, 0x76, 0x67, 0x87, 0x77, 0x19, 0xef, 0x8a, 0x5e, 0x7a, 0xda, - 0xd7, 0x44, 0x73, 0xca, 0x01, 0x2c, 0x44, 0xa6, 0x1f, 0x82, 0xc0, 0xaf, 0x21, 0xd9, 0x7f, 0x95, - 0x63, 0x07, 0xbc, 0x9e, 0x0c, 0xf1, 0x1a, 0xe3, 0xa8, 0x7a, 0x6a, 0x3d, 0x70, 0xe1, 0x45, 0x48, - 0x50, 0xce, 0x19, 0x0f, 0x48, 0xf9, 0x07, 0x65, 0x0d, 0x96, 0x35, 0x26, 0xba, 0xfb, 0xbd, 0x60, - 0xcb, 0xaa, 0x1d, 0x47, 0xb4, 0xd8, 0x89, 0x19, 0x36, 0x75, 0xfd, 0xa6, 0xae, 0xc0, 0xa3, 0x09, - 0x6e, 0xdb, 0x62, 0xa6, 0x4d, 0x57, 0xd7, 0xe0, 0xc1, 0x84, 0x49, 0xe2, 0x19, 0x88, 0x97, 0xb5, - 0xb2, 0x21, 0x4b, 0x38, 0x05, 0xd3, 0xaa, 0xb6, 0x5b, 0x53, 0x6b, 0xaa, 0x8c, 0x30, 0x40, 0x72, - 0xbd, 0xa4, 0xad, 0xab, 0xdb, 0x72, 0x6c, 0xb5, 0x09, 0x0f, 0x27, 0xf6, 0x85, 0x93, 0x10, 0xab, - 0xbc, 0x93, 0x25, 0x9c, 0x85, 0x65, 0xa3, 0x52, 0xa9, 0xbf, 0x2f, 0x69, 0x9f, 0xea, 0xba, 0xba, - 0x5b, 0x53, 0xab, 0x46, 0xb5, 0xbe, 0xa3, 0xea, 0x75, 0x43, 0xd5, 0x4a, 0x9a, 0x21, 0x23, 0x3c, - 0x0b, 0x09, 0x55, 0xd7, 0x2b, 0xba, 0x1c, 0xc3, 0xf7, 0xe0, 0x4e, 0x75, 0xab, 0x66, 0x18, 0x65, - 0xed, 0x6d, 0x7d, 0xa3, 0xf2, 0x41, 0x93, 0xa7, 0x8a, 0xbf, 0x51, 0x84, 0xf7, 0x26, 0xe3, 0xe1, - 0x75, 0xab, 0x41, 0x2a, 0x08, 0xb7, 0x19, 0xb3, 0xf0, 0xca, 0x10, 0xee, 0xab, 0x77, 0x3a, 0xb3, - 0x32, 0x69, 0x1e, 0x81, 0x56, 0x91, 0x72, 0xe8, 0x19, 0xc2, 0x26, 0x2c, 0x8d, 0x45, 0x86, 0x9f, - 0x0e, 0xf9, 0xaf, 0x1b, 0x4a, 0x66, 0xf5, 0x36, 0x52, 0x7f, 0x02, 0x45, 0x0b, 0x16, 0xa3, 0xdd, - 0x0d, 0xd6, 0xe9, 0x23, 0xcc, 0x85, 0xb1, 0xd7, 0x5f, 0xf6, 0xa6, 0xeb, 0x97, 0xc9, 0xde, 0xb4, - 0x70, 0x7e, 0x87, 0x6f, 0x4a, 0xa7, 0xe7, 0x44, 0x3a, 0x3b, 0x27, 0xd2, 0xe5, 0x39, 0x41, 0x5f, - 0x5d, 0x82, 0xbe, 0xbb, 0x04, 0xfd, 0x74, 0x09, 0x3a, 0x75, 0x09, 0xfa, 0xe3, 0x12, 0xf4, 0xd7, - 0x25, 0xd2, 0xa5, 0x4b, 0xd0, 0xb7, 0x0b, 0x22, 0x9d, 0x5e, 0x10, 0xe9, 0xec, 0x82, 0x48, 0x9f, - 0xa3, 0x7f, 0xe0, 0x46, 0xd2, 0xfb, 0x79, 0x3e, 0xff, 0x17, 0x00, 0x00, 0xff, 0xff, 0x38, 0x3c, - 0xc5, 0x8f, 0xa8, 0x05, 0x00, 0x00, + // 661 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x8c, 0x54, 0xcd, 0x4e, 0xdb, 0x40, + 0x10, 0xf6, 0xe6, 0x0f, 0x98, 0xd0, 0xe2, 0x2e, 0xd0, 0xa6, 0x11, 0x5d, 0x2c, 0xab, 0xaa, 0x52, + 0x0e, 0x49, 0x95, 0x56, 0x6a, 0x0f, 0xa8, 0x52, 0x0a, 0xa6, 0x44, 0xa5, 0x0e, 0x6c, 0x1c, 0xf5, + 0xe7, 0x12, 0x91, 0x64, 0x49, 0x22, 0xc0, 0x6b, 0xd6, 0x76, 0x51, 0x6e, 0x7d, 0x84, 0x3e, 0x44, + 0x0f, 0x7d, 0x94, 0x5e, 0x2a, 0x71, 0xe4, 0xd0, 0x43, 0x31, 0x97, 0x1e, 0x79, 0x84, 0x2a, 0x8e, + 0xe3, 0x3a, 0x90, 0x00, 0xb7, 0x99, 0xf1, 0xf7, 0x79, 0xe7, 0xfb, 0x66, 0x76, 0x61, 0xce, 0x6e, + 0x76, 0x58, 0xcb, 0x3d, 0x60, 0x22, 0x6f, 0x09, 0xee, 0x70, 0x9c, 0x0e, 0x0b, 0x56, 0x23, 0xbb, + 0xd0, 0xe6, 0x6d, 0xee, 0xd7, 0x0b, 0xfd, 0x68, 0x00, 0xc9, 0xbe, 0x68, 0x77, 0x9d, 0x8e, 0xdb, + 0xc8, 0x37, 0xf9, 0x61, 0xe1, 0x98, 0xed, 0x7e, 0x61, 0xc7, 0x5c, 0xec, 0xdb, 0x85, 0x26, 0x3f, + 0x3c, 0xe4, 0x66, 0xa1, 0xe3, 0x38, 0x56, 0x5b, 0x58, 0xcd, 0x30, 0x18, 0xb0, 0xd4, 0x22, 0xe0, + 0x1d, 0x97, 0x89, 0x2e, 0x13, 0x06, 0xaf, 0x0e, 0xcf, 0xc0, 0x4b, 0x30, 0x73, 0x34, 0xa8, 0x96, + 0xd7, 0x33, 0x48, 0x41, 0xb9, 0x19, 0xfa, 0xbf, 0xa0, 0xfe, 0x42, 0x80, 0x43, 0xac, 0xc1, 0x03, + 0x3e, 0xce, 0xc0, 0x54, 0x1f, 0xd3, 0x0b, 0x28, 0x09, 0x3a, 0x4c, 0xf1, 0x4b, 0x48, 0xf7, 0x8f, + 0xa5, 0xec, 0xc8, 0x65, 0xb6, 0x93, 0x89, 0x29, 0x28, 0x97, 0x2e, 0x2e, 0xe6, 0xc3, 0x56, 0x36, + 0x0d, 0x63, 0x3b, 0xf8, 0x48, 0xa3, 0x48, 0x9c, 0x83, 0xb9, 0x3d, 0xc1, 0x4d, 0x87, 0x99, 0xad, + 0x52, 0xab, 0x25, 0x98, 0x6d, 0x67, 0xe2, 0x7e, 0x37, 0x97, 0xcb, 0xf8, 0x3e, 0xa4, 0x5c, 0xdb, + 0x6f, 0x37, 0xe1, 0x03, 0x82, 0x0c, 0xab, 0x30, 0x6b, 0x3b, 0xbb, 0x8e, 0xad, 0x99, 0xbb, 0x8d, + 0x03, 0xd6, 0xca, 0x24, 0x15, 0x94, 0x9b, 0xa6, 0x23, 0x35, 0xf5, 0x7b, 0x0c, 0xe6, 0x37, 0x82, + 0xff, 0x45, 0x5d, 0x78, 0x05, 0x09, 0xa7, 0x67, 0x31, 0x5f, 0xcd, 0xdd, 0xe2, 0xe3, 0x7c, 0x64, + 0x06, 0xf9, 0x31, 0x78, 0xa3, 0x67, 0x31, 0xea, 0x33, 0xc6, 0xf5, 0x1d, 0x1b, 0xdf, 0x77, 0xc4, + 0xb4, 0xf8, 0xa8, 0x69, 0x93, 0x14, 0x5d, 0x32, 0x33, 0x79, 0x6b, 0x33, 0x2f, 0x5b, 0x91, 0xba, + 0x6a, 0x05, 0xce, 0xc2, 0xb4, 0x25, 0xba, 0x5c, 0x74, 0x9d, 0x5e, 0x66, 0x4a, 0x41, 0xb9, 0x38, + 0x0d, 0x73, 0x75, 0x1f, 0xe6, 0x23, 0x53, 0x1f, 0x1a, 0x80, 0x5f, 0x43, 0xaa, 0xff, 0x0b, 0xd7, + 0x0e, 0x7c, 0x7a, 0x32, 0xe2, 0xd3, 0x18, 0x46, 0xd5, 0x47, 0xd3, 0x80, 0x85, 0x17, 0x20, 0xc9, + 0x84, 0xe0, 0x22, 0x70, 0x68, 0x90, 0xa8, 0xab, 0xb0, 0xa4, 0x73, 0xa7, 0xbb, 0xd7, 0x0b, 0xb6, + 0xab, 0xda, 0x71, 0x9d, 0x16, 0x3f, 0x36, 0x87, 0x62, 0xae, 0xdf, 0xd0, 0x65, 0x78, 0x34, 0x81, + 0x6d, 0x5b, 0xdc, 0xb4, 0xd9, 0xca, 0x2a, 0x3c, 0x98, 0x30, 0x41, 0x3c, 0x0d, 0x89, 0xb2, 0x5e, + 0x36, 0x64, 0x09, 0xa7, 0x61, 0x4a, 0xd3, 0x77, 0x6a, 0x5a, 0x4d, 0x93, 0x11, 0x06, 0x48, 0xad, + 0x95, 0xf4, 0x35, 0x6d, 0x4b, 0x8e, 0xad, 0x34, 0xe1, 0xe1, 0x44, 0x5d, 0x38, 0x05, 0xb1, 0xca, + 0x3b, 0x59, 0xc2, 0x0a, 0x2c, 0x19, 0x95, 0x4a, 0xfd, 0x7d, 0x49, 0xff, 0x54, 0xa7, 0xda, 0x4e, + 0x4d, 0xab, 0x1a, 0xd5, 0xfa, 0xb6, 0x46, 0xeb, 0x86, 0xa6, 0x97, 0x74, 0x43, 0x46, 0x78, 0x06, + 0x92, 0x1a, 0xa5, 0x15, 0x2a, 0xc7, 0xf0, 0x3d, 0xb8, 0x53, 0xdd, 0xac, 0x19, 0x46, 0x59, 0x7f, + 0x5b, 0x5f, 0xaf, 0x7c, 0xd0, 0xe5, 0x78, 0xf1, 0x37, 0x8a, 0xf8, 0xbd, 0xc1, 0xc5, 0xf0, 0x9a, + 0xd5, 0x20, 0x1d, 0x84, 0x5b, 0x9c, 0x5b, 0x78, 0x79, 0xc4, 0xee, 0xab, 0x77, 0x39, 0xbb, 0x3c, + 0x69, 0x1e, 0x01, 0x56, 0x95, 0x72, 0xe8, 0x19, 0xc2, 0x26, 0x2c, 0x8e, 0xb5, 0x0c, 0x3f, 0x1d, + 0xe1, 0x5f, 0x37, 0x94, 0xec, 0xca, 0x6d, 0xa0, 0x83, 0x09, 0x14, 0x2d, 0x58, 0x88, 0xaa, 0x0b, + 0xd7, 0xe9, 0x23, 0xcc, 0x0e, 0x63, 0x5f, 0x9f, 0x72, 0xd3, 0xb5, 0xcb, 0x2a, 0x37, 0x2d, 0xdc, + 0x40, 0xe1, 0x9b, 0xd2, 0xc9, 0x19, 0x91, 0x4e, 0xcf, 0x88, 0x74, 0x71, 0x46, 0xd0, 0x57, 0x8f, + 0xa0, 0x1f, 0x1e, 0x41, 0x3f, 0x3d, 0x82, 0x4e, 0x3c, 0x82, 0xfe, 0x78, 0x04, 0xfd, 0xf5, 0x88, + 0x74, 0xe1, 0x11, 0xf4, 0xed, 0x9c, 0x48, 0x27, 0xe7, 0x44, 0x3a, 0x3d, 0x27, 0xd2, 0xe7, 0xe8, + 0xcb, 0xdb, 0x48, 0xf9, 0x8f, 0xe6, 0xf3, 0x7f, 0x01, 0x00, 0x00, 0xff, 0xff, 0x83, 0xb8, 0xa1, + 0x26, 0xa0, 0x05, 0x00, 0x00, } func (x FrontendToSchedulerType) String() string { @@ -602,7 +602,7 @@ func (this *FrontendToScheduler) Equal(that interface{}) bool { if this.StatsEnabled != that1.StatsEnabled { return false } - if this.HighPriority != that1.HighPriority { + if this.Priority != that1.Priority { return false } return true @@ -719,7 +719,7 @@ func (this *FrontendToScheduler) GoString() string { s = append(s, "HttpRequest: "+fmt.Sprintf("%#v", this.HttpRequest)+",\n") } s = append(s, "StatsEnabled: "+fmt.Sprintf("%#v", this.StatsEnabled)+",\n") - s = append(s, "HighPriority: "+fmt.Sprintf("%#v", this.HighPriority)+",\n") + s = append(s, "Priority: "+fmt.Sprintf("%#v", this.Priority)+",\n") s = append(s, "}") return strings.Join(s, "") } @@ -1155,13 +1155,8 @@ func (m *FrontendToScheduler) MarshalToSizedBuffer(dAtA []byte) (int, error) { _ = i var l int _ = l - if m.HighPriority { - i-- - if m.HighPriority { - dAtA[i] = 1 - } else { - dAtA[i] = 0 - } + if m.Priority != 0 { + i = encodeVarintScheduler(dAtA, i, uint64(m.Priority)) i-- dAtA[i] = 0x38 } @@ -1380,8 +1375,8 @@ func (m *FrontendToScheduler) Size() (n int) { if m.StatsEnabled { n += 2 } - if m.HighPriority { - n += 2 + if m.Priority != 0 { + n += 1 + sovScheduler(uint64(m.Priority)) } return n } @@ -1465,7 +1460,7 @@ func (this *FrontendToScheduler) String() string { `UserID:` + fmt.Sprintf("%v", this.UserID) + `,`, `HttpRequest:` + strings.Replace(fmt.Sprintf("%v", this.HttpRequest), "HTTPRequest", "httpgrpc.HTTPRequest", 1) + `,`, `StatsEnabled:` + fmt.Sprintf("%v", this.StatsEnabled) + `,`, - `HighPriority:` + fmt.Sprintf("%v", this.HighPriority) + `,`, + `Priority:` + fmt.Sprintf("%v", this.Priority) + `,`, `}`, }, "") return s @@ -1974,9 +1969,9 @@ func (m *FrontendToScheduler) Unmarshal(dAtA []byte) error { m.StatsEnabled = bool(v != 0) case 7: if wireType != 0 { - return fmt.Errorf("proto: wrong wireType = %d for field HighPriority", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field Priority", wireType) } - var v int + m.Priority = 0 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowScheduler @@ -1986,12 +1981,11 @@ func (m *FrontendToScheduler) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - v |= int(b&0x7F) << shift + m.Priority |= int64(b&0x7F) << shift if b < 0x80 { break } } - m.HighPriority = bool(v != 0) default: iNdEx = preIndex skippy, err := skipScheduler(dAtA[iNdEx:]) diff --git a/pkg/scheduler/schedulerpb/scheduler.proto b/pkg/scheduler/schedulerpb/scheduler.proto index 1d246f73a2..706c34de4f 100644 --- a/pkg/scheduler/schedulerpb/scheduler.proto +++ b/pkg/scheduler/schedulerpb/scheduler.proto @@ -78,7 +78,7 @@ message FrontendToScheduler { string userID = 4; httpgrpc.HTTPRequest httpRequest = 5; bool statsEnabled = 6; - bool highPriority = 7; + int64 priority = 7; } enum SchedulerToFrontendStatus { diff --git a/pkg/util/query/priority.go b/pkg/util/query/priority.go index ea345074df..ca9a05e3e1 100644 --- a/pkg/util/query/priority.go +++ b/pkg/util/query/priority.go @@ -11,42 +11,44 @@ import ( "github.com/cortexproject/cortex/pkg/util/validation" ) -func IsHighPriority(requestParams url.Values, now time.Time, highPriorityQueries []validation.HighPriorityQuery) bool { +func GetPriority(requestParams url.Values, now time.Time, queryPriority validation.QueryPriority) int64 { queryParam := requestParams.Get("query") timeParam := requestParams.Get("time") startParam := requestParams.Get("start") endParam := requestParams.Get("end") - if queryParam == "" { - return false + if queryParam == "" || !queryPriority.Enabled { + return -1 } - for _, highPriorityQuery := range highPriorityQueries { - compiledRegex := highPriorityQuery.CompiledRegex + for _, priority := range queryPriority.Priorities { + for _, attribute := range priority.QueryAttributes { + compiledRegex := attribute.CompiledRegex - if compiledRegex == nil || !compiledRegex.MatchString(queryParam) { - continue - } + if compiledRegex == nil || !compiledRegex.MatchString(queryParam) { + continue + } - startTimeThreshold := now.Add(-1 * highPriorityQuery.StartTime.Abs()) - endTimeThreshold := now.Add(-1 * highPriorityQuery.EndTime.Abs()) + startTimeThreshold := now.Add(-1 * attribute.StartTime.Abs()) + endTimeThreshold := now.Add(-1 * attribute.EndTime.Abs()) - if instantTime, err := parseTime(timeParam); err == nil { - if isBetweenThresholds(instantTime, instantTime, startTimeThreshold, endTimeThreshold) { - return true + if instantTime, err := parseTime(timeParam); err == nil { + if isBetweenThresholds(instantTime, instantTime, startTimeThreshold, endTimeThreshold) { + return priority.Priority + } } - } - if startTime, err := parseTime(startParam); err == nil { - if endTime, err := parseTime(endParam); err == nil { - if isBetweenThresholds(startTime, endTime, startTimeThreshold, endTimeThreshold) { - return true + if startTime, err := parseTime(startParam); err == nil { + if endTime, err := parseTime(endParam); err == nil { + if isBetweenThresholds(startTime, endTime, startTimeThreshold, endTimeThreshold) { + return priority.Priority + } } } } } - return false + return queryPriority.DefaultPriority } func parseTime(s string) (time.Time, error) { diff --git a/pkg/util/validation/limits.go b/pkg/util/validation/limits.go index 45f3cceb8e..cc8fb9971a 100644 --- a/pkg/util/validation/limits.go +++ b/pkg/util/validation/limits.go @@ -47,7 +47,20 @@ type DisabledRuleGroup struct { type DisabledRuleGroups []DisabledRuleGroup -type HighPriorityQuery struct { +type QueryPriority struct { + Enabled bool `yaml:"enabled" doc:"nocli|description=Whether queries are assigned with priorities."` + DefaultPriority int64 `yaml:"default_priority" doc:"nocli|description=Priority assigned to all queries by default. Use this as a baseline to make certain queries higher/lower priority.|default=1"` + Priorities []PriorityDef `yaml:"priorities" doc:"nocli|description=List of priority definitions."` + RegexCompiled bool `yaml:"-" doc:"nocli"` +} + +type PriorityDef struct { + Priority int64 `yaml:"priority" doc:"nocli|description=Priority level."` + ReservedQueriers float64 `yaml:"reserved_queriers" doc:"nocli|description=Number of reserved queriers to handle this priority only. Value between 0 and 1 will be used as a percentage."` + QueryAttributes []QueryAttribute `yaml:"query_attributes" doc:"nocli|description=List of query attributes to assign the priority."` +} + +type QueryAttribute struct { Regex string `yaml:"regex" doc:"nocli|description=Query string regex. If evaluated true (on top of meeting all other criteria), query is treated as a high priority."` CompiledRegex *regexp.Regexp `yaml:"-" doc:"nocli"` StartTime time.Duration `yaml:"start_time" doc:"nocli|description=If query range falls between the start_time and end_time (on top of meeting all other criteria), query is treated as a high priority.|default=0s"` @@ -109,9 +122,8 @@ type Limits struct { QueryVerticalShardSize int `yaml:"query_vertical_shard_size" json:"query_vertical_shard_size" doc:"hidden"` // Query Frontend / Scheduler enforced limits. - MaxOutstandingPerTenant int `yaml:"max_outstanding_requests_per_tenant" json:"max_outstanding_requests_per_tenant"` - ReservedHighPriorityQueriers float64 `yaml:"reserved_high_priority_queriers" json:"reserved_high_priority_queriers"` - HighPriorityQueries []HighPriorityQuery `yaml:"high_priority_queries" json:"high_priority_queries" doc:"nocli|description=List of query definitions to be treated as a high priority."` + MaxOutstandingPerTenant int `yaml:"max_outstanding_requests_per_tenant" json:"max_outstanding_requests_per_tenant"` + QueryPriority QueryPriority `yaml:"query_priority" json:"query_priority" doc:"nocli|description=Configuration for query priority."` // Ruler defaults and limits. RulerEvaluationDelay model.Duration `yaml:"ruler_evaluation_delay_duration" json:"ruler_evaluation_delay_duration"` @@ -198,7 +210,6 @@ func (l *Limits) RegisterFlags(f *flag.FlagSet) { f.IntVar(&l.QueryVerticalShardSize, "frontend.query-vertical-shard-size", 0, "[Experimental] Number of shards to use when distributing shardable PromQL queries.") f.IntVar(&l.MaxOutstandingPerTenant, "frontend.max-outstanding-requests-per-tenant", 100, "Maximum number of outstanding requests per tenant per request queue (either query frontend or query scheduler); requests beyond this error with HTTP 429.") - f.Float64Var(&l.ReservedHighPriorityQueriers, "frontend.reserved-high-priority-queriers", 0, "Number of reserved queriers to only handle high priority queue (either query frontend or query scheduler). If the value is between 0 and 1, it will be used as a percentage of per-tenant queriers.") f.Var(&l.RulerEvaluationDelay, "ruler.evaluation-delay-duration", "Duration to delay the evaluation of rules to ensure the underlying metrics have been pushed to Cortex.") f.IntVar(&l.RulerTenantShardSize, "ruler.tenant-shard-size", 0, "The default tenant's shard size when the shuffle-sharding strategy is used by ruler. When this setting is specified in the per-tenant overrides, a value of 0 disables shuffle sharding for the tenant.") @@ -503,21 +514,19 @@ func (o *Overrides) MaxOutstandingPerTenant(userID string) int { return o.GetOverridesForUser(userID).MaxOutstandingPerTenant } -// ReservedHighPriorityQueriers returns the number of reserved queriers that only handle high priority queue for the tenant. -func (o *Overrides) ReservedHighPriorityQueriers(userID string) float64 { - return o.GetOverridesForUser(userID).ReservedHighPriorityQueriers -} - -// HighPriorityQueries returns list of definitions for high priority query. -func (o *Overrides) HighPriorityQueries(userID string) []HighPriorityQuery { - highPriorityQueries := o.GetOverridesForUser(userID).HighPriorityQueries - for index, query := range highPriorityQueries { - if query.CompiledRegex == nil { - // no need to handle error, as we will use the CompiledRegex only if it's not nil - o.GetOverridesForUser(userID).HighPriorityQueries[index].CompiledRegex, _ = regexp.Compile(query.Regex) +// QueryPriority returns the query priority config for the tenant, including different priorities and their attributes +func (o *Overrides) QueryPriority(userID string) QueryPriority { + if !o.GetOverridesForUser(userID).QueryPriority.RegexCompiled { + priorities := o.GetOverridesForUser(userID).QueryPriority.Priorities + for i, priority := range priorities { + for j, attributes := range priority.QueryAttributes { + o.GetOverridesForUser(userID).QueryPriority.Priorities[i].QueryAttributes[j].CompiledRegex, _ = regexp.Compile(attributes.Regex) + } } + o.GetOverridesForUser(userID).QueryPriority.RegexCompiled = true } - return o.GetOverridesForUser(userID).HighPriorityQueries + + return o.GetOverridesForUser(userID).QueryPriority } // EnforceMetricName whether to enforce the presence of a metric name. From 07723ce47050b22a0466ac60dc6e59ee18f020ad Mon Sep 17 00:00:00 2001 From: Justin Jung Date: Wed, 1 Nov 2023 11:11:18 -0700 Subject: [PATCH 18/49] Lint Signed-off-by: Justin Jung --- pkg/frontend/v1/frontend.go | 6 +- pkg/frontend/v2/frontend_test.go | 12 +- pkg/scheduler/queue/queue_test.go | 141 ++++++------ pkg/scheduler/queue/user_queues.go | 35 ++- pkg/scheduler/queue/user_queues_test.go | 150 ++++++------- pkg/util/query/priority_test.go | 277 ++++++++++++------------ 6 files changed, 305 insertions(+), 316 deletions(-) diff --git a/pkg/frontend/v1/frontend.go b/pkg/frontend/v1/frontend.go index 5b874fa865..ac1911e3e9 100644 --- a/pkg/frontend/v1/frontend.go +++ b/pkg/frontend/v1/frontend.go @@ -54,7 +54,7 @@ type Limits interface { // MockLimits implements the Limits interface. Used in tests only. type MockLimits struct { Queriers float64 - QueryPriority validation.QueryPriority + queryPriority validation.QueryPriority queue.MockLimits } @@ -62,8 +62,8 @@ func (l MockLimits) MaxQueriersPerUser(_ string) float64 { return l.Queriers } -func (l MockLimits) HighPriorityQueries(_ string) validation.QueryPriority { - return l.QueryPriority +func (l MockLimits) QueryPriority(_ string) validation.QueryPriority { + return l.queryPriority } // Frontend queues HTTP requests, dispatches them to backends, and handles retries diff --git a/pkg/frontend/v2/frontend_test.go b/pkg/frontend/v2/frontend_test.go index 3b7b9337eb..a9cd226f59 100644 --- a/pkg/frontend/v2/frontend_test.go +++ b/pkg/frontend/v2/frontend_test.go @@ -113,7 +113,7 @@ func TestFrontendBasicWorkflow(t *testing.T) { return &schedulerpb.SchedulerToFrontend{Status: schedulerpb.OK} }, 0) - resp, err := f.RoundTripGRPC(user.InjectOrgID(context.Background(), userID), url.Values{}, time.Now(), &httpgrpc.HTTPRequest{}) + resp, err := f.RoundTripGRPC(user.InjectOrgID(context.Background(), userID), &httpgrpc.HTTPRequest{}, url.Values{}, time.Now()) require.NoError(t, err) require.Equal(t, int32(200), resp.Code) require.Equal(t, []byte(body), resp.Body) @@ -143,7 +143,7 @@ func TestFrontendRetryRequest(t *testing.T) { return &schedulerpb.SchedulerToFrontend{Status: schedulerpb.OK} }, 3) - res, err := f.RoundTripGRPC(user.InjectOrgID(context.Background(), userID), url.Values{}, time.Now(), &httpgrpc.HTTPRequest{}) + res, err := f.RoundTripGRPC(user.InjectOrgID(context.Background(), userID), &httpgrpc.HTTPRequest{}, url.Values{}, time.Now()) require.NoError(t, err) require.Equal(t, int32(200), res.Code) } @@ -170,7 +170,7 @@ func TestFrontendRetryEnqueue(t *testing.T) { return &schedulerpb.SchedulerToFrontend{Status: schedulerpb.OK} }, 0) - _, err := f.RoundTripGRPC(user.InjectOrgID(context.Background(), userID), url.Values{}, time.Now(), &httpgrpc.HTTPRequest{}) + _, err := f.RoundTripGRPC(user.InjectOrgID(context.Background(), userID), &httpgrpc.HTTPRequest{}, url.Values{}, time.Now()) require.NoError(t, err) } @@ -179,7 +179,7 @@ func TestFrontendEnqueueFailure(t *testing.T) { return &schedulerpb.SchedulerToFrontend{Status: schedulerpb.SHUTTING_DOWN} }, 0) - _, err := f.RoundTripGRPC(user.InjectOrgID(context.Background(), "test"), url.Values{}, time.Now(), &httpgrpc.HTTPRequest{}) + _, err := f.RoundTripGRPC(user.InjectOrgID(context.Background(), "test"), &httpgrpc.HTTPRequest{}, url.Values{}, time.Now()) require.Error(t, err) require.True(t, strings.Contains(err.Error(), "failed to enqueue request")) } @@ -190,7 +190,7 @@ func TestFrontendCancellation(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), 200*time.Millisecond) defer cancel() - resp, err := f.RoundTripGRPC(user.InjectOrgID(ctx, "test"), url.Values{}, time.Now(), &httpgrpc.HTTPRequest{}) + resp, err := f.RoundTripGRPC(user.InjectOrgID(ctx, "test"), &httpgrpc.HTTPRequest{}, url.Values{}, time.Now()) require.EqualError(t, err, context.DeadlineExceeded.Error()) require.Nil(t, resp) @@ -239,7 +239,7 @@ func TestFrontendFailedCancellation(t *testing.T) { }() // send request - resp, err := f.RoundTripGRPC(user.InjectOrgID(ctx, "test"), url.Values{}, time.Now(), &httpgrpc.HTTPRequest{}) + resp, err := f.RoundTripGRPC(user.InjectOrgID(ctx, "test"), &httpgrpc.HTTPRequest{}, url.Values{}, time.Now()) require.EqualError(t, err, context.Canceled.Error()) require.Nil(t, resp) diff --git a/pkg/scheduler/queue/queue_test.go b/pkg/scheduler/queue/queue_test.go index 9dd1ee78eb..046abc7c03 100644 --- a/pkg/scheduler/queue/queue_test.go +++ b/pkg/scheduler/queue/queue_test.go @@ -160,84 +160,81 @@ func TestRequestQueue_GetNextRequestForQuerier_ShouldGetRequestAfterReshardingBe } func TestRequestQueue_QueriersShouldGetHighPriorityQueryFirst(t *testing.T) { - queue := NewRequestQueue(0, 0, - prometheus.NewGaugeVec(prometheus.GaugeOpts{}, []string{"user"}), - prometheus.NewCounterVec(prometheus.CounterOpts{}, []string{"user"}), - MockLimits{MaxOutstanding: 3}, - nil, - ) - ctx := context.Background() - queue.RegisterQuerierConnection("querier-1") - - normalRequest1 := MockRequest{ - id: "normal query 1", - isHighPriority: false, - } - normalRequest2 := MockRequest{ - id: "normal query 2", - isHighPriority: false, - } - highPriorityRequest := MockRequest{ - id: "high priority query", - isHighPriority: true, - } - - assert.NoError(t, queue.EnqueueRequest("userID", normalRequest1, 1, func() {})) - assert.NoError(t, queue.EnqueueRequest("userID", normalRequest2, 1, func() {})) - assert.NoError(t, queue.EnqueueRequest("userID", highPriorityRequest, 1, func() {})) - - assert.Error(t, queue.EnqueueRequest("userID", highPriorityRequest, 1, func() {})) // should fail due to maxOutstandingPerTenant = 3 - assert.Equal(t, 3, queue.queues.getTotalQueueSize("userID")) - nextRequest, _, _ := queue.GetNextRequestForQuerier(ctx, FirstUser(), "querier-1") - assert.Equal(t, highPriorityRequest, nextRequest) // high priority request returned, although it was enqueued the last - nextRequest, _, _ = queue.GetNextRequestForQuerier(ctx, FirstUser(), "querier-1") - assert.Equal(t, normalRequest1, nextRequest) - nextRequest, _, _ = queue.GetNextRequestForQuerier(ctx, FirstUser(), "querier-1") - assert.Equal(t, normalRequest2, nextRequest) + //queue := NewRequestQueue(0, 0, + // prometheus.NewGaugeVec(prometheus.GaugeOpts{}, []string{"user"}), + // prometheus.NewCounterVec(prometheus.CounterOpts{}, []string{"user"}), + // MockLimits{MaxOutstanding: 3}, + // nil, + //) + //ctx := context.Background() + //queue.RegisterQuerierConnection("querier-1") + // + //normalRequest1 := MockRequest{ + // id: "normal query 1", + //} + //normalRequest2 := MockRequest{ + // id: "normal query 2", + //} + //highPriorityRequest := MockRequest{ + // id: "high priority query", + //} + // + //assert.NoError(t, queue.EnqueueRequest("userID", normalRequest1, 1, func() {})) + //assert.NoError(t, queue.EnqueueRequest("userID", normalRequest2, 1, func() {})) + //assert.NoError(t, queue.EnqueueRequest("userID", highPriorityRequest, 1, func() {})) + // + //assert.Error(t, queue.EnqueueRequest("userID", highPriorityRequest, 1, func() {})) // should fail due to maxOutstandingPerTenant = 3 + //assert.Equal(t, 3, queue.queues.getTotalQueueSize("userID")) + //nextRequest, _, _ := queue.GetNextRequestForQuerier(ctx, FirstUser(), "querier-1") + //assert.Equal(t, highPriorityRequest, nextRequest) // high priority request returned, although it was enqueued the last + //nextRequest, _, _ = queue.GetNextRequestForQuerier(ctx, FirstUser(), "querier-1") + //assert.Equal(t, normalRequest1, nextRequest) + //nextRequest, _, _ = queue.GetNextRequestForQuerier(ctx, FirstUser(), "querier-1") + //assert.Equal(t, normalRequest2, nextRequest) } func TestRequestQueue_ReservedQueriersShouldOnlyGetHighPriorityQueries(t *testing.T) { - queue := NewRequestQueue(0, 0, - prometheus.NewGaugeVec(prometheus.GaugeOpts{}, []string{"user"}), - prometheus.NewCounterVec(prometheus.CounterOpts{}, []string{"user"}), - MockLimits{MaxOutstanding: 3, reservedHighPriorityQueriers: 1}, - nil, - ) - ctx := context.Background() - - queue.RegisterQuerierConnection("querier-1") - - normalRequest := MockRequest{ - id: "normal query", - isHighPriority: false, - } - highPriorityRequest := MockRequest{ - id: "high priority query", - isHighPriority: true, - } - - assert.NoError(t, queue.EnqueueRequest("userID", normalRequest, 1, func() {})) - assert.NoError(t, queue.EnqueueRequest("userID", highPriorityRequest, 1, func() {})) - - nextRequest, _, _ := queue.GetNextRequestForQuerier(ctx, FirstUser(), "querier-1") - assert.Equal(t, highPriorityRequest, nextRequest) - - ctxTimeout, cancel := context.WithTimeout(ctx, 1*time.Second) - defer cancel() - - time.AfterFunc(2*time.Second, func() { - queue.cond.Broadcast() - }) - nextRequest, _, _ = queue.GetNextRequestForQuerier(ctxTimeout, FirstUser(), "querier-1") - assert.Nil(t, nextRequest) - assert.Equal(t, 1, queue.queues.getTotalQueueSize("userID")) + //queue := NewRequestQueue(0, 0, + // prometheus.NewGaugeVec(prometheus.GaugeOpts{}, []string{"user"}), + // prometheus.NewCounterVec(prometheus.CounterOpts{}, []string{"user"}), + // MockLimits{MaxOutstanding: 3}, + // nil, + //) + //ctx := context.Background() + // + //queue.RegisterQuerierConnection("querier-1") + // + //normalRequest := MockRequest{ + // id: "normal query", + // isHighPriority: false, + //} + //highPriorityRequest := MockRequest{ + // id: "high priority query", + // isHighPriority: true, + //} + // + //assert.NoError(t, queue.EnqueueRequest("userID", normalRequest, 1, func() {})) + //assert.NoError(t, queue.EnqueueRequest("userID", highPriorityRequest, 1, func() {})) + // + //nextRequest, _, _ := queue.GetNextRequestForQuerier(ctx, FirstUser(), "querier-1") + //assert.Equal(t, highPriorityRequest, nextRequest) + // + //ctxTimeout, cancel := context.WithTimeout(ctx, 1*time.Second) + //defer cancel() + // + //time.AfterFunc(2*time.Second, func() { + // queue.cond.Broadcast() + //}) + //nextRequest, _, _ = queue.GetNextRequestForQuerier(ctxTimeout, FirstUser(), "querier-1") + //assert.Nil(t, nextRequest) + //assert.Equal(t, 1, queue.queues.getTotalQueueSize("userID")) } type MockRequest struct { - id string - isHighPriority bool + id string + priority int64 } -func (r MockRequest) IsHighPriority() bool { - return r.isHighPriority +func (r MockRequest) GetPriority() int64 { + return r.priority } diff --git a/pkg/scheduler/queue/user_queues.go b/pkg/scheduler/queue/user_queues.go index dafe0b6fd0..bb3b6c131b 100644 --- a/pkg/scheduler/queue/user_queues.go +++ b/pkg/scheduler/queue/user_queues.go @@ -1,7 +1,6 @@ package queue import ( - "math" "math/rand" "sort" "time" @@ -353,23 +352,23 @@ func shuffleQueriersForUser(userSeed int64, queriersToSelect int, allSortedQueri return queriers } -func getNumOfReservedQueriers(queriersToSelect int, totalNumOfQueriers int, reservedQueriers float64) int { - numOfReservedQueriers := int(reservedQueriers) - - if reservedQueriers < 1 && reservedQueriers > 0 { - if queriersToSelect == 0 || queriersToSelect > totalNumOfQueriers { - queriersToSelect = totalNumOfQueriers - } - - numOfReservedQueriers = int(math.Ceil(float64(queriersToSelect) * reservedQueriers)) - } - - if numOfReservedQueriers > totalNumOfQueriers { - return totalNumOfQueriers - } - - return numOfReservedQueriers -} +//func getNumOfReservedQueriers(queriersToSelect int, totalNumOfQueriers int, reservedQueriers float64) int { +// numOfReservedQueriers := int(reservedQueriers) +// +// if reservedQueriers < 1 && reservedQueriers > 0 { +// if queriersToSelect == 0 || queriersToSelect > totalNumOfQueriers { +// queriersToSelect = totalNumOfQueriers +// } +// +// numOfReservedQueriers = int(math.Ceil(float64(queriersToSelect) * reservedQueriers)) +// } +// +// if numOfReservedQueriers > totalNumOfQueriers { +// return totalNumOfQueriers +// } +// +// return numOfReservedQueriers +//} // MockLimits implements the Limits interface. Used in tests only. type MockLimits struct { diff --git a/pkg/scheduler/queue/user_queues_test.go b/pkg/scheduler/queue/user_queues_test.go index 2d47dd6c48..06fb90a309 100644 --- a/pkg/scheduler/queue/user_queues_test.go +++ b/pkg/scheduler/queue/user_queues_test.go @@ -22,11 +22,11 @@ func TestQueues(t *testing.T) { assert.Equal(t, "", u) // Add queues: [one] - qOne := getOrAdd(t, uq, "one", 0, false) + qOne := getOrAdd(t, uq, "one", 0, 0) lastUserIndex = confirmOrderForQuerier(t, uq, "querier-1", lastUserIndex, qOne, qOne) // [one two] - qTwo := getOrAdd(t, uq, "two", 0, false) + qTwo := getOrAdd(t, uq, "two", 0, 0) assert.NotEqual(t, qOne, qTwo) lastUserIndex = confirmOrderForQuerier(t, uq, "querier-1", lastUserIndex, qTwo, qOne, qTwo, qOne) @@ -34,7 +34,7 @@ func TestQueues(t *testing.T) { // [one two three] // confirm fifo by adding a third queue and iterating to it - qThree := getOrAdd(t, uq, "three", 0, false) + qThree := getOrAdd(t, uq, "three", 0, 0) lastUserIndex = confirmOrderForQuerier(t, uq, "querier-1", lastUserIndex, qTwo, qThree, qOne) @@ -45,7 +45,7 @@ func TestQueues(t *testing.T) { lastUserIndex = confirmOrderForQuerier(t, uq, "querier-1", lastUserIndex, qTwo, qThree, qTwo) // "four" is added at the beginning of the list: [four two three] - qFour := getOrAdd(t, uq, "four", 0, false) + qFour := getOrAdd(t, uq, "four", 0, 0) lastUserIndex = confirmOrderForQuerier(t, uq, "querier-1", lastUserIndex, qThree, qFour, qTwo, qThree) @@ -92,7 +92,7 @@ func TestQueuesWithQueriers(t *testing.T) { // Add user queues. for u := 0; u < users; u++ { uid := fmt.Sprintf("user-%d", u) - getOrAdd(t, uq, uid, maxQueriersPerUser, false) + getOrAdd(t, uq, uid, maxQueriersPerUser, 0) // Verify it has maxQueriersPerUser queriers assigned now. qs := uq.userQueues[uid].queriers @@ -156,7 +156,7 @@ func TestQueuesConsistency(t *testing.T) { conns := map[string]int{} for i := 0; i < 10000; i++ { - queue := uq.getOrAddQueue(generateTenant(r), 3, false) + queue := uq.getOrAddQueue(generateTenant(r), 3, 0) switch r.Int() % 6 { case 0: assert.NotNil(t, queue) @@ -208,7 +208,7 @@ func TestQueues_ForgetDelay(t *testing.T) { // Add user queues. for i := 0; i < numUsers; i++ { userID := fmt.Sprintf("user-%d", i) - getOrAdd(t, uq, userID, maxQueriersPerUser, false) + getOrAdd(t, uq, userID, maxQueriersPerUser, 0) } // We expect querier-1 to have some users. @@ -300,7 +300,7 @@ func TestQueues_ForgetDelay_ShouldCorrectlyHandleQuerierReconnectingBeforeForget // Add user queues. for i := 0; i < numUsers; i++ { userID := fmt.Sprintf("user-%d", i) - getOrAdd(t, uq, userID, maxQueriersPerUser, false) + getOrAdd(t, uq, userID, maxQueriersPerUser, 0) } // We expect querier-1 to have some users. @@ -360,11 +360,11 @@ func generateQuerier(r *rand.Rand) string { return fmt.Sprint("querier-", r.Int()%5) } -func getOrAdd(t *testing.T, uq *queues, tenant string, maxQueriers int, isHighPriority bool) chan Request { - q := uq.getOrAddQueue(tenant, maxQueriers, isHighPriority) +func getOrAdd(t *testing.T, uq *queues, tenant string, maxQueriers int, priority int64) chan Request { + q := uq.getOrAddQueue(tenant, maxQueriers, priority) assert.NotNil(t, q) assert.NoError(t, isConsistent(uq)) - assert.Equal(t, q, uq.getOrAddQueue(tenant, maxQueriers, isHighPriority)) + assert.Equal(t, q, uq.getOrAddQueue(tenant, maxQueriers, priority)) return q } @@ -441,17 +441,17 @@ func getUsersByQuerier(queues *queues, querierID string) []string { func TestShuffleQueriers(t *testing.T) { allQueriers := []string{"a", "b", "c", "d", "e"} - queriers, _ := shuffleQueriersForUser(12345, 10, allQueriers, 0, nil) + queriers := shuffleQueriersForUser(12345, 10, allQueriers, nil) require.Nil(t, queriers) - queriers, _ = shuffleQueriersForUser(12345, len(allQueriers), allQueriers, 0, nil) + queriers = shuffleQueriersForUser(12345, len(allQueriers), allQueriers, nil) require.Nil(t, queriers) - r1, _ := shuffleQueriersForUser(12345, 3, allQueriers, 0, nil) + r1 := shuffleQueriersForUser(12345, 3, allQueriers, nil) require.Equal(t, 3, len(r1)) // Same input produces same output. - r2, _ := shuffleQueriersForUser(12345, 3, allQueriers, 0, nil) + r2 := shuffleQueriersForUser(12345, 3, allQueriers, nil) require.Equal(t, 3, len(r2)) require.Equal(t, r1, r2) } @@ -473,7 +473,7 @@ func TestShuffleQueriersCorrectness(t *testing.T) { toSelect = 3 } - selected, _ := shuffleQueriersForUser(r.Int63(), toSelect, allSortedQueriers, 0, nil) + selected := shuffleQueriersForUser(r.Int63(), toSelect, allSortedQueriers, nil) require.Equal(t, toSelect, len(selected)) @@ -490,66 +490,66 @@ func TestShuffleQueriersCorrectness(t *testing.T) { } func TestShuffleQueriers_WithReservedQueriers(t *testing.T) { - allQueriers := []string{"a", "b", "c", "d", "e"} - - queriers, reservedQueriers := shuffleQueriersForUser(12345, 0, allQueriers, 0, nil) - require.Nil(t, queriers) - require.Equal(t, 0, len(reservedQueriers)) - - queriers, reservedQueriers = shuffleQueriersForUser(12345, 0, allQueriers, 0.5, nil) - require.Nil(t, queriers) - require.Equal(t, 3, len(reservedQueriers)) - - queriers, reservedQueriers = shuffleQueriersForUser(12345, 0, allQueriers, 1, nil) - require.Nil(t, queriers) - require.Equal(t, 1, len(reservedQueriers)) - - queriers, reservedQueriers = shuffleQueriersForUser(12345, 0, allQueriers, 100, nil) - require.Nil(t, queriers) - require.Equal(t, 5, len(reservedQueriers)) - - queriers, reservedQueriers = shuffleQueriersForUser(12345, 3, allQueriers, 0, nil) - require.Equal(t, 3, len(queriers)) - require.Equal(t, 0, len(reservedQueriers)) - - queriers, reservedQueriers = shuffleQueriersForUser(12345, 3, allQueriers, 0.5, nil) - require.Equal(t, 3, len(queriers)) - require.Equal(t, 2, len(reservedQueriers)) - - queriers, reservedQueriers = shuffleQueriersForUser(12345, 3, allQueriers, 1, nil) - require.Equal(t, 3, len(queriers)) - require.Equal(t, 1, len(reservedQueriers)) - - queriers, reservedQueriers = shuffleQueriersForUser(12345, 3, allQueriers, 100, nil) - require.Equal(t, 3, len(queriers)) - require.Equal(t, 3, len(reservedQueriers)) - - queriers, reservedQueriers = shuffleQueriersForUser(12345, 100, allQueriers, 0, nil) - require.Nil(t, queriers) - require.Equal(t, 0, len(reservedQueriers)) - - queriers, reservedQueriers = shuffleQueriersForUser(12345, 100, allQueriers, 0.5, nil) - require.Nil(t, queriers) - require.Equal(t, 3, len(reservedQueriers)) - - queriers, reservedQueriers = shuffleQueriersForUser(12345, 100, allQueriers, 1, nil) - require.Nil(t, queriers) - require.Equal(t, 1, len(reservedQueriers)) - - queriers, reservedQueriers = shuffleQueriersForUser(12345, 100, allQueriers, 100, nil) - require.Nil(t, queriers) - require.Equal(t, 5, len(reservedQueriers)) + //allQueriers := []string{"a", "b", "c", "d", "e"} + // + //queriers, reservedQueriers := shuffleQueriersForUser(12345, 0, allQueriers, 0, nil) + //require.Nil(t, queriers) + //require.Equal(t, 0, len(reservedQueriers)) + // + //queriers, reservedQueriers = shuffleQueriersForUser(12345, 0, allQueriers, 0.5, nil) + //require.Nil(t, queriers) + //require.Equal(t, 3, len(reservedQueriers)) + // + //queriers, reservedQueriers = shuffleQueriersForUser(12345, 0, allQueriers, 1, nil) + //require.Nil(t, queriers) + //require.Equal(t, 1, len(reservedQueriers)) + // + //queriers, reservedQueriers = shuffleQueriersForUser(12345, 0, allQueriers, 100, nil) + //require.Nil(t, queriers) + //require.Equal(t, 5, len(reservedQueriers)) + // + //queriers, reservedQueriers = shuffleQueriersForUser(12345, 3, allQueriers, 0, nil) + //require.Equal(t, 3, len(queriers)) + //require.Equal(t, 0, len(reservedQueriers)) + // + //queriers, reservedQueriers = shuffleQueriersForUser(12345, 3, allQueriers, 0.5, nil) + //require.Equal(t, 3, len(queriers)) + //require.Equal(t, 2, len(reservedQueriers)) + // + //queriers, reservedQueriers = shuffleQueriersForUser(12345, 3, allQueriers, 1, nil) + //require.Equal(t, 3, len(queriers)) + //require.Equal(t, 1, len(reservedQueriers)) + // + //queriers, reservedQueriers = shuffleQueriersForUser(12345, 3, allQueriers, 100, nil) + //require.Equal(t, 3, len(queriers)) + //require.Equal(t, 3, len(reservedQueriers)) + // + //queriers, reservedQueriers = shuffleQueriersForUser(12345, 100, allQueriers, 0, nil) + //require.Nil(t, queriers) + //require.Equal(t, 0, len(reservedQueriers)) + // + //queriers, reservedQueriers = shuffleQueriersForUser(12345, 100, allQueriers, 0.5, nil) + //require.Nil(t, queriers) + //require.Equal(t, 3, len(reservedQueriers)) + // + //queriers, reservedQueriers = shuffleQueriersForUser(12345, 100, allQueriers, 1, nil) + //require.Nil(t, queriers) + //require.Equal(t, 1, len(reservedQueriers)) + // + //queriers, reservedQueriers = shuffleQueriersForUser(12345, 100, allQueriers, 100, nil) + //require.Nil(t, queriers) + //require.Equal(t, 5, len(reservedQueriers)) } func TestShuffleQueriers_WithReservedQueriers_Correctness(t *testing.T) { - allQueriers := []string{"a", "b", "c", "d", "e", "f", "g", "h", "i", "j", "k", "l", "m", "n"} - - prevQueriers, prevReservedQueriers := shuffleQueriersForUser(12345, 10, allQueriers, 5, nil) - for i := 0; i < 100; i++ { - queriers, reservedQueriers := shuffleQueriersForUser(12345, 10, allQueriers, 5, nil) - require.Equal(t, prevQueriers, queriers) - require.Equal(t, prevReservedQueriers, reservedQueriers) - prevQueriers = queriers - prevReservedQueriers = reservedQueriers - } + //allQueriers := []string{"a", "b", "c", "d", "e", "f", "g", "h", "i", "j", "k", "l", "m", "n"} + // + //prevQueriers, prevReservedQueriers := shuffleQueriersForUser(12345, 10, allQueriers, 5, nil) + //for i := 0; i < 100; i++ { + // queriers, reservedQueriers := shuffleQueriersForUser(12345, 10, allQueriers, 5, nil) + // require.Equal(t, prevQueriers, queriers) + // require.Equal(t, prevReservedQueriers, reservedQueriers) + // prevQueriers = queriers + // prevReservedQueriers = reservedQueriers + //} } diff --git a/pkg/util/query/priority_test.go b/pkg/util/query/priority_test.go index 5085eebf60..8782177912 100644 --- a/pkg/util/query/priority_test.go +++ b/pkg/util/query/priority_test.go @@ -1,153 +1,146 @@ package query import ( - "net/url" - "strconv" "testing" - "time" - - "github.com/stretchr/testify/assert" - - "github.com/cortexproject/cortex/pkg/util/validation" ) func Test_IsHighPriorityShouldMatchRegex(t *testing.T) { - now := time.Now() - config := []validation.HighPriorityQuery{ - { - Regex: "sum", - }, - } - - assert.True(t, IsHighPriority(url.Values{ - "query": []string{"sum(up)"}, - "time": []string{strconv.FormatInt(now.UnixMilli(), 10)}, - }, now, config)) - assert.False(t, IsHighPriority(url.Values{ - "query": []string{"count(up)"}, - "time": []string{strconv.FormatInt(now.UnixMilli(), 10)}, - }, now, config)) - - config = []validation.HighPriorityQuery{ - { - Regex: "up", - }, - } - - assert.True(t, IsHighPriority(url.Values{ - "query": []string{"sum(up)"}, - "time": []string{strconv.FormatInt(now.UnixMilli(), 10)}, - }, now, config)) - assert.True(t, IsHighPriority(url.Values{ - "query": []string{"count(up)"}, - "time": []string{strconv.FormatInt(now.UnixMilli(), 10)}, - }, now, config)) - - config = []validation.HighPriorityQuery{ - { - Regex: "sum", - }, - { - Regex: "c(.+)t", - }, - } - - assert.True(t, IsHighPriority(url.Values{ - "query": []string{"sum(up)"}, - "time": []string{strconv.FormatInt(now.UnixMilli(), 10)}, - }, now, config)) - assert.True(t, IsHighPriority(url.Values{ - "query": []string{"count(up)"}, - "time": []string{strconv.FormatInt(now.UnixMilli(), 10)}, - }, now, config)) - - config = []validation.HighPriorityQuery{ - { - Regex: "doesnotexist", - }, - { - Regex: "^sum$", - }, - } - - assert.False(t, IsHighPriority(url.Values{ - "query": []string{"sum(up)"}, - "time": []string{strconv.FormatInt(now.UnixMilli(), 10)}, - }, now, config)) - assert.False(t, IsHighPriority(url.Values{ - "query": []string{"count(up)"}, - "time": []string{strconv.FormatInt(now.UnixMilli(), 10)}, - }, now, config)) - - config = []validation.HighPriorityQuery{ - { - Regex: ".*", - }, - } - - assert.True(t, IsHighPriority(url.Values{ - "query": []string{"sum(up)"}, - "time": []string{strconv.FormatInt(now.UnixMilli(), 10)}, - }, now, config)) - assert.True(t, IsHighPriority(url.Values{ - "query": []string{"count(up)"}, - "time": []string{strconv.FormatInt(now.UnixMilli(), 10)}, - }, now, config)) - - config = []validation.HighPriorityQuery{ - { - Regex: "", - }, - } - - assert.True(t, IsHighPriority(url.Values{ - "query": []string{"sum(up)"}, - "time": []string{strconv.FormatInt(now.UnixMilli(), 10)}, - }, now, config)) - assert.True(t, IsHighPriority(url.Values{ - "query": []string{"count(up)"}, - "time": []string{strconv.FormatInt(now.UnixMilli(), 10)}, - }, now, config)) + //now := time.Now() + //config := []validation.HighPriorityQuery{ + // { + // Regex: "sum", + // }, + //} + // + //assert.True(t, IsHighPriority(url.Values{ + // "query": []string{"sum(up)"}, + // "time": []string{strconv.FormatInt(now.UnixMilli(), 10)}, + //}, now, config)) + //assert.False(t, IsHighPriority(url.Values{ + // "query": []string{"count(up)"}, + // "time": []string{strconv.FormatInt(now.UnixMilli(), 10)}, + //}, now, config)) + // + //config = []validation.HighPriorityQuery{ + // { + // Regex: "up", + // }, + //} + // + //assert.True(t, IsHighPriority(url.Values{ + // "query": []string{"sum(up)"}, + // "time": []string{strconv.FormatInt(now.UnixMilli(), 10)}, + //}, now, config)) + //assert.True(t, IsHighPriority(url.Values{ + // "query": []string{"count(up)"}, + // "time": []string{strconv.FormatInt(now.UnixMilli(), 10)}, + //}, now, config)) + // + //config = []validation.HighPriorityQuery{ + // { + // Regex: "sum", + // }, + // { + // Regex: "c(.+)t", + // }, + //} + // + //assert.True(t, IsHighPriority(url.Values{ + // "query": []string{"sum(up)"}, + // "time": []string{strconv.FormatInt(now.UnixMilli(), 10)}, + //}, now, config)) + //assert.True(t, IsHighPriority(url.Values{ + // "query": []string{"count(up)"}, + // "time": []string{strconv.FormatInt(now.UnixMilli(), 10)}, + //}, now, config)) + // + //config = []validation.HighPriorityQuery{ + // { + // Regex: "doesnotexist", + // }, + // { + // Regex: "^sum$", + // }, + //} + // + //assert.False(t, IsHighPriority(url.Values{ + // "query": []string{"sum(up)"}, + // "time": []string{strconv.FormatInt(now.UnixMilli(), 10)}, + //}, now, config)) + //assert.False(t, IsHighPriority(url.Values{ + // "query": []string{"count(up)"}, + // "time": []string{strconv.FormatInt(now.UnixMilli(), 10)}, + //}, now, config)) + // + //config = []validation.HighPriorityQuery{ + // { + // Regex: ".*", + // }, + //} + // + //assert.True(t, IsHighPriority(url.Values{ + // "query": []string{"sum(up)"}, + // "time": []string{strconv.FormatInt(now.UnixMilli(), 10)}, + //}, now, config)) + //assert.True(t, IsHighPriority(url.Values{ + // "query": []string{"count(up)"}, + // "time": []string{strconv.FormatInt(now.UnixMilli(), 10)}, + //}, now, config)) + // + //config = []validation.HighPriorityQuery{ + // { + // Regex: "", + // }, + //} + // + //assert.True(t, IsHighPriority(url.Values{ + // "query": []string{"sum(up)"}, + // "time": []string{strconv.FormatInt(now.UnixMilli(), 10)}, + //}, now, config)) + //assert.True(t, IsHighPriority(url.Values{ + // "query": []string{"count(up)"}, + // "time": []string{strconv.FormatInt(now.UnixMilli(), 10)}, + //}, now, config)) } func Test_IsHighPriorityShouldBeBetweenStartAndEndTime(t *testing.T) { - now := time.Now() - config := []validation.HighPriorityQuery{ - { - StartTime: 1 * time.Hour, - EndTime: 30 * time.Minute, - }, - } - - assert.False(t, IsHighPriority(url.Values{ - "query": []string{"sum(up)"}, - "time": []string{strconv.FormatInt(now.Add(-2*time.Hour).UnixMilli(), 10)}, - }, now, config)) - assert.True(t, IsHighPriority(url.Values{ - "query": []string{"sum(up)"}, - "time": []string{strconv.FormatInt(now.Add(-1*time.Hour).UnixMilli(), 10)}, - }, now, config)) - assert.True(t, IsHighPriority(url.Values{ - "query": []string{"sum(up)"}, - "time": []string{strconv.FormatInt(now.Add(-30*time.Minute).UnixMilli(), 10)}, - }, now, config)) - assert.False(t, IsHighPriority(url.Values{ - "query": []string{"sum(up)"}, - "time": []string{strconv.FormatInt(now.Add(-1*time.Minute).UnixMilli(), 10)}, - }, now, config)) - assert.False(t, IsHighPriority(url.Values{ - "query": []string{"sum(up)"}, - "start": []string{strconv.FormatInt(now.Add(-2*time.Hour).UnixMilli(), 10)}, - "end": []string{strconv.FormatInt(now.Add(-30*time.Minute).UnixMilli(), 10)}, - }, now, config)) - assert.True(t, IsHighPriority(url.Values{ - "query": []string{"sum(up)"}, - "start": []string{strconv.FormatInt(now.Add(-1*time.Hour).UnixMilli(), 10)}, - "end": []string{strconv.FormatInt(now.Add(-30*time.Minute).UnixMilli(), 10)}, - }, now, config)) - assert.False(t, IsHighPriority(url.Values{ - "query": []string{"sum(up)"}, - "start": []string{strconv.FormatInt(now.Add(-1*time.Hour).UnixMilli(), 10)}, - "end": []string{strconv.FormatInt(now.UnixMilli(), 10)}, - }, now, config)) + //now := time.Now() + //config := []validation.HighPriorityQuery{ + // { + // StartTime: 1 * time.Hour, + // EndTime: 30 * time.Minute, + // }, + //} + // + //assert.False(t, IsHighPriority(url.Values{ + // "query": []string{"sum(up)"}, + // "time": []string{strconv.FormatInt(now.Add(-2*time.Hour).UnixMilli(), 10)}, + //}, now, config)) + //assert.True(t, IsHighPriority(url.Values{ + // "query": []string{"sum(up)"}, + // "time": []string{strconv.FormatInt(now.Add(-1*time.Hour).UnixMilli(), 10)}, + //}, now, config)) + //assert.True(t, IsHighPriority(url.Values{ + // "query": []string{"sum(up)"}, + // "time": []string{strconv.FormatInt(now.Add(-30*time.Minute).UnixMilli(), 10)}, + //}, now, config)) + //assert.False(t, IsHighPriority(url.Values{ + // "query": []string{"sum(up)"}, + // "time": []string{strconv.FormatInt(now.Add(-1*time.Minute).UnixMilli(), 10)}, + //}, now, config)) + //assert.False(t, IsHighPriority(url.Values{ + // "query": []string{"sum(up)"}, + // "start": []string{strconv.FormatInt(now.Add(-2*time.Hour).UnixMilli(), 10)}, + // "end": []string{strconv.FormatInt(now.Add(-30*time.Minute).UnixMilli(), 10)}, + //}, now, config)) + //assert.True(t, IsHighPriority(url.Values{ + // "query": []string{"sum(up)"}, + // "start": []string{strconv.FormatInt(now.Add(-1*time.Hour).UnixMilli(), 10)}, + // "end": []string{strconv.FormatInt(now.Add(-30*time.Minute).UnixMilli(), 10)}, + //}, now, config)) + //assert.False(t, IsHighPriority(url.Values{ + // "query": []string{"sum(up)"}, + // "start": []string{strconv.FormatInt(now.Add(-1*time.Hour).UnixMilli(), 10)}, + // "end": []string{strconv.FormatInt(now.UnixMilli(), 10)}, + //}, now, config)) } From 488921cdc31233efc54dc2f9cbc90c131c6f4718 Mon Sep 17 00:00:00 2001 From: Justin Jung Date: Wed, 1 Nov 2023 11:30:40 -0700 Subject: [PATCH 19/49] Updated docs Signed-off-by: Justin Jung --- docs/configuration/config-file-reference.md | 35 ++++++++++++++----- .../single-process-config-blocks-local.yaml | 11 ++++++ 2 files changed, 37 insertions(+), 9 deletions(-) diff --git a/docs/configuration/config-file-reference.md b/docs/configuration/config-file-reference.md index 9ae1ddb8d5..29d0d146cf 100644 --- a/docs/configuration/config-file-reference.md +++ b/docs/configuration/config-file-reference.md @@ -3041,14 +3041,17 @@ The `limits_config` configures default and per-tenant limits imposed by Cortex s # CLI flag: -frontend.max-outstanding-requests-per-tenant [max_outstanding_requests_per_tenant: | default = 100] -# Number of reserved queriers to only handle high priority queue (either query -# frontend or query scheduler). If the value is between 0 and 1, it will be used -# as a percentage of per-tenant queriers. -# CLI flag: -frontend.reserved-high-priority-queriers -[reserved_high_priority_queriers: | default = 0] +# Configuration for query priority. +query_priority: + # Whether queries are assigned with priorities. + [enabled: | default = false] + + # Priority assigned to all queries by default. Must be a unique value. Use + # this as a baseline to make certain queries higher/lower priority. + [default_priority: | default = 1] -# List of query definitions to be treated as a high priority. -[high_priority_queries: | default = []] + # List of priority definitions. + [priorities: | default = []] # Duration to delay the evaluation of rules to ensure the underlying metrics # have been pushed to Cortex. @@ -5041,12 +5044,26 @@ otel: [tls_insecure_skip_verify: | default = false] ``` -### `HighPriorityQuery` +### `PriorityDef` + +```yaml +# Priority level. Must be a unique value. +[priority: | default = 2] + +# Number of reserved queriers to handle this priority only. Value between 0 and +# 1 will be used as a percentage. +[reserved_queriers: | default = 0] + +# List of query attributes to assign the priority. +[query_attributes: | default = []] +``` + +### `QueryAttribute` ```yaml # Query string regex. If evaluated true (on top of meeting all other criteria), # query is treated as a high priority. -[regex: | default = ""] +[regex: | default = ".*"] # If query range falls between the start_time and end_time (on top of meeting # all other criteria), query is treated as a high priority. diff --git a/docs/configuration/single-process-config-blocks-local.yaml b/docs/configuration/single-process-config-blocks-local.yaml index a5eb711d97..f37cfec55d 100644 --- a/docs/configuration/single-process-config-blocks-local.yaml +++ b/docs/configuration/single-process-config-blocks-local.yaml @@ -88,3 +88,14 @@ ruler_storage: backend: local local: directory: /tmp/cortex/rules + +limits: + query_priority: + enabled: true + default_priority: 1 + priorities: + - priority: 1 + reserved_queriers: 1 + query_attributes: + - start_time: 2h + end_time: 0s \ No newline at end of file From d8b97996d057c115b6e7647a021dd705ce7bbf84 Mon Sep 17 00:00:00 2001 From: Justin Jung Date: Wed, 1 Nov 2023 11:57:33 -0700 Subject: [PATCH 20/49] Skip regex compile if it is match all Signed-off-by: Justin Jung --- pkg/frontend/transport/roundtripper.go | 4 ---- pkg/scheduler/queue/user_queues.go | 2 +- pkg/util/query/priority.go | 2 +- pkg/util/validation/limits.go | 13 ++++++++----- 4 files changed, 10 insertions(+), 11 deletions(-) diff --git a/pkg/frontend/transport/roundtripper.go b/pkg/frontend/transport/roundtripper.go index aaa6f306d1..a0529ba6a4 100644 --- a/pkg/frontend/transport/roundtripper.go +++ b/pkg/frontend/transport/roundtripper.go @@ -3,11 +3,9 @@ package transport import ( "bytes" "context" - "fmt" "io" "net/http" "net/url" - "regexp" "strings" "time" @@ -39,12 +37,10 @@ func (b *buffer) Bytes() []byte { } func (a *grpcRoundTripperAdapter) RoundTrip(r *http.Request) (*http.Response, error) { - regexp.MustCompile("str") req, err := server.HTTPRequest(r) if err != nil { return nil, err } - fmt.Println(r.URL.Path) var ( resp *httpgrpc.HTTPResponse diff --git a/pkg/scheduler/queue/user_queues.go b/pkg/scheduler/queue/user_queues.go index bb3b6c131b..8107245641 100644 --- a/pkg/scheduler/queue/user_queues.go +++ b/pkg/scheduler/queue/user_queues.go @@ -16,7 +16,7 @@ type Limits interface { // of outstanding requests per tenant per request queue. MaxOutstandingPerTenant(user string) int - // QueryPriority returns query priority config for the tenant, including different priorities, + // QueryPriority returns query priority config for the tenant, including priority level, // their attributes, and how many reserved queriers each priority has. QueryPriority(user string) validation.QueryPriority } diff --git a/pkg/util/query/priority.go b/pkg/util/query/priority.go index ca9a05e3e1..67dd766ede 100644 --- a/pkg/util/query/priority.go +++ b/pkg/util/query/priority.go @@ -25,7 +25,7 @@ func GetPriority(requestParams url.Values, now time.Time, queryPriority validati for _, attribute := range priority.QueryAttributes { compiledRegex := attribute.CompiledRegex - if compiledRegex == nil || !compiledRegex.MatchString(queryParam) { + if compiledRegex != nil && !compiledRegex.MatchString(queryParam) { continue } diff --git a/pkg/util/validation/limits.go b/pkg/util/validation/limits.go index cc8fb9971a..65f848f8c3 100644 --- a/pkg/util/validation/limits.go +++ b/pkg/util/validation/limits.go @@ -48,20 +48,20 @@ type DisabledRuleGroup struct { type DisabledRuleGroups []DisabledRuleGroup type QueryPriority struct { - Enabled bool `yaml:"enabled" doc:"nocli|description=Whether queries are assigned with priorities."` - DefaultPriority int64 `yaml:"default_priority" doc:"nocli|description=Priority assigned to all queries by default. Use this as a baseline to make certain queries higher/lower priority.|default=1"` + Enabled bool `yaml:"enabled" doc:"nocli|description=Whether queries are assigned with priorities.|default=false"` + DefaultPriority int64 `yaml:"default_priority" doc:"nocli|description=Priority assigned to all queries by default. Must be a unique value. Use this as a baseline to make certain queries higher/lower priority.|default=1"` Priorities []PriorityDef `yaml:"priorities" doc:"nocli|description=List of priority definitions."` RegexCompiled bool `yaml:"-" doc:"nocli"` } type PriorityDef struct { - Priority int64 `yaml:"priority" doc:"nocli|description=Priority level."` - ReservedQueriers float64 `yaml:"reserved_queriers" doc:"nocli|description=Number of reserved queriers to handle this priority only. Value between 0 and 1 will be used as a percentage."` + Priority int64 `yaml:"priority" doc:"nocli|description=Priority level. Must be a unique value.|default=2"` + ReservedQueriers float64 `yaml:"reserved_queriers" doc:"nocli|description=Number of reserved queriers to handle this priority only. Value between 0 and 1 will be used as a percentage.|default=0"` QueryAttributes []QueryAttribute `yaml:"query_attributes" doc:"nocli|description=List of query attributes to assign the priority."` } type QueryAttribute struct { - Regex string `yaml:"regex" doc:"nocli|description=Query string regex. If evaluated true (on top of meeting all other criteria), query is treated as a high priority."` + Regex string `yaml:"regex" doc:"nocli|description=Query string regex. If evaluated true (on top of meeting all other criteria), query is treated as a high priority.|default=.*"` CompiledRegex *regexp.Regexp `yaml:"-" doc:"nocli"` StartTime time.Duration `yaml:"start_time" doc:"nocli|description=If query range falls between the start_time and end_time (on top of meeting all other criteria), query is treated as a high priority.|default=0s"` EndTime time.Duration `yaml:"end_time" doc:"nocli|description=If query range falls between the start_time and end_time (on top of meeting all other criteria), query is treated as a high priority.|default=0s"` @@ -520,6 +520,9 @@ func (o *Overrides) QueryPriority(userID string) QueryPriority { priorities := o.GetOverridesForUser(userID).QueryPriority.Priorities for i, priority := range priorities { for j, attributes := range priority.QueryAttributes { + if attributes.Regex == "" || attributes.Regex == ".*" || attributes.Regex == ".+" { + continue // don't use regex at all, if it is match all + } o.GetOverridesForUser(userID).QueryPriority.Priorities[i].QueryAttributes[j].CompiledRegex, _ = regexp.Compile(attributes.Regex) } } From e6b934841c6241ddadd9bb252d1b9d2e46210b93 Mon Sep 17 00:00:00 2001 From: Justin Jung Date: Thu, 2 Nov 2023 18:14:57 -0700 Subject: [PATCH 21/49] Updated GetPriority to handle new config structure Signed-off-by: Justin Jung --- docs/configuration/config-file-reference.md | 4 +- .../single-process-config-blocks-local.yaml | 2 +- pkg/frontend/v1/frontend.go | 5 +- pkg/frontend/v2/frontend.go | 5 +- pkg/scheduler/queue/user_queues.go | 23 +- pkg/util/query/priority.go | 12 +- pkg/util/query/priority_test.go | 319 ++++++++++-------- pkg/util/validation/limits.go | 30 +- pkg/util/validation/limits_test.go | 100 ++++++ 9 files changed, 332 insertions(+), 168 deletions(-) diff --git a/docs/configuration/config-file-reference.md b/docs/configuration/config-file-reference.md index 29d0d146cf..05b2b67103 100644 --- a/docs/configuration/config-file-reference.md +++ b/docs/configuration/config-file-reference.md @@ -3048,7 +3048,7 @@ query_priority: # Priority assigned to all queries by default. Must be a unique value. Use # this as a baseline to make certain queries higher/lower priority. - [default_priority: | default = 1] + [default_priority: | default = 0] # List of priority definitions. [priorities: | default = []] @@ -5048,7 +5048,7 @@ otel: ```yaml # Priority level. Must be a unique value. -[priority: | default = 2] +[priority: | default = 0] # Number of reserved queriers to handle this priority only. Value between 0 and # 1 will be used as a percentage. diff --git a/docs/configuration/single-process-config-blocks-local.yaml b/docs/configuration/single-process-config-blocks-local.yaml index f37cfec55d..c04e9cc07c 100644 --- a/docs/configuration/single-process-config-blocks-local.yaml +++ b/docs/configuration/single-process-config-blocks-local.yaml @@ -94,7 +94,7 @@ limits: enabled: true default_priority: 1 priorities: - - priority: 1 + - priority: 2 reserved_queriers: 1 query_attributes: - start_time: 2h diff --git a/pkg/frontend/v1/frontend.go b/pkg/frontend/v1/frontend.go index ac1911e3e9..8d7bb4145d 100644 --- a/pkg/frontend/v1/frontend.go +++ b/pkg/frontend/v1/frontend.go @@ -206,8 +206,9 @@ func (f *Frontend) RoundTripGRPC(ctx context.Context, req *httpgrpc.HTTPRequest, response: make(chan *httpgrpc.HTTPResponse, 1), } - if reqParams != nil { - request.priority = util_query.GetPriority(reqParams, ts, f.limits.QueryPriority(userID)) + queryPriority := f.limits.QueryPriority(userID) + if reqParams != nil && queryPriority.Enabled { + request.priority = util_query.GetPriority(reqParams, ts, queryPriority) } if err := f.queueRequest(ctx, &request); err != nil { diff --git a/pkg/frontend/v2/frontend.go b/pkg/frontend/v2/frontend.go index 81d0e8c802..e06f3f97c1 100644 --- a/pkg/frontend/v2/frontend.go +++ b/pkg/frontend/v2/frontend.go @@ -210,8 +210,9 @@ func (f *Frontend) RoundTripGRPC(ctx context.Context, req *httpgrpc.HTTPRequest, retryOnTooManyOutstandingRequests: f.cfg.RetryOnTooManyOutstandingRequests && f.schedulerWorkers.getWorkersCount() > 1, } - if reqParams != nil { - freq.priority = util_query.GetPriority(reqParams, ts, f.limits.QueryPriority(userID)) + queryPriority := f.limits.QueryPriority(userID) + if reqParams != nil && queryPriority.Enabled { + freq.priority = util_query.GetPriority(reqParams, ts, queryPriority) } f.requests.put(freq) diff --git a/pkg/scheduler/queue/user_queues.go b/pkg/scheduler/queue/user_queues.go index 8107245641..86031c506e 100644 --- a/pkg/scheduler/queue/user_queues.go +++ b/pkg/scheduler/queue/user_queues.go @@ -59,8 +59,7 @@ type queues struct { } type userQueue struct { - normalQueue chan Request - highPriorityQueue chan Request + normalQueue chan Request // If not nil, only these queriers can handle user requests. If nil, all queriers can. // We set this to nil if number of available queriers <= maxQueriers. @@ -77,6 +76,7 @@ type userQueue struct { } func newUserQueues(maxUserQueueSize int, forgetDelay time.Duration, limits Limits) *queues { + // Create schedule per user return &queues{ userQueues: map[string]*userQueue{}, users: nil, @@ -131,10 +131,9 @@ func (q *queues) getOrAddQueue(userID string, maxQueriers int, priority int64) c queueSize = q.maxUserQueueSize } uq = &userQueue{ - normalQueue: make(chan Request, queueSize), - highPriorityQueue: make(chan Request, queueSize), - seed: util.ShuffleShardSeed(userID, ""), - index: -1, + normalQueue: make(chan Request, queueSize), + seed: util.ShuffleShardSeed(userID, ""), + index: -1, } q.userQueues[userID] = uq @@ -167,7 +166,8 @@ func (q *queues) getOrAddQueue(userID string, maxQueriers int, priority int64) c } func (q *queues) getTotalQueueSize(userID string) int { - return len(q.userQueues[userID].normalQueue) + len(q.userQueues[userID].highPriorityQueue) + // TODO: Implement + return len(q.userQueues[userID].normalQueue) } // Finds next queue for the querier. To support fair scheduling between users, client is expected @@ -199,10 +199,11 @@ func (q *queues) getNextQueueForQuerier(lastUserIndex int, querierID string) (ch } } - _, isReserved := uq.reservedQueriers[querierID] - if isReserved || len(uq.highPriorityQueue) > 0 { - return uq.highPriorityQueue, u, uid - } + // TODO: Implement + //_, isReserved := uq.reservedQueriers[querierID] + //if isReserved || len(uq.highPriorityQueue) > 0 { + // return uq.highPriorityQueue, u, uid + //} return uq.normalQueue, u, uid } diff --git a/pkg/util/query/priority.go b/pkg/util/query/priority.go index 67dd766ede..e37dd8f0c9 100644 --- a/pkg/util/query/priority.go +++ b/pkg/util/query/priority.go @@ -18,19 +18,17 @@ func GetPriority(requestParams url.Values, now time.Time, queryPriority validati endParam := requestParams.Get("end") if queryParam == "" || !queryPriority.Enabled { - return -1 + return queryPriority.DefaultPriority } for _, priority := range queryPriority.Priorities { for _, attribute := range priority.QueryAttributes { - compiledRegex := attribute.CompiledRegex - - if compiledRegex != nil && !compiledRegex.MatchString(queryParam) { + if attribute.CompiledRegex != nil && !attribute.CompiledRegex.MatchString(queryParam) { continue } - startTimeThreshold := now.Add(-1 * attribute.StartTime.Abs()) - endTimeThreshold := now.Add(-1 * attribute.EndTime.Abs()) + startTimeThreshold := now.Add(-1 * attribute.StartTime.Abs()).Truncate(time.Second).UTC() + endTimeThreshold := now.Add(-1 * attribute.EndTime.Abs()).Round(time.Second).UTC() if instantTime, err := parseTime(timeParam); err == nil { if isBetweenThresholds(instantTime, instantTime, startTimeThreshold, endTimeThreshold) { @@ -67,5 +65,5 @@ func parseTime(s string) (time.Time, error) { } func isBetweenThresholds(start, end, startThreshold, endThreshold time.Time) bool { - return start.After(startThreshold) && end.Before(endThreshold) + return (start.Equal(startThreshold) || start.After(startThreshold)) && (end.Equal(endThreshold) || end.Before(endThreshold)) } diff --git a/pkg/util/query/priority_test.go b/pkg/util/query/priority_test.go index 8782177912..696fb33af2 100644 --- a/pkg/util/query/priority_test.go +++ b/pkg/util/query/priority_test.go @@ -1,146 +1,191 @@ package query import ( + "github.com/cortexproject/cortex/pkg/util/validation" + "github.com/stretchr/testify/assert" + "net/url" + "strconv" "testing" + "time" ) -func Test_IsHighPriorityShouldMatchRegex(t *testing.T) { - //now := time.Now() - //config := []validation.HighPriorityQuery{ - // { - // Regex: "sum", - // }, - //} - // - //assert.True(t, IsHighPriority(url.Values{ - // "query": []string{"sum(up)"}, - // "time": []string{strconv.FormatInt(now.UnixMilli(), 10)}, - //}, now, config)) - //assert.False(t, IsHighPriority(url.Values{ - // "query": []string{"count(up)"}, - // "time": []string{strconv.FormatInt(now.UnixMilli(), 10)}, - //}, now, config)) - // - //config = []validation.HighPriorityQuery{ - // { - // Regex: "up", - // }, - //} - // - //assert.True(t, IsHighPriority(url.Values{ - // "query": []string{"sum(up)"}, - // "time": []string{strconv.FormatInt(now.UnixMilli(), 10)}, - //}, now, config)) - //assert.True(t, IsHighPriority(url.Values{ - // "query": []string{"count(up)"}, - // "time": []string{strconv.FormatInt(now.UnixMilli(), 10)}, - //}, now, config)) - // - //config = []validation.HighPriorityQuery{ - // { - // Regex: "sum", - // }, - // { - // Regex: "c(.+)t", - // }, - //} - // - //assert.True(t, IsHighPriority(url.Values{ - // "query": []string{"sum(up)"}, - // "time": []string{strconv.FormatInt(now.UnixMilli(), 10)}, - //}, now, config)) - //assert.True(t, IsHighPriority(url.Values{ - // "query": []string{"count(up)"}, - // "time": []string{strconv.FormatInt(now.UnixMilli(), 10)}, - //}, now, config)) - // - //config = []validation.HighPriorityQuery{ - // { - // Regex: "doesnotexist", - // }, - // { - // Regex: "^sum$", - // }, - //} - // - //assert.False(t, IsHighPriority(url.Values{ - // "query": []string{"sum(up)"}, - // "time": []string{strconv.FormatInt(now.UnixMilli(), 10)}, - //}, now, config)) - //assert.False(t, IsHighPriority(url.Values{ - // "query": []string{"count(up)"}, - // "time": []string{strconv.FormatInt(now.UnixMilli(), 10)}, - //}, now, config)) - // - //config = []validation.HighPriorityQuery{ - // { - // Regex: ".*", - // }, - //} - // - //assert.True(t, IsHighPriority(url.Values{ - // "query": []string{"sum(up)"}, - // "time": []string{strconv.FormatInt(now.UnixMilli(), 10)}, - //}, now, config)) - //assert.True(t, IsHighPriority(url.Values{ - // "query": []string{"count(up)"}, - // "time": []string{strconv.FormatInt(now.UnixMilli(), 10)}, - //}, now, config)) - // - //config = []validation.HighPriorityQuery{ - // { - // Regex: "", - // }, - //} - // - //assert.True(t, IsHighPriority(url.Values{ - // "query": []string{"sum(up)"}, - // "time": []string{strconv.FormatInt(now.UnixMilli(), 10)}, - //}, now, config)) - //assert.True(t, IsHighPriority(url.Values{ - // "query": []string{"count(up)"}, - // "time": []string{strconv.FormatInt(now.UnixMilli(), 10)}, - //}, now, config)) +func Test_GetPriorityShouldReturnDefaultPriorityIfNotEnabledOrEmptyQueryString(t *testing.T) { + now := time.Now() + priorities := []validation.PriorityDef{ + { + Priority: 1, + QueryAttributes: []validation.QueryAttribute{ + { + Regex: ".*", + StartTime: 2 * time.Hour, + EndTime: 0 * time.Hour, + }, + }, + }, + } + + assert.Equal(t, int64(0), GetPriority(url.Values{ + "query": []string{"sum(up)"}, + "time": []string{strconv.FormatInt(now.Unix(), 10)}, + }, now, validation.QueryPriority{ + Priorities: priorities, + })) + + assert.Equal(t, int64(0), GetPriority(url.Values{ + "query": []string{""}, + "time": []string{strconv.FormatInt(now.Unix(), 10)}, + }, now, validation.QueryPriority{ + Enabled: true, + Priorities: priorities, + })) } -func Test_IsHighPriorityShouldBeBetweenStartAndEndTime(t *testing.T) { - //now := time.Now() - //config := []validation.HighPriorityQuery{ - // { - // StartTime: 1 * time.Hour, - // EndTime: 30 * time.Minute, - // }, - //} - // - //assert.False(t, IsHighPriority(url.Values{ - // "query": []string{"sum(up)"}, - // "time": []string{strconv.FormatInt(now.Add(-2*time.Hour).UnixMilli(), 10)}, - //}, now, config)) - //assert.True(t, IsHighPriority(url.Values{ - // "query": []string{"sum(up)"}, - // "time": []string{strconv.FormatInt(now.Add(-1*time.Hour).UnixMilli(), 10)}, - //}, now, config)) - //assert.True(t, IsHighPriority(url.Values{ - // "query": []string{"sum(up)"}, - // "time": []string{strconv.FormatInt(now.Add(-30*time.Minute).UnixMilli(), 10)}, - //}, now, config)) - //assert.False(t, IsHighPriority(url.Values{ - // "query": []string{"sum(up)"}, - // "time": []string{strconv.FormatInt(now.Add(-1*time.Minute).UnixMilli(), 10)}, - //}, now, config)) - //assert.False(t, IsHighPriority(url.Values{ - // "query": []string{"sum(up)"}, - // "start": []string{strconv.FormatInt(now.Add(-2*time.Hour).UnixMilli(), 10)}, - // "end": []string{strconv.FormatInt(now.Add(-30*time.Minute).UnixMilli(), 10)}, - //}, now, config)) - //assert.True(t, IsHighPriority(url.Values{ - // "query": []string{"sum(up)"}, - // "start": []string{strconv.FormatInt(now.Add(-1*time.Hour).UnixMilli(), 10)}, - // "end": []string{strconv.FormatInt(now.Add(-30*time.Minute).UnixMilli(), 10)}, - //}, now, config)) - //assert.False(t, IsHighPriority(url.Values{ - // "query": []string{"sum(up)"}, - // "start": []string{strconv.FormatInt(now.Add(-1*time.Hour).UnixMilli(), 10)}, - // "end": []string{strconv.FormatInt(now.UnixMilli(), 10)}, - //}, now, config)) +func Test_GetPriorityShouldConsiderRegex(t *testing.T) { + now := time.Now() + priorities := []validation.PriorityDef{ + { + Priority: 1, + QueryAttributes: []validation.QueryAttribute{ + { + Regex: "sum", + StartTime: 2 * time.Hour, + EndTime: 0 * time.Hour, + }, + }, + }, + } + limits, _ := validation.NewOverrides(validation.Limits{ + QueryPriority: validation.QueryPriority{ + Enabled: true, + Priorities: priorities, + }, + }, nil) + + assert.Equal(t, int64(1), GetPriority(url.Values{ + "query": []string{"sum(up)"}, + "time": []string{strconv.FormatInt(now.Unix(), 10)}, + }, now, limits.QueryPriority(""))) + assert.Equal(t, int64(0), GetPriority(url.Values{ + "query": []string{"count(up)"}, + "time": []string{strconv.FormatInt(now.Unix(), 10)}, + }, now, limits.QueryPriority(""))) + + priorities[0].QueryAttributes[0].Regex = "(^sum$|c(.+)t)" + limits, _ = validation.NewOverrides(validation.Limits{ + QueryPriority: validation.QueryPriority{ + Enabled: true, + Priorities: priorities, + }, + }, nil) + + assert.Equal(t, int64(0), GetPriority(url.Values{ + "query": []string{"sum(up)"}, + "time": []string{strconv.FormatInt(now.Unix(), 10)}, + }, now, limits.QueryPriority(""))) + assert.Equal(t, int64(1), GetPriority(url.Values{ + "query": []string{"count(up)"}, + "time": []string{strconv.FormatInt(now.Unix(), 10)}, + }, now, limits.QueryPriority(""))) + + priorities[0].QueryAttributes[0].Regex = ".*" + limits, _ = validation.NewOverrides(validation.Limits{ + QueryPriority: validation.QueryPriority{ + Enabled: true, + Priorities: priorities, + }, + }, nil) + + assert.Equal(t, int64(1), GetPriority(url.Values{ + "query": []string{"sum(up)"}, + "time": []string{strconv.FormatInt(now.Unix(), 10)}, + }, now, limits.QueryPriority(""))) + assert.Equal(t, int64(1), GetPriority(url.Values{ + "query": []string{"count(up)"}, + "time": []string{strconv.FormatInt(now.Unix(), 10)}, + }, now, limits.QueryPriority(""))) + + priorities[0].QueryAttributes[0].Regex = "" + limits, _ = validation.NewOverrides(validation.Limits{ + QueryPriority: validation.QueryPriority{ + Enabled: true, + Priorities: priorities, + }, + }, nil) + + assert.Equal(t, int64(1), GetPriority(url.Values{ + "query": []string{"sum(up)"}, + "time": []string{strconv.FormatInt(now.Unix(), 10)}, + }, now, limits.QueryPriority(""))) + assert.Equal(t, int64(1), GetPriority(url.Values{ + "query": []string{"count(up)"}, + "time": []string{strconv.FormatInt(now.Unix(), 10)}, + }, now, limits.QueryPriority(""))) +} + +func Test_GetPriorityShouldConsiderStartAndEndTime(t *testing.T) { + now := time.Now() + priorities := []validation.PriorityDef{ + { + Priority: 1, + QueryAttributes: []validation.QueryAttribute{ + { + StartTime: 45 * time.Minute, + EndTime: 15 * time.Minute, + }, + }, + }, + } + limits, _ := validation.NewOverrides(validation.Limits{ + QueryPriority: validation.QueryPriority{ + Enabled: true, + Priorities: priorities, + }, + }, nil) + + assert.Equal(t, int64(0), GetPriority(url.Values{ + "query": []string{"sum(up)"}, + "time": []string{strconv.FormatInt(now.Unix(), 10)}, + }, now, limits.QueryPriority(""))) + assert.Equal(t, int64(1), GetPriority(url.Values{ + "query": []string{"sum(up)"}, + "time": []string{strconv.FormatInt(now.Add(-30*time.Minute).Unix(), 10)}, + }, now, limits.QueryPriority(""))) + assert.Equal(t, int64(0), GetPriority(url.Values{ + "query": []string{"sum(up)"}, + "time": []string{strconv.FormatInt(now.Add(-60*time.Minute).Unix(), 10)}, + }, now, limits.QueryPriority(""))) + assert.Equal(t, int64(1), GetPriority(url.Values{ + "query": []string{"sum(up)"}, + "time": []string{strconv.FormatInt(now.Add(-45*time.Minute).Unix(), 10)}, + }, now, limits.QueryPriority(""))) + assert.Equal(t, int64(1), GetPriority(url.Values{ + "query": []string{"sum(up)"}, + "time": []string{strconv.FormatInt(now.Add(-15*time.Minute).Unix(), 10)}, + }, now, limits.QueryPriority(""))) + + assert.Equal(t, int64(1), GetPriority(url.Values{ + "query": []string{"sum(up)"}, + "start": []string{strconv.FormatInt(now.Add(-45*time.Minute).Unix(), 10)}, + "end": []string{strconv.FormatInt(now.Add(-15*time.Minute).Unix(), 10)}, + }, now, limits.QueryPriority(""))) + assert.Equal(t, int64(0), GetPriority(url.Values{ + "query": []string{"sum(up)"}, + "start": []string{strconv.FormatInt(now.Add(-50*time.Minute).Unix(), 10)}, + "end": []string{strconv.FormatInt(now.Add(-15*time.Minute).Unix(), 10)}, + }, now, limits.QueryPriority(""))) + assert.Equal(t, int64(0), GetPriority(url.Values{ + "query": []string{"sum(up)"}, + "start": []string{strconv.FormatInt(now.Add(-45*time.Minute).Unix(), 10)}, + "end": []string{strconv.FormatInt(now.Add(-10*time.Minute).Unix(), 10)}, + }, now, limits.QueryPriority(""))) + assert.Equal(t, int64(0), GetPriority(url.Values{ + "query": []string{"sum(up)"}, + "start": []string{strconv.FormatInt(now.Add(-60*time.Minute).Unix(), 10)}, + "end": []string{strconv.FormatInt(now.Add(-45*time.Minute).Unix(), 10)}, + }, now, limits.QueryPriority(""))) + assert.Equal(t, int64(0), GetPriority(url.Values{ + "query": []string{"sum(up)"}, + "start": []string{strconv.FormatInt(now.Add(-15*time.Minute).Unix(), 10)}, + "end": []string{strconv.FormatInt(now.Add(-1*time.Minute).Unix(), 10)}, + }, now, limits.QueryPriority(""))) } diff --git a/pkg/util/validation/limits.go b/pkg/util/validation/limits.go index 65f848f8c3..10db7b9bc9 100644 --- a/pkg/util/validation/limits.go +++ b/pkg/util/validation/limits.go @@ -18,6 +18,7 @@ import ( ) var errMaxGlobalSeriesPerUserValidation = errors.New("The ingester.max-global-series-per-user limit is unsupported if distributor.shard-by-all-labels is disabled") +var errDuplicateQueryPriorities = errors.New("There is a duplicate entry of priorities. Make sure they are all unique, including the default priority") // Supported values for enum limits const ( @@ -49,13 +50,13 @@ type DisabledRuleGroups []DisabledRuleGroup type QueryPriority struct { Enabled bool `yaml:"enabled" doc:"nocli|description=Whether queries are assigned with priorities.|default=false"` - DefaultPriority int64 `yaml:"default_priority" doc:"nocli|description=Priority assigned to all queries by default. Must be a unique value. Use this as a baseline to make certain queries higher/lower priority.|default=1"` + DefaultPriority int64 `yaml:"default_priority" doc:"nocli|description=Priority assigned to all queries by default. Must be a unique value. Use this as a baseline to make certain queries higher/lower priority.|default=0"` Priorities []PriorityDef `yaml:"priorities" doc:"nocli|description=List of priority definitions."` RegexCompiled bool `yaml:"-" doc:"nocli"` } type PriorityDef struct { - Priority int64 `yaml:"priority" doc:"nocli|description=Priority level. Must be a unique value.|default=2"` + Priority int64 `yaml:"priority" doc:"nocli|description=Priority level. Must be a unique value.|default=0"` ReservedQueriers float64 `yaml:"reserved_queriers" doc:"nocli|description=Number of reserved queriers to handle this priority only. Value between 0 and 1 will be used as a percentage.|default=0"` QueryAttributes []QueryAttribute `yaml:"query_attributes" doc:"nocli|description=List of query attributes to assign the priority."` } @@ -250,6 +251,19 @@ func (l *Limits) Validate(shardByAllLabels bool) error { return errMaxGlobalSeriesPerUserValidation } + if l.QueryPriority.Enabled { + queryPriority := l.QueryPriority + prioritySet := map[int64]struct{}{} + prioritySet[queryPriority.DefaultPriority] = struct{}{} + for _, priority := range queryPriority.Priorities { + if _, exists := prioritySet[priority.Priority]; exists { + return errDuplicateQueryPriorities + } + + prioritySet[priority.Priority] = struct{}{} + } + } + return nil } @@ -516,14 +530,18 @@ func (o *Overrides) MaxOutstandingPerTenant(userID string) int { // QueryPriority returns the query priority config for the tenant, including different priorities and their attributes func (o *Overrides) QueryPriority(userID string) QueryPriority { - if !o.GetOverridesForUser(userID).QueryPriority.RegexCompiled { - priorities := o.GetOverridesForUser(userID).QueryPriority.Priorities + queryPriority := o.GetOverridesForUser(userID).QueryPriority + + if !queryPriority.RegexCompiled { + priorities := queryPriority.Priorities for i, priority := range priorities { for j, attributes := range priority.QueryAttributes { if attributes.Regex == "" || attributes.Regex == ".*" || attributes.Regex == ".+" { - continue // don't use regex at all, if it is match all + // if it matches all, don't use regex + o.GetOverridesForUser(userID).QueryPriority.Priorities[i].QueryAttributes[j].CompiledRegex = nil + } else { + o.GetOverridesForUser(userID).QueryPriority.Priorities[i].QueryAttributes[j].CompiledRegex, _ = regexp.Compile(attributes.Regex) } - o.GetOverridesForUser(userID).QueryPriority.Priorities[i].QueryAttributes[j].CompiledRegex, _ = regexp.Compile(attributes.Regex) } } o.GetOverridesForUser(userID).QueryPriority.RegexCompiled = true diff --git a/pkg/util/validation/limits_test.go b/pkg/util/validation/limits_test.go index 2631354893..f2de1300e6 100644 --- a/pkg/util/validation/limits_test.go +++ b/pkg/util/validation/limits_test.go @@ -61,6 +61,47 @@ func TestLimits_Validate(t *testing.T) { shardByAllLabels: true, expected: nil, }, + "duplicate priority entries": { + limits: Limits{QueryPriority: QueryPriority{ + Enabled: true, + Priorities: []PriorityDef{ + { + Priority: 1, + }, + { + Priority: 1, + }, + }, + }}, + expected: errDuplicateQueryPriorities, + }, + "priority matches default priority": { + limits: Limits{QueryPriority: QueryPriority{ + Enabled: true, + DefaultPriority: 1, + Priorities: []PriorityDef{ + { + Priority: 1, + }, + }, + }}, + expected: errDuplicateQueryPriorities, + }, + "all priorities are unique": { + limits: Limits{QueryPriority: QueryPriority{ + Enabled: true, + DefaultPriority: 1, + Priorities: []PriorityDef{ + { + Priority: 2, + }, + { + Priority: 3, + }, + }, + }}, + expected: nil, + }, } for testName, testData := range tests { @@ -113,6 +154,65 @@ func TestOverridesManager_GetOverrides(t *testing.T) { require.Equal(t, 0, ov.MaxLabelsSizeBytes("user2")) } +func TestQueryPriority(t *testing.T) { + type testCase struct { + regex string + compiledRegexNil bool + } + testCases := []testCase{ + { + regex: "", + compiledRegexNil: true, + }, + { + regex: ".*", + compiledRegexNil: true, + }, + { + regex: ".+", + compiledRegexNil: true, + }, + { + regex: "some_metric", + compiledRegexNil: false, + }, + } + + for _, tc := range testCases { + overrides, err := NewOverrides(Limits{ + QueryPriority: QueryPriority{ + Enabled: true, + Priorities: getPriorities(tc.regex, 1*time.Hour, 0*time.Hour), + }, + }, nil) + queryPriority := overrides.QueryPriority("") + + assert.NoError(t, err) + assert.Equal(t, 1, len(queryPriority.Priorities[0].QueryAttributes)) + if tc.compiledRegexNil { + assert.Nil(t, queryPriority.Priorities[0].QueryAttributes[0].CompiledRegex) + } else { + assert.NotNil(t, queryPriority.Priorities[0].QueryAttributes[0].CompiledRegex) + } + assert.True(t, queryPriority.RegexCompiled) + } +} + +func getPriorities(regex string, startTime, endTime time.Duration) []PriorityDef { + return []PriorityDef{ + { + Priority: 1, + QueryAttributes: []QueryAttribute{ + { + Regex: regex, + StartTime: startTime, + EndTime: endTime, + }, + }, + }, + } +} + func TestLimitsLoadingFromYaml(t *testing.T) { SetDefaultLimitsForYAMLUnmarshalling(Limits{ MaxLabelNameLength: 100, From 5eb2d730647d135c78e1d86d4648f2a2bb6fc675 Mon Sep 17 00:00:00 2001 From: Justin Jung Date: Tue, 7 Nov 2023 12:19:26 -0800 Subject: [PATCH 22/49] Add priority queue to user queue Signed-off-by: Justin Jung --- pkg/frontend/v1/frontend.go | 2 +- pkg/scheduler/queue/queue.go | 15 +++----- pkg/scheduler/queue/request_queue.go | 49 ++++++++++++++++++++++++ pkg/scheduler/queue/user_queues.go | 57 +++++++++++++++++----------- pkg/scheduler/scheduler.go | 2 +- pkg/util/priority_queue.go | 29 ++++---------- 6 files changed, 99 insertions(+), 55 deletions(-) create mode 100644 pkg/scheduler/queue/request_queue.go diff --git a/pkg/frontend/v1/frontend.go b/pkg/frontend/v1/frontend.go index 8d7bb4145d..d3734e9bcf 100644 --- a/pkg/frontend/v1/frontend.go +++ b/pkg/frontend/v1/frontend.go @@ -101,7 +101,7 @@ type request struct { priority int64 } -func (r request) GetPriority() int64 { +func (r request) Priority() int64 { return r.priority } diff --git a/pkg/scheduler/queue/queue.go b/pkg/scheduler/queue/queue.go index cf20a3573a..a2e44cba83 100644 --- a/pkg/scheduler/queue/queue.go +++ b/pkg/scheduler/queue/queue.go @@ -45,7 +45,7 @@ func FirstUser() UserIndex { // Request stored into the queue. type Request interface { - GetPriority() int64 + util.PriorityOp } // RequestQueue holds incoming requests in per-user queues. It also assigns each user specified number of queriers, @@ -98,7 +98,7 @@ func (q *RequestQueue) EnqueueRequest(userID string, req Request, maxQueriers fl } shardSize := util.DynamicShardSize(maxQueriers, len(q.queues.queriers)) - queue := q.queues.getOrAddQueue(userID, shardSize, req.GetPriority()) + queue := q.queues.getOrAddQueue(userID, shardSize) maxOutstandingRequests := q.queues.limits.MaxOutstandingPerTenant(userID) if queue == nil { @@ -108,12 +108,12 @@ func (q *RequestQueue) EnqueueRequest(userID string, req Request, maxQueriers fl q.totalRequests.WithLabelValues(userID).Inc() - if q.queues.getTotalQueueSize(userID) >= maxOutstandingRequests { + if queue.length() >= maxOutstandingRequests { q.discardedRequests.WithLabelValues(userID).Inc() return ErrTooManyRequests } - queue <- req + queue.enqueueRequest(req) q.queueLength.WithLabelValues(userID).Inc() q.cond.Broadcast() // Call this function while holding a lock. This guarantees that no querier can fetch the request before function returns. @@ -150,16 +150,13 @@ FindQueue: for { queue, userID, idx := q.queues.getNextQueueForQuerier(last.last, querierID) last.last = idx - if queue == nil || len(queue) < 1 { + if queue == nil { break } // Pick next request from the queue. for { - request := <-queue - if q.queues.getTotalQueueSize(userID) == 0 { - q.queues.deleteQueue(userID) - } + request := queue.dequeueRequest() q.queueLength.WithLabelValues(userID).Dec() diff --git a/pkg/scheduler/queue/request_queue.go b/pkg/scheduler/queue/request_queue.go new file mode 100644 index 0000000000..66c66e905e --- /dev/null +++ b/pkg/scheduler/queue/request_queue.go @@ -0,0 +1,49 @@ +package queue + +import "github.com/cortexproject/cortex/pkg/util" + +type requestQueue interface { + enqueueRequest(Request) + dequeueRequest() Request + length() int +} + +type FIFORequestQueue struct { + queue chan Request +} + +func NewFIFOQueue(queue chan Request) *FIFORequestQueue { + return &FIFORequestQueue{queue: queue} +} + +func (f *FIFORequestQueue) enqueueRequest(r Request) { + f.queue <- r +} + +func (f *FIFORequestQueue) dequeueRequest() Request { + return <-f.queue +} + +func (f *FIFORequestQueue) length() int { + return len(f.queue) +} + +type PriorityRequestQueue struct { + queue *util.PriorityQueue +} + +func NewPriorityRequestQueue(queue *util.PriorityQueue) *PriorityRequestQueue { + return &PriorityRequestQueue{queue: queue} +} + +func (f *PriorityRequestQueue) enqueueRequest(r Request) { + f.queue.Enqueue(r) +} + +func (f *PriorityRequestQueue) dequeueRequest() Request { + return f.queue.Dequeue() +} + +func (f *PriorityRequestQueue) length() int { + return f.queue.Length() +} diff --git a/pkg/scheduler/queue/user_queues.go b/pkg/scheduler/queue/user_queues.go index 86031c506e..e20bc0f905 100644 --- a/pkg/scheduler/queue/user_queues.go +++ b/pkg/scheduler/queue/user_queues.go @@ -59,12 +59,12 @@ type queues struct { } type userQueue struct { - normalQueue chan Request + queue requestQueue // If not nil, only these queriers can handle user requests. If nil, all queriers can. // We set this to nil if number of available queriers <= maxQueriers. queriers map[string]struct{} - reservedQueriers map[string]struct{} + reservedQueriers map[string]int64 maxQueriers int // Seed for shuffle sharding of queriers. This seed is based on userID only and is therefore consistent @@ -76,7 +76,6 @@ type userQueue struct { } func newUserQueues(maxUserQueueSize int, forgetDelay time.Duration, limits Limits) *queues { - // Create schedule per user return &queues{ userQueues: map[string]*userQueue{}, users: nil, @@ -111,7 +110,7 @@ func (q *queues) deleteQueue(userID string) { // MaxQueriers is used to compute which queriers should handle requests for this user. // If maxQueriers is <= 0, all queriers can handle this user's requests. // If maxQueriers has changed since the last call, queriers for this are recomputed. -func (q *queues) getOrAddQueue(userID string, maxQueriers int, priority int64) chan Request { +func (q *queues) getOrAddQueue(userID string, maxQueriers int) requestQueue { // Empty user is not allowed, as that would break our users list ("" is used for free spot). if userID == "" { return nil @@ -122,6 +121,7 @@ func (q *queues) getOrAddQueue(userID string, maxQueriers int, priority int64) c } uq := q.userQueues[userID] + queryPriority := q.limits.QueryPriority(userID) if uq == nil { queueSize := q.limits.MaxOutstandingPerTenant(userID) @@ -131,10 +131,16 @@ func (q *queues) getOrAddQueue(userID string, maxQueriers int, priority int64) c queueSize = q.maxUserQueueSize } uq = &userQueue{ - normalQueue: make(chan Request, queueSize), - seed: util.ShuffleShardSeed(userID, ""), - index: -1, + seed: util.ShuffleShardSeed(userID, ""), + index: -1, } + + if queryPriority.Enabled { + uq.queue = NewPriorityRequestQueue(util.NewPriorityQueue(nil)) + } else { + uq.queue = NewFIFOQueue(make(chan Request, queueSize)) + } + q.userQueues[userID] = uq // Add user to the list of users... find first free spot, and put it there. @@ -156,24 +162,16 @@ func (q *queues) getOrAddQueue(userID string, maxQueriers int, priority int64) c if uq.maxQueriers != maxQueriers { uq.maxQueriers = maxQueriers uq.queriers = shuffleQueriersForUser(uq.seed, maxQueriers, q.sortedQueriers, nil) + uq.reservedQueriers = getReservedQueriers(uq.queriers, q.limits.QueryPriority(userID).Priorities) } - //if highPriority { - // return uq.highPriorityQueue - //} - - return uq.normalQueue -} - -func (q *queues) getTotalQueueSize(userID string) int { - // TODO: Implement - return len(q.userQueues[userID].normalQueue) + return uq.queue } // Finds next queue for the querier. To support fair scheduling between users, client is expected // to pass last user index returned by this function as argument. Is there was no previous // last user index, use -1. -func (q *queues) getNextQueueForQuerier(lastUserIndex int, querierID string) (chan Request, string, int) { +func (q *queues) getNextQueueForQuerier(lastUserIndex int, querierID string) (requestQueue, string, int) { uid := lastUserIndex for iters := 0; iters < len(q.users); iters++ { @@ -199,13 +197,12 @@ func (q *queues) getNextQueueForQuerier(lastUserIndex int, querierID string) (ch } } - // TODO: Implement - //_, isReserved := uq.reservedQueriers[querierID] - //if isReserved || len(uq.highPriorityQueue) > 0 { - // return uq.highPriorityQueue, u, uid + //TODO: jungjust, reserved queriers + //if priority, isReserved := uq.reservedQueriers[querierID]; isReserved { + // return uq.queues[priority], u, uid //} - return uq.normalQueue, u, uid + return uq.queue, u, uid } return nil, "", uid } @@ -353,6 +350,20 @@ func shuffleQueriersForUser(userSeed int64, queriersToSelect int, allSortedQueri return queriers } +func getReservedQueriers(queriers map[string]struct{}, priorities []validation.PriorityDef) map[string]int64 { + // TODO jungjust: + //reservedQueriers := make(map[string]int64) + // + //cnt := 0 + //for querierID, _ := range queriers { + // + // //reservedQueriers + // // TODO jungjust: get diff priorities + // cnt++ + //} + return map[string]int64{} +} + //func getNumOfReservedQueriers(queriersToSelect int, totalNumOfQueriers int, reservedQueriers float64) int { // numOfReservedQueriers := int(reservedQueriers) // diff --git a/pkg/scheduler/scheduler.go b/pkg/scheduler/scheduler.go index d4f7ac8541..4c3a0afcf8 100644 --- a/pkg/scheduler/scheduler.go +++ b/pkg/scheduler/scheduler.go @@ -166,7 +166,7 @@ type schedulerRequest struct { parentSpanContext opentracing.SpanContext } -func (s schedulerRequest) GetPriority() int64 { +func (s schedulerRequest) Priority() int64 { return s.priority } diff --git a/pkg/util/priority_queue.go b/pkg/util/priority_queue.go index 1c2fbadd3b..4cbc86e3ff 100644 --- a/pkg/util/priority_queue.go +++ b/pkg/util/priority_queue.go @@ -13,18 +13,16 @@ type PriorityQueue struct { cond *sync.Cond closing bool closed bool - hit map[string]struct{} queue queue lengthGauge prometheus.Gauge } -// Op is an operation on the priority queue. -type Op interface { - Key() string +// PriorityOp is an operation on the priority queue. +type PriorityOp interface { Priority() int64 // The larger the number the higher the priority. } -type queue []Op +type queue []PriorityOp func (q queue) Len() int { return len(q) } func (q queue) Less(i, j int) bool { return q[i].Priority() > q[j].Priority() } @@ -33,7 +31,7 @@ func (q queue) Swap(i, j int) { q[i], q[j] = q[j], q[i] } // Push and Pop use pointer receivers because they modify the slice's length, // not just its contents. func (q *queue) Push(x interface{}) { - *q = append(*q, x.(Op)) + *q = append(*q, x.(PriorityOp)) } func (q *queue) Pop() interface{} { @@ -47,7 +45,6 @@ func (q *queue) Pop() interface{} { // NewPriorityQueue makes a new priority queue. func NewPriorityQueue(lengthGauge prometheus.Gauge) *PriorityQueue { pq := &PriorityQueue{ - hit: map[string]struct{}{}, lengthGauge: lengthGauge, } pq.cond = sync.NewCond(&pq.lock) @@ -77,13 +74,11 @@ func (pq *PriorityQueue) DiscardAndClose() { defer pq.lock.Unlock() pq.closed = true pq.queue = nil - pq.hit = map[string]struct{}{} pq.cond.Broadcast() } -// Enqueue adds an operation to the queue in priority order. Returns -// true if added; false if the operation was already on the queue. -func (pq *PriorityQueue) Enqueue(op Op) bool { +// Enqueue adds an operation to the queue in priority order. +func (pq *PriorityQueue) Enqueue(op PriorityOp) { pq.lock.Lock() defer pq.lock.Unlock() @@ -91,23 +86,16 @@ func (pq *PriorityQueue) Enqueue(op Op) bool { panic("enqueue on closed queue") } - _, enqueued := pq.hit[op.Key()] - if enqueued { - return false - } - - pq.hit[op.Key()] = struct{}{} heap.Push(&pq.queue, op) pq.cond.Broadcast() if pq.lengthGauge != nil { pq.lengthGauge.Inc() } - return true } // Dequeue will return the op with the highest priority; block if queue is // empty; returns nil if queue is closed. -func (pq *PriorityQueue) Dequeue() Op { +func (pq *PriorityQueue) Dequeue() PriorityOp { pq.lock.Lock() defer pq.lock.Unlock() @@ -120,8 +108,7 @@ func (pq *PriorityQueue) Dequeue() Op { return nil } - op := heap.Pop(&pq.queue).(Op) - delete(pq.hit, op.Key()) + op := heap.Pop(&pq.queue).(PriorityOp) if pq.lengthGauge != nil { pq.lengthGauge.Dec() } From f064b3a7457005e516aacf6829f99fb7a518ff8d Mon Sep 17 00:00:00 2001 From: Justin Jung Date: Tue, 7 Nov 2023 16:40:51 -0800 Subject: [PATCH 23/49] Nits + priority queue test added Signed-off-by: Justin Jung --- pkg/scheduler/queue/queue.go | 3 ++ pkg/scheduler/queue/queue_test.go | 62 ++++++++++++------------- pkg/scheduler/queue/request_queue.go | 9 ++++ pkg/scheduler/queue/user_queues.go | 15 ++---- pkg/scheduler/queue/user_queues_test.go | 26 +++++------ 5 files changed, 59 insertions(+), 56 deletions(-) diff --git a/pkg/scheduler/queue/queue.go b/pkg/scheduler/queue/queue.go index a2e44cba83..98031843ab 100644 --- a/pkg/scheduler/queue/queue.go +++ b/pkg/scheduler/queue/queue.go @@ -157,6 +157,9 @@ FindQueue: // Pick next request from the queue. for { request := queue.dequeueRequest() + if queue.length() == 0 { + q.queues.deleteQueue(userID) + } q.queueLength.WithLabelValues(userID).Dec() diff --git a/pkg/scheduler/queue/queue_test.go b/pkg/scheduler/queue/queue_test.go index 046abc7c03..e77dc33e6e 100644 --- a/pkg/scheduler/queue/queue_test.go +++ b/pkg/scheduler/queue/queue_test.go @@ -3,6 +3,7 @@ package queue import ( "context" "fmt" + "github.com/cortexproject/cortex/pkg/util/validation" "strconv" "sync" "testing" @@ -160,40 +161,37 @@ func TestRequestQueue_GetNextRequestForQuerier_ShouldGetRequestAfterReshardingBe } func TestRequestQueue_QueriersShouldGetHighPriorityQueryFirst(t *testing.T) { - //queue := NewRequestQueue(0, 0, - // prometheus.NewGaugeVec(prometheus.GaugeOpts{}, []string{"user"}), - // prometheus.NewCounterVec(prometheus.CounterOpts{}, []string{"user"}), - // MockLimits{MaxOutstanding: 3}, - // nil, - //) - //ctx := context.Background() - //queue.RegisterQuerierConnection("querier-1") - // - //normalRequest1 := MockRequest{ - // id: "normal query 1", - //} - //normalRequest2 := MockRequest{ - // id: "normal query 2", - //} - //highPriorityRequest := MockRequest{ - // id: "high priority query", - //} - // - //assert.NoError(t, queue.EnqueueRequest("userID", normalRequest1, 1, func() {})) - //assert.NoError(t, queue.EnqueueRequest("userID", normalRequest2, 1, func() {})) - //assert.NoError(t, queue.EnqueueRequest("userID", highPriorityRequest, 1, func() {})) - // - //assert.Error(t, queue.EnqueueRequest("userID", highPriorityRequest, 1, func() {})) // should fail due to maxOutstandingPerTenant = 3 - //assert.Equal(t, 3, queue.queues.getTotalQueueSize("userID")) - //nextRequest, _, _ := queue.GetNextRequestForQuerier(ctx, FirstUser(), "querier-1") - //assert.Equal(t, highPriorityRequest, nextRequest) // high priority request returned, although it was enqueued the last - //nextRequest, _, _ = queue.GetNextRequestForQuerier(ctx, FirstUser(), "querier-1") - //assert.Equal(t, normalRequest1, nextRequest) - //nextRequest, _, _ = queue.GetNextRequestForQuerier(ctx, FirstUser(), "querier-1") - //assert.Equal(t, normalRequest2, nextRequest) + queue := NewRequestQueue(0, 0, + prometheus.NewGaugeVec(prometheus.GaugeOpts{}, []string{"user"}), + prometheus.NewCounterVec(prometheus.CounterOpts{}, []string{"user"}), + MockLimits{MaxOutstanding: 3, queryPriority: validation.QueryPriority{Enabled: true}}, + nil, + ) + ctx := context.Background() + queue.RegisterQuerierConnection("querier-1") + + normalRequest1 := MockRequest{ + id: "normal query 1", + } + normalRequest2 := MockRequest{ + id: "normal query 2", + } + highPriorityRequest := MockRequest{ + id: "high priority query", + priority: 1, + } + + assert.NoError(t, queue.EnqueueRequest("userID", normalRequest1, 1, func() {})) + assert.NoError(t, queue.EnqueueRequest("userID", normalRequest2, 1, func() {})) + assert.NoError(t, queue.EnqueueRequest("userID", highPriorityRequest, 1, func() {})) + + assert.Error(t, queue.EnqueueRequest("userID", highPriorityRequest, 1, func() {})) // should fail due to maxOutstandingPerTenant = 3 + nextRequest, _, _ := queue.GetNextRequestForQuerier(ctx, FirstUser(), "querier-1") + assert.Equal(t, highPriorityRequest, nextRequest) // high priority request returned, although it was enqueued the last } func TestRequestQueue_ReservedQueriersShouldOnlyGetHighPriorityQueries(t *testing.T) { + // TODO: justinjung04 //queue := NewRequestQueue(0, 0, // prometheus.NewGaugeVec(prometheus.GaugeOpts{}, []string{"user"}), // prometheus.NewCounterVec(prometheus.CounterOpts{}, []string{"user"}), @@ -235,6 +233,6 @@ type MockRequest struct { priority int64 } -func (r MockRequest) GetPriority() int64 { +func (r MockRequest) Priority() int64 { return r.priority } diff --git a/pkg/scheduler/queue/request_queue.go b/pkg/scheduler/queue/request_queue.go index 66c66e905e..d6ff100fa6 100644 --- a/pkg/scheduler/queue/request_queue.go +++ b/pkg/scheduler/queue/request_queue.go @@ -6,6 +6,7 @@ type requestQueue interface { enqueueRequest(Request) dequeueRequest() Request length() int + closeQueue() } type FIFORequestQueue struct { @@ -28,6 +29,10 @@ func (f *FIFORequestQueue) length() int { return len(f.queue) } +func (f *FIFORequestQueue) closeQueue() { + close(f.queue) +} + type PriorityRequestQueue struct { queue *util.PriorityQueue } @@ -47,3 +52,7 @@ func (f *PriorityRequestQueue) dequeueRequest() Request { func (f *PriorityRequestQueue) length() int { return f.queue.Length() } + +func (f *PriorityRequestQueue) closeQueue() { + f.queue.DiscardAndClose() +} diff --git a/pkg/scheduler/queue/user_queues.go b/pkg/scheduler/queue/user_queues.go index e20bc0f905..3eb4cf0581 100644 --- a/pkg/scheduler/queue/user_queues.go +++ b/pkg/scheduler/queue/user_queues.go @@ -97,6 +97,7 @@ func (q *queues) deleteQueue(userID string) { return } + uq.queue.closeQueue() delete(q.userQueues, userID) q.users[uq.index] = "" @@ -197,7 +198,7 @@ func (q *queues) getNextQueueForQuerier(lastUserIndex int, querierID string) (re } } - //TODO: jungjust, reserved queriers + //TODO: justinjung04, reserved queriers //if priority, isReserved := uq.reservedQueriers[querierID]; isReserved { // return uq.queues[priority], u, uid //} @@ -351,19 +352,11 @@ func shuffleQueriersForUser(userSeed int64, queriersToSelect int, allSortedQueri } func getReservedQueriers(queriers map[string]struct{}, priorities []validation.PriorityDef) map[string]int64 { - // TODO jungjust: - //reservedQueriers := make(map[string]int64) - // - //cnt := 0 - //for querierID, _ := range queriers { - // - // //reservedQueriers - // // TODO jungjust: get diff priorities - // cnt++ - //} + // TODO: justinjung04 return map[string]int64{} } +// TODO: justinjung04 //func getNumOfReservedQueriers(queriersToSelect int, totalNumOfQueriers int, reservedQueriers float64) int { // numOfReservedQueriers := int(reservedQueriers) // diff --git a/pkg/scheduler/queue/user_queues_test.go b/pkg/scheduler/queue/user_queues_test.go index 06fb90a309..48a8640d50 100644 --- a/pkg/scheduler/queue/user_queues_test.go +++ b/pkg/scheduler/queue/user_queues_test.go @@ -22,11 +22,11 @@ func TestQueues(t *testing.T) { assert.Equal(t, "", u) // Add queues: [one] - qOne := getOrAdd(t, uq, "one", 0, 0) + qOne := getOrAdd(t, uq, "one", 0) lastUserIndex = confirmOrderForQuerier(t, uq, "querier-1", lastUserIndex, qOne, qOne) // [one two] - qTwo := getOrAdd(t, uq, "two", 0, 0) + qTwo := getOrAdd(t, uq, "two", 0) assert.NotEqual(t, qOne, qTwo) lastUserIndex = confirmOrderForQuerier(t, uq, "querier-1", lastUserIndex, qTwo, qOne, qTwo, qOne) @@ -34,7 +34,7 @@ func TestQueues(t *testing.T) { // [one two three] // confirm fifo by adding a third queue and iterating to it - qThree := getOrAdd(t, uq, "three", 0, 0) + qThree := getOrAdd(t, uq, "three", 0) lastUserIndex = confirmOrderForQuerier(t, uq, "querier-1", lastUserIndex, qTwo, qThree, qOne) @@ -45,7 +45,7 @@ func TestQueues(t *testing.T) { lastUserIndex = confirmOrderForQuerier(t, uq, "querier-1", lastUserIndex, qTwo, qThree, qTwo) // "four" is added at the beginning of the list: [four two three] - qFour := getOrAdd(t, uq, "four", 0, 0) + qFour := getOrAdd(t, uq, "four", 0) lastUserIndex = confirmOrderForQuerier(t, uq, "querier-1", lastUserIndex, qThree, qFour, qTwo, qThree) @@ -92,7 +92,7 @@ func TestQueuesWithQueriers(t *testing.T) { // Add user queues. for u := 0; u < users; u++ { uid := fmt.Sprintf("user-%d", u) - getOrAdd(t, uq, uid, maxQueriersPerUser, 0) + getOrAdd(t, uq, uid, maxQueriersPerUser) // Verify it has maxQueriersPerUser queriers assigned now. qs := uq.userQueues[uid].queriers @@ -156,7 +156,7 @@ func TestQueuesConsistency(t *testing.T) { conns := map[string]int{} for i := 0; i < 10000; i++ { - queue := uq.getOrAddQueue(generateTenant(r), 3, 0) + queue := uq.getOrAddQueue(generateTenant(r), 3) switch r.Int() % 6 { case 0: assert.NotNil(t, queue) @@ -208,7 +208,7 @@ func TestQueues_ForgetDelay(t *testing.T) { // Add user queues. for i := 0; i < numUsers; i++ { userID := fmt.Sprintf("user-%d", i) - getOrAdd(t, uq, userID, maxQueriersPerUser, 0) + getOrAdd(t, uq, userID, maxQueriersPerUser) } // We expect querier-1 to have some users. @@ -300,7 +300,7 @@ func TestQueues_ForgetDelay_ShouldCorrectlyHandleQuerierReconnectingBeforeForget // Add user queues. for i := 0; i < numUsers; i++ { userID := fmt.Sprintf("user-%d", i) - getOrAdd(t, uq, userID, maxQueriersPerUser, 0) + getOrAdd(t, uq, userID, maxQueriersPerUser) } // We expect querier-1 to have some users. @@ -360,16 +360,16 @@ func generateQuerier(r *rand.Rand) string { return fmt.Sprint("querier-", r.Int()%5) } -func getOrAdd(t *testing.T, uq *queues, tenant string, maxQueriers int, priority int64) chan Request { - q := uq.getOrAddQueue(tenant, maxQueriers, priority) +func getOrAdd(t *testing.T, uq *queues, tenant string, maxQueriers int) requestQueue { + q := uq.getOrAddQueue(tenant, maxQueriers) assert.NotNil(t, q) assert.NoError(t, isConsistent(uq)) - assert.Equal(t, q, uq.getOrAddQueue(tenant, maxQueriers, priority)) + assert.Equal(t, q, uq.getOrAddQueue(tenant, maxQueriers)) return q } -func confirmOrderForQuerier(t *testing.T, uq *queues, querier string, lastUserIndex int, qs ...chan Request) int { - var n chan Request +func confirmOrderForQuerier(t *testing.T, uq *queues, querier string, lastUserIndex int, qs ...requestQueue) int { + var n requestQueue for _, q := range qs { n, _, lastUserIndex = uq.getNextQueueForQuerier(lastUserIndex, querier) assert.Equal(t, q, n) From e41e4c1ab6d2b110893ec81e7271567833f51743 Mon Sep 17 00:00:00 2001 From: Justin Jung Date: Wed, 8 Nov 2023 08:42:23 -0800 Subject: [PATCH 24/49] Create user request queue test Signed-off-by: Justin Jung --- pkg/scheduler/queue/queue.go | 7 ++- pkg/scheduler/queue/user_queues.go | 61 +++++-------------- pkg/scheduler/queue/user_queues_test.go | 6 +- ...request_queue.go => user_request_queue.go} | 13 ++-- .../queue/user_request_queue_test.go | 57 +++++++++++++++++ pkg/util/priority_queue.go | 8 ++- pkg/util/priority_queue_test.go | 7 +-- 7 files changed, 97 insertions(+), 62 deletions(-) rename pkg/scheduler/queue/{request_queue.go => user_request_queue.go} (72%) create mode 100644 pkg/scheduler/queue/user_request_queue_test.go diff --git a/pkg/scheduler/queue/queue.go b/pkg/scheduler/queue/queue.go index 98031843ab..e6c55ca53d 100644 --- a/pkg/scheduler/queue/queue.go +++ b/pkg/scheduler/queue/queue.go @@ -156,7 +156,12 @@ FindQueue: // Pick next request from the queue. for { - request := queue.dequeueRequest() + request := queue.dequeueRequest(q.queues.getMinPriority(userID, querierID)) + if request == nil { + // the queue does not contain request with the min priority, break to wait for more requests + break + } + if queue.length() == 0 { q.queues.deleteQueue(userID) } diff --git a/pkg/scheduler/queue/user_queues.go b/pkg/scheduler/queue/user_queues.go index 3eb4cf0581..1c34f41762 100644 --- a/pkg/scheduler/queue/user_queues.go +++ b/pkg/scheduler/queue/user_queues.go @@ -59,13 +59,12 @@ type queues struct { } type userQueue struct { - queue requestQueue + queue userRequestQueue // If not nil, only these queriers can handle user requests. If nil, all queriers can. // We set this to nil if number of available queriers <= maxQueriers. - queriers map[string]struct{} - reservedQueriers map[string]int64 - maxQueriers int + queriers map[string]struct{} + maxQueriers int // Seed for shuffle sharding of queriers. This seed is based on userID only and is therefore consistent // between different frontends. @@ -111,7 +110,7 @@ func (q *queues) deleteQueue(userID string) { // MaxQueriers is used to compute which queriers should handle requests for this user. // If maxQueriers is <= 0, all queriers can handle this user's requests. // If maxQueriers has changed since the last call, queriers for this are recomputed. -func (q *queues) getOrAddQueue(userID string, maxQueriers int) requestQueue { +func (q *queues) getOrAddQueue(userID string, maxQueriers int) userRequestQueue { // Empty user is not allowed, as that would break our users list ("" is used for free spot). if userID == "" { return nil @@ -139,7 +138,7 @@ func (q *queues) getOrAddQueue(userID string, maxQueriers int) requestQueue { if queryPriority.Enabled { uq.queue = NewPriorityRequestQueue(util.NewPriorityQueue(nil)) } else { - uq.queue = NewFIFOQueue(make(chan Request, queueSize)) + uq.queue = NewFIFORequestQueue(make(chan Request, queueSize)) } q.userQueues[userID] = uq @@ -163,7 +162,6 @@ func (q *queues) getOrAddQueue(userID string, maxQueriers int) requestQueue { if uq.maxQueriers != maxQueriers { uq.maxQueriers = maxQueriers uq.queriers = shuffleQueriersForUser(uq.seed, maxQueriers, q.sortedQueriers, nil) - uq.reservedQueriers = getReservedQueriers(uq.queriers, q.limits.QueryPriority(userID).Priorities) } return uq.queue @@ -172,7 +170,7 @@ func (q *queues) getOrAddQueue(userID string, maxQueriers int) requestQueue { // Finds next queue for the querier. To support fair scheduling between users, client is expected // to pass last user index returned by this function as argument. Is there was no previous // last user index, use -1. -func (q *queues) getNextQueueForQuerier(lastUserIndex int, querierID string) (requestQueue, string, int) { +func (q *queues) getNextQueueForQuerier(lastUserIndex int, querierID string) (userRequestQueue, string, int) { uid := lastUserIndex for iters := 0; iters < len(q.users); iters++ { @@ -198,7 +196,7 @@ func (q *queues) getNextQueueForQuerier(lastUserIndex int, querierID string) (re } } - //TODO: justinjung04, reserved queriers + // TODO: justinjung04, reserved queriers //if priority, isReserved := uq.reservedQueriers[querierID]; isReserved { // return uq.queues[priority], u, uid //} @@ -208,6 +206,13 @@ func (q *queues) getNextQueueForQuerier(lastUserIndex int, querierID string) (re return nil, "", uid } +func (q *queues) getMinPriority(userID string, querierID string) int64 { + // TODO: justinjung04 reserved querier + // check list of queriers and QueryPriority config + // from QueryPriority config, establish map of + return 0 +} + func (q *queues) addQuerierConnection(querierID string) { info := q.queriers[querierID] if info != nil { @@ -320,16 +325,6 @@ func (q *queues) recomputeUserQueriers() { // In that case *all* queriers should be used. // Scratchpad is used for shuffling, to avoid new allocations. If nil, new slice is allocated. func shuffleQueriersForUser(userSeed int64, queriersToSelect int, allSortedQueriers []string, scratchpad []string) map[string]struct{} { - //numOfReservedQueriers := getNumOfReservedQueriers(queriersToSelect, len(allSortedQueriers), numOfReservedQueriersFloat) - //reservedQueriers := make(map[string]struct{}, numOfReservedQueriers) - - //if queriersToSelect == 0 || len(allSortedQueriers) <= queriersToSelect { - // for i := 0; i < numOfReservedQueriers; i++ { - // reservedQueriers[allSortedQueriers[i]] = struct{}{} - // } - // return nil, reservedQueriers - //} - queriers := make(map[string]struct{}, queriersToSelect) rnd := rand.New(rand.NewSource(userSeed)) @@ -340,10 +335,6 @@ func shuffleQueriersForUser(userSeed int64, queriersToSelect int, allSortedQueri for i := 0; i < queriersToSelect; i++ { r := rnd.Intn(last + 1) queriers[scratchpad[r]] = struct{}{} - //if i < numOfReservedQueriers { - // reservedQueriers[scratchpad[r]] = struct{}{} - //} - // move selected item to the end, it won't be selected anymore. scratchpad[r], scratchpad[last] = scratchpad[last], scratchpad[r] last-- } @@ -351,30 +342,6 @@ func shuffleQueriersForUser(userSeed int64, queriersToSelect int, allSortedQueri return queriers } -func getReservedQueriers(queriers map[string]struct{}, priorities []validation.PriorityDef) map[string]int64 { - // TODO: justinjung04 - return map[string]int64{} -} - -// TODO: justinjung04 -//func getNumOfReservedQueriers(queriersToSelect int, totalNumOfQueriers int, reservedQueriers float64) int { -// numOfReservedQueriers := int(reservedQueriers) -// -// if reservedQueriers < 1 && reservedQueriers > 0 { -// if queriersToSelect == 0 || queriersToSelect > totalNumOfQueriers { -// queriersToSelect = totalNumOfQueriers -// } -// -// numOfReservedQueriers = int(math.Ceil(float64(queriersToSelect) * reservedQueriers)) -// } -// -// if numOfReservedQueriers > totalNumOfQueriers { -// return totalNumOfQueriers -// } -// -// return numOfReservedQueriers -//} - // MockLimits implements the Limits interface. Used in tests only. type MockLimits struct { MaxOutstanding int diff --git a/pkg/scheduler/queue/user_queues_test.go b/pkg/scheduler/queue/user_queues_test.go index 48a8640d50..729ae9699d 100644 --- a/pkg/scheduler/queue/user_queues_test.go +++ b/pkg/scheduler/queue/user_queues_test.go @@ -360,7 +360,7 @@ func generateQuerier(r *rand.Rand) string { return fmt.Sprint("querier-", r.Int()%5) } -func getOrAdd(t *testing.T, uq *queues, tenant string, maxQueriers int) requestQueue { +func getOrAdd(t *testing.T, uq *queues, tenant string, maxQueriers int) userRequestQueue { q := uq.getOrAddQueue(tenant, maxQueriers) assert.NotNil(t, q) assert.NoError(t, isConsistent(uq)) @@ -368,8 +368,8 @@ func getOrAdd(t *testing.T, uq *queues, tenant string, maxQueriers int) requestQ return q } -func confirmOrderForQuerier(t *testing.T, uq *queues, querier string, lastUserIndex int, qs ...requestQueue) int { - var n requestQueue +func confirmOrderForQuerier(t *testing.T, uq *queues, querier string, lastUserIndex int, qs ...userRequestQueue) int { + var n userRequestQueue for _, q := range qs { n, _, lastUserIndex = uq.getNextQueueForQuerier(lastUserIndex, querier) assert.Equal(t, q, n) diff --git a/pkg/scheduler/queue/request_queue.go b/pkg/scheduler/queue/user_request_queue.go similarity index 72% rename from pkg/scheduler/queue/request_queue.go rename to pkg/scheduler/queue/user_request_queue.go index d6ff100fa6..b8ee1ecaf7 100644 --- a/pkg/scheduler/queue/request_queue.go +++ b/pkg/scheduler/queue/user_request_queue.go @@ -2,9 +2,9 @@ package queue import "github.com/cortexproject/cortex/pkg/util" -type requestQueue interface { +type userRequestQueue interface { enqueueRequest(Request) - dequeueRequest() Request + dequeueRequest(minPriority int64) Request length() int closeQueue() } @@ -13,7 +13,7 @@ type FIFORequestQueue struct { queue chan Request } -func NewFIFOQueue(queue chan Request) *FIFORequestQueue { +func NewFIFORequestQueue(queue chan Request) *FIFORequestQueue { return &FIFORequestQueue{queue: queue} } @@ -21,7 +21,7 @@ func (f *FIFORequestQueue) enqueueRequest(r Request) { f.queue <- r } -func (f *FIFORequestQueue) dequeueRequest() Request { +func (f *FIFORequestQueue) dequeueRequest(_ int64) Request { return <-f.queue } @@ -45,7 +45,10 @@ func (f *PriorityRequestQueue) enqueueRequest(r Request) { f.queue.Enqueue(r) } -func (f *PriorityRequestQueue) dequeueRequest() Request { +func (f *PriorityRequestQueue) dequeueRequest(minPriority int64) Request { + if f.queue.Peek().Priority() < minPriority { + return nil + } return f.queue.Dequeue() } diff --git a/pkg/scheduler/queue/user_request_queue_test.go b/pkg/scheduler/queue/user_request_queue_test.go new file mode 100644 index 0000000000..aeaad7fb2a --- /dev/null +++ b/pkg/scheduler/queue/user_request_queue_test.go @@ -0,0 +1,57 @@ +package queue + +import ( + "github.com/cortexproject/cortex/pkg/util" + "github.com/stretchr/testify/assert" + "testing" +) + +func TestFIFORequestQueue(t *testing.T) { + queue := NewFIFORequestQueue(make(chan Request, 2)) + request1 := MockRequest{ + id: "request 1", + priority: 1, + } + request2 := MockRequest{ + id: "request 2", + priority: 2, + } + + queue.enqueueRequest(request1) + queue.enqueueRequest(request2) + assert.Equal(t, 2, queue.length()) + assert.Equal(t, request1, queue.dequeueRequest(0)) + assert.Equal(t, 1, queue.length()) + assert.Equal(t, request2, queue.dequeueRequest(0)) + assert.Equal(t, 0, queue.length()) + queue.closeQueue() + assert.Panics(t, func() { queue.enqueueRequest(request1) }) +} + +func TestPriorityRequestQueue(t *testing.T) { + queue := NewPriorityRequestQueue(util.NewPriorityQueue(nil)) + request1 := MockRequest{ + id: "request 1", + priority: 1, + } + request2 := MockRequest{ + id: "request 2", + priority: 2, + } + + queue.enqueueRequest(request1) + queue.enqueueRequest(request2) + assert.Equal(t, 2, queue.length()) + assert.Equal(t, request2, queue.dequeueRequest(0)) + assert.Equal(t, 1, queue.length()) + assert.Equal(t, request1, queue.dequeueRequest(0)) + assert.Equal(t, 0, queue.length()) + + queue.enqueueRequest(request1) + queue.enqueueRequest(request2) + assert.Equal(t, 2, queue.length()) + assert.Equal(t, request2, queue.dequeueRequest(2)) + + queue.closeQueue() + assert.Panics(t, func() { queue.enqueueRequest(request1) }) +} diff --git a/pkg/util/priority_queue.go b/pkg/util/priority_queue.go index 4cbc86e3ff..26e0b13832 100644 --- a/pkg/util/priority_queue.go +++ b/pkg/util/priority_queue.go @@ -93,7 +93,7 @@ func (pq *PriorityQueue) Enqueue(op PriorityOp) { } } -// Dequeue will return the op with the highest priority; block if queue is +// Dequeue will remove and return the op with the highest priority; block if queue is // empty; returns nil if queue is closed. func (pq *PriorityQueue) Dequeue() PriorityOp { pq.lock.Lock() @@ -114,3 +114,9 @@ func (pq *PriorityQueue) Dequeue() PriorityOp { } return op } + +// Peek will return the op with the highest priority without removing it from the queue +func (pq *PriorityQueue) Peek() PriorityOp { + op := pq.queue[0] + return op +} diff --git a/pkg/util/priority_queue_test.go b/pkg/util/priority_queue_test.go index 79eaf2784f..d7ee8bc202 100644 --- a/pkg/util/priority_queue_test.go +++ b/pkg/util/priority_queue_test.go @@ -2,7 +2,6 @@ package util import ( "runtime" - "strconv" "testing" "time" @@ -15,10 +14,6 @@ func (i simpleItem) Priority() int64 { return int64(i) } -func (i simpleItem) Key() string { - return strconv.FormatInt(int64(i), 10) -} - func TestPriorityQueueBasic(t *testing.T) { queue := NewPriorityQueue(nil) assert.Equal(t, 0, queue.Length(), "Expected length = 0") @@ -39,6 +34,7 @@ func TestPriorityQueuePriorities(t *testing.T) { queue.Enqueue(simpleItem(1)) queue.Enqueue(simpleItem(2)) + assert.Equal(t, simpleItem(2), queue.Peek().(simpleItem), "Expected to peek simpleItem(2)") assert.Equal(t, simpleItem(2), queue.Dequeue().(simpleItem), "Expected to dequeue simpleItem(2)") assert.Equal(t, simpleItem(1), queue.Dequeue().(simpleItem), "Expected to dequeue simpleItem(1)") @@ -51,6 +47,7 @@ func TestPriorityQueuePriorities2(t *testing.T) { queue.Enqueue(simpleItem(2)) queue.Enqueue(simpleItem(1)) + assert.Equal(t, simpleItem(2), queue.Peek().(simpleItem), "Expected to peek simpleItem(2)") assert.Equal(t, simpleItem(2), queue.Dequeue().(simpleItem), "Expected to dequeue simpleItem(2)") assert.Equal(t, simpleItem(1), queue.Dequeue().(simpleItem), "Expected to dequeue simpleItem(1)") From ca2cd1cd502f3c270f0e846fa1baefd95a4a010c Mon Sep 17 00:00:00 2001 From: Justin Jung Date: Wed, 8 Nov 2023 16:47:39 -0800 Subject: [PATCH 25/49] Add reserved querier logic Signed-off-by: Justin Jung --- pkg/frontend/v1/frontend.go | 13 +- pkg/frontend/v2/frontend.go | 14 +- pkg/scheduler/queue/queue.go | 24 ++- pkg/scheduler/queue/queue_test.go | 2 +- pkg/scheduler/queue/user_queues.go | 149 ++++++++++++------ pkg/scheduler/queue/user_queues_test.go | 10 +- pkg/scheduler/queue/user_request_queue.go | 10 +- .../queue/user_request_queue_test.go | 19 ++- pkg/util/priority_queue.go | 6 +- pkg/util/query/priority.go | 16 +- pkg/util/query/priority_test.go | 99 +++++------- pkg/util/validation/limits.go | 18 --- pkg/util/validation/limits_test.go | 1 - 13 files changed, 225 insertions(+), 156 deletions(-) diff --git a/pkg/frontend/v1/frontend.go b/pkg/frontend/v1/frontend.go index d3734e9bcf..7b952fff2b 100644 --- a/pkg/frontend/v1/frontend.go +++ b/pkg/frontend/v1/frontend.go @@ -6,6 +6,7 @@ import ( "fmt" "net/http" "net/url" + "reflect" "time" "github.com/go-kit/log" @@ -79,6 +80,9 @@ type Frontend struct { requestQueue *queue.RequestQueue activeUsers *util.ActiveUsersCleanupService + queryPriority validation.QueryPriority + compiledQueryPriority validation.QueryPriority + // Subservices manager. subservices *services.Manager subservicesWatcher *services.FailureWatcher @@ -207,8 +211,15 @@ func (f *Frontend) RoundTripGRPC(ctx context.Context, req *httpgrpc.HTTPRequest, } queryPriority := f.limits.QueryPriority(userID) + if reqParams != nil && queryPriority.Enabled { - request.priority = util_query.GetPriority(reqParams, ts, queryPriority) + queryPriorityChanged := !reflect.DeepEqual(f.queryPriority, queryPriority) + if queryPriorityChanged { + f.queryPriority = queryPriority + f.compiledQueryPriority = queryPriority + } + + request.priority = util_query.GetPriority(reqParams, ts, &f.compiledQueryPriority, queryPriorityChanged) } if err := f.queueRequest(ctx, &request); err != nil { diff --git a/pkg/frontend/v2/frontend.go b/pkg/frontend/v2/frontend.go index e06f3f97c1..c99980100e 100644 --- a/pkg/frontend/v2/frontend.go +++ b/pkg/frontend/v2/frontend.go @@ -7,6 +7,7 @@ import ( "math/rand" "net/http" "net/url" + "reflect" "sync" "time" @@ -30,6 +31,7 @@ import ( util_log "github.com/cortexproject/cortex/pkg/util/log" util_query "github.com/cortexproject/cortex/pkg/util/query" "github.com/cortexproject/cortex/pkg/util/services" + "github.com/cortexproject/cortex/pkg/util/validation" ) // Config for a Frontend. @@ -74,6 +76,9 @@ type Frontend struct { lastQueryID atomic.Uint64 + queryPriority validation.QueryPriority + compiledQueryPriority validation.QueryPriority + // frontend workers will read from this channel, and send request to scheduler. requestsCh chan *frontendRequest @@ -211,8 +216,15 @@ func (f *Frontend) RoundTripGRPC(ctx context.Context, req *httpgrpc.HTTPRequest, } queryPriority := f.limits.QueryPriority(userID) + if reqParams != nil && queryPriority.Enabled { - freq.priority = util_query.GetPriority(reqParams, ts, queryPriority) + queryPriorityChanged := !reflect.DeepEqual(f.queryPriority, queryPriority) + if queryPriorityChanged { + f.queryPriority = queryPriority + f.compiledQueryPriority = queryPriority + } + + freq.priority = util_query.GetPriority(reqParams, ts, &f.compiledQueryPriority, queryPriorityChanged) } f.requests.put(freq) diff --git a/pkg/scheduler/queue/queue.go b/pkg/scheduler/queue/queue.go index e6c55ca53d..3fc0fd16d3 100644 --- a/pkg/scheduler/queue/queue.go +++ b/pkg/scheduler/queue/queue.go @@ -98,7 +98,8 @@ func (q *RequestQueue) EnqueueRequest(userID string, req Request, maxQueriers fl } shardSize := util.DynamicShardSize(maxQueriers, len(q.queues.queriers)) - queue := q.queues.getOrAddQueue(userID, shardSize) + priorityList, priorityEnabled := q.getPriorityList(userID) + queue := q.queues.getOrAddQueue(userID, shardSize, priorityList, priorityEnabled) maxOutstandingRequests := q.queues.limits.MaxOutstandingPerTenant(userID) if queue == nil { @@ -123,6 +124,24 @@ func (q *RequestQueue) EnqueueRequest(userID string, req Request, maxQueriers fl return nil } +func (q *RequestQueue) getPriorityList(userID string) ([]int64, bool) { + var priorityList []int64 + + queryPriority := q.queues.limits.QueryPriority(userID) + + if queryPriority.Enabled { + for _, priority := range queryPriority.Priorities { + reservedQuerierShardSize := util.DynamicShardSize(priority.ReservedQueriers, len(q.queues.queriers)) + + for i := 0; i < reservedQuerierShardSize; i++ { + priorityList = append(priorityList, priority.Priority) + } + } + } + + return priorityList, queryPriority.Enabled +} + // GetNextRequestForQuerier find next user queue and takes the next request off of it. Will block if there are no requests. // By passing user index from previous call of this method, querier guarantees that it iterates over all users fairly. // If querier finds that request from the user is already expired, it can get a request for the same user by using UserIndex.ReuseLastUser. @@ -156,7 +175,8 @@ FindQueue: // Pick next request from the queue. for { - request := queue.dequeueRequest(q.queues.getMinPriority(userID, querierID)) + minPriority, checkMinPriority := q.queues.getMinPriority(userID, querierID) + request := queue.dequeueRequest(minPriority, checkMinPriority) if request == nil { // the queue does not contain request with the min priority, break to wait for more requests break diff --git a/pkg/scheduler/queue/queue_test.go b/pkg/scheduler/queue/queue_test.go index e77dc33e6e..535647bc7a 100644 --- a/pkg/scheduler/queue/queue_test.go +++ b/pkg/scheduler/queue/queue_test.go @@ -3,7 +3,6 @@ package queue import ( "context" "fmt" - "github.com/cortexproject/cortex/pkg/util/validation" "strconv" "sync" "testing" @@ -14,6 +13,7 @@ import ( "github.com/stretchr/testify/require" "github.com/cortexproject/cortex/pkg/util/services" + "github.com/cortexproject/cortex/pkg/util/validation" ) func BenchmarkGetNextRequest(b *testing.B) { diff --git a/pkg/scheduler/queue/user_queues.go b/pkg/scheduler/queue/user_queues.go index 1c34f41762..7bf0f8c80b 100644 --- a/pkg/scheduler/queue/user_queues.go +++ b/pkg/scheduler/queue/user_queues.go @@ -2,6 +2,7 @@ package queue import ( "math/rand" + "reflect" "sort" "time" @@ -63,8 +64,11 @@ type userQueue struct { // If not nil, only these queriers can handle user requests. If nil, all queriers can. // We set this to nil if number of available queriers <= maxQueriers. - queriers map[string]struct{} - maxQueriers int + queriers map[string]struct{} + reservedQueriers map[string]int64 + priorityList []int64 + priorityEnabled bool + maxQueriers int // Seed for shuffle sharding of queriers. This seed is based on userID only and is therefore consistent // between different frontends. @@ -110,7 +114,7 @@ func (q *queues) deleteQueue(userID string) { // MaxQueriers is used to compute which queriers should handle requests for this user. // If maxQueriers is <= 0, all queriers can handle this user's requests. // If maxQueriers has changed since the last call, queriers for this are recomputed. -func (q *queues) getOrAddQueue(userID string, maxQueriers int) userRequestQueue { +func (q *queues) getOrAddQueue(userID string, maxQueriers int, priorityList []int64, priorityEnabled bool) userRequestQueue { // Empty user is not allowed, as that would break our users list ("" is used for free spot). if userID == "" { return nil @@ -121,48 +125,12 @@ func (q *queues) getOrAddQueue(userID string, maxQueriers int) userRequestQueue } uq := q.userQueues[userID] - queryPriority := q.limits.QueryPriority(userID) if uq == nil { - queueSize := q.limits.MaxOutstandingPerTenant(userID) - // 0 is the default value of the flag. If the old flag is set - // then we use its value for compatibility reason. - if q.maxUserQueueSize != 0 { - queueSize = q.maxUserQueueSize - } - uq = &userQueue{ - seed: util.ShuffleShardSeed(userID, ""), - index: -1, - } - - if queryPriority.Enabled { - uq.queue = NewPriorityRequestQueue(util.NewPriorityQueue(nil)) - } else { - uq.queue = NewFIFORequestQueue(make(chan Request, queueSize)) - } - - q.userQueues[userID] = uq - - // Add user to the list of users... find first free spot, and put it there. - for ix, u := range q.users { - if u == "" { - uq.index = ix - q.users[ix] = userID - break - } - } - - // ... or add to the end. - if uq.index < 0 { - uq.index = len(q.users) - q.users = append(q.users, userID) - } + uq = q.createUserQueue(userID) } - if uq.maxQueriers != maxQueriers { - uq.maxQueriers = maxQueriers - uq.queriers = shuffleQueriersForUser(uq.seed, maxQueriers, q.sortedQueriers, nil) - } + q.updateUserQueuesAttributes(uq, userID, maxQueriers, priorityList, priorityEnabled) return uq.queue } @@ -196,21 +164,98 @@ func (q *queues) getNextQueueForQuerier(lastUserIndex int, querierID string) (us } } - // TODO: justinjung04, reserved queriers - //if priority, isReserved := uq.reservedQueriers[querierID]; isReserved { - // return uq.queues[priority], u, uid - //} - return uq.queue, u, uid } return nil, "", uid } -func (q *queues) getMinPriority(userID string, querierID string) int64 { - // TODO: justinjung04 reserved querier - // check list of queriers and QueryPriority config - // from QueryPriority config, establish map of - return 0 +func (q *queues) createUserQueue(userID string) *userQueue { + uq := &userQueue{ + seed: util.ShuffleShardSeed(userID, ""), + index: -1, + } + + uq.queue = q.createUserRequestQueue(userID) + q.userQueues[userID] = uq + + // Add user to the list of users... find first free spot, and put it there. + for ix, u := range q.users { + if u == "" { + uq.index = ix + q.users[ix] = userID + break + } + } + + // ... or add to the end. + if uq.index < 0 { + uq.index = len(q.users) + q.users = append(q.users, userID) + } + + return uq +} + +func (q *queues) createUserRequestQueue(userID string) userRequestQueue { + if q.limits.QueryPriority(userID).Enabled { + return NewPriorityRequestQueue(util.NewPriorityQueue(nil)) + } + + return NewFIFORequestQueue(make(chan Request, q.getQueueSize(userID))) +} + +func (q *queues) getQueueSize(userID string) int { + queueSize := q.limits.MaxOutstandingPerTenant(userID) + + // 0 is the default value of the flag. If the old flag is set + // then we use its value for compatibility reason. + if q.maxUserQueueSize != 0 { + queueSize = q.maxUserQueueSize + } + + return queueSize +} + +func (q *queues) updateUserQueuesAttributes(uq *userQueue, userID string, maxQueriers int, priorityList []int64, priorityEnabled bool) { + if uq.maxQueriers != maxQueriers { + uq.maxQueriers = maxQueriers + uq.queriers = shuffleQueriersForUser(uq.seed, maxQueriers, q.sortedQueriers, nil) + } + + // if query priority is newly enabled/disabled, transfer the requests to the new queue + if uq.priorityEnabled != priorityEnabled { + tmpQueue := q.createUserRequestQueue(userID) + uq.queue.closeQueue() + + for uq.queue.length() > 0 { + tmpQueue.enqueueRequest(uq.queue.dequeueRequest(0, false)) + } + uq.queue = tmpQueue + } + + if priorityEnabled && !reflect.DeepEqual(uq.priorityList, priorityList) { + reservedQueriers := make(map[string]int64) + + i := 0 + for querierID := range uq.queriers { + reservedQueriers[querierID] = priorityList[i] + i++ + if i == len(priorityList) { + break + } + } + + uq.reservedQueriers = reservedQueriers + uq.priorityList = priorityList + uq.priorityEnabled = priorityEnabled + } +} + +func (q *queues) getMinPriority(userID string, querierID string) (int64, bool) { + if priority, ok := q.userQueues[userID].reservedQueriers[querierID]; ok { + return priority, true + } + return 0, false } func (q *queues) addQuerierConnection(querierID string) { @@ -349,7 +394,7 @@ type MockLimits struct { queryPriority validation.QueryPriority } -func (l MockLimits) MaxQueriersPerUser(user string) float64 { +func (l MockLimits) MaxQueriersPerUser(_ string) float64 { return l.maxQueriersPerUser } diff --git a/pkg/scheduler/queue/user_queues_test.go b/pkg/scheduler/queue/user_queues_test.go index 729ae9699d..4b141dbf88 100644 --- a/pkg/scheduler/queue/user_queues_test.go +++ b/pkg/scheduler/queue/user_queues_test.go @@ -156,7 +156,7 @@ func TestQueuesConsistency(t *testing.T) { conns := map[string]int{} for i := 0; i < 10000; i++ { - queue := uq.getOrAddQueue(generateTenant(r), 3) + queue := uq.getOrAddQueue(generateTenant(r), 3, []int64{}, true) switch r.Int() % 6 { case 0: assert.NotNil(t, queue) @@ -352,6 +352,10 @@ func TestQueues_ForgetDelay_ShouldCorrectlyHandleQuerierReconnectingBeforeForget } } +func TestQueuesUpdatedConfig(t *testing.T) { + // TODO: justinjung04 +} + func generateTenant(r *rand.Rand) string { return fmt.Sprint("tenant-", r.Int()%5) } @@ -361,10 +365,10 @@ func generateQuerier(r *rand.Rand) string { } func getOrAdd(t *testing.T, uq *queues, tenant string, maxQueriers int) userRequestQueue { - q := uq.getOrAddQueue(tenant, maxQueriers) + q := uq.getOrAddQueue(tenant, maxQueriers, []int64{}, true) assert.NotNil(t, q) assert.NoError(t, isConsistent(uq)) - assert.Equal(t, q, uq.getOrAddQueue(tenant, maxQueriers)) + assert.Equal(t, q, uq.getOrAddQueue(tenant, maxQueriers, []int64{}, true)) return q } diff --git a/pkg/scheduler/queue/user_request_queue.go b/pkg/scheduler/queue/user_request_queue.go index b8ee1ecaf7..77afa31806 100644 --- a/pkg/scheduler/queue/user_request_queue.go +++ b/pkg/scheduler/queue/user_request_queue.go @@ -4,7 +4,7 @@ import "github.com/cortexproject/cortex/pkg/util" type userRequestQueue interface { enqueueRequest(Request) - dequeueRequest(minPriority int64) Request + dequeueRequest(minPriority int64, checkMinPriority bool) Request length() int closeQueue() } @@ -21,7 +21,7 @@ func (f *FIFORequestQueue) enqueueRequest(r Request) { f.queue <- r } -func (f *FIFORequestQueue) dequeueRequest(_ int64) Request { +func (f *FIFORequestQueue) dequeueRequest(_ int64, _ bool) Request { return <-f.queue } @@ -45,8 +45,8 @@ func (f *PriorityRequestQueue) enqueueRequest(r Request) { f.queue.Enqueue(r) } -func (f *PriorityRequestQueue) dequeueRequest(minPriority int64) Request { - if f.queue.Peek().Priority() < minPriority { +func (f *PriorityRequestQueue) dequeueRequest(minPriority int64, checkMinPriority bool) Request { + if checkMinPriority && f.queue.Peek().Priority() < minPriority { return nil } return f.queue.Dequeue() @@ -57,5 +57,5 @@ func (f *PriorityRequestQueue) length() int { } func (f *PriorityRequestQueue) closeQueue() { - f.queue.DiscardAndClose() + f.queue.Close() } diff --git a/pkg/scheduler/queue/user_request_queue_test.go b/pkg/scheduler/queue/user_request_queue_test.go index aeaad7fb2a..ec4c728f89 100644 --- a/pkg/scheduler/queue/user_request_queue_test.go +++ b/pkg/scheduler/queue/user_request_queue_test.go @@ -1,9 +1,11 @@ package queue import ( - "github.com/cortexproject/cortex/pkg/util" - "github.com/stretchr/testify/assert" "testing" + + "github.com/stretchr/testify/assert" + + "github.com/cortexproject/cortex/pkg/util" ) func TestFIFORequestQueue(t *testing.T) { @@ -20,9 +22,9 @@ func TestFIFORequestQueue(t *testing.T) { queue.enqueueRequest(request1) queue.enqueueRequest(request2) assert.Equal(t, 2, queue.length()) - assert.Equal(t, request1, queue.dequeueRequest(0)) + assert.Equal(t, request1, queue.dequeueRequest(0, false)) assert.Equal(t, 1, queue.length()) - assert.Equal(t, request2, queue.dequeueRequest(0)) + assert.Equal(t, request2, queue.dequeueRequest(0, false)) assert.Equal(t, 0, queue.length()) queue.closeQueue() assert.Panics(t, func() { queue.enqueueRequest(request1) }) @@ -42,16 +44,17 @@ func TestPriorityRequestQueue(t *testing.T) { queue.enqueueRequest(request1) queue.enqueueRequest(request2) assert.Equal(t, 2, queue.length()) - assert.Equal(t, request2, queue.dequeueRequest(0)) + assert.Equal(t, request2, queue.dequeueRequest(0, true)) assert.Equal(t, 1, queue.length()) - assert.Equal(t, request1, queue.dequeueRequest(0)) + assert.Equal(t, request1, queue.dequeueRequest(0, true)) assert.Equal(t, 0, queue.length()) queue.enqueueRequest(request1) queue.enqueueRequest(request2) assert.Equal(t, 2, queue.length()) - assert.Equal(t, request2, queue.dequeueRequest(2)) - + assert.Equal(t, request2, queue.dequeueRequest(2, true)) + assert.Equal(t, request1, queue.dequeueRequest(2, false)) + queue.closeQueue() assert.Panics(t, func() { queue.enqueueRequest(request1) }) } diff --git a/pkg/util/priority_queue.go b/pkg/util/priority_queue.go index 26e0b13832..543192e126 100644 --- a/pkg/util/priority_queue.go +++ b/pkg/util/priority_queue.go @@ -64,7 +64,11 @@ func (pq *PriorityQueue) Length() int { func (pq *PriorityQueue) Close() { pq.lock.Lock() defer pq.lock.Unlock() - pq.closing = true + if len(pq.queue) > 0 { + pq.closing = true + } else { + pq.closed = true + } pq.cond.Broadcast() } diff --git a/pkg/util/query/priority.go b/pkg/util/query/priority.go index e37dd8f0c9..e32c5df89b 100644 --- a/pkg/util/query/priority.go +++ b/pkg/util/query/priority.go @@ -3,6 +3,7 @@ package query import ( "math" "net/url" + "regexp" "strconv" "time" @@ -11,7 +12,7 @@ import ( "github.com/cortexproject/cortex/pkg/util/validation" ) -func GetPriority(requestParams url.Values, now time.Time, queryPriority validation.QueryPriority) int64 { +func GetPriority(requestParams url.Values, now time.Time, queryPriority *validation.QueryPriority, queryPriorityChanged bool) int64 { queryParam := requestParams.Get("query") timeParam := requestParams.Get("time") startParam := requestParams.Get("start") @@ -21,8 +22,17 @@ func GetPriority(requestParams url.Values, now time.Time, queryPriority validati return queryPriority.DefaultPriority } - for _, priority := range queryPriority.Priorities { - for _, attribute := range priority.QueryAttributes { + for i, priority := range queryPriority.Priorities { + for j, attribute := range priority.QueryAttributes { + if queryPriorityChanged { + compiledRegex, err := regexp.Compile(attribute.Regex) + if err != nil { + continue + } + + queryPriority.Priorities[i].QueryAttributes[j].CompiledRegex = compiledRegex + } + if attribute.CompiledRegex != nil && !attribute.CompiledRegex.MatchString(queryParam) { continue } diff --git a/pkg/util/query/priority_test.go b/pkg/util/query/priority_test.go index 696fb33af2..0a93ac355b 100644 --- a/pkg/util/query/priority_test.go +++ b/pkg/util/query/priority_test.go @@ -1,12 +1,14 @@ package query import ( - "github.com/cortexproject/cortex/pkg/util/validation" - "github.com/stretchr/testify/assert" "net/url" "strconv" "testing" "time" + + "github.com/stretchr/testify/assert" + + "github.com/cortexproject/cortex/pkg/util/validation" ) func Test_GetPriorityShouldReturnDefaultPriorityIfNotEnabledOrEmptyQueryString(t *testing.T) { @@ -23,21 +25,20 @@ func Test_GetPriorityShouldReturnDefaultPriorityIfNotEnabledOrEmptyQueryString(t }, }, } + queryPriority := validation.QueryPriority{ + Priorities: priorities, + } assert.Equal(t, int64(0), GetPriority(url.Values{ "query": []string{"sum(up)"}, "time": []string{strconv.FormatInt(now.Unix(), 10)}, - }, now, validation.QueryPriority{ - Priorities: priorities, - })) + }, now, &queryPriority, true)) + queryPriority.Enabled = true assert.Equal(t, int64(0), GetPriority(url.Values{ "query": []string{""}, "time": []string{strconv.FormatInt(now.Unix(), 10)}, - }, now, validation.QueryPriority{ - Enabled: true, - Priorities: priorities, - })) + }, now, &queryPriority, true)) } func Test_GetPriorityShouldConsiderRegex(t *testing.T) { @@ -54,72 +55,52 @@ func Test_GetPriorityShouldConsiderRegex(t *testing.T) { }, }, } - limits, _ := validation.NewOverrides(validation.Limits{ - QueryPriority: validation.QueryPriority{ - Enabled: true, - Priorities: priorities, - }, - }, nil) + queryPriority := validation.QueryPriority{ + Enabled: true, + Priorities: priorities, + } assert.Equal(t, int64(1), GetPriority(url.Values{ "query": []string{"sum(up)"}, "time": []string{strconv.FormatInt(now.Unix(), 10)}, - }, now, limits.QueryPriority(""))) + }, now, &queryPriority, true)) assert.Equal(t, int64(0), GetPriority(url.Values{ "query": []string{"count(up)"}, "time": []string{strconv.FormatInt(now.Unix(), 10)}, - }, now, limits.QueryPriority(""))) + }, now, &queryPriority, false)) - priorities[0].QueryAttributes[0].Regex = "(^sum$|c(.+)t)" - limits, _ = validation.NewOverrides(validation.Limits{ - QueryPriority: validation.QueryPriority{ - Enabled: true, - Priorities: priorities, - }, - }, nil) + queryPriority.Priorities[0].QueryAttributes[0].Regex = "(^sum$|c(.+)t)" assert.Equal(t, int64(0), GetPriority(url.Values{ "query": []string{"sum(up)"}, "time": []string{strconv.FormatInt(now.Unix(), 10)}, - }, now, limits.QueryPriority(""))) + }, now, &queryPriority, true)) assert.Equal(t, int64(1), GetPriority(url.Values{ "query": []string{"count(up)"}, "time": []string{strconv.FormatInt(now.Unix(), 10)}, - }, now, limits.QueryPriority(""))) + }, now, &queryPriority, false)) - priorities[0].QueryAttributes[0].Regex = ".*" - limits, _ = validation.NewOverrides(validation.Limits{ - QueryPriority: validation.QueryPriority{ - Enabled: true, - Priorities: priorities, - }, - }, nil) + queryPriority.Priorities[0].QueryAttributes[0].Regex = ".*" assert.Equal(t, int64(1), GetPriority(url.Values{ "query": []string{"sum(up)"}, "time": []string{strconv.FormatInt(now.Unix(), 10)}, - }, now, limits.QueryPriority(""))) + }, now, &queryPriority, true)) assert.Equal(t, int64(1), GetPriority(url.Values{ "query": []string{"count(up)"}, "time": []string{strconv.FormatInt(now.Unix(), 10)}, - }, now, limits.QueryPriority(""))) + }, now, &queryPriority, false)) - priorities[0].QueryAttributes[0].Regex = "" - limits, _ = validation.NewOverrides(validation.Limits{ - QueryPriority: validation.QueryPriority{ - Enabled: true, - Priorities: priorities, - }, - }, nil) + queryPriority.Priorities[0].QueryAttributes[0].Regex = "" assert.Equal(t, int64(1), GetPriority(url.Values{ "query": []string{"sum(up)"}, "time": []string{strconv.FormatInt(now.Unix(), 10)}, - }, now, limits.QueryPriority(""))) + }, now, &queryPriority, true)) assert.Equal(t, int64(1), GetPriority(url.Values{ "query": []string{"count(up)"}, "time": []string{strconv.FormatInt(now.Unix(), 10)}, - }, now, limits.QueryPriority(""))) + }, now, &queryPriority, false)) } func Test_GetPriorityShouldConsiderStartAndEndTime(t *testing.T) { @@ -135,57 +116,55 @@ func Test_GetPriorityShouldConsiderStartAndEndTime(t *testing.T) { }, }, } - limits, _ := validation.NewOverrides(validation.Limits{ - QueryPriority: validation.QueryPriority{ - Enabled: true, - Priorities: priorities, - }, - }, nil) + queryPriority := validation.QueryPriority{ + Enabled: true, + Priorities: priorities, + } assert.Equal(t, int64(0), GetPriority(url.Values{ "query": []string{"sum(up)"}, "time": []string{strconv.FormatInt(now.Unix(), 10)}, - }, now, limits.QueryPriority(""))) + }, now, &queryPriority, true)) assert.Equal(t, int64(1), GetPriority(url.Values{ "query": []string{"sum(up)"}, "time": []string{strconv.FormatInt(now.Add(-30*time.Minute).Unix(), 10)}, - }, now, limits.QueryPriority(""))) + }, now, &queryPriority, false)) assert.Equal(t, int64(0), GetPriority(url.Values{ "query": []string{"sum(up)"}, "time": []string{strconv.FormatInt(now.Add(-60*time.Minute).Unix(), 10)}, - }, now, limits.QueryPriority(""))) + }, now, &queryPriority, false)) assert.Equal(t, int64(1), GetPriority(url.Values{ "query": []string{"sum(up)"}, "time": []string{strconv.FormatInt(now.Add(-45*time.Minute).Unix(), 10)}, - }, now, limits.QueryPriority(""))) + }, now, &queryPriority, false)) assert.Equal(t, int64(1), GetPriority(url.Values{ "query": []string{"sum(up)"}, "time": []string{strconv.FormatInt(now.Add(-15*time.Minute).Unix(), 10)}, - }, now, limits.QueryPriority(""))) + }, now, &queryPriority, false)) assert.Equal(t, int64(1), GetPriority(url.Values{ "query": []string{"sum(up)"}, "start": []string{strconv.FormatInt(now.Add(-45*time.Minute).Unix(), 10)}, "end": []string{strconv.FormatInt(now.Add(-15*time.Minute).Unix(), 10)}, - }, now, limits.QueryPriority(""))) + }, now, &queryPriority, false)) assert.Equal(t, int64(0), GetPriority(url.Values{ "query": []string{"sum(up)"}, "start": []string{strconv.FormatInt(now.Add(-50*time.Minute).Unix(), 10)}, "end": []string{strconv.FormatInt(now.Add(-15*time.Minute).Unix(), 10)}, - }, now, limits.QueryPriority(""))) + }, now, &queryPriority, false)) assert.Equal(t, int64(0), GetPriority(url.Values{ "query": []string{"sum(up)"}, "start": []string{strconv.FormatInt(now.Add(-45*time.Minute).Unix(), 10)}, "end": []string{strconv.FormatInt(now.Add(-10*time.Minute).Unix(), 10)}, - }, now, limits.QueryPriority(""))) + }, now, &queryPriority, false)) assert.Equal(t, int64(0), GetPriority(url.Values{ "query": []string{"sum(up)"}, "start": []string{strconv.FormatInt(now.Add(-60*time.Minute).Unix(), 10)}, "end": []string{strconv.FormatInt(now.Add(-45*time.Minute).Unix(), 10)}, - }, now, limits.QueryPriority(""))) + }, now, &queryPriority, false)) assert.Equal(t, int64(0), GetPriority(url.Values{ "query": []string{"sum(up)"}, "start": []string{strconv.FormatInt(now.Add(-15*time.Minute).Unix(), 10)}, "end": []string{strconv.FormatInt(now.Add(-1*time.Minute).Unix(), 10)}, - }, now, limits.QueryPriority(""))) + }, now, &queryPriority, false)) } diff --git a/pkg/util/validation/limits.go b/pkg/util/validation/limits.go index 10db7b9bc9..0403d0c495 100644 --- a/pkg/util/validation/limits.go +++ b/pkg/util/validation/limits.go @@ -52,7 +52,6 @@ type QueryPriority struct { Enabled bool `yaml:"enabled" doc:"nocli|description=Whether queries are assigned with priorities.|default=false"` DefaultPriority int64 `yaml:"default_priority" doc:"nocli|description=Priority assigned to all queries by default. Must be a unique value. Use this as a baseline to make certain queries higher/lower priority.|default=0"` Priorities []PriorityDef `yaml:"priorities" doc:"nocli|description=List of priority definitions."` - RegexCompiled bool `yaml:"-" doc:"nocli"` } type PriorityDef struct { @@ -530,23 +529,6 @@ func (o *Overrides) MaxOutstandingPerTenant(userID string) int { // QueryPriority returns the query priority config for the tenant, including different priorities and their attributes func (o *Overrides) QueryPriority(userID string) QueryPriority { - queryPriority := o.GetOverridesForUser(userID).QueryPriority - - if !queryPriority.RegexCompiled { - priorities := queryPriority.Priorities - for i, priority := range priorities { - for j, attributes := range priority.QueryAttributes { - if attributes.Regex == "" || attributes.Regex == ".*" || attributes.Regex == ".+" { - // if it matches all, don't use regex - o.GetOverridesForUser(userID).QueryPriority.Priorities[i].QueryAttributes[j].CompiledRegex = nil - } else { - o.GetOverridesForUser(userID).QueryPriority.Priorities[i].QueryAttributes[j].CompiledRegex, _ = regexp.Compile(attributes.Regex) - } - } - } - o.GetOverridesForUser(userID).QueryPriority.RegexCompiled = true - } - return o.GetOverridesForUser(userID).QueryPriority } diff --git a/pkg/util/validation/limits_test.go b/pkg/util/validation/limits_test.go index f2de1300e6..c3b8c7e422 100644 --- a/pkg/util/validation/limits_test.go +++ b/pkg/util/validation/limits_test.go @@ -194,7 +194,6 @@ func TestQueryPriority(t *testing.T) { } else { assert.NotNil(t, queryPriority.Priorities[0].QueryAttributes[0].CompiledRegex) } - assert.True(t, queryPriority.RegexCompiled) } } From 067cc836cd5660ff62f5497078114b605f9bb9e1 Mon Sep 17 00:00:00 2001 From: Justin Jung Date: Thu, 9 Nov 2023 04:14:09 -0800 Subject: [PATCH 26/49] Fix tests Signed-off-by: Justin Jung --- pkg/scheduler/queue/queue.go | 2 +- pkg/scheduler/queue/user_queues.go | 8 ++- pkg/scheduler/queue/user_queues_test.go | 3 +- .../queue/user_request_queue_test.go | 3 + pkg/util/query/priority.go | 3 +- pkg/util/query/priority_test.go | 13 +++++ pkg/util/validation/limits_test.go | 58 ------------------- 7 files changed, 26 insertions(+), 64 deletions(-) diff --git a/pkg/scheduler/queue/queue.go b/pkg/scheduler/queue/queue.go index 3fc0fd16d3..f316229499 100644 --- a/pkg/scheduler/queue/queue.go +++ b/pkg/scheduler/queue/queue.go @@ -107,7 +107,7 @@ func (q *RequestQueue) EnqueueRequest(userID string, req Request, maxQueriers fl return errors.New("no queue found") } - q.totalRequests.WithLabelValues(userID).Inc() + q.totalRequests.WithLabelValues(userID).Inc() // TODO: justinjung04 if queue.length() >= maxOutstandingRequests { q.discardedRequests.WithLabelValues(userID).Inc() diff --git a/pkg/scheduler/queue/user_queues.go b/pkg/scheduler/queue/user_queues.go index 7bf0f8c80b..9c1e2452f2 100644 --- a/pkg/scheduler/queue/user_queues.go +++ b/pkg/scheduler/queue/user_queues.go @@ -238,11 +238,11 @@ func (q *queues) updateUserQueuesAttributes(uq *userQueue, userID string, maxQue i := 0 for querierID := range uq.queriers { - reservedQueriers[querierID] = priorityList[i] - i++ if i == len(priorityList) { break } + reservedQueriers[querierID] = priorityList[i] + i++ } uq.reservedQueriers = reservedQueriers @@ -370,6 +370,10 @@ func (q *queues) recomputeUserQueriers() { // In that case *all* queriers should be used. // Scratchpad is used for shuffling, to avoid new allocations. If nil, new slice is allocated. func shuffleQueriersForUser(userSeed int64, queriersToSelect int, allSortedQueriers []string, scratchpad []string) map[string]struct{} { + if queriersToSelect == 0 || len(allSortedQueriers) <= queriersToSelect { + return nil + } + queriers := make(map[string]struct{}, queriersToSelect) rnd := rand.New(rand.NewSource(userSeed)) diff --git a/pkg/scheduler/queue/user_queues_test.go b/pkg/scheduler/queue/user_queues_test.go index 4b141dbf88..dd4cc3f5e1 100644 --- a/pkg/scheduler/queue/user_queues_test.go +++ b/pkg/scheduler/queue/user_queues_test.go @@ -156,10 +156,9 @@ func TestQueuesConsistency(t *testing.T) { conns := map[string]int{} for i := 0; i < 10000; i++ { - queue := uq.getOrAddQueue(generateTenant(r), 3, []int64{}, true) switch r.Int() % 6 { case 0: - assert.NotNil(t, queue) + assert.NotNil(t, uq.getOrAddQueue(generateTenant(r), 3, []int64{}, false)) case 1: qid := generateQuerier(r) _, _, luid := uq.getNextQueueForQuerier(lastUserIndexes[qid], qid) diff --git a/pkg/scheduler/queue/user_request_queue_test.go b/pkg/scheduler/queue/user_request_queue_test.go index ec4c728f89..84a4ce208e 100644 --- a/pkg/scheduler/queue/user_request_queue_test.go +++ b/pkg/scheduler/queue/user_request_queue_test.go @@ -26,6 +26,7 @@ func TestFIFORequestQueue(t *testing.T) { assert.Equal(t, 1, queue.length()) assert.Equal(t, request2, queue.dequeueRequest(0, false)) assert.Equal(t, 0, queue.length()) + queue.closeQueue() assert.Panics(t, func() { queue.enqueueRequest(request1) }) } @@ -53,7 +54,9 @@ func TestPriorityRequestQueue(t *testing.T) { queue.enqueueRequest(request2) assert.Equal(t, 2, queue.length()) assert.Equal(t, request2, queue.dequeueRequest(2, true)) + assert.Equal(t, 1, queue.length()) assert.Equal(t, request1, queue.dequeueRequest(2, false)) + assert.Equal(t, 0, queue.length()) queue.closeQueue() assert.Panics(t, func() { queue.enqueueRequest(request1) }) diff --git a/pkg/util/query/priority.go b/pkg/util/query/priority.go index e32c5df89b..fa63cd3c30 100644 --- a/pkg/util/query/priority.go +++ b/pkg/util/query/priority.go @@ -30,7 +30,8 @@ func GetPriority(requestParams url.Values, now time.Time, queryPriority *validat continue } - queryPriority.Priorities[i].QueryAttributes[j].CompiledRegex = compiledRegex + attribute.CompiledRegex = compiledRegex + queryPriority.Priorities[i].QueryAttributes[j] = attribute } if attribute.CompiledRegex != nil && !attribute.CompiledRegex.MatchString(queryParam) { diff --git a/pkg/util/query/priority_test.go b/pkg/util/query/priority_test.go index 0a93ac355b..0d1193a2f3 100644 --- a/pkg/util/query/priority_test.go +++ b/pkg/util/query/priority_test.go @@ -60,10 +60,12 @@ func Test_GetPriorityShouldConsiderRegex(t *testing.T) { Priorities: priorities, } + assert.Nil(t, queryPriority.Priorities[0].QueryAttributes[0].CompiledRegex) assert.Equal(t, int64(1), GetPriority(url.Values{ "query": []string{"sum(up)"}, "time": []string{strconv.FormatInt(now.Unix(), 10)}, }, now, &queryPriority, true)) + assert.NotNil(t, queryPriority.Priorities[0].QueryAttributes[0].CompiledRegex) assert.Equal(t, int64(0), GetPriority(url.Values{ "query": []string{"count(up)"}, "time": []string{strconv.FormatInt(now.Unix(), 10)}, @@ -91,6 +93,17 @@ func Test_GetPriorityShouldConsiderRegex(t *testing.T) { "time": []string{strconv.FormatInt(now.Unix(), 10)}, }, now, &queryPriority, false)) + queryPriority.Priorities[0].QueryAttributes[0].Regex = ".+" + + assert.Equal(t, int64(1), GetPriority(url.Values{ + "query": []string{"sum(up)"}, + "time": []string{strconv.FormatInt(now.Unix(), 10)}, + }, now, &queryPriority, true)) + assert.Equal(t, int64(1), GetPriority(url.Values{ + "query": []string{"count(up)"}, + "time": []string{strconv.FormatInt(now.Unix(), 10)}, + }, now, &queryPriority, false)) + queryPriority.Priorities[0].QueryAttributes[0].Regex = "" assert.Equal(t, int64(1), GetPriority(url.Values{ diff --git a/pkg/util/validation/limits_test.go b/pkg/util/validation/limits_test.go index c3b8c7e422..01c18225c2 100644 --- a/pkg/util/validation/limits_test.go +++ b/pkg/util/validation/limits_test.go @@ -154,64 +154,6 @@ func TestOverridesManager_GetOverrides(t *testing.T) { require.Equal(t, 0, ov.MaxLabelsSizeBytes("user2")) } -func TestQueryPriority(t *testing.T) { - type testCase struct { - regex string - compiledRegexNil bool - } - testCases := []testCase{ - { - regex: "", - compiledRegexNil: true, - }, - { - regex: ".*", - compiledRegexNil: true, - }, - { - regex: ".+", - compiledRegexNil: true, - }, - { - regex: "some_metric", - compiledRegexNil: false, - }, - } - - for _, tc := range testCases { - overrides, err := NewOverrides(Limits{ - QueryPriority: QueryPriority{ - Enabled: true, - Priorities: getPriorities(tc.regex, 1*time.Hour, 0*time.Hour), - }, - }, nil) - queryPriority := overrides.QueryPriority("") - - assert.NoError(t, err) - assert.Equal(t, 1, len(queryPriority.Priorities[0].QueryAttributes)) - if tc.compiledRegexNil { - assert.Nil(t, queryPriority.Priorities[0].QueryAttributes[0].CompiledRegex) - } else { - assert.NotNil(t, queryPriority.Priorities[0].QueryAttributes[0].CompiledRegex) - } - } -} - -func getPriorities(regex string, startTime, endTime time.Duration) []PriorityDef { - return []PriorityDef{ - { - Priority: 1, - QueryAttributes: []QueryAttribute{ - { - Regex: regex, - StartTime: startTime, - EndTime: endTime, - }, - }, - }, - } -} - func TestLimitsLoadingFromYaml(t *testing.T) { SetDefaultLimitsForYAMLUnmarshalling(Limits{ MaxLabelNameLength: 100, From e1b996f8dd8e1a88782606332eb7c1303bd02888 Mon Sep 17 00:00:00 2001 From: Justin Jung Date: Thu, 9 Nov 2023 05:19:00 -0800 Subject: [PATCH 27/49] Add priority label to request and queue metrics Signed-off-by: Justin Jung --- .../single-process-config-blocks-local.yaml | 13 +------------ pkg/frontend/v1/frontend.go | 4 ++-- pkg/scheduler/queue/queue.go | 12 +++++++----- pkg/scheduler/queue/queue_test.go | 5 +++++ pkg/scheduler/queue/user_queues.go | 3 --- pkg/scheduler/queue/user_queues_test.go | 4 ---- pkg/scheduler/queue/user_request_queue.go | 9 --------- pkg/scheduler/scheduler.go | 4 ++-- 8 files changed, 17 insertions(+), 37 deletions(-) diff --git a/docs/configuration/single-process-config-blocks-local.yaml b/docs/configuration/single-process-config-blocks-local.yaml index c04e9cc07c..ce21bc3ce4 100644 --- a/docs/configuration/single-process-config-blocks-local.yaml +++ b/docs/configuration/single-process-config-blocks-local.yaml @@ -87,15 +87,4 @@ ruler: ruler_storage: backend: local local: - directory: /tmp/cortex/rules - -limits: - query_priority: - enabled: true - default_priority: 1 - priorities: - - priority: 2 - reserved_queriers: 1 - query_attributes: - - start_time: 2h - end_time: 0s \ No newline at end of file + directory: /tmp/cortex/rules \ No newline at end of file diff --git a/pkg/frontend/v1/frontend.go b/pkg/frontend/v1/frontend.go index 7b952fff2b..c98cda732f 100644 --- a/pkg/frontend/v1/frontend.go +++ b/pkg/frontend/v1/frontend.go @@ -119,11 +119,11 @@ func New(cfg Config, limits Limits, log log.Logger, registerer prometheus.Regist queueLength: promauto.With(registerer).NewGaugeVec(prometheus.GaugeOpts{ Name: "cortex_query_frontend_queue_length", Help: "Number of queries in the queue.", - }, []string{"user"}), + }, []string{"user", "priority"}), discardedRequests: promauto.With(registerer).NewCounterVec(prometheus.CounterOpts{ Name: "cortex_query_frontend_discarded_requests_total", Help: "Total number of query requests discarded.", - }, []string{"user"}), + }, []string{"user", "priority"}), queueDuration: promauto.With(registerer).NewHistogram(prometheus.HistogramOpts{ Name: "cortex_query_frontend_queue_duration_seconds", Help: "Time spend by requests queued.", diff --git a/pkg/scheduler/queue/queue.go b/pkg/scheduler/queue/queue.go index f316229499..ab0296e678 100644 --- a/pkg/scheduler/queue/queue.go +++ b/pkg/scheduler/queue/queue.go @@ -2,6 +2,7 @@ package queue import ( "context" + "strconv" "sync" "time" @@ -74,7 +75,7 @@ func NewRequestQueue(maxOutstandingPerTenant int, forgetDelay time.Duration, que totalRequests: promauto.With(registerer).NewCounterVec(prometheus.CounterOpts{ Name: "cortex_request_queue_requests_total", Help: "Total number of query requests going to the request queue.", - }, []string{"user"}), + }, []string{"user", "priority"}), discardedRequests: discardedRequests, } @@ -101,21 +102,22 @@ func (q *RequestQueue) EnqueueRequest(userID string, req Request, maxQueriers fl priorityList, priorityEnabled := q.getPriorityList(userID) queue := q.queues.getOrAddQueue(userID, shardSize, priorityList, priorityEnabled) maxOutstandingRequests := q.queues.limits.MaxOutstandingPerTenant(userID) + priority := strconv.FormatInt(req.Priority(), 10) if queue == nil { // This can only happen if userID is "". return errors.New("no queue found") } - q.totalRequests.WithLabelValues(userID).Inc() // TODO: justinjung04 + q.totalRequests.WithLabelValues(userID, priority).Inc() if queue.length() >= maxOutstandingRequests { - q.discardedRequests.WithLabelValues(userID).Inc() + q.discardedRequests.WithLabelValues(userID, priority).Inc() return ErrTooManyRequests } queue.enqueueRequest(req) - q.queueLength.WithLabelValues(userID).Inc() + q.queueLength.WithLabelValues(userID, priority).Inc() q.cond.Broadcast() // Call this function while holding a lock. This guarantees that no querier can fetch the request before function returns. if successFn != nil { @@ -186,7 +188,7 @@ FindQueue: q.queues.deleteQueue(userID) } - q.queueLength.WithLabelValues(userID).Dec() + q.queueLength.WithLabelValues(userID, strconv.FormatInt(request.Priority(), 10)).Dec() // Tell close() we've processed a request. q.cond.Broadcast() diff --git a/pkg/scheduler/queue/queue_test.go b/pkg/scheduler/queue/queue_test.go index 535647bc7a..b2de565db3 100644 --- a/pkg/scheduler/queue/queue_test.go +++ b/pkg/scheduler/queue/queue_test.go @@ -228,6 +228,11 @@ func TestRequestQueue_ReservedQueriersShouldOnlyGetHighPriorityQueries(t *testin //assert.Equal(t, 1, queue.queues.getTotalQueueSize("userID")) } +func TestRequestQueue_EnqueueRequest(t *testing.T) { + // TODO: justinjung04 + // check updated limit as well +} + type MockRequest struct { id string priority int64 diff --git a/pkg/scheduler/queue/user_queues.go b/pkg/scheduler/queue/user_queues.go index 9c1e2452f2..b44c1e5a72 100644 --- a/pkg/scheduler/queue/user_queues.go +++ b/pkg/scheduler/queue/user_queues.go @@ -100,7 +100,6 @@ func (q *queues) deleteQueue(userID string) { return } - uq.queue.closeQueue() delete(q.userQueues, userID) q.users[uq.index] = "" @@ -225,7 +224,6 @@ func (q *queues) updateUserQueuesAttributes(uq *userQueue, userID string, maxQue // if query priority is newly enabled/disabled, transfer the requests to the new queue if uq.priorityEnabled != priorityEnabled { tmpQueue := q.createUserRequestQueue(userID) - uq.queue.closeQueue() for uq.queue.length() > 0 { tmpQueue.enqueueRequest(uq.queue.dequeueRequest(0, false)) @@ -361,7 +359,6 @@ func (q *queues) recomputeUserQueriers() { scratchpad := make([]string, 0, len(q.sortedQueriers)) for _, uq := range q.userQueues { - //userID := q.users[uq.index] uq.queriers = shuffleQueriersForUser(uq.seed, uq.maxQueriers, q.sortedQueriers, scratchpad) } } diff --git a/pkg/scheduler/queue/user_queues_test.go b/pkg/scheduler/queue/user_queues_test.go index dd4cc3f5e1..c6c6e8cfc2 100644 --- a/pkg/scheduler/queue/user_queues_test.go +++ b/pkg/scheduler/queue/user_queues_test.go @@ -351,10 +351,6 @@ func TestQueues_ForgetDelay_ShouldCorrectlyHandleQuerierReconnectingBeforeForget } } -func TestQueuesUpdatedConfig(t *testing.T) { - // TODO: justinjung04 -} - func generateTenant(r *rand.Rand) string { return fmt.Sprint("tenant-", r.Int()%5) } diff --git a/pkg/scheduler/queue/user_request_queue.go b/pkg/scheduler/queue/user_request_queue.go index 77afa31806..9f95c9e8c5 100644 --- a/pkg/scheduler/queue/user_request_queue.go +++ b/pkg/scheduler/queue/user_request_queue.go @@ -6,7 +6,6 @@ type userRequestQueue interface { enqueueRequest(Request) dequeueRequest(minPriority int64, checkMinPriority bool) Request length() int - closeQueue() } type FIFORequestQueue struct { @@ -29,10 +28,6 @@ func (f *FIFORequestQueue) length() int { return len(f.queue) } -func (f *FIFORequestQueue) closeQueue() { - close(f.queue) -} - type PriorityRequestQueue struct { queue *util.PriorityQueue } @@ -55,7 +50,3 @@ func (f *PriorityRequestQueue) dequeueRequest(minPriority int64, checkMinPriorit func (f *PriorityRequestQueue) length() int { return f.queue.Length() } - -func (f *PriorityRequestQueue) closeQueue() { - f.queue.Close() -} diff --git a/pkg/scheduler/scheduler.go b/pkg/scheduler/scheduler.go index 4c3a0afcf8..2d7c61dac1 100644 --- a/pkg/scheduler/scheduler.go +++ b/pkg/scheduler/scheduler.go @@ -105,12 +105,12 @@ func NewScheduler(cfg Config, limits Limits, log log.Logger, registerer promethe s.queueLength = promauto.With(registerer).NewGaugeVec(prometheus.GaugeOpts{ Name: "cortex_query_scheduler_queue_length", Help: "Number of queries in the queue.", - }, []string{"user"}) + }, []string{"user", "priority"}) s.discardedRequests = promauto.With(registerer).NewCounterVec(prometheus.CounterOpts{ Name: "cortex_query_scheduler_discarded_requests_total", Help: "Total number of query requests discarded.", - }, []string{"user"}) + }, []string{"user", "priority"}) s.requestQueue = queue.NewRequestQueue(cfg.MaxOutstandingPerTenant, cfg.QuerierForgetDelay, s.queueLength, s.discardedRequests, s.limits, registerer) From 2f7947c334af7b737dd8ae2f38aaaef4cc96c768 Mon Sep 17 00:00:00 2001 From: Justin Jung Date: Thu, 9 Nov 2023 08:39:11 -0800 Subject: [PATCH 28/49] Fix tests Signed-off-by: Justin Jung --- .../configuration/single-process-config-blocks-local.yaml | 2 +- pkg/frontend/v1/frontend.go | 8 ++++++-- pkg/frontend/v1/frontend_test.go | 2 +- pkg/scheduler/scheduler.go | 8 ++++++-- pkg/scheduler/scheduler_test.go | 6 +++--- 5 files changed, 17 insertions(+), 9 deletions(-) diff --git a/docs/configuration/single-process-config-blocks-local.yaml b/docs/configuration/single-process-config-blocks-local.yaml index ce21bc3ce4..a5eb711d97 100644 --- a/docs/configuration/single-process-config-blocks-local.yaml +++ b/docs/configuration/single-process-config-blocks-local.yaml @@ -87,4 +87,4 @@ ruler: ruler_storage: backend: local local: - directory: /tmp/cortex/rules \ No newline at end of file + directory: /tmp/cortex/rules diff --git a/pkg/frontend/v1/frontend.go b/pkg/frontend/v1/frontend.go index c98cda732f..dbe5c28ae1 100644 --- a/pkg/frontend/v1/frontend.go +++ b/pkg/frontend/v1/frontend.go @@ -176,8 +176,12 @@ func (f *Frontend) stopping(_ error) error { } func (f *Frontend) cleanupInactiveUserMetrics(user string) { - f.queueLength.DeleteLabelValues(user) - f.discardedRequests.DeleteLabelValues(user) + f.queueLength.DeletePartialMatch(prometheus.Labels{ + "user": user, + }) + f.discardedRequests.DeletePartialMatch(prometheus.Labels{ + "user": user, + }) } // RoundTripGRPC round trips a proto (instead of a HTTP request). diff --git a/pkg/frontend/v1/frontend_test.go b/pkg/frontend/v1/frontend_test.go index 7951dd1d60..316c8c25fe 100644 --- a/pkg/frontend/v1/frontend_test.go +++ b/pkg/frontend/v1/frontend_test.go @@ -211,7 +211,7 @@ func TestFrontendMetricsCleanup(t *testing.T) { require.NoError(t, testutil.GatherAndCompare(reg, strings.NewReader(` # HELP cortex_query_frontend_queue_length Number of queries in the queue. # TYPE cortex_query_frontend_queue_length gauge - cortex_query_frontend_queue_length{user="1"} 0 + cortex_query_frontend_queue_length{priority="0",user="1"} 0 `), "cortex_query_frontend_queue_length")) fr.cleanupInactiveUserMetrics("1") diff --git a/pkg/scheduler/scheduler.go b/pkg/scheduler/scheduler.go index 2d7c61dac1..b96b659e4b 100644 --- a/pkg/scheduler/scheduler.go +++ b/pkg/scheduler/scheduler.go @@ -520,8 +520,12 @@ func (s *Scheduler) stopping(_ error) error { } func (s *Scheduler) cleanupMetricsForInactiveUser(user string) { - s.queueLength.DeleteLabelValues(user) - s.discardedRequests.DeleteLabelValues(user) + s.queueLength.DeletePartialMatch(prometheus.Labels{ + "user": user, + }) + s.discardedRequests.DeletePartialMatch(prometheus.Labels{ + "user": user, + }) } func (s *Scheduler) getConnectedFrontendClientsMetric() float64 { diff --git a/pkg/scheduler/scheduler_test.go b/pkg/scheduler/scheduler_test.go index c309ff195e..b055754133 100644 --- a/pkg/scheduler/scheduler_test.go +++ b/pkg/scheduler/scheduler_test.go @@ -430,8 +430,8 @@ func TestSchedulerMetrics(t *testing.T) { require.NoError(t, promtest.GatherAndCompare(reg, strings.NewReader(` # HELP cortex_query_scheduler_queue_length Number of queries in the queue. # TYPE cortex_query_scheduler_queue_length gauge - cortex_query_scheduler_queue_length{user="another"} 1 - cortex_query_scheduler_queue_length{user="test"} 1 + cortex_query_scheduler_queue_length{priority="0",user="another"} 1 + cortex_query_scheduler_queue_length{priority="0",user="test"} 1 `), "cortex_query_scheduler_queue_length")) scheduler.cleanupMetricsForInactiveUser("test") @@ -439,7 +439,7 @@ func TestSchedulerMetrics(t *testing.T) { require.NoError(t, promtest.GatherAndCompare(reg, strings.NewReader(` # HELP cortex_query_scheduler_queue_length Number of queries in the queue. # TYPE cortex_query_scheduler_queue_length gauge - cortex_query_scheduler_queue_length{user="another"} 1 + cortex_query_scheduler_queue_length{priority="0",user="another"} 1 `), "cortex_query_scheduler_queue_length")) } From f9df45396673eefde8a1c43acd13a48a16b2878e Mon Sep 17 00:00:00 2001 From: Justin Jung Date: Thu, 9 Nov 2023 09:15:42 -0800 Subject: [PATCH 29/49] Fix tests Signed-off-by: Justin Jung --- pkg/scheduler/queue/user_request_queue_test.go | 6 ------ 1 file changed, 6 deletions(-) diff --git a/pkg/scheduler/queue/user_request_queue_test.go b/pkg/scheduler/queue/user_request_queue_test.go index 84a4ce208e..6001f8966c 100644 --- a/pkg/scheduler/queue/user_request_queue_test.go +++ b/pkg/scheduler/queue/user_request_queue_test.go @@ -26,9 +26,6 @@ func TestFIFORequestQueue(t *testing.T) { assert.Equal(t, 1, queue.length()) assert.Equal(t, request2, queue.dequeueRequest(0, false)) assert.Equal(t, 0, queue.length()) - - queue.closeQueue() - assert.Panics(t, func() { queue.enqueueRequest(request1) }) } func TestPriorityRequestQueue(t *testing.T) { @@ -57,7 +54,4 @@ func TestPriorityRequestQueue(t *testing.T) { assert.Equal(t, 1, queue.length()) assert.Equal(t, request1, queue.dequeueRequest(2, false)) assert.Equal(t, 0, queue.length()) - - queue.closeQueue() - assert.Panics(t, func() { queue.enqueueRequest(request1) }) } From a510f77db41339bb5e547a5c1088b241d3c304c7 Mon Sep 17 00:00:00 2001 From: Justin Jung Date: Thu, 9 Nov 2023 11:53:39 -0800 Subject: [PATCH 30/49] Refactor Signed-off-by: Justin Jung --- pkg/scheduler/queue/queue.go | 54 ++++---- pkg/scheduler/queue/user_queues.go | 177 ++++++++++++------------ pkg/scheduler/queue/user_queues_test.go | 6 +- 3 files changed, 118 insertions(+), 119 deletions(-) diff --git a/pkg/scheduler/queue/queue.go b/pkg/scheduler/queue/queue.go index ab0296e678..0502e9dd46 100644 --- a/pkg/scheduler/queue/queue.go +++ b/pkg/scheduler/queue/queue.go @@ -62,9 +62,9 @@ type RequestQueue struct { queues *queues stopped bool - queueLength *prometheus.GaugeVec // Per user and reason. - totalRequests *prometheus.CounterVec // Per user. - discardedRequests *prometheus.CounterVec // Per user. + queueLength *prometheus.GaugeVec // Per user and priority. + totalRequests *prometheus.CounterVec // Per user and priority. + discardedRequests *prometheus.CounterVec // Per user and priority. } func NewRequestQueue(maxOutstandingPerTenant int, forgetDelay time.Duration, queueLength *prometheus.GaugeVec, discardedRequests *prometheus.CounterVec, limits Limits, registerer prometheus.Registerer) *RequestQueue { @@ -98,9 +98,8 @@ func (q *RequestQueue) EnqueueRequest(userID string, req Request, maxQueriers fl return ErrStopped } - shardSize := util.DynamicShardSize(maxQueriers, len(q.queues.queriers)) - priorityList, priorityEnabled := q.getPriorityList(userID) - queue := q.queues.getOrAddQueue(userID, shardSize, priorityList, priorityEnabled) + maxQuerierCount := util.DynamicShardSize(maxQueriers, len(q.queues.queriers)) + queue := q.queues.getOrAddQueue(userID, maxQuerierCount) maxOutstandingRequests := q.queues.limits.MaxOutstandingPerTenant(userID) priority := strconv.FormatInt(req.Priority(), 10) @@ -109,15 +108,19 @@ func (q *RequestQueue) EnqueueRequest(userID string, req Request, maxQueriers fl return errors.New("no queue found") } - q.totalRequests.WithLabelValues(userID, priority).Inc() + metricLabels := prometheus.Labels{ + "user": userID, + "priority": priority, + } + q.totalRequests.With(metricLabels).Inc() if queue.length() >= maxOutstandingRequests { - q.discardedRequests.WithLabelValues(userID, priority).Inc() + q.discardedRequests.With(metricLabels).Inc() return ErrTooManyRequests } queue.enqueueRequest(req) - q.queueLength.WithLabelValues(userID, priority).Inc() + q.queueLength.With(metricLabels).Inc() q.cond.Broadcast() // Call this function while holding a lock. This guarantees that no querier can fetch the request before function returns. if successFn != nil { @@ -126,24 +129,6 @@ func (q *RequestQueue) EnqueueRequest(userID string, req Request, maxQueriers fl return nil } -func (q *RequestQueue) getPriorityList(userID string) ([]int64, bool) { - var priorityList []int64 - - queryPriority := q.queues.limits.QueryPriority(userID) - - if queryPriority.Enabled { - for _, priority := range queryPriority.Priorities { - reservedQuerierShardSize := util.DynamicShardSize(priority.ReservedQueriers, len(q.queues.queriers)) - - for i := 0; i < reservedQuerierShardSize; i++ { - priorityList = append(priorityList, priority.Priority) - } - } - } - - return priorityList, queryPriority.Enabled -} - // GetNextRequestForQuerier find next user queue and takes the next request off of it. Will block if there are no requests. // By passing user index from previous call of this method, querier guarantees that it iterates over all users fairly. // If querier finds that request from the user is already expired, it can get a request for the same user by using UserIndex.ReuseLastUser. @@ -177,7 +162,7 @@ FindQueue: // Pick next request from the queue. for { - minPriority, checkMinPriority := q.queues.getMinPriority(userID, querierID) + minPriority, checkMinPriority := q.getMinPriorityForQuerier(userID, querierID) request := queue.dequeueRequest(minPriority, checkMinPriority) if request == nil { // the queue does not contain request with the min priority, break to wait for more requests @@ -188,7 +173,11 @@ FindQueue: q.queues.deleteQueue(userID) } - q.queueLength.WithLabelValues(userID, strconv.FormatInt(request.Priority(), 10)).Dec() + metricLabels := prometheus.Labels{ + "user": userID, + "priority": strconv.FormatInt(request.Priority(), 10), + } + q.queueLength.With(metricLabels).Dec() // Tell close() we've processed a request. q.cond.Broadcast() @@ -203,6 +192,13 @@ FindQueue: goto FindQueue } +func (q *RequestQueue) getMinPriorityForQuerier(userID string, querierID string) (int64, bool) { + if priority, ok := q.queues.userQueues[userID].reservedQueriers[querierID]; ok { + return priority, true + } + return 0, false +} + func (q *RequestQueue) forgetDisconnectedQueriers(_ context.Context) error { q.mtx.Lock() defer q.mtx.Unlock() diff --git a/pkg/scheduler/queue/user_queues.go b/pkg/scheduler/queue/user_queues.go index b44c1e5a72..46c0bb598a 100644 --- a/pkg/scheduler/queue/user_queues.go +++ b/pkg/scheduler/queue/user_queues.go @@ -65,10 +65,11 @@ type userQueue struct { // If not nil, only these queriers can handle user requests. If nil, all queriers can. // We set this to nil if number of available queriers <= maxQueriers. queriers map[string]struct{} + maxQueriers int + maxOutstanding int reservedQueriers map[string]int64 priorityList []int64 priorityEnabled bool - maxQueriers int // Seed for shuffle sharding of queriers. This seed is based on userID only and is therefore consistent // between different frontends. @@ -113,7 +114,8 @@ func (q *queues) deleteQueue(userID string) { // MaxQueriers is used to compute which queriers should handle requests for this user. // If maxQueriers is <= 0, all queriers can handle this user's requests. // If maxQueriers has changed since the last call, queriers for this are recomputed. -func (q *queues) getOrAddQueue(userID string, maxQueriers int, priorityList []int64, priorityEnabled bool) userRequestQueue { +// It's also responsible to store user configs and update the attributes if related configs have changed. +func (q *queues) getOrAddQueue(userID string, maxQueriers int) userRequestQueue { // Empty user is not allowed, as that would break our users list ("" is used for free spot). if userID == "" { return nil @@ -124,75 +126,69 @@ func (q *queues) getOrAddQueue(userID string, maxQueriers int, priorityList []in } uq := q.userQueues[userID] + priorityEnabled := q.limits.QueryPriority(userID).Enabled + maxOutstanding := q.limits.MaxOutstandingPerTenant(userID) + priorityList := getPriorityList(q.limits.QueryPriority(userID), len(q.queriers)) if uq == nil { - uq = q.createUserQueue(userID) - } - - q.updateUserQueuesAttributes(uq, userID, maxQueriers, priorityList, priorityEnabled) + uq = &userQueue{ + seed: util.ShuffleShardSeed(userID, ""), + index: -1, + } - return uq.queue -} + uq.queue = q.createUserRequestQueue(userID) + uq.maxOutstanding = q.limits.MaxOutstandingPerTenant(userID) + q.userQueues[userID] = uq -// Finds next queue for the querier. To support fair scheduling between users, client is expected -// to pass last user index returned by this function as argument. Is there was no previous -// last user index, use -1. -func (q *queues) getNextQueueForQuerier(lastUserIndex int, querierID string) (userRequestQueue, string, int) { - uid := lastUserIndex - - for iters := 0; iters < len(q.users); iters++ { - uid = uid + 1 - - // Don't use "mod len(q.users)", as that could skip users at the beginning of the list - // for example when q.users has shrunk since last call. - if uid >= len(q.users) { - uid = 0 + // Add user to the list of users... find first free spot, and put it there. + for ix, u := range q.users { + if u == "" { + uq.index = ix + q.users[ix] = userID + break + } } - u := q.users[uid] - if u == "" { - continue + // ... or add to the end. + if uq.index < 0 { + uq.index = len(q.users) + q.users = append(q.users, userID) } + } else if (uq.priorityEnabled != priorityEnabled) || (uq.maxOutstanding != maxOutstanding && priorityEnabled) { + tmpQueue := q.createUserRequestQueue(userID) - uq := q.userQueues[u] - - if uq.queriers != nil { - if _, ok := uq.queriers[querierID]; !ok { - // This querier is not handling the user. - continue - } + // flush to new queue + for uq.queue.length() > 0 { + tmpQueue.enqueueRequest(uq.queue.dequeueRequest(0, false)) } - return uq.queue, u, uid + uq.queue = tmpQueue + uq.maxOutstanding = q.limits.MaxOutstandingPerTenant(userID) } - return nil, "", uid -} -func (q *queues) createUserQueue(userID string) *userQueue { - uq := &userQueue{ - seed: util.ShuffleShardSeed(userID, ""), - index: -1, + if uq.maxQueriers != maxQueriers { + uq.maxQueriers = maxQueriers + uq.queriers = shuffleQueriersForUser(uq.seed, maxQueriers, q.sortedQueriers, nil) } - uq.queue = q.createUserRequestQueue(userID) - q.userQueues[userID] = uq + if priorityEnabled && !reflect.DeepEqual(uq.priorityList, priorityList) { + reservedQueriers := make(map[string]int64) - // Add user to the list of users... find first free spot, and put it there. - for ix, u := range q.users { - if u == "" { - uq.index = ix - q.users[ix] = userID - break + i := 0 + for querierID := range uq.queriers { + if i == len(priorityList) { + break + } + reservedQueriers[querierID] = priorityList[i] + i++ } - } - // ... or add to the end. - if uq.index < 0 { - uq.index = len(q.users) - q.users = append(q.users, userID) + uq.reservedQueriers = reservedQueriers + uq.priorityList = priorityList + uq.priorityEnabled = priorityEnabled } - return uq + return uq.queue } func (q *queues) createUserRequestQueue(userID string) userRequestQueue { @@ -200,10 +196,6 @@ func (q *queues) createUserRequestQueue(userID string) userRequestQueue { return NewPriorityRequestQueue(util.NewPriorityQueue(nil)) } - return NewFIFORequestQueue(make(chan Request, q.getQueueSize(userID))) -} - -func (q *queues) getQueueSize(userID string) int { queueSize := q.limits.MaxOutstandingPerTenant(userID) // 0 is the default value of the flag. If the old flag is set @@ -212,48 +204,41 @@ func (q *queues) getQueueSize(userID string) int { queueSize = q.maxUserQueueSize } - return queueSize + return NewFIFORequestQueue(make(chan Request, queueSize)) } -func (q *queues) updateUserQueuesAttributes(uq *userQueue, userID string, maxQueriers int, priorityList []int64, priorityEnabled bool) { - if uq.maxQueriers != maxQueriers { - uq.maxQueriers = maxQueriers - uq.queriers = shuffleQueriersForUser(uq.seed, maxQueriers, q.sortedQueriers, nil) - } +// Finds next queue for the querier. To support fair scheduling between users, client is expected +// to pass last user index returned by this function as argument. Is there was no previous +// last user index, use -1. +func (q *queues) getNextQueueForQuerier(lastUserIndex int, querierID string) (userRequestQueue, string, int) { + uid := lastUserIndex - // if query priority is newly enabled/disabled, transfer the requests to the new queue - if uq.priorityEnabled != priorityEnabled { - tmpQueue := q.createUserRequestQueue(userID) + for iters := 0; iters < len(q.users); iters++ { + uid = uid + 1 - for uq.queue.length() > 0 { - tmpQueue.enqueueRequest(uq.queue.dequeueRequest(0, false)) + // Don't use "mod len(q.users)", as that could skip users at the beginning of the list + // for example when q.users has shrunk since last call. + if uid >= len(q.users) { + uid = 0 } - uq.queue = tmpQueue - } - if priorityEnabled && !reflect.DeepEqual(uq.priorityList, priorityList) { - reservedQueriers := make(map[string]int64) + u := q.users[uid] + if u == "" { + continue + } - i := 0 - for querierID := range uq.queriers { - if i == len(priorityList) { - break + uq := q.userQueues[u] + + if uq.queriers != nil { + if _, ok := uq.queriers[querierID]; !ok { + // This querier is not handling the user. + continue } - reservedQueriers[querierID] = priorityList[i] - i++ } - uq.reservedQueriers = reservedQueriers - uq.priorityList = priorityList - uq.priorityEnabled = priorityEnabled - } -} - -func (q *queues) getMinPriority(userID string, querierID string) (int64, bool) { - if priority, ok := q.userQueues[userID].reservedQueriers[querierID]; ok { - return priority, true + return uq.queue, u, uid } - return 0, false + return nil, "", uid } func (q *queues) addQuerierConnection(querierID string) { @@ -388,6 +373,24 @@ func shuffleQueriersForUser(userSeed int64, queriersToSelect int, allSortedQueri return queriers } +// getPriorityList returns a list of priorities, each priority repeated as much as number of reserved queriers. +// This is used when creating map of reserved queriers. +func getPriorityList(queryPriority validation.QueryPriority, totalQuerierCount int) []int64 { + var priorityList []int64 + + if queryPriority.Enabled { + for _, priority := range queryPriority.Priorities { + reservedQuerierShardSize := util.DynamicShardSize(priority.ReservedQueriers, totalQuerierCount) + + for i := 0; i < reservedQuerierShardSize; i++ { + priorityList = append(priorityList, priority.Priority) + } + } + } + + return priorityList +} + // MockLimits implements the Limits interface. Used in tests only. type MockLimits struct { MaxOutstanding int diff --git a/pkg/scheduler/queue/user_queues_test.go b/pkg/scheduler/queue/user_queues_test.go index c6c6e8cfc2..235f1cb187 100644 --- a/pkg/scheduler/queue/user_queues_test.go +++ b/pkg/scheduler/queue/user_queues_test.go @@ -158,7 +158,7 @@ func TestQueuesConsistency(t *testing.T) { for i := 0; i < 10000; i++ { switch r.Int() % 6 { case 0: - assert.NotNil(t, uq.getOrAddQueue(generateTenant(r), 3, []int64{}, false)) + assert.NotNil(t, uq.getOrAddQueue(generateTenant(r), 3)) case 1: qid := generateQuerier(r) _, _, luid := uq.getNextQueueForQuerier(lastUserIndexes[qid], qid) @@ -360,10 +360,10 @@ func generateQuerier(r *rand.Rand) string { } func getOrAdd(t *testing.T, uq *queues, tenant string, maxQueriers int) userRequestQueue { - q := uq.getOrAddQueue(tenant, maxQueriers, []int64{}, true) + q := uq.getOrAddQueue(tenant, maxQueriers) assert.NotNil(t, q) assert.NoError(t, isConsistent(uq)) - assert.Equal(t, q, uq.getOrAddQueue(tenant, maxQueriers, []int64{}, true)) + assert.Equal(t, q, uq.getOrAddQueue(tenant, maxQueriers)) return q } From 717d746d5b99a354f6fd5deef917a82311354a1b Mon Sep 17 00:00:00 2001 From: Justin Jung Date: Fri, 10 Nov 2023 10:26:46 -0800 Subject: [PATCH 31/49] Add more tests Signed-off-by: Justin Jung --- pkg/frontend/v1/frontend.go | 2 +- pkg/scheduler/queue/queue.go | 20 ++- pkg/scheduler/queue/queue_test.go | 147 ++++++++++++------ pkg/scheduler/queue/user_queues.go | 24 +-- pkg/scheduler/queue/user_queues_test.go | 111 +++++++++++++ .../queue/user_request_queue_test.go | 1 + pkg/scheduler/scheduler.go | 2 +- pkg/util/priority_queue.go | 2 +- pkg/util/priority_queue_test.go | 18 +++ pkg/util/query/priority_test.go | 36 +++++ 10 files changed, 301 insertions(+), 62 deletions(-) diff --git a/pkg/frontend/v1/frontend.go b/pkg/frontend/v1/frontend.go index dbe5c28ae1..de5d2d0647 100644 --- a/pkg/frontend/v1/frontend.go +++ b/pkg/frontend/v1/frontend.go @@ -119,7 +119,7 @@ func New(cfg Config, limits Limits, log log.Logger, registerer prometheus.Regist queueLength: promauto.With(registerer).NewGaugeVec(prometheus.GaugeOpts{ Name: "cortex_query_frontend_queue_length", Help: "Number of queries in the queue.", - }, []string{"user", "priority"}), + }, []string{"user", "priority", "type"}), discardedRequests: promauto.With(registerer).NewCounterVec(prometheus.CounterOpts{ Name: "cortex_query_frontend_discarded_requests_total", Help: "Total number of query requests discarded.", diff --git a/pkg/scheduler/queue/queue.go b/pkg/scheduler/queue/queue.go index 0502e9dd46..aa29e061f8 100644 --- a/pkg/scheduler/queue/queue.go +++ b/pkg/scheduler/queue/queue.go @@ -62,7 +62,7 @@ type RequestQueue struct { queues *queues stopped bool - queueLength *prometheus.GaugeVec // Per user and priority. + queueLength *prometheus.GaugeVec // Per user, priority and type of the queue (fifo or priority). totalRequests *prometheus.CounterVec // Per user and priority. discardedRequests *prometheus.CounterVec // Per user and priority. } @@ -108,6 +108,10 @@ func (q *RequestQueue) EnqueueRequest(userID string, req Request, maxQueriers fl return errors.New("no queue found") } + queueType := "fifo" + if q.queues.limits.QueryPriority(userID).Enabled { + queueType = "priority" + } metricLabels := prometheus.Labels{ "user": userID, "priority": priority, @@ -120,7 +124,11 @@ func (q *RequestQueue) EnqueueRequest(userID string, req Request, maxQueriers fl } queue.enqueueRequest(req) - q.queueLength.With(metricLabels).Inc() + q.queueLength.With(prometheus.Labels{ + "user": userID, + "priority": priority, + "type": queueType, + }).Inc() q.cond.Broadcast() // Call this function while holding a lock. This guarantees that no querier can fetch the request before function returns. if successFn != nil { @@ -166,16 +174,22 @@ FindQueue: request := queue.dequeueRequest(minPriority, checkMinPriority) if request == nil { // the queue does not contain request with the min priority, break to wait for more requests - break + querierWait = true + goto FindQueue } if queue.length() == 0 { q.queues.deleteQueue(userID) } + queueType := "fifo" + if q.queues.limits.QueryPriority(userID).Enabled { + queueType = "priority" + } metricLabels := prometheus.Labels{ "user": userID, "priority": strconv.FormatInt(request.Priority(), 10), + "type": queueType, } q.queueLength.With(metricLabels).Dec() diff --git a/pkg/scheduler/queue/queue_test.go b/pkg/scheduler/queue/queue_test.go index b2de565db3..02317338d0 100644 --- a/pkg/scheduler/queue/queue_test.go +++ b/pkg/scheduler/queue/queue_test.go @@ -25,8 +25,8 @@ func BenchmarkGetNextRequest(b *testing.B) { for n := 0; n < b.N; n++ { queue := NewRequestQueue(maxOutstandingPerTenant, 0, - prometheus.NewGaugeVec(prometheus.GaugeOpts{}, []string{"user"}), - prometheus.NewCounterVec(prometheus.CounterOpts{}, []string{"user"}), + prometheus.NewGaugeVec(prometheus.GaugeOpts{}, []string{"user", "priority", "type"}), + prometheus.NewCounterVec(prometheus.CounterOpts{}, []string{"user", "priority"}), MockLimits{MaxOutstanding: 100}, nil, ) @@ -119,8 +119,8 @@ func TestRequestQueue_GetNextRequestForQuerier_ShouldGetRequestAfterReshardingBe const forgetDelay = 3 * time.Second queue := NewRequestQueue(1, forgetDelay, - prometheus.NewGaugeVec(prometheus.GaugeOpts{}, []string{"user"}), - prometheus.NewCounterVec(prometheus.CounterOpts{}, []string{"user"}), + prometheus.NewGaugeVec(prometheus.GaugeOpts{}, []string{"user", "priority", "type"}), + prometheus.NewCounterVec(prometheus.CounterOpts{}, []string{"user", "priority"}), MockLimits{MaxOutstanding: 100}, nil, ) @@ -160,11 +160,11 @@ func TestRequestQueue_GetNextRequestForQuerier_ShouldGetRequestAfterReshardingBe assert.GreaterOrEqual(t, waitTime.Milliseconds(), forgetDelay.Milliseconds()) } -func TestRequestQueue_QueriersShouldGetHighPriorityQueryFirst(t *testing.T) { +func TestQueriersShouldGetHighPriorityQueryFirst(t *testing.T) { queue := NewRequestQueue(0, 0, - prometheus.NewGaugeVec(prometheus.GaugeOpts{}, []string{"user"}), - prometheus.NewCounterVec(prometheus.CounterOpts{}, []string{"user"}), - MockLimits{MaxOutstanding: 3, queryPriority: validation.QueryPriority{Enabled: true}}, + prometheus.NewGaugeVec(prometheus.GaugeOpts{}, []string{"user", "priority", "type"}), + prometheus.NewCounterVec(prometheus.CounterOpts{}, []string{"user", "priority"}), + MockLimits{MaxOutstanding: 3, QueryPriorityVal: validation.QueryPriority{Enabled: true}}, nil, ) ctx := context.Background() @@ -190,47 +190,100 @@ func TestRequestQueue_QueriersShouldGetHighPriorityQueryFirst(t *testing.T) { assert.Equal(t, highPriorityRequest, nextRequest) // high priority request returned, although it was enqueued the last } -func TestRequestQueue_ReservedQueriersShouldOnlyGetHighPriorityQueries(t *testing.T) { - // TODO: justinjung04 - //queue := NewRequestQueue(0, 0, - // prometheus.NewGaugeVec(prometheus.GaugeOpts{}, []string{"user"}), - // prometheus.NewCounterVec(prometheus.CounterOpts{}, []string{"user"}), - // MockLimits{MaxOutstanding: 3}, - // nil, - //) - //ctx := context.Background() - // - //queue.RegisterQuerierConnection("querier-1") - // - //normalRequest := MockRequest{ - // id: "normal query", - // isHighPriority: false, - //} - //highPriorityRequest := MockRequest{ - // id: "high priority query", - // isHighPriority: true, - //} - // - //assert.NoError(t, queue.EnqueueRequest("userID", normalRequest, 1, func() {})) - //assert.NoError(t, queue.EnqueueRequest("userID", highPriorityRequest, 1, func() {})) - // - //nextRequest, _, _ := queue.GetNextRequestForQuerier(ctx, FirstUser(), "querier-1") - //assert.Equal(t, highPriorityRequest, nextRequest) - // - //ctxTimeout, cancel := context.WithTimeout(ctx, 1*time.Second) - //defer cancel() - // - //time.AfterFunc(2*time.Second, func() { - // queue.cond.Broadcast() - //}) - //nextRequest, _, _ = queue.GetNextRequestForQuerier(ctxTimeout, FirstUser(), "querier-1") - //assert.Nil(t, nextRequest) - //assert.Equal(t, 1, queue.queues.getTotalQueueSize("userID")) +func TestReservedQueriersShouldOnlyGetHighPriorityQueries(t *testing.T) { + queue := NewRequestQueue(0, 0, + prometheus.NewGaugeVec(prometheus.GaugeOpts{}, []string{"user", "priority", "type"}), + prometheus.NewCounterVec(prometheus.CounterOpts{}, []string{"user", "priority"}), + MockLimits{ + MaxOutstanding: 3, + QueryPriorityVal: validation.QueryPriority{ + Enabled: true, + Priorities: []validation.PriorityDef{ + { + Priority: 1, + ReservedQueriers: 1, + }, + }, + }, + }, + nil, + ) + ctx := context.Background() + + queue.RegisterQuerierConnection("querier-1") + + normalRequest := MockRequest{ + id: "normal query", + } + priority1Request := MockRequest{ + id: "priority 1", + priority: 1, + } + priority2Request := MockRequest{ + id: "priority 2", + priority: 2, + } + + assert.NoError(t, queue.EnqueueRequest("userID", normalRequest, 1, func() {})) + assert.NoError(t, queue.EnqueueRequest("userID", priority1Request, 1, func() {})) + assert.NoError(t, queue.EnqueueRequest("userID", priority2Request, 1, func() {})) + + nextRequest, _, _ := queue.GetNextRequestForQuerier(ctx, FirstUser(), "querier-1") + assert.Equal(t, priority2Request, nextRequest) + + nextRequest, _, _ = queue.GetNextRequestForQuerier(ctx, FirstUser(), "querier-1") + assert.Equal(t, priority1Request, nextRequest) + + ctxTimeout, cancel := context.WithTimeout(ctx, 1*time.Second) + defer cancel() + + time.AfterFunc(2*time.Second, func() { + queue.cond.Broadcast() + }) + nextRequest, _, _ = queue.GetNextRequestForQuerier(ctxTimeout, FirstUser(), "querier-1") + assert.Nil(t, nextRequest) + assert.Equal(t, 1, queue.queues.userQueues["userID"].queue.length()) } -func TestRequestQueue_EnqueueRequest(t *testing.T) { - // TODO: justinjung04 - // check updated limit as well +func TestExitingRequestsShouldPersistEvenIfTheConfigHasChanged(t *testing.T) { + limits := MockLimits{ + MaxOutstanding: 3, + } + queue := NewRequestQueue(0, 0, + prometheus.NewGaugeVec(prometheus.GaugeOpts{}, []string{"user", "priority", "type"}), + prometheus.NewCounterVec(prometheus.CounterOpts{}, []string{"user", "priority"}), + limits, + nil, + ) + + ctx := context.Background() + + queue.RegisterQuerierConnection("querier-1") + + normalRequest := MockRequest{ + id: "normal query", + } + highPriorityRequest := MockRequest{ + id: "high priority query", + priority: 1, + } + + assert.NoError(t, queue.EnqueueRequest("userID", normalRequest, 1, func() {})) + assert.NoError(t, queue.EnqueueRequest("userID", highPriorityRequest, 1, func() {})) + assert.NoError(t, queue.EnqueueRequest("userID", normalRequest, 1, func() {})) + + limits.MaxOutstanding = 4 + limits.QueryPriorityVal = validation.QueryPriority{Enabled: true} + queue.queues.limits = limits + + assert.NoError(t, queue.EnqueueRequest("userID", normalRequest, 1, func() {})) + + nextRequest, _, _ := queue.GetNextRequestForQuerier(ctx, FirstUser(), "querier-1") + assert.Equal(t, highPriorityRequest, nextRequest) + + nextRequest, _, _ = queue.GetNextRequestForQuerier(ctx, FirstUser(), "querier-1") + assert.Equal(t, normalRequest, nextRequest) + assert.Equal(t, 2, queue.queues.userQueues["userID"].queue.length()) } type MockRequest struct { diff --git a/pkg/scheduler/queue/user_queues.go b/pkg/scheduler/queue/user_queues.go index 46c0bb598a..816f87aae8 100644 --- a/pkg/scheduler/queue/user_queues.go +++ b/pkg/scheduler/queue/user_queues.go @@ -154,7 +154,7 @@ func (q *queues) getOrAddQueue(userID string, maxQueriers int) userRequestQueue uq.index = len(q.users) q.users = append(q.users, userID) } - } else if (uq.priorityEnabled != priorityEnabled) || (uq.maxOutstanding != maxOutstanding && priorityEnabled) { + } else if (uq.priorityEnabled != priorityEnabled) || (!priorityEnabled && uq.maxOutstanding != maxOutstanding) { tmpQueue := q.createUserRequestQueue(userID) // flush to new queue @@ -175,12 +175,14 @@ func (q *queues) getOrAddQueue(userID string, maxQueriers int) userRequestQueue reservedQueriers := make(map[string]int64) i := 0 - for querierID := range uq.queriers { + for _, querierID := range q.sortedQueriers { if i == len(priorityList) { break } - reservedQueriers[querierID] = priorityList[i] - i++ + if _, ok := uq.queriers[querierID]; ok || uq.queriers == nil { + reservedQueriers[querierID] = priorityList[i] + i++ + } } uq.reservedQueriers = reservedQueriers @@ -388,18 +390,22 @@ func getPriorityList(queryPriority validation.QueryPriority, totalQuerierCount i } } + if len(priorityList) > totalQuerierCount { + return []int64{} + } + return priorityList } // MockLimits implements the Limits interface. Used in tests only. type MockLimits struct { - MaxOutstanding int - maxQueriersPerUser float64 - queryPriority validation.QueryPriority + MaxOutstanding int + MaxQueriersPerUserVal float64 + QueryPriorityVal validation.QueryPriority } func (l MockLimits) MaxQueriersPerUser(_ string) float64 { - return l.maxQueriersPerUser + return l.MaxQueriersPerUserVal } func (l MockLimits) MaxOutstandingPerTenant(_ string) int { @@ -407,5 +413,5 @@ func (l MockLimits) MaxOutstandingPerTenant(_ string) int { } func (l MockLimits) QueryPriority(_ string) validation.QueryPriority { - return l.queryPriority + return l.QueryPriorityVal } diff --git a/pkg/scheduler/queue/user_queues_test.go b/pkg/scheduler/queue/user_queues_test.go index 235f1cb187..3f0495b9ce 100644 --- a/pkg/scheduler/queue/user_queues_test.go +++ b/pkg/scheduler/queue/user_queues_test.go @@ -2,6 +2,7 @@ package queue import ( "fmt" + "github.com/cortexproject/cortex/pkg/util/validation" "math" "math/rand" "sort" @@ -351,6 +352,69 @@ func TestQueues_ForgetDelay_ShouldCorrectlyHandleQuerierReconnectingBeforeForget } } +func TestGetQueue(t *testing.T) { + limits := MockLimits{ + MaxOutstanding: 3, + } + q := newUserQueues(0, 0, limits) + q.addQuerierConnection("q-1") + q.addQuerierConnection("q-2") + q.addQuerierConnection("q-3") + q.addQuerierConnection("q-4") + q.addQuerierConnection("q-5") + + queue := q.getOrAddQueue("userID", 2) + queue.enqueueRequest(MockRequest{}) + + assert.NotNil(t, q.userQueues["userID"]) + assert.Equal(t, 3, q.userQueues["userID"].maxOutstanding) + assert.Equal(t, 2, q.userQueues["userID"].maxQueriers) + assert.Equal(t, 5, len(q.queriers)) + assert.Equal(t, 2, len(q.userQueues["userID"].queriers)) + assert.Subset(t, getKeys(q.queriers), getKeys(q.userQueues["userID"].queriers)) + assert.IsType(t, &FIFORequestQueue{}, queue) + assert.Equal(t, 1, queue.length()) + + limits.MaxOutstanding = 10 + q.limits = limits + queue = q.getOrAddQueue("userID", 0) + + assert.Equal(t, 10, q.userQueues["userID"].maxOutstanding) + assert.Equal(t, 0, q.userQueues["userID"].maxQueriers) + assert.Nil(t, q.userQueues["userID"].queriers) + assert.IsType(t, &FIFORequestQueue{}, queue) + assert.Equal(t, 1, queue.length()) + + limits.QueryPriorityVal = validation.QueryPriority{Enabled: true, Priorities: []validation.PriorityDef{ + { + Priority: 1, + ReservedQueriers: 2, + }, + }} + q.limits = limits + queue = q.getOrAddQueue("userID", 3) + + assert.Equal(t, 10, q.userQueues["userID"].maxOutstanding) + assert.Equal(t, 3, q.userQueues["userID"].maxQueriers) + assert.Equal(t, 5, len(q.queriers)) + assert.Equal(t, 3, len(q.userQueues["userID"].queriers)) + assert.IsType(t, &PriorityRequestQueue{}, queue) + assert.Equal(t, 1, queue.length()) + assert.ElementsMatch(t, []int64{1, 1}, q.userQueues["userID"].priorityList) + assert.True(t, q.userQueues["userID"].priorityEnabled) + assert.Subset(t, getKeys(q.queriers), getKeys(q.userQueues["userID"].queriers)) + assert.Subset(t, getKeys(q.userQueues["userID"].queriers), getKeys(q.userQueues["userID"].reservedQueriers)) + + // check the queriers and reservedQueriers map are consistent + for i := 0; i < 10; i++ { + queriers := q.userQueues["userID"].queriers + reservedQueriers := q.userQueues["userID"].reservedQueriers + queue = q.getOrAddQueue("userID", 3) + assert.Equal(t, queriers, q.userQueues["userID"].queriers) + assert.Equal(t, reservedQueriers, q.userQueues["userID"].reservedQueriers) + } +} + func generateTenant(r *rand.Rand) string { return fmt.Sprint("tenant-", r.Int()%5) } @@ -437,6 +501,27 @@ func getUsersByQuerier(queues *queues, querierID string) []string { return userIDs } +func getKeys(x interface{}) []string { + var keys []string + + switch i := x.(type) { + case map[string]struct{}: + for k := range i { + keys = append(keys, k) + } + case map[string]*querier: + for k := range i { + keys = append(keys, k) + } + case map[string]int64: + for k := range i { + keys = append(keys, k) + } + } + + return keys +} + func TestShuffleQueriers(t *testing.T) { allQueriers := []string{"a", "b", "c", "d", "e"} @@ -552,3 +637,29 @@ func TestShuffleQueriers_WithReservedQueriers_Correctness(t *testing.T) { // prevReservedQueriers = reservedQueriers //} } + +func TestGetPriorityList(t *testing.T) { + queryPriority := validation.QueryPriority{ + Enabled: true, + Priorities: []validation.PriorityDef{ + { + Priority: 1, + ReservedQueriers: 2, + }, + { + Priority: 2, + ReservedQueriers: 3, + }, + }, + } + + assert.EqualValues(t, []int64{1, 1, 2, 2, 2}, getPriorityList(queryPriority, 10)) + assert.EqualValues(t, []int64{}, getPriorityList(queryPriority, 1)) + + queryPriority.Priorities[0].ReservedQueriers = 0.4 + queryPriority.Priorities[1].ReservedQueriers = 0.6 + assert.EqualValues(t, []int64{1, 1, 1, 1, 2, 2, 2, 2, 2, 2}, getPriorityList(queryPriority, 10)) + + queryPriority.Enabled = false + assert.Nil(t, getPriorityList(queryPriority, 10)) +} diff --git a/pkg/scheduler/queue/user_request_queue_test.go b/pkg/scheduler/queue/user_request_queue_test.go index 6001f8966c..aeaa88440d 100644 --- a/pkg/scheduler/queue/user_request_queue_test.go +++ b/pkg/scheduler/queue/user_request_queue_test.go @@ -52,6 +52,7 @@ func TestPriorityRequestQueue(t *testing.T) { assert.Equal(t, 2, queue.length()) assert.Equal(t, request2, queue.dequeueRequest(2, true)) assert.Equal(t, 1, queue.length()) + assert.Nil(t, queue.dequeueRequest(2, true)) assert.Equal(t, request1, queue.dequeueRequest(2, false)) assert.Equal(t, 0, queue.length()) } diff --git a/pkg/scheduler/scheduler.go b/pkg/scheduler/scheduler.go index b96b659e4b..65235540a3 100644 --- a/pkg/scheduler/scheduler.go +++ b/pkg/scheduler/scheduler.go @@ -105,7 +105,7 @@ func NewScheduler(cfg Config, limits Limits, log log.Logger, registerer promethe s.queueLength = promauto.With(registerer).NewGaugeVec(prometheus.GaugeOpts{ Name: "cortex_query_scheduler_queue_length", Help: "Number of queries in the queue.", - }, []string{"user", "priority"}) + }, []string{"user", "priority", "type"}) s.discardedRequests = promauto.With(registerer).NewCounterVec(prometheus.CounterOpts{ Name: "cortex_query_scheduler_discarded_requests_total", diff --git a/pkg/util/priority_queue.go b/pkg/util/priority_queue.go index 543192e126..9937c231c9 100644 --- a/pkg/util/priority_queue.go +++ b/pkg/util/priority_queue.go @@ -86,7 +86,7 @@ func (pq *PriorityQueue) Enqueue(op PriorityOp) { pq.lock.Lock() defer pq.lock.Unlock() - if pq.closed { + if pq.closing || pq.closed { panic("enqueue on closed queue") } diff --git a/pkg/util/priority_queue_test.go b/pkg/util/priority_queue_test.go index d7ee8bc202..168ed96137 100644 --- a/pkg/util/priority_queue_test.go +++ b/pkg/util/priority_queue_test.go @@ -33,9 +33,11 @@ func TestPriorityQueuePriorities(t *testing.T) { queue := NewPriorityQueue(nil) queue.Enqueue(simpleItem(1)) queue.Enqueue(simpleItem(2)) + queue.Enqueue(simpleItem(2)) assert.Equal(t, simpleItem(2), queue.Peek().(simpleItem), "Expected to peek simpleItem(2)") assert.Equal(t, simpleItem(2), queue.Dequeue().(simpleItem), "Expected to dequeue simpleItem(2)") + assert.Equal(t, simpleItem(2), queue.Dequeue().(simpleItem), "Expected to dequeue simpleItem(2)") assert.Equal(t, simpleItem(1), queue.Dequeue().(simpleItem), "Expected to dequeue simpleItem(1)") queue.Close() @@ -46,9 +48,11 @@ func TestPriorityQueuePriorities2(t *testing.T) { queue := NewPriorityQueue(nil) queue.Enqueue(simpleItem(2)) queue.Enqueue(simpleItem(1)) + queue.Enqueue(simpleItem(2)) assert.Equal(t, simpleItem(2), queue.Peek().(simpleItem), "Expected to peek simpleItem(2)") assert.Equal(t, simpleItem(2), queue.Dequeue().(simpleItem), "Expected to dequeue simpleItem(2)") + assert.Equal(t, simpleItem(2), queue.Dequeue().(simpleItem), "Expected to dequeue simpleItem(2)") assert.Equal(t, simpleItem(1), queue.Dequeue().(simpleItem), "Expected to dequeue simpleItem(1)") queue.Close() @@ -72,3 +76,17 @@ func TestPriorityQueueWait(t *testing.T) { t.Fatal("Close didn't unblock Dequeue.") } } + +func TestPriorityQueueClose(t *testing.T) { + queue := NewPriorityQueue(nil) + queue.Enqueue(simpleItem(1)) + queue.Close() + assert.Panics(t, func() { queue.Enqueue(simpleItem(2)) }) + assert.Equal(t, simpleItem(1), queue.Dequeue().(simpleItem)) + + queue = NewPriorityQueue(nil) + queue.Enqueue(simpleItem(1)) + queue.DiscardAndClose() + assert.Panics(t, func() { queue.Enqueue(simpleItem(2)) }) + assert.Nil(t, queue.Dequeue()) +} diff --git a/pkg/util/query/priority_test.go b/pkg/util/query/priority_test.go index 0d1193a2f3..f1c9f4d185 100644 --- a/pkg/util/query/priority_test.go +++ b/pkg/util/query/priority_test.go @@ -116,6 +116,42 @@ func Test_GetPriorityShouldConsiderRegex(t *testing.T) { }, now, &queryPriority, false)) } +func Test_GetPriorityShouldNotRecompileRegexIfQueryPriorityChangedIsTrue(t *testing.T) { + now := time.Now() + priorities := []validation.PriorityDef{ + { + Priority: 1, + QueryAttributes: []validation.QueryAttribute{ + { + Regex: "sum", + StartTime: 2 * time.Hour, + EndTime: 0 * time.Hour, + }, + }, + }, + } + queryPriority := validation.QueryPriority{ + Enabled: true, + Priorities: priorities, + } + + assert.Equal(t, int64(1), GetPriority(url.Values{ + "query": []string{"sum(up)"}, + "time": []string{strconv.FormatInt(now.Unix(), 10)}, + }, now, &queryPriority, true)) + + queryPriority.Priorities[0].QueryAttributes[0].Regex = "count" + + assert.Equal(t, int64(0), GetPriority(url.Values{ + "query": []string{"count(up)"}, + "time": []string{strconv.FormatInt(now.Unix(), 10)}, + }, now, &queryPriority, false)) + assert.Equal(t, int64(1), GetPriority(url.Values{ + "query": []string{"count(up)"}, + "time": []string{strconv.FormatInt(now.Unix(), 10)}, + }, now, &queryPriority, true)) +} + func Test_GetPriorityShouldConsiderStartAndEndTime(t *testing.T) { now := time.Now() priorities := []validation.PriorityDef{ From f3a977292d128e4937b41a957e4f75cb1bb2664b Mon Sep 17 00:00:00 2001 From: Justin Jung Date: Fri, 10 Nov 2023 11:01:28 -0800 Subject: [PATCH 32/49] Lint Signed-off-by: Justin Jung --- pkg/frontend/v1/frontend_test.go | 2 +- pkg/scheduler/queue/user_queues_test.go | 12 ++++++++---- pkg/scheduler/scheduler_test.go | 6 +++--- 3 files changed, 12 insertions(+), 8 deletions(-) diff --git a/pkg/frontend/v1/frontend_test.go b/pkg/frontend/v1/frontend_test.go index 316c8c25fe..1206969a93 100644 --- a/pkg/frontend/v1/frontend_test.go +++ b/pkg/frontend/v1/frontend_test.go @@ -211,7 +211,7 @@ func TestFrontendMetricsCleanup(t *testing.T) { require.NoError(t, testutil.GatherAndCompare(reg, strings.NewReader(` # HELP cortex_query_frontend_queue_length Number of queries in the queue. # TYPE cortex_query_frontend_queue_length gauge - cortex_query_frontend_queue_length{priority="0",user="1"} 0 + cortex_query_frontend_queue_length{priority="0",type="fifo",user="1"} 0 `), "cortex_query_frontend_queue_length")) fr.cleanupInactiveUserMetrics("1") diff --git a/pkg/scheduler/queue/user_queues_test.go b/pkg/scheduler/queue/user_queues_test.go index 3f0495b9ce..df4fce2b86 100644 --- a/pkg/scheduler/queue/user_queues_test.go +++ b/pkg/scheduler/queue/user_queues_test.go @@ -2,7 +2,6 @@ package queue import ( "fmt" - "github.com/cortexproject/cortex/pkg/util/validation" "math" "math/rand" "sort" @@ -11,6 +10,8 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + + "github.com/cortexproject/cortex/pkg/util/validation" ) func TestQueues(t *testing.T) { @@ -352,7 +353,7 @@ func TestQueues_ForgetDelay_ShouldCorrectlyHandleQuerierReconnectingBeforeForget } } -func TestGetQueue(t *testing.T) { +func TestGetOrAddQueueShouldUpdateProperties(t *testing.T) { limits := MockLimits{ MaxOutstanding: 3, } @@ -406,10 +407,13 @@ func TestGetQueue(t *testing.T) { assert.Subset(t, getKeys(q.userQueues["userID"].queriers), getKeys(q.userQueues["userID"].reservedQueriers)) // check the queriers and reservedQueriers map are consistent - for i := 0; i < 10; i++ { + for i := 0; i < 100; i++ { queriers := q.userQueues["userID"].queriers reservedQueriers := q.userQueues["userID"].reservedQueriers - queue = q.getOrAddQueue("userID", 3) + q.userQueues["userID"].maxQueriers = 0 // reset to trigger querier assignment + q.userQueues["userID"].priorityList = []int64{} // reset to trigger reserved querier assignment + _ = q.getOrAddQueue("userID", 3) + assert.Equal(t, queriers, q.userQueues["userID"].queriers) assert.Equal(t, reservedQueriers, q.userQueues["userID"].reservedQueriers) } diff --git a/pkg/scheduler/scheduler_test.go b/pkg/scheduler/scheduler_test.go index b055754133..798843284e 100644 --- a/pkg/scheduler/scheduler_test.go +++ b/pkg/scheduler/scheduler_test.go @@ -430,8 +430,8 @@ func TestSchedulerMetrics(t *testing.T) { require.NoError(t, promtest.GatherAndCompare(reg, strings.NewReader(` # HELP cortex_query_scheduler_queue_length Number of queries in the queue. # TYPE cortex_query_scheduler_queue_length gauge - cortex_query_scheduler_queue_length{priority="0",user="another"} 1 - cortex_query_scheduler_queue_length{priority="0",user="test"} 1 + cortex_query_scheduler_queue_length{priority="0",type="fifo",user="another"} 1 + cortex_query_scheduler_queue_length{priority="0",type="fifo",user="test"} 1 `), "cortex_query_scheduler_queue_length")) scheduler.cleanupMetricsForInactiveUser("test") @@ -439,7 +439,7 @@ func TestSchedulerMetrics(t *testing.T) { require.NoError(t, promtest.GatherAndCompare(reg, strings.NewReader(` # HELP cortex_query_scheduler_queue_length Number of queries in the queue. # TYPE cortex_query_scheduler_queue_length gauge - cortex_query_scheduler_queue_length{priority="0",user="another"} 1 + cortex_query_scheduler_queue_length{priority="0",type="fifo",user="another"} 1 `), "cortex_query_scheduler_queue_length")) } From 87c9a0ed6608fcc4da589dd9b660fc3d0d4bb6f9 Mon Sep 17 00:00:00 2001 From: Justin Jung Date: Fri, 10 Nov 2023 12:15:52 -0800 Subject: [PATCH 33/49] Update doc Signed-off-by: Justin Jung --- docs/configuration/config-file-reference.md | 9 +++------ pkg/util/validation/limits.go | 6 +++--- 2 files changed, 6 insertions(+), 9 deletions(-) diff --git a/docs/configuration/config-file-reference.md b/docs/configuration/config-file-reference.md index 05b2b67103..c4765c2948 100644 --- a/docs/configuration/config-file-reference.md +++ b/docs/configuration/config-file-reference.md @@ -5061,16 +5061,13 @@ otel: ### `QueryAttribute` ```yaml -# Query string regex. If evaluated true (on top of meeting all other criteria), -# query is treated as a high priority. +# Query string regex. [regex: | default = ".*"] -# If query range falls between the start_time and end_time (on top of meeting -# all other criteria), query is treated as a high priority. +# Query start time. [start_time: | default = 0s] -# If query range falls between the start_time and end_time (on top of meeting -# all other criteria), query is treated as a high priority. +# Query end time. [end_time: | default = 0s] ``` diff --git a/pkg/util/validation/limits.go b/pkg/util/validation/limits.go index 0403d0c495..a4561823d2 100644 --- a/pkg/util/validation/limits.go +++ b/pkg/util/validation/limits.go @@ -61,10 +61,10 @@ type PriorityDef struct { } type QueryAttribute struct { - Regex string `yaml:"regex" doc:"nocli|description=Query string regex. If evaluated true (on top of meeting all other criteria), query is treated as a high priority.|default=.*"` + Regex string `yaml:"regex" doc:"nocli|description=Query string regex.|default=.*"` CompiledRegex *regexp.Regexp `yaml:"-" doc:"nocli"` - StartTime time.Duration `yaml:"start_time" doc:"nocli|description=If query range falls between the start_time and end_time (on top of meeting all other criteria), query is treated as a high priority.|default=0s"` - EndTime time.Duration `yaml:"end_time" doc:"nocli|description=If query range falls between the start_time and end_time (on top of meeting all other criteria), query is treated as a high priority.|default=0s"` + StartTime time.Duration `yaml:"start_time" doc:"nocli|description=Query start time.|default=0s"` + EndTime time.Duration `yaml:"end_time" doc:"nocli|description=Query end time.|default=0s"` } // Limits describe all the limits for users; can be used to describe global default From 506d5c29a3d07beeaefd5210e89311e5a35b64b2 Mon Sep 17 00:00:00 2001 From: Justin Jung Date: Wed, 15 Nov 2023 04:39:52 -0800 Subject: [PATCH 34/49] Improve time comparison when assigning priority Signed-off-by: Justin Jung --- pkg/util/query/priority.go | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/pkg/util/query/priority.go b/pkg/util/query/priority.go index fa63cd3c30..2b5353266e 100644 --- a/pkg/util/query/priority.go +++ b/pkg/util/query/priority.go @@ -39,7 +39,15 @@ func GetPriority(requestParams url.Values, now time.Time, queryPriority *validat } startTimeThreshold := now.Add(-1 * attribute.StartTime.Abs()).Truncate(time.Second).UTC() - endTimeThreshold := now.Add(-1 * attribute.EndTime.Abs()).Round(time.Second).UTC() + endTimeThreshold := now.Add(-1 * attribute.EndTime.Abs()).Add(1 * time.Second).Truncate(time.Second).UTC() + + if startTime, err := parseTime(startParam); err == nil { + if endTime, err := parseTime(endParam); err == nil { + if isBetweenThresholds(startTime, endTime, startTimeThreshold, endTimeThreshold) { + return priority.Priority + } + } + } if instantTime, err := parseTime(timeParam); err == nil { if isBetweenThresholds(instantTime, instantTime, startTimeThreshold, endTimeThreshold) { @@ -47,11 +55,9 @@ func GetPriority(requestParams url.Values, now time.Time, queryPriority *validat } } - if startTime, err := parseTime(startParam); err == nil { - if endTime, err := parseTime(endParam); err == nil { - if isBetweenThresholds(startTime, endTime, startTimeThreshold, endTimeThreshold) { - return priority.Priority - } + if timeParam == "" { + if isBetweenThresholds(now, now, startTimeThreshold, endTimeThreshold) { + return priority.Priority } } } From c0f268d623e68a71677d4ff4da1003674f9bbc5e Mon Sep 17 00:00:00 2001 From: Justin Jung Date: Wed, 15 Nov 2023 04:41:57 -0800 Subject: [PATCH 35/49] Make reserved querier to match exact priority + change query length gauge vector in user request queue Signed-off-by: Justin Jung --- pkg/scheduler/queue/queue.go | 32 ++------ pkg/scheduler/queue/queue_test.go | 16 +++- pkg/scheduler/queue/user_queues.go | 12 ++- pkg/scheduler/queue/user_queues_test.go | 12 +-- pkg/scheduler/queue/user_request_queue.go | 64 ++++++++++++--- .../queue/user_request_queue_test.go | 79 ++++++++++++++++++- 6 files changed, 160 insertions(+), 55 deletions(-) diff --git a/pkg/scheduler/queue/queue.go b/pkg/scheduler/queue/queue.go index aa29e061f8..6580068900 100644 --- a/pkg/scheduler/queue/queue.go +++ b/pkg/scheduler/queue/queue.go @@ -62,16 +62,14 @@ type RequestQueue struct { queues *queues stopped bool - queueLength *prometheus.GaugeVec // Per user, priority and type of the queue (fifo or priority). totalRequests *prometheus.CounterVec // Per user and priority. discardedRequests *prometheus.CounterVec // Per user and priority. } func NewRequestQueue(maxOutstandingPerTenant int, forgetDelay time.Duration, queueLength *prometheus.GaugeVec, discardedRequests *prometheus.CounterVec, limits Limits, registerer prometheus.Registerer) *RequestQueue { q := &RequestQueue{ - queues: newUserQueues(maxOutstandingPerTenant, forgetDelay, limits), + queues: newUserQueues(maxOutstandingPerTenant, forgetDelay, limits, queueLength), connectedQuerierWorkers: atomic.NewInt32(0), - queueLength: queueLength, totalRequests: promauto.With(registerer).NewCounterVec(prometheus.CounterOpts{ Name: "cortex_request_queue_requests_total", Help: "Total number of query requests going to the request queue.", @@ -108,10 +106,6 @@ func (q *RequestQueue) EnqueueRequest(userID string, req Request, maxQueriers fl return errors.New("no queue found") } - queueType := "fifo" - if q.queues.limits.QueryPriority(userID).Enabled { - queueType = "priority" - } metricLabels := prometheus.Labels{ "user": userID, "priority": priority, @@ -124,11 +118,6 @@ func (q *RequestQueue) EnqueueRequest(userID string, req Request, maxQueriers fl } queue.enqueueRequest(req) - q.queueLength.With(prometheus.Labels{ - "user": userID, - "priority": priority, - "type": queueType, - }).Inc() q.cond.Broadcast() // Call this function while holding a lock. This guarantees that no querier can fetch the request before function returns. if successFn != nil { @@ -170,10 +159,10 @@ FindQueue: // Pick next request from the queue. for { - minPriority, checkMinPriority := q.getMinPriorityForQuerier(userID, querierID) - request := queue.dequeueRequest(minPriority, checkMinPriority) + priority, matchPriority := q.getPriorityForQuerier(userID, querierID) + request := queue.dequeueRequest(priority, matchPriority) if request == nil { - // the queue does not contain request with the min priority, break to wait for more requests + // the queue does not contain request with the min priority, wait for more requests querierWait = true goto FindQueue } @@ -182,17 +171,6 @@ FindQueue: q.queues.deleteQueue(userID) } - queueType := "fifo" - if q.queues.limits.QueryPriority(userID).Enabled { - queueType = "priority" - } - metricLabels := prometheus.Labels{ - "user": userID, - "priority": strconv.FormatInt(request.Priority(), 10), - "type": queueType, - } - q.queueLength.With(metricLabels).Dec() - // Tell close() we've processed a request. q.cond.Broadcast() @@ -206,7 +184,7 @@ FindQueue: goto FindQueue } -func (q *RequestQueue) getMinPriorityForQuerier(userID string, querierID string) (int64, bool) { +func (q *RequestQueue) getPriorityForQuerier(userID string, querierID string) (int64, bool) { if priority, ok := q.queues.userQueues[userID].reservedQueriers[querierID]; ok { return priority, true } diff --git a/pkg/scheduler/queue/queue_test.go b/pkg/scheduler/queue/queue_test.go index 02317338d0..8c7c1f0794 100644 --- a/pkg/scheduler/queue/queue_test.go +++ b/pkg/scheduler/queue/queue_test.go @@ -226,10 +226,10 @@ func TestReservedQueriersShouldOnlyGetHighPriorityQueries(t *testing.T) { assert.NoError(t, queue.EnqueueRequest("userID", normalRequest, 1, func() {})) assert.NoError(t, queue.EnqueueRequest("userID", priority1Request, 1, func() {})) - assert.NoError(t, queue.EnqueueRequest("userID", priority2Request, 1, func() {})) + assert.NoError(t, queue.EnqueueRequest("userID", priority1Request, 1, func() {})) nextRequest, _, _ := queue.GetNextRequestForQuerier(ctx, FirstUser(), "querier-1") - assert.Equal(t, priority2Request, nextRequest) + assert.Equal(t, priority1Request, nextRequest) nextRequest, _, _ = queue.GetNextRequestForQuerier(ctx, FirstUser(), "querier-1") assert.Equal(t, priority1Request, nextRequest) @@ -243,6 +243,18 @@ func TestReservedQueriersShouldOnlyGetHighPriorityQueries(t *testing.T) { nextRequest, _, _ = queue.GetNextRequestForQuerier(ctxTimeout, FirstUser(), "querier-1") assert.Nil(t, nextRequest) assert.Equal(t, 1, queue.queues.userQueues["userID"].queue.length()) + + assert.NoError(t, queue.EnqueueRequest("userID", priority2Request, 1, func() {})) + + ctxTimeout, cancel = context.WithTimeout(ctx, 1*time.Second) + defer cancel() + + time.AfterFunc(2*time.Second, func() { + queue.cond.Broadcast() + }) + nextRequest, _, _ = queue.GetNextRequestForQuerier(ctxTimeout, FirstUser(), "querier-1") + assert.Nil(t, nextRequest) + assert.Equal(t, 2, queue.queues.userQueues["userID"].queue.length()) } func TestExitingRequestsShouldPersistEvenIfTheConfigHasChanged(t *testing.T) { diff --git a/pkg/scheduler/queue/user_queues.go b/pkg/scheduler/queue/user_queues.go index 816f87aae8..a42b3f80c6 100644 --- a/pkg/scheduler/queue/user_queues.go +++ b/pkg/scheduler/queue/user_queues.go @@ -6,9 +6,10 @@ import ( "sort" "time" - "github.com/cortexproject/cortex/pkg/util/validation" + "github.com/prometheus/client_golang/prometheus" "github.com/cortexproject/cortex/pkg/util" + "github.com/cortexproject/cortex/pkg/util/validation" ) // Limits needed for the Query Scheduler - interface used for decoupling. @@ -57,6 +58,8 @@ type queues struct { sortedQueriers []string limits Limits + + queueLength *prometheus.GaugeVec // Per user, type and priority. } type userQueue struct { @@ -79,7 +82,7 @@ type userQueue struct { index int } -func newUserQueues(maxUserQueueSize int, forgetDelay time.Duration, limits Limits) *queues { +func newUserQueues(maxUserQueueSize int, forgetDelay time.Duration, limits Limits, queueLength *prometheus.GaugeVec) *queues { return &queues{ userQueues: map[string]*userQueue{}, users: nil, @@ -88,6 +91,7 @@ func newUserQueues(maxUserQueueSize int, forgetDelay time.Duration, limits Limit queriers: map[string]*querier{}, sortedQueriers: nil, limits: limits, + queueLength: queueLength, } } @@ -195,7 +199,7 @@ func (q *queues) getOrAddQueue(userID string, maxQueriers int) userRequestQueue func (q *queues) createUserRequestQueue(userID string) userRequestQueue { if q.limits.QueryPriority(userID).Enabled { - return NewPriorityRequestQueue(util.NewPriorityQueue(nil)) + return NewPriorityRequestQueue(util.NewPriorityQueue(nil), userID, q.queueLength) } queueSize := q.limits.MaxOutstandingPerTenant(userID) @@ -206,7 +210,7 @@ func (q *queues) createUserRequestQueue(userID string) userRequestQueue { queueSize = q.maxUserQueueSize } - return NewFIFORequestQueue(make(chan Request, queueSize)) + return NewFIFORequestQueue(make(chan Request, queueSize), userID, q.queueLength) } // Finds next queue for the querier. To support fair scheduling between users, client is expected diff --git a/pkg/scheduler/queue/user_queues_test.go b/pkg/scheduler/queue/user_queues_test.go index df4fce2b86..75c876c516 100644 --- a/pkg/scheduler/queue/user_queues_test.go +++ b/pkg/scheduler/queue/user_queues_test.go @@ -15,7 +15,7 @@ import ( ) func TestQueues(t *testing.T) { - uq := newUserQueues(0, 0, MockLimits{}) + uq := newUserQueues(0, 0, MockLimits{}, nil) assert.NotNil(t, uq) assert.NoError(t, isConsistent(uq)) @@ -70,7 +70,7 @@ func TestQueues(t *testing.T) { } func TestQueuesWithQueriers(t *testing.T) { - uq := newUserQueues(0, 0, MockLimits{}) + uq := newUserQueues(0, 0, MockLimits{}, nil) assert.NotNil(t, uq) assert.NoError(t, isConsistent(uq)) @@ -147,7 +147,7 @@ func TestQueuesConsistency(t *testing.T) { for testName, testData := range tests { t.Run(testName, func(t *testing.T) { - uq := newUserQueues(0, testData.forgetDelay, MockLimits{}) + uq := newUserQueues(0, testData.forgetDelay, MockLimits{}, nil) assert.NotNil(t, uq) assert.NoError(t, isConsistent(uq)) @@ -196,7 +196,7 @@ func TestQueues_ForgetDelay(t *testing.T) { ) now := time.Now() - uq := newUserQueues(0, forgetDelay, MockLimits{}) + uq := newUserQueues(0, forgetDelay, MockLimits{}, nil) assert.NotNil(t, uq) assert.NoError(t, isConsistent(uq)) @@ -288,7 +288,7 @@ func TestQueues_ForgetDelay_ShouldCorrectlyHandleQuerierReconnectingBeforeForget ) now := time.Now() - uq := newUserQueues(0, forgetDelay, MockLimits{}) + uq := newUserQueues(0, forgetDelay, MockLimits{}, nil) assert.NotNil(t, uq) assert.NoError(t, isConsistent(uq)) @@ -357,7 +357,7 @@ func TestGetOrAddQueueShouldUpdateProperties(t *testing.T) { limits := MockLimits{ MaxOutstanding: 3, } - q := newUserQueues(0, 0, limits) + q := newUserQueues(0, 0, limits, nil) q.addQuerierConnection("q-1") q.addQuerierConnection("q-2") q.addQuerierConnection("q-3") diff --git a/pkg/scheduler/queue/user_request_queue.go b/pkg/scheduler/queue/user_request_queue.go index 9f95c9e8c5..5325e94d9b 100644 --- a/pkg/scheduler/queue/user_request_queue.go +++ b/pkg/scheduler/queue/user_request_queue.go @@ -1,27 +1,50 @@ package queue -import "github.com/cortexproject/cortex/pkg/util" +import ( + "strconv" + + "github.com/prometheus/client_golang/prometheus" + + "github.com/cortexproject/cortex/pkg/util" +) type userRequestQueue interface { enqueueRequest(Request) - dequeueRequest(minPriority int64, checkMinPriority bool) Request + dequeueRequest(int64, bool) Request length() int } type FIFORequestQueue struct { - queue chan Request + queue chan Request + userID string + queueLength *prometheus.GaugeVec } -func NewFIFORequestQueue(queue chan Request) *FIFORequestQueue { - return &FIFORequestQueue{queue: queue} +func NewFIFORequestQueue(queue chan Request, userID string, queueLength *prometheus.GaugeVec) *FIFORequestQueue { + return &FIFORequestQueue{queue: queue, userID: userID, queueLength: queueLength} } func (f *FIFORequestQueue) enqueueRequest(r Request) { f.queue <- r + if f.queueLength != nil { + f.queueLength.With(prometheus.Labels{ + "user": f.userID, + "priority": strconv.FormatInt(r.Priority(), 10), + "type": "fifo", + }).Inc() + } } func (f *FIFORequestQueue) dequeueRequest(_ int64, _ bool) Request { - return <-f.queue + r := <-f.queue + if f.queueLength != nil { + f.queueLength.With(prometheus.Labels{ + "user": f.userID, + "priority": strconv.FormatInt(r.Priority(), 10), + "type": "fifo", + }).Dec() + } + return r } func (f *FIFORequestQueue) length() int { @@ -29,22 +52,39 @@ func (f *FIFORequestQueue) length() int { } type PriorityRequestQueue struct { - queue *util.PriorityQueue + queue *util.PriorityQueue + userID string + queueLength *prometheus.GaugeVec } -func NewPriorityRequestQueue(queue *util.PriorityQueue) *PriorityRequestQueue { - return &PriorityRequestQueue{queue: queue} +func NewPriorityRequestQueue(queue *util.PriorityQueue, userID string, queueLength *prometheus.GaugeVec) *PriorityRequestQueue { + return &PriorityRequestQueue{queue: queue, userID: userID, queueLength: queueLength} } func (f *PriorityRequestQueue) enqueueRequest(r Request) { f.queue.Enqueue(r) + if f.queueLength != nil { + f.queueLength.With(prometheus.Labels{ + "user": f.userID, + "priority": strconv.FormatInt(r.Priority(), 10), + "type": "priority", + }).Inc() + } } -func (f *PriorityRequestQueue) dequeueRequest(minPriority int64, checkMinPriority bool) Request { - if checkMinPriority && f.queue.Peek().Priority() < minPriority { +func (f *PriorityRequestQueue) dequeueRequest(priority int64, matchPriority bool) Request { + if matchPriority && f.queue.Peek().Priority() != priority { return nil } - return f.queue.Dequeue() + r := f.queue.Dequeue() + if f.queueLength != nil { + f.queueLength.With(prometheus.Labels{ + "user": f.userID, + "priority": strconv.FormatInt(r.Priority(), 10), + "type": "priority", + }).Dec() + } + return r } func (f *PriorityRequestQueue) length() int { diff --git a/pkg/scheduler/queue/user_request_queue_test.go b/pkg/scheduler/queue/user_request_queue_test.go index aeaa88440d..cabe792e06 100644 --- a/pkg/scheduler/queue/user_request_queue_test.go +++ b/pkg/scheduler/queue/user_request_queue_test.go @@ -1,15 +1,25 @@ package queue import ( + "strings" "testing" + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promauto" + promtest "github.com/prometheus/client_golang/prometheus/testutil" "github.com/stretchr/testify/assert" "github.com/cortexproject/cortex/pkg/util" ) func TestFIFORequestQueue(t *testing.T) { - queue := NewFIFORequestQueue(make(chan Request, 2)) + reg := prometheus.NewPedanticRegistry() + queueLength := promauto.With(reg).NewGaugeVec(prometheus.GaugeOpts{ + Name: "cortex_query_scheduler_queue_length", + Help: "Number of queries in the queue.", + }, []string{"user", "priority", "type"}) + + queue := NewFIFORequestQueue(make(chan Request, 2), "userID", queueLength) request1 := MockRequest{ id: "request 1", priority: 1, @@ -21,15 +31,40 @@ func TestFIFORequestQueue(t *testing.T) { queue.enqueueRequest(request1) queue.enqueueRequest(request2) + + assert.NoError(t, promtest.GatherAndCompare(reg, strings.NewReader(` + # HELP cortex_query_scheduler_queue_length Number of queries in the queue. + # TYPE cortex_query_scheduler_queue_length gauge + cortex_query_scheduler_queue_length{priority="1",type="fifo",user="userID"} 1 + cortex_query_scheduler_queue_length{priority="2",type="fifo",user="userID"} 1 + `), "cortex_query_scheduler_queue_length")) assert.Equal(t, 2, queue.length()) assert.Equal(t, request1, queue.dequeueRequest(0, false)) assert.Equal(t, 1, queue.length()) + assert.NoError(t, promtest.GatherAndCompare(reg, strings.NewReader(` + # HELP cortex_query_scheduler_queue_length Number of queries in the queue. + # TYPE cortex_query_scheduler_queue_length gauge + cortex_query_scheduler_queue_length{priority="1",type="fifo",user="userID"} 0 + cortex_query_scheduler_queue_length{priority="2",type="fifo",user="userID"} 1 + `), "cortex_query_scheduler_queue_length")) assert.Equal(t, request2, queue.dequeueRequest(0, false)) assert.Equal(t, 0, queue.length()) + assert.NoError(t, promtest.GatherAndCompare(reg, strings.NewReader(` + # HELP cortex_query_scheduler_queue_length Number of queries in the queue. + # TYPE cortex_query_scheduler_queue_length gauge + cortex_query_scheduler_queue_length{priority="1",type="fifo",user="userID"} 0 + cortex_query_scheduler_queue_length{priority="2",type="fifo",user="userID"} 0 + `), "cortex_query_scheduler_queue_length")) } func TestPriorityRequestQueue(t *testing.T) { - queue := NewPriorityRequestQueue(util.NewPriorityQueue(nil)) + reg := prometheus.NewPedanticRegistry() + queueLength := promauto.With(reg).NewGaugeVec(prometheus.GaugeOpts{ + Name: "cortex_query_scheduler_queue_length", + Help: "Number of queries in the queue.", + }, []string{"user", "priority", "type"}) + + queue := NewPriorityRequestQueue(util.NewPriorityQueue(nil), "userID", queueLength) request1 := MockRequest{ id: "request 1", priority: 1, @@ -41,18 +76,54 @@ func TestPriorityRequestQueue(t *testing.T) { queue.enqueueRequest(request1) queue.enqueueRequest(request2) + assert.NoError(t, promtest.GatherAndCompare(reg, strings.NewReader(` + # HELP cortex_query_scheduler_queue_length Number of queries in the queue. + # TYPE cortex_query_scheduler_queue_length gauge + cortex_query_scheduler_queue_length{priority="1",type="priority",user="userID"} 1 + cortex_query_scheduler_queue_length{priority="2",type="priority",user="userID"} 1 + `), "cortex_query_scheduler_queue_length")) assert.Equal(t, 2, queue.length()) - assert.Equal(t, request2, queue.dequeueRequest(0, true)) + assert.Equal(t, request2, queue.dequeueRequest(0, false)) assert.Equal(t, 1, queue.length()) - assert.Equal(t, request1, queue.dequeueRequest(0, true)) + assert.NoError(t, promtest.GatherAndCompare(reg, strings.NewReader(` + # HELP cortex_query_scheduler_queue_length Number of queries in the queue. + # TYPE cortex_query_scheduler_queue_length gauge + cortex_query_scheduler_queue_length{priority="1",type="priority",user="userID"} 1 + cortex_query_scheduler_queue_length{priority="2",type="priority",user="userID"} 0 + `), "cortex_query_scheduler_queue_length")) + assert.Equal(t, request1, queue.dequeueRequest(0, false)) assert.Equal(t, 0, queue.length()) + assert.NoError(t, promtest.GatherAndCompare(reg, strings.NewReader(` + # HELP cortex_query_scheduler_queue_length Number of queries in the queue. + # TYPE cortex_query_scheduler_queue_length gauge + cortex_query_scheduler_queue_length{priority="1",type="priority",user="userID"} 0 + cortex_query_scheduler_queue_length{priority="2",type="priority",user="userID"} 0 + `), "cortex_query_scheduler_queue_length")) queue.enqueueRequest(request1) queue.enqueueRequest(request2) + assert.NoError(t, promtest.GatherAndCompare(reg, strings.NewReader(` + # HELP cortex_query_scheduler_queue_length Number of queries in the queue. + # TYPE cortex_query_scheduler_queue_length gauge + cortex_query_scheduler_queue_length{priority="1",type="priority",user="userID"} 1 + cortex_query_scheduler_queue_length{priority="2",type="priority",user="userID"} 1 + `), "cortex_query_scheduler_queue_length")) assert.Equal(t, 2, queue.length()) assert.Equal(t, request2, queue.dequeueRequest(2, true)) assert.Equal(t, 1, queue.length()) assert.Nil(t, queue.dequeueRequest(2, true)) + assert.NoError(t, promtest.GatherAndCompare(reg, strings.NewReader(` + # HELP cortex_query_scheduler_queue_length Number of queries in the queue. + # TYPE cortex_query_scheduler_queue_length gauge + cortex_query_scheduler_queue_length{priority="1",type="priority",user="userID"} 1 + cortex_query_scheduler_queue_length{priority="2",type="priority",user="userID"} 0 + `), "cortex_query_scheduler_queue_length")) assert.Equal(t, request1, queue.dequeueRequest(2, false)) assert.Equal(t, 0, queue.length()) + assert.NoError(t, promtest.GatherAndCompare(reg, strings.NewReader(` + # HELP cortex_query_scheduler_queue_length Number of queries in the queue. + # TYPE cortex_query_scheduler_queue_length gauge + cortex_query_scheduler_queue_length{priority="1",type="priority",user="userID"} 0 + cortex_query_scheduler_queue_length{priority="2",type="priority",user="userID"} 0 + `), "cortex_query_scheduler_queue_length")) } From 0b987730e8151b9fd7d7a11d576220315707f695 Mon Sep 17 00:00:00 2001 From: Justin Jung Date: Wed, 15 Nov 2023 06:28:25 -0800 Subject: [PATCH 36/49] Bug fix Signed-off-by: Justin Jung --- pkg/scheduler/queue/user_queues.go | 1 + pkg/scheduler/queue/user_queues_test.go | 5 +++++ 2 files changed, 6 insertions(+) diff --git a/pkg/scheduler/queue/user_queues.go b/pkg/scheduler/queue/user_queues.go index a42b3f80c6..722539ef9f 100644 --- a/pkg/scheduler/queue/user_queues.go +++ b/pkg/scheduler/queue/user_queues.go @@ -168,6 +168,7 @@ func (q *queues) getOrAddQueue(userID string, maxQueriers int) userRequestQueue uq.queue = tmpQueue uq.maxOutstanding = q.limits.MaxOutstandingPerTenant(userID) + uq.priorityEnabled = priorityEnabled } if uq.maxQueriers != maxQueriers { diff --git a/pkg/scheduler/queue/user_queues_test.go b/pkg/scheduler/queue/user_queues_test.go index 75c876c516..ae812aac6a 100644 --- a/pkg/scheduler/queue/user_queues_test.go +++ b/pkg/scheduler/queue/user_queues_test.go @@ -406,6 +406,11 @@ func TestGetOrAddQueueShouldUpdateProperties(t *testing.T) { assert.Subset(t, getKeys(q.queriers), getKeys(q.userQueues["userID"].queriers)) assert.Subset(t, getKeys(q.userQueues["userID"].queriers), getKeys(q.userQueues["userID"].reservedQueriers)) + limits.QueryPriorityVal.Enabled = false + q.limits = limits + queue = q.getOrAddQueue("userID", 3) + assert.IsType(t, &FIFORequestQueue{}, queue) + // check the queriers and reservedQueriers map are consistent for i := 0; i < 100; i++ { queriers := q.userQueues["userID"].queriers From 3fffc3fe85f42d30a0525b8f8fbbce0603ca60cd Mon Sep 17 00:00:00 2001 From: Justin Jung Date: Wed, 15 Nov 2023 15:21:15 -0800 Subject: [PATCH 37/49] Add comments Signed-off-by: Justin Jung --- pkg/frontend/v1/frontend.go | 5 ++++- pkg/frontend/v2/frontend.go | 5 ++++- pkg/scheduler/queue/queue.go | 2 +- pkg/scheduler/queue/user_queues.go | 14 +++++++++----- pkg/util/query/priority.go | 1 + pkg/util/validation/limits.go | 1 + 6 files changed, 20 insertions(+), 8 deletions(-) diff --git a/pkg/frontend/v1/frontend.go b/pkg/frontend/v1/frontend.go index de5d2d0647..54e8ada594 100644 --- a/pkg/frontend/v1/frontend.go +++ b/pkg/frontend/v1/frontend.go @@ -80,7 +80,10 @@ type Frontend struct { requestQueue *queue.RequestQueue activeUsers *util.ActiveUsersCleanupService - queryPriority validation.QueryPriority + // Used to check whether query priority config has changed + queryPriority validation.QueryPriority + + // Populate and reuse compiled regex until query priority config changes compiledQueryPriority validation.QueryPriority // Subservices manager. diff --git a/pkg/frontend/v2/frontend.go b/pkg/frontend/v2/frontend.go index c99980100e..b05af6277f 100644 --- a/pkg/frontend/v2/frontend.go +++ b/pkg/frontend/v2/frontend.go @@ -76,7 +76,10 @@ type Frontend struct { lastQueryID atomic.Uint64 - queryPriority validation.QueryPriority + // Used to check whether query priority config has changed + queryPriority validation.QueryPriority + + // Populate and reuse compiled regex until query priority config changes compiledQueryPriority validation.QueryPriority // frontend workers will read from this channel, and send request to scheduler. diff --git a/pkg/scheduler/queue/queue.go b/pkg/scheduler/queue/queue.go index 6580068900..57f85bfb12 100644 --- a/pkg/scheduler/queue/queue.go +++ b/pkg/scheduler/queue/queue.go @@ -162,7 +162,7 @@ FindQueue: priority, matchPriority := q.getPriorityForQuerier(userID, querierID) request := queue.dequeueRequest(priority, matchPriority) if request == nil { - // the queue does not contain request with the min priority, wait for more requests + // The queue does not contain request with the priority, wait for more requests querierWait = true goto FindQueue } diff --git a/pkg/scheduler/queue/user_queues.go b/pkg/scheduler/queue/user_queues.go index 722539ef9f..b0ae8069ac 100644 --- a/pkg/scheduler/queue/user_queues.go +++ b/pkg/scheduler/queue/user_queues.go @@ -67,12 +67,16 @@ type userQueue struct { // If not nil, only these queriers can handle user requests. If nil, all queriers can. // We set this to nil if number of available queriers <= maxQueriers. - queriers map[string]struct{} - maxQueriers int - maxOutstanding int + queriers map[string]struct{} + + // Contains assigned priority for querier ID reservedQueriers map[string]int64 - priorityList []int64 - priorityEnabled bool + + // Stores last limit config for the user. When changed, re-populate queriers and reservedQueriers + maxQueriers int + maxOutstanding int + priorityList []int64 + priorityEnabled bool // Seed for shuffle sharding of queriers. This seed is based on userID only and is therefore consistent // between different frontends. diff --git a/pkg/util/query/priority.go b/pkg/util/query/priority.go index 2b5353266e..79e08c1ad8 100644 --- a/pkg/util/query/priority.go +++ b/pkg/util/query/priority.go @@ -24,6 +24,7 @@ func GetPriority(requestParams url.Values, now time.Time, queryPriority *validat for i, priority := range queryPriority.Priorities { for j, attribute := range priority.QueryAttributes { + // If query priority config changed, re-populate the compiled regex if queryPriorityChanged { compiledRegex, err := regexp.Compile(attribute.Regex) if err != nil { diff --git a/pkg/util/validation/limits.go b/pkg/util/validation/limits.go index a4561823d2..b1bebf1d42 100644 --- a/pkg/util/validation/limits.go +++ b/pkg/util/validation/limits.go @@ -250,6 +250,7 @@ func (l *Limits) Validate(shardByAllLabels bool) error { return errMaxGlobalSeriesPerUserValidation } + // If query priority is enabled, do not allow duplicate priority values if l.QueryPriority.Enabled { queryPriority := l.QueryPriority prioritySet := map[int64]struct{}{} From 2abeffbec992f38fb1842b4f68d8319a7b5d3f9e Mon Sep 17 00:00:00 2001 From: Justin Jung Date: Fri, 17 Nov 2023 08:58:58 -0800 Subject: [PATCH 38/49] Address comments Signed-off-by: Justin Jung --- pkg/frontend/v1/frontend.go | 35 +++++--- pkg/frontend/v2/frontend.go | 34 +++++--- pkg/frontend/v2/frontend_test.go | 3 +- pkg/scheduler/queue/user_request_queue.go | 24 +---- pkg/util/query/priority.go | 64 ++++++-------- pkg/util/query/priority_test.go | 101 ++++++++-------------- 6 files changed, 115 insertions(+), 146 deletions(-) diff --git a/pkg/frontend/v1/frontend.go b/pkg/frontend/v1/frontend.go index 54e8ada594..8ea66fbd58 100644 --- a/pkg/frontend/v1/frontend.go +++ b/pkg/frontend/v1/frontend.go @@ -7,6 +7,7 @@ import ( "net/http" "net/url" "reflect" + "sync" "time" "github.com/go-kit/log" @@ -81,10 +82,11 @@ type Frontend struct { activeUsers *util.ActiveUsersCleanupService // Used to check whether query priority config has changed - queryPriority validation.QueryPriority + queryPriority map[string]validation.QueryPriority + queryPriorityMtx map[string]*sync.RWMutex // Populate and reuse compiled regex until query priority config changes - compiledQueryPriority validation.QueryPriority + compiledQueryPriority map[string]validation.QueryPriority // Subservices manager. subservices *services.Manager @@ -217,16 +219,29 @@ func (f *Frontend) RoundTripGRPC(ctx context.Context, req *httpgrpc.HTTPRequest, response: make(chan *httpgrpc.HTTPResponse, 1), } - queryPriority := f.limits.QueryPriority(userID) + if reqParams != nil { + queryPriority := f.limits.QueryPriority(userID) - if reqParams != nil && queryPriority.Enabled { - queryPriorityChanged := !reflect.DeepEqual(f.queryPriority, queryPriority) - if queryPriorityChanged { - f.queryPriority = queryPriority - f.compiledQueryPriority = queryPriority - } + if queryPriority.Enabled { + if _, exists := f.queryPriorityMtx[userID]; !exists { + f.queryPriorityMtx[userID] = &sync.RWMutex{} + } + + f.queryPriorityMtx[userID].RLock() + queryPriorityChanged := !reflect.DeepEqual(f.queryPriority[userID], queryPriority) + f.queryPriorityMtx[userID].RUnlock() - request.priority = util_query.GetPriority(reqParams, ts, &f.compiledQueryPriority, queryPriorityChanged) + if queryPriorityChanged { + f.queryPriorityMtx[userID].Lock() + f.queryPriority[userID] = queryPriority + f.compiledQueryPriority[userID] = util_query.GetCompileQueryPriority(queryPriority) + f.queryPriorityMtx[userID].Unlock() + } + + f.queryPriorityMtx[userID].RLock() + request.priority = util_query.GetPriority(reqParams, ts, f.compiledQueryPriority[userID]) + f.queryPriorityMtx[userID].Unlock() + } } if err := f.queueRequest(ctx, &request); err != nil { diff --git a/pkg/frontend/v2/frontend.go b/pkg/frontend/v2/frontend.go index b05af6277f..41dd4e7749 100644 --- a/pkg/frontend/v2/frontend.go +++ b/pkg/frontend/v2/frontend.go @@ -77,10 +77,11 @@ type Frontend struct { lastQueryID atomic.Uint64 // Used to check whether query priority config has changed - queryPriority validation.QueryPriority + queryPriority map[string]validation.QueryPriority + queryPriorityMtx map[string]*sync.RWMutex // Populate and reuse compiled regex until query priority config changes - compiledQueryPriority validation.QueryPriority + compiledQueryPriority map[string]validation.QueryPriority // frontend workers will read from this channel, and send request to scheduler. requestsCh chan *frontendRequest @@ -218,16 +219,29 @@ func (f *Frontend) RoundTripGRPC(ctx context.Context, req *httpgrpc.HTTPRequest, retryOnTooManyOutstandingRequests: f.cfg.RetryOnTooManyOutstandingRequests && f.schedulerWorkers.getWorkersCount() > 1, } - queryPriority := f.limits.QueryPriority(userID) + if reqParams != nil { + queryPriority := f.limits.QueryPriority(userID) - if reqParams != nil && queryPriority.Enabled { - queryPriorityChanged := !reflect.DeepEqual(f.queryPriority, queryPriority) - if queryPriorityChanged { - f.queryPriority = queryPriority - f.compiledQueryPriority = queryPriority - } + if queryPriority.Enabled { + if _, exists := f.queryPriorityMtx[userID]; !exists { + f.queryPriorityMtx[userID] = &sync.RWMutex{} + } + + f.queryPriorityMtx[userID].RLock() + queryPriorityChanged := !reflect.DeepEqual(f.queryPriority[userID], queryPriority) + f.queryPriorityMtx[userID].RUnlock() - freq.priority = util_query.GetPriority(reqParams, ts, &f.compiledQueryPriority, queryPriorityChanged) + if queryPriorityChanged { + f.queryPriorityMtx[userID].Lock() + f.queryPriority[userID] = queryPriority + f.compiledQueryPriority[userID] = util_query.GetCompileQueryPriority(queryPriority) + f.queryPriorityMtx[userID].Unlock() + } + + f.queryPriorityMtx[userID].RLock() + freq.priority = util_query.GetPriority(reqParams, ts, f.compiledQueryPriority[userID]) + f.queryPriorityMtx[userID].Unlock() + } } f.requests.put(freq) diff --git a/pkg/frontend/v2/frontend_test.go b/pkg/frontend/v2/frontend_test.go index a9cd226f59..4c40d88b66 100644 --- a/pkg/frontend/v2/frontend_test.go +++ b/pkg/frontend/v2/frontend_test.go @@ -10,8 +10,6 @@ import ( "testing" "time" - "github.com/cortexproject/cortex/pkg/scheduler/queue" - "github.com/go-kit/log" "github.com/stretchr/testify/require" "github.com/weaveworks/common/httpgrpc" @@ -22,6 +20,7 @@ import ( "github.com/cortexproject/cortex/pkg/frontend/transport" "github.com/cortexproject/cortex/pkg/frontend/v2/frontendv2pb" "github.com/cortexproject/cortex/pkg/querier/stats" + "github.com/cortexproject/cortex/pkg/scheduler/queue" "github.com/cortexproject/cortex/pkg/scheduler/schedulerpb" "github.com/cortexproject/cortex/pkg/util/flagext" "github.com/cortexproject/cortex/pkg/util/services" diff --git a/pkg/scheduler/queue/user_request_queue.go b/pkg/scheduler/queue/user_request_queue.go index 5325e94d9b..828c9246fd 100644 --- a/pkg/scheduler/queue/user_request_queue.go +++ b/pkg/scheduler/queue/user_request_queue.go @@ -27,22 +27,14 @@ func NewFIFORequestQueue(queue chan Request, userID string, queueLength *prometh func (f *FIFORequestQueue) enqueueRequest(r Request) { f.queue <- r if f.queueLength != nil { - f.queueLength.With(prometheus.Labels{ - "user": f.userID, - "priority": strconv.FormatInt(r.Priority(), 10), - "type": "fifo", - }).Inc() + f.queueLength.WithLabelValues(f.userID, strconv.FormatInt(r.Priority(), 10), "fifo").Inc() } } func (f *FIFORequestQueue) dequeueRequest(_ int64, _ bool) Request { r := <-f.queue if f.queueLength != nil { - f.queueLength.With(prometheus.Labels{ - "user": f.userID, - "priority": strconv.FormatInt(r.Priority(), 10), - "type": "fifo", - }).Dec() + f.queueLength.WithLabelValues(f.userID, strconv.FormatInt(r.Priority(), 10), "fifo").Dec() } return r } @@ -64,11 +56,7 @@ func NewPriorityRequestQueue(queue *util.PriorityQueue, userID string, queueLeng func (f *PriorityRequestQueue) enqueueRequest(r Request) { f.queue.Enqueue(r) if f.queueLength != nil { - f.queueLength.With(prometheus.Labels{ - "user": f.userID, - "priority": strconv.FormatInt(r.Priority(), 10), - "type": "priority", - }).Inc() + f.queueLength.WithLabelValues(f.userID, strconv.FormatInt(r.Priority(), 10), "priority").Inc() } } @@ -78,11 +66,7 @@ func (f *PriorityRequestQueue) dequeueRequest(priority int64, matchPriority bool } r := f.queue.Dequeue() if f.queueLength != nil { - f.queueLength.With(prometheus.Labels{ - "user": f.userID, - "priority": strconv.FormatInt(r.Priority(), 10), - "type": "priority", - }).Dec() + f.queueLength.WithLabelValues(f.userID, strconv.FormatInt(r.Priority(), 10), "priority").Dec() } return r } diff --git a/pkg/util/query/priority.go b/pkg/util/query/priority.go index 79e08c1ad8..503736f208 100644 --- a/pkg/util/query/priority.go +++ b/pkg/util/query/priority.go @@ -1,18 +1,32 @@ package query import ( - "math" "net/url" "regexp" - "strconv" "time" - "github.com/pkg/errors" - + "github.com/cortexproject/cortex/pkg/util" "github.com/cortexproject/cortex/pkg/util/validation" ) -func GetPriority(requestParams url.Values, now time.Time, queryPriority *validation.QueryPriority, queryPriorityChanged bool) int64 { +func GetCompileQueryPriority(queryPriority validation.QueryPriority) validation.QueryPriority { + compiledQueryPriority := queryPriority + for i, priority := range compiledQueryPriority.Priorities { + for j, attribute := range priority.QueryAttributes { + compiledRegex, err := regexp.Compile(attribute.Regex) + if err != nil { + continue + } + + attribute.CompiledRegex = compiledRegex + compiledQueryPriority.Priorities[i].QueryAttributes[j] = attribute + } + } + + return compiledQueryPriority +} + +func GetPriority(requestParams url.Values, now time.Time, queryPriority validation.QueryPriority) int64 { queryParam := requestParams.Get("query") timeParam := requestParams.Get("time") startParam := requestParams.Get("start") @@ -22,19 +36,8 @@ func GetPriority(requestParams url.Values, now time.Time, queryPriority *validat return queryPriority.DefaultPriority } - for i, priority := range queryPriority.Priorities { - for j, attribute := range priority.QueryAttributes { - // If query priority config changed, re-populate the compiled regex - if queryPriorityChanged { - compiledRegex, err := regexp.Compile(attribute.Regex) - if err != nil { - continue - } - - attribute.CompiledRegex = compiledRegex - queryPriority.Priorities[i].QueryAttributes[j] = attribute - } - + for _, priority := range queryPriority.Priorities { + for _, attribute := range priority.QueryAttributes { if attribute.CompiledRegex != nil && !attribute.CompiledRegex.MatchString(queryParam) { continue } @@ -42,16 +45,16 @@ func GetPriority(requestParams url.Values, now time.Time, queryPriority *validat startTimeThreshold := now.Add(-1 * attribute.StartTime.Abs()).Truncate(time.Second).UTC() endTimeThreshold := now.Add(-1 * attribute.EndTime.Abs()).Add(1 * time.Second).Truncate(time.Second).UTC() - if startTime, err := parseTime(startParam); err == nil { - if endTime, err := parseTime(endParam); err == nil { - if isBetweenThresholds(startTime, endTime, startTimeThreshold, endTimeThreshold) { + if startTime, err := util.ParseTime(startParam); err == nil { + if endTime, err := util.ParseTime(endParam); err == nil { + if isBetweenThresholds(util.TimeFromMillis(startTime), util.TimeFromMillis(endTime), startTimeThreshold, endTimeThreshold) { return priority.Priority } } } - if instantTime, err := parseTime(timeParam); err == nil { - if isBetweenThresholds(instantTime, instantTime, startTimeThreshold, endTimeThreshold) { + if instantTime, err := util.ParseTime(timeParam); err == nil { + if isBetweenThresholds(util.TimeFromMillis(instantTime), util.TimeFromMillis(instantTime), startTimeThreshold, endTimeThreshold) { return priority.Priority } } @@ -67,21 +70,6 @@ func GetPriority(requestParams url.Values, now time.Time, queryPriority *validat return queryPriority.DefaultPriority } -func parseTime(s string) (time.Time, error) { - if s != "" { - if t, err := strconv.ParseFloat(s, 64); err == nil { - s, ns := math.Modf(t) - ns = math.Round(ns*1000) / 1000 - return time.Unix(int64(s), int64(ns*float64(time.Second))).UTC(), nil - } - if t, err := time.Parse(time.RFC3339Nano, s); err == nil { - return t, nil - } - } - - return time.Time{}, errors.Errorf("cannot parse %q to a valid timestamp", s) -} - func isBetweenThresholds(start, end, startThreshold, endThreshold time.Time) bool { return (start.Equal(startThreshold) || start.After(startThreshold)) && (end.Equal(endThreshold) || end.Before(endThreshold)) } diff --git a/pkg/util/query/priority_test.go b/pkg/util/query/priority_test.go index f1c9f4d185..646aed17c1 100644 --- a/pkg/util/query/priority_test.go +++ b/pkg/util/query/priority_test.go @@ -2,6 +2,7 @@ package query import ( "net/url" + "regexp" "strconv" "testing" "time" @@ -18,9 +19,10 @@ func Test_GetPriorityShouldReturnDefaultPriorityIfNotEnabledOrEmptyQueryString(t Priority: 1, QueryAttributes: []validation.QueryAttribute{ { - Regex: ".*", - StartTime: 2 * time.Hour, - EndTime: 0 * time.Hour, + Regex: ".*", + CompiledRegex: regexp.MustCompile(".*"), + StartTime: 2 * time.Hour, + EndTime: 0 * time.Hour, }, }, }, @@ -32,13 +34,13 @@ func Test_GetPriorityShouldReturnDefaultPriorityIfNotEnabledOrEmptyQueryString(t assert.Equal(t, int64(0), GetPriority(url.Values{ "query": []string{"sum(up)"}, "time": []string{strconv.FormatInt(now.Unix(), 10)}, - }, now, &queryPriority, true)) + }, now, queryPriority)) queryPriority.Enabled = true assert.Equal(t, int64(0), GetPriority(url.Values{ "query": []string{""}, "time": []string{strconv.FormatInt(now.Unix(), 10)}, - }, now, &queryPriority, true)) + }, now, queryPriority)) } func Test_GetPriorityShouldConsiderRegex(t *testing.T) { @@ -48,9 +50,10 @@ func Test_GetPriorityShouldConsiderRegex(t *testing.T) { Priority: 1, QueryAttributes: []validation.QueryAttribute{ { - Regex: "sum", - StartTime: 2 * time.Hour, - EndTime: 0 * time.Hour, + Regex: "sum", + CompiledRegex: regexp.MustCompile("sum"), + StartTime: 2 * time.Hour, + EndTime: 0 * time.Hour, }, }, }, @@ -60,96 +63,62 @@ func Test_GetPriorityShouldConsiderRegex(t *testing.T) { Priorities: priorities, } - assert.Nil(t, queryPriority.Priorities[0].QueryAttributes[0].CompiledRegex) assert.Equal(t, int64(1), GetPriority(url.Values{ "query": []string{"sum(up)"}, "time": []string{strconv.FormatInt(now.Unix(), 10)}, - }, now, &queryPriority, true)) - assert.NotNil(t, queryPriority.Priorities[0].QueryAttributes[0].CompiledRegex) + }, now, queryPriority)) assert.Equal(t, int64(0), GetPriority(url.Values{ "query": []string{"count(up)"}, "time": []string{strconv.FormatInt(now.Unix(), 10)}, - }, now, &queryPriority, false)) + }, now, queryPriority)) queryPriority.Priorities[0].QueryAttributes[0].Regex = "(^sum$|c(.+)t)" + queryPriority.Priorities[0].QueryAttributes[0].CompiledRegex = regexp.MustCompile("(^sum$|c(.+)t)") assert.Equal(t, int64(0), GetPriority(url.Values{ "query": []string{"sum(up)"}, "time": []string{strconv.FormatInt(now.Unix(), 10)}, - }, now, &queryPriority, true)) + }, now, queryPriority)) assert.Equal(t, int64(1), GetPriority(url.Values{ "query": []string{"count(up)"}, "time": []string{strconv.FormatInt(now.Unix(), 10)}, - }, now, &queryPriority, false)) + }, now, queryPriority)) queryPriority.Priorities[0].QueryAttributes[0].Regex = ".*" + queryPriority.Priorities[0].QueryAttributes[0].CompiledRegex = regexp.MustCompile(".*") assert.Equal(t, int64(1), GetPriority(url.Values{ "query": []string{"sum(up)"}, "time": []string{strconv.FormatInt(now.Unix(), 10)}, - }, now, &queryPriority, true)) + }, now, queryPriority)) assert.Equal(t, int64(1), GetPriority(url.Values{ "query": []string{"count(up)"}, "time": []string{strconv.FormatInt(now.Unix(), 10)}, - }, now, &queryPriority, false)) + }, now, queryPriority)) queryPriority.Priorities[0].QueryAttributes[0].Regex = ".+" + queryPriority.Priorities[0].QueryAttributes[0].CompiledRegex = regexp.MustCompile(".+") assert.Equal(t, int64(1), GetPriority(url.Values{ "query": []string{"sum(up)"}, "time": []string{strconv.FormatInt(now.Unix(), 10)}, - }, now, &queryPriority, true)) + }, now, queryPriority)) assert.Equal(t, int64(1), GetPriority(url.Values{ "query": []string{"count(up)"}, "time": []string{strconv.FormatInt(now.Unix(), 10)}, - }, now, &queryPriority, false)) + }, now, queryPriority)) queryPriority.Priorities[0].QueryAttributes[0].Regex = "" + queryPriority.Priorities[0].QueryAttributes[0].CompiledRegex = regexp.MustCompile("") assert.Equal(t, int64(1), GetPriority(url.Values{ "query": []string{"sum(up)"}, "time": []string{strconv.FormatInt(now.Unix(), 10)}, - }, now, &queryPriority, true)) + }, now, queryPriority)) assert.Equal(t, int64(1), GetPriority(url.Values{ "query": []string{"count(up)"}, "time": []string{strconv.FormatInt(now.Unix(), 10)}, - }, now, &queryPriority, false)) -} - -func Test_GetPriorityShouldNotRecompileRegexIfQueryPriorityChangedIsTrue(t *testing.T) { - now := time.Now() - priorities := []validation.PriorityDef{ - { - Priority: 1, - QueryAttributes: []validation.QueryAttribute{ - { - Regex: "sum", - StartTime: 2 * time.Hour, - EndTime: 0 * time.Hour, - }, - }, - }, - } - queryPriority := validation.QueryPriority{ - Enabled: true, - Priorities: priorities, - } - - assert.Equal(t, int64(1), GetPriority(url.Values{ - "query": []string{"sum(up)"}, - "time": []string{strconv.FormatInt(now.Unix(), 10)}, - }, now, &queryPriority, true)) - - queryPriority.Priorities[0].QueryAttributes[0].Regex = "count" - - assert.Equal(t, int64(0), GetPriority(url.Values{ - "query": []string{"count(up)"}, - "time": []string{strconv.FormatInt(now.Unix(), 10)}, - }, now, &queryPriority, false)) - assert.Equal(t, int64(1), GetPriority(url.Values{ - "query": []string{"count(up)"}, - "time": []string{strconv.FormatInt(now.Unix(), 10)}, - }, now, &queryPriority, true)) + }, now, queryPriority)) } func Test_GetPriorityShouldConsiderStartAndEndTime(t *testing.T) { @@ -173,47 +142,47 @@ func Test_GetPriorityShouldConsiderStartAndEndTime(t *testing.T) { assert.Equal(t, int64(0), GetPriority(url.Values{ "query": []string{"sum(up)"}, "time": []string{strconv.FormatInt(now.Unix(), 10)}, - }, now, &queryPriority, true)) + }, now, queryPriority)) assert.Equal(t, int64(1), GetPriority(url.Values{ "query": []string{"sum(up)"}, "time": []string{strconv.FormatInt(now.Add(-30*time.Minute).Unix(), 10)}, - }, now, &queryPriority, false)) + }, now, queryPriority)) assert.Equal(t, int64(0), GetPriority(url.Values{ "query": []string{"sum(up)"}, "time": []string{strconv.FormatInt(now.Add(-60*time.Minute).Unix(), 10)}, - }, now, &queryPriority, false)) + }, now, queryPriority)) assert.Equal(t, int64(1), GetPriority(url.Values{ "query": []string{"sum(up)"}, "time": []string{strconv.FormatInt(now.Add(-45*time.Minute).Unix(), 10)}, - }, now, &queryPriority, false)) + }, now, queryPriority)) assert.Equal(t, int64(1), GetPriority(url.Values{ "query": []string{"sum(up)"}, "time": []string{strconv.FormatInt(now.Add(-15*time.Minute).Unix(), 10)}, - }, now, &queryPriority, false)) + }, now, queryPriority)) assert.Equal(t, int64(1), GetPriority(url.Values{ "query": []string{"sum(up)"}, "start": []string{strconv.FormatInt(now.Add(-45*time.Minute).Unix(), 10)}, "end": []string{strconv.FormatInt(now.Add(-15*time.Minute).Unix(), 10)}, - }, now, &queryPriority, false)) + }, now, queryPriority)) assert.Equal(t, int64(0), GetPriority(url.Values{ "query": []string{"sum(up)"}, "start": []string{strconv.FormatInt(now.Add(-50*time.Minute).Unix(), 10)}, "end": []string{strconv.FormatInt(now.Add(-15*time.Minute).Unix(), 10)}, - }, now, &queryPriority, false)) + }, now, queryPriority)) assert.Equal(t, int64(0), GetPriority(url.Values{ "query": []string{"sum(up)"}, "start": []string{strconv.FormatInt(now.Add(-45*time.Minute).Unix(), 10)}, "end": []string{strconv.FormatInt(now.Add(-10*time.Minute).Unix(), 10)}, - }, now, &queryPriority, false)) + }, now, queryPriority)) assert.Equal(t, int64(0), GetPriority(url.Values{ "query": []string{"sum(up)"}, "start": []string{strconv.FormatInt(now.Add(-60*time.Minute).Unix(), 10)}, "end": []string{strconv.FormatInt(now.Add(-45*time.Minute).Unix(), 10)}, - }, now, &queryPriority, false)) + }, now, queryPriority)) assert.Equal(t, int64(0), GetPriority(url.Values{ "query": []string{"sum(up)"}, "start": []string{strconv.FormatInt(now.Add(-15*time.Minute).Unix(), 10)}, "end": []string{strconv.FormatInt(now.Add(-1*time.Minute).Unix(), 10)}, - }, now, &queryPriority, false)) + }, now, queryPriority)) } From ab4b9c7902cf5f0a39f5a15b177bf76a84d645f0 Mon Sep 17 00:00:00 2001 From: Justin Jung Date: Tue, 21 Nov 2023 10:52:49 -0800 Subject: [PATCH 39/49] Make reserved querier to handle priorities higher or equal Signed-off-by: Justin Jung --- docs/configuration/config-file-reference.md | 4 ++-- pkg/scheduler/queue/queue.go | 4 ++-- pkg/scheduler/queue/queue_test.go | 6 +----- pkg/scheduler/queue/user_request_queue.go | 4 ++-- pkg/scheduler/queue/user_request_queue_test.go | 17 +++++++++++++++++ pkg/util/validation/limits.go | 2 +- 6 files changed, 25 insertions(+), 12 deletions(-) diff --git a/docs/configuration/config-file-reference.md b/docs/configuration/config-file-reference.md index c4765c2948..1083b3fc6c 100644 --- a/docs/configuration/config-file-reference.md +++ b/docs/configuration/config-file-reference.md @@ -5050,8 +5050,8 @@ otel: # Priority level. Must be a unique value. [priority: | default = 0] -# Number of reserved queriers to handle this priority only. Value between 0 and -# 1 will be used as a percentage. +# Number of reserved queriers to handle priorities higher or equal to this value +# only. Value between 0 and 1 will be used as a percentage. [reserved_queriers: | default = 0] # List of query attributes to assign the priority. diff --git a/pkg/scheduler/queue/queue.go b/pkg/scheduler/queue/queue.go index 57f85bfb12..0f604fce8a 100644 --- a/pkg/scheduler/queue/queue.go +++ b/pkg/scheduler/queue/queue.go @@ -159,8 +159,8 @@ FindQueue: // Pick next request from the queue. for { - priority, matchPriority := q.getPriorityForQuerier(userID, querierID) - request := queue.dequeueRequest(priority, matchPriority) + minPriority, matchMinPriority := q.getPriorityForQuerier(userID, querierID) + request := queue.dequeueRequest(minPriority, matchMinPriority) if request == nil { // The queue does not contain request with the priority, wait for more requests querierWait = true diff --git a/pkg/scheduler/queue/queue_test.go b/pkg/scheduler/queue/queue_test.go index 8c7c1f0794..36531b3886 100644 --- a/pkg/scheduler/queue/queue_test.go +++ b/pkg/scheduler/queue/queue_test.go @@ -219,10 +219,6 @@ func TestReservedQueriersShouldOnlyGetHighPriorityQueries(t *testing.T) { id: "priority 1", priority: 1, } - priority2Request := MockRequest{ - id: "priority 2", - priority: 2, - } assert.NoError(t, queue.EnqueueRequest("userID", normalRequest, 1, func() {})) assert.NoError(t, queue.EnqueueRequest("userID", priority1Request, 1, func() {})) @@ -244,7 +240,7 @@ func TestReservedQueriersShouldOnlyGetHighPriorityQueries(t *testing.T) { assert.Nil(t, nextRequest) assert.Equal(t, 1, queue.queues.userQueues["userID"].queue.length()) - assert.NoError(t, queue.EnqueueRequest("userID", priority2Request, 1, func() {})) + assert.NoError(t, queue.EnqueueRequest("userID", normalRequest, 1, func() {})) ctxTimeout, cancel = context.WithTimeout(ctx, 1*time.Second) defer cancel() diff --git a/pkg/scheduler/queue/user_request_queue.go b/pkg/scheduler/queue/user_request_queue.go index 828c9246fd..20588988c2 100644 --- a/pkg/scheduler/queue/user_request_queue.go +++ b/pkg/scheduler/queue/user_request_queue.go @@ -60,8 +60,8 @@ func (f *PriorityRequestQueue) enqueueRequest(r Request) { } } -func (f *PriorityRequestQueue) dequeueRequest(priority int64, matchPriority bool) Request { - if matchPriority && f.queue.Peek().Priority() != priority { +func (f *PriorityRequestQueue) dequeueRequest(minPriority int64, checkMinPriority bool) Request { + if checkMinPriority && f.queue.Peek().Priority() < minPriority { return nil } r := f.queue.Dequeue() diff --git a/pkg/scheduler/queue/user_request_queue_test.go b/pkg/scheduler/queue/user_request_queue_test.go index cabe792e06..f1f2314585 100644 --- a/pkg/scheduler/queue/user_request_queue_test.go +++ b/pkg/scheduler/queue/user_request_queue_test.go @@ -73,6 +73,10 @@ func TestPriorityRequestQueue(t *testing.T) { id: "request 2", priority: 2, } + request3 := MockRequest{ + id: "request 3", + priority: 3, + } queue.enqueueRequest(request1) queue.enqueueRequest(request2) @@ -102,11 +106,22 @@ func TestPriorityRequestQueue(t *testing.T) { queue.enqueueRequest(request1) queue.enqueueRequest(request2) + queue.enqueueRequest(request3) + assert.NoError(t, promtest.GatherAndCompare(reg, strings.NewReader(` + # HELP cortex_query_scheduler_queue_length Number of queries in the queue. + # TYPE cortex_query_scheduler_queue_length gauge + cortex_query_scheduler_queue_length{priority="1",type="priority",user="userID"} 1 + cortex_query_scheduler_queue_length{priority="2",type="priority",user="userID"} 1 + cortex_query_scheduler_queue_length{priority="3",type="priority",user="userID"} 1 + `), "cortex_query_scheduler_queue_length")) + assert.Equal(t, 3, queue.length()) + assert.Equal(t, request3, queue.dequeueRequest(2, true)) assert.NoError(t, promtest.GatherAndCompare(reg, strings.NewReader(` # HELP cortex_query_scheduler_queue_length Number of queries in the queue. # TYPE cortex_query_scheduler_queue_length gauge cortex_query_scheduler_queue_length{priority="1",type="priority",user="userID"} 1 cortex_query_scheduler_queue_length{priority="2",type="priority",user="userID"} 1 + cortex_query_scheduler_queue_length{priority="3",type="priority",user="userID"} 0 `), "cortex_query_scheduler_queue_length")) assert.Equal(t, 2, queue.length()) assert.Equal(t, request2, queue.dequeueRequest(2, true)) @@ -117,6 +132,7 @@ func TestPriorityRequestQueue(t *testing.T) { # TYPE cortex_query_scheduler_queue_length gauge cortex_query_scheduler_queue_length{priority="1",type="priority",user="userID"} 1 cortex_query_scheduler_queue_length{priority="2",type="priority",user="userID"} 0 + cortex_query_scheduler_queue_length{priority="3",type="priority",user="userID"} 0 `), "cortex_query_scheduler_queue_length")) assert.Equal(t, request1, queue.dequeueRequest(2, false)) assert.Equal(t, 0, queue.length()) @@ -125,5 +141,6 @@ func TestPriorityRequestQueue(t *testing.T) { # TYPE cortex_query_scheduler_queue_length gauge cortex_query_scheduler_queue_length{priority="1",type="priority",user="userID"} 0 cortex_query_scheduler_queue_length{priority="2",type="priority",user="userID"} 0 + cortex_query_scheduler_queue_length{priority="3",type="priority",user="userID"} 0 `), "cortex_query_scheduler_queue_length")) } diff --git a/pkg/util/validation/limits.go b/pkg/util/validation/limits.go index b1bebf1d42..d5adeb9085 100644 --- a/pkg/util/validation/limits.go +++ b/pkg/util/validation/limits.go @@ -56,7 +56,7 @@ type QueryPriority struct { type PriorityDef struct { Priority int64 `yaml:"priority" doc:"nocli|description=Priority level. Must be a unique value.|default=0"` - ReservedQueriers float64 `yaml:"reserved_queriers" doc:"nocli|description=Number of reserved queriers to handle this priority only. Value between 0 and 1 will be used as a percentage.|default=0"` + ReservedQueriers float64 `yaml:"reserved_queriers" doc:"nocli|description=Number of reserved queriers to handle priorities higher or equal to this value only. Value between 0 and 1 will be used as a percentage.|default=0"` QueryAttributes []QueryAttribute `yaml:"query_attributes" doc:"nocli|description=List of query attributes to assign the priority."` } From 8086aee1b85c07ac8261d41e7d87a295c88953c4 Mon Sep 17 00:00:00 2001 From: Justin Jung Date: Tue, 21 Nov 2023 11:11:26 -0800 Subject: [PATCH 40/49] Add benchmark tests Signed-off-by: Justin Jung --- pkg/scheduler/queue/queue.go | 8 +-- pkg/scheduler/queue/queue_test.go | 103 +++++++++++++++++++++++++++++- 2 files changed, 103 insertions(+), 8 deletions(-) diff --git a/pkg/scheduler/queue/queue.go b/pkg/scheduler/queue/queue.go index 0f604fce8a..0d634debed 100644 --- a/pkg/scheduler/queue/queue.go +++ b/pkg/scheduler/queue/queue.go @@ -106,14 +106,10 @@ func (q *RequestQueue) EnqueueRequest(userID string, req Request, maxQueriers fl return errors.New("no queue found") } - metricLabels := prometheus.Labels{ - "user": userID, - "priority": priority, - } - q.totalRequests.With(metricLabels).Inc() + q.totalRequests.WithLabelValues(userID, priority).Inc() if queue.length() >= maxOutstandingRequests { - q.discardedRequests.With(metricLabels).Inc() + q.discardedRequests.WithLabelValues(userID, priority).Inc() return ErrTooManyRequests } diff --git a/pkg/scheduler/queue/queue_test.go b/pkg/scheduler/queue/queue_test.go index 36531b3886..52abe26270 100644 --- a/pkg/scheduler/queue/queue_test.go +++ b/pkg/scheduler/queue/queue_test.go @@ -84,8 +84,8 @@ func BenchmarkQueueRequest(b *testing.B) { for n := 0; n < b.N; n++ { q := NewRequestQueue(maxOutstandingPerTenant, 0, - prometheus.NewGaugeVec(prometheus.GaugeOpts{}, []string{"user"}), - prometheus.NewCounterVec(prometheus.CounterOpts{}, []string{"user"}), + prometheus.NewGaugeVec(prometheus.GaugeOpts{}, []string{"user", "priority", "type"}), + prometheus.NewCounterVec(prometheus.CounterOpts{}, []string{"user", "priority"}), MockLimits{MaxOutstanding: 100}, nil, ) @@ -115,6 +115,105 @@ func BenchmarkQueueRequest(b *testing.B) { } } +func BenchmarkGetNextRequestPriorityQueue(b *testing.B) { + const maxOutstandingPerTenant = 2 + const numTenants = 50 + const queriers = 5 + + queues := make([]*RequestQueue, 0, b.N) + + for n := 0; n < b.N; n++ { + queue := NewRequestQueue(maxOutstandingPerTenant, 0, + prometheus.NewGaugeVec(prometheus.GaugeOpts{}, []string{"user", "priority", "type"}), + prometheus.NewCounterVec(prometheus.CounterOpts{}, []string{"user", "priority"}), + MockLimits{MaxOutstanding: 100, QueryPriorityVal: validation.QueryPriority{Enabled: true}}, + nil, + ) + queues = append(queues, queue) + + for ix := 0; ix < queriers; ix++ { + queue.RegisterQuerierConnection(fmt.Sprintf("querier-%d", ix)) + } + + for i := 0; i < maxOutstandingPerTenant; i++ { + for j := 0; j < numTenants; j++ { + userID := strconv.Itoa(j) + + err := queue.EnqueueRequest(userID, MockRequest{priority: int64(i)}, 0, nil) + if err != nil { + b.Fatal(err) + } + } + } + } + + ctx := context.Background() + b.ResetTimer() + + for i := 0; i < b.N; i++ { + idx := FirstUser() + for j := 0; j < maxOutstandingPerTenant*numTenants; j++ { + querier := "" + b: + // Find querier with at least one request to avoid blocking in getNextRequestForQuerier. + for _, q := range queues[i].queues.userQueues { + for qid := range q.queriers { + querier = qid + break b + } + } + + _, nidx, err := queues[i].GetNextRequestForQuerier(ctx, idx, querier) + if err != nil { + b.Fatal(err) + } + idx = nidx + } + } +} + +func BenchmarkQueueRequestPriorityQueue(b *testing.B) { + const maxOutstandingPerTenant = 2 + const numTenants = 50 + const queriers = 5 + + queues := make([]*RequestQueue, 0, b.N) + users := make([]string, 0, numTenants) + requests := make([]MockRequest, 0, numTenants) + + for n := 0; n < b.N; n++ { + q := NewRequestQueue(maxOutstandingPerTenant, 0, + prometheus.NewGaugeVec(prometheus.GaugeOpts{}, []string{"user", "priority", "type"}), + prometheus.NewCounterVec(prometheus.CounterOpts{}, []string{"user", "priority"}), + MockLimits{MaxOutstanding: 100, QueryPriorityVal: validation.QueryPriority{Enabled: true}}, + nil, + ) + + for ix := 0; ix < queriers; ix++ { + q.RegisterQuerierConnection(fmt.Sprintf("querier-%d", ix)) + } + + queues = append(queues, q) + + for j := 0; j < numTenants; j++ { + requests = append(requests, MockRequest{id: fmt.Sprintf("%d-%d", n, j), priority: int64(j)}) + users = append(users, strconv.Itoa(j)) + } + } + + b.ResetTimer() + for n := 0; n < b.N; n++ { + for i := 0; i < maxOutstandingPerTenant; i++ { + for j := 0; j < numTenants; j++ { + err := queues[n].EnqueueRequest(users[j], requests[j], 0, nil) + if err != nil { + b.Fatal(err) + } + } + } + } +} + func TestRequestQueue_GetNextRequestForQuerier_ShouldGetRequestAfterReshardingBecauseQuerierHasBeenForgotten(t *testing.T) { const forgetDelay = 3 * time.Second From 584c6f94096ce09f486229605434eebe885fdbaa Mon Sep 17 00:00:00 2001 From: Justin Jung Date: Wed, 22 Nov 2023 11:44:55 -0800 Subject: [PATCH 41/49] Update regex to be compiled upon unmarshal Signed-off-by: Justin Jung --- pkg/frontend/transport/handler.go | 5 ++ pkg/frontend/v1/frontend.go | 28 +------ pkg/frontend/v2/frontend.go | 28 +------ pkg/util/query/priority.go | 18 ----- pkg/util/validation/limits.go | 112 ++++++++++++++++++++------- pkg/util/validation/limits_test.go | 120 +++++++++++++++++++---------- 6 files changed, 169 insertions(+), 142 deletions(-) diff --git a/pkg/frontend/transport/handler.go b/pkg/frontend/transport/handler.go index d323a138cd..6bb21b92ba 100644 --- a/pkg/frontend/transport/handler.go +++ b/pkg/frontend/transport/handler.go @@ -201,6 +201,11 @@ func (f *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) { } startTime := time.Now() + // get config + // assign priority + // embed it to the http request, header? + // extract Decode to here, to make sure all requests pass here + // log the priority as well resp, err := f.roundTripper.RoundTrip(r) queryResponseTime := time.Since(startTime) diff --git a/pkg/frontend/v1/frontend.go b/pkg/frontend/v1/frontend.go index 8ea66fbd58..ef80fd07b1 100644 --- a/pkg/frontend/v1/frontend.go +++ b/pkg/frontend/v1/frontend.go @@ -6,8 +6,6 @@ import ( "fmt" "net/http" "net/url" - "reflect" - "sync" "time" "github.com/go-kit/log" @@ -81,13 +79,6 @@ type Frontend struct { requestQueue *queue.RequestQueue activeUsers *util.ActiveUsersCleanupService - // Used to check whether query priority config has changed - queryPriority map[string]validation.QueryPriority - queryPriorityMtx map[string]*sync.RWMutex - - // Populate and reuse compiled regex until query priority config changes - compiledQueryPriority map[string]validation.QueryPriority - // Subservices manager. subservices *services.Manager subservicesWatcher *services.FailureWatcher @@ -223,24 +214,7 @@ func (f *Frontend) RoundTripGRPC(ctx context.Context, req *httpgrpc.HTTPRequest, queryPriority := f.limits.QueryPriority(userID) if queryPriority.Enabled { - if _, exists := f.queryPriorityMtx[userID]; !exists { - f.queryPriorityMtx[userID] = &sync.RWMutex{} - } - - f.queryPriorityMtx[userID].RLock() - queryPriorityChanged := !reflect.DeepEqual(f.queryPriority[userID], queryPriority) - f.queryPriorityMtx[userID].RUnlock() - - if queryPriorityChanged { - f.queryPriorityMtx[userID].Lock() - f.queryPriority[userID] = queryPriority - f.compiledQueryPriority[userID] = util_query.GetCompileQueryPriority(queryPriority) - f.queryPriorityMtx[userID].Unlock() - } - - f.queryPriorityMtx[userID].RLock() - request.priority = util_query.GetPriority(reqParams, ts, f.compiledQueryPriority[userID]) - f.queryPriorityMtx[userID].Unlock() + request.priority = util_query.GetPriority(reqParams, ts, queryPriority) } } diff --git a/pkg/frontend/v2/frontend.go b/pkg/frontend/v2/frontend.go index 41dd4e7749..b11d8c04bc 100644 --- a/pkg/frontend/v2/frontend.go +++ b/pkg/frontend/v2/frontend.go @@ -7,7 +7,6 @@ import ( "math/rand" "net/http" "net/url" - "reflect" "sync" "time" @@ -31,7 +30,6 @@ import ( util_log "github.com/cortexproject/cortex/pkg/util/log" util_query "github.com/cortexproject/cortex/pkg/util/query" "github.com/cortexproject/cortex/pkg/util/services" - "github.com/cortexproject/cortex/pkg/util/validation" ) // Config for a Frontend. @@ -76,13 +74,6 @@ type Frontend struct { lastQueryID atomic.Uint64 - // Used to check whether query priority config has changed - queryPriority map[string]validation.QueryPriority - queryPriorityMtx map[string]*sync.RWMutex - - // Populate and reuse compiled regex until query priority config changes - compiledQueryPriority map[string]validation.QueryPriority - // frontend workers will read from this channel, and send request to scheduler. requestsCh chan *frontendRequest @@ -223,24 +214,7 @@ func (f *Frontend) RoundTripGRPC(ctx context.Context, req *httpgrpc.HTTPRequest, queryPriority := f.limits.QueryPriority(userID) if queryPriority.Enabled { - if _, exists := f.queryPriorityMtx[userID]; !exists { - f.queryPriorityMtx[userID] = &sync.RWMutex{} - } - - f.queryPriorityMtx[userID].RLock() - queryPriorityChanged := !reflect.DeepEqual(f.queryPriority[userID], queryPriority) - f.queryPriorityMtx[userID].RUnlock() - - if queryPriorityChanged { - f.queryPriorityMtx[userID].Lock() - f.queryPriority[userID] = queryPriority - f.compiledQueryPriority[userID] = util_query.GetCompileQueryPriority(queryPriority) - f.queryPriorityMtx[userID].Unlock() - } - - f.queryPriorityMtx[userID].RLock() - freq.priority = util_query.GetPriority(reqParams, ts, f.compiledQueryPriority[userID]) - f.queryPriorityMtx[userID].Unlock() + freq.priority = util_query.GetPriority(reqParams, ts, queryPriority) } } diff --git a/pkg/util/query/priority.go b/pkg/util/query/priority.go index 503736f208..1a8b603d80 100644 --- a/pkg/util/query/priority.go +++ b/pkg/util/query/priority.go @@ -2,30 +2,12 @@ package query import ( "net/url" - "regexp" "time" "github.com/cortexproject/cortex/pkg/util" "github.com/cortexproject/cortex/pkg/util/validation" ) -func GetCompileQueryPriority(queryPriority validation.QueryPriority) validation.QueryPriority { - compiledQueryPriority := queryPriority - for i, priority := range compiledQueryPriority.Priorities { - for j, attribute := range priority.QueryAttributes { - compiledRegex, err := regexp.Compile(attribute.Regex) - if err != nil { - continue - } - - attribute.CompiledRegex = compiledRegex - compiledQueryPriority.Priorities[i].QueryAttributes[j] = attribute - } - } - - return compiledQueryPriority -} - func GetPriority(requestParams url.Values, now time.Time, queryPriority validation.QueryPriority) int64 { queryParam := requestParams.Get("query") timeParam := requestParams.Get("time") diff --git a/pkg/util/validation/limits.go b/pkg/util/validation/limits.go index d5adeb9085..35751c44fd 100644 --- a/pkg/util/validation/limits.go +++ b/pkg/util/validation/limits.go @@ -18,7 +18,8 @@ import ( ) var errMaxGlobalSeriesPerUserValidation = errors.New("The ingester.max-global-series-per-user limit is unsupported if distributor.shard-by-all-labels is disabled") -var errDuplicateQueryPriorities = errors.New("There is a duplicate entry of priorities. Make sure they are all unique, including the default priority") +var errDuplicateQueryPriorities = errors.New("duplicate entry of priorities found. Make sure they are all unique, including the default priority") +var errCompilingQueryPriorityRegex = errors.New("error compiling query priority regex") // Supported values for enum limits const ( @@ -49,22 +50,22 @@ type DisabledRuleGroup struct { type DisabledRuleGroups []DisabledRuleGroup type QueryPriority struct { - Enabled bool `yaml:"enabled" doc:"nocli|description=Whether queries are assigned with priorities.|default=false"` - DefaultPriority int64 `yaml:"default_priority" doc:"nocli|description=Priority assigned to all queries by default. Must be a unique value. Use this as a baseline to make certain queries higher/lower priority.|default=0"` - Priorities []PriorityDef `yaml:"priorities" doc:"nocli|description=List of priority definitions."` + Enabled bool `yaml:"enabled" json:"enabled" doc:"nocli|description=Whether queries are assigned with priorities.|default=false"` + DefaultPriority int64 `yaml:"default_priority" json:"default_priority" doc:"nocli|description=Priority assigned to all queries by default. Must be a unique value. Use this as a baseline to make certain queries higher/lower priority.|default=0"` + Priorities []PriorityDef `yaml:"priorities" json:"priorities" doc:"nocli|description=List of priority definitions."` } type PriorityDef struct { - Priority int64 `yaml:"priority" doc:"nocli|description=Priority level. Must be a unique value.|default=0"` - ReservedQueriers float64 `yaml:"reserved_queriers" doc:"nocli|description=Number of reserved queriers to handle priorities higher or equal to this value only. Value between 0 and 1 will be used as a percentage.|default=0"` - QueryAttributes []QueryAttribute `yaml:"query_attributes" doc:"nocli|description=List of query attributes to assign the priority."` + Priority int64 `yaml:"priority" json:"priority" doc:"nocli|description=Priority level. Must be a unique value.|default=0"` + ReservedQueriers float64 `yaml:"reserved_queriers" json:"reserved_queriers" doc:"nocli|description=Number of reserved queriers to handle priorities higher or equal to this value only. Value between 0 and 1 will be used as a percentage.|default=0"` + QueryAttributes []QueryAttribute `yaml:"query_attributes" json:"query_attributes" doc:"nocli|description=List of query attributes to assign the priority."` } type QueryAttribute struct { - Regex string `yaml:"regex" doc:"nocli|description=Query string regex.|default=.*"` - CompiledRegex *regexp.Regexp `yaml:"-" doc:"nocli"` - StartTime time.Duration `yaml:"start_time" doc:"nocli|description=Query start time.|default=0s"` - EndTime time.Duration `yaml:"end_time" doc:"nocli|description=Query end time.|default=0s"` + Regex string `yaml:"regex" json:"regex" doc:"nocli|description=Query string regex.|default=.*"` + StartTime time.Duration `yaml:"start_time" json:"start_time" doc:"nocli|description=Query start time.|default=0s"` + EndTime time.Duration `yaml:"end_time" json:"end_time" doc:"nocli|description=Query end time.|default=0s"` + CompiledRegex *regexp.Regexp } // Limits describe all the limits for users; can be used to describe global default @@ -122,8 +123,10 @@ type Limits struct { QueryVerticalShardSize int `yaml:"query_vertical_shard_size" json:"query_vertical_shard_size" doc:"hidden"` // Query Frontend / Scheduler enforced limits. - MaxOutstandingPerTenant int `yaml:"max_outstanding_requests_per_tenant" json:"max_outstanding_requests_per_tenant"` - QueryPriority QueryPriority `yaml:"query_priority" json:"query_priority" doc:"nocli|description=Configuration for query priority."` + MaxOutstandingPerTenant int `yaml:"max_outstanding_requests_per_tenant" json:"max_outstanding_requests_per_tenant"` + QueryPriority QueryPriority `yaml:"query_priority" json:"query_priority" doc:"nocli|description=Configuration for query priority."` + queryPriorityRegexHash string + queryPriorityCompiledRegex map[string]*regexp.Regexp // Ruler defaults and limits. RulerEvaluationDelay model.Duration `yaml:"ruler_evaluation_delay_duration" json:"ruler_evaluation_delay_duration"` @@ -250,20 +253,6 @@ func (l *Limits) Validate(shardByAllLabels bool) error { return errMaxGlobalSeriesPerUserValidation } - // If query priority is enabled, do not allow duplicate priority values - if l.QueryPriority.Enabled { - queryPriority := l.QueryPriority - prioritySet := map[int64]struct{}{} - prioritySet[queryPriority.DefaultPriority] = struct{}{} - for _, priority := range queryPriority.Priorities { - if _, exists := prioritySet[priority.Priority]; exists { - return errDuplicateQueryPriorities - } - - prioritySet[priority.Priority] = struct{}{} - } - } - return nil } @@ -280,7 +269,15 @@ func (l *Limits) UnmarshalYAML(unmarshal func(interface{}) error) error { l.copyNotificationIntegrationLimits(defaultLimits.NotificationRateLimitPerIntegration) } type plain Limits - return unmarshal((*plain)(l)) + if err := unmarshal((*plain)(l)); err != nil { + return err + } + + if err := l.compileQueryPriorityRegex(); err != nil { + return err + } + + return nil } // UnmarshalJSON implements the json.Unmarshaler interface. @@ -298,7 +295,15 @@ func (l *Limits) UnmarshalJSON(data []byte) error { dec := json.NewDecoder(bytes.NewReader(data)) dec.DisallowUnknownFields() - return dec.Decode((*plain)(l)) + if err := dec.Decode((*plain)(l)); err != nil { + return err + } + + if err := l.compileQueryPriorityRegex(); err != nil { + return err + } + + return nil } func (l *Limits) copyNotificationIntegrationLimits(defaults NotificationRateLimitMap) { @@ -308,6 +313,55 @@ func (l *Limits) copyNotificationIntegrationLimits(defaults NotificationRateLimi } } +func (l *Limits) hasQueryPriorityRegexChanged() bool { + var newHash string + for _, priority := range l.QueryPriority.Priorities { + for _, attribute := range priority.QueryAttributes { + newHash += attribute.Regex + } + } + if newHash != l.queryPriorityRegexHash { + l.queryPriorityRegexHash = newHash + return true + } + return false +} + +func (l *Limits) compileQueryPriorityRegex() error { + if l.QueryPriority.Enabled { + hasQueryPriorityRegexChanged := l.hasQueryPriorityRegexChanged() + prioritySet := map[int64]struct{}{} + newQueryPriorityCompiledRegex := map[string]*regexp.Regexp{} + + for i, priority := range l.QueryPriority.Priorities { + // Check for duplicate priority entry + if _, exists := prioritySet[priority.Priority]; exists { + return errDuplicateQueryPriorities + } + prioritySet[priority.Priority] = struct{}{} + + for j, attribute := range priority.QueryAttributes { + if hasQueryPriorityRegexChanged { + compiledRegex, err := regexp.Compile(attribute.Regex) + if err != nil { + return errors.Join(errCompilingQueryPriorityRegex, err) + } + newQueryPriorityCompiledRegex[attribute.Regex] = compiledRegex + l.QueryPriority.Priorities[i].QueryAttributes[j].CompiledRegex = compiledRegex + } else { + l.QueryPriority.Priorities[i].QueryAttributes[j].CompiledRegex = l.queryPriorityCompiledRegex[attribute.Regex] + } + } + } + + if hasQueryPriorityRegexChanged { + l.queryPriorityCompiledRegex = newQueryPriorityCompiledRegex + } + } + + return nil +} + // When we load YAML from disk, we want the various per-customer limits // to default to any values specified on the command line, not default // command line values. This global contains those values. I (Tom) cannot diff --git a/pkg/util/validation/limits_test.go b/pkg/util/validation/limits_test.go index 01c18225c2..7d4542f491 100644 --- a/pkg/util/validation/limits_test.go +++ b/pkg/util/validation/limits_test.go @@ -3,6 +3,7 @@ package validation import ( "encoding/json" "reflect" + "regexp" "strings" "testing" "time" @@ -61,47 +62,6 @@ func TestLimits_Validate(t *testing.T) { shardByAllLabels: true, expected: nil, }, - "duplicate priority entries": { - limits: Limits{QueryPriority: QueryPriority{ - Enabled: true, - Priorities: []PriorityDef{ - { - Priority: 1, - }, - { - Priority: 1, - }, - }, - }}, - expected: errDuplicateQueryPriorities, - }, - "priority matches default priority": { - limits: Limits{QueryPriority: QueryPriority{ - Enabled: true, - DefaultPriority: 1, - Priorities: []PriorityDef{ - { - Priority: 1, - }, - }, - }}, - expected: errDuplicateQueryPriorities, - }, - "all priorities are unique": { - limits: Limits{QueryPriority: QueryPriority{ - Enabled: true, - DefaultPriority: 1, - Priorities: []PriorityDef{ - { - Priority: 2, - }, - { - Priority: 3, - }, - }, - }}, - expected: nil, - }, } for testName, testData := range tests { @@ -637,3 +597,81 @@ tenant2: require.Equal(t, 3, ov.MaxDownloadedBytesPerRequest("tenant2")) require.Equal(t, 5, ov.MaxDownloadedBytesPerRequest("tenant3")) } + +func TestHasQueryPriorityRegexChanged(t *testing.T) { + l := Limits{ + QueryPriority: QueryPriority{ + Priorities: []PriorityDef{ + { + Priority: 1, + QueryAttributes: []QueryAttribute{ + { + Regex: "test", + }, + }, + }, + }, + }, + } + + require.True(t, l.hasQueryPriorityRegexChanged()) + + l.QueryPriority.Priorities[0].QueryAttributes[0].Regex = "new" + + require.True(t, l.hasQueryPriorityRegexChanged()) + + l.QueryPriority.Priorities[0].QueryAttributes[0].StartTime = 2 * time.Hour + + require.False(t, l.hasQueryPriorityRegexChanged()) + + l.QueryPriority.Priorities[0].QueryAttributes = append(l.QueryPriority.Priorities[0].QueryAttributes, QueryAttribute{Regex: "hi"}) + + require.True(t, l.hasQueryPriorityRegexChanged()) + + l.QueryPriority.Priorities[0].QueryAttributes = l.QueryPriority.Priorities[0].QueryAttributes[:1] + + require.True(t, l.hasQueryPriorityRegexChanged()) +} + +func TestCompileQueryPriorityRegex(t *testing.T) { + l := Limits{ + QueryPriority: QueryPriority{ + Enabled: true, + Priorities: []PriorityDef{ + { + Priority: 1, + QueryAttributes: []QueryAttribute{ + { + Regex: "test", + }, + }, + }, + }, + }, + } + + require.Nil(t, l.QueryPriority.Priorities[0].QueryAttributes[0].CompiledRegex) + + err := l.compileQueryPriorityRegex() + require.NoError(t, err) + require.Equal(t, regexp.MustCompile("test"), l.QueryPriority.Priorities[0].QueryAttributes[0].CompiledRegex) + + l.QueryPriority.Priorities[0].QueryAttributes[0].Regex = "new" + + err = l.compileQueryPriorityRegex() + require.NoError(t, err) + require.Equal(t, regexp.MustCompile("new"), l.QueryPriority.Priorities[0].QueryAttributes[0].CompiledRegex) + + l.QueryPriority.Priorities[0].QueryAttributes[0].CompiledRegex = nil + + err = l.compileQueryPriorityRegex() + require.NoError(t, err) + require.Equal(t, regexp.MustCompile("new"), l.QueryPriority.Priorities[0].QueryAttributes[0].CompiledRegex) + + l.QueryPriority.Enabled = false + l.QueryPriority.Priorities[0].QueryAttributes[0].CompiledRegex = nil + + err = l.compileQueryPriorityRegex() + require.NoError(t, err) + require.Nil(t, l.QueryPriority.Priorities[0].QueryAttributes[0].CompiledRegex) +} From 4001fe440a55eaeaa9b59155fddbd66ca7eddc1b Mon Sep 17 00:00:00 2001 From: Justin Jung Date: Wed, 22 Nov 2023 14:13:16 -0800 Subject: [PATCH 42/49] Make start and end time check to be skipped if not specified Signed-off-by: Justin Jung --- pkg/util/query/priority.go | 33 +++++++++++++++----- pkg/util/query/priority_test.go | 48 ++++++++++++++++++++++++++---- pkg/util/validation/limits.go | 6 ++-- pkg/util/validation/limits_test.go | 2 +- 4 files changed, 71 insertions(+), 18 deletions(-) diff --git a/pkg/util/query/priority.go b/pkg/util/query/priority.go index 1a8b603d80..2a7f61ba10 100644 --- a/pkg/util/query/priority.go +++ b/pkg/util/query/priority.go @@ -24,25 +24,22 @@ func GetPriority(requestParams url.Values, now time.Time, queryPriority validati continue } - startTimeThreshold := now.Add(-1 * attribute.StartTime.Abs()).Truncate(time.Second).UTC() - endTimeThreshold := now.Add(-1 * attribute.EndTime.Abs()).Add(1 * time.Second).Truncate(time.Second).UTC() - if startTime, err := util.ParseTime(startParam); err == nil { if endTime, err := util.ParseTime(endParam); err == nil { - if isBetweenThresholds(util.TimeFromMillis(startTime), util.TimeFromMillis(endTime), startTimeThreshold, endTimeThreshold) { + if isWithinTimeAttributes(attribute, now, startTime, endTime) { return priority.Priority } } } if instantTime, err := util.ParseTime(timeParam); err == nil { - if isBetweenThresholds(util.TimeFromMillis(instantTime), util.TimeFromMillis(instantTime), startTimeThreshold, endTimeThreshold) { + if isWithinTimeAttributes(attribute, now, instantTime, instantTime) { return priority.Priority } } if timeParam == "" { - if isBetweenThresholds(now, now, startTimeThreshold, endTimeThreshold) { + if isWithinTimeAttributes(attribute, now, util.TimeToMillis(now), util.TimeToMillis(now)) { return priority.Priority } } @@ -52,6 +49,26 @@ func GetPriority(requestParams url.Values, now time.Time, queryPriority validati return queryPriority.DefaultPriority } -func isBetweenThresholds(start, end, startThreshold, endThreshold time.Time) bool { - return (start.Equal(startThreshold) || start.After(startThreshold)) && (end.Equal(endThreshold) || end.Before(endThreshold)) +func isWithinTimeAttributes(attribute validation.QueryAttribute, now time.Time, startTime, endTime int64) bool { + if attribute.StartTime == 0 && attribute.EndTime == 0 { + return true + } + + if attribute.StartTime != 0 { + startTimeThreshold := now.Add(-1 * time.Duration(attribute.StartTime).Abs()).Truncate(time.Second) + + if util.TimeFromMillis(startTime).Before(startTimeThreshold) { + return false + } + } + + if attribute.EndTime != 0 { + endTimeThreshold := now.Add(-1 * time.Duration(attribute.EndTime).Abs()).Add(1 * time.Second).Truncate(time.Second) + + if util.TimeFromMillis(endTime).After(endTimeThreshold) { + return false + } + } + + return true } diff --git a/pkg/util/query/priority_test.go b/pkg/util/query/priority_test.go index 646aed17c1..fea552cde4 100644 --- a/pkg/util/query/priority_test.go +++ b/pkg/util/query/priority_test.go @@ -7,6 +7,7 @@ import ( "testing" "time" + "github.com/prometheus/common/model" "github.com/stretchr/testify/assert" "github.com/cortexproject/cortex/pkg/util/validation" @@ -21,8 +22,6 @@ func Test_GetPriorityShouldReturnDefaultPriorityIfNotEnabledOrEmptyQueryString(t { Regex: ".*", CompiledRegex: regexp.MustCompile(".*"), - StartTime: 2 * time.Hour, - EndTime: 0 * time.Hour, }, }, }, @@ -52,8 +51,6 @@ func Test_GetPriorityShouldConsiderRegex(t *testing.T) { { Regex: "sum", CompiledRegex: regexp.MustCompile("sum"), - StartTime: 2 * time.Hour, - EndTime: 0 * time.Hour, }, }, }, @@ -128,8 +125,8 @@ func Test_GetPriorityShouldConsiderStartAndEndTime(t *testing.T) { Priority: 1, QueryAttributes: []validation.QueryAttribute{ { - StartTime: 45 * time.Minute, - EndTime: 15 * time.Minute, + StartTime: model.Duration(45 * time.Minute), + EndTime: model.Duration(15 * time.Minute), }, }, }, @@ -186,3 +183,42 @@ func Test_GetPriorityShouldConsiderStartAndEndTime(t *testing.T) { "end": []string{strconv.FormatInt(now.Add(-1*time.Minute).Unix(), 10)}, }, now, queryPriority)) } + +func Test_GetPriorityShouldSKipStartAndEndTimeIfEmpty(t *testing.T) { + now := time.Now() + priorities := []validation.PriorityDef{ + { + Priority: 1, + QueryAttributes: []validation.QueryAttribute{ + { + Regex: "^test$", + }, + }, + }, + } + queryPriority := validation.QueryPriority{ + Enabled: true, + Priorities: priorities, + } + + assert.Equal(t, int64(1), GetPriority(url.Values{ + "query": []string{"test"}, + }, now, queryPriority)) + assert.Equal(t, int64(1), GetPriority(url.Values{ + "query": []string{"test"}, + "time": []string{strconv.FormatInt(now.Add(8760*time.Hour).Unix(), 10)}, + }, now, queryPriority)) + assert.Equal(t, int64(1), GetPriority(url.Values{ + "query": []string{"test"}, + "time": []string{strconv.FormatInt(now.Unix(), 10)}, + }, now, queryPriority)) + assert.Equal(t, int64(1), GetPriority(url.Values{ + "query": []string{"test"}, + "time": []string{strconv.FormatInt(now.Add(-8760*time.Hour).Unix(), 10)}, + }, now, queryPriority)) + assert.Equal(t, int64(1), GetPriority(url.Values{ + "query": []string{"test"}, + "start": []string{strconv.FormatInt(now.Add(-100000*time.Minute).Unix(), 10)}, + "end": []string{strconv.FormatInt(now.Add(100000*time.Minute).Unix(), 10)}, + }, now, queryPriority)) +} diff --git a/pkg/util/validation/limits.go b/pkg/util/validation/limits.go index 35751c44fd..b9b005d0df 100644 --- a/pkg/util/validation/limits.go +++ b/pkg/util/validation/limits.go @@ -62,9 +62,9 @@ type PriorityDef struct { } type QueryAttribute struct { - Regex string `yaml:"regex" json:"regex" doc:"nocli|description=Query string regex.|default=.*"` - StartTime time.Duration `yaml:"start_time" json:"start_time" doc:"nocli|description=Query start time.|default=0s"` - EndTime time.Duration `yaml:"end_time" json:"end_time" doc:"nocli|description=Query end time.|default=0s"` + Regex string `yaml:"regex" json:"regex" doc:"nocli|description=Query string regex. If set to empty string, it will not match anything.|default=\"\""` + StartTime model.Duration `yaml:"start_time" json:"start_time" doc:"nocli|description=Query start time. If set to 0, the start time won't be checked.'.|default=0"` + EndTime model.Duration `yaml:"end_time" json:"end_time" doc:"nocli|description=Query end time. If set to 0, the end time won't be checked.|default=0"` CompiledRegex *regexp.Regexp } diff --git a/pkg/util/validation/limits_test.go b/pkg/util/validation/limits_test.go index 7d4542f491..32a0bf924c 100644 --- a/pkg/util/validation/limits_test.go +++ b/pkg/util/validation/limits_test.go @@ -620,7 +620,7 @@ func TestHasQueryPriorityRegexChanged(t *testing.T) { require.True(t, l.hasQueryPriorityRegexChanged()) - l.QueryPriority.Priorities[0].QueryAttributes[0].StartTime = 2 * time.Hour + l.QueryPriority.Priorities[0].QueryAttributes[0].StartTime = model.Duration(2 * time.Hour) require.False(t, l.hasQueryPriorityRegexChanged()) From aa38a8c556756af3337a3deb8537758cceaad585 Mon Sep 17 00:00:00 2001 From: Justin Jung Date: Thu, 23 Nov 2023 16:34:44 -0800 Subject: [PATCH 43/49] Assign priority before splitting the query Signed-off-by: Justin Jung --- docs/configuration/config-file-reference.md | 16 +- pkg/frontend/transport/handler.go | 9 +- pkg/frontend/transport/roundtripper.go | 20 +- pkg/frontend/v1/frontend.go | 27 +- pkg/frontend/v2/frontend.go | 13 +- pkg/frontend/v2/frontend_scheduler_worker.go | 1 - pkg/frontend/v2/frontend_test.go | 13 +- .../tripperware/instantquery/instant_query.go | 18 +- pkg/querier/tripperware/limits.go | 9 +- pkg/querier/tripperware/priority.go | 94 ++++++ pkg/querier/tripperware/priority_test.go | 277 ++++++++++++++++++ .../tripperware/queryrange/limits_test.go | 5 + pkg/querier/tripperware/roundtrip.go | 23 +- .../tripperware/test_shard_by_query_utils.go | 6 + pkg/scheduler/scheduler.go | 10 +- pkg/scheduler/schedulerpb/scheduler.pb.go | 127 +++----- pkg/scheduler/schedulerpb/scheduler.proto | 1 - pkg/util/http.go | 1 + pkg/util/httpgrpcutil/header.go | 22 ++ pkg/util/query/priority.go | 74 ----- pkg/util/query/priority_test.go | 224 -------------- pkg/util/time.go | 14 + pkg/util/validation/limits.go | 6 +- tools/doc-generator/parser.go | 4 + 24 files changed, 533 insertions(+), 481 deletions(-) create mode 100644 pkg/querier/tripperware/priority.go create mode 100644 pkg/querier/tripperware/priority_test.go create mode 100644 pkg/util/httpgrpcutil/header.go delete mode 100644 pkg/util/query/priority.go delete mode 100644 pkg/util/query/priority_test.go diff --git a/docs/configuration/config-file-reference.md b/docs/configuration/config-file-reference.md index 1083b3fc6c..25286293dd 100644 --- a/docs/configuration/config-file-reference.md +++ b/docs/configuration/config-file-reference.md @@ -5050,8 +5050,8 @@ otel: # Priority level. Must be a unique value. [priority: | default = 0] -# Number of reserved queriers to handle priorities higher or equal to this value -# only. Value between 0 and 1 will be used as a percentage. +# Number of reserved queriers to handle priorities higher or equal to the +# priority level. Value between 0 and 1 will be used as a percentage. [reserved_queriers: | default = 0] # List of query attributes to assign the priority. @@ -5061,14 +5061,14 @@ otel: ### `QueryAttribute` ```yaml -# Query string regex. -[regex: | default = ".*"] +# Query string regex. If set to empty string, it will not match anything. +[regex: | default = ""] -# Query start time. -[start_time: | default = 0s] +# Query start time. If set to 0, the start time won't be checked. +[start_time: | default = 0] -# Query end time. -[end_time: | default = 0s] +# Query end time. If set to 0, the end time won't be checked. +[end_time: | default = 0] ``` ### `DisabledRuleGroup` diff --git a/pkg/frontend/transport/handler.go b/pkg/frontend/transport/handler.go index 6bb21b92ba..042dbd32b7 100644 --- a/pkg/frontend/transport/handler.go +++ b/pkg/frontend/transport/handler.go @@ -200,12 +200,8 @@ func (f *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) { r.Body = io.NopCloser(&buf) } + r.Header.Get("test") startTime := time.Now() - // get config - // assign priority - // embed it to the http request, header? - // extract Decode to here, to make sure all requests pass here - // log the priority as well resp, err := f.roundTripper.RoundTrip(r) queryResponseTime := time.Since(startTime) @@ -348,6 +344,9 @@ func (f *Handler) reportQueryStats(r *http.Request, userID string, queryString u if ua := r.Header.Get("User-Agent"); len(ua) > 0 { logMessage = append(logMessage, "user_agent", ua) } + if queryPriority := r.Header.Get(util.QueryPriorityHeaderKey); len(queryPriority) > 0 { + logMessage = append(logMessage, "priority", queryPriority) + } if error != nil { s, ok := status.FromError(error) diff --git a/pkg/frontend/transport/roundtripper.go b/pkg/frontend/transport/roundtripper.go index a0529ba6a4..583fc22d04 100644 --- a/pkg/frontend/transport/roundtripper.go +++ b/pkg/frontend/transport/roundtripper.go @@ -5,9 +5,6 @@ import ( "context" "io" "net/http" - "net/url" - "strings" - "time" "github.com/weaveworks/common/httpgrpc" "github.com/weaveworks/common/httpgrpc/server" @@ -15,7 +12,7 @@ import ( // GrpcRoundTripper is similar to http.RoundTripper, but works with HTTP requests converted to protobuf messages. type GrpcRoundTripper interface { - RoundTripGRPC(context.Context, *httpgrpc.HTTPRequest, url.Values, time.Time) (*httpgrpc.HTTPResponse, error) + RoundTripGRPC(context.Context, *httpgrpc.HTTPRequest) (*httpgrpc.HTTPResponse, error) } func AdaptGrpcRoundTripperToHTTPRoundTripper(r GrpcRoundTripper) http.RoundTripper { @@ -42,20 +39,7 @@ func (a *grpcRoundTripperAdapter) RoundTrip(r *http.Request) (*http.Response, er return nil, err } - var ( - resp *httpgrpc.HTTPResponse - reqValues url.Values - ts time.Time - ) - - if strings.HasSuffix(r.URL.Path, "/query") || strings.HasSuffix(r.URL.Path, "/query_range") { - if err = r.ParseForm(); err == nil { - reqValues = r.Form - ts = time.Now() - } - } - - resp, err = a.roundTripper.RoundTripGRPC(r.Context(), req, reqValues, ts) + resp, err := a.roundTripper.RoundTripGRPC(r.Context(), req) if err != nil { return nil, err } diff --git a/pkg/frontend/v1/frontend.go b/pkg/frontend/v1/frontend.go index ef80fd07b1..024c1f961a 100644 --- a/pkg/frontend/v1/frontend.go +++ b/pkg/frontend/v1/frontend.go @@ -5,7 +5,7 @@ import ( "flag" "fmt" "net/http" - "net/url" + "strconv" "time" "github.com/go-kit/log" @@ -23,7 +23,6 @@ import ( "github.com/cortexproject/cortex/pkg/tenant" "github.com/cortexproject/cortex/pkg/util" "github.com/cortexproject/cortex/pkg/util/httpgrpcutil" - util_query "github.com/cortexproject/cortex/pkg/util/query" "github.com/cortexproject/cortex/pkg/util/services" "github.com/cortexproject/cortex/pkg/util/validation" ) @@ -98,11 +97,15 @@ type request struct { request *httpgrpc.HTTPRequest err chan error response chan *httpgrpc.HTTPResponse - priority int64 } func (r request) Priority() int64 { - return r.priority + priority, err := strconv.ParseInt(httpgrpcutil.GetHeader(*r.request, util.QueryPriorityHeaderKey), 10, 64) + if err != nil { + return 0 + } + + return priority } // New creates a new frontend. Frontend implements service, and must be started and stopped. @@ -181,7 +184,7 @@ func (f *Frontend) cleanupInactiveUserMetrics(user string) { } // RoundTripGRPC round trips a proto (instead of a HTTP request). -func (f *Frontend) RoundTripGRPC(ctx context.Context, req *httpgrpc.HTTPRequest, reqParams url.Values, ts time.Time) (*httpgrpc.HTTPResponse, error) { +func (f *Frontend) RoundTripGRPC(ctx context.Context, req *httpgrpc.HTTPRequest) (*httpgrpc.HTTPResponse, error) { // Propagate trace context in gRPC too - this will be ignored if using HTTP. tracer, span := opentracing.GlobalTracer(), opentracing.SpanFromContext(ctx) if tracer != nil && span != nil { @@ -192,12 +195,6 @@ func (f *Frontend) RoundTripGRPC(ctx context.Context, req *httpgrpc.HTTPRequest, } } - tenantIDs, err := tenant.TenantIDs(ctx) - if err != nil { - return nil, err - } - userID := tenant.JoinTenantIDs(tenantIDs) - return f.retry.Do(ctx, func() (*httpgrpc.HTTPResponse, error) { request := request{ request: req, @@ -210,14 +207,6 @@ func (f *Frontend) RoundTripGRPC(ctx context.Context, req *httpgrpc.HTTPRequest, response: make(chan *httpgrpc.HTTPResponse, 1), } - if reqParams != nil { - queryPriority := f.limits.QueryPriority(userID) - - if queryPriority.Enabled { - request.priority = util_query.GetPriority(reqParams, ts, queryPriority) - } - } - if err := f.queueRequest(ctx, &request); err != nil { return nil, err } diff --git a/pkg/frontend/v2/frontend.go b/pkg/frontend/v2/frontend.go index b11d8c04bc..2df0f8f344 100644 --- a/pkg/frontend/v2/frontend.go +++ b/pkg/frontend/v2/frontend.go @@ -6,7 +6,6 @@ import ( "fmt" "math/rand" "net/http" - "net/url" "sync" "time" @@ -28,7 +27,6 @@ import ( "github.com/cortexproject/cortex/pkg/util/grpcclient" "github.com/cortexproject/cortex/pkg/util/httpgrpcutil" util_log "github.com/cortexproject/cortex/pkg/util/log" - util_query "github.com/cortexproject/cortex/pkg/util/query" "github.com/cortexproject/cortex/pkg/util/services" ) @@ -89,7 +87,6 @@ type frontendRequest struct { request *httpgrpc.HTTPRequest userID string statsEnabled bool - priority int64 cancel context.CancelFunc @@ -170,7 +167,7 @@ func (f *Frontend) stopping(_ error) error { } // RoundTripGRPC round trips a proto (instead of a HTTP request). -func (f *Frontend) RoundTripGRPC(ctx context.Context, req *httpgrpc.HTTPRequest, reqParams url.Values, ts time.Time) (*httpgrpc.HTTPResponse, error) { +func (f *Frontend) RoundTripGRPC(ctx context.Context, req *httpgrpc.HTTPRequest) (*httpgrpc.HTTPResponse, error) { if s := f.State(); s != services.Running { return nil, fmt.Errorf("frontend not running: %v", s) } @@ -210,14 +207,6 @@ func (f *Frontend) RoundTripGRPC(ctx context.Context, req *httpgrpc.HTTPRequest, retryOnTooManyOutstandingRequests: f.cfg.RetryOnTooManyOutstandingRequests && f.schedulerWorkers.getWorkersCount() > 1, } - if reqParams != nil { - queryPriority := f.limits.QueryPriority(userID) - - if queryPriority.Enabled { - freq.priority = util_query.GetPriority(reqParams, ts, queryPriority) - } - } - f.requests.put(freq) defer f.requests.delete(freq.queryID) diff --git a/pkg/frontend/v2/frontend_scheduler_worker.go b/pkg/frontend/v2/frontend_scheduler_worker.go index 2abff24bf6..bbe1e2ed74 100644 --- a/pkg/frontend/v2/frontend_scheduler_worker.go +++ b/pkg/frontend/v2/frontend_scheduler_worker.go @@ -263,7 +263,6 @@ func (w *frontendSchedulerWorker) schedulerLoop(loop schedulerpb.SchedulerForFro HttpRequest: req.request, FrontendAddress: w.frontendAddr, StatsEnabled: req.statsEnabled, - Priority: req.priority, }) if err != nil { diff --git a/pkg/frontend/v2/frontend_test.go b/pkg/frontend/v2/frontend_test.go index 4c40d88b66..6b20926e89 100644 --- a/pkg/frontend/v2/frontend_test.go +++ b/pkg/frontend/v2/frontend_test.go @@ -3,7 +3,6 @@ package v2 import ( "context" "net" - "net/url" "strconv" "strings" "sync" @@ -112,7 +111,7 @@ func TestFrontendBasicWorkflow(t *testing.T) { return &schedulerpb.SchedulerToFrontend{Status: schedulerpb.OK} }, 0) - resp, err := f.RoundTripGRPC(user.InjectOrgID(context.Background(), userID), &httpgrpc.HTTPRequest{}, url.Values{}, time.Now()) + resp, err := f.RoundTripGRPC(user.InjectOrgID(context.Background(), userID), &httpgrpc.HTTPRequest{}) require.NoError(t, err) require.Equal(t, int32(200), resp.Code) require.Equal(t, []byte(body), resp.Body) @@ -142,7 +141,7 @@ func TestFrontendRetryRequest(t *testing.T) { return &schedulerpb.SchedulerToFrontend{Status: schedulerpb.OK} }, 3) - res, err := f.RoundTripGRPC(user.InjectOrgID(context.Background(), userID), &httpgrpc.HTTPRequest{}, url.Values{}, time.Now()) + res, err := f.RoundTripGRPC(user.InjectOrgID(context.Background(), userID), &httpgrpc.HTTPRequest{}) require.NoError(t, err) require.Equal(t, int32(200), res.Code) } @@ -169,7 +168,7 @@ func TestFrontendRetryEnqueue(t *testing.T) { return &schedulerpb.SchedulerToFrontend{Status: schedulerpb.OK} }, 0) - _, err := f.RoundTripGRPC(user.InjectOrgID(context.Background(), userID), &httpgrpc.HTTPRequest{}, url.Values{}, time.Now()) + _, err := f.RoundTripGRPC(user.InjectOrgID(context.Background(), userID), &httpgrpc.HTTPRequest{}) require.NoError(t, err) } @@ -178,7 +177,7 @@ func TestFrontendEnqueueFailure(t *testing.T) { return &schedulerpb.SchedulerToFrontend{Status: schedulerpb.SHUTTING_DOWN} }, 0) - _, err := f.RoundTripGRPC(user.InjectOrgID(context.Background(), "test"), &httpgrpc.HTTPRequest{}, url.Values{}, time.Now()) + _, err := f.RoundTripGRPC(user.InjectOrgID(context.Background(), "test"), &httpgrpc.HTTPRequest{}) require.Error(t, err) require.True(t, strings.Contains(err.Error(), "failed to enqueue request")) } @@ -189,7 +188,7 @@ func TestFrontendCancellation(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), 200*time.Millisecond) defer cancel() - resp, err := f.RoundTripGRPC(user.InjectOrgID(ctx, "test"), &httpgrpc.HTTPRequest{}, url.Values{}, time.Now()) + resp, err := f.RoundTripGRPC(user.InjectOrgID(ctx, "test"), &httpgrpc.HTTPRequest{}) require.EqualError(t, err, context.DeadlineExceeded.Error()) require.Nil(t, resp) @@ -238,7 +237,7 @@ func TestFrontendFailedCancellation(t *testing.T) { }() // send request - resp, err := f.RoundTripGRPC(user.InjectOrgID(ctx, "test"), &httpgrpc.HTTPRequest{}, url.Values{}, time.Now()) + resp, err := f.RoundTripGRPC(user.InjectOrgID(ctx, "test"), &httpgrpc.HTTPRequest{}) require.EqualError(t, err, context.Canceled.Error()) require.Nil(t, resp) diff --git a/pkg/querier/tripperware/instantquery/instant_query.go b/pkg/querier/tripperware/instantquery/instant_query.go index a7350e65d5..60bea7a8d1 100644 --- a/pkg/querier/tripperware/instantquery/instant_query.go +++ b/pkg/querier/tripperware/instantquery/instant_query.go @@ -4,18 +4,17 @@ import ( "bytes" "context" "fmt" + "github.com/cortexproject/cortex/pkg/util" "io" "net/http" "net/url" "sort" - "strconv" "strings" "time" jsoniter "github.com/json-iterator/go" "github.com/opentracing/opentracing-go" otlog "github.com/opentracing/opentracing-go/log" - "github.com/pkg/errors" "github.com/prometheus/common/model" "github.com/prometheus/prometheus/model/labels" "github.com/prometheus/prometheus/model/timestamp" @@ -26,7 +25,6 @@ import ( "github.com/cortexproject/cortex/pkg/cortexpb" "github.com/cortexproject/cortex/pkg/querier/tripperware" "github.com/cortexproject/cortex/pkg/querier/tripperware/queryrange" - "github.com/cortexproject/cortex/pkg/util" "github.com/cortexproject/cortex/pkg/util/spanlogger" ) @@ -132,7 +130,7 @@ func (resp *PrometheusInstantQueryResponse) HTTPHeaders() map[string][]string { func (c instantQueryCodec) DecodeRequest(_ context.Context, r *http.Request, forwardHeaders []string) (tripperware.Request, error) { result := PrometheusRequest{Headers: map[string][]string{}} var err error - result.Time, err = parseTimeParam(r, "time", c.now().Unix()) + result.Time, err = util.ParseTimeParam(r, "time", c.now().Unix()) if err != nil { return nil, decorateWithParamName(err, "time") } @@ -630,15 +628,3 @@ func (s *PrometheusInstantQueryData) MarshalJSON() ([]byte, error) { return s.Result.GetRawBytes(), nil } } - -func parseTimeParam(r *http.Request, paramName string, defaultValue int64) (int64, error) { - val := r.FormValue(paramName) - if val == "" { - val = strconv.FormatInt(defaultValue, 10) - } - result, err := util.ParseTime(val) - if err != nil { - return 0, errors.Wrapf(err, "Invalid time value for '%s'", paramName) - } - return result, nil -} diff --git a/pkg/querier/tripperware/limits.go b/pkg/querier/tripperware/limits.go index 15ce78592f..815693b3c1 100644 --- a/pkg/querier/tripperware/limits.go +++ b/pkg/querier/tripperware/limits.go @@ -1,6 +1,10 @@ package tripperware -import "time" +import ( + "time" + + "github.com/cortexproject/cortex/pkg/util/validation" +) // Limits allows us to specify per-tenant runtime limits on the behavior of // the query handling code. @@ -21,4 +25,7 @@ type Limits interface { // QueryVerticalShardSize returns the maximum number of queriers that can handle requests for this user. QueryVerticalShardSize(userID string) int + + // QueryPriority returns the query priority config for the tenant, including different priorities and their attributes. + QueryPriority(userID string) validation.QueryPriority } diff --git a/pkg/querier/tripperware/priority.go b/pkg/querier/tripperware/priority.go new file mode 100644 index 0000000000..5ce4e0d3fa --- /dev/null +++ b/pkg/querier/tripperware/priority.go @@ -0,0 +1,94 @@ +package tripperware + +import ( + "net/http" + "strings" + "time" + + "github.com/prometheus/prometheus/promql/parser" + + "github.com/cortexproject/cortex/pkg/util" + "github.com/cortexproject/cortex/pkg/util/validation" +) + +func GetPriority(r *http.Request, userID string, limits Limits, now time.Time) (int64, error) { + isQuery := strings.HasSuffix(r.URL.Path, "/query") + isQueryRange := strings.HasSuffix(r.URL.Path, "/query_range") + queryPriority := limits.QueryPriority(userID) + query := r.FormValue("query") + + if (!isQuery && !isQueryRange) || !queryPriority.Enabled || query == "" { + return 0, nil + } + + expr, err := parser.ParseExpr(query) + if err != nil { + return 0, err + } + + var startTime, endTime int64 + if isQuery { + if t, err := util.ParseTimeParam(r, "time", now.Unix()); err == nil { + startTime = t + endTime = t + } + } else if isQueryRange { + if st, err := util.ParseTime(r.FormValue("start")); err == nil { + if et, err := util.ParseTime(r.FormValue("end")); err == nil { + startTime = st + endTime = et + } + } + } + + es := &parser.EvalStmt{ + Expr: expr, + Start: util.TimeFromMillis(startTime), + End: util.TimeFromMillis(endTime), + LookbackDelta: limits.MaxQueryLookback(userID), // this is available from querier flag. + } + + minTime, maxTime := FindMinMaxTime(es) + + for _, priority := range queryPriority.Priorities { + for _, attribute := range priority.QueryAttributes { + if attribute.Regex == "" || (attribute.CompiledRegex != nil && !attribute.CompiledRegex.MatchString(query)) { + continue + } + + if isWithinTimeAttributes(attribute, now, minTime, maxTime) { + return priority.Priority, nil + } + } + } + + return queryPriority.DefaultPriority, nil +} + +func isWithinTimeAttributes(attribute validation.QueryAttribute, now time.Time, startTime, endTime int64) bool { + if attribute.StartTime == 0 && attribute.EndTime == 0 { + return true + } + + if attribute.StartTime != 0 { + startTimeThreshold := now.Add(-1 * time.Duration(attribute.StartTime).Abs()).Truncate(time.Second).Unix() + if startTime < startTimeThreshold { + return false + } + } + + if attribute.EndTime != 0 { + endTimeThreshold := now.Add(-1 * time.Duration(attribute.EndTime).Abs()).Add(1 * time.Second).Truncate(time.Second).Unix() + if endTime > endTimeThreshold { + return false + } + } + + return true +} + +func FindMinMaxTime(s *parser.EvalStmt) (int64, int64) { + // Placeholder until Prometheus is updated to >=0.48.0 + // which includes https://github.com/prometheus/prometheus/commit/9e3df532d8294d4fe3284bde7bc96db336a33552 + return s.Start.Unix(), s.End.Unix() +} diff --git a/pkg/querier/tripperware/priority_test.go b/pkg/querier/tripperware/priority_test.go new file mode 100644 index 0000000000..2918842346 --- /dev/null +++ b/pkg/querier/tripperware/priority_test.go @@ -0,0 +1,277 @@ +package tripperware + +import ( + "bytes" + "net/http" + "regexp" + "strconv" + "testing" + "time" + + "github.com/prometheus/common/model" + "github.com/stretchr/testify/assert" + + "github.com/cortexproject/cortex/pkg/util/validation" +) + +func Test_GetPriorityShouldReturnDefaultPriorityIfNotEnabledOrEmptyQueryString(t *testing.T) { + now := time.Now() + limits := mockLimits{queryPriority: validation.QueryPriority{ + Priorities: []validation.PriorityDef{ + { + Priority: 1, + QueryAttributes: []validation.QueryAttribute{ + { + Regex: ".*", + CompiledRegex: regexp.MustCompile(".*"), + }, + }, + }, + }, + }} + + type testCase struct { + url string + queryPriorityEnabled bool + } + + tests := map[string]testCase{ + "should miss if query priority not enabled": { + url: "/query?query=up", + }, + "should miss if query string empty": { + url: "/query?query=", + queryPriorityEnabled: true, + }, + "should miss if query string empty - range query": { + url: "/query_range?query=", + queryPriorityEnabled: true, + }, + "should miss if neither instant nor range query": { + url: "/series", + queryPriorityEnabled: true, + }, + } + + for testName, testData := range tests { + t.Run(testName, func(t *testing.T) { + limits.queryPriority.Enabled = testData.queryPriorityEnabled + req, _ := http.NewRequest(http.MethodPost, testData.url, bytes.NewReader([]byte{})) + priority, err := GetPriority(req, "", limits, now) + assert.NoError(t, err) + assert.Equal(t, int64(0), priority) + }) + } +} + +func Test_GetPriorityShouldConsiderRegex(t *testing.T) { + now := time.Now() + limits := mockLimits{queryPriority: validation.QueryPriority{ + Enabled: true, + Priorities: []validation.PriorityDef{ + { + Priority: 1, + QueryAttributes: []validation.QueryAttribute{ + {}, + }, + }, + }, + }} + + type testCase struct { + regex string + query string + expectedPriority int + } + + tests := map[string]testCase{ + "should hit if regex matches": { + regex: "(^sum|c(.+)t)", + query: "sum(up)", + expectedPriority: 1, + }, + "should miss if regex doesn't match": { + regex: "(^sum|c(.+)t)", + query: "min(up)", + expectedPriority: 0, + }, + "should hit if regex matches - .*": { + regex: ".*", + query: "count(sum(up))", + expectedPriority: 1, + }, + "should hit if regex matches - .+": { + regex: ".+", + query: "count(sum(up))", + expectedPriority: 1, + }, + "should miss if regex is an empty string": { + regex: "", + query: "sum(up)", + expectedPriority: 0, + }, + } + + for testName, testData := range tests { + t.Run(testName, func(t *testing.T) { + limits.queryPriority.Priorities[0].QueryAttributes[0].Regex = testData.regex + limits.queryPriority.Priorities[0].QueryAttributes[0].CompiledRegex = regexp.MustCompile(testData.regex) + req, _ := http.NewRequest(http.MethodPost, "/query?query="+testData.query, bytes.NewReader([]byte{})) + priority, err := GetPriority(req, "", limits, now) + assert.NoError(t, err) + assert.Equal(t, int64(testData.expectedPriority), priority) + }) + } +} + +func Test_GetPriorityShouldConsiderStartAndEndTime(t *testing.T) { + now := time.Now() + limits := mockLimits{queryPriority: validation.QueryPriority{ + Enabled: true, + Priorities: []validation.PriorityDef{ + { + Priority: 1, + QueryAttributes: []validation.QueryAttribute{ + { + Regex: ".*", + CompiledRegex: regexp.MustCompile(".*"), + StartTime: model.Duration(45 * time.Minute), + EndTime: model.Duration(15 * time.Minute), + }, + }, + }, + }, + }} + + type testCase struct { + time time.Time + start time.Time + end time.Time + expectedPriority int + } + + tests := map[string]testCase{ + "should hit instant query between start and end time": { + time: now.Add(-30 * time.Minute), + expectedPriority: 1, + }, + "should hit instant query equal to start time": { + time: now.Add(-45 * time.Minute), + expectedPriority: 1, + }, + "should hit instant query equal to end time": { + time: now.Add(-15 * time.Minute), + expectedPriority: 1, + }, + "should miss instant query outside of end time": { + expectedPriority: 0, + }, + "should miss instant query outside of start time": { + time: now.Add(-60 * time.Minute), + expectedPriority: 0, + }, + "should hit range query between start and end time": { + start: now.Add(-40 * time.Minute), + end: now.Add(-20 * time.Minute), + expectedPriority: 1, + }, + "should hit range query equal to start and end time": { + start: now.Add(-45 * time.Minute), + end: now.Add(-15 * time.Minute), + expectedPriority: 1, + }, + "should miss range query outside of start time": { + start: now.Add(-50 * time.Minute), + end: now.Add(-15 * time.Minute), + expectedPriority: 0, + }, + "should miss range query completely outside of start time": { + start: now.Add(-50 * time.Minute), + end: now.Add(-45 * time.Minute), + expectedPriority: 0, + }, + "should miss range query outside of end time": { + start: now.Add(-45 * time.Minute), + end: now.Add(-10 * time.Minute), + expectedPriority: 0, + }, + "should miss range query completely outside of end time": { + start: now.Add(-15 * time.Minute), + end: now.Add(-10 * time.Minute), + expectedPriority: 0, + }, + } + + for testName, testData := range tests { + t.Run(testName, func(t *testing.T) { + var url string + if !testData.time.IsZero() { + url = "/query?query=sum(up)&time=" + strconv.FormatInt(testData.time.Unix(), 10) + } else if !testData.start.IsZero() { + url = "/query_range?query=sum(up)&start=" + strconv.FormatInt(testData.start.Unix(), 10) + url += "&end=" + strconv.FormatInt(testData.end.Unix(), 10) + } else { + url = "/query?query=sum(up)" + } + req, _ := http.NewRequest(http.MethodPost, url, bytes.NewReader([]byte{})) + priority, err := GetPriority(req, "", limits, now) + assert.NoError(t, err) + assert.Equal(t, int64(testData.expectedPriority), priority) + }) + } +} + +func Test_GetPriorityShouldNotConsiderStartAndEndTimeIfEmpty(t *testing.T) { + now := time.Now() + limits := mockLimits{queryPriority: validation.QueryPriority{ + Enabled: true, + Priorities: []validation.PriorityDef{ + { + Priority: 1, + QueryAttributes: []validation.QueryAttribute{ + { + Regex: "^sum\\(up\\)$", + }, + }, + }, + }, + }} + + type testCase struct { + time time.Time + start time.Time + end time.Time + } + + tests := map[string]testCase{ + "should hit instant query with no time": {}, + "should hit instant query with future time": { + time: now.Add(1000000 * time.Hour), + }, + "should hit instant query with very old time": { + time: now.Add(-1000000 * time.Hour), + }, + "should hit range query with very wide time window": { + start: now.Add(-1000000 * time.Hour), + end: now.Add(1000000 * time.Hour), + }, + } + + for testName, testData := range tests { + t.Run(testName, func(t *testing.T) { + var url string + if !testData.time.IsZero() { + url = "/query?query=sum(up)&time=" + strconv.FormatInt(testData.time.Unix(), 10) + } else if !testData.start.IsZero() { + url = "/query_range?query=sum(up)&start=" + strconv.FormatInt(testData.start.Unix(), 10) + url += "&end=" + strconv.FormatInt(testData.end.Unix(), 10) + } else { + url = "/query?query=sum(up)" + } + req, _ := http.NewRequest(http.MethodPost, url, bytes.NewReader([]byte{})) + priority, err := GetPriority(req, "", limits, now) + assert.NoError(t, err) + assert.Equal(t, int64(1), priority) + }) + } +} diff --git a/pkg/querier/tripperware/queryrange/limits_test.go b/pkg/querier/tripperware/queryrange/limits_test.go index 1569ea2e3a..c1e5954421 100644 --- a/pkg/querier/tripperware/queryrange/limits_test.go +++ b/pkg/querier/tripperware/queryrange/limits_test.go @@ -2,6 +2,7 @@ package queryrange import ( "context" + "github.com/cortexproject/cortex/pkg/util/validation" "testing" "time" @@ -219,6 +220,10 @@ func (m mockLimits) QueryVerticalShardSize(userID string) int { return 0 } +func (m mockLimits) QueryPriority(userID string) validation.QueryPriority { + return validation.QueryPriority{} +} + type mockHandler struct { mock.Mock } diff --git a/pkg/querier/tripperware/roundtrip.go b/pkg/querier/tripperware/roundtrip.go index 6aefe4ccec..6c7cd13508 100644 --- a/pkg/querier/tripperware/roundtrip.go +++ b/pkg/querier/tripperware/roundtrip.go @@ -19,6 +19,7 @@ import ( "context" "io" "net/http" + "strconv" "strings" "time" @@ -142,15 +143,27 @@ func NewQueryTripperware( if err != nil { return nil, err } + now := time.Now() userStr := tenant.JoinTenantIDs(tenantIDs) - activeUsers.UpdateUserTimestamp(userStr, time.Now()) + activeUsers.UpdateUserTimestamp(userStr, now) queriesPerTenant.WithLabelValues(op, userStr).Inc() - if maxSubQuerySteps > 0 && (isQuery || isQueryRange) { + if isQuery || isQueryRange { query := r.FormValue("query") - // Check subquery step size. - if err := SubQueryStepSizeCheck(query, defaultSubQueryInterval, maxSubQuerySteps); err != nil { - return nil, err + + if maxSubQuerySteps > 0 { + // Check subquery step size. + if err := SubQueryStepSizeCheck(query, defaultSubQueryInterval, maxSubQuerySteps); err != nil { + return nil, err + } + } + + if limits.QueryPriority(userStr).Enabled { + priority, err := GetPriority(r, userStr, limits, now) + if err != nil { + return nil, err + } + r.Header.Set(util.QueryPriorityHeaderKey, strconv.FormatInt(priority, 10)) } } diff --git a/pkg/querier/tripperware/test_shard_by_query_utils.go b/pkg/querier/tripperware/test_shard_by_query_utils.go index 657d7daa3a..5cbad93ca8 100644 --- a/pkg/querier/tripperware/test_shard_by_query_utils.go +++ b/pkg/querier/tripperware/test_shard_by_query_utils.go @@ -21,6 +21,7 @@ import ( "github.com/weaveworks/common/user" "github.com/cortexproject/cortex/pkg/querysharding" + "github.com/cortexproject/cortex/pkg/util/validation" ) func TestQueryShardQuery(t *testing.T, instantQueryCodec Codec, shardedPrometheusCodec Codec) { @@ -466,6 +467,7 @@ type mockLimits struct { maxQueryLength time.Duration maxCacheFreshness time.Duration shardSize int + queryPriority validation.QueryPriority } func (m mockLimits) MaxQueryLookback(string) time.Duration { @@ -488,6 +490,10 @@ func (m mockLimits) QueryVerticalShardSize(userID string) int { return m.shardSize } +func (m mockLimits) QueryPriority(userID string) validation.QueryPriority { + return m.queryPriority +} + type singleHostRoundTripper struct { host string next http.RoundTripper diff --git a/pkg/scheduler/scheduler.go b/pkg/scheduler/scheduler.go index 65235540a3..fa28485298 100644 --- a/pkg/scheduler/scheduler.go +++ b/pkg/scheduler/scheduler.go @@ -5,6 +5,7 @@ import ( "flag" "io" "net/http" + "strconv" "sync" "time" @@ -154,7 +155,6 @@ type schedulerRequest struct { queryID uint64 request *httpgrpc.HTTPRequest statsEnabled bool - priority int64 enqueueTime time.Time @@ -167,7 +167,12 @@ type schedulerRequest struct { } func (s schedulerRequest) Priority() int64 { - return s.priority + priority, err := strconv.ParseInt(httpgrpcutil.GetHeader(*s.request, util.QueryPriorityHeaderKey), 10, 64) + if err != nil { + return 0 + } + + return priority } // FrontendLoop handles connection from frontend. @@ -298,7 +303,6 @@ func (s *Scheduler) enqueueRequest(frontendContext context.Context, frontendAddr queryID: msg.QueryID, request: msg.HttpRequest, statsEnabled: msg.StatsEnabled, - priority: msg.Priority, } now := time.Now() diff --git a/pkg/scheduler/schedulerpb/scheduler.pb.go b/pkg/scheduler/schedulerpb/scheduler.pb.go index 2f352a8138..d3288f95b3 100644 --- a/pkg/scheduler/schedulerpb/scheduler.pb.go +++ b/pkg/scheduler/schedulerpb/scheduler.pb.go @@ -219,7 +219,6 @@ type FrontendToScheduler struct { UserID string `protobuf:"bytes,4,opt,name=userID,proto3" json:"userID,omitempty"` HttpRequest *httpgrpc.HTTPRequest `protobuf:"bytes,5,opt,name=httpRequest,proto3" json:"httpRequest,omitempty"` StatsEnabled bool `protobuf:"varint,6,opt,name=statsEnabled,proto3" json:"statsEnabled,omitempty"` - Priority int64 `protobuf:"varint,7,opt,name=priority,proto3" json:"priority,omitempty"` } func (m *FrontendToScheduler) Reset() { *m = FrontendToScheduler{} } @@ -296,13 +295,6 @@ func (m *FrontendToScheduler) GetStatsEnabled() bool { return false } -func (m *FrontendToScheduler) GetPriority() int64 { - if m != nil { - return m.Priority - } - return 0 -} - type SchedulerToFrontend struct { Status SchedulerToFrontendStatus `protobuf:"varint,1,opt,name=status,proto3,enum=schedulerpb.SchedulerToFrontendStatus" json:"status,omitempty"` Error string `protobuf:"bytes,2,opt,name=error,proto3" json:"error,omitempty"` @@ -446,49 +438,48 @@ func init() { func init() { proto.RegisterFile("scheduler.proto", fileDescriptor_2b3fc28395a6d9c5) } var fileDescriptor_2b3fc28395a6d9c5 = []byte{ - // 661 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x8c, 0x54, 0xcd, 0x4e, 0xdb, 0x40, - 0x10, 0xf6, 0xe6, 0x0f, 0x98, 0xd0, 0xe2, 0x2e, 0xd0, 0xa6, 0x11, 0x5d, 0x2c, 0xab, 0xaa, 0x52, - 0x0e, 0x49, 0x95, 0x56, 0x6a, 0x0f, 0xa8, 0x52, 0x0a, 0xa6, 0x44, 0xa5, 0x0e, 0x6c, 0x1c, 0xf5, - 0xe7, 0x12, 0x91, 0x64, 0x49, 0x22, 0xc0, 0x6b, 0xd6, 0x76, 0x51, 0x6e, 0x7d, 0x84, 0x3e, 0x44, - 0x0f, 0x7d, 0x94, 0x5e, 0x2a, 0x71, 0xe4, 0xd0, 0x43, 0x31, 0x97, 0x1e, 0x79, 0x84, 0x2a, 0x8e, - 0xe3, 0x3a, 0x90, 0x00, 0xb7, 0x99, 0xf1, 0xf7, 0x79, 0xe7, 0xfb, 0x66, 0x76, 0x61, 0xce, 0x6e, - 0x76, 0x58, 0xcb, 0x3d, 0x60, 0x22, 0x6f, 0x09, 0xee, 0x70, 0x9c, 0x0e, 0x0b, 0x56, 0x23, 0xbb, - 0xd0, 0xe6, 0x6d, 0xee, 0xd7, 0x0b, 0xfd, 0x68, 0x00, 0xc9, 0xbe, 0x68, 0x77, 0x9d, 0x8e, 0xdb, - 0xc8, 0x37, 0xf9, 0x61, 0xe1, 0x98, 0xed, 0x7e, 0x61, 0xc7, 0x5c, 0xec, 0xdb, 0x85, 0x26, 0x3f, - 0x3c, 0xe4, 0x66, 0xa1, 0xe3, 0x38, 0x56, 0x5b, 0x58, 0xcd, 0x30, 0x18, 0xb0, 0xd4, 0x22, 0xe0, - 0x1d, 0x97, 0x89, 0x2e, 0x13, 0x06, 0xaf, 0x0e, 0xcf, 0xc0, 0x4b, 0x30, 0x73, 0x34, 0xa8, 0x96, - 0xd7, 0x33, 0x48, 0x41, 0xb9, 0x19, 0xfa, 0xbf, 0xa0, 0xfe, 0x42, 0x80, 0x43, 0xac, 0xc1, 0x03, - 0x3e, 0xce, 0xc0, 0x54, 0x1f, 0xd3, 0x0b, 0x28, 0x09, 0x3a, 0x4c, 0xf1, 0x4b, 0x48, 0xf7, 0x8f, - 0xa5, 0xec, 0xc8, 0x65, 0xb6, 0x93, 0x89, 0x29, 0x28, 0x97, 0x2e, 0x2e, 0xe6, 0xc3, 0x56, 0x36, - 0x0d, 0x63, 0x3b, 0xf8, 0x48, 0xa3, 0x48, 0x9c, 0x83, 0xb9, 0x3d, 0xc1, 0x4d, 0x87, 0x99, 0xad, - 0x52, 0xab, 0x25, 0x98, 0x6d, 0x67, 0xe2, 0x7e, 0x37, 0x97, 0xcb, 0xf8, 0x3e, 0xa4, 0x5c, 0xdb, - 0x6f, 0x37, 0xe1, 0x03, 0x82, 0x0c, 0xab, 0x30, 0x6b, 0x3b, 0xbb, 0x8e, 0xad, 0x99, 0xbb, 0x8d, - 0x03, 0xd6, 0xca, 0x24, 0x15, 0x94, 0x9b, 0xa6, 0x23, 0x35, 0xf5, 0x7b, 0x0c, 0xe6, 0x37, 0x82, - 0xff, 0x45, 0x5d, 0x78, 0x05, 0x09, 0xa7, 0x67, 0x31, 0x5f, 0xcd, 0xdd, 0xe2, 0xe3, 0x7c, 0x64, - 0x06, 0xf9, 0x31, 0x78, 0xa3, 0x67, 0x31, 0xea, 0x33, 0xc6, 0xf5, 0x1d, 0x1b, 0xdf, 0x77, 0xc4, - 0xb4, 0xf8, 0xa8, 0x69, 0x93, 0x14, 0x5d, 0x32, 0x33, 0x79, 0x6b, 0x33, 0x2f, 0x5b, 0x91, 0xba, - 0x6a, 0x05, 0xce, 0xc2, 0xb4, 0x25, 0xba, 0x5c, 0x74, 0x9d, 0x5e, 0x66, 0x4a, 0x41, 0xb9, 0x38, - 0x0d, 0x73, 0x75, 0x1f, 0xe6, 0x23, 0x53, 0x1f, 0x1a, 0x80, 0x5f, 0x43, 0xaa, 0xff, 0x0b, 0xd7, - 0x0e, 0x7c, 0x7a, 0x32, 0xe2, 0xd3, 0x18, 0x46, 0xd5, 0x47, 0xd3, 0x80, 0x85, 0x17, 0x20, 0xc9, - 0x84, 0xe0, 0x22, 0x70, 0x68, 0x90, 0xa8, 0xab, 0xb0, 0xa4, 0x73, 0xa7, 0xbb, 0xd7, 0x0b, 0xb6, - 0xab, 0xda, 0x71, 0x9d, 0x16, 0x3f, 0x36, 0x87, 0x62, 0xae, 0xdf, 0xd0, 0x65, 0x78, 0x34, 0x81, - 0x6d, 0x5b, 0xdc, 0xb4, 0xd9, 0xca, 0x2a, 0x3c, 0x98, 0x30, 0x41, 0x3c, 0x0d, 0x89, 0xb2, 0x5e, - 0x36, 0x64, 0x09, 0xa7, 0x61, 0x4a, 0xd3, 0x77, 0x6a, 0x5a, 0x4d, 0x93, 0x11, 0x06, 0x48, 0xad, - 0x95, 0xf4, 0x35, 0x6d, 0x4b, 0x8e, 0xad, 0x34, 0xe1, 0xe1, 0x44, 0x5d, 0x38, 0x05, 0xb1, 0xca, - 0x3b, 0x59, 0xc2, 0x0a, 0x2c, 0x19, 0x95, 0x4a, 0xfd, 0x7d, 0x49, 0xff, 0x54, 0xa7, 0xda, 0x4e, - 0x4d, 0xab, 0x1a, 0xd5, 0xfa, 0xb6, 0x46, 0xeb, 0x86, 0xa6, 0x97, 0x74, 0x43, 0x46, 0x78, 0x06, - 0x92, 0x1a, 0xa5, 0x15, 0x2a, 0xc7, 0xf0, 0x3d, 0xb8, 0x53, 0xdd, 0xac, 0x19, 0x46, 0x59, 0x7f, - 0x5b, 0x5f, 0xaf, 0x7c, 0xd0, 0xe5, 0x78, 0xf1, 0x37, 0x8a, 0xf8, 0xbd, 0xc1, 0xc5, 0xf0, 0x9a, - 0xd5, 0x20, 0x1d, 0x84, 0x5b, 0x9c, 0x5b, 0x78, 0x79, 0xc4, 0xee, 0xab, 0x77, 0x39, 0xbb, 0x3c, - 0x69, 0x1e, 0x01, 0x56, 0x95, 0x72, 0xe8, 0x19, 0xc2, 0x26, 0x2c, 0x8e, 0xb5, 0x0c, 0x3f, 0x1d, - 0xe1, 0x5f, 0x37, 0x94, 0xec, 0xca, 0x6d, 0xa0, 0x83, 0x09, 0x14, 0x2d, 0x58, 0x88, 0xaa, 0x0b, - 0xd7, 0xe9, 0x23, 0xcc, 0x0e, 0x63, 0x5f, 0x9f, 0x72, 0xd3, 0xb5, 0xcb, 0x2a, 0x37, 0x2d, 0xdc, - 0x40, 0xe1, 0x9b, 0xd2, 0xc9, 0x19, 0x91, 0x4e, 0xcf, 0x88, 0x74, 0x71, 0x46, 0xd0, 0x57, 0x8f, - 0xa0, 0x1f, 0x1e, 0x41, 0x3f, 0x3d, 0x82, 0x4e, 0x3c, 0x82, 0xfe, 0x78, 0x04, 0xfd, 0xf5, 0x88, - 0x74, 0xe1, 0x11, 0xf4, 0xed, 0x9c, 0x48, 0x27, 0xe7, 0x44, 0x3a, 0x3d, 0x27, 0xd2, 0xe7, 0xe8, - 0xcb, 0xdb, 0x48, 0xf9, 0x8f, 0xe6, 0xf3, 0x7f, 0x01, 0x00, 0x00, 0xff, 0xff, 0x83, 0xb8, 0xa1, - 0x26, 0xa0, 0x05, 0x00, 0x00, + // 644 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x8c, 0x54, 0x4f, 0x4f, 0xdb, 0x4e, + 0x10, 0xf5, 0x86, 0x24, 0xc0, 0x84, 0xdf, 0x0f, 0x77, 0x81, 0x36, 0x8d, 0xe8, 0x12, 0x45, 0x55, + 0x95, 0x72, 0x48, 0xaa, 0xb4, 0x52, 0x7b, 0x40, 0x95, 0x52, 0x30, 0x25, 0x2a, 0x75, 0x60, 0xb3, + 0x51, 0xff, 0x5c, 0x22, 0x92, 0x2c, 0x09, 0x02, 0xbc, 0x66, 0x6d, 0x17, 0xe5, 0xd6, 0x63, 0x8f, + 0xfd, 0x18, 0xfd, 0x28, 0xbd, 0x54, 0xe2, 0xc8, 0xa1, 0x87, 0x62, 0x2e, 0x3d, 0xf2, 0x11, 0xaa, + 0x38, 0x76, 0xea, 0xa4, 0x0e, 0x70, 0x9b, 0x1d, 0xbf, 0xe7, 0x9d, 0xf7, 0x66, 0x66, 0x61, 0xde, + 0x6a, 0x75, 0x79, 0xdb, 0x39, 0xe2, 0xb2, 0x60, 0x4a, 0x61, 0x0b, 0x9c, 0x1a, 0x26, 0xcc, 0x66, + 0x66, 0xb1, 0x23, 0x3a, 0xc2, 0xcb, 0x17, 0xfb, 0xd1, 0x00, 0x92, 0x79, 0xd6, 0x39, 0xb0, 0xbb, + 0x4e, 0xb3, 0xd0, 0x12, 0xc7, 0xc5, 0x53, 0xbe, 0xf7, 0x89, 0x9f, 0x0a, 0x79, 0x68, 0x15, 0x5b, + 0xe2, 0xf8, 0x58, 0x18, 0xc5, 0xae, 0x6d, 0x9b, 0x1d, 0x69, 0xb6, 0x86, 0xc1, 0x80, 0x95, 0x2b, + 0x01, 0xde, 0x75, 0xb8, 0x3c, 0xe0, 0x92, 0x89, 0x5a, 0x70, 0x07, 0x5e, 0x86, 0xd9, 0x93, 0x41, + 0xb6, 0xb2, 0x91, 0x46, 0x59, 0x94, 0x9f, 0xa5, 0x7f, 0x13, 0xb9, 0x1f, 0x08, 0xf0, 0x10, 0xcb, + 0x84, 0xcf, 0xc7, 0x69, 0x98, 0xee, 0x63, 0x7a, 0x3e, 0x25, 0x4e, 0x83, 0x23, 0x7e, 0x0e, 0xa9, + 0xfe, 0xb5, 0x94, 0x9f, 0x38, 0xdc, 0xb2, 0xd3, 0xb1, 0x2c, 0xca, 0xa7, 0x4a, 0x4b, 0x85, 0x61, + 0x29, 0x5b, 0x8c, 0xed, 0xf8, 0x1f, 0x69, 0x18, 0x89, 0xf3, 0x30, 0xbf, 0x2f, 0x85, 0x61, 0x73, + 0xa3, 0x5d, 0x6e, 0xb7, 0x25, 0xb7, 0xac, 0xf4, 0x94, 0x57, 0xcd, 0x78, 0x1a, 0xdf, 0x85, 0xa4, + 0x63, 0x79, 0xe5, 0xc6, 0x3d, 0x80, 0x7f, 0xc2, 0x39, 0x98, 0xb3, 0xec, 0x3d, 0xdb, 0xd2, 0x8c, + 0xbd, 0xe6, 0x11, 0x6f, 0xa7, 0x13, 0x59, 0x94, 0x9f, 0xa1, 0x23, 0xb9, 0xdc, 0x97, 0x18, 0x2c, + 0x6c, 0xfa, 0xff, 0x0b, 0xbb, 0xf0, 0x02, 0xe2, 0x76, 0xcf, 0xe4, 0x9e, 0x9a, 0xff, 0x4b, 0x0f, + 0x0b, 0xa1, 0x1e, 0x14, 0x22, 0xf0, 0xac, 0x67, 0x72, 0xea, 0x31, 0xa2, 0xea, 0x8e, 0x45, 0xd7, + 0x1d, 0x32, 0x6d, 0x6a, 0xd4, 0xb4, 0x49, 0x8a, 0xc6, 0xcc, 0x4c, 0xdc, 0xda, 0xcc, 0x71, 0x2b, + 0x92, 0x11, 0x56, 0x1c, 0xc2, 0x42, 0xa8, 0xb3, 0x81, 0x48, 0xfc, 0x12, 0x92, 0x7d, 0x98, 0x63, + 0xf9, 0x5e, 0x3c, 0x1a, 0xf1, 0x22, 0x82, 0x51, 0xf3, 0xd0, 0xd4, 0x67, 0xe1, 0x45, 0x48, 0x70, + 0x29, 0x85, 0xf4, 0x5d, 0x18, 0x1c, 0x72, 0x6b, 0xb0, 0xac, 0x0b, 0xfb, 0x60, 0xbf, 0xe7, 0x4f, + 0x50, 0xad, 0xeb, 0xd8, 0x6d, 0x71, 0x6a, 0x04, 0x05, 0x5f, 0x3f, 0x85, 0x2b, 0xf0, 0x60, 0x02, + 0xdb, 0x32, 0x85, 0x61, 0xf1, 0xd5, 0x35, 0xb8, 0x37, 0xa1, 0x4b, 0x78, 0x06, 0xe2, 0x15, 0xbd, + 0xc2, 0x54, 0x05, 0xa7, 0x60, 0x5a, 0xd3, 0x77, 0xeb, 0x5a, 0x5d, 0x53, 0x11, 0x06, 0x48, 0xae, + 0x97, 0xf5, 0x75, 0x6d, 0x5b, 0x8d, 0xad, 0xb6, 0xe0, 0xfe, 0x44, 0x5d, 0x38, 0x09, 0xb1, 0xea, + 0x1b, 0x55, 0xc1, 0x59, 0x58, 0x66, 0xd5, 0x6a, 0xe3, 0x6d, 0x59, 0xff, 0xd0, 0xa0, 0xda, 0x6e, + 0x5d, 0xab, 0xb1, 0x5a, 0x63, 0x47, 0xa3, 0x0d, 0xa6, 0xe9, 0x65, 0x9d, 0xa9, 0x08, 0xcf, 0x42, + 0x42, 0xa3, 0xb4, 0x4a, 0xd5, 0x18, 0xbe, 0x03, 0xff, 0xd5, 0xb6, 0xea, 0x8c, 0x55, 0xf4, 0xd7, + 0x8d, 0x8d, 0xea, 0x3b, 0x5d, 0x9d, 0x2a, 0xfd, 0x44, 0x21, 0xbf, 0x37, 0x85, 0x0c, 0x56, 0xa9, + 0x0e, 0x29, 0x3f, 0xdc, 0x16, 0xc2, 0xc4, 0x2b, 0x23, 0x76, 0xff, 0xbb, 0xaf, 0x99, 0x95, 0x49, + 0xfd, 0xf0, 0xb1, 0x39, 0x25, 0x8f, 0x9e, 0x20, 0x6c, 0xc0, 0x52, 0xa4, 0x65, 0xf8, 0xf1, 0x08, + 0xff, 0xba, 0xa6, 0x64, 0x56, 0x6f, 0x03, 0x1d, 0x74, 0xa0, 0x64, 0xc2, 0x62, 0x58, 0xdd, 0x70, + 0x9c, 0xde, 0xc3, 0x5c, 0x10, 0x7b, 0xfa, 0xb2, 0x37, 0xad, 0x56, 0x26, 0x7b, 0xd3, 0xc0, 0x0d, + 0x14, 0xbe, 0x2a, 0x9f, 0x5d, 0x10, 0xe5, 0xfc, 0x82, 0x28, 0x57, 0x17, 0x04, 0x7d, 0x76, 0x09, + 0xfa, 0xe6, 0x12, 0xf4, 0xdd, 0x25, 0xe8, 0xcc, 0x25, 0xe8, 0x97, 0x4b, 0xd0, 0x6f, 0x97, 0x28, + 0x57, 0x2e, 0x41, 0x5f, 0x2f, 0x89, 0x72, 0x76, 0x49, 0x94, 0xf3, 0x4b, 0xa2, 0x7c, 0x0c, 0xbf, + 0xae, 0xcd, 0xa4, 0xf7, 0x30, 0x3e, 0xfd, 0x13, 0x00, 0x00, 0xff, 0xff, 0x88, 0x0c, 0xfe, 0x56, + 0x84, 0x05, 0x00, 0x00, } func (x FrontendToSchedulerType) String() string { @@ -602,9 +593,6 @@ func (this *FrontendToScheduler) Equal(that interface{}) bool { if this.StatsEnabled != that1.StatsEnabled { return false } - if this.Priority != that1.Priority { - return false - } return true } func (this *SchedulerToFrontend) Equal(that interface{}) bool { @@ -709,7 +697,7 @@ func (this *FrontendToScheduler) GoString() string { if this == nil { return "nil" } - s := make([]string, 0, 11) + s := make([]string, 0, 10) s = append(s, "&schedulerpb.FrontendToScheduler{") s = append(s, "Type: "+fmt.Sprintf("%#v", this.Type)+",\n") s = append(s, "FrontendAddress: "+fmt.Sprintf("%#v", this.FrontendAddress)+",\n") @@ -719,7 +707,6 @@ func (this *FrontendToScheduler) GoString() string { s = append(s, "HttpRequest: "+fmt.Sprintf("%#v", this.HttpRequest)+",\n") } s = append(s, "StatsEnabled: "+fmt.Sprintf("%#v", this.StatsEnabled)+",\n") - s = append(s, "Priority: "+fmt.Sprintf("%#v", this.Priority)+",\n") s = append(s, "}") return strings.Join(s, "") } @@ -1155,11 +1142,6 @@ func (m *FrontendToScheduler) MarshalToSizedBuffer(dAtA []byte) (int, error) { _ = i var l int _ = l - if m.Priority != 0 { - i = encodeVarintScheduler(dAtA, i, uint64(m.Priority)) - i-- - dAtA[i] = 0x38 - } if m.StatsEnabled { i-- if m.StatsEnabled { @@ -1375,9 +1357,6 @@ func (m *FrontendToScheduler) Size() (n int) { if m.StatsEnabled { n += 2 } - if m.Priority != 0 { - n += 1 + sovScheduler(uint64(m.Priority)) - } return n } @@ -1460,7 +1439,6 @@ func (this *FrontendToScheduler) String() string { `UserID:` + fmt.Sprintf("%v", this.UserID) + `,`, `HttpRequest:` + strings.Replace(fmt.Sprintf("%v", this.HttpRequest), "HTTPRequest", "httpgrpc.HTTPRequest", 1) + `,`, `StatsEnabled:` + fmt.Sprintf("%v", this.StatsEnabled) + `,`, - `Priority:` + fmt.Sprintf("%v", this.Priority) + `,`, `}`, }, "") return s @@ -1967,25 +1945,6 @@ func (m *FrontendToScheduler) Unmarshal(dAtA []byte) error { } } m.StatsEnabled = bool(v != 0) - case 7: - if wireType != 0 { - return fmt.Errorf("proto: wrong wireType = %d for field Priority", wireType) - } - m.Priority = 0 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowScheduler - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - m.Priority |= int64(b&0x7F) << shift - if b < 0x80 { - break - } - } default: iNdEx = preIndex skippy, err := skipScheduler(dAtA[iNdEx:]) diff --git a/pkg/scheduler/schedulerpb/scheduler.proto b/pkg/scheduler/schedulerpb/scheduler.proto index 706c34de4f..eea28717b8 100644 --- a/pkg/scheduler/schedulerpb/scheduler.proto +++ b/pkg/scheduler/schedulerpb/scheduler.proto @@ -78,7 +78,6 @@ message FrontendToScheduler { string userID = 4; httpgrpc.HTTPRequest httpRequest = 5; bool statsEnabled = 6; - int64 priority = 7; } enum SchedulerToFrontendStatus { diff --git a/pkg/util/http.go b/pkg/util/http.go index 09fb3df38c..41daae0fc6 100644 --- a/pkg/util/http.go +++ b/pkg/util/http.go @@ -21,6 +21,7 @@ import ( yamlv3 "gopkg.in/yaml.v3" ) +const QueryPriorityHeaderKey = "X-Cortex-Query-Priority" const messageSizeLargerErrFmt = "received message larger than max (%d vs %d)" // IsRequestBodyTooLarge returns true if the error is "http: request body too large". diff --git a/pkg/util/httpgrpcutil/header.go b/pkg/util/httpgrpcutil/header.go new file mode 100644 index 0000000000..b844e0c65f --- /dev/null +++ b/pkg/util/httpgrpcutil/header.go @@ -0,0 +1,22 @@ +package httpgrpcutil + +import ( + "github.com/weaveworks/common/httpgrpc" +) + +// GetHeader is similar to http.Header.Get, which gets the first value associated with the given key. +// If there are no values associated with the key, it returns "". +func GetHeader(r httpgrpc.HTTPRequest, key string) string { + return GetHeaderValues(r, key)[0] +} + +// GetHeaderValues is similar to http.Header.Values, which returns all values associated with the given key. +func GetHeaderValues(r httpgrpc.HTTPRequest, key string) []string { + for _, header := range r.Headers { + if header.GetKey() == key { + return header.GetValues() + } + } + + return []string{} +} diff --git a/pkg/util/query/priority.go b/pkg/util/query/priority.go deleted file mode 100644 index 2a7f61ba10..0000000000 --- a/pkg/util/query/priority.go +++ /dev/null @@ -1,74 +0,0 @@ -package query - -import ( - "net/url" - "time" - - "github.com/cortexproject/cortex/pkg/util" - "github.com/cortexproject/cortex/pkg/util/validation" -) - -func GetPriority(requestParams url.Values, now time.Time, queryPriority validation.QueryPriority) int64 { - queryParam := requestParams.Get("query") - timeParam := requestParams.Get("time") - startParam := requestParams.Get("start") - endParam := requestParams.Get("end") - - if queryParam == "" || !queryPriority.Enabled { - return queryPriority.DefaultPriority - } - - for _, priority := range queryPriority.Priorities { - for _, attribute := range priority.QueryAttributes { - if attribute.CompiledRegex != nil && !attribute.CompiledRegex.MatchString(queryParam) { - continue - } - - if startTime, err := util.ParseTime(startParam); err == nil { - if endTime, err := util.ParseTime(endParam); err == nil { - if isWithinTimeAttributes(attribute, now, startTime, endTime) { - return priority.Priority - } - } - } - - if instantTime, err := util.ParseTime(timeParam); err == nil { - if isWithinTimeAttributes(attribute, now, instantTime, instantTime) { - return priority.Priority - } - } - - if timeParam == "" { - if isWithinTimeAttributes(attribute, now, util.TimeToMillis(now), util.TimeToMillis(now)) { - return priority.Priority - } - } - } - } - - return queryPriority.DefaultPriority -} - -func isWithinTimeAttributes(attribute validation.QueryAttribute, now time.Time, startTime, endTime int64) bool { - if attribute.StartTime == 0 && attribute.EndTime == 0 { - return true - } - - if attribute.StartTime != 0 { - startTimeThreshold := now.Add(-1 * time.Duration(attribute.StartTime).Abs()).Truncate(time.Second) - - if util.TimeFromMillis(startTime).Before(startTimeThreshold) { - return false - } - } - - if attribute.EndTime != 0 { - endTimeThreshold := now.Add(-1 * time.Duration(attribute.EndTime).Abs()).Add(1 * time.Second).Truncate(time.Second) - - if util.TimeFromMillis(endTime).After(endTimeThreshold) { - return false - } - } - - return true -} diff --git a/pkg/util/query/priority_test.go b/pkg/util/query/priority_test.go deleted file mode 100644 index fea552cde4..0000000000 --- a/pkg/util/query/priority_test.go +++ /dev/null @@ -1,224 +0,0 @@ -package query - -import ( - "net/url" - "regexp" - "strconv" - "testing" - "time" - - "github.com/prometheus/common/model" - "github.com/stretchr/testify/assert" - - "github.com/cortexproject/cortex/pkg/util/validation" -) - -func Test_GetPriorityShouldReturnDefaultPriorityIfNotEnabledOrEmptyQueryString(t *testing.T) { - now := time.Now() - priorities := []validation.PriorityDef{ - { - Priority: 1, - QueryAttributes: []validation.QueryAttribute{ - { - Regex: ".*", - CompiledRegex: regexp.MustCompile(".*"), - }, - }, - }, - } - queryPriority := validation.QueryPriority{ - Priorities: priorities, - } - - assert.Equal(t, int64(0), GetPriority(url.Values{ - "query": []string{"sum(up)"}, - "time": []string{strconv.FormatInt(now.Unix(), 10)}, - }, now, queryPriority)) - - queryPriority.Enabled = true - assert.Equal(t, int64(0), GetPriority(url.Values{ - "query": []string{""}, - "time": []string{strconv.FormatInt(now.Unix(), 10)}, - }, now, queryPriority)) -} - -func Test_GetPriorityShouldConsiderRegex(t *testing.T) { - now := time.Now() - priorities := []validation.PriorityDef{ - { - Priority: 1, - QueryAttributes: []validation.QueryAttribute{ - { - Regex: "sum", - CompiledRegex: regexp.MustCompile("sum"), - }, - }, - }, - } - queryPriority := validation.QueryPriority{ - Enabled: true, - Priorities: priorities, - } - - assert.Equal(t, int64(1), GetPriority(url.Values{ - "query": []string{"sum(up)"}, - "time": []string{strconv.FormatInt(now.Unix(), 10)}, - }, now, queryPriority)) - assert.Equal(t, int64(0), GetPriority(url.Values{ - "query": []string{"count(up)"}, - "time": []string{strconv.FormatInt(now.Unix(), 10)}, - }, now, queryPriority)) - - queryPriority.Priorities[0].QueryAttributes[0].Regex = "(^sum$|c(.+)t)" - queryPriority.Priorities[0].QueryAttributes[0].CompiledRegex = regexp.MustCompile("(^sum$|c(.+)t)") - - assert.Equal(t, int64(0), GetPriority(url.Values{ - "query": []string{"sum(up)"}, - "time": []string{strconv.FormatInt(now.Unix(), 10)}, - }, now, queryPriority)) - assert.Equal(t, int64(1), GetPriority(url.Values{ - "query": []string{"count(up)"}, - "time": []string{strconv.FormatInt(now.Unix(), 10)}, - }, now, queryPriority)) - - queryPriority.Priorities[0].QueryAttributes[0].Regex = ".*" - queryPriority.Priorities[0].QueryAttributes[0].CompiledRegex = regexp.MustCompile(".*") - - assert.Equal(t, int64(1), GetPriority(url.Values{ - "query": []string{"sum(up)"}, - "time": []string{strconv.FormatInt(now.Unix(), 10)}, - }, now, queryPriority)) - assert.Equal(t, int64(1), GetPriority(url.Values{ - "query": []string{"count(up)"}, - "time": []string{strconv.FormatInt(now.Unix(), 10)}, - }, now, queryPriority)) - - queryPriority.Priorities[0].QueryAttributes[0].Regex = ".+" - queryPriority.Priorities[0].QueryAttributes[0].CompiledRegex = regexp.MustCompile(".+") - - assert.Equal(t, int64(1), GetPriority(url.Values{ - "query": []string{"sum(up)"}, - "time": []string{strconv.FormatInt(now.Unix(), 10)}, - }, now, queryPriority)) - assert.Equal(t, int64(1), GetPriority(url.Values{ - "query": []string{"count(up)"}, - "time": []string{strconv.FormatInt(now.Unix(), 10)}, - }, now, queryPriority)) - - queryPriority.Priorities[0].QueryAttributes[0].Regex = "" - queryPriority.Priorities[0].QueryAttributes[0].CompiledRegex = regexp.MustCompile("") - - assert.Equal(t, int64(1), GetPriority(url.Values{ - "query": []string{"sum(up)"}, - "time": []string{strconv.FormatInt(now.Unix(), 10)}, - }, now, queryPriority)) - assert.Equal(t, int64(1), GetPriority(url.Values{ - "query": []string{"count(up)"}, - "time": []string{strconv.FormatInt(now.Unix(), 10)}, - }, now, queryPriority)) -} - -func Test_GetPriorityShouldConsiderStartAndEndTime(t *testing.T) { - now := time.Now() - priorities := []validation.PriorityDef{ - { - Priority: 1, - QueryAttributes: []validation.QueryAttribute{ - { - StartTime: model.Duration(45 * time.Minute), - EndTime: model.Duration(15 * time.Minute), - }, - }, - }, - } - queryPriority := validation.QueryPriority{ - Enabled: true, - Priorities: priorities, - } - - assert.Equal(t, int64(0), GetPriority(url.Values{ - "query": []string{"sum(up)"}, - "time": []string{strconv.FormatInt(now.Unix(), 10)}, - }, now, queryPriority)) - assert.Equal(t, int64(1), GetPriority(url.Values{ - "query": []string{"sum(up)"}, - "time": []string{strconv.FormatInt(now.Add(-30*time.Minute).Unix(), 10)}, - }, now, queryPriority)) - assert.Equal(t, int64(0), GetPriority(url.Values{ - "query": []string{"sum(up)"}, - "time": []string{strconv.FormatInt(now.Add(-60*time.Minute).Unix(), 10)}, - }, now, queryPriority)) - assert.Equal(t, int64(1), GetPriority(url.Values{ - "query": []string{"sum(up)"}, - "time": []string{strconv.FormatInt(now.Add(-45*time.Minute).Unix(), 10)}, - }, now, queryPriority)) - assert.Equal(t, int64(1), GetPriority(url.Values{ - "query": []string{"sum(up)"}, - "time": []string{strconv.FormatInt(now.Add(-15*time.Minute).Unix(), 10)}, - }, now, queryPriority)) - - assert.Equal(t, int64(1), GetPriority(url.Values{ - "query": []string{"sum(up)"}, - "start": []string{strconv.FormatInt(now.Add(-45*time.Minute).Unix(), 10)}, - "end": []string{strconv.FormatInt(now.Add(-15*time.Minute).Unix(), 10)}, - }, now, queryPriority)) - assert.Equal(t, int64(0), GetPriority(url.Values{ - "query": []string{"sum(up)"}, - "start": []string{strconv.FormatInt(now.Add(-50*time.Minute).Unix(), 10)}, - "end": []string{strconv.FormatInt(now.Add(-15*time.Minute).Unix(), 10)}, - }, now, queryPriority)) - assert.Equal(t, int64(0), GetPriority(url.Values{ - "query": []string{"sum(up)"}, - "start": []string{strconv.FormatInt(now.Add(-45*time.Minute).Unix(), 10)}, - "end": []string{strconv.FormatInt(now.Add(-10*time.Minute).Unix(), 10)}, - }, now, queryPriority)) - assert.Equal(t, int64(0), GetPriority(url.Values{ - "query": []string{"sum(up)"}, - "start": []string{strconv.FormatInt(now.Add(-60*time.Minute).Unix(), 10)}, - "end": []string{strconv.FormatInt(now.Add(-45*time.Minute).Unix(), 10)}, - }, now, queryPriority)) - assert.Equal(t, int64(0), GetPriority(url.Values{ - "query": []string{"sum(up)"}, - "start": []string{strconv.FormatInt(now.Add(-15*time.Minute).Unix(), 10)}, - "end": []string{strconv.FormatInt(now.Add(-1*time.Minute).Unix(), 10)}, - }, now, queryPriority)) -} - -func Test_GetPriorityShouldSKipStartAndEndTimeIfEmpty(t *testing.T) { - now := time.Now() - priorities := []validation.PriorityDef{ - { - Priority: 1, - QueryAttributes: []validation.QueryAttribute{ - { - Regex: "^test$", - }, - }, - }, - } - queryPriority := validation.QueryPriority{ - Enabled: true, - Priorities: priorities, - } - - assert.Equal(t, int64(1), GetPriority(url.Values{ - "query": []string{"test"}, - }, now, queryPriority)) - assert.Equal(t, int64(1), GetPriority(url.Values{ - "query": []string{"test"}, - "time": []string{strconv.FormatInt(now.Add(8760*time.Hour).Unix(), 10)}, - }, now, queryPriority)) - assert.Equal(t, int64(1), GetPriority(url.Values{ - "query": []string{"test"}, - "time": []string{strconv.FormatInt(now.Unix(), 10)}, - }, now, queryPriority)) - assert.Equal(t, int64(1), GetPriority(url.Values{ - "query": []string{"test"}, - "time": []string{strconv.FormatInt(now.Add(-8760*time.Hour).Unix(), 10)}, - }, now, queryPriority)) - assert.Equal(t, int64(1), GetPriority(url.Values{ - "query": []string{"test"}, - "start": []string{strconv.FormatInt(now.Add(-100000*time.Minute).Unix(), 10)}, - "end": []string{strconv.FormatInt(now.Add(100000*time.Minute).Unix(), 10)}, - }, now, queryPriority)) -} diff --git a/pkg/util/time.go b/pkg/util/time.go index 8816b1d7d2..a28c84b046 100644 --- a/pkg/util/time.go +++ b/pkg/util/time.go @@ -7,6 +7,7 @@ import ( "strconv" "time" + "github.com/pkg/errors" "github.com/prometheus/common/model" "github.com/weaveworks/common/httpgrpc" ) @@ -48,6 +49,19 @@ func ParseTime(s string) (int64, error) { return 0, httpgrpc.Errorf(http.StatusBadRequest, "cannot parse %q to a valid timestamp", s) } +// ParseTimeParam parses the time request parameter into an int64, milliseconds since epoch. +func ParseTimeParam(r *http.Request, paramName string, defaultValue int64) (int64, error) { + val := r.FormValue(paramName) + if val == "" { + val = strconv.FormatInt(defaultValue, 10) + } + result, err := ParseTime(val) + if err != nil { + return 0, errors.Wrapf(err, "Invalid time value for '%s'", paramName) + } + return result, nil +} + // DurationWithJitter returns random duration from "input - input*variance" to "input + input*variance" interval. func DurationWithJitter(input time.Duration, variancePerc float64) time.Duration { // No duration? No jitter. diff --git a/pkg/util/validation/limits.go b/pkg/util/validation/limits.go index b9b005d0df..cd65e875da 100644 --- a/pkg/util/validation/limits.go +++ b/pkg/util/validation/limits.go @@ -57,13 +57,13 @@ type QueryPriority struct { type PriorityDef struct { Priority int64 `yaml:"priority" json:"priority" doc:"nocli|description=Priority level. Must be a unique value.|default=0"` - ReservedQueriers float64 `yaml:"reserved_queriers" json:"reserved_queriers" doc:"nocli|description=Number of reserved queriers to handle priorities higher or equal to this value only. Value between 0 and 1 will be used as a percentage.|default=0"` + ReservedQueriers float64 `yaml:"reserved_queriers" json:"reserved_queriers" doc:"nocli|description=Number of reserved queriers to handle priorities higher or equal to the priority level. Value between 0 and 1 will be used as a percentage.|default=0"` QueryAttributes []QueryAttribute `yaml:"query_attributes" json:"query_attributes" doc:"nocli|description=List of query attributes to assign the priority."` } type QueryAttribute struct { - Regex string `yaml:"regex" json:"regex" doc:"nocli|description=Query string regex. If set to empty string, it will not match anything.|default=\"\""` - StartTime model.Duration `yaml:"start_time" json:"start_time" doc:"nocli|description=Query start time. If set to 0, the start time won't be checked.'.|default=0"` + Regex string `yaml:"regex" json:"regex" doc:"nocli|description=Query string regex. If set to empty string, it will not match anything."` + StartTime model.Duration `yaml:"start_time" json:"start_time" doc:"nocli|description=Query start time. If set to 0, the start time won't be checked.|default=0"` EndTime model.Duration `yaml:"end_time" json:"end_time" doc:"nocli|description=Query end time. If set to 0, the end time won't be checked.|default=0"` CompiledRegex *regexp.Regexp } diff --git a/tools/doc-generator/parser.go b/tools/doc-generator/parser.go index 7a6dbfebfb..28ee31ba4f 100644 --- a/tools/doc-generator/parser.go +++ b/tools/doc-generator/parser.go @@ -409,6 +409,10 @@ func getCustomFieldEntry(parent reflect.Type, field reflect.StructField, fieldVa return nil, err } + if fieldFlag == nil { + return nil, nil + } + return &configEntry{ kind: "field", name: getFieldName(field), From 6c1813b75ecf8d7d6e65cd556550c9bdc709ffe5 Mon Sep 17 00:00:00 2001 From: Justin Jung Date: Tue, 28 Nov 2023 10:13:51 -0800 Subject: [PATCH 44/49] Attempt to fix tests Signed-off-by: Justin Jung --- pkg/frontend/transport/handler.go | 1 - pkg/querier/tripperware/priority.go | 4 ++-- pkg/util/httpgrpcutil/header.go | 7 ++++++- 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/pkg/frontend/transport/handler.go b/pkg/frontend/transport/handler.go index 042dbd32b7..700145312f 100644 --- a/pkg/frontend/transport/handler.go +++ b/pkg/frontend/transport/handler.go @@ -200,7 +200,6 @@ func (f *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) { r.Body = io.NopCloser(&buf) } - r.Header.Get("test") startTime := time.Now() resp, err := f.roundTripper.RoundTrip(r) queryResponseTime := time.Since(startTime) diff --git a/pkg/querier/tripperware/priority.go b/pkg/querier/tripperware/priority.go index 5ce4e0d3fa..b96a3b0e9a 100644 --- a/pkg/querier/tripperware/priority.go +++ b/pkg/querier/tripperware/priority.go @@ -88,7 +88,7 @@ func isWithinTimeAttributes(attribute validation.QueryAttribute, now time.Time, } func FindMinMaxTime(s *parser.EvalStmt) (int64, int64) { - // Placeholder until Prometheus is updated to >=0.48.0 - // which includes https://github.com/prometheus/prometheus/commit/9e3df532d8294d4fe3284bde7bc96db336a33552 + // Placeholder until Prometheus is updated to include + // https://github.com/prometheus/prometheus/commit/9e3df532d8294d4fe3284bde7bc96db336a33552 return s.Start.Unix(), s.End.Unix() } diff --git a/pkg/util/httpgrpcutil/header.go b/pkg/util/httpgrpcutil/header.go index b844e0c65f..ed453f01fa 100644 --- a/pkg/util/httpgrpcutil/header.go +++ b/pkg/util/httpgrpcutil/header.go @@ -7,7 +7,12 @@ import ( // GetHeader is similar to http.Header.Get, which gets the first value associated with the given key. // If there are no values associated with the key, it returns "". func GetHeader(r httpgrpc.HTTPRequest, key string) string { - return GetHeaderValues(r, key)[0] + values := GetHeaderValues(r, key) + if len(values) == 0 { + return "" + } + + return values[0] } // GetHeaderValues is similar to http.Header.Values, which returns all values associated with the given key. From cdd2a0c785aacb62ce44535a41a619db55670558 Mon Sep 17 00:00:00 2001 From: Justin Jung Date: Wed, 29 Nov 2023 10:12:44 -0800 Subject: [PATCH 45/49] Address comments Signed-off-by: Justin Jung --- pkg/cortex/modules.go | 1 + pkg/frontend/transport/handler_test.go | 80 +++++++++++++------ .../tripperware/instantquery/instant_query.go | 2 +- pkg/querier/tripperware/priority.go | 4 +- pkg/querier/tripperware/priority_test.go | 8 +- .../query_range_middlewares_test.go | 1 + pkg/querier/tripperware/roundtrip.go | 5 +- pkg/querier/tripperware/roundtrip_test.go | 1 + pkg/util/validation/limits.go | 13 ++- 9 files changed, 78 insertions(+), 37 deletions(-) diff --git a/pkg/cortex/modules.go b/pkg/cortex/modules.go index 7858483c68..4fc0969ee7 100644 --- a/pkg/cortex/modules.go +++ b/pkg/cortex/modules.go @@ -487,6 +487,7 @@ func (t *Cortex) initQueryFrontendTripperware() (serv services.Service, err erro queryAnalyzer, t.Cfg.Querier.DefaultEvaluationInterval, t.Cfg.Querier.MaxSubQuerySteps, + t.Cfg.Querier.LookbackDelta, ) return services.NewIdleService(nil, func(_ error) error { diff --git a/pkg/frontend/transport/handler_test.go b/pkg/frontend/transport/handler_test.go index 955323fa99..801bda6d81 100644 --- a/pkg/frontend/transport/handler_test.go +++ b/pkg/frontend/transport/handler_test.go @@ -21,6 +21,7 @@ import ( "github.com/weaveworks/common/user" querier_stats "github.com/cortexproject/cortex/pkg/querier/stats" + "github.com/cortexproject/cortex/pkg/util" ) type roundTripperFunc func(*http.Request) (*http.Response, error) @@ -301,34 +302,63 @@ func TestReportQueryStatsFormat(t *testing.T) { outputBuf := bytes.NewBuffer(nil) logger := log.NewSyncLogger(log.NewLogfmtLogger(outputBuf)) handler := NewHandler(HandlerConfig{QueryStatsEnabled: true}, http.DefaultTransport, logger, nil) - userID := "fake" - queryString := url.Values(map[string][]string{"query": {"up"}}) - req, err := http.NewRequest(http.MethodGet, "http://localhost:8080/prometheus/api/v1/query", nil) - require.NoError(t, err) - req.Header = http.Header{ - "User-Agent": []string{"Grafana"}, - } - resp := &http.Response{ - ContentLength: 1000, + req, _ := http.NewRequest(http.MethodGet, "http://localhost:8080/prometheus/api/v1/query", nil) + resp := &http.Response{ContentLength: 1000} + responseTime := time.Second + statusCode := http.StatusOK + + type testCase struct { + queryString url.Values + queryStats *querier_stats.QueryStats + header http.Header + responseErr error + expectedLog string } - stats := &querier_stats.QueryStats{ - Stats: querier_stats.Stats{ - WallTime: 3 * time.Second, - FetchedSeriesCount: 100, - FetchedChunksCount: 200, - FetchedSamplesCount: 300, - FetchedChunkBytes: 1024, - FetchedDataBytes: 2048, + + tests := map[string]testCase{ + "should not include query and header details if empty": { + expectedLog: `level=info msg="query stats" component=query-frontend method=GET path=/prometheus/api/v1/query response_time=1s query_wall_time_seconds=0 fetched_series_count=0 fetched_chunks_count=0 fetched_samples_count=0 fetched_chunks_bytes=0 fetched_data_bytes=0 status_code=200 response_size=1000`, + }, + "should include query length and string at the end": { + queryString: url.Values(map[string][]string{"query": {"up"}}), + expectedLog: `level=info msg="query stats" component=query-frontend method=GET path=/prometheus/api/v1/query response_time=1s query_wall_time_seconds=0 fetched_series_count=0 fetched_chunks_count=0 fetched_samples_count=0 fetched_chunks_bytes=0 fetched_data_bytes=0 status_code=200 response_size=1000 query_length=2 param_query=up`, + }, + "should include query stats": { + queryStats: &querier_stats.QueryStats{ + Stats: querier_stats.Stats{ + WallTime: 3 * time.Second, + FetchedSeriesCount: 100, + FetchedChunksCount: 200, + FetchedSamplesCount: 300, + FetchedChunkBytes: 1024, + FetchedDataBytes: 2048, + }, + }, + expectedLog: `level=info msg="query stats" component=query-frontend method=GET path=/prometheus/api/v1/query response_time=1s query_wall_time_seconds=3 fetched_series_count=100 fetched_chunks_count=200 fetched_samples_count=300 fetched_chunks_bytes=1024 fetched_data_bytes=2048 status_code=200 response_size=1000`, + }, + "should include user agent": { + header: http.Header{"User-Agent": []string{"Grafana"}}, + expectedLog: `level=info msg="query stats" component=query-frontend method=GET path=/prometheus/api/v1/query response_time=1s query_wall_time_seconds=0 fetched_series_count=0 fetched_chunks_count=0 fetched_samples_count=0 fetched_chunks_bytes=0 fetched_data_bytes=0 status_code=200 response_size=1000 user_agent=Grafana`, + }, + "should include response error": { + responseErr: errors.New("foo_err"), + expectedLog: `level=error msg="query stats" component=query-frontend method=GET path=/prometheus/api/v1/query response_time=1s query_wall_time_seconds=0 fetched_series_count=0 fetched_chunks_count=0 fetched_samples_count=0 fetched_chunks_bytes=0 fetched_data_bytes=0 status_code=200 response_size=1000 error=foo_err`, + }, + "should include query priority": { + queryString: url.Values(map[string][]string{"query": {"up"}}), + header: http.Header{util.QueryPriorityHeaderKey: []string{"99"}}, + expectedLog: `level=info msg="query stats" component=query-frontend method=GET path=/prometheus/api/v1/query response_time=1s query_wall_time_seconds=0 fetched_series_count=0 fetched_chunks_count=0 fetched_samples_count=0 fetched_chunks_bytes=0 fetched_data_bytes=0 status_code=200 response_size=1000 query_length=2 priority=99 param_query=up`, }, } - responseErr := errors.New("foo_err") - handler.reportQueryStats(req, userID, queryString, time.Second, stats, responseErr, http.StatusOK, resp) - data, err := io.ReadAll(outputBuf) - require.NoError(t, err) - - expectedLog := `level=error msg="query stats" component=query-frontend method=GET path=/prometheus/api/v1/query response_time=1s query_wall_time_seconds=3 fetched_series_count=100 fetched_chunks_count=200 fetched_samples_count=300 fetched_chunks_bytes=1024 fetched_data_bytes=2048 status_code=200 response_size=1000 query_length=2 user_agent=Grafana error=foo_err param_query=up -` - require.Equal(t, expectedLog, string(data)) + for testName, testData := range tests { + t.Run(testName, func(t *testing.T) { + req.Header = testData.header + handler.reportQueryStats(req, userID, testData.queryString, responseTime, testData.queryStats, testData.responseErr, statusCode, resp) + data, err := io.ReadAll(outputBuf) + require.NoError(t, err) + require.Equal(t, testData.expectedLog+"\n", string(data)) + }) + } } diff --git a/pkg/querier/tripperware/instantquery/instant_query.go b/pkg/querier/tripperware/instantquery/instant_query.go index 60bea7a8d1..252443f5a2 100644 --- a/pkg/querier/tripperware/instantquery/instant_query.go +++ b/pkg/querier/tripperware/instantquery/instant_query.go @@ -4,7 +4,6 @@ import ( "bytes" "context" "fmt" - "github.com/cortexproject/cortex/pkg/util" "io" "net/http" "net/url" @@ -25,6 +24,7 @@ import ( "github.com/cortexproject/cortex/pkg/cortexpb" "github.com/cortexproject/cortex/pkg/querier/tripperware" "github.com/cortexproject/cortex/pkg/querier/tripperware/queryrange" + "github.com/cortexproject/cortex/pkg/util" "github.com/cortexproject/cortex/pkg/util/spanlogger" ) diff --git a/pkg/querier/tripperware/priority.go b/pkg/querier/tripperware/priority.go index b96a3b0e9a..0bdd4c1be5 100644 --- a/pkg/querier/tripperware/priority.go +++ b/pkg/querier/tripperware/priority.go @@ -11,7 +11,7 @@ import ( "github.com/cortexproject/cortex/pkg/util/validation" ) -func GetPriority(r *http.Request, userID string, limits Limits, now time.Time) (int64, error) { +func GetPriority(r *http.Request, userID string, limits Limits, now time.Time, lookbackDelta time.Duration) (int64, error) { isQuery := strings.HasSuffix(r.URL.Path, "/query") isQueryRange := strings.HasSuffix(r.URL.Path, "/query_range") queryPriority := limits.QueryPriority(userID) @@ -45,7 +45,7 @@ func GetPriority(r *http.Request, userID string, limits Limits, now time.Time) ( Expr: expr, Start: util.TimeFromMillis(startTime), End: util.TimeFromMillis(endTime), - LookbackDelta: limits.MaxQueryLookback(userID), // this is available from querier flag. + LookbackDelta: lookbackDelta, } minTime, maxTime := FindMinMaxTime(es) diff --git a/pkg/querier/tripperware/priority_test.go b/pkg/querier/tripperware/priority_test.go index 2918842346..6cfaa45b49 100644 --- a/pkg/querier/tripperware/priority_test.go +++ b/pkg/querier/tripperware/priority_test.go @@ -57,7 +57,7 @@ func Test_GetPriorityShouldReturnDefaultPriorityIfNotEnabledOrEmptyQueryString(t t.Run(testName, func(t *testing.T) { limits.queryPriority.Enabled = testData.queryPriorityEnabled req, _ := http.NewRequest(http.MethodPost, testData.url, bytes.NewReader([]byte{})) - priority, err := GetPriority(req, "", limits, now) + priority, err := GetPriority(req, "", limits, now, 0) assert.NoError(t, err) assert.Equal(t, int64(0), priority) }) @@ -117,7 +117,7 @@ func Test_GetPriorityShouldConsiderRegex(t *testing.T) { limits.queryPriority.Priorities[0].QueryAttributes[0].Regex = testData.regex limits.queryPriority.Priorities[0].QueryAttributes[0].CompiledRegex = regexp.MustCompile(testData.regex) req, _ := http.NewRequest(http.MethodPost, "/query?query="+testData.query, bytes.NewReader([]byte{})) - priority, err := GetPriority(req, "", limits, now) + priority, err := GetPriority(req, "", limits, now, 0) assert.NoError(t, err) assert.Equal(t, int64(testData.expectedPriority), priority) }) @@ -214,7 +214,7 @@ func Test_GetPriorityShouldConsiderStartAndEndTime(t *testing.T) { url = "/query?query=sum(up)" } req, _ := http.NewRequest(http.MethodPost, url, bytes.NewReader([]byte{})) - priority, err := GetPriority(req, "", limits, now) + priority, err := GetPriority(req, "", limits, now, 0) assert.NoError(t, err) assert.Equal(t, int64(testData.expectedPriority), priority) }) @@ -269,7 +269,7 @@ func Test_GetPriorityShouldNotConsiderStartAndEndTimeIfEmpty(t *testing.T) { url = "/query?query=sum(up)" } req, _ := http.NewRequest(http.MethodPost, url, bytes.NewReader([]byte{})) - priority, err := GetPriority(req, "", limits, now) + priority, err := GetPriority(req, "", limits, now, 0) assert.NoError(t, err) assert.Equal(t, int64(1), priority) }) diff --git a/pkg/querier/tripperware/queryrange/query_range_middlewares_test.go b/pkg/querier/tripperware/queryrange/query_range_middlewares_test.go index 6276be72bd..518cdce885 100644 --- a/pkg/querier/tripperware/queryrange/query_range_middlewares_test.go +++ b/pkg/querier/tripperware/queryrange/query_range_middlewares_test.go @@ -75,6 +75,7 @@ func TestRoundTrip(t *testing.T) { qa, time.Minute, 0, + 0, ) for i, tc := range []struct { diff --git a/pkg/querier/tripperware/roundtrip.go b/pkg/querier/tripperware/roundtrip.go index 6c7cd13508..8ad3dd7fe0 100644 --- a/pkg/querier/tripperware/roundtrip.go +++ b/pkg/querier/tripperware/roundtrip.go @@ -105,6 +105,7 @@ func NewQueryTripperware( queryAnalyzer querysharding.Analyzer, defaultSubQueryInterval time.Duration, maxSubQuerySteps int64, + lookbackDelta time.Duration, ) Tripperware { // Per tenant query metrics. queriesPerTenant := promauto.With(registerer).NewCounterVec(prometheus.CounterOpts{ @@ -158,8 +159,8 @@ func NewQueryTripperware( } } - if limits.QueryPriority(userStr).Enabled { - priority, err := GetPriority(r, userStr, limits, now) + if limits != nil && limits.QueryPriority(userStr).Enabled { + priority, err := GetPriority(r, userStr, limits, now, lookbackDelta) if err != nil { return nil, err } diff --git a/pkg/querier/tripperware/roundtrip_test.go b/pkg/querier/tripperware/roundtrip_test.go index e52514ee8d..2754c17baa 100644 --- a/pkg/querier/tripperware/roundtrip_test.go +++ b/pkg/querier/tripperware/roundtrip_test.go @@ -201,6 +201,7 @@ func TestRoundTrip(t *testing.T) { querysharding.NewQueryAnalyzer(), time.Minute, tc.maxSubQuerySteps, + 0, ) resp, err := tw(downstream).RoundTrip(req) if tc.expectedErr == nil { diff --git a/pkg/util/validation/limits.go b/pkg/util/validation/limits.go index cd65e875da..7253bf3108 100644 --- a/pkg/util/validation/limits.go +++ b/pkg/util/validation/limits.go @@ -10,6 +10,7 @@ import ( "strings" "time" + "github.com/cespare/xxhash/v2" "github.com/prometheus/common/model" "github.com/prometheus/prometheus/model/relabel" "golang.org/x/time/rate" @@ -125,7 +126,7 @@ type Limits struct { // Query Frontend / Scheduler enforced limits. MaxOutstandingPerTenant int `yaml:"max_outstanding_requests_per_tenant" json:"max_outstanding_requests_per_tenant"` QueryPriority QueryPriority `yaml:"query_priority" json:"query_priority" doc:"nocli|description=Configuration for query priority."` - queryPriorityRegexHash string + queryPriorityRegexHash uint64 queryPriorityCompiledRegex map[string]*regexp.Regexp // Ruler defaults and limits. @@ -314,12 +315,18 @@ func (l *Limits) copyNotificationIntegrationLimits(defaults NotificationRateLimi } func (l *Limits) hasQueryPriorityRegexChanged() bool { - var newHash string + var newHash uint64 + + var seps = []byte{'\xff'} + h := xxhash.New() for _, priority := range l.QueryPriority.Priorities { for _, attribute := range priority.QueryAttributes { - newHash += attribute.Regex + _, _ = h.WriteString(attribute.Regex) + _, _ = h.Write(seps) } } + newHash = h.Sum64() + if newHash != l.queryPriorityRegexHash { l.queryPriorityRegexHash = newHash return true From 3f347e4d154aa0da6af9d04c6430cd4714069969 Mon Sep 17 00:00:00 2001 From: Justin Jung Date: Wed, 29 Nov 2023 17:00:07 -0800 Subject: [PATCH 46/49] Minor improvements Signed-off-by: Justin Jung --- pkg/querier/tripperware/priority.go | 10 ++++++++-- pkg/querier/tripperware/priority_test.go | 4 ++-- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/pkg/querier/tripperware/priority.go b/pkg/querier/tripperware/priority.go index 0bdd4c1be5..361008a581 100644 --- a/pkg/querier/tripperware/priority.go +++ b/pkg/querier/tripperware/priority.go @@ -26,6 +26,10 @@ func GetPriority(r *http.Request, userID string, limits Limits, now time.Time, l return 0, err } + if len(queryPriority.Priorities) == 0 { + return queryPriority.DefaultPriority, nil + } + var startTime, endTime int64 if isQuery { if t, err := util.ParseTimeParam(r, "time", now.Unix()); err == nil { @@ -52,8 +56,10 @@ func GetPriority(r *http.Request, userID string, limits Limits, now time.Time, l for _, priority := range queryPriority.Priorities { for _, attribute := range priority.QueryAttributes { - if attribute.Regex == "" || (attribute.CompiledRegex != nil && !attribute.CompiledRegex.MatchString(query)) { - continue + if attribute.Regex != "" && attribute.Regex != ".*" && attribute.Regex != ".+" { + if attribute.CompiledRegex != nil && !attribute.CompiledRegex.MatchString(query) { + continue + } } if isWithinTimeAttributes(attribute, now, minTime, maxTime) { diff --git a/pkg/querier/tripperware/priority_test.go b/pkg/querier/tripperware/priority_test.go index 6cfaa45b49..9aac92ffe3 100644 --- a/pkg/querier/tripperware/priority_test.go +++ b/pkg/querier/tripperware/priority_test.go @@ -105,10 +105,10 @@ func Test_GetPriorityShouldConsiderRegex(t *testing.T) { query: "count(sum(up))", expectedPriority: 1, }, - "should miss if regex is an empty string": { + "should hit if regex is an empty string": { regex: "", query: "sum(up)", - expectedPriority: 0, + expectedPriority: 1, }, } From 3ee0b0da92096ab018104900d9b8c20cb1471a7c Mon Sep 17 00:00:00 2001 From: Justin Jung Date: Wed, 29 Nov 2023 21:13:41 -0800 Subject: [PATCH 47/49] Rename query start end time Signed-off-by: Justin Jung --- docs/configuration/config-file-reference.md | 14 +++++++++----- pkg/querier/tripperware/priority.go | 14 +++++++------- pkg/querier/tripperware/priority_test.go | 6 ++++-- pkg/util/validation/limits.go | 10 +++++++--- pkg/util/validation/limits_test.go | 2 +- 5 files changed, 28 insertions(+), 18 deletions(-) diff --git a/docs/configuration/config-file-reference.md b/docs/configuration/config-file-reference.md index 25286293dd..49c244f87a 100644 --- a/docs/configuration/config-file-reference.md +++ b/docs/configuration/config-file-reference.md @@ -5061,14 +5061,18 @@ otel: ### `QueryAttribute` ```yaml -# Query string regex. If set to empty string, it will not match anything. +# Regex that the query string should match. If not set, it won't be checked.' [regex: | default = ""] -# Query start time. If set to 0, the start time won't be checked. -[start_time: | default = 0] +# Time window that the query should be within. If not set, it won't be checked.' +time_window: + # Start of the time window that the query should be within. If set to 0, it + # won't be checked. + [start: | default = 0] -# Query end time. If set to 0, the end time won't be checked. -[end_time: | default = 0] + # End of the time window that the query should be within. If set to 0, it + # won't be checked. + [end: | default = 0] ``` ### `DisabledRuleGroup` diff --git a/pkg/querier/tripperware/priority.go b/pkg/querier/tripperware/priority.go index 361008a581..4a8d3ae07f 100644 --- a/pkg/querier/tripperware/priority.go +++ b/pkg/querier/tripperware/priority.go @@ -62,7 +62,7 @@ func GetPriority(r *http.Request, userID string, limits Limits, now time.Time, l } } - if isWithinTimeAttributes(attribute, now, minTime, maxTime) { + if isWithinTimeAttributes(attribute.TimeWindow, now, minTime, maxTime) { return priority.Priority, nil } } @@ -71,20 +71,20 @@ func GetPriority(r *http.Request, userID string, limits Limits, now time.Time, l return queryPriority.DefaultPriority, nil } -func isWithinTimeAttributes(attribute validation.QueryAttribute, now time.Time, startTime, endTime int64) bool { - if attribute.StartTime == 0 && attribute.EndTime == 0 { +func isWithinTimeAttributes(timeWindow validation.TimeWindow, now time.Time, startTime, endTime int64) bool { + if timeWindow.Start == 0 && timeWindow.End == 0 { return true } - if attribute.StartTime != 0 { - startTimeThreshold := now.Add(-1 * time.Duration(attribute.StartTime).Abs()).Truncate(time.Second).Unix() + if timeWindow.Start != 0 { + startTimeThreshold := now.Add(-1 * time.Duration(timeWindow.Start).Abs()).Truncate(time.Second).Unix() if startTime < startTimeThreshold { return false } } - if attribute.EndTime != 0 { - endTimeThreshold := now.Add(-1 * time.Duration(attribute.EndTime).Abs()).Add(1 * time.Second).Truncate(time.Second).Unix() + if timeWindow.End != 0 { + endTimeThreshold := now.Add(-1 * time.Duration(timeWindow.End).Abs()).Add(1 * time.Second).Truncate(time.Second).Unix() if endTime > endTimeThreshold { return false } diff --git a/pkg/querier/tripperware/priority_test.go b/pkg/querier/tripperware/priority_test.go index 9aac92ffe3..e95d9170d9 100644 --- a/pkg/querier/tripperware/priority_test.go +++ b/pkg/querier/tripperware/priority_test.go @@ -135,8 +135,10 @@ func Test_GetPriorityShouldConsiderStartAndEndTime(t *testing.T) { { Regex: ".*", CompiledRegex: regexp.MustCompile(".*"), - StartTime: model.Duration(45 * time.Minute), - EndTime: model.Duration(15 * time.Minute), + TimeWindow: validation.TimeWindow{ + Start: model.Duration(45 * time.Minute), + End: model.Duration(15 * time.Minute), + }, }, }, }, diff --git a/pkg/util/validation/limits.go b/pkg/util/validation/limits.go index 7253bf3108..9531ee8555 100644 --- a/pkg/util/validation/limits.go +++ b/pkg/util/validation/limits.go @@ -63,12 +63,16 @@ type PriorityDef struct { } type QueryAttribute struct { - Regex string `yaml:"regex" json:"regex" doc:"nocli|description=Query string regex. If set to empty string, it will not match anything."` - StartTime model.Duration `yaml:"start_time" json:"start_time" doc:"nocli|description=Query start time. If set to 0, the start time won't be checked.|default=0"` - EndTime model.Duration `yaml:"end_time" json:"end_time" doc:"nocli|description=Query end time. If set to 0, the end time won't be checked.|default=0"` + Regex string `yaml:"regex" json:"regex" doc:"nocli|description=Regex that the query string should match. If not set, it won't be checked.'"` + TimeWindow TimeWindow `yaml:"time_window" json:"time_window" doc:"nocli|description=Time window that the query should be within. If not set, it won't be checked.'"` CompiledRegex *regexp.Regexp } +type TimeWindow struct { + Start model.Duration `yaml:"start" json:"start" doc:"nocli|description=Start of the time window that the query should be within. If set to 0, it won't be checked.|default=0"` + End model.Duration `yaml:"end" json:"end" doc:"nocli|description=End of the time window that the query should be within. If set to 0, it won't be checked.|default=0"` +} + // Limits describe all the limits for users; can be used to describe global default // limits via flags, or per-user limits via yaml config. type Limits struct { diff --git a/pkg/util/validation/limits_test.go b/pkg/util/validation/limits_test.go index 32a0bf924c..e3b8c6d3ff 100644 --- a/pkg/util/validation/limits_test.go +++ b/pkg/util/validation/limits_test.go @@ -620,7 +620,7 @@ func TestHasQueryPriorityRegexChanged(t *testing.T) { require.True(t, l.hasQueryPriorityRegexChanged()) - l.QueryPriority.Priorities[0].QueryAttributes[0].StartTime = model.Duration(2 * time.Hour) + l.QueryPriority.Priorities[0].QueryAttributes[0].TimeWindow.Start = model.Duration(2 * time.Hour) require.False(t, l.hasQueryPriorityRegexChanged()) From c0252f15067b4097ebfb461d31144086e5170cf8 Mon Sep 17 00:00:00 2001 From: Justin Jung Date: Wed, 29 Nov 2023 22:08:46 -0800 Subject: [PATCH 48/49] Improve tests Signed-off-by: Justin Jung --- .../tripperware/queryrange/limits_test.go | 2 +- pkg/scheduler/queue/user_queues.go | 17 ++- pkg/scheduler/queue/user_queues_test.go | 101 +++++++----------- 3 files changed, 53 insertions(+), 67 deletions(-) diff --git a/pkg/querier/tripperware/queryrange/limits_test.go b/pkg/querier/tripperware/queryrange/limits_test.go index c1e5954421..d5d4a6b230 100644 --- a/pkg/querier/tripperware/queryrange/limits_test.go +++ b/pkg/querier/tripperware/queryrange/limits_test.go @@ -2,7 +2,6 @@ package queryrange import ( "context" - "github.com/cortexproject/cortex/pkg/util/validation" "testing" "time" @@ -13,6 +12,7 @@ import ( "github.com/cortexproject/cortex/pkg/querier/tripperware" "github.com/cortexproject/cortex/pkg/util" + "github.com/cortexproject/cortex/pkg/util/validation" ) func TestLimitsMiddleware_MaxQueryLookback(t *testing.T) { diff --git a/pkg/scheduler/queue/user_queues.go b/pkg/scheduler/queue/user_queues.go index b0ae8069ac..25f562ee02 100644 --- a/pkg/scheduler/queue/user_queues.go +++ b/pkg/scheduler/queue/user_queues.go @@ -2,7 +2,6 @@ package queue import ( "math/rand" - "reflect" "sort" "time" @@ -136,7 +135,7 @@ func (q *queues) getOrAddQueue(userID string, maxQueriers int) userRequestQueue uq := q.userQueues[userID] priorityEnabled := q.limits.QueryPriority(userID).Enabled maxOutstanding := q.limits.MaxOutstandingPerTenant(userID) - priorityList := getPriorityList(q.limits.QueryPriority(userID), len(q.queriers)) + priorityList := getPriorityList(q.limits.QueryPriority(userID), maxQueriers) if uq == nil { uq = &userQueue{ @@ -180,7 +179,7 @@ func (q *queues) getOrAddQueue(userID string, maxQueriers int) userRequestQueue uq.queriers = shuffleQueriersForUser(uq.seed, maxQueriers, q.sortedQueriers, nil) } - if priorityEnabled && !reflect.DeepEqual(uq.priorityList, priorityList) { + if priorityEnabled && hasPriorityListChanged(uq.priorityList, priorityList) { reservedQueriers := make(map[string]int64) i := 0 @@ -406,6 +405,18 @@ func getPriorityList(queryPriority validation.QueryPriority, totalQuerierCount i return priorityList } +func hasPriorityListChanged(old, new []int64) bool { + if len(old) != len(new) { + return true + } + for i := range old { + if old[i] != new[i] { + return true + } + } + return false +} + // MockLimits implements the Limits interface. Used in tests only. type MockLimits struct { MaxOutstanding int diff --git a/pkg/scheduler/queue/user_queues_test.go b/pkg/scheduler/queue/user_queues_test.go index ae812aac6a..ded597baa0 100644 --- a/pkg/scheduler/queue/user_queues_test.go +++ b/pkg/scheduler/queue/user_queues_test.go @@ -399,6 +399,7 @@ func TestGetOrAddQueueShouldUpdateProperties(t *testing.T) { assert.Equal(t, 3, q.userQueues["userID"].maxQueriers) assert.Equal(t, 5, len(q.queriers)) assert.Equal(t, 3, len(q.userQueues["userID"].queriers)) + assert.Equal(t, 2, len(q.userQueues["userID"].reservedQueriers)) assert.IsType(t, &PriorityRequestQueue{}, queue) assert.Equal(t, 1, queue.length()) assert.ElementsMatch(t, []int64{1, 1}, q.userQueues["userID"].priorityList) @@ -406,6 +407,38 @@ func TestGetOrAddQueueShouldUpdateProperties(t *testing.T) { assert.Subset(t, getKeys(q.queriers), getKeys(q.userQueues["userID"].queriers)) assert.Subset(t, getKeys(q.userQueues["userID"].queriers), getKeys(q.userQueues["userID"].reservedQueriers)) + limits.QueryPriorityVal = validation.QueryPriority{Enabled: true, Priorities: []validation.PriorityDef{ + { + Priority: 1, + ReservedQueriers: 0.5, + }, + }} + q.limits = limits + _ = q.getOrAddQueue("userID", 3) + + assert.Equal(t, 10, q.userQueues["userID"].maxOutstanding) + assert.Equal(t, 3, q.userQueues["userID"].maxQueriers) + assert.Equal(t, 5, len(q.queriers)) + assert.Equal(t, 3, len(q.userQueues["userID"].queriers)) + assert.Equal(t, 2, len(q.userQueues["userID"].reservedQueriers)) + assert.ElementsMatch(t, []int64{1, 1}, q.userQueues["userID"].priorityList) + + limits.QueryPriorityVal = validation.QueryPriority{Enabled: true, Priorities: []validation.PriorityDef{ + { + Priority: 1, + ReservedQueriers: 10, + }, + }} + q.limits = limits + _ = q.getOrAddQueue("userID", 3) + + assert.Equal(t, 10, q.userQueues["userID"].maxOutstanding) + assert.Equal(t, 3, q.userQueues["userID"].maxQueriers) + assert.Equal(t, 5, len(q.queriers)) + assert.Equal(t, 3, len(q.userQueues["userID"].queriers)) + assert.Equal(t, 0, len(q.userQueues["userID"].reservedQueriers)) + assert.ElementsMatch(t, []int64{}, q.userQueues["userID"].priorityList) + limits.QueryPriorityVal.Enabled = false q.limits = limits queue = q.getOrAddQueue("userID", 3) @@ -582,69 +615,11 @@ func TestShuffleQueriersCorrectness(t *testing.T) { } } -func TestShuffleQueriers_WithReservedQueriers(t *testing.T) { - //allQueriers := []string{"a", "b", "c", "d", "e"} - // - //queriers, reservedQueriers := shuffleQueriersForUser(12345, 0, allQueriers, 0, nil) - //require.Nil(t, queriers) - //require.Equal(t, 0, len(reservedQueriers)) - // - //queriers, reservedQueriers = shuffleQueriersForUser(12345, 0, allQueriers, 0.5, nil) - //require.Nil(t, queriers) - //require.Equal(t, 3, len(reservedQueriers)) - // - //queriers, reservedQueriers = shuffleQueriersForUser(12345, 0, allQueriers, 1, nil) - //require.Nil(t, queriers) - //require.Equal(t, 1, len(reservedQueriers)) - // - //queriers, reservedQueriers = shuffleQueriersForUser(12345, 0, allQueriers, 100, nil) - //require.Nil(t, queriers) - //require.Equal(t, 5, len(reservedQueriers)) - // - //queriers, reservedQueriers = shuffleQueriersForUser(12345, 3, allQueriers, 0, nil) - //require.Equal(t, 3, len(queriers)) - //require.Equal(t, 0, len(reservedQueriers)) - // - //queriers, reservedQueriers = shuffleQueriersForUser(12345, 3, allQueriers, 0.5, nil) - //require.Equal(t, 3, len(queriers)) - //require.Equal(t, 2, len(reservedQueriers)) - // - //queriers, reservedQueriers = shuffleQueriersForUser(12345, 3, allQueriers, 1, nil) - //require.Equal(t, 3, len(queriers)) - //require.Equal(t, 1, len(reservedQueriers)) - // - //queriers, reservedQueriers = shuffleQueriersForUser(12345, 3, allQueriers, 100, nil) - //require.Equal(t, 3, len(queriers)) - //require.Equal(t, 3, len(reservedQueriers)) - // - //queriers, reservedQueriers = shuffleQueriersForUser(12345, 100, allQueriers, 0, nil) - //require.Nil(t, queriers) - //require.Equal(t, 0, len(reservedQueriers)) - // - //queriers, reservedQueriers = shuffleQueriersForUser(12345, 100, allQueriers, 0.5, nil) - //require.Nil(t, queriers) - //require.Equal(t, 3, len(reservedQueriers)) - // - //queriers, reservedQueriers = shuffleQueriersForUser(12345, 100, allQueriers, 1, nil) - //require.Nil(t, queriers) - //require.Equal(t, 1, len(reservedQueriers)) - // - //queriers, reservedQueriers = shuffleQueriersForUser(12345, 100, allQueriers, 100, nil) - //require.Nil(t, queriers) - //require.Equal(t, 5, len(reservedQueriers)) -} - -func TestShuffleQueriers_WithReservedQueriers_Correctness(t *testing.T) { - //allQueriers := []string{"a", "b", "c", "d", "e", "f", "g", "h", "i", "j", "k", "l", "m", "n"} - // - //prevQueriers, prevReservedQueriers := shuffleQueriersForUser(12345, 10, allQueriers, 5, nil) - //for i := 0; i < 100; i++ { - // queriers, reservedQueriers := shuffleQueriersForUser(12345, 10, allQueriers, 5, nil) - // require.Equal(t, prevQueriers, queriers) - // require.Equal(t, prevReservedQueriers, reservedQueriers) - // prevQueriers = queriers - // prevReservedQueriers = reservedQueriers - //} +func TestHasPriorityListChanged(t *testing.T) { + require.True(t, hasPriorityListChanged([]int64{1, 2}, []int64{1, 3})) + require.False(t, hasPriorityListChanged([]int64{1, 2}, []int64{1, 2})) + require.True(t, hasPriorityListChanged([]int64{1, 2}, []int64{1})) + require.False(t, hasPriorityListChanged([]int64{}, []int64{})) } func TestGetPriorityList(t *testing.T) { From b6eb74d4cfb927c5de553ef3ead0ab036c429a94 Mon Sep 17 00:00:00 2001 From: Justin Jung Date: Wed, 29 Nov 2023 22:17:50 -0800 Subject: [PATCH 49/49] Nit Signed-off-by: Justin Jung --- docs/configuration/config-file-reference.md | 4 ++-- pkg/util/validation/limits.go | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/configuration/config-file-reference.md b/docs/configuration/config-file-reference.md index 49c244f87a..b5ab659830 100644 --- a/docs/configuration/config-file-reference.md +++ b/docs/configuration/config-file-reference.md @@ -5061,10 +5061,10 @@ otel: ### `QueryAttribute` ```yaml -# Regex that the query string should match. If not set, it won't be checked.' +# Regex that the query string should match. If not set, it won't be checked. [regex: | default = ""] -# Time window that the query should be within. If not set, it won't be checked.' +# Time window that the query should be within. If not set, it won't be checked. time_window: # Start of the time window that the query should be within. If set to 0, it # won't be checked. diff --git a/pkg/util/validation/limits.go b/pkg/util/validation/limits.go index 9531ee8555..a10e3f1f78 100644 --- a/pkg/util/validation/limits.go +++ b/pkg/util/validation/limits.go @@ -63,8 +63,8 @@ type PriorityDef struct { } type QueryAttribute struct { - Regex string `yaml:"regex" json:"regex" doc:"nocli|description=Regex that the query string should match. If not set, it won't be checked.'"` - TimeWindow TimeWindow `yaml:"time_window" json:"time_window" doc:"nocli|description=Time window that the query should be within. If not set, it won't be checked.'"` + Regex string `yaml:"regex" json:"regex" doc:"nocli|description=Regex that the query string should match. If not set, it won't be checked."` + TimeWindow TimeWindow `yaml:"time_window" json:"time_window" doc:"nocli|description=Time window that the query should be within. If not set, it won't be checked."` CompiledRegex *regexp.Regexp }