From 0c9a4149286fa2bf68d8e7102767a2a6e744ac90 Mon Sep 17 00:00:00 2001 From: Dimitar Dimitrov Date: Wed, 2 Oct 2024 17:23:14 +0200 Subject: [PATCH] kafka replay speed: update fetch bytes estimation on multiple attempts (#9493) Without this we end up using a bad estimation for the whole fetch Signed-off-by: Dimitar Dimitrov (cherry picked from commit 6a998f6d9fd0af1fe296d7ee3f20e50d5344ffb3) --- pkg/storage/ingest/fetcher.go | 1 + 1 file changed, 1 insertion(+) diff --git a/pkg/storage/ingest/fetcher.go b/pkg/storage/ingest/fetcher.go index aeb1061fb85..7197f8a9573 100644 --- a/pkg/storage/ingest/fetcher.go +++ b/pkg/storage/ingest/fetcher.go @@ -510,6 +510,7 @@ func (r *concurrentFetchers) run(ctx context.Context, wants chan fetchWant, logg } // Next attempt will be from the last record onwards. w.startOffset = f.Records[len(f.Records)-1].Offset + 1 + w = w.UpdateBytesPerRecord(f.fetchedBytes, len(f.Records)) // This takes into account the previousFetch too. This should give us a better average than using just the records from the last attempt. // We reset the backoff if we received any records whatsoever. A received record means _some_ success. // We don't want to slow down until we hit a larger error.