From 651e8b0f7b13f86e8964812418296749eb9782cc Mon Sep 17 00:00:00 2001 From: Stepan Bagritsevich Date: Mon, 2 Dec 2024 19:27:35 +0400 Subject: [PATCH] fix(dragonfly_test): Increase string size in the test Bug207 Signed-off-by: Stepan Bagritsevich --- src/server/dragonfly_test.cc | 13 +++--- src/server/engine_shard.cc | 89 +++++++++++++++++++----------------- 2 files changed, 53 insertions(+), 49 deletions(-) diff --git a/src/server/dragonfly_test.cc b/src/server/dragonfly_test.cc index dc78857c934b..d9eaeab645aa 100644 --- a/src/server/dragonfly_test.cc +++ b/src/server/dragonfly_test.cc @@ -456,19 +456,18 @@ TEST_F(DflyEngineTest, OOM) { /// Reproduces the case where items with expiry data were evicted, /// and then written with the same key. TEST_F(DflyEngineTest, Bug207) { - /* Sometimes filling the cache is much faster than the first heartbeat, leading to OOM. - To prevent OOM during cache filling, we need to increase the max_memory_limit and eviction - threshold. */ - max_memory_limit = 1350000; // 1.35mb + max_memory_limit = 300000 * 4; + absl::FlagSaver fs; - absl::SetFlag(&FLAGS_eviction_memory_budget_threshold, 0.6); + absl::SetFlag(&FLAGS_eviction_memory_budget_threshold, 0.3); shard_set->TEST_EnableCacheMode(); ssize_t i = 0; RespExpr resp; - for (; i < 10000; ++i) { - resp = Run({"setex", StrCat("key", i), "30", "bar"}); + std::string value(1000, '.'); + for (; i < 1000; ++i) { + resp = Run({"setex", StrCat("key", i), "30", value}); ASSERT_EQ(resp, "OK"); } diff --git a/src/server/engine_shard.cc b/src/server/engine_shard.cc index f5a6fa0b269f..b1571565e328 100644 --- a/src/server/engine_shard.cc +++ b/src/server/engine_shard.cc @@ -202,28 +202,15 @@ optional GetPeriodicCycleMs() { return clock_cycle_ms; } -ssize_t CalculateMemoryBudget(size_t max_memory, size_t used_memory) { - if (max_memory >= used_memory) { - return max_memory - used_memory; +size_t CalculateHowManyBytesToEvictOnshard(size_t global_memory_limit, size_t global_used_memory, + size_t shard_memory_threshold) { + if (global_used_memory > global_memory_limit) { + // Used memory is above the limit, we need to evict all bytes + return (global_used_memory - global_memory_limit) / shard_set->size() + shard_memory_threshold; } - // negative value indicates memory overuse - return -1 * ssize_t(used_memory - max_memory); -} - -ssize_t CalculateFreeMemoryOnShard() { - // Calculate how much memory is used by all shards - const size_t current_global_rss_memory = used_mem_current.load(memory_order_relaxed); - return CalculateMemoryBudget(max_memory_limit, current_global_rss_memory) / shard_set->size(); -} -ssize_t CalculateFreeRssMemoryOnShard(size_t global_rss_memory_limit) { - // Calculate how much rss memory is used by all shards - io::Result sdata_res = io::ReadStatusInfo(); - size_t current_global_rss_memory = - sdata_res ? FetchRssMemory(sdata_res.value()) : rss_mem_current.load(memory_order_relaxed); - // Calculate how much free rss memory we have - return CalculateMemoryBudget(global_rss_memory_limit, current_global_rss_memory) / - shard_set->size(); + const size_t shard_budget = (global_memory_limit - global_used_memory) / shard_set->size(); + return shard_budget < shard_memory_threshold ? (shard_memory_threshold - shard_budget) : 0; } } // namespace @@ -744,25 +731,31 @@ void EngineShard::RetireExpiredAndEvict() { } const size_t shards_count = shard_set->size(); + const double eviction_memory_budget_threshold = GetFlag(FLAGS_eviction_memory_budget_threshold); const double rss_oom_deny_ratio = ServerState::tlocal()->rss_oom_deny_ratio; - // We start eviction when we have less than 10% of free memory - const ssize_t shard_memory_budget_threshold = - ssize_t(max_memory_limit * absl::GetFlag(FLAGS_eviction_memory_budget_threshold)) / - shards_count; + const bool should_evict = + eviction_memory_budget_threshold > 0.0 && GetFlag(FLAGS_enable_heartbeat_eviction); + // If rss_oom_deny_ratio is set, we should evict depending on rss memory too + const bool should_evict_depending_on_rss_memory = should_evict && rss_oom_deny_ratio > 0.0; + + size_t shard_memory_budget_threshold; + if (should_evict) { + // We start eviction when we have less than eviction_memory_budget_threshold * 100% of free + // memory + shard_memory_budget_threshold = + size_t(max_memory_limit * eviction_memory_budget_threshold) / shards_count; + } size_t max_rss_memory; - ssize_t shard_rss_memory_budget_threshold; // Threshold for free rss memory - if (rss_oom_deny_ratio > 0.0) { + size_t shard_rss_memory_budget_threshold; + if (should_evict_depending_on_rss_memory) { + // rss_oom_deny_ratio > 0.0, so can safely use it max_rss_memory = size_t(rss_oom_deny_ratio * max_memory_limit); - // We start eviction when we have less than 10% of free rss memory + // We start eviction when we have less than eviction_memory_budget_threshold * 100% of free rss + // memory shard_rss_memory_budget_threshold = - ssize_t(max_rss_memory * absl::GetFlag(FLAGS_eviction_memory_budget_threshold)) / - shards_count; - } else { - max_rss_memory = std::numeric_limits::max(); - // We should never evict based on rss memory - shard_rss_memory_budget_threshold = std::numeric_limits::min(); + size_t(max_rss_memory * eviction_memory_budget_threshold) / shards_count; } DbContext db_cntx; @@ -781,17 +774,29 @@ void EngineShard::RetireExpiredAndEvict() { counter_[TTL_DELETE].IncBy(stats.deleted); } - // Memory budget for this shard - const ssize_t memory_budget = CalculateFreeMemoryOnShard(); - const ssize_t rss_memory_budget = CalculateFreeRssMemoryOnShard(max_rss_memory); + if (!should_evict) + continue; + + const size_t global_used_memory = used_mem_current.load(memory_order_relaxed); + + // Calculate how many bytes we need to evict on this shard + size_t goal_bytes = CalculateHowManyBytesToEvictOnshard(max_memory_limit, global_used_memory, + shard_memory_budget_threshold); + + if (should_evict_depending_on_rss_memory) { + // Calculate how much rss memory is used by all shards + io::Result sdata_res = io::ReadStatusInfo(); + const size_t global_used_rss_memory = sdata_res ? FetchRssMemory(sdata_res.value()) + : rss_mem_current.load(memory_order_relaxed); + + // Try to evict more bytes if we are close to the rss memory limit + goal_bytes = std::max( + goal_bytes, CalculateHowManyBytesToEvictOnshard(max_rss_memory, global_used_rss_memory, + shard_rss_memory_budget_threshold)); + } - // If our budget is below the limit we need to evict - if ((memory_budget < shard_memory_budget_threshold || - rss_memory_budget < shard_rss_memory_budget_threshold) && - GetFlag(FLAGS_enable_heartbeat_eviction)) { + if (goal_bytes) { uint32_t starting_segment_id = rand() % pt->GetSegmentCount(); - const size_t goal_bytes = std::max(shard_memory_budget_threshold - memory_budget, - shard_rss_memory_budget_threshold - rss_memory_budget); db_slice.FreeMemWithEvictionStep(i, starting_segment_id, goal_bytes); } }