From 72365e1b8bc3a396ff78f74ef7e7dc5061eefe5c Mon Sep 17 00:00:00 2001 From: Ivo Dilov Date: Tue, 26 Mar 2024 14:52:21 +0200 Subject: [PATCH] Update version map unsync tolerance to 200ms Before this change version_map's reload_interval was equal to the unsync_tolerance. This means that we would ~always miss the cache. The check for outdated cache is as follows: - When reading into cache set `last_reload_time = now - unsync_tolerance` - When checking if cache is outdated we check whether `now - last_reload_time > reload_interval This change updates the unsync tolerance to 200ms. This will mean the cache is valid for 1.8s Tests uncovered that we were not flushing the version_map when clearing the storage. This change fixes this as well. --- cpp/arcticdb/util/constants.hpp | 4 +++- cpp/arcticdb/version/version_map.hpp | 2 +- cpp/arcticdb/version/version_store_api.cpp | 1 + 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/cpp/arcticdb/util/constants.hpp b/cpp/arcticdb/util/constants.hpp index d10f4984c8..4f9779af81 100644 --- a/cpp/arcticdb/util/constants.hpp +++ b/cpp/arcticdb/util/constants.hpp @@ -19,7 +19,9 @@ constexpr auto string_nan = std::numeric_limits::max() - 1; constexpr auto NaT = std::numeric_limits::min(); -static constexpr decltype(timestamp(0) - timestamp(0)) ONE_SECOND = 1'000'000'000; +static constexpr decltype(timestamp(0) - timestamp(0)) ONE_MILLISECOND = 1'000'000; + +static constexpr decltype(timestamp(0) - timestamp(0)) ONE_SECOND = 1'000 * ONE_MILLISECOND; static constexpr decltype(timestamp(0) - timestamp(0)) ONE_MINUTE = 60 * ONE_SECOND; diff --git a/cpp/arcticdb/version/version_map.hpp b/cpp/arcticdb/version/version_map.hpp index 7c7db8abbc..0b27b991cd 100644 --- a/cpp/arcticdb/version/version_map.hpp +++ b/cpp/arcticdb/version/version_map.hpp @@ -107,7 +107,7 @@ class VersionMapImpl { */ using MapType = std::map>; - static constexpr uint64_t DEFAULT_CLOCK_UNSYNC_TOLERANCE = ONE_SECOND * 2; + static constexpr uint64_t DEFAULT_CLOCK_UNSYNC_TOLERANCE = ONE_MILLISECOND * 200; static constexpr uint64_t DEFAULT_RELOAD_INTERVAL = ONE_SECOND * 2; MapType map_; bool validate_ = false; diff --git a/cpp/arcticdb/version/version_store_api.cpp b/cpp/arcticdb/version/version_store_api.cpp index d2e161dac7..071806dc87 100644 --- a/cpp/arcticdb/version/version_store_api.cpp +++ b/cpp/arcticdb/version/version_store_api.cpp @@ -1125,6 +1125,7 @@ std::vector PythonVersionStore::list_incompletes(const StreamId& st } void PythonVersionStore::clear(const bool continue_on_error) { + version_map()->flush(); if (store()->fast_delete()) { // Most storage backends have a fast deletion method for a db/collection equivalent, eg. drop() for mongo and // lmdb and iterating each key is always going to be suboptimal.