From 2dd352060a6c11cde5c5cfc65f77a8e70f55dcb6 Mon Sep 17 00:00:00 2001
From: Neil Shen <overvenus@gmail.com>
Date: Mon, 21 Oct 2024 17:02:41 +0800
Subject: [PATCH 01/16] In-memory Engine: unify failpoints and logs (#17668)

close tikv/tikv#16141

* Unify in memory engine logs
* Unify in memory engine failpoints
* Correct range cache engine to in memory engine

Signed-off-by: Neil Shen <overvenus@gmail.com>
---
 Cargo.lock                                    |   1 +
 components/crossbeam-skiplist/Cargo.toml      |   3 +
 components/crossbeam-skiplist/src/lib.rs      |   6 +
 .../tests/failpoints/test_write_batch.rs      |   4 +-
 components/in_memory_engine/src/background.rs |  85 ++++++-----
 .../in_memory_engine/src/cross_check.rs       |  20 +--
 components/in_memory_engine/src/engine.rs     |   9 +-
 components/in_memory_engine/src/read.rs       |   5 +-
 .../in_memory_engine/src/region_manager.rs    |  24 +--
 .../in_memory_engine/src/region_stats.rs      |  12 +-
 .../in_memory_engine/src/write_batch.rs       |  24 +--
 .../tests/failpoints/test_memory_engine.rs    |  61 ++++----
 tests/failpoints/cases/mod.rs                 |   2 +-
 ...che_engine.rs => test_in_memory_engine.rs} | 137 ++++++++----------
 tests/failpoints/cases/test_merge.rs          |   2 -
 15 files changed, 192 insertions(+), 203 deletions(-)
 rename tests/failpoints/cases/{test_region_cache_engine.rs => test_in_memory_engine.rs} (87%)

diff --git a/Cargo.lock b/Cargo.lock
index e6e888f6fa0..a3535198106 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -1234,6 +1234,7 @@ name = "crossbeam-skiplist"
 version = "0.1.3"
 dependencies = [
  "crossbeam-epoch",
+ "crossbeam-skiplist 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)",
  "crossbeam-utils",
 ]
 
diff --git a/components/crossbeam-skiplist/Cargo.toml b/components/crossbeam-skiplist/Cargo.toml
index 031d4633baa..e75373677a8 100644
--- a/components/crossbeam-skiplist/Cargo.toml
+++ b/components/crossbeam-skiplist/Cargo.toml
@@ -30,6 +30,9 @@ alloc = ["crossbeam-epoch/alloc"]
 
 [dependencies]
 crossbeam-epoch = { workspace = true }
+# Add the official version of `crossbeam-skiplist` to dependency tree, so that
+# our CI can keep track of upstream critical bug-fixes and security updates.
+crossbeam-skiplist-offical = { version = "0.1.3", package = "crossbeam-skiplist" }
 crossbeam-utils = { workspace = true }
 
 # Rename test and example binary names to pass TiKV jemalloc check.
diff --git a/components/crossbeam-skiplist/src/lib.rs b/components/crossbeam-skiplist/src/lib.rs
index 8320c9a1a68..222b735024a 100644
--- a/components/crossbeam-skiplist/src/lib.rs
+++ b/components/crossbeam-skiplist/src/lib.rs
@@ -257,3 +257,9 @@ pub mod set;
 #[cfg(feature = "std")]
 #[doc(inline)]
 pub use crate::{map::SkipMap, set::SkipSet};
+
+// Prevent cargo machete warnings.
+mod seal {
+    pub trait Sealed {}
+    impl Sealed for crossbeam_skiplist_offical::SkipList<(), ()> {}
+}
diff --git a/components/hybrid_engine/tests/failpoints/test_write_batch.rs b/components/hybrid_engine/tests/failpoints/test_write_batch.rs
index e1a07d094f4..0b51f0020cc 100644
--- a/components/hybrid_engine/tests/failpoints/test_write_batch.rs
+++ b/components/hybrid_engine/tests/failpoints/test_write_batch.rs
@@ -15,8 +15,8 @@ fn test_sequence_number_unique() {
         hybrid_engine_for_tests("temp", InMemoryEngineConfig::config_for_test(), |_| {}).unwrap();
 
     let (tx, rx) = sync_channel(0);
-    fail::cfg_callback("on_completes_batch_loading", move || {
-        fail::cfg("on_start_loading_region", "pause").unwrap();
+    fail::cfg_callback("ime_on_completes_batch_loading", move || {
+        fail::cfg("ime_on_start_loading_region", "pause").unwrap();
         tx.send(true).unwrap();
     })
     .unwrap();
diff --git a/components/in_memory_engine/src/background.rs b/components/in_memory_engine/src/background.rs
index 35db969f3f8..605a7270bf0 100644
--- a/components/in_memory_engine/src/background.rs
+++ b/components/in_memory_engine/src/background.rs
@@ -204,9 +204,9 @@ impl PdRangeHintService {
                 }
                 for key_range in &label_rule.data {
                     match CacheRegion::try_from(key_range) {
-                        Ok(cache_range) => {
-                            info!("ime requested to cache range"; "range" => ?&cache_range);
-                            range_manager_load_cb(&cache_range, is_add);
+                        Ok(cache_region) => {
+                            info!("ime requested to cache range"; "range" => ?&cache_region);
+                            range_manager_load_cb(&cache_region, is_add);
                         }
                         Err(e) => {
                             error!("ime unable to convert key_range rule to cache range"; "error" => ?e);
@@ -223,7 +223,6 @@ impl PdRangeHintService {
             pd_client,
         )
         .rule_filter_fn(|label_rule| {
-            info!("dbg rule"; "rule" => ?label_rule);
             label_rule
                 .labels
                 .iter()
@@ -285,29 +284,29 @@ impl BgWorkManager {
         let region_info_provider = self.region_info_provider.clone();
         range_hint_service.start(
             self.worker.remote(),
-            move |cache_range: &CacheRegion, is_add: bool| {
+            move |range: &CacheRegion, is_add: bool| {
                 let region_manager = core.region_manager();
                 if !is_add {
                     region_manager
                         .regions_map()
                         .write()
-                        .remove_manual_load_range(cache_range.clone());
-                    region_manager.evict_region(cache_range, EvictReason::Manual, None);
+                        .remove_manual_load_range(range.clone());
+                    region_manager.evict_region(range, EvictReason::Manual, None);
                     return;
                 }
 
                 region_manager
                     .regions_map()
                     .write()
-                    .add_manual_load_range(cache_range.clone());
+                    .add_manual_load_range(range.clone());
 
                 let Some(ref info_provider) = region_info_provider else {
                     warn!("ime region info provider is none, skip manual load range.");
                     return;
                 };
 
-                let start = origin_key(&cache_range.start);
-                let end = origin_end_key(&cache_range.end);
+                let start = origin_key(&range.start);
+                let end = origin_end_key(&range.end);
                 let regions = match info_provider.get_regions_in_range(start, end) {
                     Ok(r) => r,
                     Err(e) => {
@@ -332,14 +331,10 @@ impl BgWorkManager {
                 }
                 info!(
                     "ime manual load summary";
-                    "range" => ?cache_range,
+                    "range" => ?range,
                     "success" => total - failed,
                     "failed" => failed,
                 );
-                // TODO (afeinberg): This does not actually load the range. The
-                // load happens the apply thread begins to apply
-                // raft entries. To force this (for read-only
-                // use-cases) we should propose a No-Op command.
             },
         );
     }
@@ -359,7 +354,7 @@ impl BgWorkManager {
         // TODO: Spawn non-blocking tasks and make full use of the ticker.
         let interval = Duration::from_millis(100);
         let check_load_pending_interval = (|| {
-            fail_point!("background_check_load_pending_interval", |t| {
+            fail_point!("ime_background_check_load_pending_interval", |t| {
                 let t = t.unwrap().parse::<u64>().unwrap();
                 Duration::from_millis(t)
             });
@@ -379,7 +374,7 @@ impl BgWorkManager {
                             Ok(Ok(ts)) => ts,
                             err => {
                                 error!(
-                                    "ime schedule region cache engine gc failed ";
+                                    "ime schedule gc failed ";
                                     "timeout_duration" => ?tso_timeout,
                                     "error" => ?err,
                                 );
@@ -390,7 +385,7 @@ impl BgWorkManager {
                         let safe_point = TimeStamp::compose(safe_point, 0).into_inner();
                         if let Err(e) = scheduler.schedule(BackgroundTask::Gc(GcTask {safe_point})) {
                             error!(
-                                "ime schedule region cache engine gc failed";
+                                "ime schedule gc failed";
                                 "err" => ?e,
                             );
                         }
@@ -436,7 +431,7 @@ impl BgWorkManager {
                     recv(rx) -> r => {
                         if let Err(e) = r {
                             error!(
-                                "ime receive error in region cache engine gc ticker";
+                                "ime receive error in gc ticker";
                                 "err" => ?e,
                             );
                         }
@@ -589,8 +584,8 @@ impl BackgroundRunnerCore {
         delete_range_scheduler: &Scheduler<BackgroundTask>,
         safe_point: u64,
     ) -> bool {
-        fail::fail_point!("on_snapshot_load_finished");
-        fail::fail_point!("on_snapshot_load_finished2");
+        fail::fail_point!("ime_on_snapshot_load_finished");
+        fail::fail_point!("ime_on_snapshot_load_finished2");
         // We still need to check whether the snapshot is canceled during the load
         let mut regions_map = self.engine.region_manager().regions_map.write();
         let region_meta = regions_map.mut_region_meta(region.id).unwrap();
@@ -624,13 +619,13 @@ impl BackgroundRunnerCore {
         drop(regions_map);
 
         if !remove_regions.is_empty() {
-            fail::fail_point!("in_memory_engine_snapshot_load_canceled");
+            fail::fail_point!("ime_snapshot_load_canceled");
 
             if let Err(e) =
                 delete_range_scheduler.schedule_force(BackgroundTask::DeleteRegions(remove_regions))
             {
                 error!(
-                    "ime schedule delete range failed";
+                    "ime schedule delete regions failed";
                     "err" => ?e,
                 );
                 assert!(tikv_util::thread_group::is_shutdown(!cfg!(test)));
@@ -639,7 +634,7 @@ impl BackgroundRunnerCore {
             return false;
         }
 
-        fail::fail_point!("on_completes_batch_loading");
+        fail::fail_point!("ime_on_completes_batch_loading");
         true
     }
 
@@ -680,7 +675,7 @@ impl BackgroundRunnerCore {
             delete_range_scheduler.schedule_force(BackgroundTask::DeleteRegions(remove_regions))
         {
             error!(
-                "ime schedule delete range failed";
+                "ime schedule delete regions failed";
                 "err" => ?e,
             );
             assert!(tikv_util::thread_group::is_shutdown(!cfg!(test)));
@@ -718,13 +713,17 @@ impl BackgroundRunnerCore {
 
         let evict_count = regions_to_evict.len();
         let mut regions_to_delete = Vec::with_capacity(evict_count);
-        info!("ime load_evict"; "regions_to_load" => ?&regions_to_load, "regions_to_evict" => ?&regions_to_evict);
+        info!(
+            "ime load_evict";
+            "regions_to_load" => ?&regions_to_load,
+            "regions_to_evict" => ?&regions_to_evict,
+        );
         let (tx, mut rx) = mpsc::channel(evict_count + 1);
         for evict_region in regions_to_evict {
             let cache_region = CacheRegion::from_region(&evict_region);
             let tx_clone = tx.clone();
             // Bound is set to 1 so that the sender side will not be blocked
-            let deleteable_regions = self.engine.region_manager().evict_region(
+            let deletable_regions = self.engine.region_manager().evict_region(
                 &cache_region,
                 EvictReason::AutoEvict,
                 Some(Box::new(move || {
@@ -736,9 +735,9 @@ impl BackgroundRunnerCore {
             info!(
                 "ime load_evict: auto evict";
                 "region_to_evict" => ?&cache_region,
-                "evicted_regions" => ?&deleteable_regions,
+                "evicted_regions" => ?&deletable_regions,
             );
-            regions_to_delete.extend(deleteable_regions);
+            regions_to_delete.extend(deletable_regions);
         }
 
         if !regions_to_delete.is_empty() {
@@ -746,7 +745,7 @@ impl BackgroundRunnerCore {
                 .schedule_force(BackgroundTask::DeleteRegions(regions_to_delete))
             {
                 error!(
-                    "ime schedule deletet range failed";
+                    "ime schedule delete range failed";
                     "err" => ?e,
                 );
                 assert!(tikv_util::thread_group::is_shutdown(!cfg!(test)));
@@ -915,11 +914,11 @@ impl Runnable for BackgroundRunner {
         match task {
             BackgroundTask::SetRocksEngine(rocks_engine) => {
                 self.rocks_engine = Some(rocks_engine);
-                fail::fail_point!("in_memory_engine_set_rocks_engine");
+                fail::fail_point!("ime_set_rocks_engine");
             }
             BackgroundTask::Gc(t) => {
                 let seqno = (|| {
-                    fail::fail_point!("in_memory_engine_gc_oldest_seqno", |t| {
+                    fail::fail_point!("ime_gc_oldest_seqno", |t| {
                         Some(t.unwrap().parse::<u64>().unwrap())
                     });
 
@@ -939,7 +938,7 @@ impl Runnable for BackgroundRunner {
                 };
 
                 info!(
-                    "ime start a new round of gc for range cache engine";
+                    "ime start a new round of gc";
                     "safe_point" => t.safe_point,
                     "oldest_sequence" => seqno,
                 );
@@ -967,8 +966,8 @@ impl Runnable for BackgroundRunner {
                 let pd_client = self.pd_client.clone();
                 let gc_run_interval = self.config.value().gc_run_interval.0;
                 let f = async move {
-                    fail::fail_point!("before_start_loading_region");
-                    fail::fail_point!("on_start_loading_region");
+                    fail::fail_point!("ime_before_start_loading_region");
+                    fail::fail_point!("ime_on_start_loading_region");
                     let mut is_canceled = false;
                     {
                         let regions_map = core.engine.region_manager().regions_map.read();
@@ -998,7 +997,7 @@ impl Runnable for BackgroundRunner {
                         return;
                     }
 
-                    info!("ime Loading region"; "region" => ?&region);
+                    info!("ime loading region"; "region" => ?&region);
                     let start = Instant::now();
                     let iter_opt = IterOptions::new(
                         Some(KeyBuilder::from_slice(&region.start, 0, 0)),
@@ -1070,7 +1069,7 @@ impl Runnable for BackgroundRunner {
                         };
 
                         let safe_point = (|| {
-                            fail::fail_point!("in_memory_engine_safe_point_in_loading", |t| {
+                            fail::fail_point!("ime_safe_point_in_loading", |t| {
                                 t.unwrap().parse::<u64>().unwrap()
                             });
 
@@ -1100,12 +1099,12 @@ impl Runnable for BackgroundRunner {
                             let duration = start.saturating_elapsed();
                             IN_MEMORY_ENGINE_LOAD_TIME_HISTOGRAM.observe(duration.as_secs_f64());
                             info!(
-                                "ime Loading region finished";
+                                "ime loading region finished";
                                 "region" => ?region,
                                 "duration(sec)" => ?duration,
                             );
                         } else {
-                            info!("ime Loading region canceled";"region" => ?region);
+                            info!("ime loading region canceled";"region" => ?region);
                         }
                     } else {
                         info!(
@@ -1251,7 +1250,7 @@ impl Runnable for BackgroundRunner {
                         "current_count" => lock_handle.len(),
                     );
 
-                    fail::fail_point!("clean_lock_tombstone_done");
+                    fail::fail_point!("ime_clean_lock_tombstone_done");
                 };
 
                 self.lock_cleanup_remote.spawn(f);
@@ -1375,7 +1374,7 @@ impl DeleteRangeRunner {
         }
         self.engine.region_manager().on_delete_regions(regions);
 
-        fail::fail_point!("in_memory_engine_delete_range_done");
+        fail::fail_point!("ime_delete_range_done");
 
         #[cfg(test)]
         flush_epoch();
@@ -1387,7 +1386,7 @@ impl Runnable for DeleteRangeRunner {
     fn run(&mut self, task: Self::Task) {
         match task {
             BackgroundTask::DeleteRegions(regions) => {
-                fail::fail_point!("on_in_memory_engine_delete_range");
+                fail::fail_point!("ime_on_delete_range");
                 let (mut regions_to_delay, regions_to_delete) = {
                     let region_manager = self.engine.region_manager();
                     let regions_map = region_manager.regions_map.read();
@@ -1532,7 +1531,7 @@ impl Filter {
             let v = iter.value();
             if let Err(e) = self.filter_key(k.as_bytes(), v.as_bytes()) {
                 warn!(
-                    "ime Something Wrong in memory engine GC";
+                    "ime something wrong GC";
                     "error" => ?e,
                 );
             }
diff --git a/components/in_memory_engine/src/cross_check.rs b/components/in_memory_engine/src/cross_check.rs
index 778a1b00ae5..36165af1a77 100644
--- a/components/in_memory_engine/src/cross_check.rs
+++ b/components/in_memory_engine/src/cross_check.rs
@@ -227,7 +227,7 @@ impl CrossChecker {
                     Ok(write) => write,
                     Err(e) => {
                         panic!(
-                            "ime cross check fail(parse error); 
+                            "ime cross check fail(parse error);
                             cache_region={:?}; cache_key={:?}, cache_val={:?}; sequence_numer={}; Error={:?}",
                             region_snap.snapshot_meta().region,
                             log_wrappers::Value(mem_iter.key()),
@@ -272,7 +272,7 @@ impl CrossChecker {
                         Ok(write) => write,
                         Err(e) => {
                             panic!(
-                                "ime cross check fail(parse error); 
+                                "ime cross check fail(parse error);
                                 cache_region={:?}; cache_key={:?}, cache_val={:?}; sequence_numer={}; Error={:?}",
                                 region_snap.snapshot_meta().region,
                                 log_wrappers::Value(mem_iter.key()),
@@ -391,7 +391,7 @@ impl CrossChecker {
                 // CF_LOCK should always have the same view
                 if disk_key != mem_key {
                     panic!(
-                        "ime cross check fail(key not equal): lock cf not match; 
+                        "ime cross check fail(key not equal): lock cf not match;
                         cache_region={:?}; cache_key={:?}, disk_key={:?}; sequence_numer={};",
                         cached_region,
                         log_wrappers::Value(mem_key),
@@ -401,7 +401,7 @@ impl CrossChecker {
                 }
                 if mem_iter.value() != disk_iter.value() {
                     panic!(
-                        "ime cross check fail(value not equal): lock cf not match; 
+                        "ime cross check fail(value not equal): lock cf not match;
                         cache_region={:?}; key={:?}, mem_value={:?} disk_key={:?};",
                         cached_region,
                         log_wrappers::Value(mem_key),
@@ -415,7 +415,7 @@ impl CrossChecker {
             if disk_key == mem_key {
                 if mem_iter.value() != disk_iter.value() {
                     panic!(
-                        "ime cross check fail(value not equal): write cf not match; 
+                        "ime cross check fail(value not equal): write cf not match;
                         cache_region={:?}; key={:?}, mem_value={:?} disk_key={:?};",
                         cached_region,
                         log_wrappers::Value(mem_key),
@@ -433,7 +433,7 @@ impl CrossChecker {
                 Ok(write) => write,
                 Err(e) => {
                     panic!(
-                        "ime cross check fail(parse error); 
+                        "ime cross check fail(parse error);
                         cache_region={:?}; cache_key={:?}, cache_val={:?}; sequence_numer={}; Error={:?}",
                         cached_region,
                         log_wrappers::Value(mem_iter.key()),
@@ -613,7 +613,7 @@ impl CrossChecker {
             // IME and rocks enigne should have the same data view for CF_LOCK
             if *cf == CF_LOCK {
                 panic!(
-                    "ime cross check fail(key should exist): lock cf not match; 
+                    "ime cross check fail(key should exist): lock cf not match;
                     cache_region={:?}; disk_key={:?}; sequence_numer={};",
                     cached_region,
                     log_wrappers::Value(disk_iter.key()),
@@ -653,7 +653,7 @@ impl CrossChecker {
                 Ok(write) => write,
                 Err(e) => {
                     panic!(
-                        "ime cross check fail(parse error); 
+                        "ime cross check fail(parse error);
                         cache_region={:?}; cache_key={:?}, cache_val={:?}; sequence_numer={}; Error={:?}",
                         cached_region,
                         log_wrappers::Value(mem_iter.key()),
@@ -848,7 +848,7 @@ impl Runnable for CrossChecker {
             Ok(Ok(ts)) => ts,
             err => {
                 error!(
-                    "schedule range cache engine gc failed ";
+                    "ime schedule gc failed ";
                     "timeout_duration" => ?tso_timeout,
                     "error" => ?err,
                 );
@@ -870,7 +870,7 @@ impl Runnable for CrossChecker {
                     Ok(range_snap) => Some(range_snap),
                     Err(_) => {
                         warn!(
-                            "failed to get snap in cross check";
+                            "ime failed to get snap in cross check";
                             "range" => ?range,
                         );
                         None
diff --git a/components/in_memory_engine/src/engine.rs b/components/in_memory_engine/src/engine.rs
index 583b0341ac0..5128023b80e 100644
--- a/components/in_memory_engine/src/engine.rs
+++ b/components/in_memory_engine/src/engine.rs
@@ -361,7 +361,6 @@ impl RegionCacheMemoryEngine {
         region_info_provider: Option<Arc<dyn RegionInfoProvider>>,
         raft_casual_router: Option<Box<dyn CasualRouter<RocksEngine>>>,
     ) -> Self {
-        info!("ime init region cache memory engine");
         let core = Arc::new(RegionCacheMemoryEngineCore::new());
         let skiplist_engine = core.engine().clone();
 
@@ -411,15 +410,15 @@ impl RegionCacheMemoryEngine {
         evict_reason: EvictReason,
         cb: Option<Box<dyn AsyncFnOnce + Send + Sync>>,
     ) {
-        let deleteable_regions = self
+        let deletable_regions = self
             .core
             .region_manager
             .evict_region(region, evict_reason, cb);
-        if !deleteable_regions.is_empty() {
+        if !deletable_regions.is_empty() {
             // The region can be deleted directly.
             if let Err(e) = self
                 .bg_worker_manager()
-                .schedule_task(BackgroundTask::DeleteRegions(deleteable_regions))
+                .schedule_task(BackgroundTask::DeleteRegions(deletable_regions))
             {
                 error!(
                     "ime schedule delete region failed";
@@ -559,7 +558,7 @@ impl RegionCacheEngineExt for RegionCacheMemoryEngine {
                         .overlap_with_manual_load_range(&region)
                     {
                         info!(
-                            "try to load region in manual load range";
+                            "ime try to load region in manual load range";
                             "region" => ?region,
                         );
                         if let Err(e) = self.load_region(region.clone()) {
diff --git a/components/in_memory_engine/src/read.rs b/components/in_memory_engine/src/read.rs
index 44f2d76ef3a..3619c4c4555 100644
--- a/components/in_memory_engine/src/read.rs
+++ b/components/in_memory_engine/src/read.rs
@@ -137,7 +137,7 @@ impl Iterable for RegionCacheSnapshot {
             || upper_bound > self.snapshot_meta.region.end
         {
             return Err(Error::Other(box_err!(
-                "the bounderies required [{}, {}] exceeds the range of the snapshot [{}, {}]",
+                "the boundaries required [{}, {}] exceeds the range of the snapshot [{}, {}]",
                 log_wrappers::Value(&lower_bound),
                 log_wrappers::Value(&upper_bound),
                 log_wrappers::Value(&self.snapshot_meta.region.start),
@@ -177,7 +177,6 @@ impl Peekable for RegionCacheSnapshot {
         cf: &str,
         key: &[u8],
     ) -> Result<Option<Self::DbVector>> {
-        fail::fail_point!("on_region_cache_get_value");
         if !self.snapshot_meta.region.contains_key(key) {
             return Err(Error::Other(box_err!(
                 "key {} not in range[{}, {}]",
@@ -532,7 +531,7 @@ impl Iterator for RegionCacheIterator {
     }
 
     fn seek(&mut self, key: &[u8]) -> Result<bool> {
-        fail::fail_point!("on_region_cache_iterator_seek");
+        fail::fail_point!("ime_on_iterator_seek");
         let begin = Instant::now();
         self.direction = Direction::Forward;
         if let Some(ref mut extractor) = self.prefix_extractor {
diff --git a/components/in_memory_engine/src/region_manager.rs b/components/in_memory_engine/src/region_manager.rs
index e77f0c8c140..c0e14764285 100644
--- a/components/in_memory_engine/src/region_manager.rs
+++ b/components/in_memory_engine/src/region_manager.rs
@@ -206,7 +206,7 @@ impl CacheRegionMeta {
             "ime update region meta state";
             "region_id" => self.region.id,
             "epoch" => self.region.epoch_version,
-            "curr_state" => ?self.state,
+            "current_state" => ?self.state,
             "new_state" => ?new_state);
         self.state = new_state;
     }
@@ -683,11 +683,11 @@ impl RegionManager {
         Ok(())
     }
 
-    // If the snapshot is the last one in the snapshot list of one cache range in
-    // historical_regions, it means one or some evicted_ranges may be ready to be
-    // removed physically.
-    // So, we return a vector of ranges to denote the ranges that are ready to be
-    // removed.
+    // If the snapshot is the last one in the snapshot list of one cache region
+    // in historical_regions, it means one or some evicted_regions may be ready
+    // to be removed physically.
+    // So, we return a vector of ranges to denote the ranges that are ready to
+    // be removed.
     pub(crate) fn remove_region_snapshot(
         &self,
         snapshot_meta: &RegionCacheSnapshotMeta,
@@ -846,7 +846,7 @@ impl RegionManager {
             return vec![];
         }
 
-        let mut deleteable_regions = vec![];
+        let mut deletable_regions = vec![];
         for rid in evict_ids {
             if let Some(region) = self.do_evict_region(
                 rid,
@@ -859,10 +859,10 @@ impl RegionManager {
                     None
                 },
             ) {
-                deleteable_regions.push(region);
+                deletable_regions.push(region);
             }
         }
-        deleteable_regions
+        deletable_regions
     }
 
     // return the region if it can be directly deleted.
@@ -885,7 +885,7 @@ impl RegionManager {
         if prev_state == RegionState::Pending {
             let meta = regions_map.remove_region(id);
             info!(
-                "ime evict overlap pending region in cache range engine";
+                "ime evict overlap pending region";
                 "reason" => ?evict_reason,
                 "target_region" => ?evict_region,
                 "overlap_region" => ?meta.region,
@@ -905,7 +905,7 @@ impl RegionManager {
         };
 
         info!(
-            "ime evict overlap region in cache range engine";
+            "ime evict overlap region";
             "reason" => ?evict_reason,
             "target_region" => ?evict_region,
             "overlap_region" => ?meta.region,
@@ -926,7 +926,7 @@ impl RegionManager {
     }
 
     pub fn on_delete_regions(&self, regions: &[CacheRegion]) {
-        fail::fail_point!("in_memory_engine_on_delete_regions");
+        fail::fail_point!("ime_on_delete_regions");
         let mut cbs = vec![];
         {
             let mut regions_map = self.regions_map.write();
diff --git a/components/in_memory_engine/src/region_stats.rs b/components/in_memory_engine/src/region_stats.rs
index 013a56e2c9c..5f81dc82860 100644
--- a/components/in_memory_engine/src/region_stats.rs
+++ b/components/in_memory_engine/src/region_stats.rs
@@ -220,7 +220,7 @@ impl RegionStatsManager {
         };
 
         info!(
-            "IME mvcc amplification reduction filter";
+            "ime mvcc amplification reduction filter";
             "mvcc_amplification_to_filter" => mvcc_amplification_to_filter,
         );
 
@@ -377,7 +377,7 @@ impl RegionStatsManager {
         // Evict two regions each time to reduce the probability that an un-dropped
         // ongoing snapshot blocks the process
         for regions in evict_candidates.chunks(2) {
-            let mut deleteable_regions = vec![];
+            let mut deletable_regions = vec![];
 
             let (tx, mut rx) = mpsc::channel(3);
             for r in regions {
@@ -387,7 +387,7 @@ impl RegionStatsManager {
                 );
 
                 let tx_clone = tx.clone();
-                deleteable_regions.extend(evict_region(
+                deletable_regions.extend(evict_region(
                     &CacheRegion::from_region(r),
                     EvictReason::MemoryLimitReached,
                     // This callback will be executed when eviction finishes at `on_delete_regions`
@@ -400,12 +400,12 @@ impl RegionStatsManager {
                 ));
                 self.handle_region_evicted(r);
             }
-            if !deleteable_regions.is_empty() {
+            if !deletable_regions.is_empty() {
                 if let Err(e) = delete_range_scheduler
-                    .schedule_force(BackgroundTask::DeleteRegions(deleteable_regions))
+                    .schedule_force(BackgroundTask::DeleteRegions(deletable_regions))
                 {
                     error!(
-                        "ime schedule deletet range failed";
+                        "ime schedule delete regions failed";
                         "err" => ?e,
                     );
                     assert!(tikv_util::thread_group::is_shutdown(!cfg!(test)));
diff --git a/components/in_memory_engine/src/write_batch.rs b/components/in_memory_engine/src/write_batch.rs
index 3c0a15c7c2f..4a654ec2d53 100644
--- a/components/in_memory_engine/src/write_batch.rs
+++ b/components/in_memory_engine/src/write_batch.rs
@@ -171,7 +171,7 @@ impl RegionCacheWriteBatch {
         // record last region before flush.
         self.record_last_written_region();
 
-        fail::fail_point!("on_region_cache_write_batch_write_impl");
+        fail::fail_point!("ime_on_region_cache_write_batch_write_impl");
         let guard = &epoch::pin();
         let start = Instant::now();
         let mut lock_modification: u64 = 0;
@@ -188,8 +188,8 @@ impl RegionCacheWriteBatch {
         let duration = start.saturating_elapsed_secs();
         IN_MEMORY_ENGINE_WRITE_DURATION_HISTOGRAM.observe(duration);
 
-        fail::fail_point!("in_memory_engine_write_batch_consumed");
-        fail::fail_point!("before_clear_ranges_in_being_written");
+        fail::fail_point!("ime_on_region_cache_write_batch_write_consumed");
+        fail::fail_point!("ime_before_clear_regions_in_being_written");
 
         if !self.written_regions.is_empty() {
             self.engine
@@ -234,14 +234,14 @@ impl RegionCacheWriteBatch {
 
         if !self.engine.enabled() {
             let region = self.current_region.as_ref().unwrap();
-            info!("ime range cache is disabled, evict the range"; "region" => ?region);
+            info!("ime is disabled, evict the range"; "region" => ?region);
             self.evict_current_region(EvictReason::Disabled);
             return;
         }
         let memory_expect = entry_size();
         if !self.memory_acquire(memory_expect) {
             let region = self.current_region.as_ref().unwrap();
-            info!("ime memory acquire failed due to reaching hard limit"; "region" => ?region);
+            info!("ime memory acquire failed due to reaches capacity"; "region" => ?region);
             self.evict_current_region(EvictReason::MemoryLimitReached);
             return;
         }
@@ -268,13 +268,13 @@ impl RegionCacheWriteBatch {
         }
     }
 
-    // return false means the memory usage reaches to hard limit and we have no
+    // return false means the memory usage reaches to capacity and we have no
     // quota to write to the engine
     fn memory_acquire(&mut self, mem_required: usize) -> bool {
         match self.memory_controller.acquire(mem_required) {
             MemoryUsage::CapacityReached(n) => {
                 warn!(
-                    "ime the memory usage of in-memory engine reaches to hard limit";
+                    "ime the memory usage reaches capacity";
                     "region" => ?self.current_region.as_ref().unwrap(),
                     "memory_usage(MB)" => ReadableSize(n as u64).as_mb_f64(),
                 );
@@ -908,20 +908,20 @@ mod tests {
             .snapshot(CacheRegion::from_region(&r1), 1000, 1000)
             .unwrap();
 
-        // disable the range cache
+        // disable the ime
         let mut config_manager = InMemoryEngineConfigManager(config.clone());
         let mut config_change = ConfigChange::new();
         config_change.insert(String::from("enable"), ConfigValue::Bool(false));
         config_manager.dispatch(config_change).unwrap();
 
         wb.write_impl(1000).unwrap();
-        // existing snapshot can still work after the range cache is disabled, but new
+        // existing snapshot can still work after the ime is disabled, but new
         // snapshot will fail to create
         assert!(snap1.get_value(b"zkk00").unwrap().is_none());
 
         let mut wb = RegionCacheWriteBatch::from(&engine);
+        // put should trigger the evict and it won't write into ime
         wb.prepare_for_region(&r1);
-        // put should trigger the evict and it won't write into range cache
         wb.put(b"zkk01", &val1).unwrap();
         wb.write_impl(1000).unwrap();
 
@@ -931,10 +931,10 @@ mod tests {
         let snap2 = engine
             .snapshot(CacheRegion::from_region(&r2), 1000, 1000)
             .unwrap();
-        // if no new write, the range cache can still be used.
+        // if no new write, the ime can still be used.
         assert_eq!(snap2.get_value(b"zkk11").unwrap().unwrap(), &val1);
 
-        // enable the range cache again
+        // enable the ime again
         let mut config_manager = InMemoryEngineConfigManager(config.clone());
         let mut config_change = ConfigChange::new();
         config_change.insert(String::from("enable"), ConfigValue::Bool(true));
diff --git a/components/in_memory_engine/tests/failpoints/test_memory_engine.rs b/components/in_memory_engine/tests/failpoints/test_memory_engine.rs
index 8e4f62e2d2c..84bce6bdd64 100644
--- a/components/in_memory_engine/tests/failpoints/test_memory_engine.rs
+++ b/components/in_memory_engine/tests/failpoints/test_memory_engine.rs
@@ -30,7 +30,7 @@ use txn_types::{Key, TimeStamp, WriteType};
 #[test]
 fn test_set_disk_engine() {
     let (tx, rx) = sync_channel(0);
-    fail::cfg_callback("in_memory_engine_set_rocks_engine", move || {
+    fail::cfg_callback("ime_set_rocks_engine", move || {
         let _ = tx.send(true);
     })
     .unwrap();
@@ -74,7 +74,7 @@ fn test_gc_worker() {
     let write = skip_engine.cf_handle(CF_WRITE);
     let default = skip_engine.cf_handle(CF_DEFAULT);
 
-    fail::cfg("in_memory_engine_gc_oldest_seqno", "return(1000)").unwrap();
+    fail::cfg("ime_gc_oldest_seqno", "return(1000)").unwrap();
 
     let (tx, rx) = sync_channel(0);
     fail::cfg_callback("in_memory_engine_gc_finish", move || {
@@ -165,7 +165,7 @@ fn test_clean_up_tombstone() {
     let engine = RegionCacheMemoryEngine::new(InMemoryEngineContext::new_for_tests(config.clone()));
     let region = new_region(1, b"".to_vec(), b"z".to_vec());
     let (tx, rx) = sync_channel(0);
-    fail::cfg_callback("clean_lock_tombstone_done", move || {
+    fail::cfg_callback("ime_clean_lock_tombstone_done", move || {
         tx.send(true).unwrap();
     })
     .unwrap();
@@ -256,14 +256,14 @@ fn test_evict_with_loading_range() {
     // range1 and range2 will be evicted
     let r = new_region(4, b"k05".to_vec(), b"k25".to_vec());
     let engine_clone = engine.clone();
-    fail::cfg_callback("on_snapshot_load_finished", move || {
+    fail::cfg_callback("ime_on_snapshot_load_finished", move || {
         let _ = snapshot_load_tx.send(true);
         engine_clone.evict_region(&CacheRegion::from_region(&r), EvictReason::AutoEvict, None);
     })
     .unwrap();
 
     let (loading_complete_tx, loading_complete_rx) = sync_channel(0);
-    fail::cfg_callback("on_completes_batch_loading", move || {
+    fail::cfg_callback("ime_on_completes_batch_loading", move || {
         let _ = loading_complete_tx.send(true);
     })
     .unwrap();
@@ -316,12 +316,12 @@ fn test_cached_write_batch_cleared_when_load_failed() {
     engine.set_disk_engine(rocks_engine);
 
     let (tx, rx) = sync_channel(0);
-    fail::cfg_callback("on_snapshot_load_finished", move || {
+    fail::cfg_callback("ime_on_snapshot_load_finished", move || {
         let _ = tx.send(true);
     })
     .unwrap();
 
-    fail::cfg("on_snapshot_load_finished2", "pause").unwrap();
+    fail::cfg("ime_on_snapshot_load_finished2", "pause").unwrap();
 
     // range1 will be canceled in on_snapshot_load_finished whereas range2 will be
     // canceled at begin
@@ -344,7 +344,7 @@ fn test_cached_write_batch_cleared_when_load_failed() {
     wb.set_sequence_number(100).unwrap();
     wb.write().unwrap();
 
-    fail::remove("on_snapshot_load_finished2");
+    fail::remove("ime_on_snapshot_load_finished2");
 
     test_util::eventually(
         Duration::from_millis(100),
@@ -380,25 +380,28 @@ fn test_concurrency_between_delete_range_and_write_to_memory() {
     let r2 = new_region(2, b"k20".to_vec(), b"k30".to_vec());
     let r3 = new_region(3, b"k40".to_vec(), b"k50".to_vec());
     let (snapshot_load_cancel_tx, snapshot_load_cancel_rx) = sync_channel(0);
-    fail::cfg_callback("in_memory_engine_snapshot_load_canceled", move || {
+    fail::cfg_callback("ime_snapshot_load_canceled", move || {
         let _ = snapshot_load_cancel_tx.send(true);
     })
     .unwrap();
     let (snapshot_load_tx, snapshot_load_rx) = sync_channel(0);
-    fail::cfg_callback("on_snapshot_load_finished", move || {
+    fail::cfg_callback("ime_on_snapshot_load_finished", move || {
         let _ = snapshot_load_tx.send(true);
     })
     .unwrap();
-    fail::cfg("before_clear_ranges_in_being_written", "pause").unwrap();
+    fail::cfg("ime_before_clear_regions_in_being_written", "pause").unwrap();
 
     let (write_batch_consume_tx, write_batch_consume_rx) = sync_channel(0);
-    fail::cfg_callback("in_memory_engine_write_batch_consumed", move || {
-        let _ = write_batch_consume_tx.send(true);
-    })
+    fail::cfg_callback(
+        "ime_on_region_cache_write_batch_write_consumed",
+        move || {
+            let _ = write_batch_consume_tx.send(true);
+        },
+    )
     .unwrap();
 
     let (delete_range_tx, delete_range_rx) = sync_channel(0);
-    fail::cfg_callback("in_memory_engine_delete_range_done", move || {
+    fail::cfg_callback("ime_delete_range_done", move || {
         let _ = delete_range_tx.send(true);
     })
     .unwrap();
@@ -471,7 +474,7 @@ fn test_concurrency_between_delete_range_and_write_to_memory() {
     // the data has not be deleted
     verify_data(&r1, 3);
     // remove failpoint so that the range can leave write status
-    fail::remove("before_clear_ranges_in_being_written");
+    fail::remove("ime_before_clear_regions_in_being_written");
     delete_range_rx
         .recv_timeout(Duration::from_secs(5))
         .unwrap();
@@ -479,13 +482,13 @@ fn test_concurrency_between_delete_range_and_write_to_memory() {
     verify_data(&r1, 0);
 
     // Next to test range2
-    fail::cfg("before_clear_ranges_in_being_written", "pause").unwrap();
+    fail::cfg("ime_before_clear_regions_in_being_written", "pause").unwrap();
     write_batch_consume_rx
         .recv_timeout(Duration::from_secs(5))
         .unwrap();
     verify_data(&r2, 2);
     // remove failpoint so that the range can leave write status
-    fail::remove("before_clear_ranges_in_being_written");
+    fail::remove("ime_before_clear_regions_in_being_written");
     delete_range_rx
         .recv_timeout(Duration::from_secs(5))
         .unwrap();
@@ -497,7 +500,7 @@ fn test_concurrency_between_delete_range_and_write_to_memory() {
         .unwrap();
     engine.evict_region(&CacheRegion::from_region(&r3), EvictReason::AutoEvict, None);
 
-    fail::cfg("before_clear_ranges_in_being_written", "pause").unwrap();
+    fail::cfg("ime_before_clear_regions_in_being_written", "pause").unwrap();
     write_batch_consume_rx
         .recv_timeout(Duration::from_secs(5))
         .unwrap();
@@ -505,7 +508,7 @@ fn test_concurrency_between_delete_range_and_write_to_memory() {
     snapshot_load_cancel_rx
         .recv_timeout(Duration::from_secs(5))
         .unwrap();
-    fail::remove("before_clear_ranges_in_being_written");
+    fail::remove("ime_before_clear_regions_in_being_written");
     delete_range_rx
         .recv_timeout(Duration::from_secs(5))
         .unwrap();
@@ -532,7 +535,7 @@ fn test_double_delete_range_schedule() {
     let (snapshot_load_tx, snapshot_load_rx) = sync_channel(0);
     let engine_clone = engine.clone();
     let r = new_region(4, b"k00", b"k60");
-    fail::cfg_callback("on_snapshot_load_finished", move || {
+    fail::cfg_callback("ime_on_snapshot_load_finished", move || {
         let _ = snapshot_load_tx.send(true);
         // evict all ranges. So the loading ranges will also be evicted and a delete
         // range task will be scheduled.
@@ -541,7 +544,7 @@ fn test_double_delete_range_schedule() {
     .unwrap();
 
     let (delete_range_tx, delete_range_rx) = sync_channel(0);
-    fail::cfg_callback("on_in_memory_engine_delete_range", move || {
+    fail::cfg_callback("ime_on_delete_range", move || {
         let _ = delete_range_tx.send(true);
     })
     .unwrap();
@@ -614,9 +617,9 @@ fn test_load_with_gc() {
     put_data_in_rocks(b"k4", b"val", 6, 0, false, &rocks_engine, WriteType::Delete);
     put_data_in_rocks(b"k4", b"val", 5, 0, false, &rocks_engine, WriteType::Put);
 
-    fail::cfg("in_memory_engine_safe_point_in_loading", "return(6)").unwrap();
+    fail::cfg("ime_safe_point_in_loading", "return(6)").unwrap();
     let (load_tx, load_rx) = sync_channel(0);
-    fail::cfg_callback("on_completes_batch_loading", move || {
+    fail::cfg_callback("ime_on_completes_batch_loading", move || {
         let _ = load_tx.send(true);
     })
     .unwrap();
@@ -669,11 +672,11 @@ fn test_region_split_before_batch_loading_start() {
     engine.set_disk_engine(rocks_engine.clone());
 
     let (tx, rx) = sync_channel(0);
-    fail::cfg_callback("before_start_loading_region", move || {
+    fail::cfg_callback("ime_before_start_loading_region", move || {
         let _ = tx.send(());
     })
     .unwrap();
-    fail::cfg("on_start_loading_region", "pause").unwrap();
+    fail::cfg("ime_on_start_loading_region", "pause").unwrap();
 
     let region = new_region(1, b"k00", b"k30");
     let cache_region = CacheRegion::from_region(&region);
@@ -739,7 +742,7 @@ fn test_region_split_before_batch_loading_start() {
     }
 
     // unblock batch loading.
-    fail::remove("on_start_loading_region");
+    fail::remove("ime_on_start_loading_region");
 
     // all new regions should be active after batch loading task finished.
     test_util::eventually(
@@ -772,7 +775,7 @@ fn test_cb_on_eviction() {
     wb.put(b"c", b"val3").unwrap();
     wb.write().unwrap();
 
-    fail::cfg("in_memory_engine_on_delete_regions", "pause").unwrap();
+    fail::cfg("ime_on_delete_regions", "pause").unwrap();
 
     let (tx, rx) = mpsc::channel(1);
     engine.evict_region(
@@ -796,7 +799,7 @@ fn test_cb_on_eviction() {
             .await
             .unwrap_err()
     });
-    fail::remove("in_memory_engine_on_delete_regions");
+    fail::remove("ime_on_delete_regions");
     rt.block_on(async move { rx.lock().await.recv().await.unwrap() });
 
     {
diff --git a/tests/failpoints/cases/mod.rs b/tests/failpoints/cases/mod.rs
index ba78c70887f..ca6770b046f 100644
--- a/tests/failpoints/cases/mod.rs
+++ b/tests/failpoints/cases/mod.rs
@@ -17,6 +17,7 @@ mod test_gc_metrics;
 mod test_gc_worker;
 mod test_hibernate;
 mod test_import_service;
+mod test_in_memory_engine;
 mod test_kv_service;
 mod test_life;
 mod test_local_read;
@@ -28,7 +29,6 @@ mod test_pd_client_legacy;
 mod test_pending_peers;
 mod test_rawkv;
 mod test_read_execution_tracker;
-mod test_region_cache_engine;
 mod test_replica_read;
 mod test_replica_stale_read;
 mod test_server;
diff --git a/tests/failpoints/cases/test_region_cache_engine.rs b/tests/failpoints/cases/test_in_memory_engine.rs
similarity index 87%
rename from tests/failpoints/cases/test_region_cache_engine.rs
rename to tests/failpoints/cases/test_in_memory_engine.rs
index 37b89b354f2..f4e7750192f 100644
--- a/tests/failpoints/cases/test_region_cache_engine.rs
+++ b/tests/failpoints/cases/test_in_memory_engine.rs
@@ -1,3 +1,5 @@
+// Copyright 2024 TiKV Project Authors. Licensed under Apache-2.0.
+
 use std::{
     fs::File,
     io::Read,
@@ -48,6 +50,7 @@ fn copr_point_get(
     let key_range = table.get_record_range_one(row_id);
     let req = DagSelect::from(table)
         .key_ranges(vec![key_range])
+        .start_ts(get_tso(&cluster.pd_client).into())
         .build_with(ctx, &[0]);
     let cop_resp = handle_request(&endpoint, req);
     assert!(!cop_resp.has_region_error(), "{:?}", cop_resp);
@@ -98,10 +101,39 @@ fn must_copr_load_data(cluster: &mut Cluster<ServerCluster>, table: &ProductTabl
         &[(row_id, Some(&format!("name:{}", row_id)), row_id)],
         true,
         &cluster.cfg.tikv.server,
-        None,
+        Some(cluster.pd_client.clone()),
     );
 }
 
+fn async_put(
+    cluster: &mut Cluster<ServerCluster>,
+    table: &ProductTable,
+    row_id: i64,
+) -> std::thread::JoinHandle<()> {
+    let cfg = cluster.cfg.tikv.server.clone();
+    let pd_client = cluster.pd_client.clone();
+    let key = table.get_table_prefix();
+    let split_key = Key::from_raw(&key).into_encoded();
+    let ctx = cluster.get_ctx(&split_key);
+    let engine = cluster.sim.rl().storages[&ctx.get_peer().get_store_id()].clone();
+    let table_ = table.clone();
+    let (tx, rx) = unbounded();
+    let handle = std::thread::spawn(move || {
+        tx.send(()).unwrap();
+        let _ = init_data_with_details_pd_client(
+            ctx,
+            engine,
+            &table_,
+            &[(row_id, Some(&format!("name:{}", row_id)), row_id)],
+            true,
+            &cfg,
+            Some(pd_client),
+        );
+    });
+    rx.recv_timeout(Duration::from_secs(5)).unwrap();
+    handle
+}
+
 #[test]
 fn test_put_copr_get() {
     let mut cluster = new_server_cluster_with_hybrid_engine_with_no_region_cache(0, 1);
@@ -127,7 +159,7 @@ fn test_put_copr_get() {
     let product = ProductTable::new();
     must_copr_load_data(&mut cluster, &product, 1);
     let (tx, rx) = unbounded();
-    fail::cfg_callback("on_region_cache_iterator_seek", move || {
+    fail::cfg_callback("ime_on_iterator_seek", move || {
         tx.send(true).unwrap();
     })
     .unwrap();
@@ -161,7 +193,7 @@ fn test_load() {
     }
 
     let (tx, rx) = unbounded();
-    fail::cfg_callback("on_snapshot_load_finished", move || {
+    fail::cfg_callback("ime_on_snapshot_load_finished", move || {
         tx.send(true).unwrap();
     })
     .unwrap();
@@ -200,7 +232,7 @@ fn test_load() {
     rx.recv_timeout(Duration::from_secs(5)).unwrap();
 
     let (tx, rx) = unbounded();
-    fail::cfg_callback("on_region_cache_iterator_seek", move || {
+    fail::cfg_callback("ime_on_iterator_seek", move || {
         tx.send(true).unwrap();
     })
     .unwrap();
@@ -225,7 +257,7 @@ fn test_load_with_split() {
     // let channel to make load process block at finishing loading snapshot
     let (tx2, rx2) = sync_channel(0);
     let rx2 = Arc::new(Mutex::new(rx2));
-    fail::cfg_callback("on_snapshot_load_finished", move || {
+    fail::cfg_callback("ime_on_snapshot_load_finished", move || {
         tx.send(true).unwrap();
         let _ = rx2.lock().unwrap().recv().unwrap();
     })
@@ -271,7 +303,7 @@ fn test_load_with_split() {
     tx2.send(true).unwrap();
 
     let (tx, rx) = unbounded();
-    fail::cfg_callback("on_region_cache_iterator_seek", move || {
+    fail::cfg_callback("ime_on_iterator_seek", move || {
         tx.send(true).unwrap();
     })
     .unwrap();
@@ -320,30 +352,7 @@ fn test_load_with_split2() {
         let _ = handle_put_pause_rx.recv();
     })
     .unwrap();
-    let mut async_put = |table: &ProductTable, row_id| {
-        let engine = cluster.sim.rl().storages[&1].clone();
-        let cfg = cluster.cfg.tikv.server.clone();
-        let key = table.get_table_prefix();
-        let split_key = Key::from_raw(&key).into_encoded();
-        let ctx = cluster.get_ctx(&split_key);
-        let table_ = table.clone();
-        let (tx, rx) = unbounded();
-        let handle = std::thread::spawn(move || {
-            tx.send(()).unwrap();
-            let _ = init_data_with_details_pd_client(
-                ctx,
-                engine,
-                &table_,
-                &[(row_id, Some(&format!("name:{}", row_id)), row_id)],
-                true,
-                &cfg,
-                None,
-            );
-        });
-        rx.recv_timeout(Duration::from_secs(5)).unwrap();
-        handle
-    };
-    let handle1 = async_put(&product1, 2);
+    let handle1 = async_put(&mut cluster, &product1, 2);
     handle_put_rx.recv_timeout(Duration::from_secs(5)).unwrap();
 
     std::thread::sleep(Duration::from_secs(1));
@@ -356,12 +365,12 @@ fn test_load_with_split2() {
         .unwrap();
 
     let (tx, rx) = sync_channel(1);
-    fail::cfg_callback("on_snapshot_load_finished", move || {
+    fail::cfg_callback("ime_on_snapshot_load_finished", move || {
         tx.send(true).unwrap();
     })
     .unwrap();
 
-    let handle2 = async_put(&product2, 9);
+    let handle2 = async_put(&mut cluster, &product2, 9);
     let _ = rx.recv_timeout(Duration::from_secs(5)).unwrap();
 
     drop(handle_put_pause_tx);
@@ -384,7 +393,7 @@ fn test_load_with_split2() {
     handle2.join().unwrap();
 
     let (tx, rx) = unbounded();
-    fail::cfg_callback("on_region_cache_iterator_seek", move || {
+    fail::cfg_callback("ime_on_iterator_seek", move || {
         tx.send(true).unwrap();
     })
     .unwrap();
@@ -394,8 +403,8 @@ fn test_load_with_split2() {
     // ["", table2) should not cached.
     must_copr_load_data(&mut cluster, &product1, 3);
     let (tx, rx) = unbounded();
-    fail::remove("on_region_cache_iterator_seek");
-    fail::cfg_callback("on_region_cache_iterator_seek", move || {
+    fail::remove("ime_on_iterator_seek");
+    fail::cfg_callback("ime_on_iterator_seek", move || {
         tx.send(true).unwrap();
     })
     .unwrap();
@@ -415,7 +424,8 @@ fn test_load_with_eviction() {
     let region_cache_engine = cluster.sim.rl().get_region_cache_engine(1);
     // Load the whole range as if it is not splitted. Loading process should handle
     // it correctly.
-    let cache_range = CacheRegion::new(1, 0, DATA_MIN_KEY, DATA_MAX_KEY);
+    let r = cluster.get_region(b"");
+    let cache_range = CacheRegion::from_region(&r);
     region_cache_engine
         .core()
         .region_manager()
@@ -430,29 +440,9 @@ fn test_load_with_eviction() {
     let r = cluster.get_region(&split_key);
     cluster.must_split(&r, &split_key);
 
-    fail::cfg("on_region_cache_write_batch_write_impl", "pause").unwrap();
-    let mut async_put = |table: &ProductTable, row_id| {
-        let engine = cluster.sim.rl().storages[&1].clone();
-        let cfg = cluster.cfg.tikv.server.clone();
-        let pd_client = cluster.pd_client.clone();
-        let key = table.get_table_prefix();
-        let split_key = Key::from_raw(&key).into_encoded();
-        let ctx = cluster.get_ctx(&split_key);
-        let table_ = table.clone();
-        std::thread::spawn(move || {
-            let _ = init_data_with_details_pd_client(
-                ctx,
-                engine,
-                &table_,
-                &[(row_id, Some(&format!("name:{}", row_id)), row_id)],
-                true,
-                &cfg,
-                Some(pd_client),
-            );
-        })
-    };
-    let handle1 = async_put(&product1, 1);
-    let handle2 = async_put(&product2, 15);
+    fail::cfg("ime_on_region_cache_write_batch_write_impl", "pause").unwrap();
+    let handle1 = async_put(&mut cluster, &product1, 1);
+    let handle2 = async_put(&mut cluster, &product2, 1);
 
     {
         let region_cache_engine = cluster.sim.rl().get_region_cache_engine(1);
@@ -474,28 +464,19 @@ fn test_load_with_eviction() {
         );
     }
 
-    fail::remove("on_region_cache_write_batch_write_impl");
+    fail::remove("ime_on_region_cache_write_batch_write_impl");
     handle1.join().unwrap();
     handle2.join().unwrap();
 
     for (table, is_cached) in &[(product1, true), (product2, false)] {
-        fail::remove("on_region_cache_iterator_seek");
+        fail::remove("ime_on_iterator_seek");
         let (tx, rx) = unbounded();
-        fail::cfg_callback("on_region_cache_iterator_seek", move || {
+        fail::cfg_callback("ime_on_iterator_seek", move || {
             tx.send(true).unwrap();
         })
         .unwrap();
 
-        let key = table.get_table_prefix();
-        let table_key = Key::from_raw(&key).into_encoded();
-        let ctx = cluster.get_ctx(&table_key);
-        let endpoint = cluster.sim.rl().copr_endpoints[&1].clone();
-        let req = DagSelect::from(table).build_with(ctx, &[0]);
-        let cop_resp = handle_request(&endpoint, req);
-        let mut resp = SelectResponse::default();
-        resp.merge_from_bytes(cop_resp.get_data()).unwrap();
-        assert!(!cop_resp.has_region_error(), "{:?}", cop_resp);
-        assert!(cop_resp.get_other_error().is_empty(), "{:?}", cop_resp);
+        must_copr_point_get(&mut cluster, table, 1);
 
         if *is_cached {
             rx.try_recv().unwrap();
@@ -682,7 +663,7 @@ fn test_pre_load_when_transfer_ledaer() {
     cluster.run();
 
     let (tx, rx) = unbounded();
-    fail::cfg_callback("on_completes_batch_loading", move || {
+    fail::cfg_callback("ime_on_completes_batch_loading", move || {
         tx.send(true).unwrap();
     })
     .unwrap();
@@ -708,7 +689,7 @@ fn test_pre_load_when_transfer_ledaer() {
 
 #[test]
 fn test_background_loading_pending_region() {
-    fail::cfg("background_check_load_pending_interval", "return(1000)").unwrap();
+    fail::cfg("ime_background_check_load_pending_interval", "return(1000)").unwrap();
 
     let mut cluster = new_server_cluster_with_hybrid_engine_with_no_region_cache(0, 1);
     cluster.run();
@@ -737,7 +718,7 @@ fn test_delete_range() {
         cluster.run();
 
         let (tx, rx) = sync_channel(0);
-        fail::cfg_callback("on_snapshot_load_finished", move || {
+        fail::cfg_callback("ime_on_snapshot_load_finished", move || {
             tx.send(true).unwrap();
         })
         .unwrap();
@@ -759,7 +740,7 @@ fn test_delete_range() {
         rx.recv_timeout(Duration::from_secs(5)).unwrap();
 
         let (tx, rx) = unbounded();
-        fail::cfg_callback("on_region_cache_iterator_seek", move || {
+        fail::cfg_callback("ime_on_iterator_seek", move || {
             tx.send(true).unwrap();
         })
         .unwrap();
@@ -826,7 +807,7 @@ fn test_evict_on_flashback() {
     cluster.must_send_wait_flashback_msg(r.id, AdminCmdType::FinishFlashback);
 
     let (tx, rx) = unbounded();
-    fail::cfg_callback("on_region_cache_iterator_seek", move || {
+    fail::cfg_callback("ime_on_iterator_seek", move || {
         tx.send(true).unwrap();
     })
     .unwrap();
@@ -843,7 +824,7 @@ fn test_evict_on_flashback() {
 
 #[test]
 fn test_load_during_flashback() {
-    fail::cfg("background_check_load_pending_interval", "return(1000)").unwrap();
+    fail::cfg("ime_background_check_load_pending_interval", "return(1000)").unwrap();
     let mut cluster = new_server_cluster_with_hybrid_engine_with_no_region_cache(0, 1);
     cluster.cfg.raft_store.apply_batch_system.pool_size = 1;
     cluster.run();
diff --git a/tests/failpoints/cases/test_merge.rs b/tests/failpoints/cases/test_merge.rs
index 998f0dcc632..a9da6b1a233 100644
--- a/tests/failpoints/cases/test_merge.rs
+++ b/tests/failpoints/cases/test_merge.rs
@@ -2202,7 +2202,6 @@ fn test_destroy_race_during_atomic_snapshot_after_merge() {
         .when(left_filter_block.clone())
         .reserve_dropped(left_blocked_messages.clone())
         .set_msg_callback(Arc::new(move |msg: &RaftMessage| {
-            debug!("dbg left msg_callback"; "msg" => ?msg);
             if left_filter_block.load(atomic::Ordering::SeqCst) {
                 return;
             }
@@ -2226,7 +2225,6 @@ fn test_destroy_race_during_atomic_snapshot_after_merge() {
         .direction(Direction::Recv)
         .when(right_filter_block.clone())
         .set_msg_callback(Arc::new(move |msg: &RaftMessage| {
-            debug!("dbg right msg_callback"; "msg" => ?msg);
             if msg.get_to_peer().get_id() == new_peer_id {
                 let _ = new_peer_id_tx.lock().unwrap().take().map(|tx| tx.send(()));
                 if msg.get_message().get_msg_type() == MessageType::MsgSnapshot {

From d621ed9e4ea5df5fa4a98a64fbf8be61e09236db Mon Sep 17 00:00:00 2001
From: Spade  A <71589810+SpadeA-Tang@users.noreply.github.com>
Date: Tue, 22 Oct 2024 15:34:47 +0800
Subject: [PATCH 02/16] In-memory engine: fix data inconsistency due to
 unflushed data that exists when schedule load (#17685)

close tikv/tikv#17645

fix data inconsistency due to unflushed data that exists when schedule load

Signed-off-by: SpadeA-Tang <tangchenjie1210@gmail.com>

Co-authored-by: ti-chi-bot[bot] <108142056+ti-chi-bot[bot]@users.noreply.github.com>
---
 Cargo.lock                                    |  1 +
 components/in_memory_engine/Cargo.toml        |  1 +
 .../in_memory_engine/src/region_manager.rs    |  2 +-
 .../in_memory_engine/src/write_batch.rs       | 53 +++++++++++++++++++
 components/test_raftstore/src/util.rs         |  6 +++
 5 files changed, 62 insertions(+), 1 deletion(-)

diff --git a/Cargo.lock b/Cargo.lock
index a3535198106..8b61b31fb13 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -2589,6 +2589,7 @@ dependencies = [
  "serde_json",
  "slog",
  "slog-global",
+ "smallvec",
  "strum 0.20.0",
  "tempfile",
  "test_pd",
diff --git a/components/in_memory_engine/Cargo.toml b/components/in_memory_engine/Cargo.toml
index 820edc0c0f6..7bef323b930 100644
--- a/components/in_memory_engine/Cargo.toml
+++ b/components/in_memory_engine/Cargo.toml
@@ -48,6 +48,7 @@ online_config = { workspace = true }
 libc = "0.2"
 rand = "0.8"
 tokio = { version = "1.5", features = ["rt-multi-thread"] }
+smallvec = "1.4"
 
 [dev-dependencies]
 tempfile = "3.0"
diff --git a/components/in_memory_engine/src/region_manager.rs b/components/in_memory_engine/src/region_manager.rs
index c0e14764285..2c11a8b398b 100644
--- a/components/in_memory_engine/src/region_manager.rs
+++ b/components/in_memory_engine/src/region_manager.rs
@@ -1045,7 +1045,7 @@ pub enum LoadFailedReason {
     Evicting,
 }
 
-#[derive(PartialEq, Debug)]
+#[derive(PartialEq, Debug, Clone, Copy)]
 pub enum RegionCacheStatus {
     NotInCache,
     Cached,
diff --git a/components/in_memory_engine/src/write_batch.rs b/components/in_memory_engine/src/write_batch.rs
index 4a654ec2d53..b91bdb2e1c0 100644
--- a/components/in_memory_engine/src/write_batch.rs
+++ b/components/in_memory_engine/src/write_batch.rs
@@ -12,6 +12,7 @@ use engine_traits::{
     WriteBatchExt, WriteOptions, CF_DEFAULT,
 };
 use kvproto::metapb;
+use smallvec::SmallVec;
 use tikv_util::{box_err, config::ReadableSize, error, info, time::Instant, warn};
 
 use crate::{
@@ -67,6 +68,15 @@ pub struct RegionCacheWriteBatch {
 
     // record the total durations of the prepare work for write in the write batch
     prepare_for_write_duration: Duration,
+
+    // Now, we have an assumption that in one round of batch system process (PollHandler::begin ->
+    // ... -> PollHandler::end), although the same region can call `prepare_for_region`
+    // multiple times, it can only call sequentially. This is say, we will not have this:
+    // prepare_for_region(region1), prepare_for_region(region2), prepare_for_region(region1).
+    // In case to avoid this asssumption being broken, we record the regions that have called
+    // prepare_for_region and ensure that if the region is not the `currnet_region`, it is not
+    // recorded in this vec.
+    prepared_regions: SmallVec<[u64; 10]>,
 }
 
 impl std::fmt::Debug for RegionCacheWriteBatch {
@@ -92,6 +102,7 @@ impl From<&RegionCacheMemoryEngine> for RegionCacheWriteBatch {
             prepare_for_write_duration: Duration::default(),
             current_region: None,
             written_regions: vec![],
+            prepared_regions: SmallVec::new(),
         }
     }
 }
@@ -110,6 +121,7 @@ impl RegionCacheWriteBatch {
             prepare_for_write_duration: Duration::default(),
             current_region: None,
             written_regions: vec![],
+            prepared_regions: SmallVec::new(),
         }
     }
 
@@ -444,6 +456,7 @@ impl WriteBatch for RegionCacheWriteBatch {
         self.current_region = None;
         self.written_regions.clear();
         self.prepare_for_write_duration = Duration::ZERO;
+        self.prepared_regions.clear();
     }
 
     fn set_save_point(&mut self) {
@@ -472,7 +485,22 @@ impl WriteBatch for RegionCacheWriteBatch {
     }
 
     fn prepare_for_region(&mut self, region: &metapb::Region) {
+        // If the region is already prepared for write, we do not need to prepare it
+        // again. See comments for the `prepared_regions` field for more details.
+        if let Some(current_region) = &self.current_region
+            && current_region.id == region.id
+        {
+            return;
+        }
         let time = Instant::now();
+        // verify that the region is not prepared before
+        if self.prepared_regions.contains(&region.id) {
+            panic!(
+                "region {} is prepared for write before, but it is not the current region",
+                region.id
+            );
+        }
+        self.prepared_regions.push(region.id);
         // record last region for clearing region in written flags.
         self.record_last_written_region();
 
@@ -987,4 +1015,29 @@ mod tests {
             assert_eq!(meta_by_range.get_region(), &r_new);
         }
     }
+
+    #[test]
+    fn test_dirty_data_exist_when_prepare_for_region() {
+        let engine = RegionCacheMemoryEngine::new(InMemoryEngineContext::new_for_tests(Arc::new(
+            VersionTrack::new(InMemoryEngineConfig::config_for_test()),
+        )));
+        let r = new_region(1, b"", b"z");
+        let cache_region = CacheRegion::from_region(&r);
+        let mut wb = RegionCacheWriteBatch::from(&engine);
+        wb.prepare_for_region(&r);
+
+        engine
+            .core()
+            .region_manager()
+            .load_region(cache_region.clone())
+            .unwrap();
+        wb.prepare_for_region(&r);
+        wb.put(b"k1", b"val1").unwrap();
+        wb.put(b"k2", b"val2").unwrap();
+        wb.set_sequence_number(100).unwrap();
+
+        wb.write().unwrap();
+
+        assert!(engine.core().engine().data[0].is_empty());
+    }
 }
diff --git a/components/test_raftstore/src/util.rs b/components/test_raftstore/src/util.rs
index 6966483cc04..f6f513b767b 100644
--- a/components/test_raftstore/src/util.rs
+++ b/components/test_raftstore/src/util.rs
@@ -319,6 +319,12 @@ pub fn new_region_leader_cmd() -> StatusRequest {
     cmd
 }
 
+pub fn new_compute_hash_request(region_id: u64, epoch: &RegionEpoch) -> RaftCmdRequest {
+    let mut admin = AdminRequest::default();
+    admin.set_cmd_type(AdminCmdType::ComputeHash);
+    new_admin_request(region_id, epoch, admin)
+}
+
 pub fn new_admin_request(
     region_id: u64,
     epoch: &RegionEpoch,

From 4ebb3001b22d13a5ec830aa32c4bb88e385a1ef0 Mon Sep 17 00:00:00 2001
From: Spade  A <71589810+SpadeA-Tang@users.noreply.github.com>
Date: Tue, 22 Oct 2024 15:58:47 +0800
Subject: [PATCH 03/16] In-memory engine: display safe point and safe point gap
 (#17694)

ref tikv/tikv#16141

display safe point and safe point gap

Signed-off-by: SpadeA-Tang <tangchenjie1210@gmail.com>

Co-authored-by: ti-chi-bot[bot] <108142056+ti-chi-bot[bot]@users.noreply.github.com>
---
 components/in_memory_engine/src/background.rs |   33 +-
 components/in_memory_engine/src/metrics.rs    |   15 +
 metrics/grafana/tikv_details.dashboard.py     |   73 +
 metrics/grafana/tikv_details.json             | 4562 +++++++++--------
 metrics/grafana/tikv_details.json.sha256      |    2 +-
 5 files changed, 2668 insertions(+), 2017 deletions(-)

diff --git a/components/in_memory_engine/src/background.rs b/components/in_memory_engine/src/background.rs
index 605a7270bf0..3db81a21d7d 100644
--- a/components/in_memory_engine/src/background.rs
+++ b/components/in_memory_engine/src/background.rs
@@ -43,7 +43,8 @@ use crate::{
     metrics::{
         IN_MEMORY_ENGINE_CACHE_COUNT, IN_MEMORY_ENGINE_GC_FILTERED_STATIC,
         IN_MEMORY_ENGINE_GC_TIME_HISTOGRAM, IN_MEMORY_ENGINE_LOAD_TIME_HISTOGRAM,
-        IN_MEMORY_ENGINE_MEMORY_USAGE,
+        IN_MEMORY_ENGINE_MEMORY_USAGE, IN_MEMORY_ENGINE_NEWEST_SAFE_POINT,
+        IN_MEMORY_ENGINE_OLDEST_SAFE_POINT, SAFE_POINT_GAP,
     },
     region_label::{
         LabelRule, RegionLabelChangedCallback, RegionLabelRulesManager, RegionLabelServiceBuilder,
@@ -1330,10 +1331,40 @@ impl RunnableWithTimer for BackgroundRunner {
         IN_MEMORY_ENGINE_MEMORY_USAGE.set(mem_usage as i64);
 
         let mut count_by_state = [0; RegionState::COUNT];
+        let mut oldest_safe_point = u64::MAX;
+        let mut newest_safe_point = u64::MIN;
         {
             let regions_map = self.core.engine.region_manager().regions_map.read();
             for m in regions_map.regions().values() {
                 count_by_state[m.get_state() as usize] += 1;
+                if m.get_state() == RegionState::Active && m.safe_point() != 0 {
+                    oldest_safe_point = u64::min(oldest_safe_point, m.safe_point());
+                    newest_safe_point = u64::max(newest_safe_point, m.safe_point());
+                }
+            }
+        }
+
+        if oldest_safe_point != u64::MAX {
+            IN_MEMORY_ENGINE_OLDEST_SAFE_POINT.set(oldest_safe_point as i64);
+            IN_MEMORY_ENGINE_NEWEST_SAFE_POINT.set(newest_safe_point as i64);
+            if let Ok(Ok(tikv_safe_point)) =
+                block_on_timeout(self.pd_client.get_gc_safe_point(), Duration::from_secs(5))
+            {
+                if tikv_safe_point > oldest_safe_point {
+                    warn!(
+                        "ime oldest auto gc safe point is older than tikv's auto gc safe point";
+                        "tikv_safe_point" => tikv_safe_point,
+                        "ime_oldest_safe_point" => oldest_safe_point,
+                    );
+                }
+
+                let gap =
+                    TimeStamp::new(oldest_safe_point.saturating_sub(tikv_safe_point)).physical();
+                // If gap is too larger (more than a year), it means tikv safe point is not
+                // initialized, so we does not update the metrics now.
+                if gap < Duration::from_secs(365 * 24 * 3600).as_millis() as u64 {
+                    SAFE_POINT_GAP.set(oldest_safe_point as i64 - tikv_safe_point as i64);
+                }
             }
         }
 
diff --git a/components/in_memory_engine/src/metrics.rs b/components/in_memory_engine/src/metrics.rs
index a99ffcab5bb..ffdb48d6c64 100644
--- a/components/in_memory_engine/src/metrics.rs
+++ b/components/in_memory_engine/src/metrics.rs
@@ -127,6 +127,21 @@ lazy_static! {
         exponential_buckets(0.00001, 2.0, 26).unwrap()
     )
     .unwrap();
+    pub static ref IN_MEMORY_ENGINE_OLDEST_SAFE_POINT: IntGauge = register_int_gauge!(
+        "tikv_in_memory_engine_oldest_safe_point",
+        "The oldest safe point in the in-memory engine",
+    )
+    .unwrap();
+    pub static ref IN_MEMORY_ENGINE_NEWEST_SAFE_POINT: IntGauge = register_int_gauge!(
+        "tikv_in_memory_engine_newest_safe_point",
+        "The newest safe point in the in-memory engine",
+    )
+    .unwrap();
+    pub static ref SAFE_POINT_GAP: IntGauge = register_int_gauge!(
+        "tikv_safe_point_gap_with_in_memory_engine",
+        "The gap between tikv auto gc safe point and the oldest auto gc safe point in the in-memory engine",
+    )
+    .unwrap();
 }
 
 lazy_static! {
diff --git a/metrics/grafana/tikv_details.dashboard.py b/metrics/grafana/tikv_details.dashboard.py
index 802435adf68..fca208a4f25 100644
--- a/metrics/grafana/tikv_details.dashboard.py
+++ b/metrics/grafana/tikv_details.dashboard.py
@@ -4643,6 +4643,79 @@ def InMemoryEngine() -> RowPanel:
             ),
         ]
     )
+    layout.row(
+        [
+            graph_panel(
+                title="Oldest Auto GC SafePoint",
+                description="Unlike the auto gc safe point used for TiKV, the safe point for in-memory engine is per region and this is the oldest one",
+                yaxes=yaxes(left_format=UNITS.DATE_TIME_ISO),
+                targets=[
+                    target(
+                        expr=expr_max(
+                            "tikv_in_memory_engine_oldest_safe_point",
+                        )
+                        .extra("/ (2^18)")
+                        .skip_default_instance_selector(),
+                        additional_groupby=True,
+                    ),
+                ],
+            ),
+            graph_panel(
+                title="Newest Auto GC SafePoint",
+                description="Unlike the auto gc safe point used for TiKV, the safe point for in-memory engine is per region and this is the newest one",
+                yaxes=yaxes(left_format=UNITS.DATE_TIME_ISO),
+                targets=[
+                    target(
+                        expr=expr_max(
+                            "tikv_in_memory_engine_newest_safe_point",
+                        )
+                        .extra("/ (2^18)")
+                        .skip_default_instance_selector(),
+                        additional_groupby=True,
+                    ),
+                ],
+            ),
+            graph_panel(
+                title="Auto GC SafePoint Gap",
+                description="The gap between newest auto gc safe point and oldest auto gc safe point of regions cached in the in-memroy engine",
+                yaxes=yaxes(left_format=UNITS.MILLI_SECONDS),
+                targets=[
+                    target(
+                        expr=expr_operator(
+                            expr_sum(
+                                "tikv_in_memory_engine_newest_safe_point",
+                            )
+                            .extra("/ (2^18)")
+                            .skip_default_instance_selector(),
+                            "-",
+                            expr_sum(
+                                "tikv_in_memory_engine_oldest_safe_point",
+                            )
+                            .extra("/ (2^18)")
+                            .skip_default_instance_selector(),
+                        ),
+                        additional_groupby=True,
+                        legend_format="{{instance}}",
+                    ),
+                ],
+            ),
+            graph_panel(
+                title="Auto GC SafePoint Gap With TiKV",
+                description="The gap between tikv auto gc safe point and in-memory engine oldest auto gc safe point",
+                yaxes=yaxes(left_format=UNITS.MILLI_SECONDS),
+                targets=[
+                    target(
+                        expr=expr_max(
+                            "tikv_safe_point_gap_with_in_memory_engine",
+                        )
+                        .extra("/ (2^18)")
+                        .skip_default_instance_selector(),
+                        additional_groupby=True,
+                    ),
+                ],
+            ),
+        ]
+    )
     return layout.row_panel
 
 
diff --git a/metrics/grafana/tikv_details.json b/metrics/grafana/tikv_details.json
index 794658a4071..78b5c6493fc 100644
--- a/metrics/grafana/tikv_details.json
+++ b/metrics/grafana/tikv_details.json
@@ -39562,55 +39562,13 @@
             "align": false,
             "alignLevel": 0
           }
-        }
-      ],
-      "repeat": null,
-      "repeatDirection": null,
-      "span": null,
-      "targets": [],
-      "timeFrom": null,
-      "timeShift": null,
-      "title": "In Memory Engine",
-      "transformations": [],
-      "transparent": false,
-      "type": "row"
-    },
-    {
-      "cacheTimeout": null,
-      "collapsed": true,
-      "datasource": null,
-      "description": null,
-      "editable": true,
-      "error": false,
-      "fieldConfig": {
-        "defaults": {
-          "thresholds": {
-            "mode": "absolute",
-            "steps": []
-          }
-        }
-      },
-      "gridPos": {
-        "h": 7,
-        "w": 24,
-        "x": 0,
-        "y": 0
-      },
-      "height": null,
-      "hideTimeOverride": false,
-      "id": 277,
-      "interval": null,
-      "links": [],
-      "maxDataPoints": 100,
-      "maxPerRow": null,
-      "minSpan": null,
-      "panels": [
+        },
         {
           "aliasColors": {},
           "bars": false,
           "cacheTimeout": null,
           "datasource": "${DS_TEST-CLUSTER}",
-          "description": "",
+          "description": "Unlike the auto gc safe point used for TiKV, the safe point for in-memory engine is per region and this is the oldest one",
           "editable": true,
           "error": false,
           "fieldConfig": {
@@ -39631,13 +39589,13 @@
           },
           "gridPos": {
             "h": 7,
-            "w": 12,
+            "w": 6,
             "x": 0,
-            "y": 0
+            "y": 49
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 278,
+          "id": 277,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -39680,30 +39638,15 @@
           "targets": [
             {
               "datasource": "${DS_TEST-CLUSTER}",
-              "expr": "sum((\n    tikv_scheduler_write_flow\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    \n)) by (instance) ",
+              "expr": "max((\n    tikv_in_memory_engine_oldest_safe_point\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\"}\n    \n)) by (instance, $additional_groupby) / (2^18)",
               "format": "time_series",
               "hide": false,
               "instant": false,
               "interval": "",
               "intervalFactor": 1,
-              "legendFormat": "write-{{instance}}",
-              "metric": "",
-              "query": "sum((\n    tikv_scheduler_write_flow\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    \n)) by (instance) ",
-              "refId": "",
-              "step": 10,
-              "target": ""
-            },
-            {
-              "datasource": "${DS_TEST-CLUSTER}",
-              "expr": "sum((\n    tikv_scheduler_throttle_flow\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    \n)) by (instance)  != 0",
-              "format": "time_series",
-              "hide": false,
-              "instant": false,
-              "interval": "",
-              "intervalFactor": 1,
-              "legendFormat": "throttle-{{instance}}",
+              "legendFormat": "{{instance}} {{$additional_groupby}}",
               "metric": "",
-              "query": "sum((\n    tikv_scheduler_throttle_flow\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    \n)) by (instance)  != 0",
+              "query": "max((\n    tikv_in_memory_engine_oldest_safe_point\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\"}\n    \n)) by (instance, $additional_groupby) / (2^18)",
               "refId": "",
               "step": 10,
               "target": ""
@@ -39712,7 +39655,7 @@
           "thresholds": [],
           "timeFrom": null,
           "timeShift": null,
-          "title": "Scheduler flow",
+          "title": "Oldest Auto GC SafePoint",
           "tooltip": {
             "msResolution": true,
             "shared": true,
@@ -39731,7 +39674,7 @@
           "yaxes": [
             {
               "decimals": null,
-              "format": "bytes",
+              "format": "dateTimeAsIso",
               "label": null,
               "logBase": 1,
               "max": null,
@@ -39758,7 +39701,7 @@
           "bars": false,
           "cacheTimeout": null,
           "datasource": "${DS_TEST-CLUSTER}",
-          "description": "",
+          "description": "Unlike the auto gc safe point used for TiKV, the safe point for in-memory engine is per region and this is the newest one",
           "editable": true,
           "error": false,
           "fieldConfig": {
@@ -39779,13 +39722,13 @@
           },
           "gridPos": {
             "h": 7,
-            "w": 12,
-            "x": 12,
-            "y": 0
+            "w": 6,
+            "x": 6,
+            "y": 49
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 279,
+          "id": 278,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -39828,15 +39771,15 @@
           "targets": [
             {
               "datasource": "${DS_TEST-CLUSTER}",
-              "expr": "sum((\n    tikv_scheduler_discard_ratio\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    \n)) by (type)  / 10000000",
+              "expr": "max((\n    tikv_in_memory_engine_newest_safe_point\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\"}\n    \n)) by (instance, $additional_groupby) / (2^18)",
               "format": "time_series",
               "hide": false,
               "instant": false,
               "interval": "",
               "intervalFactor": 1,
-              "legendFormat": "{{type}}",
+              "legendFormat": "{{instance}} {{$additional_groupby}}",
               "metric": "",
-              "query": "sum((\n    tikv_scheduler_discard_ratio\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    \n)) by (type)  / 10000000",
+              "query": "max((\n    tikv_in_memory_engine_newest_safe_point\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\"}\n    \n)) by (instance, $additional_groupby) / (2^18)",
               "refId": "",
               "step": 10,
               "target": ""
@@ -39845,7 +39788,7 @@
           "thresholds": [],
           "timeFrom": null,
           "timeShift": null,
-          "title": "Scheduler discard ratio",
+          "title": "Newest Auto GC SafePoint",
           "tooltip": {
             "msResolution": true,
             "shared": true,
@@ -39864,7 +39807,7 @@
           "yaxes": [
             {
               "decimals": null,
-              "format": "percentunit",
+              "format": "dateTimeAsIso",
               "label": null,
               "logBase": 1,
               "max": null,
@@ -39886,117 +39829,12 @@
             "alignLevel": 0
           }
         },
-        {
-          "cacheTimeout": null,
-          "cards": {
-            "cardPadding": null,
-            "cardRound": null
-          },
-          "color": {
-            "cardColor": "#b4ff00",
-            "colorScale": "sqrt",
-            "colorScheme": "interpolateSpectral",
-            "exponent": 0.5,
-            "max": null,
-            "min": null,
-            "mode": "spectrum"
-          },
-          "dataFormat": "tsbuckets",
-          "datasource": "${DS_TEST-CLUSTER}",
-          "description": null,
-          "editable": true,
-          "error": false,
-          "fieldConfig": {
-            "defaults": {
-              "thresholds": {
-                "mode": "absolute",
-                "steps": []
-              }
-            }
-          },
-          "gridPos": {
-            "h": 7,
-            "w": 12,
-            "x": 0,
-            "y": 7
-          },
-          "heatmap": {},
-          "height": null,
-          "hideTimeOverride": false,
-          "hideZeroBuckets": true,
-          "highlightCards": true,
-          "id": 280,
-          "interval": null,
-          "legend": {
-            "show": false
-          },
-          "links": [],
-          "maxDataPoints": 512,
-          "maxPerRow": null,
-          "minSpan": null,
-          "options": {},
-          "repeat": null,
-          "repeatDirection": null,
-          "reverseYBuckets": false,
-          "span": null,
-          "targets": [
-            {
-              "datasource": "${DS_TEST-CLUSTER}",
-              "expr": "sum(increase(\n    tikv_scheduler_throttle_duration_seconds_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    [$__rate_interval]\n)) by (le) ",
-              "format": "heatmap",
-              "hide": false,
-              "instant": false,
-              "interval": "",
-              "intervalFactor": 1,
-              "legendFormat": "{{le}}",
-              "metric": "",
-              "query": "sum(increase(\n    tikv_scheduler_throttle_duration_seconds_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    [$__rate_interval]\n)) by (le) ",
-              "refId": "",
-              "step": 10,
-              "target": ""
-            }
-          ],
-          "timeFrom": null,
-          "timeShift": null,
-          "title": "Throttle duration",
-          "tooltip": {
-            "msResolution": true,
-            "shared": true,
-            "show": true,
-            "showHistogram": true,
-            "sort": 0,
-            "value_type": "individual"
-          },
-          "transformations": [],
-          "transparent": false,
-          "type": "heatmap",
-          "xAxis": {
-            "mode": "time",
-            "name": null,
-            "show": true,
-            "values": []
-          },
-          "xBucketNumber": null,
-          "xBucketSize": null,
-          "yAxis": {
-            "decimals": 1,
-            "format": "s",
-            "label": null,
-            "logBase": 1,
-            "max": null,
-            "min": null,
-            "show": true
-          },
-          "yBucketBound": "upper",
-          "yBucketNumber": null,
-          "yBucketSize": null
-        },
         {
           "aliasColors": {},
           "bars": false,
           "cacheTimeout": null,
           "datasource": "${DS_TEST-CLUSTER}",
-          "description": null,
+          "description": "The gap between newest auto gc safe point and oldest auto gc safe point of regions cached in the in-memroy engine",
           "editable": true,
           "error": false,
           "fieldConfig": {
@@ -40017,13 +39855,13 @@
           },
           "gridPos": {
             "h": 7,
-            "w": 12,
+            "w": 6,
             "x": 12,
-            "y": 7
+            "y": 49
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 281,
+          "id": 279,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -40066,15 +39904,15 @@
           "targets": [
             {
               "datasource": "${DS_TEST-CLUSTER}",
-              "expr": "((\n    tikv_scheduler_throttle_cf\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    \n))   != 0",
+              "expr": "(sum((\n    tikv_in_memory_engine_newest_safe_point\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\"}\n    \n)) by (instance, $additional_groupby) / (2^18) - sum((\n    tikv_in_memory_engine_oldest_safe_point\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\"}\n    \n)) by (instance, $additional_groupby) / (2^18))",
               "format": "time_series",
               "hide": false,
               "instant": false,
               "interval": "",
               "intervalFactor": 1,
-              "legendFormat": "{{instance}}-{{cf}}",
+              "legendFormat": "{{instance}} {{$additional_groupby}}",
               "metric": "",
-              "query": "((\n    tikv_scheduler_throttle_cf\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    \n))   != 0",
+              "query": "(sum((\n    tikv_in_memory_engine_newest_safe_point\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\"}\n    \n)) by (instance, $additional_groupby) / (2^18) - sum((\n    tikv_in_memory_engine_oldest_safe_point\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\"}\n    \n)) by (instance, $additional_groupby) / (2^18))",
               "refId": "",
               "step": 10,
               "target": ""
@@ -40083,7 +39921,7 @@
           "thresholds": [],
           "timeFrom": null,
           "timeShift": null,
-          "title": "Scheduler throttled CF",
+          "title": "Auto GC SafePoint Gap",
           "tooltip": {
             "msResolution": true,
             "shared": true,
@@ -40102,7 +39940,7 @@
           "yaxes": [
             {
               "decimals": null,
-              "format": "ops",
+              "format": "ms",
               "label": null,
               "logBase": 1,
               "max": null,
@@ -40129,7 +39967,7 @@
           "bars": false,
           "cacheTimeout": null,
           "datasource": "${DS_TEST-CLUSTER}",
-          "description": "",
+          "description": "The gap between tikv auto gc safe point and in-memory engine oldest auto gc safe point",
           "editable": true,
           "error": false,
           "fieldConfig": {
@@ -40150,13 +39988,13 @@
           },
           "gridPos": {
             "h": 7,
-            "w": 12,
-            "x": 0,
-            "y": 14
+            "w": 6,
+            "x": 18,
+            "y": 49
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 282,
+          "id": 280,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -40199,15 +40037,15 @@
           "targets": [
             {
               "datasource": "${DS_TEST-CLUSTER}",
-              "expr": "sum(rate(\n    tikv_scheduler_throttle_action_total\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    [$__rate_interval]\n)) by (type, cf, $additional_groupby) ",
+              "expr": "max((\n    tikv_safe_point_gap_with_in_memory_engine\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\"}\n    \n)) by (instance, $additional_groupby) / (2^18)",
               "format": "time_series",
               "hide": false,
               "instant": false,
               "interval": "",
               "intervalFactor": 1,
-              "legendFormat": "{{type}}-{{cf}} {{$additional_groupby}}",
+              "legendFormat": "{{instance}} {{$additional_groupby}}",
               "metric": "",
-              "query": "sum(rate(\n    tikv_scheduler_throttle_action_total\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    [$__rate_interval]\n)) by (type, cf, $additional_groupby) ",
+              "query": "max((\n    tikv_safe_point_gap_with_in_memory_engine\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\"}\n    \n)) by (instance, $additional_groupby) / (2^18)",
               "refId": "",
               "step": 10,
               "target": ""
@@ -40216,7 +40054,7 @@
           "thresholds": [],
           "timeFrom": null,
           "timeShift": null,
-          "title": "Flow controller actions",
+          "title": "Auto GC SafePoint Gap With TiKV",
           "tooltip": {
             "msResolution": true,
             "shared": true,
@@ -40235,7 +40073,7 @@
           "yaxes": [
             {
               "decimals": null,
-              "format": "ops",
+              "format": "ms",
               "label": null,
               "logBase": 1,
               "max": null,
@@ -40256,7 +40094,49 @@
             "align": false,
             "alignLevel": 0
           }
-        },
+        }
+      ],
+      "repeat": null,
+      "repeatDirection": null,
+      "span": null,
+      "targets": [],
+      "timeFrom": null,
+      "timeShift": null,
+      "title": "In Memory Engine",
+      "transformations": [],
+      "transparent": false,
+      "type": "row"
+    },
+    {
+      "cacheTimeout": null,
+      "collapsed": true,
+      "datasource": null,
+      "description": null,
+      "editable": true,
+      "error": false,
+      "fieldConfig": {
+        "defaults": {
+          "thresholds": {
+            "mode": "absolute",
+            "steps": []
+          }
+        }
+      },
+      "gridPos": {
+        "h": 7,
+        "w": 24,
+        "x": 0,
+        "y": 0
+      },
+      "height": null,
+      "hideTimeOverride": false,
+      "id": 281,
+      "interval": null,
+      "links": [],
+      "maxDataPoints": 100,
+      "maxPerRow": null,
+      "minSpan": null,
+      "panels": [
         {
           "aliasColors": {},
           "bars": false,
@@ -40284,12 +40164,12 @@
           "gridPos": {
             "h": 7,
             "w": 12,
-            "x": 12,
-            "y": 14
+            "x": 0,
+            "y": 0
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 283,
+          "id": 282,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -40332,60 +40212,30 @@
           "targets": [
             {
               "datasource": "${DS_TEST-CLUSTER}",
-              "expr": "sum((\n    tikv_scheduler_l0_flow\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    \n)) by (instance, cf) ",
-              "format": "time_series",
-              "hide": false,
-              "instant": false,
-              "interval": "",
-              "intervalFactor": 1,
-              "legendFormat": "{{cf}}_l0_flow-{{instance}}",
-              "metric": "",
-              "query": "sum((\n    tikv_scheduler_l0_flow\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    \n)) by (instance, cf) ",
-              "refId": "",
-              "step": 10,
-              "target": ""
-            },
-            {
-              "datasource": "${DS_TEST-CLUSTER}",
-              "expr": "sum((\n    tikv_scheduler_flush_flow\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    \n)) by (instance, cf) ",
-              "format": "time_series",
-              "hide": false,
-              "instant": false,
-              "interval": "",
-              "intervalFactor": 1,
-              "legendFormat": "{{cf}}_flush_flow-{{instance}}",
-              "metric": "",
-              "query": "sum((\n    tikv_scheduler_flush_flow\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    \n)) by (instance, cf) ",
-              "refId": "",
-              "step": 10,
-              "target": ""
-            },
-            {
-              "datasource": "${DS_TEST-CLUSTER}",
-              "expr": "sum((\n    tikv_scheduler_l0_flow\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    \n)) by (instance) ",
+              "expr": "sum((\n    tikv_scheduler_write_flow\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    \n)) by (instance) ",
               "format": "time_series",
               "hide": false,
               "instant": false,
               "interval": "",
               "intervalFactor": 1,
-              "legendFormat": "total_l0_flow-{{instance}}",
+              "legendFormat": "write-{{instance}}",
               "metric": "",
-              "query": "sum((\n    tikv_scheduler_l0_flow\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    \n)) by (instance) ",
+              "query": "sum((\n    tikv_scheduler_write_flow\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    \n)) by (instance) ",
               "refId": "",
               "step": 10,
               "target": ""
             },
             {
               "datasource": "${DS_TEST-CLUSTER}",
-              "expr": "sum((\n    tikv_scheduler_flush_flow\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    \n)) by (instance) ",
+              "expr": "sum((\n    tikv_scheduler_throttle_flow\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    \n)) by (instance)  != 0",
               "format": "time_series",
               "hide": false,
               "instant": false,
               "interval": "",
               "intervalFactor": 1,
-              "legendFormat": "total_flush_flow-{{instance}}",
+              "legendFormat": "throttle-{{instance}}",
               "metric": "",
-              "query": "sum((\n    tikv_scheduler_flush_flow\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    \n)) by (instance) ",
+              "query": "sum((\n    tikv_scheduler_throttle_flow\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    \n)) by (instance)  != 0",
               "refId": "",
               "step": 10,
               "target": ""
@@ -40394,7 +40244,7 @@
           "thresholds": [],
           "timeFrom": null,
           "timeShift": null,
-          "title": "Flush/L0 flow",
+          "title": "Scheduler flow",
           "tooltip": {
             "msResolution": true,
             "shared": true,
@@ -40462,12 +40312,12 @@
           "gridPos": {
             "h": 7,
             "w": 12,
-            "x": 0,
-            "y": 21
+            "x": 12,
+            "y": 0
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 284,
+          "id": 283,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -40510,45 +40360,15 @@
           "targets": [
             {
               "datasource": "${DS_TEST-CLUSTER}",
-              "expr": "max((\n    tikv_scheduler_l0\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    \n)) by (instance) ",
-              "format": "time_series",
-              "hide": false,
-              "instant": false,
-              "interval": "",
-              "intervalFactor": 1,
-              "legendFormat": "l0-{{instance}}",
-              "metric": "",
-              "query": "max((\n    tikv_scheduler_l0\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    \n)) by (instance) ",
-              "refId": "",
-              "step": 10,
-              "target": ""
-            },
-            {
-              "datasource": "${DS_TEST-CLUSTER}",
-              "expr": "max((\n    tikv_scheduler_memtable\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    \n)) by (instance) ",
-              "format": "time_series",
-              "hide": false,
-              "instant": false,
-              "interval": "",
-              "intervalFactor": 1,
-              "legendFormat": "memtable-{{instance}}",
-              "metric": "",
-              "query": "max((\n    tikv_scheduler_memtable\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    \n)) by (instance) ",
-              "refId": "",
-              "step": 10,
-              "target": ""
-            },
-            {
-              "datasource": "${DS_TEST-CLUSTER}",
-              "expr": "max((\n    tikv_scheduler_l0_avg\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    \n)) by (instance) ",
+              "expr": "sum((\n    tikv_scheduler_discard_ratio\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    \n)) by (type)  / 10000000",
               "format": "time_series",
               "hide": false,
               "instant": false,
               "interval": "",
               "intervalFactor": 1,
-              "legendFormat": "avg_l0-{{instance}}",
+              "legendFormat": "{{type}}",
               "metric": "",
-              "query": "max((\n    tikv_scheduler_l0_avg\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    \n)) by (instance) ",
+              "query": "sum((\n    tikv_scheduler_discard_ratio\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    \n)) by (type)  / 10000000",
               "refId": "",
               "step": 10,
               "target": ""
@@ -40557,7 +40377,7 @@
           "thresholds": [],
           "timeFrom": null,
           "timeShift": null,
-          "title": "Flow controller factors",
+          "title": "Scheduler discard ratio",
           "tooltip": {
             "msResolution": true,
             "shared": true,
@@ -40576,7 +40396,7 @@
           "yaxes": [
             {
               "decimals": null,
-              "format": "none",
+              "format": "percentunit",
               "label": null,
               "logBase": 1,
               "max": null,
@@ -40599,11 +40419,23 @@
           }
         },
         {
-          "aliasColors": {},
-          "bars": false,
           "cacheTimeout": null,
+          "cards": {
+            "cardPadding": null,
+            "cardRound": null
+          },
+          "color": {
+            "cardColor": "#b4ff00",
+            "colorScale": "sqrt",
+            "colorScheme": "interpolateSpectral",
+            "exponent": 0.5,
+            "max": null,
+            "min": null,
+            "mode": "spectrum"
+          },
+          "dataFormat": "tsbuckets",
           "datasource": "${DS_TEST-CLUSTER}",
-          "description": "",
+          "description": null,
           "editable": true,
           "error": false,
           "fieldConfig": {
@@ -40614,144 +40446,89 @@
               }
             }
           },
-          "fill": 1,
-          "fillGradient": 1,
-          "grid": {
-            "threshold1": null,
-            "threshold1Color": "rgba(216, 200, 27, 0.27)",
-            "threshold2": null,
-            "threshold2Color": "rgba(234, 112, 112, 0.22)"
-          },
           "gridPos": {
             "h": 7,
             "w": 12,
-            "x": 12,
-            "y": 21
+            "x": 0,
+            "y": 7
           },
+          "heatmap": {},
           "height": null,
           "hideTimeOverride": false,
-          "id": 285,
+          "hideZeroBuckets": true,
+          "highlightCards": true,
+          "id": 284,
           "interval": null,
-          "isNew": true,
           "legend": {
-            "alignAsTable": true,
-            "avg": false,
-            "current": true,
-            "hideEmpty": true,
-            "hideZero": true,
-            "max": true,
-            "min": false,
-            "rightSide": true,
-            "show": true,
-            "sideWidth": null,
-            "sort": "max",
-            "sortDesc": true,
-            "total": false,
-            "values": true
+            "show": false
           },
-          "lines": true,
-          "linewidth": 1,
           "links": [],
-          "maxDataPoints": null,
+          "maxDataPoints": 512,
           "maxPerRow": null,
           "minSpan": null,
-          "nullPointMode": "null as zero",
-          "options": {
-            "alertThreshold": true,
-            "dataLinks": []
-          },
-          "percentage": false,
-          "pointradius": 5,
-          "points": false,
-          "renderer": "flot",
+          "options": {},
           "repeat": null,
           "repeatDirection": null,
-          "seriesOverrides": [],
+          "reverseYBuckets": false,
           "span": null,
-          "stack": false,
-          "steppedLine": false,
           "targets": [
             {
               "datasource": "${DS_TEST-CLUSTER}",
-              "expr": "sum((\n    tikv_engine_pending_compaction_bytes\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"kv\"}\n    \n)) by (cf, $additional_groupby) ",
-              "format": "time_series",
+              "expr": "sum(increase(\n    tikv_scheduler_throttle_duration_seconds_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    [$__rate_interval]\n)) by (le) ",
+              "format": "heatmap",
               "hide": false,
               "instant": false,
               "interval": "",
               "intervalFactor": 1,
-              "legendFormat": "{{cf}} {{$additional_groupby}}",
-              "metric": "",
-              "query": "sum((\n    tikv_engine_pending_compaction_bytes\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"kv\"}\n    \n)) by (cf, $additional_groupby) ",
-              "refId": "",
-              "step": 10,
-              "target": ""
-            },
-            {
-              "datasource": "${DS_TEST-CLUSTER}",
-              "expr": "sum((\n    tikv_scheduler_pending_compaction_bytes\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    \n)) by (cf, $additional_groupby)  / 10000000",
-              "format": "time_series",
-              "hide": true,
-              "instant": false,
-              "interval": "",
-              "intervalFactor": 1,
-              "legendFormat": "pending-bytes {{$additional_groupby}}",
+              "legendFormat": "{{le}}",
               "metric": "",
-              "query": "sum((\n    tikv_scheduler_pending_compaction_bytes\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    \n)) by (cf, $additional_groupby)  / 10000000",
+              "query": "sum(increase(\n    tikv_scheduler_throttle_duration_seconds_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    [$__rate_interval]\n)) by (le) ",
               "refId": "",
               "step": 10,
               "target": ""
             }
           ],
-          "thresholds": [],
           "timeFrom": null,
           "timeShift": null,
-          "title": "Compaction pending bytes",
+          "title": "Throttle duration",
           "tooltip": {
             "msResolution": true,
             "shared": true,
+            "show": true,
+            "showHistogram": true,
             "sort": 0,
             "value_type": "individual"
           },
           "transformations": [],
           "transparent": false,
-          "type": "graph",
-          "xaxis": {
+          "type": "heatmap",
+          "xAxis": {
             "mode": "time",
             "name": null,
             "show": true,
             "values": []
           },
-          "yaxes": [
-            {
-              "decimals": null,
-              "format": "bytes",
-              "label": null,
-              "logBase": 1,
-              "max": null,
-              "min": null,
-              "show": true
-            },
-            {
-              "decimals": null,
-              "format": "short",
-              "label": null,
-              "logBase": 1,
-              "max": null,
-              "min": null,
-              "show": true
-            }
-          ],
-          "yaxis": {
-            "align": false,
-            "alignLevel": 0
-          }
+          "xBucketNumber": null,
+          "xBucketSize": null,
+          "yAxis": {
+            "decimals": 1,
+            "format": "s",
+            "label": null,
+            "logBase": 1,
+            "max": null,
+            "min": null,
+            "show": true
+          },
+          "yBucketBound": "upper",
+          "yBucketNumber": null,
+          "yBucketSize": null
         },
         {
           "aliasColors": {},
           "bars": false,
           "cacheTimeout": null,
           "datasource": "${DS_TEST-CLUSTER}",
-          "description": "Throttle time for txn storage commands in 1 minute.",
+          "description": null,
           "editable": true,
           "error": false,
           "fieldConfig": {
@@ -40773,12 +40550,12 @@
           "gridPos": {
             "h": 7,
             "w": 12,
-            "x": 0,
-            "y": 28
+            "x": 12,
+            "y": 7
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 286,
+          "id": 285,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -40821,15 +40598,15 @@
           "targets": [
             {
               "datasource": "${DS_TEST-CLUSTER}",
-              "expr": "sum(rate(\n    tikv_txn_command_throttle_time_total\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    [$__rate_interval]\n)) by (type, $additional_groupby) ",
+              "expr": "((\n    tikv_scheduler_throttle_cf\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    \n))   != 0",
               "format": "time_series",
               "hide": false,
               "instant": false,
               "interval": "",
               "intervalFactor": 1,
-              "legendFormat": "{{type}} {{$additional_groupby}}",
+              "legendFormat": "{{instance}}-{{cf}}",
               "metric": "",
-              "query": "sum(rate(\n    tikv_txn_command_throttle_time_total\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    [$__rate_interval]\n)) by (type, $additional_groupby) ",
+              "query": "((\n    tikv_scheduler_throttle_cf\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    \n))   != 0",
               "refId": "",
               "step": 10,
               "target": ""
@@ -40838,7 +40615,7 @@
           "thresholds": [],
           "timeFrom": null,
           "timeShift": null,
-          "title": "Txn command throttled duration",
+          "title": "Scheduler throttled CF",
           "tooltip": {
             "msResolution": true,
             "shared": true,
@@ -40857,7 +40634,7 @@
           "yaxes": [
             {
               "decimals": null,
-              "format": "\u00b5s",
+              "format": "ops",
               "label": null,
               "logBase": 1,
               "max": null,
@@ -40884,7 +40661,7 @@
           "bars": false,
           "cacheTimeout": null,
           "datasource": "${DS_TEST-CLUSTER}",
-          "description": "Throttle time for non-txn related processing like analyze or dag in 1 minute.",
+          "description": "",
           "editable": true,
           "error": false,
           "fieldConfig": {
@@ -40906,12 +40683,12 @@
           "gridPos": {
             "h": 7,
             "w": 12,
-            "x": 12,
-            "y": 28
+            "x": 0,
+            "y": 14
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 287,
+          "id": 286,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -40954,15 +40731,15 @@
           "targets": [
             {
               "datasource": "${DS_TEST-CLUSTER}",
-              "expr": "sum(rate(\n    tikv_non_txn_command_throttle_time_total\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    [$__rate_interval]\n)) by (type, $additional_groupby) ",
+              "expr": "sum(rate(\n    tikv_scheduler_throttle_action_total\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    [$__rate_interval]\n)) by (type, cf, $additional_groupby) ",
               "format": "time_series",
               "hide": false,
               "instant": false,
               "interval": "",
               "intervalFactor": 1,
-              "legendFormat": "{{type}} {{$additional_groupby}}",
+              "legendFormat": "{{type}}-{{cf}} {{$additional_groupby}}",
               "metric": "",
-              "query": "sum(rate(\n    tikv_non_txn_command_throttle_time_total\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    [$__rate_interval]\n)) by (type, $additional_groupby) ",
+              "query": "sum(rate(\n    tikv_scheduler_throttle_action_total\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    [$__rate_interval]\n)) by (type, cf, $additional_groupby) ",
               "refId": "",
               "step": 10,
               "target": ""
@@ -40971,7 +40748,7 @@
           "thresholds": [],
           "timeFrom": null,
           "timeShift": null,
-          "title": "Non-txn command throttled duration",
+          "title": "Flow controller actions",
           "tooltip": {
             "msResolution": true,
             "shared": true,
@@ -40990,7 +40767,7 @@
           "yaxes": [
             {
               "decimals": null,
-              "format": "\u00b5s",
+              "format": "ops",
               "label": null,
               "logBase": 1,
               "max": null,
@@ -41011,55 +40788,13 @@
             "align": false,
             "alignLevel": 0
           }
-        }
-      ],
-      "repeat": null,
-      "repeatDirection": null,
-      "span": null,
-      "targets": [],
-      "timeFrom": null,
-      "timeShift": null,
-      "title": "Flow Control",
-      "transformations": [],
-      "transparent": false,
-      "type": "row"
-    },
-    {
-      "cacheTimeout": null,
-      "collapsed": true,
-      "datasource": null,
-      "description": null,
-      "editable": true,
-      "error": false,
-      "fieldConfig": {
-        "defaults": {
-          "thresholds": {
-            "mode": "absolute",
-            "steps": []
-          }
-        }
-      },
-      "gridPos": {
-        "h": 7,
-        "w": 24,
-        "x": 0,
-        "y": 0
-      },
-      "height": null,
-      "hideTimeOverride": false,
-      "id": 288,
-      "interval": null,
-      "links": [],
-      "maxDataPoints": 100,
-      "maxPerRow": null,
-      "minSpan": null,
-      "panels": [
+        },
         {
           "aliasColors": {},
           "bars": false,
           "cacheTimeout": null,
           "datasource": "${DS_TEST-CLUSTER}",
-          "description": "The total number of commands on each stage",
+          "description": "",
           "editable": true,
           "error": false,
           "fieldConfig": {
@@ -41081,12 +40816,12 @@
           "gridPos": {
             "h": 7,
             "w": 12,
-            "x": 0,
-            "y": 0
+            "x": 12,
+            "y": 14
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 289,
+          "id": 287,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -41129,30 +40864,60 @@
           "targets": [
             {
               "datasource": "${DS_TEST-CLUSTER}",
-              "expr": "sum(rate(\n    tikv_scheduler_too_busy_total\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    [$__rate_interval]\n)) by (stage, $additional_groupby) ",
+              "expr": "sum((\n    tikv_scheduler_l0_flow\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    \n)) by (instance, cf) ",
               "format": "time_series",
               "hide": false,
               "instant": false,
               "interval": "",
               "intervalFactor": 1,
-              "legendFormat": "{{stage}} {{$additional_groupby}}",
+              "legendFormat": "{{cf}}_l0_flow-{{instance}}",
               "metric": "",
-              "query": "sum(rate(\n    tikv_scheduler_too_busy_total\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    [$__rate_interval]\n)) by (stage, $additional_groupby) ",
+              "query": "sum((\n    tikv_scheduler_l0_flow\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    \n)) by (instance, cf) ",
               "refId": "",
               "step": 10,
               "target": ""
             },
             {
               "datasource": "${DS_TEST-CLUSTER}",
-              "expr": "sum(rate(\n    tikv_scheduler_stage_total\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    [$__rate_interval]\n)) by (stage, $additional_groupby) ",
+              "expr": "sum((\n    tikv_scheduler_flush_flow\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    \n)) by (instance, cf) ",
               "format": "time_series",
               "hide": false,
               "instant": false,
               "interval": "",
               "intervalFactor": 1,
-              "legendFormat": "{{stage}} {{$additional_groupby}}",
+              "legendFormat": "{{cf}}_flush_flow-{{instance}}",
               "metric": "",
-              "query": "sum(rate(\n    tikv_scheduler_stage_total\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    [$__rate_interval]\n)) by (stage, $additional_groupby) ",
+              "query": "sum((\n    tikv_scheduler_flush_flow\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    \n)) by (instance, cf) ",
+              "refId": "",
+              "step": 10,
+              "target": ""
+            },
+            {
+              "datasource": "${DS_TEST-CLUSTER}",
+              "expr": "sum((\n    tikv_scheduler_l0_flow\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    \n)) by (instance) ",
+              "format": "time_series",
+              "hide": false,
+              "instant": false,
+              "interval": "",
+              "intervalFactor": 1,
+              "legendFormat": "total_l0_flow-{{instance}}",
+              "metric": "",
+              "query": "sum((\n    tikv_scheduler_l0_flow\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    \n)) by (instance) ",
+              "refId": "",
+              "step": 10,
+              "target": ""
+            },
+            {
+              "datasource": "${DS_TEST-CLUSTER}",
+              "expr": "sum((\n    tikv_scheduler_flush_flow\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    \n)) by (instance) ",
+              "format": "time_series",
+              "hide": false,
+              "instant": false,
+              "interval": "",
+              "intervalFactor": 1,
+              "legendFormat": "total_flush_flow-{{instance}}",
+              "metric": "",
+              "query": "sum((\n    tikv_scheduler_flush_flow\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    \n)) by (instance) ",
               "refId": "",
               "step": 10,
               "target": ""
@@ -41161,7 +40926,7 @@
           "thresholds": [],
           "timeFrom": null,
           "timeShift": null,
-          "title": "Scheduler stage total",
+          "title": "Flush/L0 flow",
           "tooltip": {
             "msResolution": true,
             "shared": true,
@@ -41180,7 +40945,7 @@
           "yaxes": [
             {
               "decimals": null,
-              "format": "ops",
+              "format": "bytes",
               "label": null,
               "logBase": 1,
               "max": null,
@@ -41207,7 +40972,7 @@
           "bars": false,
           "cacheTimeout": null,
           "datasource": "${DS_TEST-CLUSTER}",
-          "description": "The count of different priority commands",
+          "description": "",
           "editable": true,
           "error": false,
           "fieldConfig": {
@@ -41229,12 +40994,12 @@
           "gridPos": {
             "h": 7,
             "w": 12,
-            "x": 12,
-            "y": 0
+            "x": 0,
+            "y": 21
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 290,
+          "id": 288,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -41277,15 +41042,45 @@
           "targets": [
             {
               "datasource": "${DS_TEST-CLUSTER}",
-              "expr": "sum(rate(\n    tikv_scheduler_commands_pri_total\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    [$__rate_interval]\n)) by (priority, $additional_groupby) ",
+              "expr": "max((\n    tikv_scheduler_l0\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    \n)) by (instance) ",
               "format": "time_series",
               "hide": false,
               "instant": false,
               "interval": "",
               "intervalFactor": 1,
-              "legendFormat": "{{priority}} {{$additional_groupby}}",
+              "legendFormat": "l0-{{instance}}",
               "metric": "",
-              "query": "sum(rate(\n    tikv_scheduler_commands_pri_total\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    [$__rate_interval]\n)) by (priority, $additional_groupby) ",
+              "query": "max((\n    tikv_scheduler_l0\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    \n)) by (instance) ",
+              "refId": "",
+              "step": 10,
+              "target": ""
+            },
+            {
+              "datasource": "${DS_TEST-CLUSTER}",
+              "expr": "max((\n    tikv_scheduler_memtable\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    \n)) by (instance) ",
+              "format": "time_series",
+              "hide": false,
+              "instant": false,
+              "interval": "",
+              "intervalFactor": 1,
+              "legendFormat": "memtable-{{instance}}",
+              "metric": "",
+              "query": "max((\n    tikv_scheduler_memtable\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    \n)) by (instance) ",
+              "refId": "",
+              "step": 10,
+              "target": ""
+            },
+            {
+              "datasource": "${DS_TEST-CLUSTER}",
+              "expr": "max((\n    tikv_scheduler_l0_avg\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    \n)) by (instance) ",
+              "format": "time_series",
+              "hide": false,
+              "instant": false,
+              "interval": "",
+              "intervalFactor": 1,
+              "legendFormat": "avg_l0-{{instance}}",
+              "metric": "",
+              "query": "max((\n    tikv_scheduler_l0_avg\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    \n)) by (instance) ",
               "refId": "",
               "step": 10,
               "target": ""
@@ -41294,7 +41089,7 @@
           "thresholds": [],
           "timeFrom": null,
           "timeShift": null,
-          "title": "Scheduler priority commands",
+          "title": "Flow controller factors",
           "tooltip": {
             "msResolution": true,
             "shared": true,
@@ -41313,7 +41108,7 @@
           "yaxes": [
             {
               "decimals": null,
-              "format": "ops",
+              "format": "none",
               "label": null,
               "logBase": 1,
               "max": null,
@@ -41340,7 +41135,7 @@
           "bars": false,
           "cacheTimeout": null,
           "datasource": "${DS_TEST-CLUSTER}",
-          "description": "The count of pending commands per TiKV instance",
+          "description": "",
           "editable": true,
           "error": false,
           "fieldConfig": {
@@ -41362,12 +41157,12 @@
           "gridPos": {
             "h": 7,
             "w": 12,
-            "x": 0,
-            "y": 7
+            "x": 12,
+            "y": 21
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 291,
+          "id": 289,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -41410,15 +41205,30 @@
           "targets": [
             {
               "datasource": "${DS_TEST-CLUSTER}",
-              "expr": "sum((\n    tikv_scheduler_contex_total\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    \n)) by (instance) ",
+              "expr": "sum((\n    tikv_engine_pending_compaction_bytes\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"kv\"}\n    \n)) by (cf, $additional_groupby) ",
               "format": "time_series",
               "hide": false,
               "instant": false,
               "interval": "",
               "intervalFactor": 1,
-              "legendFormat": "{{instance}}",
+              "legendFormat": "{{cf}} {{$additional_groupby}}",
               "metric": "",
-              "query": "sum((\n    tikv_scheduler_contex_total\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    \n)) by (instance) ",
+              "query": "sum((\n    tikv_engine_pending_compaction_bytes\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"kv\"}\n    \n)) by (cf, $additional_groupby) ",
+              "refId": "",
+              "step": 10,
+              "target": ""
+            },
+            {
+              "datasource": "${DS_TEST-CLUSTER}",
+              "expr": "sum((\n    tikv_scheduler_pending_compaction_bytes\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    \n)) by (cf, $additional_groupby)  / 10000000",
+              "format": "time_series",
+              "hide": true,
+              "instant": false,
+              "interval": "",
+              "intervalFactor": 1,
+              "legendFormat": "pending-bytes {{$additional_groupby}}",
+              "metric": "",
+              "query": "sum((\n    tikv_scheduler_pending_compaction_bytes\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    \n)) by (cf, $additional_groupby)  / 10000000",
               "refId": "",
               "step": 10,
               "target": ""
@@ -41427,7 +41237,7 @@
           "thresholds": [],
           "timeFrom": null,
           "timeShift": null,
-          "title": "Scheduler pending commands",
+          "title": "Compaction pending bytes",
           "tooltip": {
             "msResolution": true,
             "shared": true,
@@ -41446,7 +41256,7 @@
           "yaxes": [
             {
               "decimals": null,
-              "format": "none",
+              "format": "bytes",
               "label": null,
               "logBase": 1,
               "max": null,
@@ -41473,7 +41283,7 @@
           "bars": false,
           "cacheTimeout": null,
           "datasource": "${DS_TEST-CLUSTER}",
-          "description": "The count of running commands per TiKV instance",
+          "description": "Throttle time for txn storage commands in 1 minute.",
           "editable": true,
           "error": false,
           "fieldConfig": {
@@ -41495,12 +41305,12 @@
           "gridPos": {
             "h": 7,
             "w": 12,
-            "x": 12,
-            "y": 7
+            "x": 0,
+            "y": 28
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 292,
+          "id": 290,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -41543,15 +41353,15 @@
           "targets": [
             {
               "datasource": "${DS_TEST-CLUSTER}",
-              "expr": "sum((\n    tikv_scheduler_running_commands\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    \n)) by (instance) ",
+              "expr": "sum(rate(\n    tikv_txn_command_throttle_time_total\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    [$__rate_interval]\n)) by (type, $additional_groupby) ",
               "format": "time_series",
               "hide": false,
               "instant": false,
               "interval": "",
               "intervalFactor": 1,
-              "legendFormat": "{{instance}}",
+              "legendFormat": "{{type}} {{$additional_groupby}}",
               "metric": "",
-              "query": "sum((\n    tikv_scheduler_running_commands\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    \n)) by (instance) ",
+              "query": "sum(rate(\n    tikv_txn_command_throttle_time_total\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    [$__rate_interval]\n)) by (type, $additional_groupby) ",
               "refId": "",
               "step": 10,
               "target": ""
@@ -41560,7 +41370,7 @@
           "thresholds": [],
           "timeFrom": null,
           "timeShift": null,
-          "title": "Scheduler running commands",
+          "title": "Txn command throttled duration",
           "tooltip": {
             "msResolution": true,
             "shared": true,
@@ -41579,7 +41389,7 @@
           "yaxes": [
             {
               "decimals": null,
-              "format": "none",
+              "format": "\u00b5s",
               "label": null,
               "logBase": 1,
               "max": null,
@@ -41606,7 +41416,7 @@
           "bars": false,
           "cacheTimeout": null,
           "datasource": "${DS_TEST-CLUSTER}",
-          "description": "The total writing bytes of commands on each stage",
+          "description": "Throttle time for non-txn related processing like analyze or dag in 1 minute.",
           "editable": true,
           "error": false,
           "fieldConfig": {
@@ -41628,12 +41438,12 @@
           "gridPos": {
             "h": 7,
             "w": 12,
-            "x": 0,
-            "y": 14
+            "x": 12,
+            "y": 28
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 293,
+          "id": 291,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -41676,15 +41486,15 @@
           "targets": [
             {
               "datasource": "${DS_TEST-CLUSTER}",
-              "expr": "sum((\n    tikv_scheduler_writing_bytes\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    \n)) by (instance) ",
+              "expr": "sum(rate(\n    tikv_non_txn_command_throttle_time_total\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    [$__rate_interval]\n)) by (type, $additional_groupby) ",
               "format": "time_series",
               "hide": false,
               "instant": false,
               "interval": "",
               "intervalFactor": 1,
-              "legendFormat": "{{instance}}",
+              "legendFormat": "{{type}} {{$additional_groupby}}",
               "metric": "",
-              "query": "sum((\n    tikv_scheduler_writing_bytes\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    \n)) by (instance) ",
+              "query": "sum(rate(\n    tikv_non_txn_command_throttle_time_total\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    [$__rate_interval]\n)) by (type, $additional_groupby) ",
               "refId": "",
               "step": 10,
               "target": ""
@@ -41693,7 +41503,7 @@
           "thresholds": [],
           "timeFrom": null,
           "timeShift": null,
-          "title": "Scheduler writing bytes",
+          "title": "Non-txn command throttled duration",
           "tooltip": {
             "msResolution": true,
             "shared": true,
@@ -41712,7 +41522,7 @@
           "yaxes": [
             {
               "decimals": null,
-              "format": "bytes",
+              "format": "\u00b5s",
               "label": null,
               "logBase": 1,
               "max": null,
@@ -41733,13 +41543,55 @@
             "align": false,
             "alignLevel": 0
           }
-        },
+        }
+      ],
+      "repeat": null,
+      "repeatDirection": null,
+      "span": null,
+      "targets": [],
+      "timeFrom": null,
+      "timeShift": null,
+      "title": "Flow Control",
+      "transformations": [],
+      "transparent": false,
+      "type": "row"
+    },
+    {
+      "cacheTimeout": null,
+      "collapsed": true,
+      "datasource": null,
+      "description": null,
+      "editable": true,
+      "error": false,
+      "fieldConfig": {
+        "defaults": {
+          "thresholds": {
+            "mode": "absolute",
+            "steps": []
+          }
+        }
+      },
+      "gridPos": {
+        "h": 7,
+        "w": 24,
+        "x": 0,
+        "y": 0
+      },
+      "height": null,
+      "hideTimeOverride": false,
+      "id": 292,
+      "interval": null,
+      "links": [],
+      "maxDataPoints": 100,
+      "maxPerRow": null,
+      "minSpan": null,
+      "panels": [
         {
           "aliasColors": {},
           "bars": false,
           "cacheTimeout": null,
           "datasource": "${DS_TEST-CLUSTER}",
-          "description": "The number of bytes used by scheduler",
+          "description": "The total number of commands on each stage",
           "editable": true,
           "error": false,
           "fieldConfig": {
@@ -41761,12 +41613,12 @@
           "gridPos": {
             "h": 7,
             "w": 12,
-            "x": 12,
-            "y": 14
+            "x": 0,
+            "y": 0
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 294,
+          "id": 293,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -41809,30 +41661,30 @@
           "targets": [
             {
               "datasource": "${DS_TEST-CLUSTER}",
-              "expr": "sum((\n    tikv_scheduler_memory_quota_size\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"in_use\"}\n    \n)) by (instance) ",
+              "expr": "sum(rate(\n    tikv_scheduler_too_busy_total\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    [$__rate_interval]\n)) by (stage, $additional_groupby) ",
               "format": "time_series",
               "hide": false,
               "instant": false,
               "interval": "",
               "intervalFactor": 1,
-              "legendFormat": "{{instance}}",
+              "legendFormat": "{{stage}} {{$additional_groupby}}",
               "metric": "",
-              "query": "sum((\n    tikv_scheduler_memory_quota_size\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"in_use\"}\n    \n)) by (instance) ",
+              "query": "sum(rate(\n    tikv_scheduler_too_busy_total\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    [$__rate_interval]\n)) by (stage, $additional_groupby) ",
               "refId": "",
               "step": 10,
               "target": ""
             },
             {
               "datasource": "${DS_TEST-CLUSTER}",
-              "expr": "sum((\n    tikv_scheduler_memory_quota_size\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"capacity\"}\n    \n)) by (instance) ",
+              "expr": "sum(rate(\n    tikv_scheduler_stage_total\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    [$__rate_interval]\n)) by (stage, $additional_groupby) ",
               "format": "time_series",
-              "hide": true,
+              "hide": false,
               "instant": false,
               "interval": "",
               "intervalFactor": 1,
-              "legendFormat": "{{instance}}",
+              "legendFormat": "{{stage}} {{$additional_groupby}}",
               "metric": "",
-              "query": "sum((\n    tikv_scheduler_memory_quota_size\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"capacity\"}\n    \n)) by (instance) ",
+              "query": "sum(rate(\n    tikv_scheduler_stage_total\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    [$__rate_interval]\n)) by (stage, $additional_groupby) ",
               "refId": "",
               "step": 10,
               "target": ""
@@ -41841,7 +41693,7 @@
           "thresholds": [],
           "timeFrom": null,
           "timeShift": null,
-          "title": "Scheduler memory quota",
+          "title": "Scheduler stage total",
           "tooltip": {
             "msResolution": true,
             "shared": true,
@@ -41860,7 +41712,7 @@
           "yaxes": [
             {
               "decimals": null,
-              "format": "bytes",
+              "format": "ops",
               "label": null,
               "logBase": 1,
               "max": null,
@@ -41882,159 +41734,12 @@
             "alignLevel": 0
           }
         },
-        {
-          "cacheTimeout": null,
-          "cards": {
-            "cardPadding": null,
-            "cardRound": null
-          },
-          "color": {
-            "cardColor": "#b4ff00",
-            "colorScale": "sqrt",
-            "colorScheme": "interpolateSpectral",
-            "exponent": 0.5,
-            "max": null,
-            "min": null,
-            "mode": "spectrum"
-          },
-          "dataFormat": "tsbuckets",
-          "datasource": "${DS_TEST-CLUSTER}",
-          "description": null,
-          "editable": true,
-          "error": false,
-          "fieldConfig": {
-            "defaults": {
-              "thresholds": {
-                "mode": "absolute",
-                "steps": []
-              }
-            }
-          },
-          "gridPos": {
-            "h": 7,
-            "w": 24,
-            "x": 0,
-            "y": 21
-          },
-          "heatmap": {},
-          "height": null,
-          "hideTimeOverride": false,
-          "hideZeroBuckets": true,
-          "highlightCards": true,
-          "id": 295,
-          "interval": null,
-          "legend": {
-            "show": false
-          },
-          "links": [],
-          "maxDataPoints": 512,
-          "maxPerRow": null,
-          "minSpan": null,
-          "options": {},
-          "repeat": null,
-          "repeatDirection": null,
-          "reverseYBuckets": false,
-          "span": null,
-          "targets": [
-            {
-              "datasource": "${DS_TEST-CLUSTER}",
-              "expr": "sum(increase(\n    tikv_yatp_pool_schedule_wait_duration_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"sched-worker.*\"}\n    [$__rate_interval]\n)) by (le) ",
-              "format": "heatmap",
-              "hide": false,
-              "instant": false,
-              "interval": "",
-              "intervalFactor": 1,
-              "legendFormat": "{{le}}",
-              "metric": "",
-              "query": "sum(increase(\n    tikv_yatp_pool_schedule_wait_duration_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"sched-worker.*\"}\n    [$__rate_interval]\n)) by (le) ",
-              "refId": "",
-              "step": 10,
-              "target": ""
-            }
-          ],
-          "timeFrom": null,
-          "timeShift": null,
-          "title": "Txn Scheduler Pool Wait Duration",
-          "tooltip": {
-            "msResolution": true,
-            "shared": true,
-            "show": true,
-            "showHistogram": true,
-            "sort": 0,
-            "value_type": "individual"
-          },
-          "transformations": [],
-          "transparent": false,
-          "type": "heatmap",
-          "xAxis": {
-            "mode": "time",
-            "name": null,
-            "show": true,
-            "values": []
-          },
-          "xBucketNumber": null,
-          "xBucketSize": null,
-          "yAxis": {
-            "decimals": 1,
-            "format": "s",
-            "label": null,
-            "logBase": 1,
-            "max": null,
-            "min": null,
-            "show": true
-          },
-          "yBucketBound": "upper",
-          "yBucketNumber": null,
-          "yBucketSize": null
-        }
-      ],
-      "repeat": null,
-      "repeatDirection": null,
-      "span": null,
-      "targets": [],
-      "timeFrom": null,
-      "timeShift": null,
-      "title": "Scheduler",
-      "transformations": [],
-      "transparent": false,
-      "type": "row"
-    },
-    {
-      "cacheTimeout": null,
-      "collapsed": true,
-      "datasource": null,
-      "description": null,
-      "editable": true,
-      "error": false,
-      "fieldConfig": {
-        "defaults": {
-          "thresholds": {
-            "mode": "absolute",
-            "steps": []
-          }
-        }
-      },
-      "gridPos": {
-        "h": 7,
-        "w": 24,
-        "x": 0,
-        "y": 0
-      },
-      "height": null,
-      "hideTimeOverride": false,
-      "id": 296,
-      "interval": null,
-      "links": [],
-      "maxDataPoints": 100,
-      "maxPerRow": null,
-      "minSpan": null,
-      "panels": [
         {
           "aliasColors": {},
           "bars": false,
           "cacheTimeout": null,
           "datasource": "${DS_TEST-CLUSTER}",
-          "description": "The time used by each level in the yatp thread pool per second. Level 0 refers to small queries.",
+          "description": "The count of different priority commands",
           "editable": true,
           "error": false,
           "fieldConfig": {
@@ -42056,12 +41761,12 @@
           "gridPos": {
             "h": 7,
             "w": 12,
-            "x": 0,
+            "x": 12,
             "y": 0
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 297,
+          "id": 294,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -42104,15 +41809,15 @@
           "targets": [
             {
               "datasource": "${DS_TEST-CLUSTER}",
-              "expr": "sum(rate(\n    tikv_multilevel_level_elapsed\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"sched-worker.*\"}\n    [$__rate_interval]\n)) by (level, $additional_groupby) ",
+              "expr": "sum(rate(\n    tikv_scheduler_commands_pri_total\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    [$__rate_interval]\n)) by (priority, $additional_groupby) ",
               "format": "time_series",
               "hide": false,
               "instant": false,
               "interval": "",
               "intervalFactor": 1,
-              "legendFormat": "{{level}} {{$additional_groupby}}",
+              "legendFormat": "{{priority}} {{$additional_groupby}}",
               "metric": "",
-              "query": "sum(rate(\n    tikv_multilevel_level_elapsed\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"sched-worker.*\"}\n    [$__rate_interval]\n)) by (level, $additional_groupby) ",
+              "query": "sum(rate(\n    tikv_scheduler_commands_pri_total\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    [$__rate_interval]\n)) by (priority, $additional_groupby) ",
               "refId": "",
               "step": 10,
               "target": ""
@@ -42121,7 +41826,7 @@
           "thresholds": [],
           "timeFrom": null,
           "timeShift": null,
-          "title": "Time used by level",
+          "title": "Scheduler priority commands",
           "tooltip": {
             "msResolution": true,
             "shared": true,
@@ -42140,7 +41845,7 @@
           "yaxes": [
             {
               "decimals": null,
-              "format": "\u00b5s",
+              "format": "ops",
               "label": null,
               "logBase": 1,
               "max": null,
@@ -42167,7 +41872,7 @@
           "bars": false,
           "cacheTimeout": null,
           "datasource": "${DS_TEST-CLUSTER}",
-          "description": "The chance that level 0 (small) tasks are scheduled in the yatp thread pool.",
+          "description": "The count of pending commands per TiKV instance",
           "editable": true,
           "error": false,
           "fieldConfig": {
@@ -42189,12 +41894,12 @@
           "gridPos": {
             "h": 7,
             "w": 12,
-            "x": 12,
-            "y": 0
+            "x": 0,
+            "y": 7
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 298,
+          "id": 295,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -42237,7 +41942,7 @@
           "targets": [
             {
               "datasource": "${DS_TEST-CLUSTER}",
-              "expr": "((\n    tikv_multilevel_level0_chance\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"sched-worker.*\"}\n    \n))  ",
+              "expr": "sum((\n    tikv_scheduler_contex_total\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    \n)) by (instance) ",
               "format": "time_series",
               "hide": false,
               "instant": false,
@@ -42245,7 +41950,7 @@
               "intervalFactor": 1,
               "legendFormat": "{{instance}}",
               "metric": "",
-              "query": "((\n    tikv_multilevel_level0_chance\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"sched-worker.*\"}\n    \n))  ",
+              "query": "sum((\n    tikv_scheduler_contex_total\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    \n)) by (instance) ",
               "refId": "",
               "step": 10,
               "target": ""
@@ -42254,7 +41959,7 @@
           "thresholds": [],
           "timeFrom": null,
           "timeShift": null,
-          "title": "Level 0 chance",
+          "title": "Scheduler pending commands",
           "tooltip": {
             "msResolution": true,
             "shared": true,
@@ -42273,7 +41978,7 @@
           "yaxes": [
             {
               "decimals": null,
-              "format": "percentunit",
+              "format": "none",
               "label": null,
               "logBase": 1,
               "max": null,
@@ -42300,7 +42005,7 @@
           "bars": false,
           "cacheTimeout": null,
           "datasource": "${DS_TEST-CLUSTER}",
-          "description": "The number of concurrently running tasks in the yatp thread pool.",
+          "description": "The count of running commands per TiKV instance",
           "editable": true,
           "error": false,
           "fieldConfig": {
@@ -42322,12 +42027,12 @@
           "gridPos": {
             "h": 7,
             "w": 12,
-            "x": 0,
+            "x": 12,
             "y": 7
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 299,
+          "id": 296,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -42370,15 +42075,15 @@
           "targets": [
             {
               "datasource": "${DS_TEST-CLUSTER}",
-              "expr": "sum(avg_over_time(\n    tikv_scheduler_running_commands\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    [1m]\n)) by (instance, $additional_groupby) ",
+              "expr": "sum((\n    tikv_scheduler_running_commands\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    \n)) by (instance) ",
               "format": "time_series",
               "hide": false,
               "instant": false,
               "interval": "",
               "intervalFactor": 1,
-              "legendFormat": "{{instance}} {{$additional_groupby}}",
+              "legendFormat": "{{instance}}",
               "metric": "",
-              "query": "sum(avg_over_time(\n    tikv_scheduler_running_commands\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    [1m]\n)) by (instance, $additional_groupby) ",
+              "query": "sum((\n    tikv_scheduler_running_commands\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    \n)) by (instance) ",
               "refId": "",
               "step": 10,
               "target": ""
@@ -42387,7 +42092,7 @@
           "thresholds": [],
           "timeFrom": null,
           "timeShift": null,
-          "title": "Running tasks",
+          "title": "Scheduler running commands",
           "tooltip": {
             "msResolution": true,
             "shared": true,
@@ -42429,23 +42134,11 @@
           }
         },
         {
+          "aliasColors": {},
+          "bars": false,
           "cacheTimeout": null,
-          "cards": {
-            "cardPadding": null,
-            "cardRound": null
-          },
-          "color": {
-            "cardColor": "#b4ff00",
-            "colorScale": "sqrt",
-            "colorScheme": "interpolateSpectral",
-            "exponent": 0.5,
-            "max": null,
-            "min": null,
-            "mode": "spectrum"
-          },
-          "dataFormat": "tsbuckets",
           "datasource": "${DS_TEST-CLUSTER}",
-          "description": null,
+          "description": "The total writing bytes of commands on each stage",
           "editable": true,
           "error": false,
           "fieldConfig": {
@@ -42456,89 +42149,129 @@
               }
             }
           },
+          "fill": 1,
+          "fillGradient": 1,
+          "grid": {
+            "threshold1": null,
+            "threshold1Color": "rgba(216, 200, 27, 0.27)",
+            "threshold2": null,
+            "threshold2Color": "rgba(234, 112, 112, 0.22)"
+          },
           "gridPos": {
             "h": 7,
             "w": 12,
-            "x": 12,
-            "y": 7
+            "x": 0,
+            "y": 14
           },
-          "heatmap": {},
           "height": null,
           "hideTimeOverride": false,
-          "hideZeroBuckets": true,
-          "highlightCards": true,
-          "id": 300,
+          "id": 297,
           "interval": null,
+          "isNew": true,
           "legend": {
-            "show": false
+            "alignAsTable": true,
+            "avg": false,
+            "current": true,
+            "hideEmpty": true,
+            "hideZero": true,
+            "max": true,
+            "min": false,
+            "rightSide": true,
+            "show": true,
+            "sideWidth": null,
+            "sort": "max",
+            "sortDesc": true,
+            "total": false,
+            "values": true
           },
+          "lines": true,
+          "linewidth": 1,
           "links": [],
-          "maxDataPoints": 512,
+          "maxDataPoints": null,
           "maxPerRow": null,
           "minSpan": null,
-          "options": {},
+          "nullPointMode": "null as zero",
+          "options": {
+            "alertThreshold": true,
+            "dataLinks": []
+          },
+          "percentage": false,
+          "pointradius": 5,
+          "points": false,
+          "renderer": "flot",
           "repeat": null,
           "repeatDirection": null,
-          "reverseYBuckets": false,
+          "seriesOverrides": [],
           "span": null,
+          "stack": false,
+          "steppedLine": false,
           "targets": [
             {
               "datasource": "${DS_TEST-CLUSTER}",
-              "expr": "sum(increase(\n    tikv_yatp_pool_schedule_wait_duration_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"sched-worker.*\"}\n    [$__rate_interval]\n)) by (le) ",
-              "format": "heatmap",
+              "expr": "sum((\n    tikv_scheduler_writing_bytes\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    \n)) by (instance) ",
+              "format": "time_series",
               "hide": false,
               "instant": false,
               "interval": "",
               "intervalFactor": 1,
-              "legendFormat": "{{le}}",
+              "legendFormat": "{{instance}}",
               "metric": "",
-              "query": "sum(increase(\n    tikv_yatp_pool_schedule_wait_duration_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"sched-worker.*\"}\n    [$__rate_interval]\n)) by (le) ",
+              "query": "sum((\n    tikv_scheduler_writing_bytes\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    \n)) by (instance) ",
               "refId": "",
               "step": 10,
               "target": ""
             }
           ],
+          "thresholds": [],
           "timeFrom": null,
           "timeShift": null,
-          "title": "Wait Duration",
+          "title": "Scheduler writing bytes",
           "tooltip": {
             "msResolution": true,
             "shared": true,
-            "show": true,
-            "showHistogram": true,
             "sort": 0,
             "value_type": "individual"
           },
           "transformations": [],
           "transparent": false,
-          "type": "heatmap",
-          "xAxis": {
+          "type": "graph",
+          "xaxis": {
             "mode": "time",
             "name": null,
             "show": true,
             "values": []
           },
-          "xBucketNumber": null,
-          "xBucketSize": null,
-          "yAxis": {
-            "decimals": 1,
-            "format": "s",
-            "label": null,
-            "logBase": 1,
-            "max": null,
-            "min": null,
-            "show": true
-          },
-          "yBucketBound": "upper",
-          "yBucketNumber": null,
-          "yBucketSize": null
+          "yaxes": [
+            {
+              "decimals": null,
+              "format": "bytes",
+              "label": null,
+              "logBase": 1,
+              "max": null,
+              "min": null,
+              "show": true
+            },
+            {
+              "decimals": null,
+              "format": "short",
+              "label": null,
+              "logBase": 1,
+              "max": null,
+              "min": null,
+              "show": true
+            }
+          ],
+          "yaxis": {
+            "align": false,
+            "alignLevel": 0
+          }
         },
         {
           "aliasColors": {},
           "bars": false,
           "cacheTimeout": null,
           "datasource": "${DS_TEST-CLUSTER}",
-          "description": "Task execution time during one schedule.",
+          "description": "The number of bytes used by scheduler",
           "editable": true,
           "error": false,
           "fieldConfig": {
@@ -42560,12 +42293,811 @@
           "gridPos": {
             "h": 7,
             "w": 12,
-            "x": 0,
+            "x": 12,
             "y": 14
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 301,
+          "id": 298,
+          "interval": null,
+          "isNew": true,
+          "legend": {
+            "alignAsTable": true,
+            "avg": false,
+            "current": true,
+            "hideEmpty": true,
+            "hideZero": true,
+            "max": true,
+            "min": false,
+            "rightSide": true,
+            "show": true,
+            "sideWidth": null,
+            "sort": "max",
+            "sortDesc": true,
+            "total": false,
+            "values": true
+          },
+          "lines": true,
+          "linewidth": 1,
+          "links": [],
+          "maxDataPoints": null,
+          "maxPerRow": null,
+          "minSpan": null,
+          "nullPointMode": "null as zero",
+          "options": {
+            "alertThreshold": true,
+            "dataLinks": []
+          },
+          "percentage": false,
+          "pointradius": 5,
+          "points": false,
+          "renderer": "flot",
+          "repeat": null,
+          "repeatDirection": null,
+          "seriesOverrides": [],
+          "span": null,
+          "stack": false,
+          "steppedLine": false,
+          "targets": [
+            {
+              "datasource": "${DS_TEST-CLUSTER}",
+              "expr": "sum((\n    tikv_scheduler_memory_quota_size\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"in_use\"}\n    \n)) by (instance) ",
+              "format": "time_series",
+              "hide": false,
+              "instant": false,
+              "interval": "",
+              "intervalFactor": 1,
+              "legendFormat": "{{instance}}",
+              "metric": "",
+              "query": "sum((\n    tikv_scheduler_memory_quota_size\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"in_use\"}\n    \n)) by (instance) ",
+              "refId": "",
+              "step": 10,
+              "target": ""
+            },
+            {
+              "datasource": "${DS_TEST-CLUSTER}",
+              "expr": "sum((\n    tikv_scheduler_memory_quota_size\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"capacity\"}\n    \n)) by (instance) ",
+              "format": "time_series",
+              "hide": true,
+              "instant": false,
+              "interval": "",
+              "intervalFactor": 1,
+              "legendFormat": "{{instance}}",
+              "metric": "",
+              "query": "sum((\n    tikv_scheduler_memory_quota_size\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"capacity\"}\n    \n)) by (instance) ",
+              "refId": "",
+              "step": 10,
+              "target": ""
+            }
+          ],
+          "thresholds": [],
+          "timeFrom": null,
+          "timeShift": null,
+          "title": "Scheduler memory quota",
+          "tooltip": {
+            "msResolution": true,
+            "shared": true,
+            "sort": 0,
+            "value_type": "individual"
+          },
+          "transformations": [],
+          "transparent": false,
+          "type": "graph",
+          "xaxis": {
+            "mode": "time",
+            "name": null,
+            "show": true,
+            "values": []
+          },
+          "yaxes": [
+            {
+              "decimals": null,
+              "format": "bytes",
+              "label": null,
+              "logBase": 1,
+              "max": null,
+              "min": null,
+              "show": true
+            },
+            {
+              "decimals": null,
+              "format": "short",
+              "label": null,
+              "logBase": 1,
+              "max": null,
+              "min": null,
+              "show": true
+            }
+          ],
+          "yaxis": {
+            "align": false,
+            "alignLevel": 0
+          }
+        },
+        {
+          "cacheTimeout": null,
+          "cards": {
+            "cardPadding": null,
+            "cardRound": null
+          },
+          "color": {
+            "cardColor": "#b4ff00",
+            "colorScale": "sqrt",
+            "colorScheme": "interpolateSpectral",
+            "exponent": 0.5,
+            "max": null,
+            "min": null,
+            "mode": "spectrum"
+          },
+          "dataFormat": "tsbuckets",
+          "datasource": "${DS_TEST-CLUSTER}",
+          "description": null,
+          "editable": true,
+          "error": false,
+          "fieldConfig": {
+            "defaults": {
+              "thresholds": {
+                "mode": "absolute",
+                "steps": []
+              }
+            }
+          },
+          "gridPos": {
+            "h": 7,
+            "w": 24,
+            "x": 0,
+            "y": 21
+          },
+          "heatmap": {},
+          "height": null,
+          "hideTimeOverride": false,
+          "hideZeroBuckets": true,
+          "highlightCards": true,
+          "id": 299,
+          "interval": null,
+          "legend": {
+            "show": false
+          },
+          "links": [],
+          "maxDataPoints": 512,
+          "maxPerRow": null,
+          "minSpan": null,
+          "options": {},
+          "repeat": null,
+          "repeatDirection": null,
+          "reverseYBuckets": false,
+          "span": null,
+          "targets": [
+            {
+              "datasource": "${DS_TEST-CLUSTER}",
+              "expr": "sum(increase(\n    tikv_yatp_pool_schedule_wait_duration_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"sched-worker.*\"}\n    [$__rate_interval]\n)) by (le) ",
+              "format": "heatmap",
+              "hide": false,
+              "instant": false,
+              "interval": "",
+              "intervalFactor": 1,
+              "legendFormat": "{{le}}",
+              "metric": "",
+              "query": "sum(increase(\n    tikv_yatp_pool_schedule_wait_duration_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"sched-worker.*\"}\n    [$__rate_interval]\n)) by (le) ",
+              "refId": "",
+              "step": 10,
+              "target": ""
+            }
+          ],
+          "timeFrom": null,
+          "timeShift": null,
+          "title": "Txn Scheduler Pool Wait Duration",
+          "tooltip": {
+            "msResolution": true,
+            "shared": true,
+            "show": true,
+            "showHistogram": true,
+            "sort": 0,
+            "value_type": "individual"
+          },
+          "transformations": [],
+          "transparent": false,
+          "type": "heatmap",
+          "xAxis": {
+            "mode": "time",
+            "name": null,
+            "show": true,
+            "values": []
+          },
+          "xBucketNumber": null,
+          "xBucketSize": null,
+          "yAxis": {
+            "decimals": 1,
+            "format": "s",
+            "label": null,
+            "logBase": 1,
+            "max": null,
+            "min": null,
+            "show": true
+          },
+          "yBucketBound": "upper",
+          "yBucketNumber": null,
+          "yBucketSize": null
+        }
+      ],
+      "repeat": null,
+      "repeatDirection": null,
+      "span": null,
+      "targets": [],
+      "timeFrom": null,
+      "timeShift": null,
+      "title": "Scheduler",
+      "transformations": [],
+      "transparent": false,
+      "type": "row"
+    },
+    {
+      "cacheTimeout": null,
+      "collapsed": true,
+      "datasource": null,
+      "description": null,
+      "editable": true,
+      "error": false,
+      "fieldConfig": {
+        "defaults": {
+          "thresholds": {
+            "mode": "absolute",
+            "steps": []
+          }
+        }
+      },
+      "gridPos": {
+        "h": 7,
+        "w": 24,
+        "x": 0,
+        "y": 0
+      },
+      "height": null,
+      "hideTimeOverride": false,
+      "id": 300,
+      "interval": null,
+      "links": [],
+      "maxDataPoints": 100,
+      "maxPerRow": null,
+      "minSpan": null,
+      "panels": [
+        {
+          "aliasColors": {},
+          "bars": false,
+          "cacheTimeout": null,
+          "datasource": "${DS_TEST-CLUSTER}",
+          "description": "The time used by each level in the yatp thread pool per second. Level 0 refers to small queries.",
+          "editable": true,
+          "error": false,
+          "fieldConfig": {
+            "defaults": {
+              "thresholds": {
+                "mode": "absolute",
+                "steps": []
+              }
+            }
+          },
+          "fill": 1,
+          "fillGradient": 1,
+          "grid": {
+            "threshold1": null,
+            "threshold1Color": "rgba(216, 200, 27, 0.27)",
+            "threshold2": null,
+            "threshold2Color": "rgba(234, 112, 112, 0.22)"
+          },
+          "gridPos": {
+            "h": 7,
+            "w": 12,
+            "x": 0,
+            "y": 0
+          },
+          "height": null,
+          "hideTimeOverride": false,
+          "id": 301,
+          "interval": null,
+          "isNew": true,
+          "legend": {
+            "alignAsTable": true,
+            "avg": false,
+            "current": true,
+            "hideEmpty": true,
+            "hideZero": true,
+            "max": true,
+            "min": false,
+            "rightSide": true,
+            "show": true,
+            "sideWidth": null,
+            "sort": "max",
+            "sortDesc": true,
+            "total": false,
+            "values": true
+          },
+          "lines": true,
+          "linewidth": 1,
+          "links": [],
+          "maxDataPoints": null,
+          "maxPerRow": null,
+          "minSpan": null,
+          "nullPointMode": "null as zero",
+          "options": {
+            "alertThreshold": true,
+            "dataLinks": []
+          },
+          "percentage": false,
+          "pointradius": 5,
+          "points": false,
+          "renderer": "flot",
+          "repeat": null,
+          "repeatDirection": null,
+          "seriesOverrides": [],
+          "span": null,
+          "stack": false,
+          "steppedLine": false,
+          "targets": [
+            {
+              "datasource": "${DS_TEST-CLUSTER}",
+              "expr": "sum(rate(\n    tikv_multilevel_level_elapsed\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"sched-worker.*\"}\n    [$__rate_interval]\n)) by (level, $additional_groupby) ",
+              "format": "time_series",
+              "hide": false,
+              "instant": false,
+              "interval": "",
+              "intervalFactor": 1,
+              "legendFormat": "{{level}} {{$additional_groupby}}",
+              "metric": "",
+              "query": "sum(rate(\n    tikv_multilevel_level_elapsed\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"sched-worker.*\"}\n    [$__rate_interval]\n)) by (level, $additional_groupby) ",
+              "refId": "",
+              "step": 10,
+              "target": ""
+            }
+          ],
+          "thresholds": [],
+          "timeFrom": null,
+          "timeShift": null,
+          "title": "Time used by level",
+          "tooltip": {
+            "msResolution": true,
+            "shared": true,
+            "sort": 0,
+            "value_type": "individual"
+          },
+          "transformations": [],
+          "transparent": false,
+          "type": "graph",
+          "xaxis": {
+            "mode": "time",
+            "name": null,
+            "show": true,
+            "values": []
+          },
+          "yaxes": [
+            {
+              "decimals": null,
+              "format": "\u00b5s",
+              "label": null,
+              "logBase": 1,
+              "max": null,
+              "min": null,
+              "show": true
+            },
+            {
+              "decimals": null,
+              "format": "short",
+              "label": null,
+              "logBase": 1,
+              "max": null,
+              "min": null,
+              "show": true
+            }
+          ],
+          "yaxis": {
+            "align": false,
+            "alignLevel": 0
+          }
+        },
+        {
+          "aliasColors": {},
+          "bars": false,
+          "cacheTimeout": null,
+          "datasource": "${DS_TEST-CLUSTER}",
+          "description": "The chance that level 0 (small) tasks are scheduled in the yatp thread pool.",
+          "editable": true,
+          "error": false,
+          "fieldConfig": {
+            "defaults": {
+              "thresholds": {
+                "mode": "absolute",
+                "steps": []
+              }
+            }
+          },
+          "fill": 1,
+          "fillGradient": 1,
+          "grid": {
+            "threshold1": null,
+            "threshold1Color": "rgba(216, 200, 27, 0.27)",
+            "threshold2": null,
+            "threshold2Color": "rgba(234, 112, 112, 0.22)"
+          },
+          "gridPos": {
+            "h": 7,
+            "w": 12,
+            "x": 12,
+            "y": 0
+          },
+          "height": null,
+          "hideTimeOverride": false,
+          "id": 302,
+          "interval": null,
+          "isNew": true,
+          "legend": {
+            "alignAsTable": true,
+            "avg": false,
+            "current": true,
+            "hideEmpty": true,
+            "hideZero": true,
+            "max": true,
+            "min": false,
+            "rightSide": true,
+            "show": true,
+            "sideWidth": null,
+            "sort": "max",
+            "sortDesc": true,
+            "total": false,
+            "values": true
+          },
+          "lines": true,
+          "linewidth": 1,
+          "links": [],
+          "maxDataPoints": null,
+          "maxPerRow": null,
+          "minSpan": null,
+          "nullPointMode": "null as zero",
+          "options": {
+            "alertThreshold": true,
+            "dataLinks": []
+          },
+          "percentage": false,
+          "pointradius": 5,
+          "points": false,
+          "renderer": "flot",
+          "repeat": null,
+          "repeatDirection": null,
+          "seriesOverrides": [],
+          "span": null,
+          "stack": false,
+          "steppedLine": false,
+          "targets": [
+            {
+              "datasource": "${DS_TEST-CLUSTER}",
+              "expr": "((\n    tikv_multilevel_level0_chance\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"sched-worker.*\"}\n    \n))  ",
+              "format": "time_series",
+              "hide": false,
+              "instant": false,
+              "interval": "",
+              "intervalFactor": 1,
+              "legendFormat": "{{instance}}",
+              "metric": "",
+              "query": "((\n    tikv_multilevel_level0_chance\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"sched-worker.*\"}\n    \n))  ",
+              "refId": "",
+              "step": 10,
+              "target": ""
+            }
+          ],
+          "thresholds": [],
+          "timeFrom": null,
+          "timeShift": null,
+          "title": "Level 0 chance",
+          "tooltip": {
+            "msResolution": true,
+            "shared": true,
+            "sort": 0,
+            "value_type": "individual"
+          },
+          "transformations": [],
+          "transparent": false,
+          "type": "graph",
+          "xaxis": {
+            "mode": "time",
+            "name": null,
+            "show": true,
+            "values": []
+          },
+          "yaxes": [
+            {
+              "decimals": null,
+              "format": "percentunit",
+              "label": null,
+              "logBase": 1,
+              "max": null,
+              "min": null,
+              "show": true
+            },
+            {
+              "decimals": null,
+              "format": "short",
+              "label": null,
+              "logBase": 1,
+              "max": null,
+              "min": null,
+              "show": true
+            }
+          ],
+          "yaxis": {
+            "align": false,
+            "alignLevel": 0
+          }
+        },
+        {
+          "aliasColors": {},
+          "bars": false,
+          "cacheTimeout": null,
+          "datasource": "${DS_TEST-CLUSTER}",
+          "description": "The number of concurrently running tasks in the yatp thread pool.",
+          "editable": true,
+          "error": false,
+          "fieldConfig": {
+            "defaults": {
+              "thresholds": {
+                "mode": "absolute",
+                "steps": []
+              }
+            }
+          },
+          "fill": 1,
+          "fillGradient": 1,
+          "grid": {
+            "threshold1": null,
+            "threshold1Color": "rgba(216, 200, 27, 0.27)",
+            "threshold2": null,
+            "threshold2Color": "rgba(234, 112, 112, 0.22)"
+          },
+          "gridPos": {
+            "h": 7,
+            "w": 12,
+            "x": 0,
+            "y": 7
+          },
+          "height": null,
+          "hideTimeOverride": false,
+          "id": 303,
+          "interval": null,
+          "isNew": true,
+          "legend": {
+            "alignAsTable": true,
+            "avg": false,
+            "current": true,
+            "hideEmpty": true,
+            "hideZero": true,
+            "max": true,
+            "min": false,
+            "rightSide": true,
+            "show": true,
+            "sideWidth": null,
+            "sort": "max",
+            "sortDesc": true,
+            "total": false,
+            "values": true
+          },
+          "lines": true,
+          "linewidth": 1,
+          "links": [],
+          "maxDataPoints": null,
+          "maxPerRow": null,
+          "minSpan": null,
+          "nullPointMode": "null as zero",
+          "options": {
+            "alertThreshold": true,
+            "dataLinks": []
+          },
+          "percentage": false,
+          "pointradius": 5,
+          "points": false,
+          "renderer": "flot",
+          "repeat": null,
+          "repeatDirection": null,
+          "seriesOverrides": [],
+          "span": null,
+          "stack": false,
+          "steppedLine": false,
+          "targets": [
+            {
+              "datasource": "${DS_TEST-CLUSTER}",
+              "expr": "sum(avg_over_time(\n    tikv_scheduler_running_commands\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    [1m]\n)) by (instance, $additional_groupby) ",
+              "format": "time_series",
+              "hide": false,
+              "instant": false,
+              "interval": "",
+              "intervalFactor": 1,
+              "legendFormat": "{{instance}} {{$additional_groupby}}",
+              "metric": "",
+              "query": "sum(avg_over_time(\n    tikv_scheduler_running_commands\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    [1m]\n)) by (instance, $additional_groupby) ",
+              "refId": "",
+              "step": 10,
+              "target": ""
+            }
+          ],
+          "thresholds": [],
+          "timeFrom": null,
+          "timeShift": null,
+          "title": "Running tasks",
+          "tooltip": {
+            "msResolution": true,
+            "shared": true,
+            "sort": 0,
+            "value_type": "individual"
+          },
+          "transformations": [],
+          "transparent": false,
+          "type": "graph",
+          "xaxis": {
+            "mode": "time",
+            "name": null,
+            "show": true,
+            "values": []
+          },
+          "yaxes": [
+            {
+              "decimals": null,
+              "format": "none",
+              "label": null,
+              "logBase": 1,
+              "max": null,
+              "min": null,
+              "show": true
+            },
+            {
+              "decimals": null,
+              "format": "short",
+              "label": null,
+              "logBase": 1,
+              "max": null,
+              "min": null,
+              "show": true
+            }
+          ],
+          "yaxis": {
+            "align": false,
+            "alignLevel": 0
+          }
+        },
+        {
+          "cacheTimeout": null,
+          "cards": {
+            "cardPadding": null,
+            "cardRound": null
+          },
+          "color": {
+            "cardColor": "#b4ff00",
+            "colorScale": "sqrt",
+            "colorScheme": "interpolateSpectral",
+            "exponent": 0.5,
+            "max": null,
+            "min": null,
+            "mode": "spectrum"
+          },
+          "dataFormat": "tsbuckets",
+          "datasource": "${DS_TEST-CLUSTER}",
+          "description": null,
+          "editable": true,
+          "error": false,
+          "fieldConfig": {
+            "defaults": {
+              "thresholds": {
+                "mode": "absolute",
+                "steps": []
+              }
+            }
+          },
+          "gridPos": {
+            "h": 7,
+            "w": 12,
+            "x": 12,
+            "y": 7
+          },
+          "heatmap": {},
+          "height": null,
+          "hideTimeOverride": false,
+          "hideZeroBuckets": true,
+          "highlightCards": true,
+          "id": 304,
+          "interval": null,
+          "legend": {
+            "show": false
+          },
+          "links": [],
+          "maxDataPoints": 512,
+          "maxPerRow": null,
+          "minSpan": null,
+          "options": {},
+          "repeat": null,
+          "repeatDirection": null,
+          "reverseYBuckets": false,
+          "span": null,
+          "targets": [
+            {
+              "datasource": "${DS_TEST-CLUSTER}",
+              "expr": "sum(increase(\n    tikv_yatp_pool_schedule_wait_duration_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"sched-worker.*\"}\n    [$__rate_interval]\n)) by (le) ",
+              "format": "heatmap",
+              "hide": false,
+              "instant": false,
+              "interval": "",
+              "intervalFactor": 1,
+              "legendFormat": "{{le}}",
+              "metric": "",
+              "query": "sum(increase(\n    tikv_yatp_pool_schedule_wait_duration_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"sched-worker.*\"}\n    [$__rate_interval]\n)) by (le) ",
+              "refId": "",
+              "step": 10,
+              "target": ""
+            }
+          ],
+          "timeFrom": null,
+          "timeShift": null,
+          "title": "Wait Duration",
+          "tooltip": {
+            "msResolution": true,
+            "shared": true,
+            "show": true,
+            "showHistogram": true,
+            "sort": 0,
+            "value_type": "individual"
+          },
+          "transformations": [],
+          "transparent": false,
+          "type": "heatmap",
+          "xAxis": {
+            "mode": "time",
+            "name": null,
+            "show": true,
+            "values": []
+          },
+          "xBucketNumber": null,
+          "xBucketSize": null,
+          "yAxis": {
+            "decimals": 1,
+            "format": "s",
+            "label": null,
+            "logBase": 1,
+            "max": null,
+            "min": null,
+            "show": true
+          },
+          "yBucketBound": "upper",
+          "yBucketNumber": null,
+          "yBucketSize": null
+        },
+        {
+          "aliasColors": {},
+          "bars": false,
+          "cacheTimeout": null,
+          "datasource": "${DS_TEST-CLUSTER}",
+          "description": "Task execution time during one schedule.",
+          "editable": true,
+          "error": false,
+          "fieldConfig": {
+            "defaults": {
+              "thresholds": {
+                "mode": "absolute",
+                "steps": []
+              }
+            }
+          },
+          "fill": 1,
+          "fillGradient": 1,
+          "grid": {
+            "threshold1": null,
+            "threshold1Color": "rgba(216, 200, 27, 0.27)",
+            "threshold2": null,
+            "threshold2Color": "rgba(234, 112, 112, 0.22)"
+          },
+          "gridPos": {
+            "h": 7,
+            "w": 12,
+            "x": 0,
+            "y": 14
+          },
+          "height": null,
+          "hideTimeOverride": false,
+          "id": 305,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -42766,7 +43298,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 302,
+          "id": 306,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -42967,7 +43499,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 303,
+          "id": 307,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -43171,7 +43703,7 @@
       },
       "height": null,
       "hideTimeOverride": false,
-      "id": 304,
+      "id": 308,
       "interval": null,
       "links": [],
       "maxDataPoints": 100,
@@ -43210,7 +43742,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 305,
+          "id": 309,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -43358,7 +43890,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 306,
+          "id": 310,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -43559,7 +44091,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 307,
+          "id": 311,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -43760,7 +44292,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 308,
+          "id": 312,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -43961,7 +44493,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 309,
+          "id": 313,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -44162,7 +44694,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 310,
+          "id": 314,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -44295,7 +44827,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 311,
+          "id": 315,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -44428,7 +44960,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 312,
+          "id": 316,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -44561,7 +45093,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 313,
+          "id": 317,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -44694,7 +45226,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 314,
+          "id": 318,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -44902,7 +45434,7 @@
           "hideTimeOverride": false,
           "hideZeroBuckets": true,
           "highlightCards": true,
-          "id": 315,
+          "id": 319,
           "interval": null,
           "legend": {
             "show": false
@@ -45003,7 +45535,7 @@
       },
       "height": null,
       "hideTimeOverride": false,
-      "id": 316,
+      "id": 320,
       "interval": null,
       "links": [],
       "maxDataPoints": 100,
@@ -45049,7 +45581,7 @@
           "hideTimeOverride": false,
           "hideZeroBuckets": true,
           "highlightCards": true,
-          "id": 317,
+          "id": 321,
           "interval": null,
           "legend": {
             "show": false
@@ -45147,7 +45679,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 318,
+          "id": 322,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -45348,7 +45880,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 319,
+          "id": 323,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -45481,7 +46013,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 320,
+          "id": 324,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -45614,341 +46146,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 321,
-          "interval": null,
-          "isNew": true,
-          "legend": {
-            "alignAsTable": true,
-            "avg": false,
-            "current": true,
-            "hideEmpty": true,
-            "hideZero": true,
-            "max": true,
-            "min": false,
-            "rightSide": true,
-            "show": true,
-            "sideWidth": null,
-            "sort": "max",
-            "sortDesc": true,
-            "total": false,
-            "values": true
-          },
-          "lines": true,
-          "linewidth": 1,
-          "links": [],
-          "maxDataPoints": null,
-          "maxPerRow": null,
-          "minSpan": null,
-          "nullPointMode": "null as zero",
-          "options": {
-            "alertThreshold": true,
-            "dataLinks": []
-          },
-          "percentage": false,
-          "pointradius": 5,
-          "points": false,
-          "renderer": "flot",
-          "repeat": null,
-          "repeatDirection": null,
-          "seriesOverrides": [],
-          "span": null,
-          "stack": false,
-          "steppedLine": false,
-          "targets": [
-            {
-              "datasource": "${DS_TEST-CLUSTER}",
-              "expr": "sum(rate(\n    tikv_coprocessor_scan_keys_sum\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    [$__rate_interval]\n)) by (req, $additional_groupby) ",
-              "format": "time_series",
-              "hide": false,
-              "instant": false,
-              "interval": "",
-              "intervalFactor": 1,
-              "legendFormat": "{{req}} {{$additional_groupby}}",
-              "metric": "",
-              "query": "sum(rate(\n    tikv_coprocessor_scan_keys_sum\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    [$__rate_interval]\n)) by (req, $additional_groupby) ",
-              "refId": "",
-              "step": 10,
-              "target": ""
-            }
-          ],
-          "thresholds": [],
-          "timeFrom": null,
-          "timeShift": null,
-          "title": "KV Cursor Operations",
-          "tooltip": {
-            "msResolution": true,
-            "shared": true,
-            "sort": 0,
-            "value_type": "individual"
-          },
-          "transformations": [],
-          "transparent": false,
-          "type": "graph",
-          "xaxis": {
-            "mode": "time",
-            "name": null,
-            "show": true,
-            "values": []
-          },
-          "yaxes": [
-            {
-              "decimals": null,
-              "format": "none",
-              "label": null,
-              "logBase": 1,
-              "max": null,
-              "min": null,
-              "show": true
-            },
-            {
-              "decimals": null,
-              "format": "short",
-              "label": null,
-              "logBase": 1,
-              "max": null,
-              "min": null,
-              "show": true
-            }
-          ],
-          "yaxis": {
-            "align": false,
-            "alignLevel": 0
-          }
-        },
-        {
-          "aliasColors": {},
-          "bars": false,
-          "cacheTimeout": null,
-          "datasource": "${DS_TEST-CLUSTER}",
-          "description": "",
-          "editable": true,
-          "error": false,
-          "fieldConfig": {
-            "defaults": {
-              "thresholds": {
-                "mode": "absolute",
-                "steps": []
-              }
-            }
-          },
-          "fill": 1,
-          "fillGradient": 1,
-          "grid": {
-            "threshold1": null,
-            "threshold1Color": "rgba(216, 200, 27, 0.27)",
-            "threshold2": null,
-            "threshold2Color": "rgba(234, 112, 112, 0.22)"
-          },
-          "gridPos": {
-            "h": 7,
-            "w": 12,
-            "x": 12,
-            "y": 14
-          },
-          "height": null,
-          "hideTimeOverride": false,
-          "id": 322,
-          "interval": null,
-          "isNew": true,
-          "legend": {
-            "alignAsTable": true,
-            "avg": false,
-            "current": true,
-            "hideEmpty": true,
-            "hideZero": true,
-            "max": true,
-            "min": false,
-            "rightSide": true,
-            "show": true,
-            "sideWidth": null,
-            "sort": "max",
-            "sortDesc": true,
-            "total": false,
-            "values": true
-          },
-          "lines": true,
-          "linewidth": 1,
-          "links": [],
-          "maxDataPoints": null,
-          "maxPerRow": null,
-          "minSpan": null,
-          "nullPointMode": "null as zero",
-          "options": {
-            "alertThreshold": true,
-            "dataLinks": []
-          },
-          "percentage": false,
-          "pointradius": 5,
-          "points": false,
-          "renderer": "flot",
-          "repeat": null,
-          "repeatDirection": null,
-          "seriesOverrides": [
-            {
-              "alias": "count",
-              "bars": false,
-              "dashLength": 1,
-              "dashes": true,
-              "fill": 2,
-              "fillBelowTo": null,
-              "lines": true,
-              "spaceLength": 1,
-              "transform": "negative-Y",
-              "yaxis": 2,
-              "zindex": -3
-            },
-            {
-              "alias": "avg",
-              "bars": false,
-              "fill": 7,
-              "fillBelowTo": null,
-              "lines": true,
-              "yaxis": 1,
-              "zindex": 0
-            }
-          ],
-          "span": null,
-          "stack": false,
-          "steppedLine": false,
-          "targets": [
-            {
-              "datasource": "${DS_TEST-CLUSTER}",
-              "expr": "histogram_quantile(0.9999,(\n    sum(rate(\n    tikv_coprocessor_scan_keys_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    [$__rate_interval]\n)) by (req, le, $additional_groupby) \n    \n    \n))  ",
-              "format": "time_series",
-              "hide": false,
-              "instant": false,
-              "interval": "",
-              "intervalFactor": 1,
-              "legendFormat": "99.99%-{{req}} {{$additional_groupby}}",
-              "metric": "",
-              "query": "histogram_quantile(0.9999,(\n    sum(rate(\n    tikv_coprocessor_scan_keys_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    [$__rate_interval]\n)) by (req, le, $additional_groupby) \n    \n    \n))  ",
-              "refId": "",
-              "step": 10,
-              "target": ""
-            },
-            {
-              "datasource": "${DS_TEST-CLUSTER}",
-              "expr": "histogram_quantile(0.99,(\n    sum(rate(\n    tikv_coprocessor_scan_keys_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    [$__rate_interval]\n)) by (req, le, $additional_groupby) \n    \n    \n))  ",
-              "format": "time_series",
-              "hide": false,
-              "instant": false,
-              "interval": "",
-              "intervalFactor": 1,
-              "legendFormat": "99%-{{req}} {{$additional_groupby}}",
-              "metric": "",
-              "query": "histogram_quantile(0.99,(\n    sum(rate(\n    tikv_coprocessor_scan_keys_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    [$__rate_interval]\n)) by (req, le, $additional_groupby) \n    \n    \n))  ",
-              "refId": "",
-              "step": 10,
-              "target": ""
-            },
-            {
-              "datasource": "${DS_TEST-CLUSTER}",
-              "expr": "(sum(rate(\n    tikv_coprocessor_scan_keys_sum\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    [$__rate_interval]\n)) by (req, $additional_groupby)  / sum(rate(\n    tikv_coprocessor_scan_keys_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    [$__rate_interval]\n)) by (req, $additional_groupby) )",
-              "format": "time_series",
-              "hide": false,
-              "instant": false,
-              "interval": "",
-              "intervalFactor": 1,
-              "legendFormat": "avg-{{req}} {{$additional_groupby}}",
-              "metric": "",
-              "query": "(sum(rate(\n    tikv_coprocessor_scan_keys_sum\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    [$__rate_interval]\n)) by (req, $additional_groupby)  / sum(rate(\n    tikv_coprocessor_scan_keys_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    [$__rate_interval]\n)) by (req, $additional_groupby) )",
-              "refId": "",
-              "step": 10,
-              "target": ""
-            },
-            {
-              "datasource": "${DS_TEST-CLUSTER}",
-              "expr": "sum(rate(\n    tikv_coprocessor_scan_keys_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    [$__rate_interval]\n)) by (req, $additional_groupby) ",
-              "format": "time_series",
-              "hide": true,
-              "instant": false,
-              "interval": "",
-              "intervalFactor": 1,
-              "legendFormat": "count-{{req}} {{$additional_groupby}}",
-              "metric": "",
-              "query": "sum(rate(\n    tikv_coprocessor_scan_keys_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    [$__rate_interval]\n)) by (req, $additional_groupby) ",
-              "refId": "",
-              "step": 10,
-              "target": ""
-            }
-          ],
-          "thresholds": [],
-          "timeFrom": null,
-          "timeShift": null,
-          "title": "KV Cursor Operations",
-          "tooltip": {
-            "msResolution": true,
-            "shared": true,
-            "sort": 0,
-            "value_type": "individual"
-          },
-          "transformations": [],
-          "transparent": false,
-          "type": "graph",
-          "xaxis": {
-            "mode": "time",
-            "name": null,
-            "show": true,
-            "values": []
-          },
-          "yaxes": [
-            {
-              "decimals": null,
-              "format": "short",
-              "label": null,
-              "logBase": 1,
-              "max": null,
-              "min": null,
-              "show": true
-            },
-            {
-              "decimals": null,
-              "format": "short",
-              "label": null,
-              "logBase": 1,
-              "max": null,
-              "min": null,
-              "show": true
-            }
-          ],
-          "yaxis": {
-            "align": false,
-            "alignLevel": 0
-          }
-        },
-        {
-          "aliasColors": {},
-          "bars": false,
-          "cacheTimeout": null,
-          "datasource": "${DS_TEST-CLUSTER}",
-          "description": null,
-          "editable": true,
-          "error": false,
-          "fieldConfig": {
-            "defaults": {
-              "thresholds": {
-                "mode": "absolute",
-                "steps": []
-              }
-            }
-          },
-          "fill": 1,
-          "fillGradient": 1,
-          "grid": {
-            "threshold1": null,
-            "threshold1Color": "rgba(216, 200, 27, 0.27)",
-            "threshold2": null,
-            "threshold2Color": "rgba(234, 112, 112, 0.22)"
-          },
-          "gridPos": {
-            "h": 7,
-            "w": 12,
-            "x": 0,
-            "y": 21
-          },
-          "height": null,
-          "hideTimeOverride": false,
-          "id": 323,
+          "id": 325,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -45991,15 +46189,15 @@
           "targets": [
             {
               "datasource": "${DS_TEST-CLUSTER}",
-              "expr": "sum(rate(\n    tikv_coprocessor_rocksdb_perf\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",metric=\"internal_delete_skipped_count\"}\n    [$__rate_interval]\n)) by (req, $additional_groupby) ",
+              "expr": "sum(rate(\n    tikv_coprocessor_scan_keys_sum\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    [$__rate_interval]\n)) by (req, $additional_groupby) ",
               "format": "time_series",
               "hide": false,
               "instant": false,
               "interval": "",
               "intervalFactor": 1,
-              "legendFormat": "delete_skipped-{{req}} {{$additional_groupby}}",
+              "legendFormat": "{{req}} {{$additional_groupby}}",
               "metric": "",
-              "query": "sum(rate(\n    tikv_coprocessor_rocksdb_perf\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",metric=\"internal_delete_skipped_count\"}\n    [$__rate_interval]\n)) by (req, $additional_groupby) ",
+              "query": "sum(rate(\n    tikv_coprocessor_scan_keys_sum\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    [$__rate_interval]\n)) by (req, $additional_groupby) ",
               "refId": "",
               "step": 10,
               "target": ""
@@ -46008,7 +46206,7 @@
           "thresholds": [],
           "timeFrom": null,
           "timeShift": null,
-          "title": "Total RocksDB Perf Statistics",
+          "title": "KV Cursor Operations",
           "tooltip": {
             "msResolution": true,
             "shared": true,
@@ -46054,7 +46252,7 @@
           "bars": false,
           "cacheTimeout": null,
           "datasource": "${DS_TEST-CLUSTER}",
-          "description": null,
+          "description": "",
           "editable": true,
           "error": false,
           "fieldConfig": {
@@ -46077,11 +46275,11 @@
             "h": 7,
             "w": 12,
             "x": 12,
-            "y": 21
+            "y": 14
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 324,
+          "id": 326,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -46117,22 +46315,90 @@
           "renderer": "flot",
           "repeat": null,
           "repeatDirection": null,
-          "seriesOverrides": [],
+          "seriesOverrides": [
+            {
+              "alias": "count",
+              "bars": false,
+              "dashLength": 1,
+              "dashes": true,
+              "fill": 2,
+              "fillBelowTo": null,
+              "lines": true,
+              "spaceLength": 1,
+              "transform": "negative-Y",
+              "yaxis": 2,
+              "zindex": -3
+            },
+            {
+              "alias": "avg",
+              "bars": false,
+              "fill": 7,
+              "fillBelowTo": null,
+              "lines": true,
+              "yaxis": 1,
+              "zindex": 0
+            }
+          ],
           "span": null,
           "stack": false,
           "steppedLine": false,
           "targets": [
             {
               "datasource": "${DS_TEST-CLUSTER}",
-              "expr": "sum(rate(\n    tikv_coprocessor_response_bytes\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    [$__rate_interval]\n)) by (instance) ",
+              "expr": "histogram_quantile(0.9999,(\n    sum(rate(\n    tikv_coprocessor_scan_keys_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    [$__rate_interval]\n)) by (req, le, $additional_groupby) \n    \n    \n))  ",
               "format": "time_series",
               "hide": false,
               "instant": false,
               "interval": "",
               "intervalFactor": 1,
-              "legendFormat": "{{instance}}",
+              "legendFormat": "99.99%-{{req}} {{$additional_groupby}}",
               "metric": "",
-              "query": "sum(rate(\n    tikv_coprocessor_response_bytes\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    [$__rate_interval]\n)) by (instance) ",
+              "query": "histogram_quantile(0.9999,(\n    sum(rate(\n    tikv_coprocessor_scan_keys_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    [$__rate_interval]\n)) by (req, le, $additional_groupby) \n    \n    \n))  ",
+              "refId": "",
+              "step": 10,
+              "target": ""
+            },
+            {
+              "datasource": "${DS_TEST-CLUSTER}",
+              "expr": "histogram_quantile(0.99,(\n    sum(rate(\n    tikv_coprocessor_scan_keys_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    [$__rate_interval]\n)) by (req, le, $additional_groupby) \n    \n    \n))  ",
+              "format": "time_series",
+              "hide": false,
+              "instant": false,
+              "interval": "",
+              "intervalFactor": 1,
+              "legendFormat": "99%-{{req}} {{$additional_groupby}}",
+              "metric": "",
+              "query": "histogram_quantile(0.99,(\n    sum(rate(\n    tikv_coprocessor_scan_keys_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    [$__rate_interval]\n)) by (req, le, $additional_groupby) \n    \n    \n))  ",
+              "refId": "",
+              "step": 10,
+              "target": ""
+            },
+            {
+              "datasource": "${DS_TEST-CLUSTER}",
+              "expr": "(sum(rate(\n    tikv_coprocessor_scan_keys_sum\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    [$__rate_interval]\n)) by (req, $additional_groupby)  / sum(rate(\n    tikv_coprocessor_scan_keys_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    [$__rate_interval]\n)) by (req, $additional_groupby) )",
+              "format": "time_series",
+              "hide": false,
+              "instant": false,
+              "interval": "",
+              "intervalFactor": 1,
+              "legendFormat": "avg-{{req}} {{$additional_groupby}}",
+              "metric": "",
+              "query": "(sum(rate(\n    tikv_coprocessor_scan_keys_sum\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    [$__rate_interval]\n)) by (req, $additional_groupby)  / sum(rate(\n    tikv_coprocessor_scan_keys_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    [$__rate_interval]\n)) by (req, $additional_groupby) )",
+              "refId": "",
+              "step": 10,
+              "target": ""
+            },
+            {
+              "datasource": "${DS_TEST-CLUSTER}",
+              "expr": "sum(rate(\n    tikv_coprocessor_scan_keys_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    [$__rate_interval]\n)) by (req, $additional_groupby) ",
+              "format": "time_series",
+              "hide": true,
+              "instant": false,
+              "interval": "",
+              "intervalFactor": 1,
+              "legendFormat": "count-{{req}} {{$additional_groupby}}",
+              "metric": "",
+              "query": "sum(rate(\n    tikv_coprocessor_scan_keys_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    [$__rate_interval]\n)) by (req, $additional_groupby) ",
               "refId": "",
               "step": 10,
               "target": ""
@@ -46141,7 +46407,7 @@
           "thresholds": [],
           "timeFrom": null,
           "timeShift": null,
-          "title": "Total Response Size",
+          "title": "KV Cursor Operations",
           "tooltip": {
             "msResolution": true,
             "shared": true,
@@ -46160,7 +46426,7 @@
           "yaxes": [
             {
               "decimals": null,
-              "format": "bytes",
+              "format": "short",
               "label": null,
               "logBase": 1,
               "max": null,
@@ -46187,7 +46453,7 @@
           "bars": false,
           "cacheTimeout": null,
           "datasource": "${DS_TEST-CLUSTER}",
-          "description": "Total bytes of memory used by coprocessor requests",
+          "description": null,
           "editable": true,
           "error": false,
           "fieldConfig": {
@@ -46208,13 +46474,13 @@
           },
           "gridPos": {
             "h": 7,
-            "w": 24,
+            "w": 12,
             "x": 0,
-            "y": 28
+            "y": 21
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 325,
+          "id": 327,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -46257,15 +46523,15 @@
           "targets": [
             {
               "datasource": "${DS_TEST-CLUSTER}",
-              "expr": "sum((\n    tikv_coprocessor_memory_quota\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    \n)) by (instance, type) ",
+              "expr": "sum(rate(\n    tikv_coprocessor_rocksdb_perf\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",metric=\"internal_delete_skipped_count\"}\n    [$__rate_interval]\n)) by (req, $additional_groupby) ",
               "format": "time_series",
               "hide": false,
               "instant": false,
               "interval": "",
               "intervalFactor": 1,
-              "legendFormat": "{{instance}}-{{type}}",
+              "legendFormat": "delete_skipped-{{req}} {{$additional_groupby}}",
               "metric": "",
-              "query": "sum((\n    tikv_coprocessor_memory_quota\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    \n)) by (instance, type) ",
+              "query": "sum(rate(\n    tikv_coprocessor_rocksdb_perf\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",metric=\"internal_delete_skipped_count\"}\n    [$__rate_interval]\n)) by (req, $additional_groupby) ",
               "refId": "",
               "step": 10,
               "target": ""
@@ -46274,7 +46540,7 @@
           "thresholds": [],
           "timeFrom": null,
           "timeShift": null,
-          "title": "Memory Quota",
+          "title": "Total RocksDB Perf Statistics",
           "tooltip": {
             "msResolution": true,
             "shared": true,
@@ -46293,7 +46559,7 @@
           "yaxes": [
             {
               "decimals": null,
-              "format": "bytes",
+              "format": "none",
               "label": null,
               "logBase": 1,
               "max": null,
@@ -46314,55 +46580,13 @@
             "align": false,
             "alignLevel": 0
           }
-        }
-      ],
-      "repeat": null,
-      "repeatDirection": null,
-      "span": null,
-      "targets": [],
-      "timeFrom": null,
-      "timeShift": null,
-      "title": "Coprocessor Overview",
-      "transformations": [],
-      "transparent": false,
-      "type": "row"
-    },
-    {
-      "cacheTimeout": null,
-      "collapsed": true,
-      "datasource": null,
-      "description": null,
-      "editable": true,
-      "error": false,
-      "fieldConfig": {
-        "defaults": {
-          "thresholds": {
-            "mode": "absolute",
-            "steps": []
-          }
-        }
-      },
-      "gridPos": {
-        "h": 7,
-        "w": 24,
-        "x": 0,
-        "y": 0
-      },
-      "height": null,
-      "hideTimeOverride": false,
-      "id": 326,
-      "interval": null,
-      "links": [],
-      "maxDataPoints": 100,
-      "maxPerRow": null,
-      "minSpan": null,
-      "panels": [
+        },
         {
           "aliasColors": {},
           "bars": false,
           "cacheTimeout": null,
           "datasource": "${DS_TEST-CLUSTER}",
-          "description": "The time consumed when handling coprocessor requests",
+          "description": null,
           "editable": true,
           "error": false,
           "fieldConfig": {
@@ -46384,12 +46608,12 @@
           "gridPos": {
             "h": 7,
             "w": 12,
-            "x": 0,
-            "y": 0
+            "x": 12,
+            "y": 21
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 327,
+          "id": 328,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -46425,90 +46649,22 @@
           "renderer": "flot",
           "repeat": null,
           "repeatDirection": null,
-          "seriesOverrides": [
-            {
-              "alias": "count",
-              "bars": false,
-              "dashLength": 1,
-              "dashes": true,
-              "fill": 2,
-              "fillBelowTo": null,
-              "lines": true,
-              "spaceLength": 1,
-              "transform": "negative-Y",
-              "yaxis": 2,
-              "zindex": -3
-            },
-            {
-              "alias": "avg",
-              "bars": false,
-              "fill": 7,
-              "fillBelowTo": null,
-              "lines": true,
-              "yaxis": 1,
-              "zindex": 0
-            }
-          ],
+          "seriesOverrides": [],
           "span": null,
           "stack": false,
           "steppedLine": false,
           "targets": [
             {
               "datasource": "${DS_TEST-CLUSTER}",
-              "expr": "histogram_quantile(0.9999,(\n    sum(rate(\n    tikv_coprocessor_request_handle_seconds_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    [$__rate_interval]\n)) by (req, le, $additional_groupby) \n    \n    \n))  ",
-              "format": "time_series",
-              "hide": false,
-              "instant": false,
-              "interval": "",
-              "intervalFactor": 1,
-              "legendFormat": "99.99%-{{req}} {{$additional_groupby}}",
-              "metric": "",
-              "query": "histogram_quantile(0.9999,(\n    sum(rate(\n    tikv_coprocessor_request_handle_seconds_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    [$__rate_interval]\n)) by (req, le, $additional_groupby) \n    \n    \n))  ",
-              "refId": "",
-              "step": 10,
-              "target": ""
-            },
-            {
-              "datasource": "${DS_TEST-CLUSTER}",
-              "expr": "histogram_quantile(0.99,(\n    sum(rate(\n    tikv_coprocessor_request_handle_seconds_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    [$__rate_interval]\n)) by (req, le, $additional_groupby) \n    \n    \n))  ",
+              "expr": "sum(rate(\n    tikv_coprocessor_response_bytes\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    [$__rate_interval]\n)) by (instance) ",
               "format": "time_series",
               "hide": false,
               "instant": false,
               "interval": "",
               "intervalFactor": 1,
-              "legendFormat": "99%-{{req}} {{$additional_groupby}}",
-              "metric": "",
-              "query": "histogram_quantile(0.99,(\n    sum(rate(\n    tikv_coprocessor_request_handle_seconds_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    [$__rate_interval]\n)) by (req, le, $additional_groupby) \n    \n    \n))  ",
-              "refId": "",
-              "step": 10,
-              "target": ""
-            },
-            {
-              "datasource": "${DS_TEST-CLUSTER}",
-              "expr": "(sum(rate(\n    tikv_coprocessor_request_handle_seconds_sum\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    [$__rate_interval]\n)) by (req, $additional_groupby)  / sum(rate(\n    tikv_coprocessor_request_handle_seconds_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    [$__rate_interval]\n)) by (req, $additional_groupby) )",
-              "format": "time_series",
-              "hide": true,
-              "instant": false,
-              "interval": "",
-              "intervalFactor": 1,
-              "legendFormat": "avg-{{req}} {{$additional_groupby}}",
-              "metric": "",
-              "query": "(sum(rate(\n    tikv_coprocessor_request_handle_seconds_sum\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    [$__rate_interval]\n)) by (req, $additional_groupby)  / sum(rate(\n    tikv_coprocessor_request_handle_seconds_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    [$__rate_interval]\n)) by (req, $additional_groupby) )",
-              "refId": "",
-              "step": 10,
-              "target": ""
-            },
-            {
-              "datasource": "${DS_TEST-CLUSTER}",
-              "expr": "sum(rate(\n    tikv_coprocessor_request_handle_seconds_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    [$__rate_interval]\n)) by (req, $additional_groupby) ",
-              "format": "time_series",
-              "hide": true,
-              "instant": false,
-              "interval": "",
-              "intervalFactor": 1,
-              "legendFormat": "count-{{req}} {{$additional_groupby}}",
+              "legendFormat": "{{instance}}",
               "metric": "",
-              "query": "sum(rate(\n    tikv_coprocessor_request_handle_seconds_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    [$__rate_interval]\n)) by (req, $additional_groupby) ",
+              "query": "sum(rate(\n    tikv_coprocessor_response_bytes\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    [$__rate_interval]\n)) by (instance) ",
               "refId": "",
               "step": 10,
               "target": ""
@@ -46517,7 +46673,7 @@
           "thresholds": [],
           "timeFrom": null,
           "timeShift": null,
-          "title": "Handle duration",
+          "title": "Total Response Size",
           "tooltip": {
             "msResolution": true,
             "shared": true,
@@ -46536,7 +46692,7 @@
           "yaxes": [
             {
               "decimals": null,
-              "format": "s",
+              "format": "bytes",
               "label": null,
               "logBase": 1,
               "max": null,
@@ -46563,7 +46719,7 @@
           "bars": false,
           "cacheTimeout": null,
           "datasource": "${DS_TEST-CLUSTER}",
-          "description": "The time consumed to handle coprocessor requests per TiKV instance",
+          "description": "Total bytes of memory used by coprocessor requests",
           "editable": true,
           "error": false,
           "fieldConfig": {
@@ -46584,13 +46740,13 @@
           },
           "gridPos": {
             "h": 7,
-            "w": 12,
-            "x": 12,
-            "y": 0
+            "w": 24,
+            "x": 0,
+            "y": 28
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 328,
+          "id": 329,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -46626,90 +46782,22 @@
           "renderer": "flot",
           "repeat": null,
           "repeatDirection": null,
-          "seriesOverrides": [
-            {
-              "alias": "count",
-              "bars": false,
-              "dashLength": 1,
-              "dashes": true,
-              "fill": 2,
-              "fillBelowTo": null,
-              "lines": true,
-              "spaceLength": 1,
-              "transform": "negative-Y",
-              "yaxis": 2,
-              "zindex": -3
-            },
-            {
-              "alias": "avg",
-              "bars": false,
-              "fill": 7,
-              "fillBelowTo": null,
-              "lines": true,
-              "yaxis": 1,
-              "zindex": 0
-            }
-          ],
+          "seriesOverrides": [],
           "span": null,
           "stack": false,
           "steppedLine": false,
           "targets": [
             {
               "datasource": "${DS_TEST-CLUSTER}",
-              "expr": "histogram_quantile(0.9999,(\n    sum(rate(\n    tikv_coprocessor_request_handle_seconds_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    [$__rate_interval]\n)) by (req, instance, le) \n    \n    \n))  ",
-              "format": "time_series",
-              "hide": false,
-              "instant": false,
-              "interval": "",
-              "intervalFactor": 1,
-              "legendFormat": "99.99%-{{req}}-{{instance}}",
-              "metric": "",
-              "query": "histogram_quantile(0.9999,(\n    sum(rate(\n    tikv_coprocessor_request_handle_seconds_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    [$__rate_interval]\n)) by (req, instance, le) \n    \n    \n))  ",
-              "refId": "",
-              "step": 10,
-              "target": ""
-            },
-            {
-              "datasource": "${DS_TEST-CLUSTER}",
-              "expr": "histogram_quantile(0.99,(\n    sum(rate(\n    tikv_coprocessor_request_handle_seconds_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    [$__rate_interval]\n)) by (req, instance, le) \n    \n    \n))  ",
+              "expr": "sum((\n    tikv_coprocessor_memory_quota\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    \n)) by (instance, type) ",
               "format": "time_series",
               "hide": false,
               "instant": false,
               "interval": "",
               "intervalFactor": 1,
-              "legendFormat": "99%-{{req}}-{{instance}}",
-              "metric": "",
-              "query": "histogram_quantile(0.99,(\n    sum(rate(\n    tikv_coprocessor_request_handle_seconds_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    [$__rate_interval]\n)) by (req, instance, le) \n    \n    \n))  ",
-              "refId": "",
-              "step": 10,
-              "target": ""
-            },
-            {
-              "datasource": "${DS_TEST-CLUSTER}",
-              "expr": "(sum(rate(\n    tikv_coprocessor_request_handle_seconds_sum\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    [$__rate_interval]\n)) by (req, instance)  / sum(rate(\n    tikv_coprocessor_request_handle_seconds_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    [$__rate_interval]\n)) by (req, instance) )",
-              "format": "time_series",
-              "hide": true,
-              "instant": false,
-              "interval": "",
-              "intervalFactor": 1,
-              "legendFormat": "avg-{{req}}-{{instance}}",
-              "metric": "",
-              "query": "(sum(rate(\n    tikv_coprocessor_request_handle_seconds_sum\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    [$__rate_interval]\n)) by (req, instance)  / sum(rate(\n    tikv_coprocessor_request_handle_seconds_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    [$__rate_interval]\n)) by (req, instance) )",
-              "refId": "",
-              "step": 10,
-              "target": ""
-            },
-            {
-              "datasource": "${DS_TEST-CLUSTER}",
-              "expr": "sum(rate(\n    tikv_coprocessor_request_handle_seconds_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    [$__rate_interval]\n)) by (req, instance) ",
-              "format": "time_series",
-              "hide": true,
-              "instant": false,
-              "interval": "",
-              "intervalFactor": 1,
-              "legendFormat": "count-{{req}}-{{instance}}",
+              "legendFormat": "{{instance}}-{{type}}",
               "metric": "",
-              "query": "sum(rate(\n    tikv_coprocessor_request_handle_seconds_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    [$__rate_interval]\n)) by (req, instance) ",
+              "query": "sum((\n    tikv_coprocessor_memory_quota\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    \n)) by (instance, type) ",
               "refId": "",
               "step": 10,
               "target": ""
@@ -46718,7 +46806,7 @@
           "thresholds": [],
           "timeFrom": null,
           "timeShift": null,
-          "title": "Handle duration by store",
+          "title": "Memory Quota",
           "tooltip": {
             "msResolution": true,
             "shared": true,
@@ -46737,7 +46825,7 @@
           "yaxes": [
             {
               "decimals": null,
-              "format": "s",
+              "format": "bytes",
               "label": null,
               "logBase": 1,
               "max": null,
@@ -46758,13 +46846,55 @@
             "align": false,
             "alignLevel": 0
           }
-        },
+        }
+      ],
+      "repeat": null,
+      "repeatDirection": null,
+      "span": null,
+      "targets": [],
+      "timeFrom": null,
+      "timeShift": null,
+      "title": "Coprocessor Overview",
+      "transformations": [],
+      "transparent": false,
+      "type": "row"
+    },
+    {
+      "cacheTimeout": null,
+      "collapsed": true,
+      "datasource": null,
+      "description": null,
+      "editable": true,
+      "error": false,
+      "fieldConfig": {
+        "defaults": {
+          "thresholds": {
+            "mode": "absolute",
+            "steps": []
+          }
+        }
+      },
+      "gridPos": {
+        "h": 7,
+        "w": 24,
+        "x": 0,
+        "y": 0
+      },
+      "height": null,
+      "hideTimeOverride": false,
+      "id": 330,
+      "interval": null,
+      "links": [],
+      "maxDataPoints": 100,
+      "maxPerRow": null,
+      "minSpan": null,
+      "panels": [
         {
           "aliasColors": {},
           "bars": false,
           "cacheTimeout": null,
           "datasource": "${DS_TEST-CLUSTER}",
-          "description": "The time consumed when coprocessor requests are wait for being handled",
+          "description": "The time consumed when handling coprocessor requests",
           "editable": true,
           "error": false,
           "fieldConfig": {
@@ -46787,11 +46917,11 @@
             "h": 7,
             "w": 12,
             "x": 0,
-            "y": 7
+            "y": 0
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 329,
+          "id": 331,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -46857,7 +46987,7 @@
           "targets": [
             {
               "datasource": "${DS_TEST-CLUSTER}",
-              "expr": "histogram_quantile(0.9999,(\n    sum(rate(\n    tikv_coprocessor_request_wait_seconds_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"all\"}\n    [$__rate_interval]\n)) by (req, le, $additional_groupby) \n    \n    \n))  ",
+              "expr": "histogram_quantile(0.9999,(\n    sum(rate(\n    tikv_coprocessor_request_handle_seconds_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    [$__rate_interval]\n)) by (req, le, $additional_groupby) \n    \n    \n))  ",
               "format": "time_series",
               "hide": false,
               "instant": false,
@@ -46865,14 +46995,14 @@
               "intervalFactor": 1,
               "legendFormat": "99.99%-{{req}} {{$additional_groupby}}",
               "metric": "",
-              "query": "histogram_quantile(0.9999,(\n    sum(rate(\n    tikv_coprocessor_request_wait_seconds_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"all\"}\n    [$__rate_interval]\n)) by (req, le, $additional_groupby) \n    \n    \n))  ",
+              "query": "histogram_quantile(0.9999,(\n    sum(rate(\n    tikv_coprocessor_request_handle_seconds_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    [$__rate_interval]\n)) by (req, le, $additional_groupby) \n    \n    \n))  ",
               "refId": "",
               "step": 10,
               "target": ""
             },
             {
               "datasource": "${DS_TEST-CLUSTER}",
-              "expr": "histogram_quantile(0.99,(\n    sum(rate(\n    tikv_coprocessor_request_wait_seconds_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"all\"}\n    [$__rate_interval]\n)) by (req, le, $additional_groupby) \n    \n    \n))  ",
+              "expr": "histogram_quantile(0.99,(\n    sum(rate(\n    tikv_coprocessor_request_handle_seconds_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    [$__rate_interval]\n)) by (req, le, $additional_groupby) \n    \n    \n))  ",
               "format": "time_series",
               "hide": false,
               "instant": false,
@@ -46880,14 +47010,14 @@
               "intervalFactor": 1,
               "legendFormat": "99%-{{req}} {{$additional_groupby}}",
               "metric": "",
-              "query": "histogram_quantile(0.99,(\n    sum(rate(\n    tikv_coprocessor_request_wait_seconds_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"all\"}\n    [$__rate_interval]\n)) by (req, le, $additional_groupby) \n    \n    \n))  ",
+              "query": "histogram_quantile(0.99,(\n    sum(rate(\n    tikv_coprocessor_request_handle_seconds_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    [$__rate_interval]\n)) by (req, le, $additional_groupby) \n    \n    \n))  ",
               "refId": "",
               "step": 10,
               "target": ""
             },
             {
               "datasource": "${DS_TEST-CLUSTER}",
-              "expr": "(sum(rate(\n    tikv_coprocessor_request_wait_seconds_sum\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"all\"}\n    [$__rate_interval]\n)) by (req, $additional_groupby)  / sum(rate(\n    tikv_coprocessor_request_wait_seconds_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"all\"}\n    [$__rate_interval]\n)) by (req, $additional_groupby) )",
+              "expr": "(sum(rate(\n    tikv_coprocessor_request_handle_seconds_sum\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    [$__rate_interval]\n)) by (req, $additional_groupby)  / sum(rate(\n    tikv_coprocessor_request_handle_seconds_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    [$__rate_interval]\n)) by (req, $additional_groupby) )",
               "format": "time_series",
               "hide": true,
               "instant": false,
@@ -46895,14 +47025,14 @@
               "intervalFactor": 1,
               "legendFormat": "avg-{{req}} {{$additional_groupby}}",
               "metric": "",
-              "query": "(sum(rate(\n    tikv_coprocessor_request_wait_seconds_sum\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"all\"}\n    [$__rate_interval]\n)) by (req, $additional_groupby)  / sum(rate(\n    tikv_coprocessor_request_wait_seconds_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"all\"}\n    [$__rate_interval]\n)) by (req, $additional_groupby) )",
+              "query": "(sum(rate(\n    tikv_coprocessor_request_handle_seconds_sum\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    [$__rate_interval]\n)) by (req, $additional_groupby)  / sum(rate(\n    tikv_coprocessor_request_handle_seconds_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    [$__rate_interval]\n)) by (req, $additional_groupby) )",
               "refId": "",
               "step": 10,
               "target": ""
             },
             {
               "datasource": "${DS_TEST-CLUSTER}",
-              "expr": "sum(rate(\n    tikv_coprocessor_request_wait_seconds_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"all\"}\n    [$__rate_interval]\n)) by (req, $additional_groupby) ",
+              "expr": "sum(rate(\n    tikv_coprocessor_request_handle_seconds_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    [$__rate_interval]\n)) by (req, $additional_groupby) ",
               "format": "time_series",
               "hide": true,
               "instant": false,
@@ -46910,7 +47040,7 @@
               "intervalFactor": 1,
               "legendFormat": "count-{{req}} {{$additional_groupby}}",
               "metric": "",
-              "query": "sum(rate(\n    tikv_coprocessor_request_wait_seconds_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"all\"}\n    [$__rate_interval]\n)) by (req, $additional_groupby) ",
+              "query": "sum(rate(\n    tikv_coprocessor_request_handle_seconds_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    [$__rate_interval]\n)) by (req, $additional_groupby) ",
               "refId": "",
               "step": 10,
               "target": ""
@@ -46919,7 +47049,7 @@
           "thresholds": [],
           "timeFrom": null,
           "timeShift": null,
-          "title": "Wait duration",
+          "title": "Handle duration",
           "tooltip": {
             "msResolution": true,
             "shared": true,
@@ -46965,7 +47095,7 @@
           "bars": false,
           "cacheTimeout": null,
           "datasource": "${DS_TEST-CLUSTER}",
-          "description": "The time consumed when coprocessor requests are wait for being handled in each TiKV instance",
+          "description": "The time consumed to handle coprocessor requests per TiKV instance",
           "editable": true,
           "error": false,
           "fieldConfig": {
@@ -46988,11 +47118,11 @@
             "h": 7,
             "w": 12,
             "x": 12,
-            "y": 7
+            "y": 0
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 330,
+          "id": 332,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -47058,7 +47188,7 @@
           "targets": [
             {
               "datasource": "${DS_TEST-CLUSTER}",
-              "expr": "histogram_quantile(0.9999,(\n    sum(rate(\n    tikv_coprocessor_request_wait_seconds_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"all\"}\n    [$__rate_interval]\n)) by (req, instance, le) \n    \n    \n))  ",
+              "expr": "histogram_quantile(0.9999,(\n    sum(rate(\n    tikv_coprocessor_request_handle_seconds_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    [$__rate_interval]\n)) by (req, instance, le) \n    \n    \n))  ",
               "format": "time_series",
               "hide": false,
               "instant": false,
@@ -47066,14 +47196,14 @@
               "intervalFactor": 1,
               "legendFormat": "99.99%-{{req}}-{{instance}}",
               "metric": "",
-              "query": "histogram_quantile(0.9999,(\n    sum(rate(\n    tikv_coprocessor_request_wait_seconds_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"all\"}\n    [$__rate_interval]\n)) by (req, instance, le) \n    \n    \n))  ",
+              "query": "histogram_quantile(0.9999,(\n    sum(rate(\n    tikv_coprocessor_request_handle_seconds_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    [$__rate_interval]\n)) by (req, instance, le) \n    \n    \n))  ",
               "refId": "",
               "step": 10,
               "target": ""
             },
             {
               "datasource": "${DS_TEST-CLUSTER}",
-              "expr": "histogram_quantile(0.99,(\n    sum(rate(\n    tikv_coprocessor_request_wait_seconds_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"all\"}\n    [$__rate_interval]\n)) by (req, instance, le) \n    \n    \n))  ",
+              "expr": "histogram_quantile(0.99,(\n    sum(rate(\n    tikv_coprocessor_request_handle_seconds_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    [$__rate_interval]\n)) by (req, instance, le) \n    \n    \n))  ",
               "format": "time_series",
               "hide": false,
               "instant": false,
@@ -47081,14 +47211,14 @@
               "intervalFactor": 1,
               "legendFormat": "99%-{{req}}-{{instance}}",
               "metric": "",
-              "query": "histogram_quantile(0.99,(\n    sum(rate(\n    tikv_coprocessor_request_wait_seconds_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"all\"}\n    [$__rate_interval]\n)) by (req, instance, le) \n    \n    \n))  ",
+              "query": "histogram_quantile(0.99,(\n    sum(rate(\n    tikv_coprocessor_request_handle_seconds_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    [$__rate_interval]\n)) by (req, instance, le) \n    \n    \n))  ",
               "refId": "",
               "step": 10,
               "target": ""
             },
             {
               "datasource": "${DS_TEST-CLUSTER}",
-              "expr": "(sum(rate(\n    tikv_coprocessor_request_wait_seconds_sum\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"all\"}\n    [$__rate_interval]\n)) by (req, instance)  / sum(rate(\n    tikv_coprocessor_request_wait_seconds_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"all\"}\n    [$__rate_interval]\n)) by (req, instance) )",
+              "expr": "(sum(rate(\n    tikv_coprocessor_request_handle_seconds_sum\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    [$__rate_interval]\n)) by (req, instance)  / sum(rate(\n    tikv_coprocessor_request_handle_seconds_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    [$__rate_interval]\n)) by (req, instance) )",
               "format": "time_series",
               "hide": true,
               "instant": false,
@@ -47096,14 +47226,14 @@
               "intervalFactor": 1,
               "legendFormat": "avg-{{req}}-{{instance}}",
               "metric": "",
-              "query": "(sum(rate(\n    tikv_coprocessor_request_wait_seconds_sum\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"all\"}\n    [$__rate_interval]\n)) by (req, instance)  / sum(rate(\n    tikv_coprocessor_request_wait_seconds_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"all\"}\n    [$__rate_interval]\n)) by (req, instance) )",
+              "query": "(sum(rate(\n    tikv_coprocessor_request_handle_seconds_sum\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    [$__rate_interval]\n)) by (req, instance)  / sum(rate(\n    tikv_coprocessor_request_handle_seconds_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    [$__rate_interval]\n)) by (req, instance) )",
               "refId": "",
               "step": 10,
               "target": ""
             },
             {
               "datasource": "${DS_TEST-CLUSTER}",
-              "expr": "sum(rate(\n    tikv_coprocessor_request_wait_seconds_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"all\"}\n    [$__rate_interval]\n)) by (req, instance) ",
+              "expr": "sum(rate(\n    tikv_coprocessor_request_handle_seconds_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    [$__rate_interval]\n)) by (req, instance) ",
               "format": "time_series",
               "hide": true,
               "instant": false,
@@ -47111,7 +47241,7 @@
               "intervalFactor": 1,
               "legendFormat": "count-{{req}}-{{instance}}",
               "metric": "",
-              "query": "sum(rate(\n    tikv_coprocessor_request_wait_seconds_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"all\"}\n    [$__rate_interval]\n)) by (req, instance) ",
+              "query": "sum(rate(\n    tikv_coprocessor_request_handle_seconds_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    [$__rate_interval]\n)) by (req, instance) ",
               "refId": "",
               "step": 10,
               "target": ""
@@ -47120,7 +47250,7 @@
           "thresholds": [],
           "timeFrom": null,
           "timeShift": null,
-          "title": "Wait duration by store",
+          "title": "Handle duration by store",
           "tooltip": {
             "msResolution": true,
             "shared": true,
@@ -47166,7 +47296,7 @@
           "bars": false,
           "cacheTimeout": null,
           "datasource": "${DS_TEST-CLUSTER}",
-          "description": null,
+          "description": "The time consumed when coprocessor requests are wait for being handled",
           "editable": true,
           "error": false,
           "fieldConfig": {
@@ -47189,11 +47319,11 @@
             "h": 7,
             "w": 12,
             "x": 0,
-            "y": 14
+            "y": 7
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 331,
+          "id": 333,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -47229,22 +47359,90 @@
           "renderer": "flot",
           "repeat": null,
           "repeatDirection": null,
-          "seriesOverrides": [],
+          "seriesOverrides": [
+            {
+              "alias": "count",
+              "bars": false,
+              "dashLength": 1,
+              "dashes": true,
+              "fill": 2,
+              "fillBelowTo": null,
+              "lines": true,
+              "spaceLength": 1,
+              "transform": "negative-Y",
+              "yaxis": 2,
+              "zindex": -3
+            },
+            {
+              "alias": "avg",
+              "bars": false,
+              "fill": 7,
+              "fillBelowTo": null,
+              "lines": true,
+              "yaxis": 1,
+              "zindex": 0
+            }
+          ],
           "span": null,
           "stack": false,
           "steppedLine": false,
           "targets": [
             {
               "datasource": "${DS_TEST-CLUSTER}",
-              "expr": "sum(rate(\n    tikv_coprocessor_dag_request_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    [$__rate_interval]\n)) by (vec_type, $additional_groupby) ",
+              "expr": "histogram_quantile(0.9999,(\n    sum(rate(\n    tikv_coprocessor_request_wait_seconds_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"all\"}\n    [$__rate_interval]\n)) by (req, le, $additional_groupby) \n    \n    \n))  ",
               "format": "time_series",
               "hide": false,
               "instant": false,
               "interval": "",
               "intervalFactor": 1,
-              "legendFormat": "{{vec_type}} {{$additional_groupby}}",
+              "legendFormat": "99.99%-{{req}} {{$additional_groupby}}",
               "metric": "",
-              "query": "sum(rate(\n    tikv_coprocessor_dag_request_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    [$__rate_interval]\n)) by (vec_type, $additional_groupby) ",
+              "query": "histogram_quantile(0.9999,(\n    sum(rate(\n    tikv_coprocessor_request_wait_seconds_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"all\"}\n    [$__rate_interval]\n)) by (req, le, $additional_groupby) \n    \n    \n))  ",
+              "refId": "",
+              "step": 10,
+              "target": ""
+            },
+            {
+              "datasource": "${DS_TEST-CLUSTER}",
+              "expr": "histogram_quantile(0.99,(\n    sum(rate(\n    tikv_coprocessor_request_wait_seconds_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"all\"}\n    [$__rate_interval]\n)) by (req, le, $additional_groupby) \n    \n    \n))  ",
+              "format": "time_series",
+              "hide": false,
+              "instant": false,
+              "interval": "",
+              "intervalFactor": 1,
+              "legendFormat": "99%-{{req}} {{$additional_groupby}}",
+              "metric": "",
+              "query": "histogram_quantile(0.99,(\n    sum(rate(\n    tikv_coprocessor_request_wait_seconds_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"all\"}\n    [$__rate_interval]\n)) by (req, le, $additional_groupby) \n    \n    \n))  ",
+              "refId": "",
+              "step": 10,
+              "target": ""
+            },
+            {
+              "datasource": "${DS_TEST-CLUSTER}",
+              "expr": "(sum(rate(\n    tikv_coprocessor_request_wait_seconds_sum\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"all\"}\n    [$__rate_interval]\n)) by (req, $additional_groupby)  / sum(rate(\n    tikv_coprocessor_request_wait_seconds_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"all\"}\n    [$__rate_interval]\n)) by (req, $additional_groupby) )",
+              "format": "time_series",
+              "hide": true,
+              "instant": false,
+              "interval": "",
+              "intervalFactor": 1,
+              "legendFormat": "avg-{{req}} {{$additional_groupby}}",
+              "metric": "",
+              "query": "(sum(rate(\n    tikv_coprocessor_request_wait_seconds_sum\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"all\"}\n    [$__rate_interval]\n)) by (req, $additional_groupby)  / sum(rate(\n    tikv_coprocessor_request_wait_seconds_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"all\"}\n    [$__rate_interval]\n)) by (req, $additional_groupby) )",
+              "refId": "",
+              "step": 10,
+              "target": ""
+            },
+            {
+              "datasource": "${DS_TEST-CLUSTER}",
+              "expr": "sum(rate(\n    tikv_coprocessor_request_wait_seconds_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"all\"}\n    [$__rate_interval]\n)) by (req, $additional_groupby) ",
+              "format": "time_series",
+              "hide": true,
+              "instant": false,
+              "interval": "",
+              "intervalFactor": 1,
+              "legendFormat": "count-{{req}} {{$additional_groupby}}",
+              "metric": "",
+              "query": "sum(rate(\n    tikv_coprocessor_request_wait_seconds_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"all\"}\n    [$__rate_interval]\n)) by (req, $additional_groupby) ",
               "refId": "",
               "step": 10,
               "target": ""
@@ -47253,7 +47451,7 @@
           "thresholds": [],
           "timeFrom": null,
           "timeShift": null,
-          "title": "Total DAG Requests",
+          "title": "Wait duration",
           "tooltip": {
             "msResolution": true,
             "shared": true,
@@ -47272,7 +47470,7 @@
           "yaxes": [
             {
               "decimals": null,
-              "format": "none",
+              "format": "s",
               "label": null,
               "logBase": 1,
               "max": null,
@@ -47299,7 +47497,7 @@
           "bars": false,
           "cacheTimeout": null,
           "datasource": "${DS_TEST-CLUSTER}",
-          "description": "The total number of DAG executors",
+          "description": "The time consumed when coprocessor requests are wait for being handled in each TiKV instance",
           "editable": true,
           "error": false,
           "fieldConfig": {
@@ -47322,11 +47520,11 @@
             "h": 7,
             "w": 12,
             "x": 12,
-            "y": 14
+            "y": 7
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 332,
+          "id": 334,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -47362,22 +47560,90 @@
           "renderer": "flot",
           "repeat": null,
           "repeatDirection": null,
-          "seriesOverrides": [],
+          "seriesOverrides": [
+            {
+              "alias": "count",
+              "bars": false,
+              "dashLength": 1,
+              "dashes": true,
+              "fill": 2,
+              "fillBelowTo": null,
+              "lines": true,
+              "spaceLength": 1,
+              "transform": "negative-Y",
+              "yaxis": 2,
+              "zindex": -3
+            },
+            {
+              "alias": "avg",
+              "bars": false,
+              "fill": 7,
+              "fillBelowTo": null,
+              "lines": true,
+              "yaxis": 1,
+              "zindex": 0
+            }
+          ],
           "span": null,
           "stack": false,
           "steppedLine": false,
           "targets": [
             {
               "datasource": "${DS_TEST-CLUSTER}",
-              "expr": "sum(rate(\n    tikv_coprocessor_executor_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    [$__rate_interval]\n)) by (type, $additional_groupby) ",
+              "expr": "histogram_quantile(0.9999,(\n    sum(rate(\n    tikv_coprocessor_request_wait_seconds_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"all\"}\n    [$__rate_interval]\n)) by (req, instance, le) \n    \n    \n))  ",
               "format": "time_series",
               "hide": false,
               "instant": false,
               "interval": "",
               "intervalFactor": 1,
-              "legendFormat": "{{type}} {{$additional_groupby}}",
+              "legendFormat": "99.99%-{{req}}-{{instance}}",
               "metric": "",
-              "query": "sum(rate(\n    tikv_coprocessor_executor_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    [$__rate_interval]\n)) by (type, $additional_groupby) ",
+              "query": "histogram_quantile(0.9999,(\n    sum(rate(\n    tikv_coprocessor_request_wait_seconds_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"all\"}\n    [$__rate_interval]\n)) by (req, instance, le) \n    \n    \n))  ",
+              "refId": "",
+              "step": 10,
+              "target": ""
+            },
+            {
+              "datasource": "${DS_TEST-CLUSTER}",
+              "expr": "histogram_quantile(0.99,(\n    sum(rate(\n    tikv_coprocessor_request_wait_seconds_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"all\"}\n    [$__rate_interval]\n)) by (req, instance, le) \n    \n    \n))  ",
+              "format": "time_series",
+              "hide": false,
+              "instant": false,
+              "interval": "",
+              "intervalFactor": 1,
+              "legendFormat": "99%-{{req}}-{{instance}}",
+              "metric": "",
+              "query": "histogram_quantile(0.99,(\n    sum(rate(\n    tikv_coprocessor_request_wait_seconds_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"all\"}\n    [$__rate_interval]\n)) by (req, instance, le) \n    \n    \n))  ",
+              "refId": "",
+              "step": 10,
+              "target": ""
+            },
+            {
+              "datasource": "${DS_TEST-CLUSTER}",
+              "expr": "(sum(rate(\n    tikv_coprocessor_request_wait_seconds_sum\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"all\"}\n    [$__rate_interval]\n)) by (req, instance)  / sum(rate(\n    tikv_coprocessor_request_wait_seconds_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"all\"}\n    [$__rate_interval]\n)) by (req, instance) )",
+              "format": "time_series",
+              "hide": true,
+              "instant": false,
+              "interval": "",
+              "intervalFactor": 1,
+              "legendFormat": "avg-{{req}}-{{instance}}",
+              "metric": "",
+              "query": "(sum(rate(\n    tikv_coprocessor_request_wait_seconds_sum\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"all\"}\n    [$__rate_interval]\n)) by (req, instance)  / sum(rate(\n    tikv_coprocessor_request_wait_seconds_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"all\"}\n    [$__rate_interval]\n)) by (req, instance) )",
+              "refId": "",
+              "step": 10,
+              "target": ""
+            },
+            {
+              "datasource": "${DS_TEST-CLUSTER}",
+              "expr": "sum(rate(\n    tikv_coprocessor_request_wait_seconds_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"all\"}\n    [$__rate_interval]\n)) by (req, instance) ",
+              "format": "time_series",
+              "hide": true,
+              "instant": false,
+              "interval": "",
+              "intervalFactor": 1,
+              "legendFormat": "count-{{req}}-{{instance}}",
+              "metric": "",
+              "query": "sum(rate(\n    tikv_coprocessor_request_wait_seconds_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"all\"}\n    [$__rate_interval]\n)) by (req, instance) ",
               "refId": "",
               "step": 10,
               "target": ""
@@ -47386,7 +47652,7 @@
           "thresholds": [],
           "timeFrom": null,
           "timeShift": null,
-          "title": "Total DAG Executors",
+          "title": "Wait duration by store",
           "tooltip": {
             "msResolution": true,
             "shared": true,
@@ -47405,7 +47671,7 @@
           "yaxes": [
             {
               "decimals": null,
-              "format": "ops",
+              "format": "s",
               "label": null,
               "logBase": 1,
               "max": null,
@@ -47455,11 +47721,11 @@
             "h": 7,
             "w": 12,
             "x": 0,
-            "y": 21
+            "y": 14
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 333,
+          "id": 335,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -47502,15 +47768,15 @@
           "targets": [
             {
               "datasource": "${DS_TEST-CLUSTER}",
-              "expr": "sum(rate(\n    tikv_coprocessor_scan_details\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",req=~\"select|select_by_in_memory_engine\"}\n    [$__rate_interval]\n)) by (tag, $additional_groupby) ",
+              "expr": "sum(rate(\n    tikv_coprocessor_dag_request_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    [$__rate_interval]\n)) by (vec_type, $additional_groupby) ",
               "format": "time_series",
               "hide": false,
               "instant": false,
               "interval": "",
               "intervalFactor": 1,
-              "legendFormat": "{{tag}} {{$additional_groupby}}",
+              "legendFormat": "{{vec_type}} {{$additional_groupby}}",
               "metric": "",
-              "query": "sum(rate(\n    tikv_coprocessor_scan_details\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",req=~\"select|select_by_in_memory_engine\"}\n    [$__rate_interval]\n)) by (tag, $additional_groupby) ",
+              "query": "sum(rate(\n    tikv_coprocessor_dag_request_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    [$__rate_interval]\n)) by (vec_type, $additional_groupby) ",
               "refId": "",
               "step": 10,
               "target": ""
@@ -47519,7 +47785,7 @@
           "thresholds": [],
           "timeFrom": null,
           "timeShift": null,
-          "title": "Total Ops Details (Table Scan)",
+          "title": "Total DAG Requests",
           "tooltip": {
             "msResolution": true,
             "shared": true,
@@ -47538,7 +47804,7 @@
           "yaxes": [
             {
               "decimals": null,
-              "format": "ops",
+              "format": "none",
               "label": null,
               "logBase": 1,
               "max": null,
@@ -47565,7 +47831,7 @@
           "bars": false,
           "cacheTimeout": null,
           "datasource": "${DS_TEST-CLUSTER}",
-          "description": null,
+          "description": "The total number of DAG executors",
           "editable": true,
           "error": false,
           "fieldConfig": {
@@ -47588,11 +47854,11 @@
             "h": 7,
             "w": 12,
             "x": 12,
-            "y": 21
+            "y": 14
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 334,
+          "id": 336,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -47635,15 +47901,15 @@
           "targets": [
             {
               "datasource": "${DS_TEST-CLUSTER}",
-              "expr": "sum(rate(\n    tikv_coprocessor_scan_details\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",req=~\"index|index_by_region_cache\"}\n    [$__rate_interval]\n)) by (tag, $additional_groupby) ",
+              "expr": "sum(rate(\n    tikv_coprocessor_executor_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    [$__rate_interval]\n)) by (type, $additional_groupby) ",
               "format": "time_series",
               "hide": false,
               "instant": false,
               "interval": "",
               "intervalFactor": 1,
-              "legendFormat": "{{tag}} {{$additional_groupby}}",
+              "legendFormat": "{{type}} {{$additional_groupby}}",
               "metric": "",
-              "query": "sum(rate(\n    tikv_coprocessor_scan_details\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",req=~\"index|index_by_region_cache\"}\n    [$__rate_interval]\n)) by (tag, $additional_groupby) ",
+              "query": "sum(rate(\n    tikv_coprocessor_executor_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    [$__rate_interval]\n)) by (type, $additional_groupby) ",
               "refId": "",
               "step": 10,
               "target": ""
@@ -47652,7 +47918,7 @@
           "thresholds": [],
           "timeFrom": null,
           "timeShift": null,
-          "title": "Total Ops Details (Index Scan)",
+          "title": "Total DAG Executors",
           "tooltip": {
             "msResolution": true,
             "shared": true,
@@ -47721,11 +47987,11 @@
             "h": 7,
             "w": 12,
             "x": 0,
-            "y": 28
+            "y": 21
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 335,
+          "id": 337,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -47768,15 +48034,15 @@
           "targets": [
             {
               "datasource": "${DS_TEST-CLUSTER}",
-              "expr": "sum(rate(\n    tikv_coprocessor_scan_details\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",req=~\"select|select_by_in_memory_engine\"}\n    [$__rate_interval]\n)) by (cf, tag, $additional_groupby) ",
+              "expr": "sum(rate(\n    tikv_coprocessor_scan_details\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",req=~\"select|select_by_in_memory_engine\"}\n    [$__rate_interval]\n)) by (tag, $additional_groupby) ",
               "format": "time_series",
               "hide": false,
               "instant": false,
               "interval": "",
               "intervalFactor": 1,
-              "legendFormat": "{{cf}}-{{tag}} {{$additional_groupby}}",
+              "legendFormat": "{{tag}} {{$additional_groupby}}",
               "metric": "",
-              "query": "sum(rate(\n    tikv_coprocessor_scan_details\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",req=~\"select|select_by_in_memory_engine\"}\n    [$__rate_interval]\n)) by (cf, tag, $additional_groupby) ",
+              "query": "sum(rate(\n    tikv_coprocessor_scan_details\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",req=~\"select|select_by_in_memory_engine\"}\n    [$__rate_interval]\n)) by (tag, $additional_groupby) ",
               "refId": "",
               "step": 10,
               "target": ""
@@ -47785,7 +48051,7 @@
           "thresholds": [],
           "timeFrom": null,
           "timeShift": null,
-          "title": "Total Ops Details by CF (Table Scan)",
+          "title": "Total Ops Details (Table Scan)",
           "tooltip": {
             "msResolution": true,
             "shared": true,
@@ -47854,11 +48120,277 @@
             "h": 7,
             "w": 12,
             "x": 12,
-            "y": 28
+            "y": 21
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 336,
+          "id": 338,
+          "interval": null,
+          "isNew": true,
+          "legend": {
+            "alignAsTable": true,
+            "avg": false,
+            "current": true,
+            "hideEmpty": true,
+            "hideZero": true,
+            "max": true,
+            "min": false,
+            "rightSide": true,
+            "show": true,
+            "sideWidth": null,
+            "sort": "max",
+            "sortDesc": true,
+            "total": false,
+            "values": true
+          },
+          "lines": true,
+          "linewidth": 1,
+          "links": [],
+          "maxDataPoints": null,
+          "maxPerRow": null,
+          "minSpan": null,
+          "nullPointMode": "null as zero",
+          "options": {
+            "alertThreshold": true,
+            "dataLinks": []
+          },
+          "percentage": false,
+          "pointradius": 5,
+          "points": false,
+          "renderer": "flot",
+          "repeat": null,
+          "repeatDirection": null,
+          "seriesOverrides": [],
+          "span": null,
+          "stack": false,
+          "steppedLine": false,
+          "targets": [
+            {
+              "datasource": "${DS_TEST-CLUSTER}",
+              "expr": "sum(rate(\n    tikv_coprocessor_scan_details\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",req=~\"index|index_by_region_cache\"}\n    [$__rate_interval]\n)) by (tag, $additional_groupby) ",
+              "format": "time_series",
+              "hide": false,
+              "instant": false,
+              "interval": "",
+              "intervalFactor": 1,
+              "legendFormat": "{{tag}} {{$additional_groupby}}",
+              "metric": "",
+              "query": "sum(rate(\n    tikv_coprocessor_scan_details\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",req=~\"index|index_by_region_cache\"}\n    [$__rate_interval]\n)) by (tag, $additional_groupby) ",
+              "refId": "",
+              "step": 10,
+              "target": ""
+            }
+          ],
+          "thresholds": [],
+          "timeFrom": null,
+          "timeShift": null,
+          "title": "Total Ops Details (Index Scan)",
+          "tooltip": {
+            "msResolution": true,
+            "shared": true,
+            "sort": 0,
+            "value_type": "individual"
+          },
+          "transformations": [],
+          "transparent": false,
+          "type": "graph",
+          "xaxis": {
+            "mode": "time",
+            "name": null,
+            "show": true,
+            "values": []
+          },
+          "yaxes": [
+            {
+              "decimals": null,
+              "format": "ops",
+              "label": null,
+              "logBase": 1,
+              "max": null,
+              "min": null,
+              "show": true
+            },
+            {
+              "decimals": null,
+              "format": "short",
+              "label": null,
+              "logBase": 1,
+              "max": null,
+              "min": null,
+              "show": true
+            }
+          ],
+          "yaxis": {
+            "align": false,
+            "alignLevel": 0
+          }
+        },
+        {
+          "aliasColors": {},
+          "bars": false,
+          "cacheTimeout": null,
+          "datasource": "${DS_TEST-CLUSTER}",
+          "description": null,
+          "editable": true,
+          "error": false,
+          "fieldConfig": {
+            "defaults": {
+              "thresholds": {
+                "mode": "absolute",
+                "steps": []
+              }
+            }
+          },
+          "fill": 1,
+          "fillGradient": 1,
+          "grid": {
+            "threshold1": null,
+            "threshold1Color": "rgba(216, 200, 27, 0.27)",
+            "threshold2": null,
+            "threshold2Color": "rgba(234, 112, 112, 0.22)"
+          },
+          "gridPos": {
+            "h": 7,
+            "w": 12,
+            "x": 0,
+            "y": 28
+          },
+          "height": null,
+          "hideTimeOverride": false,
+          "id": 339,
+          "interval": null,
+          "isNew": true,
+          "legend": {
+            "alignAsTable": true,
+            "avg": false,
+            "current": true,
+            "hideEmpty": true,
+            "hideZero": true,
+            "max": true,
+            "min": false,
+            "rightSide": true,
+            "show": true,
+            "sideWidth": null,
+            "sort": "max",
+            "sortDesc": true,
+            "total": false,
+            "values": true
+          },
+          "lines": true,
+          "linewidth": 1,
+          "links": [],
+          "maxDataPoints": null,
+          "maxPerRow": null,
+          "minSpan": null,
+          "nullPointMode": "null as zero",
+          "options": {
+            "alertThreshold": true,
+            "dataLinks": []
+          },
+          "percentage": false,
+          "pointradius": 5,
+          "points": false,
+          "renderer": "flot",
+          "repeat": null,
+          "repeatDirection": null,
+          "seriesOverrides": [],
+          "span": null,
+          "stack": false,
+          "steppedLine": false,
+          "targets": [
+            {
+              "datasource": "${DS_TEST-CLUSTER}",
+              "expr": "sum(rate(\n    tikv_coprocessor_scan_details\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",req=~\"select|select_by_in_memory_engine\"}\n    [$__rate_interval]\n)) by (cf, tag, $additional_groupby) ",
+              "format": "time_series",
+              "hide": false,
+              "instant": false,
+              "interval": "",
+              "intervalFactor": 1,
+              "legendFormat": "{{cf}}-{{tag}} {{$additional_groupby}}",
+              "metric": "",
+              "query": "sum(rate(\n    tikv_coprocessor_scan_details\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",req=~\"select|select_by_in_memory_engine\"}\n    [$__rate_interval]\n)) by (cf, tag, $additional_groupby) ",
+              "refId": "",
+              "step": 10,
+              "target": ""
+            }
+          ],
+          "thresholds": [],
+          "timeFrom": null,
+          "timeShift": null,
+          "title": "Total Ops Details by CF (Table Scan)",
+          "tooltip": {
+            "msResolution": true,
+            "shared": true,
+            "sort": 0,
+            "value_type": "individual"
+          },
+          "transformations": [],
+          "transparent": false,
+          "type": "graph",
+          "xaxis": {
+            "mode": "time",
+            "name": null,
+            "show": true,
+            "values": []
+          },
+          "yaxes": [
+            {
+              "decimals": null,
+              "format": "ops",
+              "label": null,
+              "logBase": 1,
+              "max": null,
+              "min": null,
+              "show": true
+            },
+            {
+              "decimals": null,
+              "format": "short",
+              "label": null,
+              "logBase": 1,
+              "max": null,
+              "min": null,
+              "show": true
+            }
+          ],
+          "yaxis": {
+            "align": false,
+            "alignLevel": 0
+          }
+        },
+        {
+          "aliasColors": {},
+          "bars": false,
+          "cacheTimeout": null,
+          "datasource": "${DS_TEST-CLUSTER}",
+          "description": null,
+          "editable": true,
+          "error": false,
+          "fieldConfig": {
+            "defaults": {
+              "thresholds": {
+                "mode": "absolute",
+                "steps": []
+              }
+            }
+          },
+          "fill": 1,
+          "fillGradient": 1,
+          "grid": {
+            "threshold1": null,
+            "threshold1Color": "rgba(216, 200, 27, 0.27)",
+            "threshold2": null,
+            "threshold2Color": "rgba(234, 112, 112, 0.22)"
+          },
+          "gridPos": {
+            "h": 7,
+            "w": 12,
+            "x": 12,
+            "y": 28
+          },
+          "height": null,
+          "hideTimeOverride": false,
+          "id": 340,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -47998,7 +48530,7 @@
           "hideTimeOverride": false,
           "hideZeroBuckets": true,
           "highlightCards": true,
-          "id": 337,
+          "id": 341,
           "interval": null,
           "legend": {
             "show": false
@@ -48096,7 +48628,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 338,
+          "id": 342,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -48300,7 +48832,7 @@
       },
       "height": null,
       "hideTimeOverride": false,
-      "id": 339,
+      "id": 343,
       "interval": null,
       "links": [],
       "maxDataPoints": 100,
@@ -48339,7 +48871,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 340,
+          "id": 344,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -48472,7 +49004,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 341,
+          "id": 345,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -48605,7 +49137,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 342,
+          "id": 346,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -48745,7 +49277,7 @@
           "hideTimeOverride": false,
           "hideZeroBuckets": true,
           "highlightCards": true,
-          "id": 343,
+          "id": 347,
           "interval": null,
           "legend": {
             "show": false
@@ -48843,7 +49375,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 344,
+          "id": 348,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -49044,7 +49576,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 345,
+          "id": 349,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -49245,7 +49777,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 346,
+          "id": 350,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -49449,7 +49981,7 @@
       },
       "height": null,
       "hideTimeOverride": false,
-      "id": 347,
+      "id": 351,
       "interval": null,
       "links": [],
       "maxDataPoints": 100,
@@ -49488,7 +50020,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 348,
+          "id": 352,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -49666,7 +50198,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 349,
+          "id": 353,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -49867,7 +50399,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 350,
+          "id": 354,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -50000,7 +50532,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 351,
+          "id": 355,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -50133,7 +50665,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 352,
+          "id": 356,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -50266,7 +50798,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 353,
+          "id": 357,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -50399,7 +50931,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 354,
+          "id": 358,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -50532,7 +51064,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 355,
+          "id": 359,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -50661,7 +51193,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 356,
+          "id": 360,
           "interval": null,
           "links": [],
           "maxDataPoints": 100,
@@ -50736,7 +51268,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 357,
+          "id": 361,
           "interval": null,
           "links": [],
           "maxDataPoints": 100,
@@ -50815,7 +51347,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 358,
+          "id": 362,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -51068,7 +51600,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 359,
+          "id": 363,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -51201,7 +51733,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 360,
+          "id": 364,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -51337,7 +51869,7 @@
       },
       "height": null,
       "hideTimeOverride": false,
-      "id": 361,
+      "id": 365,
       "interval": null,
       "links": [],
       "maxDataPoints": 100,
@@ -51376,7 +51908,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 362,
+          "id": 366,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -51524,7 +52056,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 363,
+          "id": 367,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -51657,7 +52189,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 364,
+          "id": 368,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -51858,7 +52390,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 365,
+          "id": 369,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -52006,7 +52538,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 366,
+          "id": 370,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -52207,7 +52739,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 367,
+          "id": 371,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -52340,7 +52872,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 368,
+          "id": 372,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -52473,7 +53005,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 369,
+          "id": 373,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -52606,7 +53138,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 370,
+          "id": 374,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -52739,7 +53271,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 371,
+          "id": 375,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -52879,7 +53411,7 @@
           "hideTimeOverride": false,
           "hideZeroBuckets": true,
           "highlightCards": true,
-          "id": 372,
+          "id": 376,
           "interval": null,
           "legend": {
             "show": false
@@ -52977,383 +53509,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 373,
-          "interval": null,
-          "isNew": true,
-          "legend": {
-            "alignAsTable": true,
-            "avg": false,
-            "current": true,
-            "hideEmpty": true,
-            "hideZero": true,
-            "max": true,
-            "min": false,
-            "rightSide": true,
-            "show": true,
-            "sideWidth": null,
-            "sort": "max",
-            "sortDesc": true,
-            "total": false,
-            "values": true
-          },
-          "lines": true,
-          "linewidth": 1,
-          "links": [],
-          "maxDataPoints": null,
-          "maxPerRow": null,
-          "minSpan": null,
-          "nullPointMode": "null as zero",
-          "options": {
-            "alertThreshold": true,
-            "dataLinks": []
-          },
-          "percentage": false,
-          "pointradius": 5,
-          "points": false,
-          "renderer": "flot",
-          "repeat": null,
-          "repeatDirection": null,
-          "seriesOverrides": [
-            {
-              "alias": "count",
-              "bars": false,
-              "dashLength": 1,
-              "dashes": true,
-              "fill": 2,
-              "fillBelowTo": null,
-              "lines": true,
-              "spaceLength": 1,
-              "transform": "negative-Y",
-              "yaxis": 2,
-              "zindex": -3
-            },
-            {
-              "alias": "avg",
-              "bars": false,
-              "fill": 7,
-              "fillBelowTo": null,
-              "lines": true,
-              "yaxis": 1,
-              "zindex": 0
-            }
-          ],
-          "span": null,
-          "stack": false,
-          "steppedLine": false,
-          "targets": [
-            {
-              "datasource": "${DS_TEST-CLUSTER}",
-              "expr": "histogram_quantile(0.9999,(\n    sum(rate(\n    tikv_storage_mvcc_scan_lock_read_duration_seconds_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    [$__rate_interval]\n)) by (type, le, $additional_groupby) \n    \n    \n))  ",
-              "format": "time_series",
-              "hide": false,
-              "instant": false,
-              "interval": "",
-              "intervalFactor": 1,
-              "legendFormat": "99.99%-{{type}} {{$additional_groupby}}",
-              "metric": "",
-              "query": "histogram_quantile(0.9999,(\n    sum(rate(\n    tikv_storage_mvcc_scan_lock_read_duration_seconds_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    [$__rate_interval]\n)) by (type, le, $additional_groupby) \n    \n    \n))  ",
-              "refId": "",
-              "step": 10,
-              "target": ""
-            },
-            {
-              "datasource": "${DS_TEST-CLUSTER}",
-              "expr": "histogram_quantile(0.99,(\n    sum(rate(\n    tikv_storage_mvcc_scan_lock_read_duration_seconds_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    [$__rate_interval]\n)) by (type, le, $additional_groupby) \n    \n    \n))  ",
-              "format": "time_series",
-              "hide": false,
-              "instant": false,
-              "interval": "",
-              "intervalFactor": 1,
-              "legendFormat": "99%-{{type}} {{$additional_groupby}}",
-              "metric": "",
-              "query": "histogram_quantile(0.99,(\n    sum(rate(\n    tikv_storage_mvcc_scan_lock_read_duration_seconds_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    [$__rate_interval]\n)) by (type, le, $additional_groupby) \n    \n    \n))  ",
-              "refId": "",
-              "step": 10,
-              "target": ""
-            },
-            {
-              "datasource": "${DS_TEST-CLUSTER}",
-              "expr": "(sum(rate(\n    tikv_storage_mvcc_scan_lock_read_duration_seconds_sum\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    [$__rate_interval]\n)) by (type, $additional_groupby)  / sum(rate(\n    tikv_storage_mvcc_scan_lock_read_duration_seconds_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    [$__rate_interval]\n)) by (type, $additional_groupby) )",
-              "format": "time_series",
-              "hide": true,
-              "instant": false,
-              "interval": "",
-              "intervalFactor": 1,
-              "legendFormat": "avg-{{type}} {{$additional_groupby}}",
-              "metric": "",
-              "query": "(sum(rate(\n    tikv_storage_mvcc_scan_lock_read_duration_seconds_sum\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    [$__rate_interval]\n)) by (type, $additional_groupby)  / sum(rate(\n    tikv_storage_mvcc_scan_lock_read_duration_seconds_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    [$__rate_interval]\n)) by (type, $additional_groupby) )",
-              "refId": "",
-              "step": 10,
-              "target": ""
-            },
-            {
-              "datasource": "${DS_TEST-CLUSTER}",
-              "expr": "sum(rate(\n    tikv_storage_mvcc_scan_lock_read_duration_seconds_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    [$__rate_interval]\n)) by (type, $additional_groupby) ",
-              "format": "time_series",
-              "hide": true,
-              "instant": false,
-              "interval": "",
-              "intervalFactor": 1,
-              "legendFormat": "count-{{type}} {{$additional_groupby}}",
-              "metric": "",
-              "query": "sum(rate(\n    tikv_storage_mvcc_scan_lock_read_duration_seconds_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    [$__rate_interval]\n)) by (type, $additional_groupby) ",
-              "refId": "",
-              "step": 10,
-              "target": ""
-            }
-          ],
-          "thresholds": [],
-          "timeFrom": null,
-          "timeShift": null,
-          "title": "In-memory scan lock read duration",
-          "tooltip": {
-            "msResolution": true,
-            "shared": true,
-            "sort": 0,
-            "value_type": "individual"
-          },
-          "transformations": [],
-          "transparent": false,
-          "type": "graph",
-          "xaxis": {
-            "mode": "time",
-            "name": null,
-            "show": true,
-            "values": []
-          },
-          "yaxes": [
-            {
-              "decimals": null,
-              "format": "s",
-              "label": null,
-              "logBase": 2,
-              "max": null,
-              "min": null,
-              "show": true
-            },
-            {
-              "decimals": null,
-              "format": "short",
-              "label": null,
-              "logBase": 1,
-              "max": null,
-              "min": null,
-              "show": true
-            }
-          ],
-          "yaxis": {
-            "align": false,
-            "alignLevel": 0
-          }
-        }
-      ],
-      "repeat": null,
-      "repeatDirection": null,
-      "span": null,
-      "targets": [],
-      "timeFrom": null,
-      "timeShift": null,
-      "title": "Pessimistic Locking",
-      "transformations": [],
-      "transparent": false,
-      "type": "row"
-    },
-    {
-      "cacheTimeout": null,
-      "collapsed": true,
-      "datasource": null,
-      "description": null,
-      "editable": true,
-      "error": false,
-      "fieldConfig": {
-        "defaults": {
-          "thresholds": {
-            "mode": "absolute",
-            "steps": []
-          }
-        }
-      },
-      "gridPos": {
-        "h": 7,
-        "w": 24,
-        "x": 0,
-        "y": 0
-      },
-      "height": null,
-      "hideTimeOverride": false,
-      "id": 374,
-      "interval": null,
-      "links": [],
-      "maxDataPoints": 100,
-      "maxPerRow": null,
-      "minSpan": null,
-      "panels": [
-        {
-          "aliasColors": {},
-          "bars": false,
-          "cacheTimeout": null,
-          "datasource": "${DS_TEST-CLUSTER}",
-          "description": "The number of tasks handled by worker",
-          "editable": true,
-          "error": false,
-          "fieldConfig": {
-            "defaults": {
-              "thresholds": {
-                "mode": "absolute",
-                "steps": []
-              }
-            }
-          },
-          "fill": 1,
-          "fillGradient": 1,
-          "grid": {
-            "threshold1": null,
-            "threshold1Color": "rgba(216, 200, 27, 0.27)",
-            "threshold2": null,
-            "threshold2Color": "rgba(234, 112, 112, 0.22)"
-          },
-          "gridPos": {
-            "h": 7,
-            "w": 12,
-            "x": 0,
-            "y": 0
-          },
-          "height": null,
-          "hideTimeOverride": false,
-          "id": 375,
-          "interval": null,
-          "isNew": true,
-          "legend": {
-            "alignAsTable": true,
-            "avg": false,
-            "current": true,
-            "hideEmpty": true,
-            "hideZero": true,
-            "max": true,
-            "min": false,
-            "rightSide": true,
-            "show": true,
-            "sideWidth": null,
-            "sort": "max",
-            "sortDesc": true,
-            "total": false,
-            "values": true
-          },
-          "lines": true,
-          "linewidth": 1,
-          "links": [],
-          "maxDataPoints": null,
-          "maxPerRow": null,
-          "minSpan": null,
-          "nullPointMode": "null as zero",
-          "options": {
-            "alertThreshold": true,
-            "dataLinks": []
-          },
-          "percentage": false,
-          "pointradius": 5,
-          "points": false,
-          "renderer": "flot",
-          "repeat": null,
-          "repeatDirection": null,
-          "seriesOverrides": [],
-          "span": null,
-          "stack": false,
-          "steppedLine": false,
-          "targets": [
-            {
-              "datasource": "${DS_TEST-CLUSTER}",
-              "expr": "sum(rate(\n    tikv_worker_handled_task_total\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    [$__rate_interval]\n)) by (name, $additional_groupby) ",
-              "format": "time_series",
-              "hide": false,
-              "instant": false,
-              "interval": "",
-              "intervalFactor": 1,
-              "legendFormat": "{{name}} {{$additional_groupby}}",
-              "metric": "",
-              "query": "sum(rate(\n    tikv_worker_handled_task_total\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    [$__rate_interval]\n)) by (name, $additional_groupby) ",
-              "refId": "",
-              "step": 10,
-              "target": ""
-            }
-          ],
-          "thresholds": [],
-          "timeFrom": null,
-          "timeShift": null,
-          "title": "Worker handled tasks",
-          "tooltip": {
-            "msResolution": true,
-            "shared": true,
-            "sort": 0,
-            "value_type": "individual"
-          },
-          "transformations": [],
-          "transparent": false,
-          "type": "graph",
-          "xaxis": {
-            "mode": "time",
-            "name": null,
-            "show": true,
-            "values": []
-          },
-          "yaxes": [
-            {
-              "decimals": null,
-              "format": "ops",
-              "label": null,
-              "logBase": 1,
-              "max": null,
-              "min": null,
-              "show": true
-            },
-            {
-              "decimals": null,
-              "format": "short",
-              "label": null,
-              "logBase": 1,
-              "max": null,
-              "min": null,
-              "show": true
-            }
-          ],
-          "yaxis": {
-            "align": false,
-            "alignLevel": 0
-          }
-        },
-        {
-          "aliasColors": {},
-          "bars": false,
-          "cacheTimeout": null,
-          "datasource": "${DS_TEST-CLUSTER}",
-          "description": "Current pending and running tasks of worker",
-          "editable": true,
-          "error": false,
-          "fieldConfig": {
-            "defaults": {
-              "thresholds": {
-                "mode": "absolute",
-                "steps": []
-              }
-            }
-          },
-          "fill": 1,
-          "fillGradient": 1,
-          "grid": {
-            "threshold1": null,
-            "threshold1Color": "rgba(216, 200, 27, 0.27)",
-            "threshold2": null,
-            "threshold2Color": "rgba(234, 112, 112, 0.22)"
-          },
-          "gridPos": {
-            "h": 7,
-            "w": 12,
-            "x": 12,
-            "y": 0
-          },
-          "height": null,
-          "hideTimeOverride": false,
-          "id": 376,
+          "id": 377,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -53389,22 +53545,90 @@
           "renderer": "flot",
           "repeat": null,
           "repeatDirection": null,
-          "seriesOverrides": [],
+          "seriesOverrides": [
+            {
+              "alias": "count",
+              "bars": false,
+              "dashLength": 1,
+              "dashes": true,
+              "fill": 2,
+              "fillBelowTo": null,
+              "lines": true,
+              "spaceLength": 1,
+              "transform": "negative-Y",
+              "yaxis": 2,
+              "zindex": -3
+            },
+            {
+              "alias": "avg",
+              "bars": false,
+              "fill": 7,
+              "fillBelowTo": null,
+              "lines": true,
+              "yaxis": 1,
+              "zindex": 0
+            }
+          ],
           "span": null,
           "stack": false,
           "steppedLine": false,
           "targets": [
             {
               "datasource": "${DS_TEST-CLUSTER}",
-              "expr": "sum((\n    tikv_worker_pending_task_total\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    \n)) by (name, $additional_groupby) ",
+              "expr": "histogram_quantile(0.9999,(\n    sum(rate(\n    tikv_storage_mvcc_scan_lock_read_duration_seconds_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    [$__rate_interval]\n)) by (type, le, $additional_groupby) \n    \n    \n))  ",
               "format": "time_series",
               "hide": false,
               "instant": false,
               "interval": "",
               "intervalFactor": 1,
-              "legendFormat": "{{name}} {{$additional_groupby}}",
+              "legendFormat": "99.99%-{{type}} {{$additional_groupby}}",
               "metric": "",
-              "query": "sum((\n    tikv_worker_pending_task_total\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    \n)) by (name, $additional_groupby) ",
+              "query": "histogram_quantile(0.9999,(\n    sum(rate(\n    tikv_storage_mvcc_scan_lock_read_duration_seconds_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    [$__rate_interval]\n)) by (type, le, $additional_groupby) \n    \n    \n))  ",
+              "refId": "",
+              "step": 10,
+              "target": ""
+            },
+            {
+              "datasource": "${DS_TEST-CLUSTER}",
+              "expr": "histogram_quantile(0.99,(\n    sum(rate(\n    tikv_storage_mvcc_scan_lock_read_duration_seconds_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    [$__rate_interval]\n)) by (type, le, $additional_groupby) \n    \n    \n))  ",
+              "format": "time_series",
+              "hide": false,
+              "instant": false,
+              "interval": "",
+              "intervalFactor": 1,
+              "legendFormat": "99%-{{type}} {{$additional_groupby}}",
+              "metric": "",
+              "query": "histogram_quantile(0.99,(\n    sum(rate(\n    tikv_storage_mvcc_scan_lock_read_duration_seconds_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    [$__rate_interval]\n)) by (type, le, $additional_groupby) \n    \n    \n))  ",
+              "refId": "",
+              "step": 10,
+              "target": ""
+            },
+            {
+              "datasource": "${DS_TEST-CLUSTER}",
+              "expr": "(sum(rate(\n    tikv_storage_mvcc_scan_lock_read_duration_seconds_sum\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    [$__rate_interval]\n)) by (type, $additional_groupby)  / sum(rate(\n    tikv_storage_mvcc_scan_lock_read_duration_seconds_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    [$__rate_interval]\n)) by (type, $additional_groupby) )",
+              "format": "time_series",
+              "hide": true,
+              "instant": false,
+              "interval": "",
+              "intervalFactor": 1,
+              "legendFormat": "avg-{{type}} {{$additional_groupby}}",
+              "metric": "",
+              "query": "(sum(rate(\n    tikv_storage_mvcc_scan_lock_read_duration_seconds_sum\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    [$__rate_interval]\n)) by (type, $additional_groupby)  / sum(rate(\n    tikv_storage_mvcc_scan_lock_read_duration_seconds_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    [$__rate_interval]\n)) by (type, $additional_groupby) )",
+              "refId": "",
+              "step": 10,
+              "target": ""
+            },
+            {
+              "datasource": "${DS_TEST-CLUSTER}",
+              "expr": "sum(rate(\n    tikv_storage_mvcc_scan_lock_read_duration_seconds_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    [$__rate_interval]\n)) by (type, $additional_groupby) ",
+              "format": "time_series",
+              "hide": true,
+              "instant": false,
+              "interval": "",
+              "intervalFactor": 1,
+              "legendFormat": "count-{{type}} {{$additional_groupby}}",
+              "metric": "",
+              "query": "sum(rate(\n    tikv_storage_mvcc_scan_lock_read_duration_seconds_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    [$__rate_interval]\n)) by (type, $additional_groupby) ",
               "refId": "",
               "step": 10,
               "target": ""
@@ -53413,7 +53637,7 @@
           "thresholds": [],
           "timeFrom": null,
           "timeShift": null,
-          "title": "Worker pending tasks",
+          "title": "In-memory scan lock read duration",
           "tooltip": {
             "msResolution": true,
             "shared": true,
@@ -53432,9 +53656,9 @@
           "yaxes": [
             {
               "decimals": null,
-              "format": "none",
+              "format": "s",
               "label": null,
-              "logBase": 1,
+              "logBase": 2,
               "max": null,
               "min": null,
               "show": true
@@ -53453,13 +53677,55 @@
             "align": false,
             "alignLevel": 0
           }
-        },
+        }
+      ],
+      "repeat": null,
+      "repeatDirection": null,
+      "span": null,
+      "targets": [],
+      "timeFrom": null,
+      "timeShift": null,
+      "title": "Pessimistic Locking",
+      "transformations": [],
+      "transparent": false,
+      "type": "row"
+    },
+    {
+      "cacheTimeout": null,
+      "collapsed": true,
+      "datasource": null,
+      "description": null,
+      "editable": true,
+      "error": false,
+      "fieldConfig": {
+        "defaults": {
+          "thresholds": {
+            "mode": "absolute",
+            "steps": []
+          }
+        }
+      },
+      "gridPos": {
+        "h": 7,
+        "w": 24,
+        "x": 0,
+        "y": 0
+      },
+      "height": null,
+      "hideTimeOverride": false,
+      "id": 378,
+      "interval": null,
+      "links": [],
+      "maxDataPoints": 100,
+      "maxPerRow": null,
+      "minSpan": null,
+      "panels": [
         {
           "aliasColors": {},
           "bars": false,
           "cacheTimeout": null,
           "datasource": "${DS_TEST-CLUSTER}",
-          "description": "The number of tasks handled by future_pool",
+          "description": "The number of tasks handled by worker",
           "editable": true,
           "error": false,
           "fieldConfig": {
@@ -53482,11 +53748,11 @@
             "h": 7,
             "w": 12,
             "x": 0,
-            "y": 7
+            "y": 0
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 377,
+          "id": 379,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -53529,7 +53795,7 @@
           "targets": [
             {
               "datasource": "${DS_TEST-CLUSTER}",
-              "expr": "sum(rate(\n    tikv_futurepool_handled_task_total\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    [$__rate_interval]\n)) by (name, $additional_groupby) ",
+              "expr": "sum(rate(\n    tikv_worker_handled_task_total\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    [$__rate_interval]\n)) by (name, $additional_groupby) ",
               "format": "time_series",
               "hide": false,
               "instant": false,
@@ -53537,7 +53803,7 @@
               "intervalFactor": 1,
               "legendFormat": "{{name}} {{$additional_groupby}}",
               "metric": "",
-              "query": "sum(rate(\n    tikv_futurepool_handled_task_total\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    [$__rate_interval]\n)) by (name, $additional_groupby) ",
+              "query": "sum(rate(\n    tikv_worker_handled_task_total\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    [$__rate_interval]\n)) by (name, $additional_groupby) ",
               "refId": "",
               "step": 10,
               "target": ""
@@ -53546,7 +53812,7 @@
           "thresholds": [],
           "timeFrom": null,
           "timeShift": null,
-          "title": "FuturePool handled tasks",
+          "title": "Worker handled tasks",
           "tooltip": {
             "msResolution": true,
             "shared": true,
@@ -53592,7 +53858,7 @@
           "bars": false,
           "cacheTimeout": null,
           "datasource": "${DS_TEST-CLUSTER}",
-          "description": "Current pending and running tasks of future_pool",
+          "description": "Current pending and running tasks of worker",
           "editable": true,
           "error": false,
           "fieldConfig": {
@@ -53615,11 +53881,11 @@
             "h": 7,
             "w": 12,
             "x": 12,
-            "y": 7
+            "y": 0
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 378,
+          "id": 380,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -53662,7 +53928,7 @@
           "targets": [
             {
               "datasource": "${DS_TEST-CLUSTER}",
-              "expr": "sum(avg_over_time(\n    tikv_futurepool_pending_task_total\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    [1m]\n)) by (name, $additional_groupby) ",
+              "expr": "sum((\n    tikv_worker_pending_task_total\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    \n)) by (name, $additional_groupby) ",
               "format": "time_series",
               "hide": false,
               "instant": false,
@@ -53670,7 +53936,7 @@
               "intervalFactor": 1,
               "legendFormat": "{{name}} {{$additional_groupby}}",
               "metric": "",
-              "query": "sum(avg_over_time(\n    tikv_futurepool_pending_task_total\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    [1m]\n)) by (name, $additional_groupby) ",
+              "query": "sum((\n    tikv_worker_pending_task_total\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    \n)) by (name, $additional_groupby) ",
               "refId": "",
               "step": 10,
               "target": ""
@@ -53679,7 +53945,7 @@
           "thresholds": [],
           "timeFrom": null,
           "timeShift": null,
-          "title": "FuturePool pending tasks",
+          "title": "Worker pending tasks",
           "tooltip": {
             "msResolution": true,
             "shared": true,
@@ -53719,55 +53985,13 @@
             "align": false,
             "alignLevel": 0
           }
-        }
-      ],
-      "repeat": null,
-      "repeatDirection": null,
-      "span": null,
-      "targets": [],
-      "timeFrom": null,
-      "timeShift": null,
-      "title": "Task",
-      "transformations": [],
-      "transparent": false,
-      "type": "row"
-    },
-    {
-      "cacheTimeout": null,
-      "collapsed": true,
-      "datasource": null,
-      "description": null,
-      "editable": true,
-      "error": false,
-      "fieldConfig": {
-        "defaults": {
-          "thresholds": {
-            "mode": "absolute",
-            "steps": []
-          }
-        }
-      },
-      "gridPos": {
-        "h": 7,
-        "w": 24,
-        "x": 0,
-        "y": 0
-      },
-      "height": null,
-      "hideTimeOverride": false,
-      "id": 379,
-      "interval": null,
-      "links": [],
-      "maxDataPoints": 100,
-      "maxPerRow": null,
-      "minSpan": null,
-      "panels": [
+        },
         {
           "aliasColors": {},
           "bars": false,
           "cacheTimeout": null,
           "datasource": "${DS_TEST-CLUSTER}",
-          "description": "The count of requests that TiKV sends to PD",
+          "description": "The number of tasks handled by future_pool",
           "editable": true,
           "error": false,
           "fieldConfig": {
@@ -53790,11 +54014,11 @@
             "h": 7,
             "w": 12,
             "x": 0,
-            "y": 0
+            "y": 7
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 380,
+          "id": 381,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -53837,15 +54061,15 @@
           "targets": [
             {
               "datasource": "${DS_TEST-CLUSTER}",
-              "expr": "sum(rate(\n    tikv_pd_request_duration_seconds_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    [$__rate_interval]\n)) by (type, $additional_groupby) ",
+              "expr": "sum(rate(\n    tikv_futurepool_handled_task_total\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    [$__rate_interval]\n)) by (name, $additional_groupby) ",
               "format": "time_series",
               "hide": false,
               "instant": false,
               "interval": "",
               "intervalFactor": 1,
-              "legendFormat": "{{type}} {{$additional_groupby}}",
+              "legendFormat": "{{name}} {{$additional_groupby}}",
               "metric": "",
-              "query": "sum(rate(\n    tikv_pd_request_duration_seconds_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    [$__rate_interval]\n)) by (type, $additional_groupby) ",
+              "query": "sum(rate(\n    tikv_futurepool_handled_task_total\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    [$__rate_interval]\n)) by (name, $additional_groupby) ",
               "refId": "",
               "step": 10,
               "target": ""
@@ -53854,7 +54078,7 @@
           "thresholds": [],
           "timeFrom": null,
           "timeShift": null,
-          "title": "PD requests",
+          "title": "FuturePool handled tasks",
           "tooltip": {
             "msResolution": true,
             "shared": true,
@@ -53900,7 +54124,7 @@
           "bars": false,
           "cacheTimeout": null,
           "datasource": "${DS_TEST-CLUSTER}",
-          "description": "The time consumed by requests that TiKV sends to PD",
+          "description": "Current pending and running tasks of future_pool",
           "editable": true,
           "error": false,
           "fieldConfig": {
@@ -53923,11 +54147,11 @@
             "h": 7,
             "w": 12,
             "x": 12,
-            "y": 0
+            "y": 7
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 381,
+          "id": 382,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -53970,15 +54194,15 @@
           "targets": [
             {
               "datasource": "${DS_TEST-CLUSTER}",
-              "expr": "(sum(rate(\n    tikv_pd_request_duration_seconds_sum\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    [$__rate_interval]\n)) by (type, $additional_groupby)  / sum(rate(\n    tikv_pd_request_duration_seconds_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    [$__rate_interval]\n)) by (type, $additional_groupby) )",
+              "expr": "sum(avg_over_time(\n    tikv_futurepool_pending_task_total\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    [1m]\n)) by (name, $additional_groupby) ",
               "format": "time_series",
               "hide": false,
               "instant": false,
               "interval": "",
               "intervalFactor": 1,
-              "legendFormat": "{{type}} {{$additional_groupby}}",
+              "legendFormat": "{{name}} {{$additional_groupby}}",
               "metric": "",
-              "query": "(sum(rate(\n    tikv_pd_request_duration_seconds_sum\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    [$__rate_interval]\n)) by (type, $additional_groupby)  / sum(rate(\n    tikv_pd_request_duration_seconds_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    [$__rate_interval]\n)) by (type, $additional_groupby) )",
+              "query": "sum(avg_over_time(\n    tikv_futurepool_pending_task_total\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    [1m]\n)) by (name, $additional_groupby) ",
               "refId": "",
               "step": 10,
               "target": ""
@@ -53987,7 +54211,7 @@
           "thresholds": [],
           "timeFrom": null,
           "timeShift": null,
-          "title": "PD request duration (average)",
+          "title": "FuturePool pending tasks",
           "tooltip": {
             "msResolution": true,
             "shared": true,
@@ -54006,7 +54230,7 @@
           "yaxes": [
             {
               "decimals": null,
-              "format": "s",
+              "format": "none",
               "label": null,
               "logBase": 1,
               "max": null,
@@ -54027,13 +54251,55 @@
             "align": false,
             "alignLevel": 0
           }
-        },
+        }
+      ],
+      "repeat": null,
+      "repeatDirection": null,
+      "span": null,
+      "targets": [],
+      "timeFrom": null,
+      "timeShift": null,
+      "title": "Task",
+      "transformations": [],
+      "transparent": false,
+      "type": "row"
+    },
+    {
+      "cacheTimeout": null,
+      "collapsed": true,
+      "datasource": null,
+      "description": null,
+      "editable": true,
+      "error": false,
+      "fieldConfig": {
+        "defaults": {
+          "thresholds": {
+            "mode": "absolute",
+            "steps": []
+          }
+        }
+      },
+      "gridPos": {
+        "h": 7,
+        "w": 24,
+        "x": 0,
+        "y": 0
+      },
+      "height": null,
+      "hideTimeOverride": false,
+      "id": 383,
+      "interval": null,
+      "links": [],
+      "maxDataPoints": 100,
+      "maxPerRow": null,
+      "minSpan": null,
+      "panels": [
         {
           "aliasColors": {},
           "bars": false,
           "cacheTimeout": null,
           "datasource": "${DS_TEST-CLUSTER}",
-          "description": "The total number of PD heartbeat messages",
+          "description": "The count of requests that TiKV sends to PD",
           "editable": true,
           "error": false,
           "fieldConfig": {
@@ -54056,11 +54322,277 @@
             "h": 7,
             "w": 12,
             "x": 0,
-            "y": 7
+            "y": 0
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 382,
+          "id": 384,
+          "interval": null,
+          "isNew": true,
+          "legend": {
+            "alignAsTable": true,
+            "avg": false,
+            "current": true,
+            "hideEmpty": true,
+            "hideZero": true,
+            "max": true,
+            "min": false,
+            "rightSide": true,
+            "show": true,
+            "sideWidth": null,
+            "sort": "max",
+            "sortDesc": true,
+            "total": false,
+            "values": true
+          },
+          "lines": true,
+          "linewidth": 1,
+          "links": [],
+          "maxDataPoints": null,
+          "maxPerRow": null,
+          "minSpan": null,
+          "nullPointMode": "null as zero",
+          "options": {
+            "alertThreshold": true,
+            "dataLinks": []
+          },
+          "percentage": false,
+          "pointradius": 5,
+          "points": false,
+          "renderer": "flot",
+          "repeat": null,
+          "repeatDirection": null,
+          "seriesOverrides": [],
+          "span": null,
+          "stack": false,
+          "steppedLine": false,
+          "targets": [
+            {
+              "datasource": "${DS_TEST-CLUSTER}",
+              "expr": "sum(rate(\n    tikv_pd_request_duration_seconds_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    [$__rate_interval]\n)) by (type, $additional_groupby) ",
+              "format": "time_series",
+              "hide": false,
+              "instant": false,
+              "interval": "",
+              "intervalFactor": 1,
+              "legendFormat": "{{type}} {{$additional_groupby}}",
+              "metric": "",
+              "query": "sum(rate(\n    tikv_pd_request_duration_seconds_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    [$__rate_interval]\n)) by (type, $additional_groupby) ",
+              "refId": "",
+              "step": 10,
+              "target": ""
+            }
+          ],
+          "thresholds": [],
+          "timeFrom": null,
+          "timeShift": null,
+          "title": "PD requests",
+          "tooltip": {
+            "msResolution": true,
+            "shared": true,
+            "sort": 0,
+            "value_type": "individual"
+          },
+          "transformations": [],
+          "transparent": false,
+          "type": "graph",
+          "xaxis": {
+            "mode": "time",
+            "name": null,
+            "show": true,
+            "values": []
+          },
+          "yaxes": [
+            {
+              "decimals": null,
+              "format": "ops",
+              "label": null,
+              "logBase": 1,
+              "max": null,
+              "min": null,
+              "show": true
+            },
+            {
+              "decimals": null,
+              "format": "short",
+              "label": null,
+              "logBase": 1,
+              "max": null,
+              "min": null,
+              "show": true
+            }
+          ],
+          "yaxis": {
+            "align": false,
+            "alignLevel": 0
+          }
+        },
+        {
+          "aliasColors": {},
+          "bars": false,
+          "cacheTimeout": null,
+          "datasource": "${DS_TEST-CLUSTER}",
+          "description": "The time consumed by requests that TiKV sends to PD",
+          "editable": true,
+          "error": false,
+          "fieldConfig": {
+            "defaults": {
+              "thresholds": {
+                "mode": "absolute",
+                "steps": []
+              }
+            }
+          },
+          "fill": 1,
+          "fillGradient": 1,
+          "grid": {
+            "threshold1": null,
+            "threshold1Color": "rgba(216, 200, 27, 0.27)",
+            "threshold2": null,
+            "threshold2Color": "rgba(234, 112, 112, 0.22)"
+          },
+          "gridPos": {
+            "h": 7,
+            "w": 12,
+            "x": 12,
+            "y": 0
+          },
+          "height": null,
+          "hideTimeOverride": false,
+          "id": 385,
+          "interval": null,
+          "isNew": true,
+          "legend": {
+            "alignAsTable": true,
+            "avg": false,
+            "current": true,
+            "hideEmpty": true,
+            "hideZero": true,
+            "max": true,
+            "min": false,
+            "rightSide": true,
+            "show": true,
+            "sideWidth": null,
+            "sort": "max",
+            "sortDesc": true,
+            "total": false,
+            "values": true
+          },
+          "lines": true,
+          "linewidth": 1,
+          "links": [],
+          "maxDataPoints": null,
+          "maxPerRow": null,
+          "minSpan": null,
+          "nullPointMode": "null as zero",
+          "options": {
+            "alertThreshold": true,
+            "dataLinks": []
+          },
+          "percentage": false,
+          "pointradius": 5,
+          "points": false,
+          "renderer": "flot",
+          "repeat": null,
+          "repeatDirection": null,
+          "seriesOverrides": [],
+          "span": null,
+          "stack": false,
+          "steppedLine": false,
+          "targets": [
+            {
+              "datasource": "${DS_TEST-CLUSTER}",
+              "expr": "(sum(rate(\n    tikv_pd_request_duration_seconds_sum\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    [$__rate_interval]\n)) by (type, $additional_groupby)  / sum(rate(\n    tikv_pd_request_duration_seconds_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    [$__rate_interval]\n)) by (type, $additional_groupby) )",
+              "format": "time_series",
+              "hide": false,
+              "instant": false,
+              "interval": "",
+              "intervalFactor": 1,
+              "legendFormat": "{{type}} {{$additional_groupby}}",
+              "metric": "",
+              "query": "(sum(rate(\n    tikv_pd_request_duration_seconds_sum\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    [$__rate_interval]\n)) by (type, $additional_groupby)  / sum(rate(\n    tikv_pd_request_duration_seconds_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    [$__rate_interval]\n)) by (type, $additional_groupby) )",
+              "refId": "",
+              "step": 10,
+              "target": ""
+            }
+          ],
+          "thresholds": [],
+          "timeFrom": null,
+          "timeShift": null,
+          "title": "PD request duration (average)",
+          "tooltip": {
+            "msResolution": true,
+            "shared": true,
+            "sort": 0,
+            "value_type": "individual"
+          },
+          "transformations": [],
+          "transparent": false,
+          "type": "graph",
+          "xaxis": {
+            "mode": "time",
+            "name": null,
+            "show": true,
+            "values": []
+          },
+          "yaxes": [
+            {
+              "decimals": null,
+              "format": "s",
+              "label": null,
+              "logBase": 1,
+              "max": null,
+              "min": null,
+              "show": true
+            },
+            {
+              "decimals": null,
+              "format": "short",
+              "label": null,
+              "logBase": 1,
+              "max": null,
+              "min": null,
+              "show": true
+            }
+          ],
+          "yaxis": {
+            "align": false,
+            "alignLevel": 0
+          }
+        },
+        {
+          "aliasColors": {},
+          "bars": false,
+          "cacheTimeout": null,
+          "datasource": "${DS_TEST-CLUSTER}",
+          "description": "The total number of PD heartbeat messages",
+          "editable": true,
+          "error": false,
+          "fieldConfig": {
+            "defaults": {
+              "thresholds": {
+                "mode": "absolute",
+                "steps": []
+              }
+            }
+          },
+          "fill": 1,
+          "fillGradient": 1,
+          "grid": {
+            "threshold1": null,
+            "threshold1Color": "rgba(216, 200, 27, 0.27)",
+            "threshold2": null,
+            "threshold2Color": "rgba(234, 112, 112, 0.22)"
+          },
+          "gridPos": {
+            "h": 7,
+            "w": 12,
+            "x": 0,
+            "y": 7
+          },
+          "height": null,
+          "hideTimeOverride": false,
+          "id": 386,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -54208,7 +54740,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 383,
+          "id": 387,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -54341,7 +54873,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 384,
+          "id": 388,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -54474,7 +55006,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 385,
+          "id": 389,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -54607,7 +55139,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 386,
+          "id": 390,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -54743,7 +55275,7 @@
       },
       "height": null,
       "hideTimeOverride": false,
-      "id": 387,
+      "id": 391,
       "interval": null,
       "links": [],
       "maxDataPoints": 100,
@@ -54782,7 +55314,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 388,
+          "id": 392,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -54915,7 +55447,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 389,
+          "id": 393,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -55048,7 +55580,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 390,
+          "id": 394,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -55181,7 +55713,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 391,
+          "id": 395,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -55314,7 +55846,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 392,
+          "id": 396,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -55447,7 +55979,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 393,
+          "id": 397,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -55583,7 +56115,7 @@
       },
       "height": null,
       "hideTimeOverride": false,
-      "id": 394,
+      "id": 398,
       "interval": null,
       "links": [],
       "maxDataPoints": 100,
@@ -55622,7 +56154,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 395,
+          "id": 399,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -55755,7 +56287,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 396,
+          "id": 400,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -55888,7 +56420,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 397,
+          "id": 401,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -56036,7 +56568,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 398,
+          "id": 402,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -56199,7 +56731,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 399,
+          "id": 403,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -56332,7 +56864,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 400,
+          "id": 404,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -56465,7 +56997,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 401,
+          "id": 405,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -56613,7 +57145,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 402,
+          "id": 406,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -56761,7 +57293,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 403,
+          "id": 407,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -56897,7 +57429,7 @@
       },
       "height": null,
       "hideTimeOverride": false,
-      "id": 404,
+      "id": 408,
       "interval": null,
       "links": [],
       "maxDataPoints": 100,
@@ -56936,7 +57468,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 405,
+          "id": 409,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -57069,7 +57601,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 406,
+          "id": 410,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -57202,7 +57734,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 407,
+          "id": 411,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -57335,7 +57867,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 408,
+          "id": 412,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -57468,7 +58000,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 409,
+          "id": 413,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -57601,7 +58133,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 410,
+          "id": 414,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -57734,7 +58266,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 411,
+          "id": 415,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -57867,7 +58399,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 412,
+          "id": 416,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -58000,7 +58532,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 413,
+          "id": 417,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -58140,7 +58672,7 @@
           "hideTimeOverride": false,
           "hideZeroBuckets": true,
           "highlightCards": true,
-          "id": 414,
+          "id": 418,
           "interval": null,
           "legend": {
             "show": false
@@ -58238,7 +58770,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 415,
+          "id": 419,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -58371,7 +58903,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 416,
+          "id": 420,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -58519,7 +59051,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 417,
+          "id": 421,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -58667,7 +59199,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 418,
+          "id": 422,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -58807,7 +59339,7 @@
           "hideTimeOverride": false,
           "hideZeroBuckets": true,
           "highlightCards": true,
-          "id": 419,
+          "id": 423,
           "interval": null,
           "legend": {
             "show": false
@@ -58905,7 +59437,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 420,
+          "id": 424,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -59038,7 +59570,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 421,
+          "id": 425,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -59174,7 +59706,7 @@
       },
       "height": null,
       "hideTimeOverride": false,
-      "id": 422,
+      "id": 426,
       "interval": null,
       "links": [],
       "maxDataPoints": 100,
@@ -59213,7 +59745,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 423,
+          "id": 427,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -59346,7 +59878,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 424,
+          "id": 428,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -59509,7 +60041,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 425,
+          "id": 429,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -59657,7 +60189,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 426,
+          "id": 430,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -59790,7 +60322,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 427,
+          "id": 431,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -59930,7 +60462,7 @@
           "hideTimeOverride": false,
           "hideZeroBuckets": true,
           "highlightCards": true,
-          "id": 428,
+          "id": 432,
           "interval": null,
           "legend": {
             "show": false
@@ -60035,7 +60567,7 @@
           "hideTimeOverride": false,
           "hideZeroBuckets": true,
           "highlightCards": true,
-          "id": 429,
+          "id": 433,
           "interval": null,
           "legend": {
             "show": false
@@ -60140,7 +60672,7 @@
           "hideTimeOverride": false,
           "hideZeroBuckets": true,
           "highlightCards": true,
-          "id": 430,
+          "id": 434,
           "interval": null,
           "legend": {
             "show": false
@@ -60238,7 +60770,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 431,
+          "id": 435,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -60378,7 +60910,7 @@
           "hideTimeOverride": false,
           "hideZeroBuckets": true,
           "highlightCards": true,
-          "id": 432,
+          "id": 436,
           "interval": null,
           "legend": {
             "show": false
@@ -60483,7 +61015,7 @@
           "hideTimeOverride": false,
           "hideZeroBuckets": true,
           "highlightCards": true,
-          "id": 433,
+          "id": 437,
           "interval": null,
           "legend": {
             "show": false
@@ -60588,7 +61120,7 @@
           "hideTimeOverride": false,
           "hideZeroBuckets": true,
           "highlightCards": true,
-          "id": 434,
+          "id": 438,
           "interval": null,
           "legend": {
             "show": false
@@ -60686,7 +61218,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 435,
+          "id": 439,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -60819,7 +61351,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 436,
+          "id": 440,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -60952,7 +61484,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 437,
+          "id": 441,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -61092,7 +61624,7 @@
           "hideTimeOverride": false,
           "hideZeroBuckets": true,
           "highlightCards": true,
-          "id": 438,
+          "id": 442,
           "interval": null,
           "legend": {
             "show": false
@@ -61190,7 +61722,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 439,
+          "id": 443,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -61326,7 +61858,7 @@
       },
       "height": null,
       "hideTimeOverride": false,
-      "id": 440,
+      "id": 444,
       "interval": null,
       "links": [],
       "maxDataPoints": 100,
@@ -61365,7 +61897,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 441,
+          "id": 445,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -61528,7 +62060,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 442,
+          "id": 446,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -61661,7 +62193,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 443,
+          "id": 447,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -61801,7 +62333,7 @@
           "hideTimeOverride": false,
           "hideZeroBuckets": true,
           "highlightCards": true,
-          "id": 444,
+          "id": 448,
           "interval": null,
           "legend": {
             "show": false
@@ -61906,7 +62438,7 @@
           "hideTimeOverride": false,
           "hideZeroBuckets": true,
           "highlightCards": true,
-          "id": 445,
+          "id": 449,
           "interval": null,
           "legend": {
             "show": false
@@ -62004,7 +62536,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 446,
+          "id": 450,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -62159,7 +62691,7 @@
           "hideTimeOverride": false,
           "hideZeroBuckets": true,
           "highlightCards": true,
-          "id": 447,
+          "id": 451,
           "interval": null,
           "legend": {
             "show": false
@@ -62264,7 +62796,7 @@
           "hideTimeOverride": false,
           "hideZeroBuckets": true,
           "highlightCards": true,
-          "id": 448,
+          "id": 452,
           "interval": null,
           "legend": {
             "show": false
@@ -62369,7 +62901,7 @@
           "hideTimeOverride": false,
           "hideZeroBuckets": true,
           "highlightCards": true,
-          "id": 449,
+          "id": 453,
           "interval": null,
           "legend": {
             "show": false
@@ -62467,7 +62999,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 450,
+          "id": 454,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -62637,7 +63169,7 @@
           "hideTimeOverride": false,
           "hideZeroBuckets": true,
           "highlightCards": true,
-          "id": 451,
+          "id": 455,
           "interval": null,
           "legend": {
             "show": false
@@ -62735,7 +63267,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 452,
+          "id": 456,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -62936,7 +63468,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 453,
+          "id": 457,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -63137,7 +63669,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 454,
+          "id": 458,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -63270,7 +63802,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 455,
+          "id": 459,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -63433,7 +63965,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 456,
+          "id": 460,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -63566,7 +64098,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 457,
+          "id": 461,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -63699,7 +64231,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 458,
+          "id": 462,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -63900,7 +64432,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 459,
+          "id": 463,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -64033,7 +64565,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 460,
+          "id": 464,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -64173,7 +64705,7 @@
           "hideTimeOverride": false,
           "hideZeroBuckets": true,
           "highlightCards": true,
-          "id": 461,
+          "id": 465,
           "interval": null,
           "legend": {
             "show": false
@@ -64278,7 +64810,7 @@
           "hideTimeOverride": false,
           "hideZeroBuckets": true,
           "highlightCards": true,
-          "id": 462,
+          "id": 466,
           "interval": null,
           "legend": {
             "show": false
@@ -64383,7 +64915,7 @@
           "hideTimeOverride": false,
           "hideZeroBuckets": true,
           "highlightCards": true,
-          "id": 463,
+          "id": 467,
           "interval": null,
           "legend": {
             "show": false
@@ -64488,7 +65020,7 @@
           "hideTimeOverride": false,
           "hideZeroBuckets": true,
           "highlightCards": true,
-          "id": 464,
+          "id": 468,
           "interval": null,
           "legend": {
             "show": false
@@ -64593,7 +65125,7 @@
           "hideTimeOverride": false,
           "hideZeroBuckets": true,
           "highlightCards": true,
-          "id": 465,
+          "id": 469,
           "interval": null,
           "legend": {
             "show": false
@@ -64698,7 +65230,7 @@
           "hideTimeOverride": false,
           "hideZeroBuckets": true,
           "highlightCards": true,
-          "id": 466,
+          "id": 470,
           "interval": null,
           "legend": {
             "show": false
@@ -64803,7 +65335,7 @@
           "hideTimeOverride": false,
           "hideZeroBuckets": true,
           "highlightCards": true,
-          "id": 467,
+          "id": 471,
           "interval": null,
           "legend": {
             "show": false
@@ -64901,7 +65433,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 468,
+          "id": 472,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -65049,7 +65581,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 469,
+          "id": 473,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -65182,7 +65714,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 470,
+          "id": 474,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -65315,7 +65847,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 471,
+          "id": 475,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -65463,7 +65995,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 472,
+          "id": 476,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -65599,7 +66131,7 @@
       },
       "height": null,
       "hideTimeOverride": false,
-      "id": 473,
+      "id": 477,
       "interval": null,
       "links": [],
       "maxDataPoints": 100,
@@ -65650,7 +66182,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 474,
+          "id": 478,
           "interval": null,
           "links": [],
           "maxDataPoints": 100,
@@ -65746,7 +66278,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 475,
+          "id": 479,
           "interval": null,
           "links": [],
           "maxDataPoints": 100,
@@ -65821,7 +66353,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 476,
+          "id": 480,
           "interval": null,
           "links": [],
           "maxDataPoints": 100,
@@ -65896,7 +66428,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 477,
+          "id": 481,
           "interval": null,
           "links": [],
           "maxDataPoints": 100,
@@ -65971,7 +66503,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 478,
+          "id": 482,
           "interval": null,
           "links": [],
           "maxDataPoints": 100,
@@ -66046,7 +66578,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 479,
+          "id": 483,
           "interval": null,
           "links": [],
           "maxDataPoints": 100,
@@ -66121,7 +66653,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 480,
+          "id": 484,
           "interval": null,
           "links": [],
           "maxDataPoints": 100,
@@ -66196,7 +66728,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 481,
+          "id": 485,
           "interval": null,
           "links": [],
           "maxDataPoints": 100,
@@ -66275,7 +66807,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 482,
+          "id": 486,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -66408,7 +66940,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 483,
+          "id": 487,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -66541,7 +67073,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 484,
+          "id": 488,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -66674,7 +67206,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 485,
+          "id": 489,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -66807,7 +67339,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 486,
+          "id": 490,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -66940,7 +67472,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 487,
+          "id": 491,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -67088,7 +67620,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 488,
+          "id": 492,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -67221,7 +67753,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 489,
+          "id": 493,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -67354,7 +67886,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 490,
+          "id": 494,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -67520,7 +68052,7 @@
           "hideTimeOverride": false,
           "hideZeroBuckets": true,
           "highlightCards": true,
-          "id": 491,
+          "id": 495,
           "interval": null,
           "legend": {
             "show": false
@@ -67625,7 +68157,7 @@
           "hideTimeOverride": false,
           "hideZeroBuckets": true,
           "highlightCards": true,
-          "id": 492,
+          "id": 496,
           "interval": null,
           "legend": {
             "show": false
@@ -67730,7 +68262,7 @@
           "hideTimeOverride": false,
           "hideZeroBuckets": true,
           "highlightCards": true,
-          "id": 493,
+          "id": 497,
           "interval": null,
           "legend": {
             "show": false
@@ -67835,7 +68367,7 @@
           "hideTimeOverride": false,
           "hideZeroBuckets": true,
           "highlightCards": true,
-          "id": 494,
+          "id": 498,
           "interval": null,
           "legend": {
             "show": false
@@ -67940,7 +68472,7 @@
           "hideTimeOverride": false,
           "hideZeroBuckets": true,
           "highlightCards": true,
-          "id": 495,
+          "id": 499,
           "interval": null,
           "legend": {
             "show": false
@@ -68045,7 +68577,7 @@
           "hideTimeOverride": false,
           "hideZeroBuckets": true,
           "highlightCards": true,
-          "id": 496,
+          "id": 500,
           "interval": null,
           "legend": {
             "show": false
@@ -68150,7 +68682,7 @@
           "hideTimeOverride": false,
           "hideZeroBuckets": true,
           "highlightCards": true,
-          "id": 497,
+          "id": 501,
           "interval": null,
           "legend": {
             "show": false
@@ -68255,7 +68787,7 @@
           "hideTimeOverride": false,
           "hideZeroBuckets": true,
           "highlightCards": true,
-          "id": 498,
+          "id": 502,
           "interval": null,
           "legend": {
             "show": false
@@ -68353,7 +68885,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 499,
+          "id": 503,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -68486,7 +69018,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 500,
+          "id": 504,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -68619,7 +69151,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 501,
+          "id": 505,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -68752,7 +69284,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 502,
+          "id": 506,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -68885,7 +69417,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 503,
+          "id": 507,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -69018,7 +69550,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 504,
+          "id": 508,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -69151,7 +69683,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 505,
+          "id": 509,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -69284,7 +69816,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 506,
+          "id": 510,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -69424,7 +69956,7 @@
           "hideTimeOverride": false,
           "hideZeroBuckets": true,
           "highlightCards": true,
-          "id": 507,
+          "id": 511,
           "interval": null,
           "legend": {
             "show": false
@@ -69529,7 +70061,7 @@
           "hideTimeOverride": false,
           "hideZeroBuckets": true,
           "highlightCards": true,
-          "id": 508,
+          "id": 512,
           "interval": null,
           "legend": {
             "show": false
@@ -69627,7 +70159,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 509,
+          "id": 513,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -69760,7 +70292,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 510,
+          "id": 514,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -69893,7 +70425,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 511,
+          "id": 515,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -70026,7 +70558,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 512,
+          "id": 516,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -70159,7 +70691,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 513,
+          "id": 517,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -70292,7 +70824,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 514,
+          "id": 518,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -70428,7 +70960,7 @@
       },
       "height": null,
       "hideTimeOverride": false,
-      "id": 515,
+      "id": 519,
       "interval": null,
       "links": [],
       "maxDataPoints": 100,
@@ -70467,7 +70999,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 516,
+          "id": 520,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -70615,7 +71147,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 517,
+          "id": 521,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -70748,7 +71280,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 518,
+          "id": 522,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -70881,7 +71413,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 519,
+          "id": 523,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -71017,7 +71549,7 @@
       },
       "height": null,
       "hideTimeOverride": false,
-      "id": 520,
+      "id": 524,
       "interval": null,
       "links": [],
       "maxDataPoints": 100,
@@ -71056,7 +71588,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 521,
+          "id": 525,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -71189,7 +71721,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 522,
+          "id": 526,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -71322,7 +71854,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 523,
+          "id": 527,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -71455,7 +71987,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 524,
+          "id": 528,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -71588,7 +72120,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 525,
+          "id": 529,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -71721,7 +72253,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 526,
+          "id": 530,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -71857,7 +72389,7 @@
       },
       "height": null,
       "hideTimeOverride": false,
-      "id": 527,
+      "id": 531,
       "interval": null,
       "links": [],
       "maxDataPoints": 100,
@@ -71896,7 +72428,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 528,
+          "id": 532,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -72029,7 +72561,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 529,
+          "id": 533,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -72165,7 +72697,7 @@
       },
       "height": null,
       "hideTimeOverride": false,
-      "id": 530,
+      "id": 534,
       "interval": null,
       "links": [],
       "maxDataPoints": 100,
@@ -72204,7 +72736,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 531,
+          "id": 535,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -72405,7 +72937,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 532,
+          "id": 536,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -72541,7 +73073,7 @@
       },
       "height": null,
       "hideTimeOverride": false,
-      "id": 533,
+      "id": 537,
       "interval": null,
       "links": [],
       "maxDataPoints": 100,
@@ -72580,7 +73112,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 534,
+          "id": 538,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -72713,7 +73245,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 535,
+          "id": 539,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -72846,7 +73378,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 536,
+          "id": 540,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -72979,7 +73511,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 537,
+          "id": 541,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -73112,7 +73644,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 538,
+          "id": 542,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -73260,7 +73792,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 539,
+          "id": 543,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -73464,7 +73996,7 @@
       },
       "height": null,
       "hideTimeOverride": false,
-      "id": 540,
+      "id": 544,
       "interval": null,
       "links": [],
       "maxDataPoints": 100,
@@ -73503,7 +74035,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 541,
+          "id": 545,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -73636,7 +74168,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 542,
+          "id": 546,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -73769,7 +74301,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 543,
+          "id": 547,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -73902,7 +74434,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 544,
+          "id": 548,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -74035,7 +74567,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 545,
+          "id": 549,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -74232,7 +74764,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 546,
+          "id": 550,
           "interval": null,
           "links": [],
           "maxDataPoints": 100,
diff --git a/metrics/grafana/tikv_details.json.sha256 b/metrics/grafana/tikv_details.json.sha256
index 6407352f3ba..15e5d660270 100644
--- a/metrics/grafana/tikv_details.json.sha256
+++ b/metrics/grafana/tikv_details.json.sha256
@@ -1 +1 @@
-2167c0eda2495c7f8e3169596247e12503f2c2fa95cd539161909f03091c65bd  ./metrics/grafana/tikv_details.json
+d9f42f10c324e6378e1bf46aec7514321d487608a2c96ef5d385c9a0e36798e1  ./metrics/grafana/tikv_details.json

From 5a703ad8412d687e18efbe14c419967c323360a5 Mon Sep 17 00:00:00 2001
From: glorv <glorvs@163.com>
Date: Wed, 23 Oct 2024 13:51:44 +0800
Subject: [PATCH 04/16] in_memory_engine: add a debug API for IME to show
 current cached regions metadata (#17677)

ref tikv/tikv#16141

Add a "/cached_regions" http inteface to fetch current cached regions metadata.

Signed-off-by: glorv <glorvs@163.com>

Co-authored-by: ti-chi-bot[bot] <108142056+ti-chi-bot[bot]@users.noreply.github.com>
---
 .../in_memory_engine/src/region_manager.rs    |  8 +-
 components/server/src/server.rs               | 21 +++--
 components/server/src/server2.rs              |  1 +
 src/server/status_server/mod.rs               | 89 +++++++++++++++++++
 tests/integrations/server/status_server.rs    |  1 +
 5 files changed, 110 insertions(+), 10 deletions(-)

diff --git a/components/in_memory_engine/src/region_manager.rs b/components/in_memory_engine/src/region_manager.rs
index 2c11a8b398b..ad7278669d8 100644
--- a/components/in_memory_engine/src/region_manager.rs
+++ b/components/in_memory_engine/src/region_manager.rs
@@ -173,7 +173,7 @@ impl CacheRegionMeta {
         self.region.contains_range(region)
     }
 
-    pub(crate) fn safe_point(&self) -> u64 {
+    pub fn safe_point(&self) -> u64 {
         self.safe_point
     }
 
@@ -233,7 +233,7 @@ impl CacheRegionMeta {
         self.in_gc.store(in_gc, Ordering::Release);
     }
 
-    pub(crate) fn is_in_gc(&self) -> bool {
+    pub fn is_in_gc(&self) -> bool {
         self.in_gc.load(Ordering::Acquire)
     }
 
@@ -244,7 +244,7 @@ impl CacheRegionMeta {
     }
 
     #[inline]
-    pub(crate) fn is_written(&self) -> bool {
+    pub fn is_written(&self) -> bool {
         self.is_written.load(Ordering::Relaxed)
     }
 
@@ -473,7 +473,7 @@ impl RegionMetaMap {
         self.regions.get_mut(&id)
     }
 
-    pub(crate) fn regions(&self) -> &HashMap<u64, CacheRegionMeta> {
+    pub fn regions(&self) -> &HashMap<u64, CacheRegionMeta> {
         &self.regions
     }
 
diff --git a/components/server/src/server.rs b/components/server/src/server.rs
index 532debfab8b..19f47fa725f 100644
--- a/components/server/src/server.rs
+++ b/components/server/src/server.rs
@@ -47,6 +47,7 @@ use hybrid_engine::observer::{
 };
 use in_memory_engine::{
     config::InMemoryEngineConfigManager, InMemoryEngineContext, InMemoryEngineStatistics,
+    RegionCacheMemoryEngine,
 };
 use kvproto::{
     brpb::create_backup, cdcpb::create_change_data, deadlock::create_deadlock,
@@ -169,7 +170,7 @@ fn run_impl<CER, F>(
     tikv.core.init_encryption();
     let fetcher = tikv.core.init_io_utility();
     let listener = tikv.core.init_flow_receiver();
-    let (engines, engines_info) = tikv.init_raw_engines(listener);
+    let (engines, engines_info, in_memory_engine) = tikv.init_raw_engines(listener);
     tikv.init_engines(engines.clone());
     let server_config = tikv.init_servers();
     tikv.register_services();
@@ -177,7 +178,7 @@ fn run_impl<CER, F>(
     tikv.init_cgroup_monitor();
     tikv.init_storage_stats_task(engines);
     tikv.run_server(server_config);
-    tikv.run_status_server();
+    tikv.run_status_server(in_memory_engine);
     tikv.core.init_quota_tuning_task(tikv.quota_limiter.clone());
 
     // Build a background worker for handling signals.
@@ -1507,7 +1508,7 @@ where
             .unwrap_or_else(|e| fatal!("failed to start server: {}", e));
     }
 
-    fn run_status_server(&mut self) {
+    fn run_status_server(&mut self, in_memory_engine: Option<RegionCacheMemoryEngine>) {
         // Create a status server.
         let status_enabled = !self.core.config.server.status_addr.is_empty();
         if status_enabled {
@@ -1518,6 +1519,7 @@ where
                 self.engines.as_ref().unwrap().engine.raft_extension(),
                 self.resource_manager.clone(),
                 self.grpc_service_mgr.clone(),
+                in_memory_engine,
             ) {
                 Ok(status_server) => Box::new(status_server),
                 Err(e) => {
@@ -1594,7 +1596,11 @@ where
     fn init_raw_engines(
         &mut self,
         flow_listener: engine_rocks::FlowListener,
-    ) -> (Engines<RocksEngine, CER>, Arc<EnginesResourceInfo>) {
+    ) -> (
+        Engines<RocksEngine, CER>,
+        Arc<EnginesResourceInfo>,
+        Option<RegionCacheMemoryEngine>,
+    ) {
         let block_cache = self.core.config.storage.block_cache.build_shared_cache();
         let env = self
             .core
@@ -1657,7 +1663,7 @@ where
         let in_memory_engine_context =
             InMemoryEngineContext::new(in_memory_engine_config.clone(), self.pd_client.clone());
         let in_memory_engine_statistics = in_memory_engine_context.statistics();
-        if self.core.config.in_memory_engine.enable {
+        let ime_engine = if self.core.config.in_memory_engine.enable {
             let in_memory_engine = build_hybrid_engine(
                 in_memory_engine_context,
                 kv_engine.clone(),
@@ -1676,6 +1682,9 @@ where
             let snapshot_observer =
                 HybridSnapshotObserver::new(in_memory_engine.region_cache_engine().clone());
             snapshot_observer.register_to(self.coprocessor_host.as_mut().unwrap());
+            Some(in_memory_engine.region_cache_engine().clone())
+        } else {
+            None
         };
         let in_memory_engine_config_manager = InMemoryEngineConfigManager(in_memory_engine_config);
         self.kv_statistics = Some(factory.rocks_statistics());
@@ -1713,7 +1722,7 @@ where
             180, // max_samples_to_preserve
         ));
 
-        (engines, engines_info)
+        (engines, engines_info, ime_engine)
     }
 }
 
diff --git a/components/server/src/server2.rs b/components/server/src/server2.rs
index 0d8b26b9d80..74a9ffaffa9 100644
--- a/components/server/src/server2.rs
+++ b/components/server/src/server2.rs
@@ -1326,6 +1326,7 @@ where
                 self.engines.as_ref().unwrap().engine.raft_extension(),
                 self.resource_manager.clone(),
                 self.grpc_service_mgr.clone(),
+                None,
             ) {
                 Ok(status_server) => Box::new(status_server),
                 Err(e) => {
diff --git a/src/server/status_server/mod.rs b/src/server/status_server/mod.rs
index e87a214f2a6..5c20befb2fe 100644
--- a/src/server/status_server/mod.rs
+++ b/src/server/status_server/mod.rs
@@ -34,6 +34,7 @@ use hyper::{
     service::{make_service_fn, service_fn},
     Body, Method, Request, Response, Server, StatusCode,
 };
+use in_memory_engine::RegionCacheMemoryEngine;
 use kvproto::resource_manager::ResourceGroup;
 use metrics::STATUS_REQUEST_DURATION;
 use online_config::OnlineConfig;
@@ -95,6 +96,7 @@ pub struct StatusServer<R> {
     security_config: Arc<SecurityConfig>,
     resource_manager: Option<Arc<ResourceGroupManager>>,
     grpc_service_mgr: GrpcServiceManager,
+    in_memory_engine: Option<RegionCacheMemoryEngine>,
 }
 
 impl<R> StatusServer<R>
@@ -108,6 +110,7 @@ where
         router: R,
         resource_manager: Option<Arc<ResourceGroupManager>>,
         grpc_service_mgr: GrpcServiceManager,
+        in_memory_engine: Option<RegionCacheMemoryEngine>,
     ) -> Result<Self> {
         let thread_pool = Builder::new_multi_thread()
             .enable_all()
@@ -130,6 +133,7 @@ where
             security_config,
             resource_manager,
             grpc_service_mgr,
+            in_memory_engine,
         })
     }
 
@@ -613,6 +617,7 @@ where
         let router = self.router.clone();
         let resource_manager = self.resource_manager.clone();
         let grpc_service_mgr = self.grpc_service_mgr.clone();
+        let in_memory_engine = self.in_memory_engine.clone();
         // Start to serve.
         let server = builder.serve(make_service_fn(move |conn: &C| {
             let x509 = conn.get_x509();
@@ -620,6 +625,7 @@ where
             let cfg_controller = cfg_controller.clone();
             let router = router.clone();
             let resource_manager = resource_manager.clone();
+            let in_memory_engine = in_memory_engine.clone();
             let grpc_service_mgr = grpc_service_mgr.clone();
             async move {
                 // Create a status service.
@@ -630,6 +636,7 @@ where
                     let router = router.clone();
                     let resource_manager = resource_manager.clone();
                     let grpc_service_mgr = grpc_service_mgr.clone();
+                    let in_memory_engine = in_memory_engine.clone();
                     async move {
                         let path = req.uri().path().to_owned();
                         let method = req.method().to_owned();
@@ -734,6 +741,7 @@ where
                                 Self::handle_resume_grpc(grpc_service_mgr)
                             }
                             (Method::GET, "/async_tasks") => Self::dump_async_trace(),
+                            (Method::GET, "debug/ime/cached_regions") => Self::handle_dumple_cached_regions(in_memory_engine.as_ref()),
                             _ => {
                                 is_unknown_path = true;
                                 Ok(make_response(StatusCode::NOT_FOUND, "path not found"))
@@ -816,6 +824,73 @@ where
             )),
         }
     }
+
+    fn handle_dumple_cached_regions(
+        engine: Option<&RegionCacheMemoryEngine>,
+    ) -> hyper::Result<Response<Body>> {
+        // We use this function to workaround the false-positive check in
+        // `scripts/check-redact-log`.
+        fn to_hex_string(data: &[u8]) -> String {
+            hex::ToHex::encode_hex_upper(&data)
+        }
+
+        let Some(engine) = engine else {
+            return Ok(make_response(
+                StatusCode::BAD_REQUEST,
+                "In memory engine is not enabled",
+            ));
+        };
+        let body = {
+            let regions_map = engine.core().region_manager().regions_map().read();
+            let mut cached_regions = Vec::with_capacity(regions_map.regions().len());
+            for r in regions_map.regions().values() {
+                let rg_meta = CachedRegion {
+                    id: r.get_region().id,
+                    epoch_version: r.get_region().epoch_version,
+                    start: to_hex_string(keys::origin_key(&r.get_region().start)),
+                    end: to_hex_string(keys::origin_key(&r.get_region().end)),
+                    in_gc: r.is_in_gc(),
+                    safe_point: r.safe_point(),
+                    state: format!("{:?}", r.get_state()),
+                    is_written: r.is_written(),
+                };
+                cached_regions.push(rg_meta);
+            }
+            // order by region range.
+            cached_regions.sort();
+            match serde_json::to_vec(&cached_regions) {
+                Ok(body) => body,
+                Err(err) => {
+                    return Ok(make_response(
+                        StatusCode::INTERNAL_SERVER_ERROR,
+                        format!("fails to json: {}", err),
+                    ));
+                }
+            }
+        };
+        match Response::builder()
+            .header("content-type", "application/json")
+            .body(hyper::Body::from(body))
+        {
+            Ok(resp) => Ok(resp),
+            Err(err) => Ok(make_response(
+                StatusCode::INTERNAL_SERVER_ERROR,
+                format!("fails to build response: {}", err),
+            )),
+        }
+    }
+}
+
+#[derive(Serialize, Ord, PartialOrd, PartialEq, Eq)]
+struct CachedRegion {
+    start: String,
+    end: String,
+    id: u64,
+    epoch_version: u64,
+    in_gc: bool,
+    safe_point: u64,
+    state: String,
+    is_written: bool,
 }
 
 #[derive(Serialize)]
@@ -1170,6 +1245,7 @@ mod tests {
             MockRouter,
             None,
             GrpcServiceManager::dummy(),
+            None,
         )
         .unwrap();
         let addr = "127.0.0.1:0".to_owned();
@@ -1218,6 +1294,7 @@ mod tests {
             MockRouter,
             None,
             GrpcServiceManager::dummy(),
+            None,
         )
         .unwrap();
         let addr = "127.0.0.1:0".to_owned();
@@ -1270,6 +1347,7 @@ mod tests {
                 MockRouter,
                 None,
                 GrpcServiceManager::dummy(),
+                None,
             )
             .unwrap();
             let addr = "127.0.0.1:0".to_owned();
@@ -1332,6 +1410,7 @@ mod tests {
             MockRouter,
             None,
             GrpcServiceManager::dummy(),
+            None,
         )
         .unwrap();
         let addr = "127.0.0.1:0".to_owned();
@@ -1448,6 +1527,7 @@ mod tests {
             MockRouter,
             None,
             GrpcServiceManager::dummy(),
+            None,
         )
         .unwrap();
         let addr = "127.0.0.1:0".to_owned();
@@ -1492,6 +1572,7 @@ mod tests {
             MockRouter,
             None,
             GrpcServiceManager::dummy(),
+            None,
         )
         .unwrap();
         let addr = "127.0.0.1:0".to_owned();
@@ -1528,6 +1609,7 @@ mod tests {
             MockRouter,
             None,
             GrpcServiceManager::dummy(),
+            None,
         )
         .unwrap();
         let addr = "127.0.0.1:0".to_owned();
@@ -1600,6 +1682,7 @@ mod tests {
             MockRouter,
             None,
             GrpcServiceManager::dummy(),
+            None,
         )
         .unwrap();
         let addr = "127.0.0.1:0".to_owned();
@@ -1630,6 +1713,7 @@ mod tests {
             MockRouter,
             None,
             GrpcServiceManager::dummy(),
+            None,
         )
         .unwrap();
         let addr = "127.0.0.1:0".to_owned();
@@ -1663,6 +1747,7 @@ mod tests {
             MockRouter,
             None,
             GrpcServiceManager::dummy(),
+            None,
         )
         .unwrap();
         let addr = "127.0.0.1:0".to_owned();
@@ -1714,6 +1799,7 @@ mod tests {
             MockRouter,
             None,
             GrpcServiceManager::dummy(),
+            None,
         )
         .unwrap();
         let addr = "127.0.0.1:0".to_owned();
@@ -1769,6 +1855,7 @@ mod tests {
             MockRouter,
             None,
             GrpcServiceManager::dummy(),
+            None,
         )
         .unwrap();
         let addr = "127.0.0.1:0".to_owned();
@@ -1823,6 +1910,7 @@ mod tests {
                 MockRouter,
                 None,
                 GrpcServiceManager::dummy(),
+                None,
             )
             .unwrap();
             let addr = "127.0.0.1:0".to_owned();
@@ -1860,6 +1948,7 @@ mod tests {
                 MockRouter,
                 None,
                 GrpcServiceManager::dummy(),
+                None,
             )
             .unwrap();
             let addr = "127.0.0.1:0".to_owned();
diff --git a/tests/integrations/server/status_server.rs b/tests/integrations/server/status_server.rs
index 90d1122b13a..3673a8aa20b 100644
--- a/tests/integrations/server/status_server.rs
+++ b/tests/integrations/server/status_server.rs
@@ -47,6 +47,7 @@ fn test_region_meta_endpoint() {
         router,
         None,
         GrpcServiceManager::dummy(),
+        None,
     )
     .unwrap();
     let addr = format!("127.0.0.1:{}", test_util::alloc_port());

From 01e7d054b6f74b8ba6669a5ad116cef420e91dee Mon Sep 17 00:00:00 2001
From: Spade  A <71589810+SpadeA-Tang@users.noreply.github.com>
Date: Fri, 25 Oct 2024 13:45:35 +0800
Subject: [PATCH 05/16] In-memory engine: display region cache hit related
 metrics (#17697)

ref tikv/tikv#16141

display region cache hit related metrics

Signed-off-by: SpadeA-Tang <tangchenjie1210@gmail.com>

Co-authored-by: ti-chi-bot[bot] <108142056+ti-chi-bot[bot]@users.noreply.github.com>
---
 components/hybrid_engine/src/engine.rs        |  59 +-
 components/hybrid_engine/src/lib.rs           |   2 +-
 components/hybrid_engine/src/metrics.rs       |   4 +-
 .../hybrid_engine/src/observer/snapshot.rs    |  57 +-
 components/hybrid_engine/src/snapshot.rs      |   5 +-
 metrics/grafana/tikv_details.dashboard.py     |  54 +-
 metrics/grafana/tikv_details.json             | 747 +++++++++++-------
 metrics/grafana/tikv_details.json.sha256      |   2 +-
 8 files changed, 549 insertions(+), 381 deletions(-)

diff --git a/components/hybrid_engine/src/engine.rs b/components/hybrid_engine/src/engine.rs
index eb2f4afef74..39c8cadde17 100644
--- a/components/hybrid_engine/src/engine.rs
+++ b/components/hybrid_engine/src/engine.rs
@@ -1,18 +1,13 @@
 // Copyright 2023 TiKV Project Authors. Licensed under Apache-2.0.
 
 use engine_traits::{
-    CacheRegion, FailedReason, KvEngine, Mutable, Peekable, ReadOptions, RegionCacheEngine, Result,
-    SnapshotContext, SnapshotMiscExt, SyncMutable, WriteBatch, WriteBatchExt,
+    KvEngine, Mutable, Peekable, ReadOptions, RegionCacheEngine, Result, SnapshotContext,
+    SnapshotMiscExt, SyncMutable, WriteBatch, WriteBatchExt,
 };
 use keys::DATA_PREFIX_KEY;
 use kvproto::metapb::{self, RegionEpoch};
 
-use crate::{
-    metrics::{
-        IN_MEMORY_ENGINE_SNAPSHOT_ACQUIRE_FAILED_REASON_COUNT_STAIC, SNAPSHOT_TYPE_COUNT_STATIC,
-    },
-    snapshot::HybridEngineSnapshot,
-};
+use crate::snapshot::HybridEngineSnapshot;
 
 /// This engine is structured with both a disk engine and an region cache
 /// engine. The disk engine houses the complete database data, whereas the
@@ -52,41 +47,6 @@ where
     }
 }
 
-pub fn new_in_memory_snapshot<EC: RegionCacheEngine>(
-    region_cache_engine: &EC,
-    region: CacheRegion,
-    read_ts: u64,
-    sequence_number: u64,
-) -> Option<EC::Snapshot> {
-    match region_cache_engine.snapshot(region, read_ts, sequence_number) {
-        Ok(snap) => {
-            SNAPSHOT_TYPE_COUNT_STATIC.region_cache_engine.inc();
-            Some(snap)
-        }
-        Err(FailedReason::TooOldRead) => {
-            IN_MEMORY_ENGINE_SNAPSHOT_ACQUIRE_FAILED_REASON_COUNT_STAIC
-                .too_old_read
-                .inc();
-            SNAPSHOT_TYPE_COUNT_STATIC.rocksdb.inc();
-            None
-        }
-        Err(FailedReason::NotCached) => {
-            IN_MEMORY_ENGINE_SNAPSHOT_ACQUIRE_FAILED_REASON_COUNT_STAIC
-                .not_cached
-                .inc();
-            SNAPSHOT_TYPE_COUNT_STATIC.rocksdb.inc();
-            None
-        }
-        Err(FailedReason::EpochNotMatch) => {
-            IN_MEMORY_ENGINE_SNAPSHOT_ACQUIRE_FAILED_REASON_COUNT_STAIC
-                .epoch_not_match
-                .inc();
-            SNAPSHOT_TYPE_COUNT_STATIC.rocksdb.inc();
-            None
-        }
-    }
-}
-
 impl<EK, EC> HybridEngine<EK, EC>
 where
     EK: KvEngine,
@@ -104,12 +64,13 @@ where
         let region_cache_snap = if !self.region_cache_engine.enabled() {
             None
         } else if let Some(ctx) = ctx {
-            new_in_memory_snapshot(
-                &self.region_cache_engine,
-                ctx.region.unwrap(),
-                ctx.read_ts,
-                disk_snap.sequence_number(),
-            )
+            self.region_cache_engine
+                .snapshot(
+                    ctx.region.unwrap(),
+                    ctx.read_ts,
+                    disk_snap.sequence_number(),
+                )
+                .ok()
         } else {
             None
         };
diff --git a/components/hybrid_engine/src/lib.rs b/components/hybrid_engine/src/lib.rs
index ed2bf64857a..c2301f1c897 100644
--- a/components/hybrid_engine/src/lib.rs
+++ b/components/hybrid_engine/src/lib.rs
@@ -29,5 +29,5 @@ mod ttl_properties;
 pub mod util;
 mod write_batch;
 
-pub use engine::{new_in_memory_snapshot, HybridEngine};
+pub use engine::HybridEngine;
 pub use snapshot::HybridEngineSnapshot;
diff --git a/components/hybrid_engine/src/metrics.rs b/components/hybrid_engine/src/metrics.rs
index 7f23640fbd1..d961d3d6455 100644
--- a/components/hybrid_engine/src/metrics.rs
+++ b/components/hybrid_engine/src/metrics.rs
@@ -5,9 +5,11 @@ use prometheus::{register_int_counter_vec, IntCounterVec};
 use prometheus_static_metric::{auto_flush_from, make_auto_flush_static_metric};
 
 make_auto_flush_static_metric! {
+    // We may acquire ime snapshot even not in coprocessor request. We count it as wasted.
     pub label_enum SnapshotType {
         rocksdb,
-        region_cache_engine,
+        in_memory_engine,
+        wasted,
     }
 
     pub struct SnapshotTypeCountVec: LocalIntCounter {
diff --git a/components/hybrid_engine/src/observer/snapshot.rs b/components/hybrid_engine/src/observer/snapshot.rs
index e3e8f15e0bd..edf8d9729f9 100644
--- a/components/hybrid_engine/src/observer/snapshot.rs
+++ b/components/hybrid_engine/src/observer/snapshot.rs
@@ -1,20 +1,69 @@
 // Copyright 2024 TiKV Project Authors. Licensed under Apache-2.0.
 
-use engine_traits::{CacheRegion, KvEngine};
+use std::result;
+
+use engine_traits::{CacheRegion, FailedReason, KvEngine, RegionCacheEngine};
 use in_memory_engine::{RegionCacheMemoryEngine, RegionCacheSnapshot};
 use kvproto::metapb::Region;
 use raftstore::coprocessor::{
     dispatcher::BoxSnapshotObserver, CoprocessorHost, ObservedSnapshot, SnapshotObserver,
 };
 
-use crate::new_in_memory_snapshot;
+use crate::metrics::{
+    IN_MEMORY_ENGINE_SNAPSHOT_ACQUIRE_FAILED_REASON_COUNT_STAIC, SNAPSHOT_TYPE_COUNT_STATIC,
+};
 
 /// RegionCacheSnapshotPin pins data of a RegionCacheMemoryEngine during taking
 /// snapshot. It prevents the data from being evicted or deleted from the cache.
 // TODO: Remove it, theoretically it can be remove if we don't need an
 // in-memory engine snapshot when a region is removed or splitted.
 pub struct RegionCacheSnapshotPin {
-    pub snap: Option<RegionCacheSnapshot>,
+    pub snap: Option<result::Result<RegionCacheSnapshot, FailedReason>>,
+}
+
+impl Drop for RegionCacheSnapshotPin {
+    fn drop(&mut self) {
+        if matches!(self.snap, Some(Ok(_))) {
+            // ime snapshot is acquired successfully but not used in coprocessor request.
+            SNAPSHOT_TYPE_COUNT_STATIC.wasted.inc();
+        }
+    }
+}
+
+impl RegionCacheSnapshotPin {
+    pub fn take(&mut self) -> Option<RegionCacheSnapshot> {
+        if let Some(snap_result) = self.snap.take() {
+            match snap_result {
+                Ok(snap) => {
+                    SNAPSHOT_TYPE_COUNT_STATIC.in_memory_engine.inc();
+                    Some(snap)
+                }
+                Err(FailedReason::TooOldRead) => {
+                    IN_MEMORY_ENGINE_SNAPSHOT_ACQUIRE_FAILED_REASON_COUNT_STAIC
+                        .too_old_read
+                        .inc();
+                    SNAPSHOT_TYPE_COUNT_STATIC.rocksdb.inc();
+                    None
+                }
+                Err(FailedReason::NotCached) => {
+                    IN_MEMORY_ENGINE_SNAPSHOT_ACQUIRE_FAILED_REASON_COUNT_STAIC
+                        .not_cached
+                        .inc();
+                    SNAPSHOT_TYPE_COUNT_STATIC.rocksdb.inc();
+                    None
+                }
+                Err(FailedReason::EpochNotMatch) => {
+                    IN_MEMORY_ENGINE_SNAPSHOT_ACQUIRE_FAILED_REASON_COUNT_STAIC
+                        .epoch_not_match
+                        .inc();
+                    SNAPSHOT_TYPE_COUNT_STATIC.rocksdb.inc();
+                    None
+                }
+            }
+        } else {
+            None
+        }
+    }
 }
 
 impl ObservedSnapshot for RegionCacheSnapshotPin {}
@@ -47,7 +96,7 @@ impl SnapshotObserver for HybridSnapshotObserver {
         // data from being evicted or deleted from the cache.
         // The data should be released when the snapshot is dropped.
         let region = CacheRegion::from_region(region);
-        let snap = new_in_memory_snapshot(&self.cache_engine, region, read_ts, sequence_number);
+        let snap = Some(self.cache_engine.snapshot(region, read_ts, sequence_number));
         Box::new(RegionCacheSnapshotPin { snap })
     }
 }
diff --git a/components/hybrid_engine/src/snapshot.rs b/components/hybrid_engine/src/snapshot.rs
index 46032e561d2..48f16c8f16b 100644
--- a/components/hybrid_engine/src/snapshot.rs
+++ b/components/hybrid_engine/src/snapshot.rs
@@ -62,8 +62,9 @@ where
         let mut region_cache_snap = None;
         if let Some(snap_pin) = snap_pin {
             let snap_any: Box<dyn Any> = snap_pin;
-            let region_cache_snap_pin: Box<RegionCacheSnapshotPin> = snap_any.downcast().unwrap();
-            region_cache_snap = region_cache_snap_pin.snap;
+            let mut region_cache_snap_pin: Box<RegionCacheSnapshotPin> =
+                snap_any.downcast().unwrap();
+            region_cache_snap = region_cache_snap_pin.take();
         }
         HybridEngineSnapshot {
             disk_snap,
diff --git a/metrics/grafana/tikv_details.dashboard.py b/metrics/grafana/tikv_details.dashboard.py
index fca208a4f25..85c6f32f452 100644
--- a/metrics/grafana/tikv_details.dashboard.py
+++ b/metrics/grafana/tikv_details.dashboard.py
@@ -4375,43 +4375,52 @@ def InMemoryEngine() -> RowPanel:
     layout.row(
         [
             graph_panel(
-                title="Snapshot Type Count",
-                description="Count of each snapshot type",
+                title="Region Cache Hit",
+                description="Count of region cache hit",
                 targets=[
                     target(
                         expr=expr_sum_rate(
                             "tikv_snapshot_type_count",
-                            by_labels=["type"],
+                            label_selectors=['type="in_memory_engine"'],
+                            by_labels=["instance"],
                         ),
-                        legend_format="{{type}}",
-                        additional_groupby=True,
+                        legend_format="count-{{instance}}",
                     ),
                 ],
             ),
             graph_panel(
-                title="Snapshot Failed Reason",
-                description="Reasons for why rance cache snapshot is not acquired",
+                title="Region Cache Hit Rate",
+                description="Region cache hit rate",
+                yaxes=yaxes(left_format=UNITS.PERCENT_UNIT),
                 targets=[
                     target(
-                        expr=expr_sum_rate(
-                            "tikv_in_memory_engine_snapshot_acquire_failed_reason_count",
-                            by_labels=["type"],
+                        expr=expr_operator(
+                            expr_sum_rate(
+                                "tikv_snapshot_type_count",
+                                label_selectors=['type="in_memory_engine"'],
+                                by_labels=["instance"],
+                            ),
+                            "/",
+                            expr_sum_rate(
+                                "tikv_snapshot_type_count",
+                                by_labels=["instance"],
+                            ),
                         ),
-                        legend_format="{{type}}",
+                        legend_format="rate-{{instance}}",
                         additional_groupby=True,
                     ),
                 ],
             ),
             graph_panel(
-                title="Region Count",
-                description="The count of different types of region",
+                title="Region Cache Miss Reason",
+                description="Reasons for region cache miss",
                 targets=[
                     target(
-                        expr=expr_avg(
-                            "tikv_in_memory_engine_cache_count",
+                        expr=expr_sum_rate(
+                            "tikv_in_memory_engine_snapshot_acquire_failed_reason_count",
                             by_labels=["instance", "type"],
                         ),
-                        legend_format="{{instance}}--{{type}}",
+                        legend_format="{{type}}-{{instance}}",
                     ),
                 ],
             ),
@@ -4432,6 +4441,19 @@ def InMemoryEngine() -> RowPanel:
                     ),
                 ],
             ),
+            graph_panel(
+                title="Region Count",
+                description="The count of different types of region",
+                targets=[
+                    target(
+                        expr=expr_avg(
+                            "tikv_in_memory_engine_cache_count",
+                            by_labels=["instance", "type"],
+                        ),
+                        legend_format="{{instance}}--{{type}}",
+                    ),
+                ],
+            ),
             graph_panel(
                 title="GC Filter",
                 description="Rang cache engine garbage collection information",
diff --git a/metrics/grafana/tikv_details.json b/metrics/grafana/tikv_details.json
index 78b5c6493fc..9238472e7c1 100644
--- a/metrics/grafana/tikv_details.json
+++ b/metrics/grafana/tikv_details.json
@@ -37324,7 +37324,7 @@
           "bars": false,
           "cacheTimeout": null,
           "datasource": "${DS_TEST-CLUSTER}",
-          "description": "Count of each snapshot type",
+          "description": "Count of region cache hit",
           "editable": true,
           "error": false,
           "fieldConfig": {
@@ -37394,15 +37394,15 @@
           "targets": [
             {
               "datasource": "${DS_TEST-CLUSTER}",
-              "expr": "sum(rate(\n    tikv_snapshot_type_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    [$__rate_interval]\n)) by (type, $additional_groupby) ",
+              "expr": "sum(rate(\n    tikv_snapshot_type_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"in_memory_engine\"}\n    [$__rate_interval]\n)) by (instance) ",
               "format": "time_series",
               "hide": false,
               "instant": false,
               "interval": "",
               "intervalFactor": 1,
-              "legendFormat": "{{type}} {{$additional_groupby}}",
+              "legendFormat": "count-{{instance}}",
               "metric": "",
-              "query": "sum(rate(\n    tikv_snapshot_type_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    [$__rate_interval]\n)) by (type, $additional_groupby) ",
+              "query": "sum(rate(\n    tikv_snapshot_type_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"in_memory_engine\"}\n    [$__rate_interval]\n)) by (instance) ",
               "refId": "",
               "step": 10,
               "target": ""
@@ -37411,7 +37411,7 @@
           "thresholds": [],
           "timeFrom": null,
           "timeShift": null,
-          "title": "Snapshot Type Count",
+          "title": "Region Cache Hit",
           "tooltip": {
             "msResolution": true,
             "shared": true,
@@ -37457,7 +37457,7 @@
           "bars": false,
           "cacheTimeout": null,
           "datasource": "${DS_TEST-CLUSTER}",
-          "description": "Reasons for why rance cache snapshot is not acquired",
+          "description": "Region cache hit rate",
           "editable": true,
           "error": false,
           "fieldConfig": {
@@ -37527,15 +37527,15 @@
           "targets": [
             {
               "datasource": "${DS_TEST-CLUSTER}",
-              "expr": "sum(rate(\n    tikv_in_memory_engine_snapshot_acquire_failed_reason_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    [$__rate_interval]\n)) by (type, $additional_groupby) ",
+              "expr": "(sum(rate(\n    tikv_snapshot_type_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"in_memory_engine\"}\n    [$__rate_interval]\n)) by (instance, $additional_groupby)  / sum(rate(\n    tikv_snapshot_type_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    [$__rate_interval]\n)) by (instance, $additional_groupby) )",
               "format": "time_series",
               "hide": false,
               "instant": false,
               "interval": "",
               "intervalFactor": 1,
-              "legendFormat": "{{type}} {{$additional_groupby}}",
+              "legendFormat": "rate-{{instance}} {{$additional_groupby}}",
               "metric": "",
-              "query": "sum(rate(\n    tikv_in_memory_engine_snapshot_acquire_failed_reason_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    [$__rate_interval]\n)) by (type, $additional_groupby) ",
+              "query": "(sum(rate(\n    tikv_snapshot_type_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"in_memory_engine\"}\n    [$__rate_interval]\n)) by (instance, $additional_groupby)  / sum(rate(\n    tikv_snapshot_type_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    [$__rate_interval]\n)) by (instance, $additional_groupby) )",
               "refId": "",
               "step": 10,
               "target": ""
@@ -37544,7 +37544,7 @@
           "thresholds": [],
           "timeFrom": null,
           "timeShift": null,
-          "title": "Snapshot Failed Reason",
+          "title": "Region Cache Hit Rate",
           "tooltip": {
             "msResolution": true,
             "shared": true,
@@ -37563,7 +37563,7 @@
           "yaxes": [
             {
               "decimals": null,
-              "format": "none",
+              "format": "percentunit",
               "label": null,
               "logBase": 1,
               "max": null,
@@ -37590,7 +37590,7 @@
           "bars": false,
           "cacheTimeout": null,
           "datasource": "${DS_TEST-CLUSTER}",
-          "description": "The count of different types of region",
+          "description": "Reasons for region cache miss",
           "editable": true,
           "error": false,
           "fieldConfig": {
@@ -37660,15 +37660,15 @@
           "targets": [
             {
               "datasource": "${DS_TEST-CLUSTER}",
-              "expr": "avg((\n    tikv_in_memory_engine_cache_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    \n)) by (instance, type) ",
+              "expr": "sum(rate(\n    tikv_in_memory_engine_snapshot_acquire_failed_reason_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    [$__rate_interval]\n)) by (instance, type) ",
               "format": "time_series",
               "hide": false,
               "instant": false,
               "interval": "",
               "intervalFactor": 1,
-              "legendFormat": "{{instance}}--{{type}}",
+              "legendFormat": "{{type}}-{{instance}}",
               "metric": "",
-              "query": "avg((\n    tikv_in_memory_engine_cache_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    \n)) by (instance, type) ",
+              "query": "sum(rate(\n    tikv_in_memory_engine_snapshot_acquire_failed_reason_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    [$__rate_interval]\n)) by (instance, type) ",
               "refId": "",
               "step": 10,
               "target": ""
@@ -37677,7 +37677,7 @@
           "thresholds": [],
           "timeFrom": null,
           "timeShift": null,
-          "title": "Region Count",
+          "title": "Region Cache Miss Reason",
           "tooltip": {
             "msResolution": true,
             "shared": true,
@@ -37744,7 +37744,7 @@
           },
           "gridPos": {
             "h": 7,
-            "w": 8,
+            "w": 6,
             "x": 0,
             "y": 7
           },
@@ -37856,7 +37856,7 @@
           "bars": false,
           "cacheTimeout": null,
           "datasource": "${DS_TEST-CLUSTER}",
-          "description": "Rang cache engine garbage collection information",
+          "description": "The count of different types of region",
           "editable": true,
           "error": false,
           "fieldConfig": {
@@ -37877,8 +37877,8 @@
           },
           "gridPos": {
             "h": 7,
-            "w": 8,
-            "x": 8,
+            "w": 6,
+            "x": 6,
             "y": 7
           },
           "height": null,
@@ -37923,6 +37923,139 @@
           "span": null,
           "stack": false,
           "steppedLine": false,
+          "targets": [
+            {
+              "datasource": "${DS_TEST-CLUSTER}",
+              "expr": "avg((\n    tikv_in_memory_engine_cache_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    \n)) by (instance, type) ",
+              "format": "time_series",
+              "hide": false,
+              "instant": false,
+              "interval": "",
+              "intervalFactor": 1,
+              "legendFormat": "{{instance}}--{{type}}",
+              "metric": "",
+              "query": "avg((\n    tikv_in_memory_engine_cache_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    \n)) by (instance, type) ",
+              "refId": "",
+              "step": 10,
+              "target": ""
+            }
+          ],
+          "thresholds": [],
+          "timeFrom": null,
+          "timeShift": null,
+          "title": "Region Count",
+          "tooltip": {
+            "msResolution": true,
+            "shared": true,
+            "sort": 0,
+            "value_type": "individual"
+          },
+          "transformations": [],
+          "transparent": false,
+          "type": "graph",
+          "xaxis": {
+            "mode": "time",
+            "name": null,
+            "show": true,
+            "values": []
+          },
+          "yaxes": [
+            {
+              "decimals": null,
+              "format": "none",
+              "label": null,
+              "logBase": 1,
+              "max": null,
+              "min": null,
+              "show": true
+            },
+            {
+              "decimals": null,
+              "format": "short",
+              "label": null,
+              "logBase": 1,
+              "max": null,
+              "min": null,
+              "show": true
+            }
+          ],
+          "yaxis": {
+            "align": false,
+            "alignLevel": 0
+          }
+        },
+        {
+          "aliasColors": {},
+          "bars": false,
+          "cacheTimeout": null,
+          "datasource": "${DS_TEST-CLUSTER}",
+          "description": "Rang cache engine garbage collection information",
+          "editable": true,
+          "error": false,
+          "fieldConfig": {
+            "defaults": {
+              "thresholds": {
+                "mode": "absolute",
+                "steps": []
+              }
+            }
+          },
+          "fill": 1,
+          "fillGradient": 1,
+          "grid": {
+            "threshold1": null,
+            "threshold1Color": "rgba(216, 200, 27, 0.27)",
+            "threshold2": null,
+            "threshold2Color": "rgba(234, 112, 112, 0.22)"
+          },
+          "gridPos": {
+            "h": 7,
+            "w": 6,
+            "x": 12,
+            "y": 7
+          },
+          "height": null,
+          "hideTimeOverride": false,
+          "id": 266,
+          "interval": null,
+          "isNew": true,
+          "legend": {
+            "alignAsTable": true,
+            "avg": false,
+            "current": true,
+            "hideEmpty": true,
+            "hideZero": true,
+            "max": true,
+            "min": false,
+            "rightSide": true,
+            "show": true,
+            "sideWidth": null,
+            "sort": "max",
+            "sortDesc": true,
+            "total": false,
+            "values": true
+          },
+          "lines": true,
+          "linewidth": 1,
+          "links": [],
+          "maxDataPoints": null,
+          "maxPerRow": null,
+          "minSpan": null,
+          "nullPointMode": "null as zero",
+          "options": {
+            "alertThreshold": true,
+            "dataLinks": []
+          },
+          "percentage": false,
+          "pointradius": 5,
+          "points": false,
+          "renderer": "flot",
+          "repeat": null,
+          "repeatDirection": null,
+          "seriesOverrides": [],
+          "span": null,
+          "stack": false,
+          "steppedLine": false,
           "targets": [
             {
               "datasource": "${DS_TEST-CLUSTER}",
@@ -38014,8 +38147,8 @@
           },
           "gridPos": {
             "h": 7,
-            "w": 8,
-            "x": 16,
+            "w": 6,
+            "x": 18,
             "y": 7
           },
           "heatmap": {},
@@ -38023,7 +38156,7 @@
           "hideTimeOverride": false,
           "hideZeroBuckets": true,
           "highlightCards": true,
-          "id": 266,
+          "id": 267,
           "interval": null,
           "legend": {
             "show": false
@@ -38128,7 +38261,7 @@
           "hideTimeOverride": false,
           "hideZeroBuckets": true,
           "highlightCards": true,
-          "id": 267,
+          "id": 268,
           "interval": null,
           "legend": {
             "show": false
@@ -38226,7 +38359,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 268,
+          "id": 269,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -38366,7 +38499,7 @@
           "hideTimeOverride": false,
           "hideZeroBuckets": true,
           "highlightCards": true,
-          "id": 269,
+          "id": 270,
           "interval": null,
           "legend": {
             "show": false
@@ -38464,7 +38597,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 270,
+          "id": 271,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -38604,7 +38737,7 @@
           "hideTimeOverride": false,
           "hideZeroBuckets": true,
           "highlightCards": true,
-          "id": 271,
+          "id": 272,
           "interval": null,
           "legend": {
             "show": false
@@ -38702,7 +38835,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 272,
+          "id": 273,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -38910,7 +39043,7 @@
           "hideTimeOverride": false,
           "hideZeroBuckets": true,
           "highlightCards": true,
-          "id": 273,
+          "id": 274,
           "interval": null,
           "legend": {
             "show": false
@@ -39008,7 +39141,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 274,
+          "id": 275,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -39209,7 +39342,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 275,
+          "id": 276,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -39417,7 +39550,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 276,
+          "id": 277,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -39595,7 +39728,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 277,
+          "id": 278,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -39728,7 +39861,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 278,
+          "id": 279,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -39861,7 +39994,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 279,
+          "id": 280,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -39994,7 +40127,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 280,
+          "id": 281,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -40130,7 +40263,7 @@
       },
       "height": null,
       "hideTimeOverride": false,
-      "id": 281,
+      "id": 282,
       "interval": null,
       "links": [],
       "maxDataPoints": 100,
@@ -40169,7 +40302,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 282,
+          "id": 283,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -40317,7 +40450,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 283,
+          "id": 284,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -40457,7 +40590,7 @@
           "hideTimeOverride": false,
           "hideZeroBuckets": true,
           "highlightCards": true,
-          "id": 284,
+          "id": 285,
           "interval": null,
           "legend": {
             "show": false
@@ -40555,7 +40688,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 285,
+          "id": 286,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -40688,7 +40821,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 286,
+          "id": 287,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -40821,7 +40954,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 287,
+          "id": 288,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -40999,7 +41132,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 288,
+          "id": 289,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -41162,7 +41295,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 289,
+          "id": 290,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -41310,7 +41443,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 290,
+          "id": 291,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -41443,7 +41576,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 291,
+          "id": 292,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -41579,7 +41712,7 @@
       },
       "height": null,
       "hideTimeOverride": false,
-      "id": 292,
+      "id": 293,
       "interval": null,
       "links": [],
       "maxDataPoints": 100,
@@ -41618,7 +41751,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 293,
+          "id": 294,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -41766,7 +41899,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 294,
+          "id": 295,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -41899,7 +42032,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 295,
+          "id": 296,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -42032,7 +42165,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 296,
+          "id": 297,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -42165,7 +42298,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 297,
+          "id": 298,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -42298,7 +42431,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 298,
+          "id": 299,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -42453,7 +42586,7 @@
           "hideTimeOverride": false,
           "hideZeroBuckets": true,
           "highlightCards": true,
-          "id": 299,
+          "id": 300,
           "interval": null,
           "legend": {
             "show": false
@@ -42554,7 +42687,7 @@
       },
       "height": null,
       "hideTimeOverride": false,
-      "id": 300,
+      "id": 301,
       "interval": null,
       "links": [],
       "maxDataPoints": 100,
@@ -42593,7 +42726,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 301,
+          "id": 302,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -42726,7 +42859,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 302,
+          "id": 303,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -42859,7 +42992,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 303,
+          "id": 304,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -42999,7 +43132,7 @@
           "hideTimeOverride": false,
           "hideZeroBuckets": true,
           "highlightCards": true,
-          "id": 304,
+          "id": 305,
           "interval": null,
           "legend": {
             "show": false
@@ -43097,7 +43230,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 305,
+          "id": 306,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -43298,7 +43431,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 306,
+          "id": 307,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -43499,7 +43632,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 307,
+          "id": 308,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -43703,7 +43836,7 @@
       },
       "height": null,
       "hideTimeOverride": false,
-      "id": 308,
+      "id": 309,
       "interval": null,
       "links": [],
       "maxDataPoints": 100,
@@ -43742,7 +43875,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 309,
+          "id": 310,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -43890,7 +44023,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 310,
+          "id": 311,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -44091,7 +44224,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 311,
+          "id": 312,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -44292,7 +44425,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 312,
+          "id": 313,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -44493,7 +44626,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 313,
+          "id": 314,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -44694,7 +44827,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 314,
+          "id": 315,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -44827,7 +44960,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 315,
+          "id": 316,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -44960,7 +45093,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 316,
+          "id": 317,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -45093,7 +45226,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 317,
+          "id": 318,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -45226,7 +45359,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 318,
+          "id": 319,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -45434,7 +45567,7 @@
           "hideTimeOverride": false,
           "hideZeroBuckets": true,
           "highlightCards": true,
-          "id": 319,
+          "id": 320,
           "interval": null,
           "legend": {
             "show": false
@@ -45535,7 +45668,7 @@
       },
       "height": null,
       "hideTimeOverride": false,
-      "id": 320,
+      "id": 321,
       "interval": null,
       "links": [],
       "maxDataPoints": 100,
@@ -45581,7 +45714,7 @@
           "hideTimeOverride": false,
           "hideZeroBuckets": true,
           "highlightCards": true,
-          "id": 321,
+          "id": 322,
           "interval": null,
           "legend": {
             "show": false
@@ -45679,7 +45812,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 322,
+          "id": 323,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -45880,7 +46013,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 323,
+          "id": 324,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -46013,7 +46146,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 324,
+          "id": 325,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -46146,7 +46279,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 325,
+          "id": 326,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -46279,7 +46412,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 326,
+          "id": 327,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -46480,7 +46613,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 327,
+          "id": 328,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -46613,7 +46746,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 328,
+          "id": 329,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -46746,7 +46879,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 329,
+          "id": 330,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -46882,7 +47015,7 @@
       },
       "height": null,
       "hideTimeOverride": false,
-      "id": 330,
+      "id": 331,
       "interval": null,
       "links": [],
       "maxDataPoints": 100,
@@ -46921,7 +47054,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 331,
+          "id": 332,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -47122,7 +47255,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 332,
+          "id": 333,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -47323,7 +47456,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 333,
+          "id": 334,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -47524,7 +47657,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 334,
+          "id": 335,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -47725,7 +47858,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 335,
+          "id": 336,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -47858,7 +47991,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 336,
+          "id": 337,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -47991,7 +48124,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 337,
+          "id": 338,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -48124,7 +48257,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 338,
+          "id": 339,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -48257,7 +48390,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 339,
+          "id": 340,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -48390,7 +48523,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 340,
+          "id": 341,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -48530,7 +48663,7 @@
           "hideTimeOverride": false,
           "hideZeroBuckets": true,
           "highlightCards": true,
-          "id": 341,
+          "id": 342,
           "interval": null,
           "legend": {
             "show": false
@@ -48628,7 +48761,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 342,
+          "id": 343,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -48832,7 +48965,7 @@
       },
       "height": null,
       "hideTimeOverride": false,
-      "id": 343,
+      "id": 344,
       "interval": null,
       "links": [],
       "maxDataPoints": 100,
@@ -48871,7 +49004,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 344,
+          "id": 345,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -49004,7 +49137,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 345,
+          "id": 346,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -49137,7 +49270,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 346,
+          "id": 347,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -49277,7 +49410,7 @@
           "hideTimeOverride": false,
           "hideZeroBuckets": true,
           "highlightCards": true,
-          "id": 347,
+          "id": 348,
           "interval": null,
           "legend": {
             "show": false
@@ -49375,7 +49508,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 348,
+          "id": 349,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -49576,7 +49709,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 349,
+          "id": 350,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -49777,7 +49910,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 350,
+          "id": 351,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -49981,7 +50114,7 @@
       },
       "height": null,
       "hideTimeOverride": false,
-      "id": 351,
+      "id": 352,
       "interval": null,
       "links": [],
       "maxDataPoints": 100,
@@ -50020,7 +50153,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 352,
+          "id": 353,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -50198,7 +50331,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 353,
+          "id": 354,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -50399,7 +50532,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 354,
+          "id": 355,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -50532,7 +50665,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 355,
+          "id": 356,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -50665,7 +50798,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 356,
+          "id": 357,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -50798,7 +50931,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 357,
+          "id": 358,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -50931,7 +51064,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 358,
+          "id": 359,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -51064,7 +51197,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 359,
+          "id": 360,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -51193,7 +51326,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 360,
+          "id": 361,
           "interval": null,
           "links": [],
           "maxDataPoints": 100,
@@ -51268,7 +51401,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 361,
+          "id": 362,
           "interval": null,
           "links": [],
           "maxDataPoints": 100,
@@ -51347,7 +51480,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 362,
+          "id": 363,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -51600,7 +51733,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 363,
+          "id": 364,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -51733,7 +51866,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 364,
+          "id": 365,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -51869,7 +52002,7 @@
       },
       "height": null,
       "hideTimeOverride": false,
-      "id": 365,
+      "id": 366,
       "interval": null,
       "links": [],
       "maxDataPoints": 100,
@@ -51908,7 +52041,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 366,
+          "id": 367,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -52056,7 +52189,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 367,
+          "id": 368,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -52189,7 +52322,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 368,
+          "id": 369,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -52390,7 +52523,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 369,
+          "id": 370,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -52538,7 +52671,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 370,
+          "id": 371,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -52739,7 +52872,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 371,
+          "id": 372,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -52872,7 +53005,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 372,
+          "id": 373,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -53005,7 +53138,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 373,
+          "id": 374,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -53138,7 +53271,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 374,
+          "id": 375,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -53271,7 +53404,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 375,
+          "id": 376,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -53411,7 +53544,7 @@
           "hideTimeOverride": false,
           "hideZeroBuckets": true,
           "highlightCards": true,
-          "id": 376,
+          "id": 377,
           "interval": null,
           "legend": {
             "show": false
@@ -53509,7 +53642,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 377,
+          "id": 378,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -53713,7 +53846,7 @@
       },
       "height": null,
       "hideTimeOverride": false,
-      "id": 378,
+      "id": 379,
       "interval": null,
       "links": [],
       "maxDataPoints": 100,
@@ -53752,7 +53885,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 379,
+          "id": 380,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -53885,7 +54018,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 380,
+          "id": 381,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -54018,7 +54151,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 381,
+          "id": 382,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -54151,7 +54284,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 382,
+          "id": 383,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -54287,7 +54420,7 @@
       },
       "height": null,
       "hideTimeOverride": false,
-      "id": 383,
+      "id": 384,
       "interval": null,
       "links": [],
       "maxDataPoints": 100,
@@ -54326,7 +54459,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 384,
+          "id": 385,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -54459,7 +54592,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 385,
+          "id": 386,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -54592,7 +54725,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 386,
+          "id": 387,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -54740,7 +54873,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 387,
+          "id": 388,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -54873,7 +55006,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 388,
+          "id": 389,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -55006,7 +55139,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 389,
+          "id": 390,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -55139,7 +55272,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 390,
+          "id": 391,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -55275,7 +55408,7 @@
       },
       "height": null,
       "hideTimeOverride": false,
-      "id": 391,
+      "id": 392,
       "interval": null,
       "links": [],
       "maxDataPoints": 100,
@@ -55314,7 +55447,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 392,
+          "id": 393,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -55447,7 +55580,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 393,
+          "id": 394,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -55580,7 +55713,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 394,
+          "id": 395,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -55713,7 +55846,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 395,
+          "id": 396,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -55846,7 +55979,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 396,
+          "id": 397,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -55979,7 +56112,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 397,
+          "id": 398,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -56115,7 +56248,7 @@
       },
       "height": null,
       "hideTimeOverride": false,
-      "id": 398,
+      "id": 399,
       "interval": null,
       "links": [],
       "maxDataPoints": 100,
@@ -56154,7 +56287,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 399,
+          "id": 400,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -56287,7 +56420,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 400,
+          "id": 401,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -56420,7 +56553,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 401,
+          "id": 402,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -56568,7 +56701,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 402,
+          "id": 403,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -56731,7 +56864,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 403,
+          "id": 404,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -56864,7 +56997,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 404,
+          "id": 405,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -56997,7 +57130,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 405,
+          "id": 406,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -57145,7 +57278,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 406,
+          "id": 407,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -57293,7 +57426,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 407,
+          "id": 408,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -57429,7 +57562,7 @@
       },
       "height": null,
       "hideTimeOverride": false,
-      "id": 408,
+      "id": 409,
       "interval": null,
       "links": [],
       "maxDataPoints": 100,
@@ -57468,7 +57601,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 409,
+          "id": 410,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -57601,7 +57734,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 410,
+          "id": 411,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -57734,7 +57867,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 411,
+          "id": 412,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -57867,7 +58000,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 412,
+          "id": 413,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -58000,7 +58133,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 413,
+          "id": 414,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -58133,7 +58266,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 414,
+          "id": 415,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -58266,7 +58399,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 415,
+          "id": 416,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -58399,7 +58532,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 416,
+          "id": 417,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -58532,7 +58665,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 417,
+          "id": 418,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -58672,7 +58805,7 @@
           "hideTimeOverride": false,
           "hideZeroBuckets": true,
           "highlightCards": true,
-          "id": 418,
+          "id": 419,
           "interval": null,
           "legend": {
             "show": false
@@ -58770,7 +58903,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 419,
+          "id": 420,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -58903,7 +59036,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 420,
+          "id": 421,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -59051,7 +59184,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 421,
+          "id": 422,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -59199,7 +59332,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 422,
+          "id": 423,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -59339,7 +59472,7 @@
           "hideTimeOverride": false,
           "hideZeroBuckets": true,
           "highlightCards": true,
-          "id": 423,
+          "id": 424,
           "interval": null,
           "legend": {
             "show": false
@@ -59437,7 +59570,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 424,
+          "id": 425,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -59570,7 +59703,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 425,
+          "id": 426,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -59706,7 +59839,7 @@
       },
       "height": null,
       "hideTimeOverride": false,
-      "id": 426,
+      "id": 427,
       "interval": null,
       "links": [],
       "maxDataPoints": 100,
@@ -59745,7 +59878,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 427,
+          "id": 428,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -59878,7 +60011,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 428,
+          "id": 429,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -60041,7 +60174,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 429,
+          "id": 430,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -60189,7 +60322,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 430,
+          "id": 431,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -60322,7 +60455,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 431,
+          "id": 432,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -60462,7 +60595,7 @@
           "hideTimeOverride": false,
           "hideZeroBuckets": true,
           "highlightCards": true,
-          "id": 432,
+          "id": 433,
           "interval": null,
           "legend": {
             "show": false
@@ -60567,7 +60700,7 @@
           "hideTimeOverride": false,
           "hideZeroBuckets": true,
           "highlightCards": true,
-          "id": 433,
+          "id": 434,
           "interval": null,
           "legend": {
             "show": false
@@ -60672,7 +60805,7 @@
           "hideTimeOverride": false,
           "hideZeroBuckets": true,
           "highlightCards": true,
-          "id": 434,
+          "id": 435,
           "interval": null,
           "legend": {
             "show": false
@@ -60770,7 +60903,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 435,
+          "id": 436,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -60910,7 +61043,7 @@
           "hideTimeOverride": false,
           "hideZeroBuckets": true,
           "highlightCards": true,
-          "id": 436,
+          "id": 437,
           "interval": null,
           "legend": {
             "show": false
@@ -61015,7 +61148,7 @@
           "hideTimeOverride": false,
           "hideZeroBuckets": true,
           "highlightCards": true,
-          "id": 437,
+          "id": 438,
           "interval": null,
           "legend": {
             "show": false
@@ -61120,7 +61253,7 @@
           "hideTimeOverride": false,
           "hideZeroBuckets": true,
           "highlightCards": true,
-          "id": 438,
+          "id": 439,
           "interval": null,
           "legend": {
             "show": false
@@ -61218,7 +61351,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 439,
+          "id": 440,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -61351,7 +61484,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 440,
+          "id": 441,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -61484,7 +61617,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 441,
+          "id": 442,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -61624,7 +61757,7 @@
           "hideTimeOverride": false,
           "hideZeroBuckets": true,
           "highlightCards": true,
-          "id": 442,
+          "id": 443,
           "interval": null,
           "legend": {
             "show": false
@@ -61722,7 +61855,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 443,
+          "id": 444,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -61858,7 +61991,7 @@
       },
       "height": null,
       "hideTimeOverride": false,
-      "id": 444,
+      "id": 445,
       "interval": null,
       "links": [],
       "maxDataPoints": 100,
@@ -61897,7 +62030,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 445,
+          "id": 446,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -62060,7 +62193,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 446,
+          "id": 447,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -62193,7 +62326,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 447,
+          "id": 448,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -62333,7 +62466,7 @@
           "hideTimeOverride": false,
           "hideZeroBuckets": true,
           "highlightCards": true,
-          "id": 448,
+          "id": 449,
           "interval": null,
           "legend": {
             "show": false
@@ -62438,7 +62571,7 @@
           "hideTimeOverride": false,
           "hideZeroBuckets": true,
           "highlightCards": true,
-          "id": 449,
+          "id": 450,
           "interval": null,
           "legend": {
             "show": false
@@ -62536,7 +62669,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 450,
+          "id": 451,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -62691,7 +62824,7 @@
           "hideTimeOverride": false,
           "hideZeroBuckets": true,
           "highlightCards": true,
-          "id": 451,
+          "id": 452,
           "interval": null,
           "legend": {
             "show": false
@@ -62796,7 +62929,7 @@
           "hideTimeOverride": false,
           "hideZeroBuckets": true,
           "highlightCards": true,
-          "id": 452,
+          "id": 453,
           "interval": null,
           "legend": {
             "show": false
@@ -62901,7 +63034,7 @@
           "hideTimeOverride": false,
           "hideZeroBuckets": true,
           "highlightCards": true,
-          "id": 453,
+          "id": 454,
           "interval": null,
           "legend": {
             "show": false
@@ -62999,7 +63132,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 454,
+          "id": 455,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -63169,7 +63302,7 @@
           "hideTimeOverride": false,
           "hideZeroBuckets": true,
           "highlightCards": true,
-          "id": 455,
+          "id": 456,
           "interval": null,
           "legend": {
             "show": false
@@ -63267,7 +63400,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 456,
+          "id": 457,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -63468,7 +63601,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 457,
+          "id": 458,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -63669,7 +63802,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 458,
+          "id": 459,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -63802,7 +63935,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 459,
+          "id": 460,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -63965,7 +64098,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 460,
+          "id": 461,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -64098,7 +64231,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 461,
+          "id": 462,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -64231,7 +64364,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 462,
+          "id": 463,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -64432,7 +64565,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 463,
+          "id": 464,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -64565,7 +64698,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 464,
+          "id": 465,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -64705,7 +64838,7 @@
           "hideTimeOverride": false,
           "hideZeroBuckets": true,
           "highlightCards": true,
-          "id": 465,
+          "id": 466,
           "interval": null,
           "legend": {
             "show": false
@@ -64810,7 +64943,7 @@
           "hideTimeOverride": false,
           "hideZeroBuckets": true,
           "highlightCards": true,
-          "id": 466,
+          "id": 467,
           "interval": null,
           "legend": {
             "show": false
@@ -64915,7 +65048,7 @@
           "hideTimeOverride": false,
           "hideZeroBuckets": true,
           "highlightCards": true,
-          "id": 467,
+          "id": 468,
           "interval": null,
           "legend": {
             "show": false
@@ -65020,7 +65153,7 @@
           "hideTimeOverride": false,
           "hideZeroBuckets": true,
           "highlightCards": true,
-          "id": 468,
+          "id": 469,
           "interval": null,
           "legend": {
             "show": false
@@ -65125,7 +65258,7 @@
           "hideTimeOverride": false,
           "hideZeroBuckets": true,
           "highlightCards": true,
-          "id": 469,
+          "id": 470,
           "interval": null,
           "legend": {
             "show": false
@@ -65230,7 +65363,7 @@
           "hideTimeOverride": false,
           "hideZeroBuckets": true,
           "highlightCards": true,
-          "id": 470,
+          "id": 471,
           "interval": null,
           "legend": {
             "show": false
@@ -65335,7 +65468,7 @@
           "hideTimeOverride": false,
           "hideZeroBuckets": true,
           "highlightCards": true,
-          "id": 471,
+          "id": 472,
           "interval": null,
           "legend": {
             "show": false
@@ -65433,7 +65566,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 472,
+          "id": 473,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -65581,7 +65714,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 473,
+          "id": 474,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -65714,7 +65847,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 474,
+          "id": 475,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -65847,7 +65980,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 475,
+          "id": 476,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -65995,7 +66128,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 476,
+          "id": 477,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -66131,7 +66264,7 @@
       },
       "height": null,
       "hideTimeOverride": false,
-      "id": 477,
+      "id": 478,
       "interval": null,
       "links": [],
       "maxDataPoints": 100,
@@ -66182,7 +66315,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 478,
+          "id": 479,
           "interval": null,
           "links": [],
           "maxDataPoints": 100,
@@ -66278,7 +66411,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 479,
+          "id": 480,
           "interval": null,
           "links": [],
           "maxDataPoints": 100,
@@ -66353,7 +66486,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 480,
+          "id": 481,
           "interval": null,
           "links": [],
           "maxDataPoints": 100,
@@ -66428,7 +66561,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 481,
+          "id": 482,
           "interval": null,
           "links": [],
           "maxDataPoints": 100,
@@ -66503,7 +66636,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 482,
+          "id": 483,
           "interval": null,
           "links": [],
           "maxDataPoints": 100,
@@ -66578,7 +66711,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 483,
+          "id": 484,
           "interval": null,
           "links": [],
           "maxDataPoints": 100,
@@ -66653,7 +66786,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 484,
+          "id": 485,
           "interval": null,
           "links": [],
           "maxDataPoints": 100,
@@ -66728,7 +66861,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 485,
+          "id": 486,
           "interval": null,
           "links": [],
           "maxDataPoints": 100,
@@ -66807,7 +66940,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 486,
+          "id": 487,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -66940,7 +67073,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 487,
+          "id": 488,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -67073,7 +67206,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 488,
+          "id": 489,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -67206,7 +67339,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 489,
+          "id": 490,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -67339,7 +67472,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 490,
+          "id": 491,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -67472,7 +67605,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 491,
+          "id": 492,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -67620,7 +67753,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 492,
+          "id": 493,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -67753,7 +67886,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 493,
+          "id": 494,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -67886,7 +68019,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 494,
+          "id": 495,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -68052,7 +68185,7 @@
           "hideTimeOverride": false,
           "hideZeroBuckets": true,
           "highlightCards": true,
-          "id": 495,
+          "id": 496,
           "interval": null,
           "legend": {
             "show": false
@@ -68157,7 +68290,7 @@
           "hideTimeOverride": false,
           "hideZeroBuckets": true,
           "highlightCards": true,
-          "id": 496,
+          "id": 497,
           "interval": null,
           "legend": {
             "show": false
@@ -68262,7 +68395,7 @@
           "hideTimeOverride": false,
           "hideZeroBuckets": true,
           "highlightCards": true,
-          "id": 497,
+          "id": 498,
           "interval": null,
           "legend": {
             "show": false
@@ -68367,7 +68500,7 @@
           "hideTimeOverride": false,
           "hideZeroBuckets": true,
           "highlightCards": true,
-          "id": 498,
+          "id": 499,
           "interval": null,
           "legend": {
             "show": false
@@ -68472,7 +68605,7 @@
           "hideTimeOverride": false,
           "hideZeroBuckets": true,
           "highlightCards": true,
-          "id": 499,
+          "id": 500,
           "interval": null,
           "legend": {
             "show": false
@@ -68577,7 +68710,7 @@
           "hideTimeOverride": false,
           "hideZeroBuckets": true,
           "highlightCards": true,
-          "id": 500,
+          "id": 501,
           "interval": null,
           "legend": {
             "show": false
@@ -68682,7 +68815,7 @@
           "hideTimeOverride": false,
           "hideZeroBuckets": true,
           "highlightCards": true,
-          "id": 501,
+          "id": 502,
           "interval": null,
           "legend": {
             "show": false
@@ -68787,7 +68920,7 @@
           "hideTimeOverride": false,
           "hideZeroBuckets": true,
           "highlightCards": true,
-          "id": 502,
+          "id": 503,
           "interval": null,
           "legend": {
             "show": false
@@ -68885,7 +69018,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 503,
+          "id": 504,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -69018,7 +69151,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 504,
+          "id": 505,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -69151,7 +69284,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 505,
+          "id": 506,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -69284,7 +69417,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 506,
+          "id": 507,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -69417,7 +69550,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 507,
+          "id": 508,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -69550,7 +69683,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 508,
+          "id": 509,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -69683,7 +69816,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 509,
+          "id": 510,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -69816,7 +69949,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 510,
+          "id": 511,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -69956,7 +70089,7 @@
           "hideTimeOverride": false,
           "hideZeroBuckets": true,
           "highlightCards": true,
-          "id": 511,
+          "id": 512,
           "interval": null,
           "legend": {
             "show": false
@@ -70061,7 +70194,7 @@
           "hideTimeOverride": false,
           "hideZeroBuckets": true,
           "highlightCards": true,
-          "id": 512,
+          "id": 513,
           "interval": null,
           "legend": {
             "show": false
@@ -70159,7 +70292,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 513,
+          "id": 514,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -70292,7 +70425,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 514,
+          "id": 515,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -70425,7 +70558,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 515,
+          "id": 516,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -70558,7 +70691,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 516,
+          "id": 517,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -70691,7 +70824,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 517,
+          "id": 518,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -70824,7 +70957,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 518,
+          "id": 519,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -70960,7 +71093,7 @@
       },
       "height": null,
       "hideTimeOverride": false,
-      "id": 519,
+      "id": 520,
       "interval": null,
       "links": [],
       "maxDataPoints": 100,
@@ -70999,7 +71132,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 520,
+          "id": 521,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -71147,7 +71280,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 521,
+          "id": 522,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -71280,7 +71413,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 522,
+          "id": 523,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -71413,7 +71546,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 523,
+          "id": 524,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -71549,7 +71682,7 @@
       },
       "height": null,
       "hideTimeOverride": false,
-      "id": 524,
+      "id": 525,
       "interval": null,
       "links": [],
       "maxDataPoints": 100,
@@ -71588,7 +71721,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 525,
+          "id": 526,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -71721,7 +71854,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 526,
+          "id": 527,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -71854,7 +71987,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 527,
+          "id": 528,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -71987,7 +72120,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 528,
+          "id": 529,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -72120,7 +72253,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 529,
+          "id": 530,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -72253,7 +72386,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 530,
+          "id": 531,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -72389,7 +72522,7 @@
       },
       "height": null,
       "hideTimeOverride": false,
-      "id": 531,
+      "id": 532,
       "interval": null,
       "links": [],
       "maxDataPoints": 100,
@@ -72428,7 +72561,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 532,
+          "id": 533,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -72561,7 +72694,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 533,
+          "id": 534,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -72697,7 +72830,7 @@
       },
       "height": null,
       "hideTimeOverride": false,
-      "id": 534,
+      "id": 535,
       "interval": null,
       "links": [],
       "maxDataPoints": 100,
@@ -72736,7 +72869,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 535,
+          "id": 536,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -72937,7 +73070,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 536,
+          "id": 537,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -73073,7 +73206,7 @@
       },
       "height": null,
       "hideTimeOverride": false,
-      "id": 537,
+      "id": 538,
       "interval": null,
       "links": [],
       "maxDataPoints": 100,
@@ -73112,7 +73245,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 538,
+          "id": 539,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -73245,7 +73378,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 539,
+          "id": 540,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -73378,7 +73511,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 540,
+          "id": 541,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -73511,7 +73644,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 541,
+          "id": 542,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -73644,7 +73777,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 542,
+          "id": 543,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -73792,7 +73925,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 543,
+          "id": 544,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -73996,7 +74129,7 @@
       },
       "height": null,
       "hideTimeOverride": false,
-      "id": 544,
+      "id": 545,
       "interval": null,
       "links": [],
       "maxDataPoints": 100,
@@ -74035,7 +74168,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 545,
+          "id": 546,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -74168,7 +74301,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 546,
+          "id": 547,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -74301,7 +74434,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 547,
+          "id": 548,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -74434,7 +74567,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 548,
+          "id": 549,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -74567,7 +74700,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 549,
+          "id": 550,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -74764,7 +74897,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 550,
+          "id": 551,
           "interval": null,
           "links": [],
           "maxDataPoints": 100,
diff --git a/metrics/grafana/tikv_details.json.sha256 b/metrics/grafana/tikv_details.json.sha256
index 15e5d660270..583b5f9dd80 100644
--- a/metrics/grafana/tikv_details.json.sha256
+++ b/metrics/grafana/tikv_details.json.sha256
@@ -1 +1 @@
-d9f42f10c324e6378e1bf46aec7514321d487608a2c96ef5d385c9a0e36798e1  ./metrics/grafana/tikv_details.json
+45396974bfba476b17c24761788798fd35751eb0714365f30e12b4013a05fd4b  ./metrics/grafana/tikv_details.json

From a38f5fbe42394f830951632113523fbbac3eb37e Mon Sep 17 00:00:00 2001
From: glorv <glorvs@163.com>
Date: Fri, 25 Oct 2024 14:50:41 +0800
Subject: [PATCH 06/16] in_memory_engine: add a benchmark for region load
 (#17698)

ref tikv/tikv#16141

Add a benchmark to test in-memory-engine load region performance.

Signed-off-by: glorv <glorvs@163.com>

Co-authored-by: ti-chi-bot[bot] <108142056+ti-chi-bot[bot]@users.noreply.github.com>
---
 Cargo.lock                                    |   1 +
 components/in_memory_engine/Cargo.toml        |   6 +
 .../in_memory_engine/benches/load_region.rs   | 149 +++++++++
 components/in_memory_engine/src/background.rs | 314 ++++++++++--------
 components/in_memory_engine/src/engine.rs     |   7 +
 5 files changed, 331 insertions(+), 146 deletions(-)
 create mode 100644 components/in_memory_engine/benches/load_region.rs

diff --git a/Cargo.lock b/Cargo.lock
index 8b61b31fb13..a8f81c05a4e 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -2563,6 +2563,7 @@ version = "0.0.1"
 dependencies = [
  "bytes",
  "collections",
+ "criterion",
  "crossbeam",
  "crossbeam-skiplist 0.1.3",
  "dashmap",
diff --git a/components/in_memory_engine/Cargo.toml b/components/in_memory_engine/Cargo.toml
index 7bef323b930..a29ad0d6daf 100644
--- a/components/in_memory_engine/Cargo.toml
+++ b/components/in_memory_engine/Cargo.toml
@@ -14,6 +14,11 @@ name = "failpoints"
 path = "tests/failpoints/mod.rs"
 required-features = ["failpoints"]
 
+[[bench]]
+name = "load_region"
+path = "benches/load_region.rs"
+harness = false
+
 [dependencies]
 engine_traits = { workspace = true }
 collections = { workspace = true }
@@ -51,6 +56,7 @@ tokio = { version = "1.5", features = ["rt-multi-thread"] }
 smallvec = "1.4"
 
 [dev-dependencies]
+criterion = "0.3"
 tempfile = "3.0"
 test_pd = { workspace = true }
 test_util = { workspace = true }
diff --git a/components/in_memory_engine/benches/load_region.rs b/components/in_memory_engine/benches/load_region.rs
new file mode 100644
index 00000000000..77315ffc3cb
--- /dev/null
+++ b/components/in_memory_engine/benches/load_region.rs
@@ -0,0 +1,149 @@
+// Copyright 2024 TiKV Project Authors. Licensed under Apache-2.0.
+
+#![feature(test)]
+
+use std::sync::Arc;
+
+use criterion::*;
+use engine_rocks::{util::new_engine, RocksEngine};
+use engine_traits::{
+    CacheRegion, KvEngine, Mutable, RegionCacheEngine, WriteBatch, WriteBatchExt, CF_DEFAULT,
+    CF_WRITE, DATA_CFS,
+};
+use futures::future::ready;
+use in_memory_engine::{BackgroundRunner, *};
+use keys::{DATA_MAX_KEY, DATA_MIN_KEY};
+use pd_client::PdClient;
+use raftstore::coprocessor::config::SPLIT_SIZE;
+use rand::{thread_rng, RngCore};
+use tikv_util::config::VersionTrack;
+use txn_types::{Key, TimeStamp, Write, WriteType};
+
+/// Benches the performace of background load region
+fn bench_load_region(c: &mut Criterion) {
+    for value_size in [32, 128, 512, 4096] {
+        bench_with_args(c, 128, value_size, SPLIT_SIZE.0 as usize, 100);
+    }
+}
+
+fn bench_with_args(
+    c: &mut Criterion,
+    key_size: usize,
+    value_size: usize,
+    region_size: usize,
+    mvcc_amp_thres: usize,
+) {
+    use std::time::Duration;
+
+    let rocks_engine = prepare_data(key_size, value_size, region_size, mvcc_amp_thres);
+    let mut group = c.benchmark_group("load_region");
+    // the bench is slow and workload is not useful.
+    group.warm_up_time(Duration::from_millis(1)).sample_size(10);
+    group.bench_function(format!("value size {}", value_size), |b| {
+        b.iter_with_large_drop(|| {
+            load_region(&rocks_engine);
+        })
+    });
+}
+
+fn prepare_data(
+    key_size: usize,
+    value_size: usize,
+    region_size: usize,
+    mvcc_amp_thres: usize,
+) -> RocksEngine {
+    let path = tempfile::Builder::new()
+        .prefix("bench_load")
+        .tempdir()
+        .unwrap();
+    let path_str = path.path().to_str().unwrap();
+    let rocks_engine = new_engine(path_str, DATA_CFS).unwrap();
+
+    // prepare for test data
+    let mut r = thread_rng();
+    let mut wb = rocks_engine.write_batch();
+    let mut ts_count = 1;
+    let count = (region_size + key_size + value_size - 1) / (key_size + value_size);
+    let mut key = vec![0u8; key_size];
+    r.fill_bytes(&mut key[..key_size - 8]);
+    let mut key_version_count = 0;
+    let mut mvcc_version = r.next_u32() as usize % (mvcc_amp_thres * 2) + 1;
+    for _i in 0..count {
+        if key_version_count >= mvcc_version {
+            r.fill_bytes(&mut key[..key_size - 8]);
+            mvcc_version = r.next_u32() as usize % (mvcc_amp_thres * 2) + 1;
+            key_version_count = 0;
+        }
+
+        let ts = TimeStamp::new(ts_count);
+        let k = keys::data_key(Key::from_raw(&key).append_ts(ts).as_encoded());
+        let mut value = vec![0u8; value_size];
+        r.fill_bytes(&mut value);
+
+        let v = if value_size <= 256 {
+            Some(value)
+        } else {
+            wb.put_cf(CF_DEFAULT, &k, &value).unwrap();
+            None
+        };
+        let w = Write::new(WriteType::Put, ts, v);
+        wb.put_cf(CF_WRITE, &k, &w.as_ref().to_bytes()).unwrap();
+
+        key_version_count += 1;
+        ts_count += 1;
+    }
+
+    wb.write().unwrap();
+
+    rocks_engine
+}
+
+fn load_region(rocks_engine: &RocksEngine) {
+    let mut engine = RegionCacheMemoryEngine::new(InMemoryEngineContext::new_for_tests(Arc::new(
+        VersionTrack::new(InMemoryEngineConfig::config_for_test()),
+    )));
+    engine.set_disk_engine(rocks_engine.clone());
+    let memory_controller = engine.memory_controller();
+
+    // do load
+    let config = Arc::new(VersionTrack::new(InMemoryEngineConfig::config_for_test()));
+    let (worker, _) = BackgroundRunner::new(
+        engine.core().clone(),
+        memory_controller.clone(),
+        None,
+        config,
+        Arc::new(MockTsPdClient::new()),
+        None,
+    );
+
+    let region = CacheRegion::new(1, 1, DATA_MIN_KEY, DATA_MAX_KEY);
+    engine.load_region(region.clone()).unwrap();
+    // update region state to loading to avoid background load.
+    engine.must_set_region_state(1, RegionState::Loading);
+
+    let snapshot = Arc::new(rocks_engine.snapshot());
+    worker.run_load_region(region, snapshot);
+}
+
+struct MockTsPdClient {
+    ts: TimeStamp,
+}
+
+impl MockTsPdClient {
+    fn new() -> Self {
+        // use now to build a big enough timestamp to ensure gc can run.
+        let now = TimeStamp::physical_now();
+        Self {
+            ts: TimeStamp::compose(now, 0),
+        }
+    }
+}
+
+impl PdClient for MockTsPdClient {
+    fn get_tso(&self) -> pd_client::PdFuture<txn_types::TimeStamp> {
+        Box::pin(ready(Ok(self.ts)))
+    }
+}
+
+criterion_group!(benches, bench_load_region);
+criterion_main!(benches);
diff --git a/components/in_memory_engine/src/background.rs b/components/in_memory_engine/src/background.rs
index 3db81a21d7d..23b593b60d0 100644
--- a/components/in_memory_engine/src/background.rs
+++ b/components/in_memory_engine/src/background.rs
@@ -906,6 +906,167 @@ impl BackgroundRunner {
             delete_range_scheduler,
         )
     }
+
+    // used for benchmark.
+    pub fn run_load_region(&self, region: CacheRegion, snapshot: Arc<RocksSnapshot>) {
+        Self::do_load_region(
+            region,
+            snapshot,
+            self.core.clone(),
+            self.delete_range_scheduler.clone(),
+            self.pd_client.clone(),
+            self.config.value().gc_run_interval.0,
+        )
+    }
+
+    fn do_load_region(
+        region: CacheRegion,
+        snapshot: Arc<RocksSnapshot>,
+        core: BackgroundRunnerCore,
+        delete_range_scheduler: Scheduler<BackgroundTask>,
+        pd_client: Arc<dyn PdClient>,
+        gc_run_interval: Duration,
+    ) {
+        fail::fail_point!("ime_before_start_loading_region");
+        fail::fail_point!("ime_on_start_loading_region");
+        let mut is_canceled = false;
+        {
+            let regions_map = core.engine.region_manager().regions_map.read();
+            let region_meta = regions_map.region_meta(region.id).unwrap();
+            // if loading is canceled, we skip the batch load.
+            // NOTE: here we don't check the region epoch version change,
+            // We will handle possible region split and partial cancelation
+            // in `on_snapshot_load_canceled` and `on_snapshot_load_finished`.
+            if region_meta.get_state() != RegionState::Loading {
+                assert_eq!(region_meta.get_state(), RegionState::LoadingCanceled);
+                is_canceled = true;
+            }
+        }
+        let skiplist_engine = core.engine.engine.clone();
+
+        if core.memory_controller.reached_stop_load_threshold() {
+            // We are running out of memory, so cancel the load.
+            is_canceled = true;
+        }
+
+        if is_canceled {
+            info!(
+                "ime snapshot load canceled";
+                "region" => ?region,
+            );
+            core.on_snapshot_load_failed(&region, &delete_range_scheduler, false);
+            return;
+        }
+
+        info!("ime loading region"; "region" => ?&region);
+        let start = Instant::now();
+        let iter_opt = IterOptions::new(
+            Some(KeyBuilder::from_slice(&region.start, 0, 0)),
+            Some(KeyBuilder::from_slice(&region.end, 0, 0)),
+            false,
+        );
+
+        let safe_point = 'load_snapshot: {
+            for &cf in DATA_CFS {
+                let handle = skiplist_engine.cf_handle(cf);
+                let seq = snapshot.sequence_number();
+                let guard = &epoch::pin();
+                match snapshot.iterator_opt(cf, iter_opt.clone()) {
+                    Ok(mut iter) => {
+                        iter.seek_to_first().unwrap();
+                        while iter.valid().unwrap() {
+                            // use the sequence number from RocksDB snapshot here as
+                            // the kv is clearly visible
+                            let mut encoded_key = encode_key(iter.key(), seq, ValueType::Value);
+                            let mut val = InternalBytes::from_vec(iter.value().to_vec());
+
+                            let mem_size = RegionCacheWriteBatchEntry::calc_put_entry_size(
+                                iter.key(),
+                                val.as_bytes(),
+                            );
+
+                            // todo(SpadeA): we can batch acquire the memory size
+                            // here.
+                            if let MemoryUsage::CapacityReached(n) =
+                                core.memory_controller.acquire(mem_size)
+                            {
+                                warn!(
+                                    "ime stop loading snapshot due to memory reaching capacity";
+                                    "region" => ?region,
+                                    "memory_usage(MB)" => ReadableSize(n as u64).as_mb_f64(),
+                                );
+                                break 'load_snapshot None;
+                            }
+
+                            encoded_key.set_memory_controller(core.memory_controller.clone());
+                            val.set_memory_controller(core.memory_controller.clone());
+                            handle.insert(encoded_key, val, guard);
+                            iter.next().unwrap();
+                        }
+                    }
+                    Err(e) => {
+                        error!("ime creating rocksdb iterator failed"; "cf" => cf, "err" => %e);
+                        break 'load_snapshot None;
+                    }
+                }
+            }
+            // gc the range
+            let tso_timeout = std::cmp::min(gc_run_interval, TIMTOUT_FOR_TSO);
+            let now = match block_on_timeout(pd_client.get_tso(), tso_timeout) {
+                Ok(Ok(ts)) => ts,
+                err => {
+                    error!(
+                        "ime get timestamp failed, skip gc loaded region";
+                        "timeout_duration" => ?tso_timeout,
+                        "error" => ?err,
+                    );
+                    // Get timestamp fail so don't do gc.
+                    break 'load_snapshot Some(0);
+                }
+            };
+
+            let safe_point = (|| {
+                fail::fail_point!("ime_safe_point_in_loading", |t| {
+                    t.unwrap().parse::<u64>().unwrap()
+                });
+
+                let safe_point = now
+                    .physical()
+                    .saturating_sub(gc_run_interval.as_millis() as u64);
+                TimeStamp::compose(safe_point, 0).into_inner()
+            })();
+
+            let mut filter = Filter::new(
+                safe_point,
+                u64::MAX,
+                skiplist_engine.cf_handle(CF_DEFAULT),
+                skiplist_engine.cf_handle(CF_WRITE),
+            );
+            filter.filter_keys_in_region(&region);
+
+            Some(safe_point)
+        };
+
+        if let Some(safe_point) = safe_point {
+            if core.on_snapshot_load_finished(&region, &delete_range_scheduler, safe_point) {
+                let duration = start.saturating_elapsed();
+                IN_MEMORY_ENGINE_LOAD_TIME_HISTOGRAM.observe(duration.as_secs_f64());
+                info!(
+                    "ime loading region finished";
+                    "region" => ?region,
+                    "duration(sec)" => ?duration,
+                );
+            } else {
+                info!("ime loading region canceled";"region" => ?region);
+            }
+        } else {
+            info!(
+                "ime snapshot load failed";
+                "region" => ?region,
+            );
+            core.on_snapshot_load_failed(&region, &delete_range_scheduler, true);
+        }
+    }
 }
 
 impl Runnable for BackgroundRunner {
@@ -967,153 +1128,14 @@ impl Runnable for BackgroundRunner {
                 let pd_client = self.pd_client.clone();
                 let gc_run_interval = self.config.value().gc_run_interval.0;
                 let f = async move {
-                    fail::fail_point!("ime_before_start_loading_region");
-                    fail::fail_point!("ime_on_start_loading_region");
-                    let mut is_canceled = false;
-                    {
-                        let regions_map = core.engine.region_manager().regions_map.read();
-                        let region_meta = regions_map.region_meta(region.id).unwrap();
-                        // if loading is canceled, we skip the batch load.
-                        // NOTE: here we don't check the region epoch version change,
-                        // We will handle possible region split and partial cancelation
-                        // in `on_snapshot_load_canceled` and `on_snapshot_load_finished`.
-                        if region_meta.get_state() != RegionState::Loading {
-                            assert_eq!(region_meta.get_state(), RegionState::LoadingCanceled);
-                            is_canceled = true;
-                        }
-                    }
-                    let skiplist_engine = core.engine.engine.clone();
-
-                    if core.memory_controller.reached_stop_load_threshold() {
-                        // We are running out of memory, so cancel the load.
-                        is_canceled = true;
-                    }
-
-                    if is_canceled {
-                        info!(
-                            "ime snapshot load canceled";
-                            "region" => ?region,
-                        );
-                        core.on_snapshot_load_failed(&region, &delete_range_scheduler, false);
-                        return;
-                    }
-
-                    info!("ime loading region"; "region" => ?&region);
-                    let start = Instant::now();
-                    let iter_opt = IterOptions::new(
-                        Some(KeyBuilder::from_slice(&region.start, 0, 0)),
-                        Some(KeyBuilder::from_slice(&region.end, 0, 0)),
-                        false,
+                    Self::do_load_region(
+                        region,
+                        snapshot,
+                        core,
+                        delete_range_scheduler,
+                        pd_client,
+                        gc_run_interval,
                     );
-
-                    let safe_point = 'load_snapshot: {
-                        for &cf in DATA_CFS {
-                            let handle = skiplist_engine.cf_handle(cf);
-                            let seq = snapshot.sequence_number();
-                            let guard = &epoch::pin();
-                            match snapshot.iterator_opt(cf, iter_opt.clone()) {
-                                Ok(mut iter) => {
-                                    iter.seek_to_first().unwrap();
-                                    while iter.valid().unwrap() {
-                                        // use the sequence number from RocksDB snapshot here as
-                                        // the kv is clearly visible
-                                        let mut encoded_key =
-                                            encode_key(iter.key(), seq, ValueType::Value);
-                                        let mut val =
-                                            InternalBytes::from_vec(iter.value().to_vec());
-
-                                        let mem_size =
-                                            RegionCacheWriteBatchEntry::calc_put_entry_size(
-                                                iter.key(),
-                                                val.as_bytes(),
-                                            );
-
-                                        // todo(SpadeA): we can batch acquire the memory size
-                                        // here.
-                                        if let MemoryUsage::CapacityReached(n) =
-                                            core.memory_controller.acquire(mem_size)
-                                        {
-                                            warn!(
-                                                "ime stop loading snapshot due to memory reaching capacity";
-                                                "region" => ?region,
-                                                "memory_usage(MB)" => ReadableSize(n as u64).as_mb_f64(),
-                                            );
-                                            break 'load_snapshot None;
-                                        }
-
-                                        encoded_key
-                                            .set_memory_controller(core.memory_controller.clone());
-                                        val.set_memory_controller(core.memory_controller.clone());
-                                        handle.insert(encoded_key, val, guard);
-                                        iter.next().unwrap();
-                                    }
-                                }
-                                Err(e) => {
-                                    error!("ime creating rocksdb iterator failed"; "cf" => cf, "err" => %e);
-                                    break 'load_snapshot None;
-                                }
-                            }
-                        }
-                        // gc the range
-                        let tso_timeout = std::cmp::min(gc_run_interval, TIMTOUT_FOR_TSO);
-                        let now = match block_on_timeout(pd_client.get_tso(), tso_timeout) {
-                            Ok(Ok(ts)) => ts,
-                            err => {
-                                error!(
-                                    "ime get timestamp failed, skip gc loaded region";
-                                    "timeout_duration" => ?tso_timeout,
-                                    "error" => ?err,
-                                );
-                                // Get timestamp fail so don't do gc.
-                                break 'load_snapshot Some(0);
-                            }
-                        };
-
-                        let safe_point = (|| {
-                            fail::fail_point!("ime_safe_point_in_loading", |t| {
-                                t.unwrap().parse::<u64>().unwrap()
-                            });
-
-                            let safe_point = now
-                                .physical()
-                                .saturating_sub(gc_run_interval.as_millis() as u64);
-                            TimeStamp::compose(safe_point, 0).into_inner()
-                        })();
-
-                        let mut filter = Filter::new(
-                            safe_point,
-                            u64::MAX,
-                            skiplist_engine.cf_handle(CF_DEFAULT),
-                            skiplist_engine.cf_handle(CF_WRITE),
-                        );
-                        filter.filter_keys_in_region(&region);
-
-                        Some(safe_point)
-                    };
-
-                    if let Some(safe_point) = safe_point {
-                        if core.on_snapshot_load_finished(
-                            &region,
-                            &delete_range_scheduler,
-                            safe_point,
-                        ) {
-                            let duration = start.saturating_elapsed();
-                            IN_MEMORY_ENGINE_LOAD_TIME_HISTOGRAM.observe(duration.as_secs_f64());
-                            info!(
-                                "ime loading region finished";
-                                "region" => ?region,
-                                "duration(sec)" => ?duration,
-                            );
-                        } else {
-                            info!("ime loading region canceled";"region" => ?region);
-                        }
-                    } else {
-                        info!(
-                            "ime snapshot load failed";
-                            "region" => ?region,
-                        );
-                        core.on_snapshot_load_failed(&region, &delete_range_scheduler, true);
-                    }
                 };
                 self.region_load_remote.spawn(f);
             }
diff --git a/components/in_memory_engine/src/engine.rs b/components/in_memory_engine/src/engine.rs
index 5128023b80e..24c5cca5eeb 100644
--- a/components/in_memory_engine/src/engine.rs
+++ b/components/in_memory_engine/src/engine.rs
@@ -401,6 +401,13 @@ impl RegionCacheMemoryEngine {
         self.core.region_manager().load_region(cache_region)
     }
 
+    // Used for benchmark.
+    pub fn must_set_region_state(&self, id: u64, state: RegionState) {
+        let mut regions_map = self.core.region_manager().regions_map().write();
+        let meta = regions_map.mut_region_meta(id).unwrap();
+        meta.set_state(state);
+    }
+
     /// Evict a region from the in-memory engine. After this call, the region
     /// will not be readable, but the data of the region may not be deleted
     /// immediately due to some ongoing snapshots.

From f0713b6070b8b606f0d077221df353684a1178a4 Mon Sep 17 00:00:00 2001
From: Spade  A <71589810+SpadeA-Tang@users.noreply.github.com>
Date: Fri, 25 Oct 2024 15:57:53 +0800
Subject: [PATCH 07/16] In-memory engine: display put/delete operations per
 second (#17695)

ref tikv/tikv#16141

display put/delete operations per second

Signed-off-by: SpadeA-Tang <u6748471@anu.edu.au>
Signed-off-by: SpadeA-Tang <tangchenjie1210@gmail.com>

Co-authored-by: ti-chi-bot[bot] <108142056+ti-chi-bot[bot]@users.noreply.github.com>
---
 components/in_memory_engine/src/metrics.rs    |  55 ++
 .../in_memory_engine/src/write_batch.rs       |  17 +-
 metrics/grafana/tikv_details.dashboard.py     |  19 +
 metrics/grafana/tikv_details.json             | 757 ++++++++++--------
 metrics/grafana/tikv_details.json.sha256      |   2 +-
 5 files changed, 536 insertions(+), 314 deletions(-)

diff --git a/components/in_memory_engine/src/metrics.rs b/components/in_memory_engine/src/metrics.rs
index ffdb48d6c64..7743c5ff09b 100644
--- a/components/in_memory_engine/src/metrics.rs
+++ b/components/in_memory_engine/src/metrics.rs
@@ -45,6 +45,17 @@ make_auto_flush_static_metric! {
         manual,
     }
 
+    pub label_enum OperationType {
+        put,
+        delete,
+    }
+
+    pub label_enum CF {
+        default,
+        lock,
+        write,
+    }
+
     pub struct GcFilteredCountVec: LocalIntCounter {
         "type" => KeyCountType,
     }
@@ -56,6 +67,11 @@ make_auto_flush_static_metric! {
     pub struct EvictionDurationVec: LocalHistogram {
         "type" => EvictReasonType,
     }
+
+    pub struct OperationTypeForCF: LocalIntCounter {
+        "type" => OperationType,
+        "cf" => CF,
+    }
 }
 
 lazy_static! {
@@ -127,6 +143,11 @@ lazy_static! {
         exponential_buckets(0.00001, 2.0, 26).unwrap()
     )
     .unwrap();
+    pub static ref IN_MEMORY_ENGINE_KV_OPERATIONS: IntCounterVec = register_int_counter_vec!(
+        "tikv_in_memory_engine_kv_operations",
+        "Number of kv operations",
+        &["type", "cf"]
+    ).unwrap();
     pub static ref IN_MEMORY_ENGINE_OLDEST_SAFE_POINT: IntGauge = register_int_gauge!(
         "tikv_in_memory_engine_oldest_safe_point",
         "The oldest safe point in the in-memory engine",
@@ -155,6 +176,8 @@ lazy_static! {
         IN_MEMORY_ENGINE_EVICTION_DURATION_HISTOGRAM,
         EvictionDurationVec
     );
+    pub static ref IN_MEMORY_ENGINE_OPERATION_STATIC: OperationTypeForCF =
+        auto_flush_from!(IN_MEMORY_ENGINE_KV_OPERATIONS, OperationTypeForCF);
 }
 
 pub fn flush_in_memory_engine_statistics(statistics: &Arc<InMemoryEngineStatistics>) {
@@ -239,3 +262,35 @@ pub(crate) fn observe_eviction_duration(secs: f64, evict_reason: EvictReason) {
             .observe(secs),
     }
 }
+
+pub(crate) fn count_operations_for_cfs(put_operations: &[u64], delete_operations: &[u64]) {
+    // according to `cf_to_id`, we have 0 for CF_DEFAULT, 1 for CF_LOCK, and 2 for
+    // CF_WRITE
+    assert_eq!(put_operations.len(), 3);
+    assert_eq!(delete_operations.len(), 3);
+    IN_MEMORY_ENGINE_OPERATION_STATIC
+        .put
+        .default
+        .inc_by(put_operations[0]);
+    IN_MEMORY_ENGINE_OPERATION_STATIC
+        .put
+        .lock
+        .inc_by(put_operations[1]);
+    IN_MEMORY_ENGINE_OPERATION_STATIC
+        .put
+        .write
+        .inc_by(put_operations[2]);
+
+    IN_MEMORY_ENGINE_OPERATION_STATIC
+        .delete
+        .default
+        .inc_by(delete_operations[0]);
+    IN_MEMORY_ENGINE_OPERATION_STATIC
+        .delete
+        .lock
+        .inc_by(delete_operations[1]);
+    IN_MEMORY_ENGINE_OPERATION_STATIC
+        .delete
+        .write
+        .inc_by(delete_operations[2]);
+}
diff --git a/components/in_memory_engine/src/write_batch.rs b/components/in_memory_engine/src/write_batch.rs
index b91bdb2e1c0..4ebe0bef20f 100644
--- a/components/in_memory_engine/src/write_batch.rs
+++ b/components/in_memory_engine/src/write_batch.rs
@@ -21,7 +21,7 @@ use crate::{
     keys::{encode_key, InternalBytes, ValueType, ENC_KEY_SEQ_LENGTH},
     memory_controller::{MemoryController, MemoryUsage},
     metrics::{
-        IN_MEMORY_ENGINE_PREPARE_FOR_WRITE_DURATION_HISTOGRAM,
+        count_operations_for_cfs, IN_MEMORY_ENGINE_PREPARE_FOR_WRITE_DURATION_HISTOGRAM,
         IN_MEMORY_ENGINE_WRITE_DURATION_HISTOGRAM,
     },
     region_manager::RegionCacheStatus,
@@ -188,17 +188,28 @@ impl RegionCacheWriteBatch {
         let start = Instant::now();
         let mut lock_modification: u64 = 0;
         let engine = self.engine.core.engine();
+
+        // record the number of insertions and deletions for each cf
+        let mut put = [0, 0, 0];
+        let mut delete = [0, 0, 0];
         // Some entries whose ranges may be marked as evicted above, but it does not
         // matter, they will be deleted later.
         std::mem::take(&mut self.buffer).into_iter().for_each(|e| {
             if is_lock_cf(e.cf) {
                 lock_modification += e.data_size() as u64;
             }
+            if e.is_insertion() {
+                put[e.cf] += 1;
+            } else {
+                delete[e.cf] += 1;
+            }
+
             e.write_to_memory(seq, &engine, self.memory_controller.clone(), guard);
             seq += 1;
         });
         let duration = start.saturating_elapsed_secs();
         IN_MEMORY_ENGINE_WRITE_DURATION_HISTOGRAM.observe(duration);
+        count_operations_for_cfs(&put, &delete);
 
         fail::fail_point!("ime_on_region_cache_write_batch_write_consumed");
         fail::fail_point!("ime_before_clear_regions_in_being_written");
@@ -348,6 +359,10 @@ pub(crate) struct RegionCacheWriteBatchEntry {
 }
 
 impl RegionCacheWriteBatchEntry {
+    pub fn is_insertion(&self) -> bool {
+        matches!(self.inner, WriteBatchEntryInternal::PutValue(_))
+    }
+
     pub fn put_value(cf: &str, key: &[u8], value: &[u8]) -> Self {
         Self {
             cf: cf_to_id(cf),
diff --git a/metrics/grafana/tikv_details.dashboard.py b/metrics/grafana/tikv_details.dashboard.py
index 85c6f32f452..046a902c1ad 100644
--- a/metrics/grafana/tikv_details.dashboard.py
+++ b/metrics/grafana/tikv_details.dashboard.py
@@ -4372,6 +4372,25 @@ def CoprocessorDetail() -> RowPanel:
 
 def InMemoryEngine() -> RowPanel:
     layout = Layout(title="In Memory Engine")
+    layout.row(
+        [
+            graph_panel(
+                title="OPS",
+                description="Operation per second for cf",
+                yaxes=yaxes(left_format=UNITS.OPS_PER_SEC),
+                targets=[
+                    target(
+                        expr=expr_sum_rate(
+                            "tikv_in_memory_engine_kv_operations",
+                            by_labels=["instance", "type"],
+                        ),
+                        legend_format="{{type}}-{{instance}}",
+                        additional_groupby=True,
+                    ),
+                ],
+            )
+        ]
+    )
     layout.row(
         [
             graph_panel(
diff --git a/metrics/grafana/tikv_details.json b/metrics/grafana/tikv_details.json
index 9238472e7c1..100001338fd 100644
--- a/metrics/grafana/tikv_details.json
+++ b/metrics/grafana/tikv_details.json
@@ -37324,7 +37324,7 @@
           "bars": false,
           "cacheTimeout": null,
           "datasource": "${DS_TEST-CLUSTER}",
-          "description": "Count of region cache hit",
+          "description": "Operation per second for cf",
           "editable": true,
           "error": false,
           "fieldConfig": {
@@ -37345,7 +37345,7 @@
           },
           "gridPos": {
             "h": 7,
-            "w": 8,
+            "w": 24,
             "x": 0,
             "y": 0
           },
@@ -37391,6 +37391,139 @@
           "span": null,
           "stack": false,
           "steppedLine": false,
+          "targets": [
+            {
+              "datasource": "${DS_TEST-CLUSTER}",
+              "expr": "sum(rate(\n    tikv_in_memory_engine_kv_operations\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    [$__rate_interval]\n)) by (instance, type, $additional_groupby) ",
+              "format": "time_series",
+              "hide": false,
+              "instant": false,
+              "interval": "",
+              "intervalFactor": 1,
+              "legendFormat": "{{type}}-{{instance}} {{$additional_groupby}}",
+              "metric": "",
+              "query": "sum(rate(\n    tikv_in_memory_engine_kv_operations\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    [$__rate_interval]\n)) by (instance, type, $additional_groupby) ",
+              "refId": "",
+              "step": 10,
+              "target": ""
+            }
+          ],
+          "thresholds": [],
+          "timeFrom": null,
+          "timeShift": null,
+          "title": "OPS",
+          "tooltip": {
+            "msResolution": true,
+            "shared": true,
+            "sort": 0,
+            "value_type": "individual"
+          },
+          "transformations": [],
+          "transparent": false,
+          "type": "graph",
+          "xaxis": {
+            "mode": "time",
+            "name": null,
+            "show": true,
+            "values": []
+          },
+          "yaxes": [
+            {
+              "decimals": null,
+              "format": "ops",
+              "label": null,
+              "logBase": 1,
+              "max": null,
+              "min": null,
+              "show": true
+            },
+            {
+              "decimals": null,
+              "format": "short",
+              "label": null,
+              "logBase": 1,
+              "max": null,
+              "min": null,
+              "show": true
+            }
+          ],
+          "yaxis": {
+            "align": false,
+            "alignLevel": 0
+          }
+        },
+        {
+          "aliasColors": {},
+          "bars": false,
+          "cacheTimeout": null,
+          "datasource": "${DS_TEST-CLUSTER}",
+          "description": "Count of region cache hit",
+          "editable": true,
+          "error": false,
+          "fieldConfig": {
+            "defaults": {
+              "thresholds": {
+                "mode": "absolute",
+                "steps": []
+              }
+            }
+          },
+          "fill": 1,
+          "fillGradient": 1,
+          "grid": {
+            "threshold1": null,
+            "threshold1Color": "rgba(216, 200, 27, 0.27)",
+            "threshold2": null,
+            "threshold2Color": "rgba(234, 112, 112, 0.22)"
+          },
+          "gridPos": {
+            "h": 7,
+            "w": 8,
+            "x": 0,
+            "y": 7
+          },
+          "height": null,
+          "hideTimeOverride": false,
+          "id": 262,
+          "interval": null,
+          "isNew": true,
+          "legend": {
+            "alignAsTable": true,
+            "avg": false,
+            "current": true,
+            "hideEmpty": true,
+            "hideZero": true,
+            "max": true,
+            "min": false,
+            "rightSide": true,
+            "show": true,
+            "sideWidth": null,
+            "sort": "max",
+            "sortDesc": true,
+            "total": false,
+            "values": true
+          },
+          "lines": true,
+          "linewidth": 1,
+          "links": [],
+          "maxDataPoints": null,
+          "maxPerRow": null,
+          "minSpan": null,
+          "nullPointMode": "null as zero",
+          "options": {
+            "alertThreshold": true,
+            "dataLinks": []
+          },
+          "percentage": false,
+          "pointradius": 5,
+          "points": false,
+          "renderer": "flot",
+          "repeat": null,
+          "repeatDirection": null,
+          "seriesOverrides": [],
+          "span": null,
+          "stack": false,
+          "steppedLine": false,
           "targets": [
             {
               "datasource": "${DS_TEST-CLUSTER}",
@@ -37480,11 +37613,11 @@
             "h": 7,
             "w": 8,
             "x": 8,
-            "y": 0
+            "y": 7
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 262,
+          "id": 263,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -37613,11 +37746,11 @@
             "h": 7,
             "w": 8,
             "x": 16,
-            "y": 0
+            "y": 7
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 263,
+          "id": 264,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -37746,11 +37879,11 @@
             "h": 7,
             "w": 6,
             "x": 0,
-            "y": 7
+            "y": 14
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 264,
+          "id": 265,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -37879,11 +38012,11 @@
             "h": 7,
             "w": 6,
             "x": 6,
-            "y": 7
+            "y": 14
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 265,
+          "id": 266,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -38012,11 +38145,11 @@
             "h": 7,
             "w": 6,
             "x": 12,
-            "y": 7
+            "y": 14
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 266,
+          "id": 267,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -38149,14 +38282,14 @@
             "h": 7,
             "w": 6,
             "x": 18,
-            "y": 7
+            "y": 14
           },
           "heatmap": {},
           "height": null,
           "hideTimeOverride": false,
           "hideZeroBuckets": true,
           "highlightCards": true,
-          "id": 267,
+          "id": 268,
           "interval": null,
           "legend": {
             "show": false
@@ -38254,14 +38387,14 @@
             "h": 7,
             "w": 12,
             "x": 0,
-            "y": 14
+            "y": 21
           },
           "heatmap": {},
           "height": null,
           "hideTimeOverride": false,
           "hideZeroBuckets": true,
           "highlightCards": true,
-          "id": 268,
+          "id": 269,
           "interval": null,
           "legend": {
             "show": false
@@ -38355,11 +38488,11 @@
             "h": 7,
             "w": 12,
             "x": 12,
-            "y": 14
+            "y": 21
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 269,
+          "id": 270,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -38492,14 +38625,14 @@
             "h": 7,
             "w": 12,
             "x": 0,
-            "y": 21
+            "y": 28
           },
           "heatmap": {},
           "height": null,
           "hideTimeOverride": false,
           "hideZeroBuckets": true,
           "highlightCards": true,
-          "id": 270,
+          "id": 271,
           "interval": null,
           "legend": {
             "show": false
@@ -38593,11 +38726,11 @@
             "h": 7,
             "w": 12,
             "x": 12,
-            "y": 21
+            "y": 28
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 271,
+          "id": 272,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -38730,14 +38863,14 @@
             "h": 7,
             "w": 12,
             "x": 0,
-            "y": 28
+            "y": 35
           },
           "heatmap": {},
           "height": null,
           "hideTimeOverride": false,
           "hideZeroBuckets": true,
           "highlightCards": true,
-          "id": 272,
+          "id": 273,
           "interval": null,
           "legend": {
             "show": false
@@ -38831,11 +38964,11 @@
             "h": 7,
             "w": 12,
             "x": 12,
-            "y": 28
+            "y": 35
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 273,
+          "id": 274,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -39036,14 +39169,14 @@
             "h": 7,
             "w": 12,
             "x": 0,
-            "y": 35
+            "y": 42
           },
           "heatmap": {},
           "height": null,
           "hideTimeOverride": false,
           "hideZeroBuckets": true,
           "highlightCards": true,
-          "id": 274,
+          "id": 275,
           "interval": null,
           "legend": {
             "show": false
@@ -39137,11 +39270,11 @@
             "h": 7,
             "w": 12,
             "x": 12,
-            "y": 35
+            "y": 42
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 275,
+          "id": 276,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -39338,11 +39471,11 @@
             "h": 7,
             "w": 12,
             "x": 0,
-            "y": 42
+            "y": 49
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 276,
+          "id": 277,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -39546,11 +39679,11 @@
             "h": 7,
             "w": 12,
             "x": 12,
-            "y": 42
+            "y": 49
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 277,
+          "id": 278,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -39724,11 +39857,11 @@
             "h": 7,
             "w": 6,
             "x": 0,
-            "y": 49
+            "y": 56
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 278,
+          "id": 279,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -39857,11 +39990,11 @@
             "h": 7,
             "w": 6,
             "x": 6,
-            "y": 49
+            "y": 56
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 279,
+          "id": 280,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -39990,11 +40123,11 @@
             "h": 7,
             "w": 6,
             "x": 12,
-            "y": 49
+            "y": 56
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 280,
+          "id": 281,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -40123,11 +40256,11 @@
             "h": 7,
             "w": 6,
             "x": 18,
-            "y": 49
+            "y": 56
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 281,
+          "id": 282,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -40263,7 +40396,7 @@
       },
       "height": null,
       "hideTimeOverride": false,
-      "id": 282,
+      "id": 283,
       "interval": null,
       "links": [],
       "maxDataPoints": 100,
@@ -40302,7 +40435,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 283,
+          "id": 284,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -40450,7 +40583,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 284,
+          "id": 285,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -40590,7 +40723,7 @@
           "hideTimeOverride": false,
           "hideZeroBuckets": true,
           "highlightCards": true,
-          "id": 285,
+          "id": 286,
           "interval": null,
           "legend": {
             "show": false
@@ -40688,7 +40821,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 286,
+          "id": 287,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -40821,7 +40954,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 287,
+          "id": 288,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -40954,7 +41087,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 288,
+          "id": 289,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -41132,7 +41265,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 289,
+          "id": 290,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -41295,7 +41428,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 290,
+          "id": 291,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -41443,7 +41576,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 291,
+          "id": 292,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -41576,7 +41709,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 292,
+          "id": 293,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -41712,7 +41845,7 @@
       },
       "height": null,
       "hideTimeOverride": false,
-      "id": 293,
+      "id": 294,
       "interval": null,
       "links": [],
       "maxDataPoints": 100,
@@ -41751,7 +41884,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 294,
+          "id": 295,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -41899,7 +42032,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 295,
+          "id": 296,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -42032,7 +42165,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 296,
+          "id": 297,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -42165,7 +42298,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 297,
+          "id": 298,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -42298,7 +42431,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 298,
+          "id": 299,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -42431,7 +42564,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 299,
+          "id": 300,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -42586,7 +42719,7 @@
           "hideTimeOverride": false,
           "hideZeroBuckets": true,
           "highlightCards": true,
-          "id": 300,
+          "id": 301,
           "interval": null,
           "legend": {
             "show": false
@@ -42687,7 +42820,7 @@
       },
       "height": null,
       "hideTimeOverride": false,
-      "id": 301,
+      "id": 302,
       "interval": null,
       "links": [],
       "maxDataPoints": 100,
@@ -42726,7 +42859,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 302,
+          "id": 303,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -42859,7 +42992,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 303,
+          "id": 304,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -42992,7 +43125,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 304,
+          "id": 305,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -43132,7 +43265,7 @@
           "hideTimeOverride": false,
           "hideZeroBuckets": true,
           "highlightCards": true,
-          "id": 305,
+          "id": 306,
           "interval": null,
           "legend": {
             "show": false
@@ -43230,7 +43363,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 306,
+          "id": 307,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -43431,7 +43564,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 307,
+          "id": 308,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -43632,7 +43765,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 308,
+          "id": 309,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -43836,7 +43969,7 @@
       },
       "height": null,
       "hideTimeOverride": false,
-      "id": 309,
+      "id": 310,
       "interval": null,
       "links": [],
       "maxDataPoints": 100,
@@ -43875,7 +44008,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 310,
+          "id": 311,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -44023,7 +44156,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 311,
+          "id": 312,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -44224,7 +44357,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 312,
+          "id": 313,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -44425,7 +44558,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 313,
+          "id": 314,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -44626,7 +44759,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 314,
+          "id": 315,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -44827,7 +44960,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 315,
+          "id": 316,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -44960,7 +45093,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 316,
+          "id": 317,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -45093,7 +45226,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 317,
+          "id": 318,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -45226,7 +45359,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 318,
+          "id": 319,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -45359,7 +45492,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 319,
+          "id": 320,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -45567,7 +45700,7 @@
           "hideTimeOverride": false,
           "hideZeroBuckets": true,
           "highlightCards": true,
-          "id": 320,
+          "id": 321,
           "interval": null,
           "legend": {
             "show": false
@@ -45668,7 +45801,7 @@
       },
       "height": null,
       "hideTimeOverride": false,
-      "id": 321,
+      "id": 322,
       "interval": null,
       "links": [],
       "maxDataPoints": 100,
@@ -45714,7 +45847,7 @@
           "hideTimeOverride": false,
           "hideZeroBuckets": true,
           "highlightCards": true,
-          "id": 322,
+          "id": 323,
           "interval": null,
           "legend": {
             "show": false
@@ -45812,7 +45945,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 323,
+          "id": 324,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -46013,7 +46146,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 324,
+          "id": 325,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -46146,7 +46279,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 325,
+          "id": 326,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -46279,7 +46412,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 326,
+          "id": 327,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -46412,7 +46545,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 327,
+          "id": 328,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -46613,7 +46746,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 328,
+          "id": 329,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -46746,7 +46879,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 329,
+          "id": 330,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -46879,7 +47012,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 330,
+          "id": 331,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -47015,7 +47148,7 @@
       },
       "height": null,
       "hideTimeOverride": false,
-      "id": 331,
+      "id": 332,
       "interval": null,
       "links": [],
       "maxDataPoints": 100,
@@ -47054,7 +47187,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 332,
+          "id": 333,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -47255,7 +47388,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 333,
+          "id": 334,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -47456,7 +47589,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 334,
+          "id": 335,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -47657,7 +47790,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 335,
+          "id": 336,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -47858,7 +47991,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 336,
+          "id": 337,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -47991,7 +48124,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 337,
+          "id": 338,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -48124,7 +48257,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 338,
+          "id": 339,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -48257,7 +48390,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 339,
+          "id": 340,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -48390,7 +48523,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 340,
+          "id": 341,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -48523,7 +48656,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 341,
+          "id": 342,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -48663,7 +48796,7 @@
           "hideTimeOverride": false,
           "hideZeroBuckets": true,
           "highlightCards": true,
-          "id": 342,
+          "id": 343,
           "interval": null,
           "legend": {
             "show": false
@@ -48761,7 +48894,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 343,
+          "id": 344,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -48965,7 +49098,7 @@
       },
       "height": null,
       "hideTimeOverride": false,
-      "id": 344,
+      "id": 345,
       "interval": null,
       "links": [],
       "maxDataPoints": 100,
@@ -49004,7 +49137,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 345,
+          "id": 346,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -49137,7 +49270,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 346,
+          "id": 347,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -49270,7 +49403,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 347,
+          "id": 348,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -49410,7 +49543,7 @@
           "hideTimeOverride": false,
           "hideZeroBuckets": true,
           "highlightCards": true,
-          "id": 348,
+          "id": 349,
           "interval": null,
           "legend": {
             "show": false
@@ -49508,7 +49641,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 349,
+          "id": 350,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -49709,7 +49842,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 350,
+          "id": 351,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -49910,7 +50043,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 351,
+          "id": 352,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -50114,7 +50247,7 @@
       },
       "height": null,
       "hideTimeOverride": false,
-      "id": 352,
+      "id": 353,
       "interval": null,
       "links": [],
       "maxDataPoints": 100,
@@ -50153,7 +50286,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 353,
+          "id": 354,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -50331,7 +50464,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 354,
+          "id": 355,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -50532,7 +50665,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 355,
+          "id": 356,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -50665,7 +50798,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 356,
+          "id": 357,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -50798,7 +50931,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 357,
+          "id": 358,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -50931,7 +51064,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 358,
+          "id": 359,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -51064,7 +51197,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 359,
+          "id": 360,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -51197,7 +51330,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 360,
+          "id": 361,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -51326,7 +51459,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 361,
+          "id": 362,
           "interval": null,
           "links": [],
           "maxDataPoints": 100,
@@ -51401,7 +51534,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 362,
+          "id": 363,
           "interval": null,
           "links": [],
           "maxDataPoints": 100,
@@ -51480,7 +51613,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 363,
+          "id": 364,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -51733,7 +51866,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 364,
+          "id": 365,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -51866,7 +51999,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 365,
+          "id": 366,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -52002,7 +52135,7 @@
       },
       "height": null,
       "hideTimeOverride": false,
-      "id": 366,
+      "id": 367,
       "interval": null,
       "links": [],
       "maxDataPoints": 100,
@@ -52041,7 +52174,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 367,
+          "id": 368,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -52189,7 +52322,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 368,
+          "id": 369,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -52322,7 +52455,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 369,
+          "id": 370,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -52523,7 +52656,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 370,
+          "id": 371,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -52671,7 +52804,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 371,
+          "id": 372,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -52872,7 +53005,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 372,
+          "id": 373,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -53005,7 +53138,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 373,
+          "id": 374,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -53138,7 +53271,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 374,
+          "id": 375,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -53271,7 +53404,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 375,
+          "id": 376,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -53404,7 +53537,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 376,
+          "id": 377,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -53544,7 +53677,7 @@
           "hideTimeOverride": false,
           "hideZeroBuckets": true,
           "highlightCards": true,
-          "id": 377,
+          "id": 378,
           "interval": null,
           "legend": {
             "show": false
@@ -53642,7 +53775,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 378,
+          "id": 379,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -53846,7 +53979,7 @@
       },
       "height": null,
       "hideTimeOverride": false,
-      "id": 379,
+      "id": 380,
       "interval": null,
       "links": [],
       "maxDataPoints": 100,
@@ -53885,7 +54018,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 380,
+          "id": 381,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -54018,7 +54151,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 381,
+          "id": 382,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -54151,7 +54284,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 382,
+          "id": 383,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -54284,7 +54417,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 383,
+          "id": 384,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -54420,7 +54553,7 @@
       },
       "height": null,
       "hideTimeOverride": false,
-      "id": 384,
+      "id": 385,
       "interval": null,
       "links": [],
       "maxDataPoints": 100,
@@ -54459,7 +54592,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 385,
+          "id": 386,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -54592,7 +54725,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 386,
+          "id": 387,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -54725,7 +54858,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 387,
+          "id": 388,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -54873,7 +55006,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 388,
+          "id": 389,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -55006,7 +55139,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 389,
+          "id": 390,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -55139,7 +55272,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 390,
+          "id": 391,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -55272,7 +55405,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 391,
+          "id": 392,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -55408,7 +55541,7 @@
       },
       "height": null,
       "hideTimeOverride": false,
-      "id": 392,
+      "id": 393,
       "interval": null,
       "links": [],
       "maxDataPoints": 100,
@@ -55447,7 +55580,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 393,
+          "id": 394,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -55580,7 +55713,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 394,
+          "id": 395,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -55713,7 +55846,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 395,
+          "id": 396,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -55846,7 +55979,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 396,
+          "id": 397,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -55979,7 +56112,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 397,
+          "id": 398,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -56112,7 +56245,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 398,
+          "id": 399,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -56248,7 +56381,7 @@
       },
       "height": null,
       "hideTimeOverride": false,
-      "id": 399,
+      "id": 400,
       "interval": null,
       "links": [],
       "maxDataPoints": 100,
@@ -56287,7 +56420,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 400,
+          "id": 401,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -56420,7 +56553,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 401,
+          "id": 402,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -56553,7 +56686,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 402,
+          "id": 403,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -56701,7 +56834,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 403,
+          "id": 404,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -56864,7 +56997,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 404,
+          "id": 405,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -56997,7 +57130,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 405,
+          "id": 406,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -57130,7 +57263,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 406,
+          "id": 407,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -57278,7 +57411,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 407,
+          "id": 408,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -57426,7 +57559,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 408,
+          "id": 409,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -57562,7 +57695,7 @@
       },
       "height": null,
       "hideTimeOverride": false,
-      "id": 409,
+      "id": 410,
       "interval": null,
       "links": [],
       "maxDataPoints": 100,
@@ -57601,7 +57734,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 410,
+          "id": 411,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -57734,7 +57867,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 411,
+          "id": 412,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -57867,7 +58000,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 412,
+          "id": 413,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -58000,7 +58133,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 413,
+          "id": 414,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -58133,7 +58266,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 414,
+          "id": 415,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -58266,7 +58399,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 415,
+          "id": 416,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -58399,7 +58532,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 416,
+          "id": 417,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -58532,7 +58665,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 417,
+          "id": 418,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -58665,7 +58798,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 418,
+          "id": 419,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -58805,7 +58938,7 @@
           "hideTimeOverride": false,
           "hideZeroBuckets": true,
           "highlightCards": true,
-          "id": 419,
+          "id": 420,
           "interval": null,
           "legend": {
             "show": false
@@ -58903,7 +59036,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 420,
+          "id": 421,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -59036,7 +59169,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 421,
+          "id": 422,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -59184,7 +59317,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 422,
+          "id": 423,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -59332,7 +59465,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 423,
+          "id": 424,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -59472,7 +59605,7 @@
           "hideTimeOverride": false,
           "hideZeroBuckets": true,
           "highlightCards": true,
-          "id": 424,
+          "id": 425,
           "interval": null,
           "legend": {
             "show": false
@@ -59570,7 +59703,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 425,
+          "id": 426,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -59703,7 +59836,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 426,
+          "id": 427,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -59839,7 +59972,7 @@
       },
       "height": null,
       "hideTimeOverride": false,
-      "id": 427,
+      "id": 428,
       "interval": null,
       "links": [],
       "maxDataPoints": 100,
@@ -59878,7 +60011,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 428,
+          "id": 429,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -60011,7 +60144,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 429,
+          "id": 430,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -60174,7 +60307,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 430,
+          "id": 431,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -60322,7 +60455,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 431,
+          "id": 432,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -60455,7 +60588,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 432,
+          "id": 433,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -60595,7 +60728,7 @@
           "hideTimeOverride": false,
           "hideZeroBuckets": true,
           "highlightCards": true,
-          "id": 433,
+          "id": 434,
           "interval": null,
           "legend": {
             "show": false
@@ -60700,7 +60833,7 @@
           "hideTimeOverride": false,
           "hideZeroBuckets": true,
           "highlightCards": true,
-          "id": 434,
+          "id": 435,
           "interval": null,
           "legend": {
             "show": false
@@ -60805,7 +60938,7 @@
           "hideTimeOverride": false,
           "hideZeroBuckets": true,
           "highlightCards": true,
-          "id": 435,
+          "id": 436,
           "interval": null,
           "legend": {
             "show": false
@@ -60903,7 +61036,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 436,
+          "id": 437,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -61043,7 +61176,7 @@
           "hideTimeOverride": false,
           "hideZeroBuckets": true,
           "highlightCards": true,
-          "id": 437,
+          "id": 438,
           "interval": null,
           "legend": {
             "show": false
@@ -61148,7 +61281,7 @@
           "hideTimeOverride": false,
           "hideZeroBuckets": true,
           "highlightCards": true,
-          "id": 438,
+          "id": 439,
           "interval": null,
           "legend": {
             "show": false
@@ -61253,7 +61386,7 @@
           "hideTimeOverride": false,
           "hideZeroBuckets": true,
           "highlightCards": true,
-          "id": 439,
+          "id": 440,
           "interval": null,
           "legend": {
             "show": false
@@ -61351,7 +61484,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 440,
+          "id": 441,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -61484,7 +61617,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 441,
+          "id": 442,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -61617,7 +61750,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 442,
+          "id": 443,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -61757,7 +61890,7 @@
           "hideTimeOverride": false,
           "hideZeroBuckets": true,
           "highlightCards": true,
-          "id": 443,
+          "id": 444,
           "interval": null,
           "legend": {
             "show": false
@@ -61855,7 +61988,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 444,
+          "id": 445,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -61991,7 +62124,7 @@
       },
       "height": null,
       "hideTimeOverride": false,
-      "id": 445,
+      "id": 446,
       "interval": null,
       "links": [],
       "maxDataPoints": 100,
@@ -62030,7 +62163,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 446,
+          "id": 447,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -62193,7 +62326,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 447,
+          "id": 448,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -62326,7 +62459,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 448,
+          "id": 449,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -62466,7 +62599,7 @@
           "hideTimeOverride": false,
           "hideZeroBuckets": true,
           "highlightCards": true,
-          "id": 449,
+          "id": 450,
           "interval": null,
           "legend": {
             "show": false
@@ -62571,7 +62704,7 @@
           "hideTimeOverride": false,
           "hideZeroBuckets": true,
           "highlightCards": true,
-          "id": 450,
+          "id": 451,
           "interval": null,
           "legend": {
             "show": false
@@ -62669,7 +62802,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 451,
+          "id": 452,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -62824,7 +62957,7 @@
           "hideTimeOverride": false,
           "hideZeroBuckets": true,
           "highlightCards": true,
-          "id": 452,
+          "id": 453,
           "interval": null,
           "legend": {
             "show": false
@@ -62929,7 +63062,7 @@
           "hideTimeOverride": false,
           "hideZeroBuckets": true,
           "highlightCards": true,
-          "id": 453,
+          "id": 454,
           "interval": null,
           "legend": {
             "show": false
@@ -63034,7 +63167,7 @@
           "hideTimeOverride": false,
           "hideZeroBuckets": true,
           "highlightCards": true,
-          "id": 454,
+          "id": 455,
           "interval": null,
           "legend": {
             "show": false
@@ -63132,7 +63265,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 455,
+          "id": 456,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -63302,7 +63435,7 @@
           "hideTimeOverride": false,
           "hideZeroBuckets": true,
           "highlightCards": true,
-          "id": 456,
+          "id": 457,
           "interval": null,
           "legend": {
             "show": false
@@ -63400,7 +63533,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 457,
+          "id": 458,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -63601,7 +63734,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 458,
+          "id": 459,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -63802,7 +63935,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 459,
+          "id": 460,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -63935,7 +64068,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 460,
+          "id": 461,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -64098,7 +64231,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 461,
+          "id": 462,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -64231,7 +64364,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 462,
+          "id": 463,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -64364,7 +64497,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 463,
+          "id": 464,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -64565,7 +64698,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 464,
+          "id": 465,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -64698,7 +64831,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 465,
+          "id": 466,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -64838,7 +64971,7 @@
           "hideTimeOverride": false,
           "hideZeroBuckets": true,
           "highlightCards": true,
-          "id": 466,
+          "id": 467,
           "interval": null,
           "legend": {
             "show": false
@@ -64943,7 +65076,7 @@
           "hideTimeOverride": false,
           "hideZeroBuckets": true,
           "highlightCards": true,
-          "id": 467,
+          "id": 468,
           "interval": null,
           "legend": {
             "show": false
@@ -65048,7 +65181,7 @@
           "hideTimeOverride": false,
           "hideZeroBuckets": true,
           "highlightCards": true,
-          "id": 468,
+          "id": 469,
           "interval": null,
           "legend": {
             "show": false
@@ -65153,7 +65286,7 @@
           "hideTimeOverride": false,
           "hideZeroBuckets": true,
           "highlightCards": true,
-          "id": 469,
+          "id": 470,
           "interval": null,
           "legend": {
             "show": false
@@ -65258,7 +65391,7 @@
           "hideTimeOverride": false,
           "hideZeroBuckets": true,
           "highlightCards": true,
-          "id": 470,
+          "id": 471,
           "interval": null,
           "legend": {
             "show": false
@@ -65363,7 +65496,7 @@
           "hideTimeOverride": false,
           "hideZeroBuckets": true,
           "highlightCards": true,
-          "id": 471,
+          "id": 472,
           "interval": null,
           "legend": {
             "show": false
@@ -65468,7 +65601,7 @@
           "hideTimeOverride": false,
           "hideZeroBuckets": true,
           "highlightCards": true,
-          "id": 472,
+          "id": 473,
           "interval": null,
           "legend": {
             "show": false
@@ -65566,7 +65699,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 473,
+          "id": 474,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -65714,7 +65847,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 474,
+          "id": 475,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -65847,7 +65980,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 475,
+          "id": 476,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -65980,7 +66113,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 476,
+          "id": 477,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -66128,7 +66261,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 477,
+          "id": 478,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -66264,7 +66397,7 @@
       },
       "height": null,
       "hideTimeOverride": false,
-      "id": 478,
+      "id": 479,
       "interval": null,
       "links": [],
       "maxDataPoints": 100,
@@ -66315,7 +66448,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 479,
+          "id": 480,
           "interval": null,
           "links": [],
           "maxDataPoints": 100,
@@ -66411,7 +66544,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 480,
+          "id": 481,
           "interval": null,
           "links": [],
           "maxDataPoints": 100,
@@ -66486,7 +66619,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 481,
+          "id": 482,
           "interval": null,
           "links": [],
           "maxDataPoints": 100,
@@ -66561,7 +66694,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 482,
+          "id": 483,
           "interval": null,
           "links": [],
           "maxDataPoints": 100,
@@ -66636,7 +66769,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 483,
+          "id": 484,
           "interval": null,
           "links": [],
           "maxDataPoints": 100,
@@ -66711,7 +66844,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 484,
+          "id": 485,
           "interval": null,
           "links": [],
           "maxDataPoints": 100,
@@ -66786,7 +66919,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 485,
+          "id": 486,
           "interval": null,
           "links": [],
           "maxDataPoints": 100,
@@ -66861,7 +66994,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 486,
+          "id": 487,
           "interval": null,
           "links": [],
           "maxDataPoints": 100,
@@ -66940,7 +67073,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 487,
+          "id": 488,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -67073,7 +67206,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 488,
+          "id": 489,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -67206,7 +67339,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 489,
+          "id": 490,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -67339,7 +67472,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 490,
+          "id": 491,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -67472,7 +67605,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 491,
+          "id": 492,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -67605,7 +67738,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 492,
+          "id": 493,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -67753,7 +67886,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 493,
+          "id": 494,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -67886,7 +68019,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 494,
+          "id": 495,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -68019,7 +68152,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 495,
+          "id": 496,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -68185,7 +68318,7 @@
           "hideTimeOverride": false,
           "hideZeroBuckets": true,
           "highlightCards": true,
-          "id": 496,
+          "id": 497,
           "interval": null,
           "legend": {
             "show": false
@@ -68290,7 +68423,7 @@
           "hideTimeOverride": false,
           "hideZeroBuckets": true,
           "highlightCards": true,
-          "id": 497,
+          "id": 498,
           "interval": null,
           "legend": {
             "show": false
@@ -68395,7 +68528,7 @@
           "hideTimeOverride": false,
           "hideZeroBuckets": true,
           "highlightCards": true,
-          "id": 498,
+          "id": 499,
           "interval": null,
           "legend": {
             "show": false
@@ -68500,7 +68633,7 @@
           "hideTimeOverride": false,
           "hideZeroBuckets": true,
           "highlightCards": true,
-          "id": 499,
+          "id": 500,
           "interval": null,
           "legend": {
             "show": false
@@ -68605,7 +68738,7 @@
           "hideTimeOverride": false,
           "hideZeroBuckets": true,
           "highlightCards": true,
-          "id": 500,
+          "id": 501,
           "interval": null,
           "legend": {
             "show": false
@@ -68710,7 +68843,7 @@
           "hideTimeOverride": false,
           "hideZeroBuckets": true,
           "highlightCards": true,
-          "id": 501,
+          "id": 502,
           "interval": null,
           "legend": {
             "show": false
@@ -68815,7 +68948,7 @@
           "hideTimeOverride": false,
           "hideZeroBuckets": true,
           "highlightCards": true,
-          "id": 502,
+          "id": 503,
           "interval": null,
           "legend": {
             "show": false
@@ -68920,7 +69053,7 @@
           "hideTimeOverride": false,
           "hideZeroBuckets": true,
           "highlightCards": true,
-          "id": 503,
+          "id": 504,
           "interval": null,
           "legend": {
             "show": false
@@ -69018,7 +69151,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 504,
+          "id": 505,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -69151,7 +69284,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 505,
+          "id": 506,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -69284,7 +69417,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 506,
+          "id": 507,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -69417,7 +69550,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 507,
+          "id": 508,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -69550,7 +69683,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 508,
+          "id": 509,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -69683,7 +69816,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 509,
+          "id": 510,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -69816,7 +69949,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 510,
+          "id": 511,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -69949,7 +70082,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 511,
+          "id": 512,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -70089,7 +70222,7 @@
           "hideTimeOverride": false,
           "hideZeroBuckets": true,
           "highlightCards": true,
-          "id": 512,
+          "id": 513,
           "interval": null,
           "legend": {
             "show": false
@@ -70194,7 +70327,7 @@
           "hideTimeOverride": false,
           "hideZeroBuckets": true,
           "highlightCards": true,
-          "id": 513,
+          "id": 514,
           "interval": null,
           "legend": {
             "show": false
@@ -70292,7 +70425,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 514,
+          "id": 515,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -70425,7 +70558,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 515,
+          "id": 516,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -70558,7 +70691,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 516,
+          "id": 517,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -70691,7 +70824,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 517,
+          "id": 518,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -70824,7 +70957,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 518,
+          "id": 519,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -70957,7 +71090,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 519,
+          "id": 520,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -71093,7 +71226,7 @@
       },
       "height": null,
       "hideTimeOverride": false,
-      "id": 520,
+      "id": 521,
       "interval": null,
       "links": [],
       "maxDataPoints": 100,
@@ -71132,7 +71265,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 521,
+          "id": 522,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -71280,7 +71413,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 522,
+          "id": 523,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -71413,7 +71546,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 523,
+          "id": 524,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -71546,7 +71679,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 524,
+          "id": 525,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -71682,7 +71815,7 @@
       },
       "height": null,
       "hideTimeOverride": false,
-      "id": 525,
+      "id": 526,
       "interval": null,
       "links": [],
       "maxDataPoints": 100,
@@ -71721,7 +71854,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 526,
+          "id": 527,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -71854,7 +71987,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 527,
+          "id": 528,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -71987,7 +72120,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 528,
+          "id": 529,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -72120,7 +72253,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 529,
+          "id": 530,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -72253,7 +72386,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 530,
+          "id": 531,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -72386,7 +72519,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 531,
+          "id": 532,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -72522,7 +72655,7 @@
       },
       "height": null,
       "hideTimeOverride": false,
-      "id": 532,
+      "id": 533,
       "interval": null,
       "links": [],
       "maxDataPoints": 100,
@@ -72561,7 +72694,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 533,
+          "id": 534,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -72694,7 +72827,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 534,
+          "id": 535,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -72830,7 +72963,7 @@
       },
       "height": null,
       "hideTimeOverride": false,
-      "id": 535,
+      "id": 536,
       "interval": null,
       "links": [],
       "maxDataPoints": 100,
@@ -72869,7 +73002,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 536,
+          "id": 537,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -73070,7 +73203,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 537,
+          "id": 538,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -73206,7 +73339,7 @@
       },
       "height": null,
       "hideTimeOverride": false,
-      "id": 538,
+      "id": 539,
       "interval": null,
       "links": [],
       "maxDataPoints": 100,
@@ -73245,7 +73378,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 539,
+          "id": 540,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -73378,7 +73511,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 540,
+          "id": 541,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -73511,7 +73644,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 541,
+          "id": 542,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -73644,7 +73777,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 542,
+          "id": 543,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -73777,7 +73910,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 543,
+          "id": 544,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -73925,7 +74058,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 544,
+          "id": 545,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -74129,7 +74262,7 @@
       },
       "height": null,
       "hideTimeOverride": false,
-      "id": 545,
+      "id": 546,
       "interval": null,
       "links": [],
       "maxDataPoints": 100,
@@ -74168,7 +74301,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 546,
+          "id": 547,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -74301,7 +74434,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 547,
+          "id": 548,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -74434,7 +74567,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 548,
+          "id": 549,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -74567,7 +74700,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 549,
+          "id": 550,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -74700,7 +74833,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 550,
+          "id": 551,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -74897,7 +75030,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 551,
+          "id": 552,
           "interval": null,
           "links": [],
           "maxDataPoints": 100,
diff --git a/metrics/grafana/tikv_details.json.sha256 b/metrics/grafana/tikv_details.json.sha256
index 583b5f9dd80..2faf9115739 100644
--- a/metrics/grafana/tikv_details.json.sha256
+++ b/metrics/grafana/tikv_details.json.sha256
@@ -1 +1 @@
-45396974bfba476b17c24761788798fd35751eb0714365f30e12b4013a05fd4b  ./metrics/grafana/tikv_details.json
+f470e00da409e47ddb137011b21448eaf3c0698d82803e4431e1be36bb5fb23a  ./metrics/grafana/tikv_details.json

From 4b5b9e506965fe71a562c4060779ee28e612c15c Mon Sep 17 00:00:00 2001
From: glorv <glorvs@163.com>
Date: Fri, 25 Oct 2024 18:22:51 +0800
Subject: [PATCH 08/16] In-memory Engine: fix the panic in write batch prepare
 (#17703)

close tikv/tikv#17700

Call `wb.clear` even if the write batch is empty to avoid leaving
`RegionCacheWriteBatch` in a corrupted state.

Signed-off-by: glorv <glorvs@163.com>

Co-authored-by: ti-chi-bot[bot] <108142056+ti-chi-bot[bot]@users.noreply.github.com>
---
 .../in_memory_engine/src/write_batch.rs       |  17 +++
 components/raftstore/src/store/fsm/apply.rs   |  11 ++
 components/test_raftstore/src/cluster.rs      |   7 +-
 components/test_raftstore/src/node.rs         |   6 +-
 components/test_raftstore/src/server.rs       |   8 +-
 .../failpoints/cases/test_in_memory_engine.rs | 108 +++++++++++++++++-
 6 files changed, 149 insertions(+), 8 deletions(-)

diff --git a/components/in_memory_engine/src/write_batch.rs b/components/in_memory_engine/src/write_batch.rs
index 4ebe0bef20f..19346eecb3c 100644
--- a/components/in_memory_engine/src/write_batch.rs
+++ b/components/in_memory_engine/src/write_batch.rs
@@ -463,6 +463,23 @@ impl WriteBatch for RegionCacheWriteBatch {
     }
 
     fn clear(&mut self) {
+        // `current_region` is some means `write_impl` is not called, so we need to
+        // clear the `in_written` flag.
+        // This can happen when apply fsm do `commit`(e.g. after handling Msg::Change),
+        // and then do not handle other kvs. Thus, the write batch is empty,
+        // and `write_impl` is not called.
+        if self.current_region.is_some() {
+            self.record_last_written_region();
+            // region's `in_written` is not cleaned as `write_impl` is not called,
+            // so we should do it here.
+            if !self.written_regions.is_empty() {
+                self.engine
+                    .core
+                    .region_manager()
+                    .clear_regions_in_being_written(&self.written_regions);
+            }
+        }
+
         self.region_cache_status = RegionCacheStatus::NotInCache;
         self.buffer.clear();
         self.save_points.clear();
diff --git a/components/raftstore/src/store/fsm/apply.rs b/components/raftstore/src/store/fsm/apply.rs
index bceb69acb44..c4002708655 100644
--- a/components/raftstore/src/store/fsm/apply.rs
+++ b/components/raftstore/src/store/fsm/apply.rs
@@ -629,6 +629,16 @@ where
             }
             self.kv_wb_last_bytes = 0;
             self.kv_wb_last_keys = 0;
+        } else {
+            fail_point!(
+                "after_write_to_db_skip_write_node_1",
+                self.store_id == 1,
+                |_| { unreachable!() }
+            );
+            // We call `clear` here because some WriteBatch impl may have some internal
+            // state that need to be reset even if the write batch is empty.
+            // Please refer to `RegionCacheWriteBatch::clear` for more details.
+            self.kv_wb_mut().clear();
         }
         if !self.delete_ssts.is_empty() {
             let tag = self.tag.clone();
@@ -4720,6 +4730,7 @@ where
             }
             handle_result = HandleResult::KeepProcessing;
         }
+        fail_point!("before_handle_normal");
         fail_point!("before_handle_normal_3", normal.delegate.id() == 3, |_| {
             HandleResult::KeepProcessing
         });
diff --git a/components/test_raftstore/src/cluster.rs b/components/test_raftstore/src/cluster.rs
index 28676ed32e3..549c0e35f41 100644
--- a/components/test_raftstore/src/cluster.rs
+++ b/components/test_raftstore/src/cluster.rs
@@ -39,7 +39,7 @@ use raftstore::{
         fsm::{
             create_raft_batch_system,
             store::{StoreMeta, PENDING_MSG_CAP},
-            RaftBatchSystem, RaftRouter,
+            ApplyRouter, RaftBatchSystem, RaftRouter,
         },
         transport::CasualRouter,
         *,
@@ -104,6 +104,7 @@ pub trait Simulator {
     fn get_snap_dir(&self, node_id: u64) -> String;
     fn get_snap_mgr(&self, node_id: u64) -> &SnapManager;
     fn get_router(&self, node_id: u64) -> Option<RaftRouter<RocksEngine, RaftTestEngine>>;
+    fn get_apply_router(&self, node_id: u64) -> Option<ApplyRouter<RocksEngine>>;
     fn add_send_filter(&mut self, node_id: u64, filter: Box<dyn Filter>);
     fn clear_send_filters(&mut self, node_id: u64);
     fn add_recv_filter(&mut self, node_id: u64, filter: Box<dyn Filter>);
@@ -1919,6 +1920,10 @@ impl<T: Simulator> Cluster<T> {
         self.sim.rl().get_router(node_id)
     }
 
+    pub fn get_apply_router(&self, node_id: u64) -> Option<ApplyRouter<RocksEngine>> {
+        self.sim.rl().get_apply_router(node_id)
+    }
+
     pub fn refresh_region_bucket_keys(
         &mut self,
         region: &metapb::Region,
diff --git a/components/test_raftstore/src/node.rs b/components/test_raftstore/src/node.rs
index 305ecebdc3e..26319d43e27 100644
--- a/components/test_raftstore/src/node.rs
+++ b/components/test_raftstore/src/node.rs
@@ -26,7 +26,7 @@ use raftstore::{
     router::{LocalReadRouter, RaftStoreRouter, ReadContext, ServerRaftStoreRouter},
     store::{
         config::RaftstoreConfigManager,
-        fsm::{store::StoreMeta, RaftBatchSystem, RaftRouter},
+        fsm::{store::StoreMeta, ApplyRouter, RaftBatchSystem, RaftRouter},
         SnapManagerBuilder, *,
     },
     Result,
@@ -516,6 +516,10 @@ impl Simulator for NodeCluster {
     fn get_router(&self, node_id: u64) -> Option<RaftRouter<RocksEngine, RaftTestEngine>> {
         self.nodes.get(&node_id).map(|node| node.get_router())
     }
+
+    fn get_apply_router(&self, node_id: u64) -> Option<ApplyRouter<RocksEngine>> {
+        self.nodes.get(&node_id).map(|node| node.get_apply_router())
+    }
 }
 
 // Compare to server cluster, node cluster does not have server layer and
diff --git a/components/test_raftstore/src/server.rs b/components/test_raftstore/src/server.rs
index 408103bc6a9..06be7650ccd 100644
--- a/components/test_raftstore/src/server.rs
+++ b/components/test_raftstore/src/server.rs
@@ -227,10 +227,6 @@ impl ServerCluster {
         self.addrs.get(node_id).unwrap()
     }
 
-    pub fn get_apply_router(&self, node_id: u64) -> ApplyRouter<RocksEngine> {
-        self.metas.get(&node_id).unwrap().raw_apply_router.clone()
-    }
-
     pub fn get_server_router(&self, node_id: u64) -> SimulateStoreTransport {
         self.metas.get(&node_id).unwrap().sim_router.clone()
     }
@@ -888,6 +884,10 @@ impl Simulator for ServerCluster {
     fn get_router(&self, node_id: u64) -> Option<RaftRouter<RocksEngine, RaftTestEngine>> {
         self.metas.get(&node_id).map(|m| m.raw_router.clone())
     }
+
+    fn get_apply_router(&self, node_id: u64) -> Option<ApplyRouter<RocksEngine>> {
+        self.metas.get(&node_id).map(|m| m.raw_apply_router.clone())
+    }
 }
 
 impl Cluster<ServerCluster> {
diff --git a/tests/failpoints/cases/test_in_memory_engine.rs b/tests/failpoints/cases/test_in_memory_engine.rs
index f4e7750192f..a0a5efb4b78 100644
--- a/tests/failpoints/cases/test_in_memory_engine.rs
+++ b/tests/failpoints/cases/test_in_memory_engine.rs
@@ -21,13 +21,20 @@ use kvproto::{
     raft_cmdpb::{AdminCmdType, CmdType, RaftCmdRequest, RaftRequestHeader, Request},
 };
 use protobuf::Message;
+use raftstore::{
+    coprocessor::ObserveHandle,
+    store::{
+        fsm::{apply::ChangeObserver, ApplyTask},
+        msg::Callback,
+    },
+};
 use tempfile::tempdir;
 use test_coprocessor::{
     handle_request, init_data_with_details_pd_client, DagChunkSpliter, DagSelect, ProductTable,
 };
 use test_raftstore::{
-    get_tso, new_peer, new_server_cluster_with_hybrid_engine_with_no_region_cache, Cluster,
-    ServerCluster,
+    get_tso, new_peer, new_put_cf_cmd, new_server_cluster_with_hybrid_engine_with_no_region_cache,
+    Cluster, ServerCluster,
 };
 use test_util::eventually;
 use tidb_query_datatype::{
@@ -858,3 +865,100 @@ fn test_load_during_flashback() {
         );
     }
 }
+
+// This case test that ApplyFsm handles `Msg::Change` at the end of one round.
+// This will let the `flush` call at the end flushes an empty WriteBatch, so so
+// internal state of IME's write-batch may not be cleared.
+#[test]
+fn test_apply_prepared_but_not_write() {
+    let mut cluster = new_server_cluster_with_hybrid_engine_with_no_region_cache(0, 1);
+    cluster.cfg.raft_store.apply_batch_system.pool_size = 1;
+    cluster.run();
+
+    let r = cluster.get_region(b"k");
+    cluster.must_split(&r, b"k10");
+
+    let r1 = cluster.get_region(b"k");
+    let r2 = cluster.get_region(b"k20");
+
+    // load both regions at store 1
+    let region_cache_engine = cluster.sim.rl().get_region_cache_engine(1);
+    region_cache_engine
+        .core()
+        .region_manager()
+        .load_region(CacheRegion::from_region(&r1))
+        .unwrap();
+    region_cache_engine
+        .core()
+        .region_manager()
+        .load_region(CacheRegion::from_region(&r2))
+        .unwrap();
+
+    // first pause apply fsm.
+    fail::cfg("before_handle_normal", "pause").unwrap();
+
+    let (tx, rx) = unbounded();
+    fail::cfg_callback("after_write_to_db_skip_write_node_1", move || {
+        tx.send(()).unwrap();
+    })
+    .unwrap();
+
+    // propose a write to triggers sending a fake Msg::Change.
+    let req = test_raftstore::util::new_request(
+        r2.id,
+        r2.get_region_epoch().clone(),
+        vec![new_put_cf_cmd(CF_DEFAULT, b"k20", b"v1")],
+        false,
+    );
+    cluster
+        .call_command_on_leader(req, Duration::from_millis(20))
+        .unwrap_err();
+
+    // schedule a Msg::Change to trigger a explicit commit, it will lead to a empty
+    // flush at the end of this poll.
+    let apply_router = cluster.get_apply_router(1).unwrap();
+    apply_router.schedule_task(
+        r2.id,
+        ApplyTask::Change {
+            cmd: ChangeObserver::from_rts(r2.id, ObserveHandle::default()),
+            region_epoch: r2.get_region_epoch().clone(),
+            cb: Callback::Read {
+                cb: Box::new(|_| {}),
+                tracker: Default::default(),
+            },
+        },
+    );
+
+    // resume apply fsm to let it handle raft entries and the fake `Change` msg.
+    fail::remove("before_handle_normal");
+    rx.recv_timeout(Duration::from_secs(1)).unwrap();
+
+    // block apply again to batch the write of the 2 regions.
+    fail::cfg("before_handle_normal", "pause").unwrap();
+
+    // propose 2 write for 2 regions.
+    let req = test_raftstore::util::new_request(
+        r1.id,
+        r1.get_region_epoch().clone(),
+        vec![new_put_cf_cmd(CF_DEFAULT, b"k1", b"v2")],
+        false,
+    );
+    cluster
+        .call_command_on_leader(req, Duration::from_millis(10))
+        .unwrap_err();
+    let req = test_raftstore::util::new_request(
+        r2.id,
+        r2.get_region_epoch().clone(),
+        vec![new_put_cf_cmd(CF_DEFAULT, b"k20", b"v2")],
+        false,
+    );
+    cluster
+        .call_command_on_leader(req, Duration::from_millis(10))
+        .unwrap_err();
+
+    // resume apply fsm, should handle new writes successfully.
+    fail::remove("before_handle_normal");
+    fail::remove("after_write_to_db_skip_write_node_1");
+
+    assert_eq!(cluster.must_get(b"k1").unwrap(), b"v2");
+}

From b0e2f87b19b75d0136cfa9c39412fba7a12ff87a Mon Sep 17 00:00:00 2001
From: Spade  A <71589810+SpadeA-Tang@users.noreply.github.com>
Date: Mon, 28 Oct 2024 10:47:04 +0800
Subject: [PATCH 09/16] In-memory engine: copy read throughput and latency
 panels to in-memory engine (#17702)

ref tikv/tikv#16141

copy read throughput and latency panels to in-memory engine

Signed-off-by: SpadeA-Tang <u6748471@anu.edu.au>
---
 metrics/grafana/tikv_details.dashboard.py |   32 +-
 metrics/grafana/tikv_details.json         | 2829 ++++++++++++---------
 metrics/grafana/tikv_details.json.sha256  |    2 +-
 3 files changed, 1621 insertions(+), 1242 deletions(-)

diff --git a/metrics/grafana/tikv_details.dashboard.py b/metrics/grafana/tikv_details.dashboard.py
index 046a902c1ad..b58a7f28b9f 100644
--- a/metrics/grafana/tikv_details.dashboard.py
+++ b/metrics/grafana/tikv_details.dashboard.py
@@ -4388,7 +4388,37 @@ def InMemoryEngine() -> RowPanel:
                         additional_groupby=True,
                     ),
                 ],
-            )
+            ),
+            graph_panel(
+                title="Read MBps",
+                description="The total bytes of read in RocksDB and in-memory engine(the same with panel Cluster/MBps for read)",
+                yaxes=yaxes(left_format=UNITS.BYTES_IEC),
+                targets=[
+                    target(
+                        expr=expr_sum_rate(
+                            "tikv_engine_flow_bytes",
+                            label_selectors=['type=~"bytes_read|iter_bytes_read"'],
+                        ),
+                        legend_format=r"rocksdb-{{instance}}",
+                    ),
+                    target(
+                        expr=expr_sum_rate(
+                            "tikv_in_memory_engine_flow",
+                            label_selectors=['type=~"bytes_read|iter_bytes_read"'],
+                        ),
+                        legend_format=r"in-memory-engine-{{instance}}",
+                    ),
+                ],
+            ),
+            graph_panel_histogram_quantiles(
+                title="Coprocessor Handle duration",
+                description="The time consumed when handling coprocessor requests",
+                yaxes=yaxes(left_format=UNITS.SECONDS),
+                metric="tikv_coprocessor_request_handle_seconds",
+                by_labels=["req"],
+                hide_avg=True,
+                hide_count=True,
+            ),
         ]
     )
     layout.row(
diff --git a/metrics/grafana/tikv_details.json b/metrics/grafana/tikv_details.json
index 100001338fd..04c80e7ddd5 100644
--- a/metrics/grafana/tikv_details.json
+++ b/metrics/grafana/tikv_details.json
@@ -37345,7 +37345,7 @@
           },
           "gridPos": {
             "h": 7,
-            "w": 24,
+            "w": 8,
             "x": 0,
             "y": 0
           },
@@ -37457,7 +37457,7 @@
           "bars": false,
           "cacheTimeout": null,
           "datasource": "${DS_TEST-CLUSTER}",
-          "description": "Count of region cache hit",
+          "description": "The total bytes of read in RocksDB and in-memory engine(the same with panel Cluster/MBps for read)",
           "editable": true,
           "error": false,
           "fieldConfig": {
@@ -37479,8 +37479,8 @@
           "gridPos": {
             "h": 7,
             "w": 8,
-            "x": 0,
-            "y": 7
+            "x": 8,
+            "y": 0
           },
           "height": null,
           "hideTimeOverride": false,
@@ -37527,15 +37527,30 @@
           "targets": [
             {
               "datasource": "${DS_TEST-CLUSTER}",
-              "expr": "sum(rate(\n    tikv_snapshot_type_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"in_memory_engine\"}\n    [$__rate_interval]\n)) by (instance) ",
+              "expr": "sum(rate(\n    tikv_engine_flow_bytes\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=~\"bytes_read|iter_bytes_read\"}\n    [$__rate_interval]\n)) by (instance) ",
               "format": "time_series",
               "hide": false,
               "instant": false,
               "interval": "",
               "intervalFactor": 1,
-              "legendFormat": "count-{{instance}}",
+              "legendFormat": "rocksdb-{{instance}}",
               "metric": "",
-              "query": "sum(rate(\n    tikv_snapshot_type_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"in_memory_engine\"}\n    [$__rate_interval]\n)) by (instance) ",
+              "query": "sum(rate(\n    tikv_engine_flow_bytes\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=~\"bytes_read|iter_bytes_read\"}\n    [$__rate_interval]\n)) by (instance) ",
+              "refId": "",
+              "step": 10,
+              "target": ""
+            },
+            {
+              "datasource": "${DS_TEST-CLUSTER}",
+              "expr": "sum(rate(\n    tikv_in_memory_engine_flow\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=~\"bytes_read|iter_bytes_read\"}\n    [$__rate_interval]\n)) by (instance) ",
+              "format": "time_series",
+              "hide": false,
+              "instant": false,
+              "interval": "",
+              "intervalFactor": 1,
+              "legendFormat": "in-memory-engine-{{instance}}",
+              "metric": "",
+              "query": "sum(rate(\n    tikv_in_memory_engine_flow\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=~\"bytes_read|iter_bytes_read\"}\n    [$__rate_interval]\n)) by (instance) ",
               "refId": "",
               "step": 10,
               "target": ""
@@ -37544,7 +37559,7 @@
           "thresholds": [],
           "timeFrom": null,
           "timeShift": null,
-          "title": "Region Cache Hit",
+          "title": "Read MBps",
           "tooltip": {
             "msResolution": true,
             "shared": true,
@@ -37563,7 +37578,7 @@
           "yaxes": [
             {
               "decimals": null,
-              "format": "none",
+              "format": "bytes",
               "label": null,
               "logBase": 1,
               "max": null,
@@ -37590,7 +37605,7 @@
           "bars": false,
           "cacheTimeout": null,
           "datasource": "${DS_TEST-CLUSTER}",
-          "description": "Region cache hit rate",
+          "description": "The time consumed when handling coprocessor requests",
           "editable": true,
           "error": false,
           "fieldConfig": {
@@ -37612,8 +37627,8 @@
           "gridPos": {
             "h": 7,
             "w": 8,
-            "x": 8,
-            "y": 7
+            "x": 16,
+            "y": 0
           },
           "height": null,
           "hideTimeOverride": false,
@@ -37653,155 +37668,90 @@
           "renderer": "flot",
           "repeat": null,
           "repeatDirection": null,
-          "seriesOverrides": [],
+          "seriesOverrides": [
+            {
+              "alias": "count",
+              "bars": false,
+              "dashLength": 1,
+              "dashes": true,
+              "fill": 2,
+              "fillBelowTo": null,
+              "lines": true,
+              "spaceLength": 1,
+              "transform": "negative-Y",
+              "yaxis": 2,
+              "zindex": -3
+            },
+            {
+              "alias": "avg",
+              "bars": false,
+              "fill": 7,
+              "fillBelowTo": null,
+              "lines": true,
+              "yaxis": 1,
+              "zindex": 0
+            }
+          ],
           "span": null,
           "stack": false,
           "steppedLine": false,
           "targets": [
             {
               "datasource": "${DS_TEST-CLUSTER}",
-              "expr": "(sum(rate(\n    tikv_snapshot_type_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"in_memory_engine\"}\n    [$__rate_interval]\n)) by (instance, $additional_groupby)  / sum(rate(\n    tikv_snapshot_type_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    [$__rate_interval]\n)) by (instance, $additional_groupby) )",
+              "expr": "histogram_quantile(0.9999,(\n    sum(rate(\n    tikv_coprocessor_request_handle_seconds_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    [$__rate_interval]\n)) by (req, le, $additional_groupby) \n    \n    \n))  ",
               "format": "time_series",
               "hide": false,
               "instant": false,
               "interval": "",
               "intervalFactor": 1,
-              "legendFormat": "rate-{{instance}} {{$additional_groupby}}",
+              "legendFormat": "99.99%-{{req}} {{$additional_groupby}}",
               "metric": "",
-              "query": "(sum(rate(\n    tikv_snapshot_type_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"in_memory_engine\"}\n    [$__rate_interval]\n)) by (instance, $additional_groupby)  / sum(rate(\n    tikv_snapshot_type_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    [$__rate_interval]\n)) by (instance, $additional_groupby) )",
+              "query": "histogram_quantile(0.9999,(\n    sum(rate(\n    tikv_coprocessor_request_handle_seconds_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    [$__rate_interval]\n)) by (req, le, $additional_groupby) \n    \n    \n))  ",
               "refId": "",
               "step": 10,
               "target": ""
-            }
-          ],
-          "thresholds": [],
-          "timeFrom": null,
-          "timeShift": null,
-          "title": "Region Cache Hit Rate",
-          "tooltip": {
-            "msResolution": true,
-            "shared": true,
-            "sort": 0,
-            "value_type": "individual"
-          },
-          "transformations": [],
-          "transparent": false,
-          "type": "graph",
-          "xaxis": {
-            "mode": "time",
-            "name": null,
-            "show": true,
-            "values": []
-          },
-          "yaxes": [
+            },
             {
-              "decimals": null,
-              "format": "percentunit",
-              "label": null,
-              "logBase": 1,
-              "max": null,
-              "min": null,
-              "show": true
+              "datasource": "${DS_TEST-CLUSTER}",
+              "expr": "histogram_quantile(0.99,(\n    sum(rate(\n    tikv_coprocessor_request_handle_seconds_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    [$__rate_interval]\n)) by (req, le, $additional_groupby) \n    \n    \n))  ",
+              "format": "time_series",
+              "hide": false,
+              "instant": false,
+              "interval": "",
+              "intervalFactor": 1,
+              "legendFormat": "99%-{{req}} {{$additional_groupby}}",
+              "metric": "",
+              "query": "histogram_quantile(0.99,(\n    sum(rate(\n    tikv_coprocessor_request_handle_seconds_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    [$__rate_interval]\n)) by (req, le, $additional_groupby) \n    \n    \n))  ",
+              "refId": "",
+              "step": 10,
+              "target": ""
             },
             {
-              "decimals": null,
-              "format": "short",
-              "label": null,
-              "logBase": 1,
-              "max": null,
-              "min": null,
-              "show": true
-            }
-          ],
-          "yaxis": {
-            "align": false,
-            "alignLevel": 0
-          }
-        },
-        {
-          "aliasColors": {},
-          "bars": false,
-          "cacheTimeout": null,
-          "datasource": "${DS_TEST-CLUSTER}",
-          "description": "Reasons for region cache miss",
-          "editable": true,
-          "error": false,
-          "fieldConfig": {
-            "defaults": {
-              "thresholds": {
-                "mode": "absolute",
-                "steps": []
-              }
-            }
-          },
-          "fill": 1,
-          "fillGradient": 1,
-          "grid": {
-            "threshold1": null,
-            "threshold1Color": "rgba(216, 200, 27, 0.27)",
-            "threshold2": null,
-            "threshold2Color": "rgba(234, 112, 112, 0.22)"
-          },
-          "gridPos": {
-            "h": 7,
-            "w": 8,
-            "x": 16,
-            "y": 7
-          },
-          "height": null,
-          "hideTimeOverride": false,
-          "id": 264,
-          "interval": null,
-          "isNew": true,
-          "legend": {
-            "alignAsTable": true,
-            "avg": false,
-            "current": true,
-            "hideEmpty": true,
-            "hideZero": true,
-            "max": true,
-            "min": false,
-            "rightSide": true,
-            "show": true,
-            "sideWidth": null,
-            "sort": "max",
-            "sortDesc": true,
-            "total": false,
-            "values": true
-          },
-          "lines": true,
-          "linewidth": 1,
-          "links": [],
-          "maxDataPoints": null,
-          "maxPerRow": null,
-          "minSpan": null,
-          "nullPointMode": "null as zero",
-          "options": {
-            "alertThreshold": true,
-            "dataLinks": []
-          },
-          "percentage": false,
-          "pointradius": 5,
-          "points": false,
-          "renderer": "flot",
-          "repeat": null,
-          "repeatDirection": null,
-          "seriesOverrides": [],
-          "span": null,
-          "stack": false,
-          "steppedLine": false,
-          "targets": [
+              "datasource": "${DS_TEST-CLUSTER}",
+              "expr": "(sum(rate(\n    tikv_coprocessor_request_handle_seconds_sum\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    [$__rate_interval]\n)) by (req, $additional_groupby)  / sum(rate(\n    tikv_coprocessor_request_handle_seconds_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    [$__rate_interval]\n)) by (req, $additional_groupby) )",
+              "format": "time_series",
+              "hide": true,
+              "instant": false,
+              "interval": "",
+              "intervalFactor": 1,
+              "legendFormat": "avg-{{req}} {{$additional_groupby}}",
+              "metric": "",
+              "query": "(sum(rate(\n    tikv_coprocessor_request_handle_seconds_sum\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    [$__rate_interval]\n)) by (req, $additional_groupby)  / sum(rate(\n    tikv_coprocessor_request_handle_seconds_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    [$__rate_interval]\n)) by (req, $additional_groupby) )",
+              "refId": "",
+              "step": 10,
+              "target": ""
+            },
             {
               "datasource": "${DS_TEST-CLUSTER}",
-              "expr": "sum(rate(\n    tikv_in_memory_engine_snapshot_acquire_failed_reason_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    [$__rate_interval]\n)) by (instance, type) ",
+              "expr": "sum(rate(\n    tikv_coprocessor_request_handle_seconds_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    [$__rate_interval]\n)) by (req, $additional_groupby) ",
               "format": "time_series",
-              "hide": false,
+              "hide": true,
               "instant": false,
               "interval": "",
               "intervalFactor": 1,
-              "legendFormat": "{{type}}-{{instance}}",
+              "legendFormat": "count-{{req}} {{$additional_groupby}}",
               "metric": "",
-              "query": "sum(rate(\n    tikv_in_memory_engine_snapshot_acquire_failed_reason_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    [$__rate_interval]\n)) by (instance, type) ",
+              "query": "sum(rate(\n    tikv_coprocessor_request_handle_seconds_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    [$__rate_interval]\n)) by (req, $additional_groupby) ",
               "refId": "",
               "step": 10,
               "target": ""
@@ -37810,7 +37760,7 @@
           "thresholds": [],
           "timeFrom": null,
           "timeShift": null,
-          "title": "Region Cache Miss Reason",
+          "title": "Coprocessor Handle duration",
           "tooltip": {
             "msResolution": true,
             "shared": true,
@@ -37829,7 +37779,7 @@
           "yaxes": [
             {
               "decimals": null,
-              "format": "none",
+              "format": "s",
               "label": null,
               "logBase": 1,
               "max": null,
@@ -37856,7 +37806,7 @@
           "bars": false,
           "cacheTimeout": null,
           "datasource": "${DS_TEST-CLUSTER}",
-          "description": "The memory usage of the in-memory engine",
+          "description": "Count of region cache hit",
           "editable": true,
           "error": false,
           "fieldConfig": {
@@ -37877,13 +37827,13 @@
           },
           "gridPos": {
             "h": 7,
-            "w": 6,
+            "w": 8,
             "x": 0,
-            "y": 14
+            "y": 7
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 265,
+          "id": 264,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -37926,15 +37876,15 @@
           "targets": [
             {
               "datasource": "${DS_TEST-CLUSTER}",
-              "expr": "avg((\n    tikv_in_memory_engine_memory_usage_bytes\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    \n)) by (instance) ",
+              "expr": "sum(rate(\n    tikv_snapshot_type_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"in_memory_engine\"}\n    [$__rate_interval]\n)) by (instance) ",
               "format": "time_series",
               "hide": false,
               "instant": false,
               "interval": "",
               "intervalFactor": 1,
-              "legendFormat": "{{instance}}",
+              "legendFormat": "count-{{instance}}",
               "metric": "",
-              "query": "avg((\n    tikv_in_memory_engine_memory_usage_bytes\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    \n)) by (instance) ",
+              "query": "sum(rate(\n    tikv_snapshot_type_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"in_memory_engine\"}\n    [$__rate_interval]\n)) by (instance) ",
               "refId": "",
               "step": 10,
               "target": ""
@@ -37943,7 +37893,7 @@
           "thresholds": [],
           "timeFrom": null,
           "timeShift": null,
-          "title": "Memory Usage",
+          "title": "Region Cache Hit",
           "tooltip": {
             "msResolution": true,
             "shared": true,
@@ -37962,7 +37912,7 @@
           "yaxes": [
             {
               "decimals": null,
-              "format": "bytes",
+              "format": "none",
               "label": null,
               "logBase": 1,
               "max": null,
@@ -37989,7 +37939,7 @@
           "bars": false,
           "cacheTimeout": null,
           "datasource": "${DS_TEST-CLUSTER}",
-          "description": "The count of different types of region",
+          "description": "Region cache hit rate",
           "editable": true,
           "error": false,
           "fieldConfig": {
@@ -38010,13 +37960,13 @@
           },
           "gridPos": {
             "h": 7,
-            "w": 6,
-            "x": 6,
-            "y": 14
+            "w": 8,
+            "x": 8,
+            "y": 7
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 266,
+          "id": 265,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -38059,15 +38009,15 @@
           "targets": [
             {
               "datasource": "${DS_TEST-CLUSTER}",
-              "expr": "avg((\n    tikv_in_memory_engine_cache_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    \n)) by (instance, type) ",
+              "expr": "(sum(rate(\n    tikv_snapshot_type_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"in_memory_engine\"}\n    [$__rate_interval]\n)) by (instance, $additional_groupby)  / sum(rate(\n    tikv_snapshot_type_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    [$__rate_interval]\n)) by (instance, $additional_groupby) )",
               "format": "time_series",
               "hide": false,
               "instant": false,
               "interval": "",
               "intervalFactor": 1,
-              "legendFormat": "{{instance}}--{{type}}",
+              "legendFormat": "rate-{{instance}} {{$additional_groupby}}",
               "metric": "",
-              "query": "avg((\n    tikv_in_memory_engine_cache_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    \n)) by (instance, type) ",
+              "query": "(sum(rate(\n    tikv_snapshot_type_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"in_memory_engine\"}\n    [$__rate_interval]\n)) by (instance, $additional_groupby)  / sum(rate(\n    tikv_snapshot_type_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    [$__rate_interval]\n)) by (instance, $additional_groupby) )",
               "refId": "",
               "step": 10,
               "target": ""
@@ -38076,7 +38026,7 @@
           "thresholds": [],
           "timeFrom": null,
           "timeShift": null,
-          "title": "Region Count",
+          "title": "Region Cache Hit Rate",
           "tooltip": {
             "msResolution": true,
             "shared": true,
@@ -38095,7 +38045,406 @@
           "yaxes": [
             {
               "decimals": null,
-              "format": "none",
+              "format": "percentunit",
+              "label": null,
+              "logBase": 1,
+              "max": null,
+              "min": null,
+              "show": true
+            },
+            {
+              "decimals": null,
+              "format": "short",
+              "label": null,
+              "logBase": 1,
+              "max": null,
+              "min": null,
+              "show": true
+            }
+          ],
+          "yaxis": {
+            "align": false,
+            "alignLevel": 0
+          }
+        },
+        {
+          "aliasColors": {},
+          "bars": false,
+          "cacheTimeout": null,
+          "datasource": "${DS_TEST-CLUSTER}",
+          "description": "Reasons for region cache miss",
+          "editable": true,
+          "error": false,
+          "fieldConfig": {
+            "defaults": {
+              "thresholds": {
+                "mode": "absolute",
+                "steps": []
+              }
+            }
+          },
+          "fill": 1,
+          "fillGradient": 1,
+          "grid": {
+            "threshold1": null,
+            "threshold1Color": "rgba(216, 200, 27, 0.27)",
+            "threshold2": null,
+            "threshold2Color": "rgba(234, 112, 112, 0.22)"
+          },
+          "gridPos": {
+            "h": 7,
+            "w": 8,
+            "x": 16,
+            "y": 7
+          },
+          "height": null,
+          "hideTimeOverride": false,
+          "id": 266,
+          "interval": null,
+          "isNew": true,
+          "legend": {
+            "alignAsTable": true,
+            "avg": false,
+            "current": true,
+            "hideEmpty": true,
+            "hideZero": true,
+            "max": true,
+            "min": false,
+            "rightSide": true,
+            "show": true,
+            "sideWidth": null,
+            "sort": "max",
+            "sortDesc": true,
+            "total": false,
+            "values": true
+          },
+          "lines": true,
+          "linewidth": 1,
+          "links": [],
+          "maxDataPoints": null,
+          "maxPerRow": null,
+          "minSpan": null,
+          "nullPointMode": "null as zero",
+          "options": {
+            "alertThreshold": true,
+            "dataLinks": []
+          },
+          "percentage": false,
+          "pointradius": 5,
+          "points": false,
+          "renderer": "flot",
+          "repeat": null,
+          "repeatDirection": null,
+          "seriesOverrides": [],
+          "span": null,
+          "stack": false,
+          "steppedLine": false,
+          "targets": [
+            {
+              "datasource": "${DS_TEST-CLUSTER}",
+              "expr": "sum(rate(\n    tikv_in_memory_engine_snapshot_acquire_failed_reason_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    [$__rate_interval]\n)) by (instance, type) ",
+              "format": "time_series",
+              "hide": false,
+              "instant": false,
+              "interval": "",
+              "intervalFactor": 1,
+              "legendFormat": "{{type}}-{{instance}}",
+              "metric": "",
+              "query": "sum(rate(\n    tikv_in_memory_engine_snapshot_acquire_failed_reason_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    [$__rate_interval]\n)) by (instance, type) ",
+              "refId": "",
+              "step": 10,
+              "target": ""
+            }
+          ],
+          "thresholds": [],
+          "timeFrom": null,
+          "timeShift": null,
+          "title": "Region Cache Miss Reason",
+          "tooltip": {
+            "msResolution": true,
+            "shared": true,
+            "sort": 0,
+            "value_type": "individual"
+          },
+          "transformations": [],
+          "transparent": false,
+          "type": "graph",
+          "xaxis": {
+            "mode": "time",
+            "name": null,
+            "show": true,
+            "values": []
+          },
+          "yaxes": [
+            {
+              "decimals": null,
+              "format": "none",
+              "label": null,
+              "logBase": 1,
+              "max": null,
+              "min": null,
+              "show": true
+            },
+            {
+              "decimals": null,
+              "format": "short",
+              "label": null,
+              "logBase": 1,
+              "max": null,
+              "min": null,
+              "show": true
+            }
+          ],
+          "yaxis": {
+            "align": false,
+            "alignLevel": 0
+          }
+        },
+        {
+          "aliasColors": {},
+          "bars": false,
+          "cacheTimeout": null,
+          "datasource": "${DS_TEST-CLUSTER}",
+          "description": "The memory usage of the in-memory engine",
+          "editable": true,
+          "error": false,
+          "fieldConfig": {
+            "defaults": {
+              "thresholds": {
+                "mode": "absolute",
+                "steps": []
+              }
+            }
+          },
+          "fill": 1,
+          "fillGradient": 1,
+          "grid": {
+            "threshold1": null,
+            "threshold1Color": "rgba(216, 200, 27, 0.27)",
+            "threshold2": null,
+            "threshold2Color": "rgba(234, 112, 112, 0.22)"
+          },
+          "gridPos": {
+            "h": 7,
+            "w": 6,
+            "x": 0,
+            "y": 14
+          },
+          "height": null,
+          "hideTimeOverride": false,
+          "id": 267,
+          "interval": null,
+          "isNew": true,
+          "legend": {
+            "alignAsTable": true,
+            "avg": false,
+            "current": true,
+            "hideEmpty": true,
+            "hideZero": true,
+            "max": true,
+            "min": false,
+            "rightSide": true,
+            "show": true,
+            "sideWidth": null,
+            "sort": "max",
+            "sortDesc": true,
+            "total": false,
+            "values": true
+          },
+          "lines": true,
+          "linewidth": 1,
+          "links": [],
+          "maxDataPoints": null,
+          "maxPerRow": null,
+          "minSpan": null,
+          "nullPointMode": "null as zero",
+          "options": {
+            "alertThreshold": true,
+            "dataLinks": []
+          },
+          "percentage": false,
+          "pointradius": 5,
+          "points": false,
+          "renderer": "flot",
+          "repeat": null,
+          "repeatDirection": null,
+          "seriesOverrides": [],
+          "span": null,
+          "stack": false,
+          "steppedLine": false,
+          "targets": [
+            {
+              "datasource": "${DS_TEST-CLUSTER}",
+              "expr": "avg((\n    tikv_in_memory_engine_memory_usage_bytes\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    \n)) by (instance) ",
+              "format": "time_series",
+              "hide": false,
+              "instant": false,
+              "interval": "",
+              "intervalFactor": 1,
+              "legendFormat": "{{instance}}",
+              "metric": "",
+              "query": "avg((\n    tikv_in_memory_engine_memory_usage_bytes\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    \n)) by (instance) ",
+              "refId": "",
+              "step": 10,
+              "target": ""
+            }
+          ],
+          "thresholds": [],
+          "timeFrom": null,
+          "timeShift": null,
+          "title": "Memory Usage",
+          "tooltip": {
+            "msResolution": true,
+            "shared": true,
+            "sort": 0,
+            "value_type": "individual"
+          },
+          "transformations": [],
+          "transparent": false,
+          "type": "graph",
+          "xaxis": {
+            "mode": "time",
+            "name": null,
+            "show": true,
+            "values": []
+          },
+          "yaxes": [
+            {
+              "decimals": null,
+              "format": "bytes",
+              "label": null,
+              "logBase": 1,
+              "max": null,
+              "min": null,
+              "show": true
+            },
+            {
+              "decimals": null,
+              "format": "short",
+              "label": null,
+              "logBase": 1,
+              "max": null,
+              "min": null,
+              "show": true
+            }
+          ],
+          "yaxis": {
+            "align": false,
+            "alignLevel": 0
+          }
+        },
+        {
+          "aliasColors": {},
+          "bars": false,
+          "cacheTimeout": null,
+          "datasource": "${DS_TEST-CLUSTER}",
+          "description": "The count of different types of region",
+          "editable": true,
+          "error": false,
+          "fieldConfig": {
+            "defaults": {
+              "thresholds": {
+                "mode": "absolute",
+                "steps": []
+              }
+            }
+          },
+          "fill": 1,
+          "fillGradient": 1,
+          "grid": {
+            "threshold1": null,
+            "threshold1Color": "rgba(216, 200, 27, 0.27)",
+            "threshold2": null,
+            "threshold2Color": "rgba(234, 112, 112, 0.22)"
+          },
+          "gridPos": {
+            "h": 7,
+            "w": 6,
+            "x": 6,
+            "y": 14
+          },
+          "height": null,
+          "hideTimeOverride": false,
+          "id": 268,
+          "interval": null,
+          "isNew": true,
+          "legend": {
+            "alignAsTable": true,
+            "avg": false,
+            "current": true,
+            "hideEmpty": true,
+            "hideZero": true,
+            "max": true,
+            "min": false,
+            "rightSide": true,
+            "show": true,
+            "sideWidth": null,
+            "sort": "max",
+            "sortDesc": true,
+            "total": false,
+            "values": true
+          },
+          "lines": true,
+          "linewidth": 1,
+          "links": [],
+          "maxDataPoints": null,
+          "maxPerRow": null,
+          "minSpan": null,
+          "nullPointMode": "null as zero",
+          "options": {
+            "alertThreshold": true,
+            "dataLinks": []
+          },
+          "percentage": false,
+          "pointradius": 5,
+          "points": false,
+          "renderer": "flot",
+          "repeat": null,
+          "repeatDirection": null,
+          "seriesOverrides": [],
+          "span": null,
+          "stack": false,
+          "steppedLine": false,
+          "targets": [
+            {
+              "datasource": "${DS_TEST-CLUSTER}",
+              "expr": "avg((\n    tikv_in_memory_engine_cache_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    \n)) by (instance, type) ",
+              "format": "time_series",
+              "hide": false,
+              "instant": false,
+              "interval": "",
+              "intervalFactor": 1,
+              "legendFormat": "{{instance}}--{{type}}",
+              "metric": "",
+              "query": "avg((\n    tikv_in_memory_engine_cache_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    \n)) by (instance, type) ",
+              "refId": "",
+              "step": 10,
+              "target": ""
+            }
+          ],
+          "thresholds": [],
+          "timeFrom": null,
+          "timeShift": null,
+          "title": "Region Count",
+          "tooltip": {
+            "msResolution": true,
+            "shared": true,
+            "sort": 0,
+            "value_type": "individual"
+          },
+          "transformations": [],
+          "transparent": false,
+          "type": "graph",
+          "xaxis": {
+            "mode": "time",
+            "name": null,
+            "show": true,
+            "values": []
+          },
+          "yaxes": [
+            {
+              "decimals": null,
+              "format": "none",
               "label": null,
               "logBase": 1,
               "max": null,
@@ -38149,7 +38498,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 267,
+          "id": 269,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -38289,7 +38638,7 @@
           "hideTimeOverride": false,
           "hideZeroBuckets": true,
           "highlightCards": true,
-          "id": 268,
+          "id": 270,
           "interval": null,
           "legend": {
             "show": false
@@ -38394,7 +38743,7 @@
           "hideTimeOverride": false,
           "hideZeroBuckets": true,
           "highlightCards": true,
-          "id": 269,
+          "id": 271,
           "interval": null,
           "legend": {
             "show": false
@@ -38492,7 +38841,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 270,
+          "id": 272,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -38632,7 +38981,7 @@
           "hideTimeOverride": false,
           "hideZeroBuckets": true,
           "highlightCards": true,
-          "id": 271,
+          "id": 273,
           "interval": null,
           "legend": {
             "show": false
@@ -38730,7 +39079,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 272,
+          "id": 274,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -38870,7 +39219,7 @@
           "hideTimeOverride": false,
           "hideZeroBuckets": true,
           "highlightCards": true,
-          "id": 273,
+          "id": 275,
           "interval": null,
           "legend": {
             "show": false
@@ -38968,7 +39317,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 274,
+          "id": 276,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -39176,7 +39525,7 @@
           "hideTimeOverride": false,
           "hideZeroBuckets": true,
           "highlightCards": true,
-          "id": 275,
+          "id": 277,
           "interval": null,
           "legend": {
             "show": false
@@ -39274,7 +39623,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 276,
+          "id": 278,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -39475,7 +39824,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 277,
+          "id": 279,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -39683,7 +40032,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 278,
+          "id": 280,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -39861,7 +40210,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 279,
+          "id": 281,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -39994,7 +40343,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 280,
+          "id": 282,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -40127,7 +40476,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 281,
+          "id": 283,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -40260,7 +40609,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 282,
+          "id": 284,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -40396,7 +40745,7 @@
       },
       "height": null,
       "hideTimeOverride": false,
-      "id": 283,
+      "id": 285,
       "interval": null,
       "links": [],
       "maxDataPoints": 100,
@@ -40435,7 +40784,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 284,
+          "id": 286,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -40583,7 +40932,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 285,
+          "id": 287,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -40723,7 +41072,7 @@
           "hideTimeOverride": false,
           "hideZeroBuckets": true,
           "highlightCards": true,
-          "id": 286,
+          "id": 288,
           "interval": null,
           "legend": {
             "show": false
@@ -40821,7 +41170,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 287,
+          "id": 289,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -40954,7 +41303,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 288,
+          "id": 290,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -41087,7 +41436,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 289,
+          "id": 291,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -41265,170 +41614,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 290,
-          "interval": null,
-          "isNew": true,
-          "legend": {
-            "alignAsTable": true,
-            "avg": false,
-            "current": true,
-            "hideEmpty": true,
-            "hideZero": true,
-            "max": true,
-            "min": false,
-            "rightSide": true,
-            "show": true,
-            "sideWidth": null,
-            "sort": "max",
-            "sortDesc": true,
-            "total": false,
-            "values": true
-          },
-          "lines": true,
-          "linewidth": 1,
-          "links": [],
-          "maxDataPoints": null,
-          "maxPerRow": null,
-          "minSpan": null,
-          "nullPointMode": "null as zero",
-          "options": {
-            "alertThreshold": true,
-            "dataLinks": []
-          },
-          "percentage": false,
-          "pointradius": 5,
-          "points": false,
-          "renderer": "flot",
-          "repeat": null,
-          "repeatDirection": null,
-          "seriesOverrides": [],
-          "span": null,
-          "stack": false,
-          "steppedLine": false,
-          "targets": [
-            {
-              "datasource": "${DS_TEST-CLUSTER}",
-              "expr": "max((\n    tikv_scheduler_l0\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    \n)) by (instance) ",
-              "format": "time_series",
-              "hide": false,
-              "instant": false,
-              "interval": "",
-              "intervalFactor": 1,
-              "legendFormat": "l0-{{instance}}",
-              "metric": "",
-              "query": "max((\n    tikv_scheduler_l0\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    \n)) by (instance) ",
-              "refId": "",
-              "step": 10,
-              "target": ""
-            },
-            {
-              "datasource": "${DS_TEST-CLUSTER}",
-              "expr": "max((\n    tikv_scheduler_memtable\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    \n)) by (instance) ",
-              "format": "time_series",
-              "hide": false,
-              "instant": false,
-              "interval": "",
-              "intervalFactor": 1,
-              "legendFormat": "memtable-{{instance}}",
-              "metric": "",
-              "query": "max((\n    tikv_scheduler_memtable\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    \n)) by (instance) ",
-              "refId": "",
-              "step": 10,
-              "target": ""
-            },
-            {
-              "datasource": "${DS_TEST-CLUSTER}",
-              "expr": "max((\n    tikv_scheduler_l0_avg\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    \n)) by (instance) ",
-              "format": "time_series",
-              "hide": false,
-              "instant": false,
-              "interval": "",
-              "intervalFactor": 1,
-              "legendFormat": "avg_l0-{{instance}}",
-              "metric": "",
-              "query": "max((\n    tikv_scheduler_l0_avg\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    \n)) by (instance) ",
-              "refId": "",
-              "step": 10,
-              "target": ""
-            }
-          ],
-          "thresholds": [],
-          "timeFrom": null,
-          "timeShift": null,
-          "title": "Flow controller factors",
-          "tooltip": {
-            "msResolution": true,
-            "shared": true,
-            "sort": 0,
-            "value_type": "individual"
-          },
-          "transformations": [],
-          "transparent": false,
-          "type": "graph",
-          "xaxis": {
-            "mode": "time",
-            "name": null,
-            "show": true,
-            "values": []
-          },
-          "yaxes": [
-            {
-              "decimals": null,
-              "format": "none",
-              "label": null,
-              "logBase": 1,
-              "max": null,
-              "min": null,
-              "show": true
-            },
-            {
-              "decimals": null,
-              "format": "short",
-              "label": null,
-              "logBase": 1,
-              "max": null,
-              "min": null,
-              "show": true
-            }
-          ],
-          "yaxis": {
-            "align": false,
-            "alignLevel": 0
-          }
-        },
-        {
-          "aliasColors": {},
-          "bars": false,
-          "cacheTimeout": null,
-          "datasource": "${DS_TEST-CLUSTER}",
-          "description": "",
-          "editable": true,
-          "error": false,
-          "fieldConfig": {
-            "defaults": {
-              "thresholds": {
-                "mode": "absolute",
-                "steps": []
-              }
-            }
-          },
-          "fill": 1,
-          "fillGradient": 1,
-          "grid": {
-            "threshold1": null,
-            "threshold1Color": "rgba(216, 200, 27, 0.27)",
-            "threshold2": null,
-            "threshold2Color": "rgba(234, 112, 112, 0.22)"
-          },
-          "gridPos": {
-            "h": 7,
-            "w": 12,
-            "x": 12,
-            "y": 21
-          },
-          "height": null,
-          "hideTimeOverride": false,
-          "id": 291,
+          "id": 292,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -41471,30 +41657,45 @@
           "targets": [
             {
               "datasource": "${DS_TEST-CLUSTER}",
-              "expr": "sum((\n    tikv_engine_pending_compaction_bytes\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"kv\"}\n    \n)) by (cf, $additional_groupby) ",
+              "expr": "max((\n    tikv_scheduler_l0\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    \n)) by (instance) ",
               "format": "time_series",
               "hide": false,
               "instant": false,
               "interval": "",
               "intervalFactor": 1,
-              "legendFormat": "{{cf}} {{$additional_groupby}}",
+              "legendFormat": "l0-{{instance}}",
               "metric": "",
-              "query": "sum((\n    tikv_engine_pending_compaction_bytes\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"kv\"}\n    \n)) by (cf, $additional_groupby) ",
+              "query": "max((\n    tikv_scheduler_l0\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    \n)) by (instance) ",
               "refId": "",
               "step": 10,
               "target": ""
             },
             {
               "datasource": "${DS_TEST-CLUSTER}",
-              "expr": "sum((\n    tikv_scheduler_pending_compaction_bytes\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    \n)) by (cf, $additional_groupby)  / 10000000",
+              "expr": "max((\n    tikv_scheduler_memtable\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    \n)) by (instance) ",
               "format": "time_series",
-              "hide": true,
+              "hide": false,
               "instant": false,
               "interval": "",
               "intervalFactor": 1,
-              "legendFormat": "pending-bytes {{$additional_groupby}}",
+              "legendFormat": "memtable-{{instance}}",
               "metric": "",
-              "query": "sum((\n    tikv_scheduler_pending_compaction_bytes\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    \n)) by (cf, $additional_groupby)  / 10000000",
+              "query": "max((\n    tikv_scheduler_memtable\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    \n)) by (instance) ",
+              "refId": "",
+              "step": 10,
+              "target": ""
+            },
+            {
+              "datasource": "${DS_TEST-CLUSTER}",
+              "expr": "max((\n    tikv_scheduler_l0_avg\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    \n)) by (instance) ",
+              "format": "time_series",
+              "hide": false,
+              "instant": false,
+              "interval": "",
+              "intervalFactor": 1,
+              "legendFormat": "avg_l0-{{instance}}",
+              "metric": "",
+              "query": "max((\n    tikv_scheduler_l0_avg\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    \n)) by (instance) ",
               "refId": "",
               "step": 10,
               "target": ""
@@ -41503,7 +41704,7 @@
           "thresholds": [],
           "timeFrom": null,
           "timeShift": null,
-          "title": "Compaction pending bytes",
+          "title": "Flow controller factors",
           "tooltip": {
             "msResolution": true,
             "shared": true,
@@ -41522,7 +41723,7 @@
           "yaxes": [
             {
               "decimals": null,
-              "format": "bytes",
+              "format": "none",
               "label": null,
               "logBase": 1,
               "max": null,
@@ -41549,7 +41750,7 @@
           "bars": false,
           "cacheTimeout": null,
           "datasource": "${DS_TEST-CLUSTER}",
-          "description": "Throttle time for txn storage commands in 1 minute.",
+          "description": "",
           "editable": true,
           "error": false,
           "fieldConfig": {
@@ -41571,12 +41772,12 @@
           "gridPos": {
             "h": 7,
             "w": 12,
-            "x": 0,
-            "y": 28
+            "x": 12,
+            "y": 21
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 292,
+          "id": 293,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -41619,15 +41820,30 @@
           "targets": [
             {
               "datasource": "${DS_TEST-CLUSTER}",
-              "expr": "sum(rate(\n    tikv_txn_command_throttle_time_total\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    [$__rate_interval]\n)) by (type, $additional_groupby) ",
+              "expr": "sum((\n    tikv_engine_pending_compaction_bytes\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"kv\"}\n    \n)) by (cf, $additional_groupby) ",
               "format": "time_series",
               "hide": false,
               "instant": false,
               "interval": "",
               "intervalFactor": 1,
-              "legendFormat": "{{type}} {{$additional_groupby}}",
+              "legendFormat": "{{cf}} {{$additional_groupby}}",
               "metric": "",
-              "query": "sum(rate(\n    tikv_txn_command_throttle_time_total\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    [$__rate_interval]\n)) by (type, $additional_groupby) ",
+              "query": "sum((\n    tikv_engine_pending_compaction_bytes\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"kv\"}\n    \n)) by (cf, $additional_groupby) ",
+              "refId": "",
+              "step": 10,
+              "target": ""
+            },
+            {
+              "datasource": "${DS_TEST-CLUSTER}",
+              "expr": "sum((\n    tikv_scheduler_pending_compaction_bytes\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    \n)) by (cf, $additional_groupby)  / 10000000",
+              "format": "time_series",
+              "hide": true,
+              "instant": false,
+              "interval": "",
+              "intervalFactor": 1,
+              "legendFormat": "pending-bytes {{$additional_groupby}}",
+              "metric": "",
+              "query": "sum((\n    tikv_scheduler_pending_compaction_bytes\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    \n)) by (cf, $additional_groupby)  / 10000000",
               "refId": "",
               "step": 10,
               "target": ""
@@ -41636,7 +41852,7 @@
           "thresholds": [],
           "timeFrom": null,
           "timeShift": null,
-          "title": "Txn command throttled duration",
+          "title": "Compaction pending bytes",
           "tooltip": {
             "msResolution": true,
             "shared": true,
@@ -41655,7 +41871,7 @@
           "yaxes": [
             {
               "decimals": null,
-              "format": "\u00b5s",
+              "format": "bytes",
               "label": null,
               "logBase": 1,
               "max": null,
@@ -41682,7 +41898,7 @@
           "bars": false,
           "cacheTimeout": null,
           "datasource": "${DS_TEST-CLUSTER}",
-          "description": "Throttle time for non-txn related processing like analyze or dag in 1 minute.",
+          "description": "Throttle time for txn storage commands in 1 minute.",
           "editable": true,
           "error": false,
           "fieldConfig": {
@@ -41704,12 +41920,12 @@
           "gridPos": {
             "h": 7,
             "w": 12,
-            "x": 12,
+            "x": 0,
             "y": 28
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 293,
+          "id": 294,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -41752,7 +41968,7 @@
           "targets": [
             {
               "datasource": "${DS_TEST-CLUSTER}",
-              "expr": "sum(rate(\n    tikv_non_txn_command_throttle_time_total\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    [$__rate_interval]\n)) by (type, $additional_groupby) ",
+              "expr": "sum(rate(\n    tikv_txn_command_throttle_time_total\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    [$__rate_interval]\n)) by (type, $additional_groupby) ",
               "format": "time_series",
               "hide": false,
               "instant": false,
@@ -41760,7 +41976,7 @@
               "intervalFactor": 1,
               "legendFormat": "{{type}} {{$additional_groupby}}",
               "metric": "",
-              "query": "sum(rate(\n    tikv_non_txn_command_throttle_time_total\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    [$__rate_interval]\n)) by (type, $additional_groupby) ",
+              "query": "sum(rate(\n    tikv_txn_command_throttle_time_total\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    [$__rate_interval]\n)) by (type, $additional_groupby) ",
               "refId": "",
               "step": 10,
               "target": ""
@@ -41769,7 +41985,7 @@
           "thresholds": [],
           "timeFrom": null,
           "timeShift": null,
-          "title": "Non-txn command throttled duration",
+          "title": "Txn command throttled duration",
           "tooltip": {
             "msResolution": true,
             "shared": true,
@@ -41809,55 +42025,13 @@
             "align": false,
             "alignLevel": 0
           }
-        }
-      ],
-      "repeat": null,
-      "repeatDirection": null,
-      "span": null,
-      "targets": [],
-      "timeFrom": null,
-      "timeShift": null,
-      "title": "Flow Control",
-      "transformations": [],
-      "transparent": false,
-      "type": "row"
-    },
-    {
-      "cacheTimeout": null,
-      "collapsed": true,
-      "datasource": null,
-      "description": null,
-      "editable": true,
-      "error": false,
-      "fieldConfig": {
-        "defaults": {
-          "thresholds": {
-            "mode": "absolute",
-            "steps": []
-          }
-        }
-      },
-      "gridPos": {
-        "h": 7,
-        "w": 24,
-        "x": 0,
-        "y": 0
-      },
-      "height": null,
-      "hideTimeOverride": false,
-      "id": 294,
-      "interval": null,
-      "links": [],
-      "maxDataPoints": 100,
-      "maxPerRow": null,
-      "minSpan": null,
-      "panels": [
+        },
         {
           "aliasColors": {},
           "bars": false,
           "cacheTimeout": null,
           "datasource": "${DS_TEST-CLUSTER}",
-          "description": "The total number of commands on each stage",
+          "description": "Throttle time for non-txn related processing like analyze or dag in 1 minute.",
           "editable": true,
           "error": false,
           "fieldConfig": {
@@ -41879,8 +42053,8 @@
           "gridPos": {
             "h": 7,
             "w": 12,
-            "x": 0,
-            "y": 0
+            "x": 12,
+            "y": 28
           },
           "height": null,
           "hideTimeOverride": false,
@@ -41927,30 +42101,15 @@
           "targets": [
             {
               "datasource": "${DS_TEST-CLUSTER}",
-              "expr": "sum(rate(\n    tikv_scheduler_too_busy_total\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    [$__rate_interval]\n)) by (stage, $additional_groupby) ",
-              "format": "time_series",
-              "hide": false,
-              "instant": false,
-              "interval": "",
-              "intervalFactor": 1,
-              "legendFormat": "{{stage}} {{$additional_groupby}}",
-              "metric": "",
-              "query": "sum(rate(\n    tikv_scheduler_too_busy_total\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    [$__rate_interval]\n)) by (stage, $additional_groupby) ",
-              "refId": "",
-              "step": 10,
-              "target": ""
-            },
-            {
-              "datasource": "${DS_TEST-CLUSTER}",
-              "expr": "sum(rate(\n    tikv_scheduler_stage_total\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    [$__rate_interval]\n)) by (stage, $additional_groupby) ",
+              "expr": "sum(rate(\n    tikv_non_txn_command_throttle_time_total\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    [$__rate_interval]\n)) by (type, $additional_groupby) ",
               "format": "time_series",
               "hide": false,
               "instant": false,
               "interval": "",
               "intervalFactor": 1,
-              "legendFormat": "{{stage}} {{$additional_groupby}}",
+              "legendFormat": "{{type}} {{$additional_groupby}}",
               "metric": "",
-              "query": "sum(rate(\n    tikv_scheduler_stage_total\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    [$__rate_interval]\n)) by (stage, $additional_groupby) ",
+              "query": "sum(rate(\n    tikv_non_txn_command_throttle_time_total\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    [$__rate_interval]\n)) by (type, $additional_groupby) ",
               "refId": "",
               "step": 10,
               "target": ""
@@ -41959,7 +42118,7 @@
           "thresholds": [],
           "timeFrom": null,
           "timeShift": null,
-          "title": "Scheduler stage total",
+          "title": "Non-txn command throttled duration",
           "tooltip": {
             "msResolution": true,
             "shared": true,
@@ -41978,7 +42137,7 @@
           "yaxes": [
             {
               "decimals": null,
-              "format": "ops",
+              "format": "\u00b5s",
               "label": null,
               "logBase": 1,
               "max": null,
@@ -41999,13 +42158,55 @@
             "align": false,
             "alignLevel": 0
           }
-        },
+        }
+      ],
+      "repeat": null,
+      "repeatDirection": null,
+      "span": null,
+      "targets": [],
+      "timeFrom": null,
+      "timeShift": null,
+      "title": "Flow Control",
+      "transformations": [],
+      "transparent": false,
+      "type": "row"
+    },
+    {
+      "cacheTimeout": null,
+      "collapsed": true,
+      "datasource": null,
+      "description": null,
+      "editable": true,
+      "error": false,
+      "fieldConfig": {
+        "defaults": {
+          "thresholds": {
+            "mode": "absolute",
+            "steps": []
+          }
+        }
+      },
+      "gridPos": {
+        "h": 7,
+        "w": 24,
+        "x": 0,
+        "y": 0
+      },
+      "height": null,
+      "hideTimeOverride": false,
+      "id": 296,
+      "interval": null,
+      "links": [],
+      "maxDataPoints": 100,
+      "maxPerRow": null,
+      "minSpan": null,
+      "panels": [
         {
           "aliasColors": {},
           "bars": false,
           "cacheTimeout": null,
           "datasource": "${DS_TEST-CLUSTER}",
-          "description": "The count of different priority commands",
+          "description": "The total number of commands on each stage",
           "editable": true,
           "error": false,
           "fieldConfig": {
@@ -42027,12 +42228,12 @@
           "gridPos": {
             "h": 7,
             "w": 12,
-            "x": 12,
+            "x": 0,
             "y": 0
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 296,
+          "id": 297,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -42075,15 +42276,30 @@
           "targets": [
             {
               "datasource": "${DS_TEST-CLUSTER}",
-              "expr": "sum(rate(\n    tikv_scheduler_commands_pri_total\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    [$__rate_interval]\n)) by (priority, $additional_groupby) ",
+              "expr": "sum(rate(\n    tikv_scheduler_too_busy_total\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    [$__rate_interval]\n)) by (stage, $additional_groupby) ",
               "format": "time_series",
               "hide": false,
               "instant": false,
               "interval": "",
               "intervalFactor": 1,
-              "legendFormat": "{{priority}} {{$additional_groupby}}",
+              "legendFormat": "{{stage}} {{$additional_groupby}}",
               "metric": "",
-              "query": "sum(rate(\n    tikv_scheduler_commands_pri_total\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    [$__rate_interval]\n)) by (priority, $additional_groupby) ",
+              "query": "sum(rate(\n    tikv_scheduler_too_busy_total\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    [$__rate_interval]\n)) by (stage, $additional_groupby) ",
+              "refId": "",
+              "step": 10,
+              "target": ""
+            },
+            {
+              "datasource": "${DS_TEST-CLUSTER}",
+              "expr": "sum(rate(\n    tikv_scheduler_stage_total\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    [$__rate_interval]\n)) by (stage, $additional_groupby) ",
+              "format": "time_series",
+              "hide": false,
+              "instant": false,
+              "interval": "",
+              "intervalFactor": 1,
+              "legendFormat": "{{stage}} {{$additional_groupby}}",
+              "metric": "",
+              "query": "sum(rate(\n    tikv_scheduler_stage_total\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    [$__rate_interval]\n)) by (stage, $additional_groupby) ",
               "refId": "",
               "step": 10,
               "target": ""
@@ -42092,7 +42308,7 @@
           "thresholds": [],
           "timeFrom": null,
           "timeShift": null,
-          "title": "Scheduler priority commands",
+          "title": "Scheduler stage total",
           "tooltip": {
             "msResolution": true,
             "shared": true,
@@ -42138,7 +42354,7 @@
           "bars": false,
           "cacheTimeout": null,
           "datasource": "${DS_TEST-CLUSTER}",
-          "description": "The count of pending commands per TiKV instance",
+          "description": "The count of different priority commands",
           "editable": true,
           "error": false,
           "fieldConfig": {
@@ -42160,12 +42376,12 @@
           "gridPos": {
             "h": 7,
             "w": 12,
-            "x": 0,
-            "y": 7
+            "x": 12,
+            "y": 0
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 297,
+          "id": 298,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -42208,15 +42424,15 @@
           "targets": [
             {
               "datasource": "${DS_TEST-CLUSTER}",
-              "expr": "sum((\n    tikv_scheduler_contex_total\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    \n)) by (instance) ",
+              "expr": "sum(rate(\n    tikv_scheduler_commands_pri_total\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    [$__rate_interval]\n)) by (priority, $additional_groupby) ",
               "format": "time_series",
               "hide": false,
               "instant": false,
               "interval": "",
               "intervalFactor": 1,
-              "legendFormat": "{{instance}}",
+              "legendFormat": "{{priority}} {{$additional_groupby}}",
               "metric": "",
-              "query": "sum((\n    tikv_scheduler_contex_total\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    \n)) by (instance) ",
+              "query": "sum(rate(\n    tikv_scheduler_commands_pri_total\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    [$__rate_interval]\n)) by (priority, $additional_groupby) ",
               "refId": "",
               "step": 10,
               "target": ""
@@ -42225,7 +42441,7 @@
           "thresholds": [],
           "timeFrom": null,
           "timeShift": null,
-          "title": "Scheduler pending commands",
+          "title": "Scheduler priority commands",
           "tooltip": {
             "msResolution": true,
             "shared": true,
@@ -42244,7 +42460,7 @@
           "yaxes": [
             {
               "decimals": null,
-              "format": "none",
+              "format": "ops",
               "label": null,
               "logBase": 1,
               "max": null,
@@ -42271,7 +42487,7 @@
           "bars": false,
           "cacheTimeout": null,
           "datasource": "${DS_TEST-CLUSTER}",
-          "description": "The count of running commands per TiKV instance",
+          "description": "The count of pending commands per TiKV instance",
           "editable": true,
           "error": false,
           "fieldConfig": {
@@ -42293,12 +42509,12 @@
           "gridPos": {
             "h": 7,
             "w": 12,
-            "x": 12,
+            "x": 0,
             "y": 7
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 298,
+          "id": 299,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -42341,7 +42557,7 @@
           "targets": [
             {
               "datasource": "${DS_TEST-CLUSTER}",
-              "expr": "sum((\n    tikv_scheduler_running_commands\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    \n)) by (instance) ",
+              "expr": "sum((\n    tikv_scheduler_contex_total\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    \n)) by (instance) ",
               "format": "time_series",
               "hide": false,
               "instant": false,
@@ -42349,7 +42565,7 @@
               "intervalFactor": 1,
               "legendFormat": "{{instance}}",
               "metric": "",
-              "query": "sum((\n    tikv_scheduler_running_commands\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    \n)) by (instance) ",
+              "query": "sum((\n    tikv_scheduler_contex_total\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    \n)) by (instance) ",
               "refId": "",
               "step": 10,
               "target": ""
@@ -42358,7 +42574,7 @@
           "thresholds": [],
           "timeFrom": null,
           "timeShift": null,
-          "title": "Scheduler running commands",
+          "title": "Scheduler pending commands",
           "tooltip": {
             "msResolution": true,
             "shared": true,
@@ -42404,7 +42620,7 @@
           "bars": false,
           "cacheTimeout": null,
           "datasource": "${DS_TEST-CLUSTER}",
-          "description": "The total writing bytes of commands on each stage",
+          "description": "The count of running commands per TiKV instance",
           "editable": true,
           "error": false,
           "fieldConfig": {
@@ -42426,12 +42642,145 @@
           "gridPos": {
             "h": 7,
             "w": 12,
-            "x": 0,
-            "y": 14
+            "x": 12,
+            "y": 7
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 299,
+          "id": 300,
+          "interval": null,
+          "isNew": true,
+          "legend": {
+            "alignAsTable": true,
+            "avg": false,
+            "current": true,
+            "hideEmpty": true,
+            "hideZero": true,
+            "max": true,
+            "min": false,
+            "rightSide": true,
+            "show": true,
+            "sideWidth": null,
+            "sort": "max",
+            "sortDesc": true,
+            "total": false,
+            "values": true
+          },
+          "lines": true,
+          "linewidth": 1,
+          "links": [],
+          "maxDataPoints": null,
+          "maxPerRow": null,
+          "minSpan": null,
+          "nullPointMode": "null as zero",
+          "options": {
+            "alertThreshold": true,
+            "dataLinks": []
+          },
+          "percentage": false,
+          "pointradius": 5,
+          "points": false,
+          "renderer": "flot",
+          "repeat": null,
+          "repeatDirection": null,
+          "seriesOverrides": [],
+          "span": null,
+          "stack": false,
+          "steppedLine": false,
+          "targets": [
+            {
+              "datasource": "${DS_TEST-CLUSTER}",
+              "expr": "sum((\n    tikv_scheduler_running_commands\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    \n)) by (instance) ",
+              "format": "time_series",
+              "hide": false,
+              "instant": false,
+              "interval": "",
+              "intervalFactor": 1,
+              "legendFormat": "{{instance}}",
+              "metric": "",
+              "query": "sum((\n    tikv_scheduler_running_commands\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    \n)) by (instance) ",
+              "refId": "",
+              "step": 10,
+              "target": ""
+            }
+          ],
+          "thresholds": [],
+          "timeFrom": null,
+          "timeShift": null,
+          "title": "Scheduler running commands",
+          "tooltip": {
+            "msResolution": true,
+            "shared": true,
+            "sort": 0,
+            "value_type": "individual"
+          },
+          "transformations": [],
+          "transparent": false,
+          "type": "graph",
+          "xaxis": {
+            "mode": "time",
+            "name": null,
+            "show": true,
+            "values": []
+          },
+          "yaxes": [
+            {
+              "decimals": null,
+              "format": "none",
+              "label": null,
+              "logBase": 1,
+              "max": null,
+              "min": null,
+              "show": true
+            },
+            {
+              "decimals": null,
+              "format": "short",
+              "label": null,
+              "logBase": 1,
+              "max": null,
+              "min": null,
+              "show": true
+            }
+          ],
+          "yaxis": {
+            "align": false,
+            "alignLevel": 0
+          }
+        },
+        {
+          "aliasColors": {},
+          "bars": false,
+          "cacheTimeout": null,
+          "datasource": "${DS_TEST-CLUSTER}",
+          "description": "The total writing bytes of commands on each stage",
+          "editable": true,
+          "error": false,
+          "fieldConfig": {
+            "defaults": {
+              "thresholds": {
+                "mode": "absolute",
+                "steps": []
+              }
+            }
+          },
+          "fill": 1,
+          "fillGradient": 1,
+          "grid": {
+            "threshold1": null,
+            "threshold1Color": "rgba(216, 200, 27, 0.27)",
+            "threshold2": null,
+            "threshold2Color": "rgba(234, 112, 112, 0.22)"
+          },
+          "gridPos": {
+            "h": 7,
+            "w": 12,
+            "x": 0,
+            "y": 14
+          },
+          "height": null,
+          "hideTimeOverride": false,
+          "id": 301,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -42564,7 +42913,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 300,
+          "id": 302,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -42719,7 +43068,7 @@
           "hideTimeOverride": false,
           "hideZeroBuckets": true,
           "highlightCards": true,
-          "id": 301,
+          "id": 303,
           "interval": null,
           "legend": {
             "show": false
@@ -42820,7 +43169,7 @@
       },
       "height": null,
       "hideTimeOverride": false,
-      "id": 302,
+      "id": 304,
       "interval": null,
       "links": [],
       "maxDataPoints": 100,
@@ -42859,7 +43208,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 303,
+          "id": 305,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -42992,7 +43341,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 304,
+          "id": 306,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -43125,7 +43474,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 305,
+          "id": 307,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -43265,7 +43614,7 @@
           "hideTimeOverride": false,
           "hideZeroBuckets": true,
           "highlightCards": true,
-          "id": 306,
+          "id": 308,
           "interval": null,
           "legend": {
             "show": false
@@ -43363,7 +43712,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 307,
+          "id": 309,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -43564,7 +43913,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 308,
+          "id": 310,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -43765,7 +44114,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 309,
+          "id": 311,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -43969,7 +44318,7 @@
       },
       "height": null,
       "hideTimeOverride": false,
-      "id": 310,
+      "id": 312,
       "interval": null,
       "links": [],
       "maxDataPoints": 100,
@@ -44008,7 +44357,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 311,
+          "id": 313,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -44156,7 +44505,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 312,
+          "id": 314,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -44357,7 +44706,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 313,
+          "id": 315,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -44558,7 +44907,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 314,
+          "id": 316,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -44759,7 +45108,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 315,
+          "id": 317,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -44960,7 +45309,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 316,
+          "id": 318,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -45093,7 +45442,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 317,
+          "id": 319,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -45226,7 +45575,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 318,
+          "id": 320,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -45359,7 +45708,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 319,
+          "id": 321,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -45492,7 +45841,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 320,
+          "id": 322,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -45700,7 +46049,7 @@
           "hideTimeOverride": false,
           "hideZeroBuckets": true,
           "highlightCards": true,
-          "id": 321,
+          "id": 323,
           "interval": null,
           "legend": {
             "show": false
@@ -45801,7 +46150,7 @@
       },
       "height": null,
       "hideTimeOverride": false,
-      "id": 322,
+      "id": 324,
       "interval": null,
       "links": [],
       "maxDataPoints": 100,
@@ -45847,7 +46196,7 @@
           "hideTimeOverride": false,
           "hideZeroBuckets": true,
           "highlightCards": true,
-          "id": 323,
+          "id": 325,
           "interval": null,
           "legend": {
             "show": false
@@ -45945,7 +46294,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 324,
+          "id": 326,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -46146,7 +46495,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 325,
+          "id": 327,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -46279,7 +46628,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 326,
+          "id": 328,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -46412,7 +46761,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 327,
+          "id": 329,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -46545,7 +46894,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 328,
+          "id": 330,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -46746,7 +47095,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 329,
+          "id": 331,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -46879,7 +47228,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 330,
+          "id": 332,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -47012,7 +47361,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 331,
+          "id": 333,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -47148,7 +47497,7 @@
       },
       "height": null,
       "hideTimeOverride": false,
-      "id": 332,
+      "id": 334,
       "interval": null,
       "links": [],
       "maxDataPoints": 100,
@@ -47187,7 +47536,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 333,
+          "id": 335,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -47388,7 +47737,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 334,
+          "id": 336,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -47589,7 +47938,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 335,
+          "id": 337,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -47790,7 +48139,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 336,
+          "id": 338,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -47991,7 +48340,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 337,
+          "id": 339,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -48124,7 +48473,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 338,
+          "id": 340,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -48257,7 +48606,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 339,
+          "id": 341,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -48390,7 +48739,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 340,
+          "id": 342,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -48523,7 +48872,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 341,
+          "id": 343,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -48656,7 +49005,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 342,
+          "id": 344,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -48796,7 +49145,7 @@
           "hideTimeOverride": false,
           "hideZeroBuckets": true,
           "highlightCards": true,
-          "id": 343,
+          "id": 345,
           "interval": null,
           "legend": {
             "show": false
@@ -48894,7 +49243,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 344,
+          "id": 346,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -49098,7 +49447,7 @@
       },
       "height": null,
       "hideTimeOverride": false,
-      "id": 345,
+      "id": 347,
       "interval": null,
       "links": [],
       "maxDataPoints": 100,
@@ -49137,7 +49486,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 346,
+          "id": 348,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -49270,7 +49619,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 347,
+          "id": 349,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -49403,7 +49752,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 348,
+          "id": 350,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -49543,7 +49892,7 @@
           "hideTimeOverride": false,
           "hideZeroBuckets": true,
           "highlightCards": true,
-          "id": 349,
+          "id": 351,
           "interval": null,
           "legend": {
             "show": false
@@ -49641,7 +49990,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 350,
+          "id": 352,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -49842,7 +50191,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 351,
+          "id": 353,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -50043,7 +50392,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 352,
+          "id": 354,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -50247,7 +50596,7 @@
       },
       "height": null,
       "hideTimeOverride": false,
-      "id": 353,
+      "id": 355,
       "interval": null,
       "links": [],
       "maxDataPoints": 100,
@@ -50286,7 +50635,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 354,
+          "id": 356,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -50464,7 +50813,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 355,
+          "id": 357,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -50665,7 +51014,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 356,
+          "id": 358,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -50798,7 +51147,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 357,
+          "id": 359,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -50931,7 +51280,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 358,
+          "id": 360,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -51064,7 +51413,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 359,
+          "id": 361,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -51197,7 +51546,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 360,
+          "id": 362,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -51330,7 +51679,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 361,
+          "id": 363,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -51459,7 +51808,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 362,
+          "id": 364,
           "interval": null,
           "links": [],
           "maxDataPoints": 100,
@@ -51534,7 +51883,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 363,
+          "id": 365,
           "interval": null,
           "links": [],
           "maxDataPoints": 100,
@@ -51613,7 +51962,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 364,
+          "id": 366,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -51866,7 +52215,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 365,
+          "id": 367,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -51999,7 +52348,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 366,
+          "id": 368,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -52135,7 +52484,7 @@
       },
       "height": null,
       "hideTimeOverride": false,
-      "id": 367,
+      "id": 369,
       "interval": null,
       "links": [],
       "maxDataPoints": 100,
@@ -52174,7 +52523,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 368,
+          "id": 370,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -52322,7 +52671,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 369,
+          "id": 371,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -52455,7 +52804,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 370,
+          "id": 372,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -52656,7 +53005,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 371,
+          "id": 373,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -52804,7 +53153,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 372,
+          "id": 374,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -53005,7 +53354,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 373,
+          "id": 375,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -53138,7 +53487,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 374,
+          "id": 376,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -53271,7 +53620,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 375,
+          "id": 377,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -53404,7 +53753,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 376,
+          "id": 378,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -53537,7 +53886,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 377,
+          "id": 379,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -53677,7 +54026,7 @@
           "hideTimeOverride": false,
           "hideZeroBuckets": true,
           "highlightCards": true,
-          "id": 378,
+          "id": 380,
           "interval": null,
           "legend": {
             "show": false
@@ -53775,7 +54124,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 379,
+          "id": 381,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -53979,7 +54328,7 @@
       },
       "height": null,
       "hideTimeOverride": false,
-      "id": 380,
+      "id": 382,
       "interval": null,
       "links": [],
       "maxDataPoints": 100,
@@ -54018,272 +54367,6 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 381,
-          "interval": null,
-          "isNew": true,
-          "legend": {
-            "alignAsTable": true,
-            "avg": false,
-            "current": true,
-            "hideEmpty": true,
-            "hideZero": true,
-            "max": true,
-            "min": false,
-            "rightSide": true,
-            "show": true,
-            "sideWidth": null,
-            "sort": "max",
-            "sortDesc": true,
-            "total": false,
-            "values": true
-          },
-          "lines": true,
-          "linewidth": 1,
-          "links": [],
-          "maxDataPoints": null,
-          "maxPerRow": null,
-          "minSpan": null,
-          "nullPointMode": "null as zero",
-          "options": {
-            "alertThreshold": true,
-            "dataLinks": []
-          },
-          "percentage": false,
-          "pointradius": 5,
-          "points": false,
-          "renderer": "flot",
-          "repeat": null,
-          "repeatDirection": null,
-          "seriesOverrides": [],
-          "span": null,
-          "stack": false,
-          "steppedLine": false,
-          "targets": [
-            {
-              "datasource": "${DS_TEST-CLUSTER}",
-              "expr": "sum(rate(\n    tikv_worker_handled_task_total\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    [$__rate_interval]\n)) by (name, $additional_groupby) ",
-              "format": "time_series",
-              "hide": false,
-              "instant": false,
-              "interval": "",
-              "intervalFactor": 1,
-              "legendFormat": "{{name}} {{$additional_groupby}}",
-              "metric": "",
-              "query": "sum(rate(\n    tikv_worker_handled_task_total\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    [$__rate_interval]\n)) by (name, $additional_groupby) ",
-              "refId": "",
-              "step": 10,
-              "target": ""
-            }
-          ],
-          "thresholds": [],
-          "timeFrom": null,
-          "timeShift": null,
-          "title": "Worker handled tasks",
-          "tooltip": {
-            "msResolution": true,
-            "shared": true,
-            "sort": 0,
-            "value_type": "individual"
-          },
-          "transformations": [],
-          "transparent": false,
-          "type": "graph",
-          "xaxis": {
-            "mode": "time",
-            "name": null,
-            "show": true,
-            "values": []
-          },
-          "yaxes": [
-            {
-              "decimals": null,
-              "format": "ops",
-              "label": null,
-              "logBase": 1,
-              "max": null,
-              "min": null,
-              "show": true
-            },
-            {
-              "decimals": null,
-              "format": "short",
-              "label": null,
-              "logBase": 1,
-              "max": null,
-              "min": null,
-              "show": true
-            }
-          ],
-          "yaxis": {
-            "align": false,
-            "alignLevel": 0
-          }
-        },
-        {
-          "aliasColors": {},
-          "bars": false,
-          "cacheTimeout": null,
-          "datasource": "${DS_TEST-CLUSTER}",
-          "description": "Current pending and running tasks of worker",
-          "editable": true,
-          "error": false,
-          "fieldConfig": {
-            "defaults": {
-              "thresholds": {
-                "mode": "absolute",
-                "steps": []
-              }
-            }
-          },
-          "fill": 1,
-          "fillGradient": 1,
-          "grid": {
-            "threshold1": null,
-            "threshold1Color": "rgba(216, 200, 27, 0.27)",
-            "threshold2": null,
-            "threshold2Color": "rgba(234, 112, 112, 0.22)"
-          },
-          "gridPos": {
-            "h": 7,
-            "w": 12,
-            "x": 12,
-            "y": 0
-          },
-          "height": null,
-          "hideTimeOverride": false,
-          "id": 382,
-          "interval": null,
-          "isNew": true,
-          "legend": {
-            "alignAsTable": true,
-            "avg": false,
-            "current": true,
-            "hideEmpty": true,
-            "hideZero": true,
-            "max": true,
-            "min": false,
-            "rightSide": true,
-            "show": true,
-            "sideWidth": null,
-            "sort": "max",
-            "sortDesc": true,
-            "total": false,
-            "values": true
-          },
-          "lines": true,
-          "linewidth": 1,
-          "links": [],
-          "maxDataPoints": null,
-          "maxPerRow": null,
-          "minSpan": null,
-          "nullPointMode": "null as zero",
-          "options": {
-            "alertThreshold": true,
-            "dataLinks": []
-          },
-          "percentage": false,
-          "pointradius": 5,
-          "points": false,
-          "renderer": "flot",
-          "repeat": null,
-          "repeatDirection": null,
-          "seriesOverrides": [],
-          "span": null,
-          "stack": false,
-          "steppedLine": false,
-          "targets": [
-            {
-              "datasource": "${DS_TEST-CLUSTER}",
-              "expr": "sum((\n    tikv_worker_pending_task_total\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    \n)) by (name, $additional_groupby) ",
-              "format": "time_series",
-              "hide": false,
-              "instant": false,
-              "interval": "",
-              "intervalFactor": 1,
-              "legendFormat": "{{name}} {{$additional_groupby}}",
-              "metric": "",
-              "query": "sum((\n    tikv_worker_pending_task_total\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    \n)) by (name, $additional_groupby) ",
-              "refId": "",
-              "step": 10,
-              "target": ""
-            }
-          ],
-          "thresholds": [],
-          "timeFrom": null,
-          "timeShift": null,
-          "title": "Worker pending tasks",
-          "tooltip": {
-            "msResolution": true,
-            "shared": true,
-            "sort": 0,
-            "value_type": "individual"
-          },
-          "transformations": [],
-          "transparent": false,
-          "type": "graph",
-          "xaxis": {
-            "mode": "time",
-            "name": null,
-            "show": true,
-            "values": []
-          },
-          "yaxes": [
-            {
-              "decimals": null,
-              "format": "none",
-              "label": null,
-              "logBase": 1,
-              "max": null,
-              "min": null,
-              "show": true
-            },
-            {
-              "decimals": null,
-              "format": "short",
-              "label": null,
-              "logBase": 1,
-              "max": null,
-              "min": null,
-              "show": true
-            }
-          ],
-          "yaxis": {
-            "align": false,
-            "alignLevel": 0
-          }
-        },
-        {
-          "aliasColors": {},
-          "bars": false,
-          "cacheTimeout": null,
-          "datasource": "${DS_TEST-CLUSTER}",
-          "description": "The number of tasks handled by future_pool",
-          "editable": true,
-          "error": false,
-          "fieldConfig": {
-            "defaults": {
-              "thresholds": {
-                "mode": "absolute",
-                "steps": []
-              }
-            }
-          },
-          "fill": 1,
-          "fillGradient": 1,
-          "grid": {
-            "threshold1": null,
-            "threshold1Color": "rgba(216, 200, 27, 0.27)",
-            "threshold2": null,
-            "threshold2Color": "rgba(234, 112, 112, 0.22)"
-          },
-          "gridPos": {
-            "h": 7,
-            "w": 12,
-            "x": 0,
-            "y": 7
-          },
-          "height": null,
-          "hideTimeOverride": false,
           "id": 383,
           "interval": null,
           "isNew": true,
@@ -54324,6 +54407,272 @@
           "span": null,
           "stack": false,
           "steppedLine": false,
+          "targets": [
+            {
+              "datasource": "${DS_TEST-CLUSTER}",
+              "expr": "sum(rate(\n    tikv_worker_handled_task_total\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    [$__rate_interval]\n)) by (name, $additional_groupby) ",
+              "format": "time_series",
+              "hide": false,
+              "instant": false,
+              "interval": "",
+              "intervalFactor": 1,
+              "legendFormat": "{{name}} {{$additional_groupby}}",
+              "metric": "",
+              "query": "sum(rate(\n    tikv_worker_handled_task_total\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    [$__rate_interval]\n)) by (name, $additional_groupby) ",
+              "refId": "",
+              "step": 10,
+              "target": ""
+            }
+          ],
+          "thresholds": [],
+          "timeFrom": null,
+          "timeShift": null,
+          "title": "Worker handled tasks",
+          "tooltip": {
+            "msResolution": true,
+            "shared": true,
+            "sort": 0,
+            "value_type": "individual"
+          },
+          "transformations": [],
+          "transparent": false,
+          "type": "graph",
+          "xaxis": {
+            "mode": "time",
+            "name": null,
+            "show": true,
+            "values": []
+          },
+          "yaxes": [
+            {
+              "decimals": null,
+              "format": "ops",
+              "label": null,
+              "logBase": 1,
+              "max": null,
+              "min": null,
+              "show": true
+            },
+            {
+              "decimals": null,
+              "format": "short",
+              "label": null,
+              "logBase": 1,
+              "max": null,
+              "min": null,
+              "show": true
+            }
+          ],
+          "yaxis": {
+            "align": false,
+            "alignLevel": 0
+          }
+        },
+        {
+          "aliasColors": {},
+          "bars": false,
+          "cacheTimeout": null,
+          "datasource": "${DS_TEST-CLUSTER}",
+          "description": "Current pending and running tasks of worker",
+          "editable": true,
+          "error": false,
+          "fieldConfig": {
+            "defaults": {
+              "thresholds": {
+                "mode": "absolute",
+                "steps": []
+              }
+            }
+          },
+          "fill": 1,
+          "fillGradient": 1,
+          "grid": {
+            "threshold1": null,
+            "threshold1Color": "rgba(216, 200, 27, 0.27)",
+            "threshold2": null,
+            "threshold2Color": "rgba(234, 112, 112, 0.22)"
+          },
+          "gridPos": {
+            "h": 7,
+            "w": 12,
+            "x": 12,
+            "y": 0
+          },
+          "height": null,
+          "hideTimeOverride": false,
+          "id": 384,
+          "interval": null,
+          "isNew": true,
+          "legend": {
+            "alignAsTable": true,
+            "avg": false,
+            "current": true,
+            "hideEmpty": true,
+            "hideZero": true,
+            "max": true,
+            "min": false,
+            "rightSide": true,
+            "show": true,
+            "sideWidth": null,
+            "sort": "max",
+            "sortDesc": true,
+            "total": false,
+            "values": true
+          },
+          "lines": true,
+          "linewidth": 1,
+          "links": [],
+          "maxDataPoints": null,
+          "maxPerRow": null,
+          "minSpan": null,
+          "nullPointMode": "null as zero",
+          "options": {
+            "alertThreshold": true,
+            "dataLinks": []
+          },
+          "percentage": false,
+          "pointradius": 5,
+          "points": false,
+          "renderer": "flot",
+          "repeat": null,
+          "repeatDirection": null,
+          "seriesOverrides": [],
+          "span": null,
+          "stack": false,
+          "steppedLine": false,
+          "targets": [
+            {
+              "datasource": "${DS_TEST-CLUSTER}",
+              "expr": "sum((\n    tikv_worker_pending_task_total\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    \n)) by (name, $additional_groupby) ",
+              "format": "time_series",
+              "hide": false,
+              "instant": false,
+              "interval": "",
+              "intervalFactor": 1,
+              "legendFormat": "{{name}} {{$additional_groupby}}",
+              "metric": "",
+              "query": "sum((\n    tikv_worker_pending_task_total\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    \n)) by (name, $additional_groupby) ",
+              "refId": "",
+              "step": 10,
+              "target": ""
+            }
+          ],
+          "thresholds": [],
+          "timeFrom": null,
+          "timeShift": null,
+          "title": "Worker pending tasks",
+          "tooltip": {
+            "msResolution": true,
+            "shared": true,
+            "sort": 0,
+            "value_type": "individual"
+          },
+          "transformations": [],
+          "transparent": false,
+          "type": "graph",
+          "xaxis": {
+            "mode": "time",
+            "name": null,
+            "show": true,
+            "values": []
+          },
+          "yaxes": [
+            {
+              "decimals": null,
+              "format": "none",
+              "label": null,
+              "logBase": 1,
+              "max": null,
+              "min": null,
+              "show": true
+            },
+            {
+              "decimals": null,
+              "format": "short",
+              "label": null,
+              "logBase": 1,
+              "max": null,
+              "min": null,
+              "show": true
+            }
+          ],
+          "yaxis": {
+            "align": false,
+            "alignLevel": 0
+          }
+        },
+        {
+          "aliasColors": {},
+          "bars": false,
+          "cacheTimeout": null,
+          "datasource": "${DS_TEST-CLUSTER}",
+          "description": "The number of tasks handled by future_pool",
+          "editable": true,
+          "error": false,
+          "fieldConfig": {
+            "defaults": {
+              "thresholds": {
+                "mode": "absolute",
+                "steps": []
+              }
+            }
+          },
+          "fill": 1,
+          "fillGradient": 1,
+          "grid": {
+            "threshold1": null,
+            "threshold1Color": "rgba(216, 200, 27, 0.27)",
+            "threshold2": null,
+            "threshold2Color": "rgba(234, 112, 112, 0.22)"
+          },
+          "gridPos": {
+            "h": 7,
+            "w": 12,
+            "x": 0,
+            "y": 7
+          },
+          "height": null,
+          "hideTimeOverride": false,
+          "id": 385,
+          "interval": null,
+          "isNew": true,
+          "legend": {
+            "alignAsTable": true,
+            "avg": false,
+            "current": true,
+            "hideEmpty": true,
+            "hideZero": true,
+            "max": true,
+            "min": false,
+            "rightSide": true,
+            "show": true,
+            "sideWidth": null,
+            "sort": "max",
+            "sortDesc": true,
+            "total": false,
+            "values": true
+          },
+          "lines": true,
+          "linewidth": 1,
+          "links": [],
+          "maxDataPoints": null,
+          "maxPerRow": null,
+          "minSpan": null,
+          "nullPointMode": "null as zero",
+          "options": {
+            "alertThreshold": true,
+            "dataLinks": []
+          },
+          "percentage": false,
+          "pointradius": 5,
+          "points": false,
+          "renderer": "flot",
+          "repeat": null,
+          "repeatDirection": null,
+          "seriesOverrides": [],
+          "span": null,
+          "stack": false,
+          "steppedLine": false,
           "targets": [
             {
               "datasource": "${DS_TEST-CLUSTER}",
@@ -54417,7 +54766,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 384,
+          "id": 386,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -54553,7 +54902,7 @@
       },
       "height": null,
       "hideTimeOverride": false,
-      "id": 385,
+      "id": 387,
       "interval": null,
       "links": [],
       "maxDataPoints": 100,
@@ -54592,7 +54941,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 386,
+          "id": 388,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -54725,7 +55074,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 387,
+          "id": 389,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -54858,7 +55207,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 388,
+          "id": 390,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -55006,7 +55355,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 389,
+          "id": 391,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -55139,7 +55488,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 390,
+          "id": 392,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -55272,7 +55621,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 391,
+          "id": 393,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -55405,7 +55754,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 392,
+          "id": 394,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -55541,7 +55890,7 @@
       },
       "height": null,
       "hideTimeOverride": false,
-      "id": 393,
+      "id": 395,
       "interval": null,
       "links": [],
       "maxDataPoints": 100,
@@ -55580,272 +55929,6 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 394,
-          "interval": null,
-          "isNew": true,
-          "legend": {
-            "alignAsTable": true,
-            "avg": false,
-            "current": true,
-            "hideEmpty": true,
-            "hideZero": true,
-            "max": true,
-            "min": false,
-            "rightSide": true,
-            "show": true,
-            "sideWidth": null,
-            "sort": "max",
-            "sortDesc": true,
-            "total": false,
-            "values": true
-          },
-          "lines": true,
-          "linewidth": 1,
-          "links": [],
-          "maxDataPoints": null,
-          "maxPerRow": null,
-          "minSpan": null,
-          "nullPointMode": "null as zero",
-          "options": {
-            "alertThreshold": true,
-            "dataLinks": []
-          },
-          "percentage": false,
-          "pointradius": 5,
-          "points": false,
-          "renderer": "flot",
-          "repeat": null,
-          "repeatDirection": null,
-          "seriesOverrides": [],
-          "span": null,
-          "stack": false,
-          "steppedLine": false,
-          "targets": [
-            {
-              "datasource": "${DS_TEST-CLUSTER}",
-              "expr": "histogram_quantile(0.99,(\n    sum(rate(\n    tikv_raftstore_inspect_duration_seconds_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    [$__rate_interval]\n)) by (instance, type, le) \n    \n    \n))  ",
-              "format": "time_series",
-              "hide": false,
-              "instant": false,
-              "interval": "",
-              "intervalFactor": 1,
-              "legendFormat": "{{instance}}-{{type}}",
-              "metric": "",
-              "query": "histogram_quantile(0.99,(\n    sum(rate(\n    tikv_raftstore_inspect_duration_seconds_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    [$__rate_interval]\n)) by (instance, type, le) \n    \n    \n))  ",
-              "refId": "",
-              "step": 10,
-              "target": ""
-            }
-          ],
-          "thresholds": [],
-          "timeFrom": null,
-          "timeShift": null,
-          "title": "Inspected duration per server",
-          "tooltip": {
-            "msResolution": true,
-            "shared": true,
-            "sort": 0,
-            "value_type": "individual"
-          },
-          "transformations": [],
-          "transparent": false,
-          "type": "graph",
-          "xaxis": {
-            "mode": "time",
-            "name": null,
-            "show": true,
-            "values": []
-          },
-          "yaxes": [
-            {
-              "decimals": null,
-              "format": "s",
-              "label": null,
-              "logBase": 1,
-              "max": null,
-              "min": null,
-              "show": true
-            },
-            {
-              "decimals": null,
-              "format": "short",
-              "label": null,
-              "logBase": 1,
-              "max": null,
-              "min": null,
-              "show": true
-            }
-          ],
-          "yaxis": {
-            "align": false,
-            "alignLevel": 0
-          }
-        },
-        {
-          "aliasColors": {},
-          "bars": false,
-          "cacheTimeout": null,
-          "datasource": "${DS_TEST-CLUSTER}",
-          "description": "The slow score of stores",
-          "editable": true,
-          "error": false,
-          "fieldConfig": {
-            "defaults": {
-              "thresholds": {
-                "mode": "absolute",
-                "steps": []
-              }
-            }
-          },
-          "fill": 1,
-          "fillGradient": 1,
-          "grid": {
-            "threshold1": null,
-            "threshold1Color": "rgba(216, 200, 27, 0.27)",
-            "threshold2": null,
-            "threshold2Color": "rgba(234, 112, 112, 0.22)"
-          },
-          "gridPos": {
-            "h": 7,
-            "w": 12,
-            "x": 12,
-            "y": 0
-          },
-          "height": null,
-          "hideTimeOverride": false,
-          "id": 395,
-          "interval": null,
-          "isNew": true,
-          "legend": {
-            "alignAsTable": true,
-            "avg": false,
-            "current": true,
-            "hideEmpty": true,
-            "hideZero": true,
-            "max": true,
-            "min": false,
-            "rightSide": true,
-            "show": true,
-            "sideWidth": null,
-            "sort": "max",
-            "sortDesc": true,
-            "total": false,
-            "values": true
-          },
-          "lines": true,
-          "linewidth": 1,
-          "links": [],
-          "maxDataPoints": null,
-          "maxPerRow": null,
-          "minSpan": null,
-          "nullPointMode": "null as zero",
-          "options": {
-            "alertThreshold": true,
-            "dataLinks": []
-          },
-          "percentage": false,
-          "pointradius": 5,
-          "points": false,
-          "renderer": "flot",
-          "repeat": null,
-          "repeatDirection": null,
-          "seriesOverrides": [],
-          "span": null,
-          "stack": false,
-          "steppedLine": false,
-          "targets": [
-            {
-              "datasource": "${DS_TEST-CLUSTER}",
-              "expr": "sum((\n    tikv_raftstore_slow_score\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    \n)) by (instance) ",
-              "format": "time_series",
-              "hide": false,
-              "instant": false,
-              "interval": "",
-              "intervalFactor": 1,
-              "legendFormat": "{{instance}}",
-              "metric": "",
-              "query": "sum((\n    tikv_raftstore_slow_score\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    \n)) by (instance) ",
-              "refId": "",
-              "step": 10,
-              "target": ""
-            }
-          ],
-          "thresholds": [],
-          "timeFrom": null,
-          "timeShift": null,
-          "title": "Store Slow Score",
-          "tooltip": {
-            "msResolution": true,
-            "shared": true,
-            "sort": 0,
-            "value_type": "individual"
-          },
-          "transformations": [],
-          "transparent": false,
-          "type": "graph",
-          "xaxis": {
-            "mode": "time",
-            "name": null,
-            "show": true,
-            "values": []
-          },
-          "yaxes": [
-            {
-              "decimals": null,
-              "format": "none",
-              "label": null,
-              "logBase": 1,
-              "max": null,
-              "min": null,
-              "show": true
-            },
-            {
-              "decimals": null,
-              "format": "short",
-              "label": null,
-              "logBase": 1,
-              "max": null,
-              "min": null,
-              "show": true
-            }
-          ],
-          "yaxis": {
-            "align": false,
-            "alignLevel": 0
-          }
-        },
-        {
-          "aliasColors": {},
-          "bars": false,
-          "cacheTimeout": null,
-          "datasource": "${DS_TEST-CLUSTER}",
-          "description": "The changing trend of the slowness on I/O operations. 'value > 0' means the related store might have a slow trend.",
-          "editable": true,
-          "error": false,
-          "fieldConfig": {
-            "defaults": {
-              "thresholds": {
-                "mode": "absolute",
-                "steps": []
-              }
-            }
-          },
-          "fill": 1,
-          "fillGradient": 1,
-          "grid": {
-            "threshold1": null,
-            "threshold1Color": "rgba(216, 200, 27, 0.27)",
-            "threshold2": null,
-            "threshold2Color": "rgba(234, 112, 112, 0.22)"
-          },
-          "gridPos": {
-            "h": 7,
-            "w": 12,
-            "x": 0,
-            "y": 7
-          },
-          "height": null,
-          "hideTimeOverride": false,
           "id": 396,
           "interval": null,
           "isNew": true,
@@ -55886,6 +55969,272 @@
           "span": null,
           "stack": false,
           "steppedLine": false,
+          "targets": [
+            {
+              "datasource": "${DS_TEST-CLUSTER}",
+              "expr": "histogram_quantile(0.99,(\n    sum(rate(\n    tikv_raftstore_inspect_duration_seconds_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    [$__rate_interval]\n)) by (instance, type, le) \n    \n    \n))  ",
+              "format": "time_series",
+              "hide": false,
+              "instant": false,
+              "interval": "",
+              "intervalFactor": 1,
+              "legendFormat": "{{instance}}-{{type}}",
+              "metric": "",
+              "query": "histogram_quantile(0.99,(\n    sum(rate(\n    tikv_raftstore_inspect_duration_seconds_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    [$__rate_interval]\n)) by (instance, type, le) \n    \n    \n))  ",
+              "refId": "",
+              "step": 10,
+              "target": ""
+            }
+          ],
+          "thresholds": [],
+          "timeFrom": null,
+          "timeShift": null,
+          "title": "Inspected duration per server",
+          "tooltip": {
+            "msResolution": true,
+            "shared": true,
+            "sort": 0,
+            "value_type": "individual"
+          },
+          "transformations": [],
+          "transparent": false,
+          "type": "graph",
+          "xaxis": {
+            "mode": "time",
+            "name": null,
+            "show": true,
+            "values": []
+          },
+          "yaxes": [
+            {
+              "decimals": null,
+              "format": "s",
+              "label": null,
+              "logBase": 1,
+              "max": null,
+              "min": null,
+              "show": true
+            },
+            {
+              "decimals": null,
+              "format": "short",
+              "label": null,
+              "logBase": 1,
+              "max": null,
+              "min": null,
+              "show": true
+            }
+          ],
+          "yaxis": {
+            "align": false,
+            "alignLevel": 0
+          }
+        },
+        {
+          "aliasColors": {},
+          "bars": false,
+          "cacheTimeout": null,
+          "datasource": "${DS_TEST-CLUSTER}",
+          "description": "The slow score of stores",
+          "editable": true,
+          "error": false,
+          "fieldConfig": {
+            "defaults": {
+              "thresholds": {
+                "mode": "absolute",
+                "steps": []
+              }
+            }
+          },
+          "fill": 1,
+          "fillGradient": 1,
+          "grid": {
+            "threshold1": null,
+            "threshold1Color": "rgba(216, 200, 27, 0.27)",
+            "threshold2": null,
+            "threshold2Color": "rgba(234, 112, 112, 0.22)"
+          },
+          "gridPos": {
+            "h": 7,
+            "w": 12,
+            "x": 12,
+            "y": 0
+          },
+          "height": null,
+          "hideTimeOverride": false,
+          "id": 397,
+          "interval": null,
+          "isNew": true,
+          "legend": {
+            "alignAsTable": true,
+            "avg": false,
+            "current": true,
+            "hideEmpty": true,
+            "hideZero": true,
+            "max": true,
+            "min": false,
+            "rightSide": true,
+            "show": true,
+            "sideWidth": null,
+            "sort": "max",
+            "sortDesc": true,
+            "total": false,
+            "values": true
+          },
+          "lines": true,
+          "linewidth": 1,
+          "links": [],
+          "maxDataPoints": null,
+          "maxPerRow": null,
+          "minSpan": null,
+          "nullPointMode": "null as zero",
+          "options": {
+            "alertThreshold": true,
+            "dataLinks": []
+          },
+          "percentage": false,
+          "pointradius": 5,
+          "points": false,
+          "renderer": "flot",
+          "repeat": null,
+          "repeatDirection": null,
+          "seriesOverrides": [],
+          "span": null,
+          "stack": false,
+          "steppedLine": false,
+          "targets": [
+            {
+              "datasource": "${DS_TEST-CLUSTER}",
+              "expr": "sum((\n    tikv_raftstore_slow_score\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    \n)) by (instance) ",
+              "format": "time_series",
+              "hide": false,
+              "instant": false,
+              "interval": "",
+              "intervalFactor": 1,
+              "legendFormat": "{{instance}}",
+              "metric": "",
+              "query": "sum((\n    tikv_raftstore_slow_score\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    \n)) by (instance) ",
+              "refId": "",
+              "step": 10,
+              "target": ""
+            }
+          ],
+          "thresholds": [],
+          "timeFrom": null,
+          "timeShift": null,
+          "title": "Store Slow Score",
+          "tooltip": {
+            "msResolution": true,
+            "shared": true,
+            "sort": 0,
+            "value_type": "individual"
+          },
+          "transformations": [],
+          "transparent": false,
+          "type": "graph",
+          "xaxis": {
+            "mode": "time",
+            "name": null,
+            "show": true,
+            "values": []
+          },
+          "yaxes": [
+            {
+              "decimals": null,
+              "format": "none",
+              "label": null,
+              "logBase": 1,
+              "max": null,
+              "min": null,
+              "show": true
+            },
+            {
+              "decimals": null,
+              "format": "short",
+              "label": null,
+              "logBase": 1,
+              "max": null,
+              "min": null,
+              "show": true
+            }
+          ],
+          "yaxis": {
+            "align": false,
+            "alignLevel": 0
+          }
+        },
+        {
+          "aliasColors": {},
+          "bars": false,
+          "cacheTimeout": null,
+          "datasource": "${DS_TEST-CLUSTER}",
+          "description": "The changing trend of the slowness on I/O operations. 'value > 0' means the related store might have a slow trend.",
+          "editable": true,
+          "error": false,
+          "fieldConfig": {
+            "defaults": {
+              "thresholds": {
+                "mode": "absolute",
+                "steps": []
+              }
+            }
+          },
+          "fill": 1,
+          "fillGradient": 1,
+          "grid": {
+            "threshold1": null,
+            "threshold1Color": "rgba(216, 200, 27, 0.27)",
+            "threshold2": null,
+            "threshold2Color": "rgba(234, 112, 112, 0.22)"
+          },
+          "gridPos": {
+            "h": 7,
+            "w": 12,
+            "x": 0,
+            "y": 7
+          },
+          "height": null,
+          "hideTimeOverride": false,
+          "id": 398,
+          "interval": null,
+          "isNew": true,
+          "legend": {
+            "alignAsTable": true,
+            "avg": false,
+            "current": true,
+            "hideEmpty": true,
+            "hideZero": true,
+            "max": true,
+            "min": false,
+            "rightSide": true,
+            "show": true,
+            "sideWidth": null,
+            "sort": "max",
+            "sortDesc": true,
+            "total": false,
+            "values": true
+          },
+          "lines": true,
+          "linewidth": 1,
+          "links": [],
+          "maxDataPoints": null,
+          "maxPerRow": null,
+          "minSpan": null,
+          "nullPointMode": "null as zero",
+          "options": {
+            "alertThreshold": true,
+            "dataLinks": []
+          },
+          "percentage": false,
+          "pointradius": 5,
+          "points": false,
+          "renderer": "flot",
+          "repeat": null,
+          "repeatDirection": null,
+          "seriesOverrides": [],
+          "span": null,
+          "stack": false,
+          "steppedLine": false,
           "targets": [
             {
               "datasource": "${DS_TEST-CLUSTER}",
@@ -55979,7 +56328,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 397,
+          "id": 399,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -56112,7 +56461,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 398,
+          "id": 400,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -56245,7 +56594,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 399,
+          "id": 401,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -56381,7 +56730,7 @@
       },
       "height": null,
       "hideTimeOverride": false,
-      "id": 400,
+      "id": 402,
       "interval": null,
       "links": [],
       "maxDataPoints": 100,
@@ -56420,7 +56769,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 401,
+          "id": 403,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -56553,7 +56902,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 402,
+          "id": 404,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -56686,7 +57035,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 403,
+          "id": 405,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -56834,7 +57183,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 404,
+          "id": 406,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -56997,7 +57346,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 405,
+          "id": 407,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -57130,7 +57479,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 406,
+          "id": 408,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -57263,7 +57612,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 407,
+          "id": 409,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -57411,7 +57760,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 408,
+          "id": 410,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -57559,7 +57908,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 409,
+          "id": 411,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -57695,7 +58044,7 @@
       },
       "height": null,
       "hideTimeOverride": false,
-      "id": 410,
+      "id": 412,
       "interval": null,
       "links": [],
       "maxDataPoints": 100,
@@ -57734,7 +58083,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 411,
+          "id": 413,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -57867,7 +58216,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 412,
+          "id": 414,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -58000,7 +58349,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 413,
+          "id": 415,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -58133,7 +58482,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 414,
+          "id": 416,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -58266,7 +58615,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 415,
+          "id": 417,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -58399,7 +58748,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 416,
+          "id": 418,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -58532,7 +58881,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 417,
+          "id": 419,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -58665,7 +59014,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 418,
+          "id": 420,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -58798,7 +59147,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 419,
+          "id": 421,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -58938,7 +59287,7 @@
           "hideTimeOverride": false,
           "hideZeroBuckets": true,
           "highlightCards": true,
-          "id": 420,
+          "id": 422,
           "interval": null,
           "legend": {
             "show": false
@@ -59036,7 +59385,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 421,
+          "id": 423,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -59169,7 +59518,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 422,
+          "id": 424,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -59317,7 +59666,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 423,
+          "id": 425,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -59465,7 +59814,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 424,
+          "id": 426,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -59605,7 +59954,7 @@
           "hideTimeOverride": false,
           "hideZeroBuckets": true,
           "highlightCards": true,
-          "id": 425,
+          "id": 427,
           "interval": null,
           "legend": {
             "show": false
@@ -59703,7 +60052,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 426,
+          "id": 428,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -59836,7 +60185,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 427,
+          "id": 429,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -59972,7 +60321,7 @@
       },
       "height": null,
       "hideTimeOverride": false,
-      "id": 428,
+      "id": 430,
       "interval": null,
       "links": [],
       "maxDataPoints": 100,
@@ -60011,7 +60360,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 429,
+          "id": 431,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -60144,7 +60493,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 430,
+          "id": 432,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -60307,7 +60656,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 431,
+          "id": 433,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -60455,7 +60804,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 432,
+          "id": 434,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -60588,7 +60937,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 433,
+          "id": 435,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -60728,7 +61077,7 @@
           "hideTimeOverride": false,
           "hideZeroBuckets": true,
           "highlightCards": true,
-          "id": 434,
+          "id": 436,
           "interval": null,
           "legend": {
             "show": false
@@ -60833,7 +61182,7 @@
           "hideTimeOverride": false,
           "hideZeroBuckets": true,
           "highlightCards": true,
-          "id": 435,
+          "id": 437,
           "interval": null,
           "legend": {
             "show": false
@@ -60938,7 +61287,7 @@
           "hideTimeOverride": false,
           "hideZeroBuckets": true,
           "highlightCards": true,
-          "id": 436,
+          "id": 438,
           "interval": null,
           "legend": {
             "show": false
@@ -61036,7 +61385,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 437,
+          "id": 439,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -61176,7 +61525,7 @@
           "hideTimeOverride": false,
           "hideZeroBuckets": true,
           "highlightCards": true,
-          "id": 438,
+          "id": 440,
           "interval": null,
           "legend": {
             "show": false
@@ -61281,7 +61630,7 @@
           "hideTimeOverride": false,
           "hideZeroBuckets": true,
           "highlightCards": true,
-          "id": 439,
+          "id": 441,
           "interval": null,
           "legend": {
             "show": false
@@ -61386,7 +61735,7 @@
           "hideTimeOverride": false,
           "hideZeroBuckets": true,
           "highlightCards": true,
-          "id": 440,
+          "id": 442,
           "interval": null,
           "legend": {
             "show": false
@@ -61484,7 +61833,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 441,
+          "id": 443,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -61617,7 +61966,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 442,
+          "id": 444,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -61750,7 +62099,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 443,
+          "id": 445,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -61890,7 +62239,7 @@
           "hideTimeOverride": false,
           "hideZeroBuckets": true,
           "highlightCards": true,
-          "id": 444,
+          "id": 446,
           "interval": null,
           "legend": {
             "show": false
@@ -61988,7 +62337,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 445,
+          "id": 447,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -62124,7 +62473,7 @@
       },
       "height": null,
       "hideTimeOverride": false,
-      "id": 446,
+      "id": 448,
       "interval": null,
       "links": [],
       "maxDataPoints": 100,
@@ -62163,7 +62512,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 447,
+          "id": 449,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -62326,7 +62675,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 448,
+          "id": 450,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -62459,7 +62808,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 449,
+          "id": 451,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -62599,7 +62948,7 @@
           "hideTimeOverride": false,
           "hideZeroBuckets": true,
           "highlightCards": true,
-          "id": 450,
+          "id": 452,
           "interval": null,
           "legend": {
             "show": false
@@ -62704,7 +63053,7 @@
           "hideTimeOverride": false,
           "hideZeroBuckets": true,
           "highlightCards": true,
-          "id": 451,
+          "id": 453,
           "interval": null,
           "legend": {
             "show": false
@@ -62802,7 +63151,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 452,
+          "id": 454,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -62957,7 +63306,7 @@
           "hideTimeOverride": false,
           "hideZeroBuckets": true,
           "highlightCards": true,
-          "id": 453,
+          "id": 455,
           "interval": null,
           "legend": {
             "show": false
@@ -63062,7 +63411,7 @@
           "hideTimeOverride": false,
           "hideZeroBuckets": true,
           "highlightCards": true,
-          "id": 454,
+          "id": 456,
           "interval": null,
           "legend": {
             "show": false
@@ -63167,7 +63516,7 @@
           "hideTimeOverride": false,
           "hideZeroBuckets": true,
           "highlightCards": true,
-          "id": 455,
+          "id": 457,
           "interval": null,
           "legend": {
             "show": false
@@ -63265,7 +63614,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 456,
+          "id": 458,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -63435,7 +63784,7 @@
           "hideTimeOverride": false,
           "hideZeroBuckets": true,
           "highlightCards": true,
-          "id": 457,
+          "id": 459,
           "interval": null,
           "legend": {
             "show": false
@@ -63533,7 +63882,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 458,
+          "id": 460,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -63734,7 +64083,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 459,
+          "id": 461,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -63935,7 +64284,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 460,
+          "id": 462,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -64068,7 +64417,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 461,
+          "id": 463,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -64231,7 +64580,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 462,
+          "id": 464,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -64364,7 +64713,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 463,
+          "id": 465,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -64497,7 +64846,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 464,
+          "id": 466,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -64698,7 +65047,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 465,
+          "id": 467,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -64831,7 +65180,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 466,
+          "id": 468,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -64971,7 +65320,7 @@
           "hideTimeOverride": false,
           "hideZeroBuckets": true,
           "highlightCards": true,
-          "id": 467,
+          "id": 469,
           "interval": null,
           "legend": {
             "show": false
@@ -65076,7 +65425,7 @@
           "hideTimeOverride": false,
           "hideZeroBuckets": true,
           "highlightCards": true,
-          "id": 468,
+          "id": 470,
           "interval": null,
           "legend": {
             "show": false
@@ -65181,7 +65530,7 @@
           "hideTimeOverride": false,
           "hideZeroBuckets": true,
           "highlightCards": true,
-          "id": 469,
+          "id": 471,
           "interval": null,
           "legend": {
             "show": false
@@ -65286,7 +65635,7 @@
           "hideTimeOverride": false,
           "hideZeroBuckets": true,
           "highlightCards": true,
-          "id": 470,
+          "id": 472,
           "interval": null,
           "legend": {
             "show": false
@@ -65391,7 +65740,7 @@
           "hideTimeOverride": false,
           "hideZeroBuckets": true,
           "highlightCards": true,
-          "id": 471,
+          "id": 473,
           "interval": null,
           "legend": {
             "show": false
@@ -65496,7 +65845,7 @@
           "hideTimeOverride": false,
           "hideZeroBuckets": true,
           "highlightCards": true,
-          "id": 472,
+          "id": 474,
           "interval": null,
           "legend": {
             "show": false
@@ -65601,7 +65950,7 @@
           "hideTimeOverride": false,
           "hideZeroBuckets": true,
           "highlightCards": true,
-          "id": 473,
+          "id": 475,
           "interval": null,
           "legend": {
             "show": false
@@ -65699,7 +66048,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 474,
+          "id": 476,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -65847,7 +66196,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 475,
+          "id": 477,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -65980,7 +66329,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 476,
+          "id": 478,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -66113,7 +66462,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 477,
+          "id": 479,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -66261,7 +66610,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 478,
+          "id": 480,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -66397,7 +66746,7 @@
       },
       "height": null,
       "hideTimeOverride": false,
-      "id": 479,
+      "id": 481,
       "interval": null,
       "links": [],
       "maxDataPoints": 100,
@@ -66448,7 +66797,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 480,
+          "id": 482,
           "interval": null,
           "links": [],
           "maxDataPoints": 100,
@@ -66544,7 +66893,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 481,
+          "id": 483,
           "interval": null,
           "links": [],
           "maxDataPoints": 100,
@@ -66619,7 +66968,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 482,
+          "id": 484,
           "interval": null,
           "links": [],
           "maxDataPoints": 100,
@@ -66694,7 +67043,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 483,
+          "id": 485,
           "interval": null,
           "links": [],
           "maxDataPoints": 100,
@@ -66769,7 +67118,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 484,
+          "id": 486,
           "interval": null,
           "links": [],
           "maxDataPoints": 100,
@@ -66844,7 +67193,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 485,
+          "id": 487,
           "interval": null,
           "links": [],
           "maxDataPoints": 100,
@@ -66919,7 +67268,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 486,
+          "id": 488,
           "interval": null,
           "links": [],
           "maxDataPoints": 100,
@@ -66994,7 +67343,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 487,
+          "id": 489,
           "interval": null,
           "links": [],
           "maxDataPoints": 100,
@@ -67073,7 +67422,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 488,
+          "id": 490,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -67206,7 +67555,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 489,
+          "id": 491,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -67339,7 +67688,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 490,
+          "id": 492,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -67472,7 +67821,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 491,
+          "id": 493,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -67605,7 +67954,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 492,
+          "id": 494,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -67738,7 +68087,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 493,
+          "id": 495,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -67886,7 +68235,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 494,
+          "id": 496,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -68019,7 +68368,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 495,
+          "id": 497,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -68152,7 +68501,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 496,
+          "id": 498,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -68318,7 +68667,7 @@
           "hideTimeOverride": false,
           "hideZeroBuckets": true,
           "highlightCards": true,
-          "id": 497,
+          "id": 499,
           "interval": null,
           "legend": {
             "show": false
@@ -68423,7 +68772,7 @@
           "hideTimeOverride": false,
           "hideZeroBuckets": true,
           "highlightCards": true,
-          "id": 498,
+          "id": 500,
           "interval": null,
           "legend": {
             "show": false
@@ -68528,7 +68877,7 @@
           "hideTimeOverride": false,
           "hideZeroBuckets": true,
           "highlightCards": true,
-          "id": 499,
+          "id": 501,
           "interval": null,
           "legend": {
             "show": false
@@ -68633,7 +68982,7 @@
           "hideTimeOverride": false,
           "hideZeroBuckets": true,
           "highlightCards": true,
-          "id": 500,
+          "id": 502,
           "interval": null,
           "legend": {
             "show": false
@@ -68738,7 +69087,7 @@
           "hideTimeOverride": false,
           "hideZeroBuckets": true,
           "highlightCards": true,
-          "id": 501,
+          "id": 503,
           "interval": null,
           "legend": {
             "show": false
@@ -68843,7 +69192,7 @@
           "hideTimeOverride": false,
           "hideZeroBuckets": true,
           "highlightCards": true,
-          "id": 502,
+          "id": 504,
           "interval": null,
           "legend": {
             "show": false
@@ -68948,7 +69297,7 @@
           "hideTimeOverride": false,
           "hideZeroBuckets": true,
           "highlightCards": true,
-          "id": 503,
+          "id": 505,
           "interval": null,
           "legend": {
             "show": false
@@ -69053,7 +69402,7 @@
           "hideTimeOverride": false,
           "hideZeroBuckets": true,
           "highlightCards": true,
-          "id": 504,
+          "id": 506,
           "interval": null,
           "legend": {
             "show": false
@@ -69151,7 +69500,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 505,
+          "id": 507,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -69284,7 +69633,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 506,
+          "id": 508,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -69417,7 +69766,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 507,
+          "id": 509,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -69550,7 +69899,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 508,
+          "id": 510,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -69683,7 +70032,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 509,
+          "id": 511,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -69816,7 +70165,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 510,
+          "id": 512,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -69949,7 +70298,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 511,
+          "id": 513,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -70082,7 +70431,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 512,
+          "id": 514,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -70222,7 +70571,7 @@
           "hideTimeOverride": false,
           "hideZeroBuckets": true,
           "highlightCards": true,
-          "id": 513,
+          "id": 515,
           "interval": null,
           "legend": {
             "show": false
@@ -70327,7 +70676,7 @@
           "hideTimeOverride": false,
           "hideZeroBuckets": true,
           "highlightCards": true,
-          "id": 514,
+          "id": 516,
           "interval": null,
           "legend": {
             "show": false
@@ -70425,7 +70774,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 515,
+          "id": 517,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -70558,7 +70907,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 516,
+          "id": 518,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -70691,7 +71040,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 517,
+          "id": 519,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -70824,7 +71173,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 518,
+          "id": 520,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -70957,7 +71306,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 519,
+          "id": 521,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -71090,7 +71439,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 520,
+          "id": 522,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -71226,7 +71575,7 @@
       },
       "height": null,
       "hideTimeOverride": false,
-      "id": 521,
+      "id": 523,
       "interval": null,
       "links": [],
       "maxDataPoints": 100,
@@ -71265,7 +71614,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 522,
+          "id": 524,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -71413,7 +71762,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 523,
+          "id": 525,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -71546,7 +71895,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 524,
+          "id": 526,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -71679,7 +72028,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 525,
+          "id": 527,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -71815,7 +72164,7 @@
       },
       "height": null,
       "hideTimeOverride": false,
-      "id": 526,
+      "id": 528,
       "interval": null,
       "links": [],
       "maxDataPoints": 100,
@@ -71854,7 +72203,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 527,
+          "id": 529,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -71987,7 +72336,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 528,
+          "id": 530,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -72120,7 +72469,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 529,
+          "id": 531,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -72253,7 +72602,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 530,
+          "id": 532,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -72386,7 +72735,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 531,
+          "id": 533,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -72519,7 +72868,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 532,
+          "id": 534,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -72655,7 +73004,7 @@
       },
       "height": null,
       "hideTimeOverride": false,
-      "id": 533,
+      "id": 535,
       "interval": null,
       "links": [],
       "maxDataPoints": 100,
@@ -72694,7 +73043,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 534,
+          "id": 536,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -72827,7 +73176,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 535,
+          "id": 537,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -72963,7 +73312,7 @@
       },
       "height": null,
       "hideTimeOverride": false,
-      "id": 536,
+      "id": 538,
       "interval": null,
       "links": [],
       "maxDataPoints": 100,
@@ -73002,7 +73351,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 537,
+          "id": 539,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -73203,7 +73552,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 538,
+          "id": 540,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -73339,7 +73688,7 @@
       },
       "height": null,
       "hideTimeOverride": false,
-      "id": 539,
+      "id": 541,
       "interval": null,
       "links": [],
       "maxDataPoints": 100,
@@ -73378,7 +73727,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 540,
+          "id": 542,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -73511,7 +73860,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 541,
+          "id": 543,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -73644,7 +73993,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 542,
+          "id": 544,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -73777,7 +74126,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 543,
+          "id": 545,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -73910,7 +74259,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 544,
+          "id": 546,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -74058,7 +74407,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 545,
+          "id": 547,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -74262,7 +74611,7 @@
       },
       "height": null,
       "hideTimeOverride": false,
-      "id": 546,
+      "id": 548,
       "interval": null,
       "links": [],
       "maxDataPoints": 100,
@@ -74301,7 +74650,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 547,
+          "id": 549,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -74434,7 +74783,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 548,
+          "id": 550,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -74567,7 +74916,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 549,
+          "id": 551,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -74700,7 +75049,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 550,
+          "id": 552,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -74833,7 +75182,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 551,
+          "id": 553,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -75030,7 +75379,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 552,
+          "id": 554,
           "interval": null,
           "links": [],
           "maxDataPoints": 100,
diff --git a/metrics/grafana/tikv_details.json.sha256 b/metrics/grafana/tikv_details.json.sha256
index 2faf9115739..513411203b4 100644
--- a/metrics/grafana/tikv_details.json.sha256
+++ b/metrics/grafana/tikv_details.json.sha256
@@ -1 +1 @@
-f470e00da409e47ddb137011b21448eaf3c0698d82803e4431e1be36bb5fb23a  ./metrics/grafana/tikv_details.json
+a002799e26cac25ad5ffcf9c15414f3837389edb9b1559f91a297457cd22b507  ./metrics/grafana/tikv_details.json

From 2798c83236aab96708d94cc631ad96086062dab4 Mon Sep 17 00:00:00 2001
From: Spade  A <71589810+SpadeA-Tang@users.noreply.github.com>
Date: Mon, 28 Oct 2024 15:31:24 +0800
Subject: [PATCH 10/16] In-memory engine: evict region when destroy peer
 (#17712)

ref tikv/tikv#16141, close tikv/tikv#17644

Evict region when destroy peer

Signed-off-by: SpadeA-Tang <tangchenjie1210@gmail.com>
Signed-off-by: SpadeA-Tang <u6748471@anu.edu.au>
---
 .../engine_traits/src/region_cache_engine.rs  |  1 +
 .../src/observer/load_eviction.rs             | 21 +++++--
 components/in_memory_engine/src/metrics.rs    |  4 ++
 .../raftstore/src/coprocessor/dispatcher.rs   | 26 ++++++++
 components/raftstore/src/coprocessor/mod.rs   |  5 ++
 components/raftstore/src/store/fsm/peer.rs    |  1 +
 .../failpoints/cases/test_in_memory_engine.rs | 60 +++++++++++++++++++
 7 files changed, 114 insertions(+), 4 deletions(-)

diff --git a/components/engine_traits/src/region_cache_engine.rs b/components/engine_traits/src/region_cache_engine.rs
index 85ba49b0654..18ba85d6073 100644
--- a/components/engine_traits/src/region_cache_engine.rs
+++ b/components/engine_traits/src/region_cache_engine.rs
@@ -57,6 +57,7 @@ pub enum EvictReason {
     ApplySnapshot,
     Flashback,
     Manual,
+    PeerDestroy,
 }
 
 /// RegionCacheEngine works as a region cache caching some regions (in Memory or
diff --git a/components/hybrid_engine/src/observer/load_eviction.rs b/components/hybrid_engine/src/observer/load_eviction.rs
index 2aab6491461..c6388ea6b08 100644
--- a/components/hybrid_engine/src/observer/load_eviction.rs
+++ b/components/hybrid_engine/src/observer/load_eviction.rs
@@ -10,10 +10,10 @@ use kvproto::{
 };
 use raft::StateRole;
 use raftstore::coprocessor::{
-    dispatcher::BoxExtraMessageObserver, AdminObserver, ApplyCtxInfo, ApplySnapshotObserver,
-    BoxAdminObserver, BoxApplySnapshotObserver, BoxQueryObserver, BoxRoleObserver, Cmd,
-    Coprocessor, CoprocessorHost, ExtraMessageObserver, ObserverContext, QueryObserver,
-    RegionState, RoleObserver,
+    dispatcher::{BoxDestroyPeerObserver, BoxExtraMessageObserver},
+    AdminObserver, ApplyCtxInfo, ApplySnapshotObserver, BoxAdminObserver, BoxApplySnapshotObserver,
+    BoxQueryObserver, BoxRoleObserver, Cmd, Coprocessor, CoprocessorHost, DestroyPeerObserver,
+    ExtraMessageObserver, ObserverContext, QueryObserver, RegionState, RoleObserver,
 };
 use tikv_util::info;
 
@@ -53,6 +53,10 @@ impl LoadEvictionObserver {
         coprocessor_host
             .registry
             .register_extra_message_observer(priority, BoxExtraMessageObserver::new(self.clone()));
+        // Eviction the cached region when the peer is destroyed.
+        coprocessor_host
+            .registry
+            .register_destroy_peer_observer(priority, BoxDestroyPeerObserver::new(self.clone()));
     }
 
     fn post_exec_cmd(
@@ -268,6 +272,15 @@ impl ExtraMessageObserver for LoadEvictionObserver {
     }
 }
 
+impl DestroyPeerObserver for LoadEvictionObserver {
+    fn on_destroy_peer(&self, r: &Region) {
+        self.cache_engine.on_region_event(RegionEvent::Eviction {
+            region: CacheRegion::from_region(r),
+            reason: EvictReason::PeerDestroy,
+        });
+    }
+}
+
 #[cfg(test)]
 mod tests {
     use std::sync::Mutex;
diff --git a/components/in_memory_engine/src/metrics.rs b/components/in_memory_engine/src/metrics.rs
index 7743c5ff09b..9b037c1c914 100644
--- a/components/in_memory_engine/src/metrics.rs
+++ b/components/in_memory_engine/src/metrics.rs
@@ -43,6 +43,7 @@ make_auto_flush_static_metric! {
         apply_snapshot,
         flashback,
         manual,
+        destroy_peer,
     }
 
     pub label_enum OperationType {
@@ -260,6 +261,9 @@ pub(crate) fn observe_eviction_duration(secs: f64, evict_reason: EvictReason) {
         EvictReason::Manual => IN_MEMORY_ENGINE_EVICTION_DURATION_HISTOGRAM_STATIC
             .manual
             .observe(secs),
+        EvictReason::PeerDestroy => IN_MEMORY_ENGINE_EVICTION_DURATION_HISTOGRAM_STATIC
+            .destroy_peer
+            .observe(secs),
     }
 }
 
diff --git a/components/raftstore/src/coprocessor/dispatcher.rs b/components/raftstore/src/coprocessor/dispatcher.rs
index 258421c751d..adc8cf3e38c 100644
--- a/components/raftstore/src/coprocessor/dispatcher.rs
+++ b/components/raftstore/src/coprocessor/dispatcher.rs
@@ -315,6 +315,11 @@ impl_box_observer!(
     SnapshotObserver,
     WrappedBoxSnapshotObserver
 );
+impl_box_observer!(
+    BoxDestroyPeerObserver,
+    DestroyPeerObserver,
+    WrappedBoxDestroyPeerObserver
+);
 
 /// Registry contains all registered coprocessors.
 #[derive(Clone)]
@@ -336,6 +341,7 @@ where
     raft_message_observers: Vec<Entry<BoxRaftMessageObserver>>,
     extra_message_observers: Vec<Entry<BoxExtraMessageObserver>>,
     region_heartbeat_observers: Vec<Entry<BoxRegionHeartbeatObserver>>,
+    destroy_peer_observers: Vec<Entry<BoxDestroyPeerObserver>>,
     // For now, `write_batch_observer` and `snapshot_observer` can only have one
     // observer solely because of simplicity. However, it is possible to have
     // multiple observers in the future if needed.
@@ -361,6 +367,7 @@ impl<E: KvEngine> Default for Registry<E> {
             raft_message_observers: Default::default(),
             extra_message_observers: Default::default(),
             region_heartbeat_observers: Default::default(),
+            destroy_peer_observers: Default::default(),
             write_batch_observer: None,
             snapshot_observer: None,
         }
@@ -448,6 +455,14 @@ impl<E: KvEngine> Registry<E> {
         push!(priority, qo, self.region_heartbeat_observers);
     }
 
+    pub fn register_destroy_peer_observer(
+        &mut self,
+        priority: u32,
+        destroy_peer_observer: BoxDestroyPeerObserver,
+    ) {
+        push!(priority, destroy_peer_observer, self.destroy_peer_observers);
+    }
+
     pub fn register_write_batch_observer(&mut self, write_batch_observer: BoxWriteBatchObserver) {
         self.write_batch_observer = Some(write_batch_observer);
     }
@@ -973,6 +988,17 @@ impl<E: KvEngine> CoprocessorHost<E> {
         WriteBatchWrapper::new(wb, observable_wb)
     }
 
+    pub fn on_destroy_peer(&self, region: &Region) {
+        if self.registry.destroy_peer_observers.is_empty() {
+            return;
+        }
+
+        for observer in &self.registry.destroy_peer_observers {
+            let observer = observer.observer.inner();
+            observer.on_destroy_peer(region);
+        }
+    }
+
     pub fn on_snapshot(
         &self,
         region: &Region,
diff --git a/components/raftstore/src/coprocessor/mod.rs b/components/raftstore/src/coprocessor/mod.rs
index 2b6d5186f0b..b17f6df5ad3 100644
--- a/components/raftstore/src/coprocessor/mod.rs
+++ b/components/raftstore/src/coprocessor/mod.rs
@@ -611,6 +611,11 @@ pub trait UpdateSafeTsObserver: Coprocessor {
     fn on_update_safe_ts(&self, _: u64, _: u64, _: u64) {}
 }
 
+pub trait DestroyPeerObserver: Coprocessor {
+    /// Hook to call when destroying a peer.
+    fn on_destroy_peer(&self, _: &Region) {}
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;
diff --git a/components/raftstore/src/store/fsm/peer.rs b/components/raftstore/src/store/fsm/peer.rs
index 1d76c9f8872..d5ec227e849 100644
--- a/components/raftstore/src/store/fsm/peer.rs
+++ b/components/raftstore/src/store/fsm/peer.rs
@@ -3952,6 +3952,7 @@ where
 
     // [PerformanceCriticalPath] TODO: spin off the I/O code (self.fsm.peer.destroy)
     fn destroy_peer(&mut self, merged_by_target: bool) -> bool {
+        self.ctx.coprocessor_host.on_destroy_peer(self.region());
         fail_point!("destroy_peer");
         // Mark itself as pending_remove
         self.fsm.peer.pending_remove = true;
diff --git a/tests/failpoints/cases/test_in_memory_engine.rs b/tests/failpoints/cases/test_in_memory_engine.rs
index a0a5efb4b78..42b23154ed3 100644
--- a/tests/failpoints/cases/test_in_memory_engine.rs
+++ b/tests/failpoints/cases/test_in_memory_engine.rs
@@ -19,6 +19,7 @@ use kvproto::{
     import_sstpb::SstMeta,
     kvrpcpb::Context,
     raft_cmdpb::{AdminCmdType, CmdType, RaftCmdRequest, RaftRequestHeader, Request},
+    raft_serverpb::RaftMessage,
 };
 use protobuf::Message;
 use raftstore::{
@@ -962,3 +963,62 @@ fn test_apply_prepared_but_not_write() {
 
     assert_eq!(cluster.must_get(b"k1").unwrap(), b"v2");
 }
+
+#[test]
+fn test_eviction_when_destroy_peer() {
+    let mut cluster = new_server_cluster_with_hybrid_engine_with_no_region_cache(0, 1);
+    cluster.run();
+
+    let t1 = ProductTable::new();
+    let t2 = ProductTable::new();
+
+    let key = t2.get_table_prefix();
+    let split_key = Key::from_raw(&key).into_encoded();
+    let r = cluster.get_region(&split_key);
+    cluster.must_split(&r, &split_key);
+    let r = cluster.get_region(&split_key);
+
+    let (tx, rx) = sync_channel(0);
+    fail::cfg_callback("ime_on_snapshot_load_finished", move || {
+        tx.send(true).unwrap();
+    })
+    .unwrap();
+    {
+        let region_cache_engine = cluster.sim.rl().get_region_cache_engine(1);
+        let cache_region = CacheRegion::from_region(&r);
+        region_cache_engine
+            .core()
+            .region_manager()
+            .load_region(cache_region)
+            .unwrap();
+    }
+
+    must_copr_load_data(&mut cluster, &t1, 1);
+    must_copr_load_data(&mut cluster, &t2, 1);
+
+    rx.recv_timeout(Duration::from_secs(2)).unwrap();
+
+    let (tx, rx) = sync_channel(0);
+    fail::cfg_callback("destroy_peer", move || {
+        tx.send(true).unwrap();
+    })
+    .unwrap();
+
+    let router = cluster.get_router(1).unwrap();
+    let mut raft_msg = RaftMessage::default();
+    raft_msg.set_region_id(r.get_id());
+    raft_msg.set_is_tombstone(true);
+    raft_msg.set_from_peer(new_peer(0, 0));
+    raft_msg.set_to_peer(r.get_peers()[0].clone());
+    let mut epoch = r.get_region_epoch().clone();
+    epoch.set_version(epoch.get_version() + 1);
+    raft_msg.set_region_epoch(epoch);
+    router.send_raft_message(raft_msg).unwrap();
+
+    rx.recv_timeout(Duration::from_secs(2)).unwrap();
+
+    {
+        let region_cache_engine = cluster.sim.rl().get_region_cache_engine(1);
+        assert!(!region_cache_engine.region_cached(&r));
+    }
+}

From a18c8d384507dcfcb08dab81efd5c9eabf0ad7d5 Mon Sep 17 00:00:00 2001
From: Spade  A <71589810+SpadeA-Tang@users.noreply.github.com>
Date: Mon, 28 Oct 2024 21:08:36 +0800
Subject: [PATCH 11/16] *: fix compile fail for test (#17706)

close tikv/tikv#17707

fix compile fail for test

Signed-off-by: SpadeA-Tang <tangchenjie1210@gmail.com>

Co-authored-by: ti-chi-bot[bot] <108142056+ti-chi-bot[bot]@users.noreply.github.com>
---
 components/hybrid_engine/Cargo.toml        | 3 ++-
 components/raftstore/src/store/fsm/peer.rs | 4 ++++
 2 files changed, 6 insertions(+), 1 deletion(-)

diff --git a/components/hybrid_engine/Cargo.toml b/components/hybrid_engine/Cargo.toml
index 1c4680e65ce..883a0e2e128 100644
--- a/components/hybrid_engine/Cargo.toml
+++ b/components/hybrid_engine/Cargo.toml
@@ -6,7 +6,7 @@ publish = false
 license = "Apache-2.0"
 
 [features]
-testexport = ["raftstore/testexport"]
+testexport = []
 failpoints = ["fail/failpoints"]
 
 [[test]]
@@ -39,3 +39,4 @@ keys = { workspace = true }
 [dev-dependencies]
 tempfile = "3.0"
 test_util = { workspace = true }
+raftstore = { workspace = true, features = ["testexport"] }
diff --git a/components/raftstore/src/store/fsm/peer.rs b/components/raftstore/src/store/fsm/peer.rs
index d5ec227e849..5f5a9f6b773 100644
--- a/components/raftstore/src/store/fsm/peer.rs
+++ b/components/raftstore/src/store/fsm/peer.rs
@@ -2663,21 +2663,25 @@ where
                 && !is_initialized_peer
                 && msg_type == target_msg_type
         };
+        #[cfg(feature = "failpoints")]
         fail_point!(
             "on_snap_msg_1000_2",
             fp_enable(MessageType::MsgSnapshot),
             |_| Ok(())
         );
+        #[cfg(feature = "failpoints")]
         fail_point!(
             "on_vote_msg_1000_2",
             fp_enable(MessageType::MsgRequestVote),
             |_| Ok(())
         );
+        #[cfg(feature = "failpoints")]
         fail_point!(
             "on_append_msg_1000_2",
             fp_enable(MessageType::MsgAppend),
             |_| Ok(())
         );
+        #[cfg(feature = "failpoints")]
         fail_point!(
             "on_heartbeat_msg_1000_2",
             fp_enable(MessageType::MsgHeartbeat),

From 88880858b9af7e6f99801f4dca9f118c382eaa1b Mon Sep 17 00:00:00 2001
From: Yang Zhang <yang.zhang@pingcap.com>
Date: Mon, 28 Oct 2024 23:46:46 -0700
Subject: [PATCH 12/16] RocksDB: Fix SST ingestion (#17726)

close tikv/tikv#17691

Update RocksDB to include fixes for ingestion

* https://github.com/tikv/rocksdb/commit/31eb76b1bc82740c9b85a2462804130857df0577
* https://github.com/tikv/rocksdb/commit/cf9cd5b89217fe49d22194af8ae67aaecf0ba231

Signed-off-by: Yang Zhang <yang.zhang@pingcap.com>
---
 Cargo.lock | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index a8f81c05a4e..64e1d923080 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -2883,7 +2883,7 @@ dependencies = [
 [[package]]
 name = "librocksdb_sys"
 version = "0.1.0"
-source = "git+https://github.com/tikv/rust-rocksdb.git#78b26b39bc179a0358ad0e5611b9d9934edabefd"
+source = "git+https://github.com/tikv/rust-rocksdb.git#f728853552a1419c665c17ea6155f1c4f828c36c"
 dependencies = [
  "bindgen 0.65.1",
  "bzip2-sys",
@@ -2902,7 +2902,7 @@ dependencies = [
 [[package]]
 name = "libtitan_sys"
 version = "0.0.1"
-source = "git+https://github.com/tikv/rust-rocksdb.git#78b26b39bc179a0358ad0e5611b9d9934edabefd"
+source = "git+https://github.com/tikv/rust-rocksdb.git#f728853552a1419c665c17ea6155f1c4f828c36c"
 dependencies = [
  "bzip2-sys",
  "cc",
@@ -4704,7 +4704,7 @@ dependencies = [
 [[package]]
 name = "rocksdb"
 version = "0.3.0"
-source = "git+https://github.com/tikv/rust-rocksdb.git#78b26b39bc179a0358ad0e5611b9d9934edabefd"
+source = "git+https://github.com/tikv/rust-rocksdb.git#f728853552a1419c665c17ea6155f1c4f828c36c"
 dependencies = [
  "libc 0.2.151",
  "librocksdb_sys",

From 43101c5bf328e0912968777f8bf7f3a1c7c8ca10 Mon Sep 17 00:00:00 2001
From: Spade  A <71589810+SpadeA-Tang@users.noreply.github.com>
Date: Tue, 29 Oct 2024 16:06:03 +0800
Subject: [PATCH 13/16] compaction-filter: add test for triggering compaction
 in CheckAndCompact (#17725)

close tikv/tikv#17269

add test for triggering compaction in CheckAndCompact

Signed-off-by: SpadeA-Tang <tangchenjie1210@gmail.com>
---
 .../raftstore/src/store/worker/compact.rs     | 39 +++++++++++++++++++
 1 file changed, 39 insertions(+)

diff --git a/components/raftstore/src/store/worker/compact.rs b/components/raftstore/src/store/worker/compact.rs
index 0c1fc74ddab..3bf6b1358e4 100644
--- a/components/raftstore/src/store/worker/compact.rs
+++ b/components/raftstore/src/store/worker/compact.rs
@@ -870,4 +870,43 @@ mod tests {
             .unwrap();
         assert_eq!(stats.num_entries - stats.num_versions, 0);
     }
+
+    #[test]
+    fn test_need_compact() {
+        // many tombstone case
+        let range_stats = RangeStats {
+            num_entries: 1000,
+            num_versions: 200,
+            num_deletes: 0,
+            num_rows: 200,
+        };
+        assert!(need_compact(
+            &range_stats,
+            &CompactThreshold::new(10, 30, 100, 100)
+        ));
+
+        // many mvcc put case
+        let range_stats = RangeStats {
+            num_entries: 1000,
+            num_versions: 1000,
+            num_deletes: 0,
+            num_rows: 200,
+        };
+        assert!(need_compact(
+            &range_stats,
+            &CompactThreshold::new(100, 100, 100, 30)
+        ));
+
+        // many mvcc delete case
+        let range_stats = RangeStats {
+            num_entries: 1000,
+            num_versions: 1000,
+            num_deletes: 800,
+            num_rows: 1000,
+        };
+        assert!(need_compact(
+            &range_stats,
+            &CompactThreshold::new(100, 100, 100, 30)
+        ));
+    }
 }

From 31a21609bc35849183ff0d0946e181ebae0c9c73 Mon Sep 17 00:00:00 2001
From: Neil Shen <overvenus@gmail.com>
Date: Tue, 29 Oct 2024 16:31:01 +0800
Subject: [PATCH 14/16] In-memory Engine: Remove boilerplate code (#17380)

ref tikv/tikv#17181

This commit is a follow-up of #17359. Since `HybridEngine` is now
integrated into TiKV through coprocessor observers and RaftKv, it no
longer needs to implement the Engine traits. This commit removes the
unnecessary boilerplate code for `HybridEngine`.

Signed-off-by: Neil Shen <overvenus@gmail.com>

Co-authored-by: ti-chi-bot[bot] <108142056+ti-chi-bot[bot]@users.noreply.github.com>
---
 Cargo.toml                                    |   1 -
 components/backup-stream/src/observer.rs      |   6 +-
 components/cdc/src/observer.rs                |   8 +-
 .../cdc/tests/failpoints/test_register.rs     |   2 +-
 components/engine_panic/src/engine.rs         |   2 +-
 components/engine_traits/src/engine.rs        |  13 -
 .../engine_traits/src/region_cache_engine.rs  |   8 +-
 components/engine_traits/src/snapshot.rs      |   7 +-
 components/engine_traits/src/write_batch.rs   |   6 -
 components/hybrid_engine/Cargo.toml           |  12 +-
 components/hybrid_engine/src/cf_names.rs      |  15 -
 components/hybrid_engine/src/cf_options.rs    |  21 --
 components/hybrid_engine/src/checkpoint.rs    |  22 --
 components/hybrid_engine/src/compact.rs       |  65 ----
 components/hybrid_engine/src/db_options.rs    |  21 --
 components/hybrid_engine/src/db_vector.rs     |   4 +-
 components/hybrid_engine/src/engine.rs        | 159 ++-------
 .../hybrid_engine/src/engine_iterator.rs      |   4 +-
 .../hybrid_engine/src/flow_control_factors.rs |  23 --
 .../hybrid_engine/src/hybrid_metrics.rs       |  25 --
 components/hybrid_engine/src/import.rs        |  17 -
 components/hybrid_engine/src/iterable.rs      |  19 --
 components/hybrid_engine/src/lib.rs           |  24 +-
 components/hybrid_engine/src/metrics.rs       |   2 +-
 components/hybrid_engine/src/misc.rs          | 222 -------------
 .../hybrid_engine/src/mvcc_properties.rs      |  23 --
 .../src/observer/load_eviction.rs             |   8 +-
 components/hybrid_engine/src/observer/mod.rs  |   2 +
 .../src/observer/test_write_batch.rs          | 278 ++++++++++++++++
 .../hybrid_engine/src/observer/write_batch.rs |  30 +-
 components/hybrid_engine/src/perf_context.rs  |  20 --
 .../hybrid_engine/src/range_properties.rs     |  60 ----
 .../hybrid_engine/src/region_cache_engine.rs  |  26 --
 components/hybrid_engine/src/snapshot.rs      |  25 +-
 components/hybrid_engine/src/sst.rs           |  53 ---
 .../hybrid_engine/src/table_properties.rs     |  21 --
 .../hybrid_engine/src/ttl_properties.rs       |  21 --
 components/hybrid_engine/src/util.rs          |   2 +-
 components/hybrid_engine/src/write_batch.rs   | 302 ------------------
 .../hybrid_engine/tests/failpoints/mod.rs     |   3 -
 .../tests/failpoints/test_write_batch.rs      | 102 ------
 .../in_memory_engine/src/write_batch.rs       |  62 ++--
 .../raftstore/src/coprocessor/dispatcher.rs   |   2 +-
 components/raftstore/src/coprocessor/mod.rs   |   3 +-
 .../src/coprocessor/read_write/write_batch.rs |  24 +-
 components/server/src/server.rs               |   5 +-
 components/test_raftstore/src/cluster.rs      |  46 ---
 components/test_raftstore/src/lib.rs          |   1 -
 .../test_raftstore/src/region_cache_engine.rs |  22 --
 components/test_raftstore/src/server.rs       |   8 +-
 components/tikv_kv/src/lib.rs                 |   7 +-
 components/tikv_kv/src/raftstore_impls.rs     |   4 +-
 metrics/grafana/tikv_details.dashboard.py     |   4 +-
 metrics/grafana/tikv_details.json             |   8 +-
 metrics/grafana/tikv_details.json.sha256      |   2 +-
 src/coprocessor/endpoint.rs                   |   2 +-
 src/coprocessor/metrics.rs                    |   2 +-
 src/coprocessor/tracker.rs                    |   8 +-
 src/server/lock_manager/deadlock.rs           |  14 +-
 .../failpoints/cases/test_in_memory_engine.rs |  44 +--
 .../integrations/raftstore/test_lease_read.rs |   3 -
 61 files changed, 461 insertions(+), 1494 deletions(-)
 delete mode 100644 components/hybrid_engine/src/cf_names.rs
 delete mode 100644 components/hybrid_engine/src/cf_options.rs
 delete mode 100644 components/hybrid_engine/src/checkpoint.rs
 delete mode 100644 components/hybrid_engine/src/compact.rs
 delete mode 100644 components/hybrid_engine/src/db_options.rs
 delete mode 100644 components/hybrid_engine/src/flow_control_factors.rs
 delete mode 100644 components/hybrid_engine/src/hybrid_metrics.rs
 delete mode 100644 components/hybrid_engine/src/import.rs
 delete mode 100644 components/hybrid_engine/src/iterable.rs
 delete mode 100644 components/hybrid_engine/src/misc.rs
 delete mode 100644 components/hybrid_engine/src/mvcc_properties.rs
 create mode 100644 components/hybrid_engine/src/observer/test_write_batch.rs
 delete mode 100644 components/hybrid_engine/src/perf_context.rs
 delete mode 100644 components/hybrid_engine/src/range_properties.rs
 delete mode 100644 components/hybrid_engine/src/region_cache_engine.rs
 delete mode 100644 components/hybrid_engine/src/sst.rs
 delete mode 100644 components/hybrid_engine/src/table_properties.rs
 delete mode 100644 components/hybrid_engine/src/ttl_properties.rs
 delete mode 100644 components/hybrid_engine/src/write_batch.rs
 delete mode 100644 components/hybrid_engine/tests/failpoints/mod.rs
 delete mode 100644 components/hybrid_engine/tests/failpoints/test_write_batch.rs
 delete mode 100644 components/test_raftstore/src/region_cache_engine.rs

diff --git a/Cargo.toml b/Cargo.toml
index 5bc6344553c..317fe797421 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -35,7 +35,6 @@ testexport = [
   "engine_traits/testexport",
   "engine_rocks/testexport",
   "engine_panic/testexport",
-  "hybrid_engine/testexport",
   "encryption/testexport",
 ]
 test-engine-kv-rocksdb = ["engine_test/test-engine-kv-rocksdb"]
diff --git a/components/backup-stream/src/observer.rs b/components/backup-stream/src/observer.rs
index e72cba90cd8..071f09cf3d9 100644
--- a/components/backup-stream/src/observer.rs
+++ b/components/backup-stream/src/observer.rs
@@ -296,7 +296,7 @@ mod tests {
         // Test region out of range won't be added to observe list.
         let r = fake_region(43, b"0010", b"0042");
         let mut ctx = ObserverContext::new(&r);
-        o.on_role_change(&mut ctx, &RoleChange::new(StateRole::Leader));
+        o.on_role_change(&mut ctx, &RoleChange::new_for_test(StateRole::Leader));
         let task = rx.recv_timeout(Duration::from_millis(20));
         assert!(task.is_err(), "it is {:?}", task);
         assert!(!subs.is_observing(43));
@@ -311,7 +311,7 @@ mod tests {
         // Test give up subscripting when become follower.
         let r = fake_region(42, b"0008", b"0009");
         let mut ctx = ObserverContext::new(&r);
-        o.on_role_change(&mut ctx, &RoleChange::new(StateRole::Follower));
+        o.on_role_change(&mut ctx, &RoleChange::new_for_test(StateRole::Follower));
         let task = rx.recv_timeout(Duration::from_millis(20));
         assert_matches!(
             task,
@@ -333,7 +333,7 @@ mod tests {
             RegionChangeEvent::Update(RegionChangeReason::Split),
             StateRole::Leader,
         );
-        o.on_role_change(&mut ctx, &RoleChange::new(StateRole::Leader));
+        o.on_role_change(&mut ctx, &RoleChange::new_for_test(StateRole::Leader));
         let task = rx.recv_timeout(Duration::from_millis(20));
         assert!(task.is_err(), "it is {:?}", task);
     }
diff --git a/components/cdc/src/observer.rs b/components/cdc/src/observer.rs
index 061dd9daa36..fda4a456217 100644
--- a/components/cdc/src/observer.rs
+++ b/components/cdc/src/observer.rs
@@ -263,7 +263,7 @@ mod tests {
         region.mut_peers().push(new_peer(3, 3));
 
         let mut ctx = ObserverContext::new(&region);
-        observer.on_role_change(&mut ctx, &RoleChange::new(StateRole::Follower));
+        observer.on_role_change(&mut ctx, &RoleChange::new_for_test(StateRole::Follower));
         rx.recv_timeout(Duration::from_millis(10)).unwrap_err();
 
         let oid = ObserveId::new();
@@ -329,7 +329,7 @@ mod tests {
         };
 
         // No event if it changes to leader.
-        observer.on_role_change(&mut ctx, &RoleChange::new(StateRole::Leader));
+        observer.on_role_change(&mut ctx, &RoleChange::new_for_test(StateRole::Leader));
         rx.recv_timeout(Duration::from_millis(10)).unwrap_err();
 
         // unsubscribed fail if observer id is different.
@@ -338,13 +338,13 @@ mod tests {
         // No event if it is unsubscribed.
         let oid_ = observer.unsubscribe_region(1, oid).unwrap();
         assert_eq!(oid_, oid);
-        observer.on_role_change(&mut ctx, &RoleChange::new(StateRole::Follower));
+        observer.on_role_change(&mut ctx, &RoleChange::new_for_test(StateRole::Follower));
         rx.recv_timeout(Duration::from_millis(10)).unwrap_err();
 
         // No event if it is unsubscribed.
         region.set_id(999);
         let mut ctx = ObserverContext::new(&region);
-        observer.on_role_change(&mut ctx, &RoleChange::new(StateRole::Follower));
+        observer.on_role_change(&mut ctx, &RoleChange::new_for_test(StateRole::Follower));
         rx.recv_timeout(Duration::from_millis(10)).unwrap_err();
     }
 }
diff --git a/components/cdc/tests/failpoints/test_register.rs b/components/cdc/tests/failpoints/test_register.rs
index 2b6be3744af..2128dff08e1 100644
--- a/components/cdc/tests/failpoints/test_register.rs
+++ b/components/cdc/tests/failpoints/test_register.rs
@@ -97,7 +97,7 @@ fn test_region_ready_after_deregister_impl<F: KvFormat>() {
         .obs
         .get(&leader.get_store_id())
         .unwrap()
-        .on_role_change(&mut context, &RoleChange::new(StateRole::Follower));
+        .on_role_change(&mut context, &RoleChange::new_for_test(StateRole::Follower));
 
     // Then CDC should not panic
     fail::remove(fp);
diff --git a/components/engine_panic/src/engine.rs b/components/engine_panic/src/engine.rs
index 52dc8cfb499..950165b7bb2 100644
--- a/components/engine_panic/src/engine.rs
+++ b/components/engine_panic/src/engine.rs
@@ -2,7 +2,7 @@
 
 use engine_traits::{
     IterMetricsCollector, IterOptions, Iterable, Iterator, KvEngine, MetricsExt, Peekable,
-    ReadOptions, Result, SnapshotContext, SyncMutable, WriteOptions,
+    ReadOptions, Result, SyncMutable, WriteOptions,
 };
 
 use crate::{db_vector::PanicDbVector, snapshot::PanicSnapshot, write_batch::PanicWriteBatch};
diff --git a/components/engine_traits/src/engine.rs b/components/engine_traits/src/engine.rs
index a797ae7feb2..cc90f2ce075 100644
--- a/components/engine_traits/src/engine.rs
+++ b/components/engine_traits/src/engine.rs
@@ -78,16 +78,3 @@ pub trait KvEngine:
     #[cfg(feature = "testexport")]
     fn inner_refcount(&self) -> usize;
 }
-
-#[derive(Debug, Clone)]
-pub struct SnapshotContext {
-    pub region: Option<CacheRegion>,
-    pub read_ts: u64,
-}
-
-impl SnapshotContext {
-    pub fn set_region(&mut self, region: CacheRegion) {
-        assert!(self.region.is_none());
-        self.region = Some(region);
-    }
-}
diff --git a/components/engine_traits/src/region_cache_engine.rs b/components/engine_traits/src/region_cache_engine.rs
index 18ba85d6073..757629d3fdd 100644
--- a/components/engine_traits/src/region_cache_engine.rs
+++ b/components/engine_traits/src/region_cache_engine.rs
@@ -93,13 +93,13 @@ pub trait RegionCacheEngine:
 }
 
 pub trait RegionCacheEngineExt {
-    // TODO(SpadeA): try to find a better way to reduce coupling degree of range
-    // cache engine and kv engine
+    // TODO(SpadeA): try to find a better way to reduce coupling degree of
+    // region cache engine and kv engine
     fn on_region_event(&self, event: RegionEvent);
 
-    fn region_cached(&self, range: &Region) -> bool;
+    fn region_cached(&self, region: &Region) -> bool;
 
-    fn load_region(&self, range: &Region);
+    fn load_region(&self, region: &Region);
 }
 
 /// A service that should run in the background to retrieve and apply cache
diff --git a/components/engine_traits/src/snapshot.rs b/components/engine_traits/src/snapshot.rs
index fcf9b891c92..e8f61fc9d68 100644
--- a/components/engine_traits/src/snapshot.rs
+++ b/components/engine_traits/src/snapshot.rs
@@ -13,10 +13,9 @@ where
     Self:
         'static + Peekable + Iterable + CfNamesExt + SnapshotMiscExt + Send + Sync + Sized + Debug,
 {
-    /// Whether the snapshot acquired hit the cached region in the region cache
-    /// engine. It always returns false if the region cahce engine is not
-    /// enabled.
-    fn region_cache_engine_hit(&self) -> bool {
+    /// Whether the snapshot acquired hit the in memory engine. It always
+    /// returns false if the in memory engine is disabled.
+    fn in_memory_engine_hit(&self) -> bool {
         false
     }
 }
diff --git a/components/engine_traits/src/write_batch.rs b/components/engine_traits/src/write_batch.rs
index e1160852f93..222f7045b6c 100644
--- a/components/engine_traits/src/write_batch.rs
+++ b/components/engine_traits/src/write_batch.rs
@@ -1,7 +1,5 @@
 // Copyright 2019 TiKV Project Authors. Licensed under Apache-2.0.
 
-use kvproto::metapb;
-
 use crate::{errors::Result, options::WriteOptions};
 
 /// Engines that can create write batches
@@ -138,8 +136,4 @@ pub trait WriteBatch: Mutable {
     fn merge(&mut self, src: Self) -> Result<()>
     where
         Self: Sized;
-
-    /// It declares that the following consecutive write will be within this
-    /// region.
-    fn prepare_for_region(&mut self, _: &metapb::Region) {}
 }
diff --git a/components/hybrid_engine/Cargo.toml b/components/hybrid_engine/Cargo.toml
index 883a0e2e128..3a23f9927da 100644
--- a/components/hybrid_engine/Cargo.toml
+++ b/components/hybrid_engine/Cargo.toml
@@ -5,15 +5,6 @@ edition = "2021"
 publish = false
 license = "Apache-2.0"
 
-[features]
-testexport = []
-failpoints = ["fail/failpoints"]
-
-[[test]]
-name = "failpoints"
-path = "tests/failpoints/mod.rs"
-required-features = ["failpoints"]
-
 [dependencies]
 engine_traits = { workspace = true }
 txn_types = { workspace = true }
@@ -30,7 +21,6 @@ prometheus = { version = "0.13", default-features = false, features = [
 prometheus-static-metric = "0.5"
 lazy_static = "1.4.0"
 crossbeam = { workspace = true }
-fail = "0.5"
 raftstore = { workspace = true }
 raft = { workspace = true }
 kvproto = { workspace = true }
@@ -39,4 +29,4 @@ keys = { workspace = true }
 [dev-dependencies]
 tempfile = "3.0"
 test_util = { workspace = true }
-raftstore = { workspace = true, features = ["testexport"] }
+fail = { version = "0.5", features = ["failpoints"] }
diff --git a/components/hybrid_engine/src/cf_names.rs b/components/hybrid_engine/src/cf_names.rs
deleted file mode 100644
index 990fb4d0f76..00000000000
--- a/components/hybrid_engine/src/cf_names.rs
+++ /dev/null
@@ -1,15 +0,0 @@
-// Copyright 2023 TiKV Project Authors. Licensed under Apache-2.0.
-
-use engine_traits::{CfNamesExt, KvEngine, RegionCacheEngine};
-
-use crate::engine::HybridEngine;
-
-impl<EK, EC> CfNamesExt for HybridEngine<EK, EC>
-where
-    EK: KvEngine,
-    EC: RegionCacheEngine,
-{
-    fn cf_names(&self) -> Vec<&str> {
-        self.disk_engine().cf_names()
-    }
-}
diff --git a/components/hybrid_engine/src/cf_options.rs b/components/hybrid_engine/src/cf_options.rs
deleted file mode 100644
index 61fe08da536..00000000000
--- a/components/hybrid_engine/src/cf_options.rs
+++ /dev/null
@@ -1,21 +0,0 @@
-// Copyright 2023 TiKV Project Authors. Licensed under Apache-2.0.
-
-use engine_traits::{CfOptionsExt, KvEngine, RegionCacheEngine, Result};
-
-use crate::engine::HybridEngine;
-
-impl<EK, EC> CfOptionsExt for HybridEngine<EK, EC>
-where
-    EK: KvEngine,
-    EC: RegionCacheEngine,
-{
-    type CfOptions = EK::CfOptions;
-
-    fn get_options_cf(&self, cf: &str) -> Result<Self::CfOptions> {
-        self.disk_engine().get_options_cf(cf)
-    }
-
-    fn set_options_cf(&self, cf: &str, options: &[(&str, &str)]) -> Result<()> {
-        self.disk_engine().set_options_cf(cf, options)
-    }
-}
diff --git a/components/hybrid_engine/src/checkpoint.rs b/components/hybrid_engine/src/checkpoint.rs
deleted file mode 100644
index 7d9bdb022ea..00000000000
--- a/components/hybrid_engine/src/checkpoint.rs
+++ /dev/null
@@ -1,22 +0,0 @@
-// Copyright 2023 TiKV Project Authors. Licensed under Apache-2.0.
-
-use engine_traits::{Checkpointable, KvEngine, RegionCacheEngine, Result};
-
-use crate::engine::HybridEngine;
-
-impl<EK, EC> Checkpointable for HybridEngine<EK, EC>
-where
-    EK: KvEngine,
-    EC: RegionCacheEngine,
-{
-    type Checkpointer = EK::Checkpointer;
-
-    fn new_checkpointer(&self) -> Result<Self::Checkpointer> {
-        self.disk_engine().new_checkpointer()
-    }
-
-    fn merge(&self, dbs: &[&Self]) -> Result<()> {
-        let disk_dbs: Vec<_> = dbs.iter().map(|&db| db.disk_engine()).collect();
-        self.disk_engine().merge(&disk_dbs)
-    }
-}
diff --git a/components/hybrid_engine/src/compact.rs b/components/hybrid_engine/src/compact.rs
deleted file mode 100644
index 63cf8c54cb0..00000000000
--- a/components/hybrid_engine/src/compact.rs
+++ /dev/null
@@ -1,65 +0,0 @@
-// Copyright 2023 TiKV Project Authors. Licensed under Apache-2.0.
-
-use engine_traits::{CompactExt, KvEngine, ManualCompactionOptions, RegionCacheEngine, Result};
-
-use crate::engine::HybridEngine;
-
-impl<EK, EC> CompactExt for HybridEngine<EK, EC>
-where
-    EK: KvEngine,
-    EC: RegionCacheEngine,
-{
-    type CompactedEvent = EK::CompactedEvent;
-
-    fn auto_compactions_is_disabled(&self) -> Result<bool> {
-        self.disk_engine().auto_compactions_is_disabled()
-    }
-
-    fn compact_range_cf(
-        &self,
-        cf: &str,
-        start_key: Option<&[u8]>,
-        end_key: Option<&[u8]>,
-        compaction_option: ManualCompactionOptions,
-    ) -> Result<()> {
-        self.disk_engine()
-            .compact_range_cf(cf, start_key, end_key, compaction_option)
-    }
-
-    fn compact_files_in_range_cf(
-        &self,
-        cf: &str,
-        start: Option<&[u8]>,
-        end: Option<&[u8]>,
-        output_level: Option<i32>,
-    ) -> Result<()> {
-        self.disk_engine()
-            .compact_files_in_range_cf(cf, start, end, output_level)
-    }
-
-    fn compact_files_in_range(
-        &self,
-        start: Option<&[u8]>,
-        end: Option<&[u8]>,
-        output_level: Option<i32>,
-    ) -> Result<()> {
-        self.disk_engine()
-            .compact_files_in_range(start, end, output_level)
-    }
-
-    fn compact_files_cf(
-        &self,
-        cf: &str,
-        files: Vec<String>,
-        output_level: Option<i32>,
-        max_subcompactions: u32,
-        exclude_l0: bool,
-    ) -> Result<()> {
-        self.disk_engine()
-            .compact_files_cf(cf, files, output_level, max_subcompactions, exclude_l0)
-    }
-
-    fn check_in_range(&self, start: Option<&[u8]>, end: Option<&[u8]>) -> Result<()> {
-        self.disk_engine().check_in_range(start, end)
-    }
-}
diff --git a/components/hybrid_engine/src/db_options.rs b/components/hybrid_engine/src/db_options.rs
deleted file mode 100644
index 6b4be90a43f..00000000000
--- a/components/hybrid_engine/src/db_options.rs
+++ /dev/null
@@ -1,21 +0,0 @@
-// Copyright 2023 TiKV Project Authors. Licensed under Apache-2.0.
-
-use engine_traits::{DbOptionsExt, KvEngine, RegionCacheEngine, Result};
-
-use crate::engine::HybridEngine;
-
-impl<EK, EC> DbOptionsExt for HybridEngine<EK, EC>
-where
-    EK: KvEngine,
-    EC: RegionCacheEngine,
-{
-    type DbOptions = EK::DbOptions;
-
-    fn get_db_options(&self) -> Self::DbOptions {
-        self.disk_engine().get_db_options()
-    }
-
-    fn set_db_options(&self, options: &[(&str, &str)]) -> Result<()> {
-        self.disk_engine().set_db_options(options)
-    }
-}
diff --git a/components/hybrid_engine/src/db_vector.rs b/components/hybrid_engine/src/db_vector.rs
index de149b7699f..1b286e182e7 100644
--- a/components/hybrid_engine/src/db_vector.rs
+++ b/components/hybrid_engine/src/db_vector.rs
@@ -28,7 +28,7 @@ where
     EK: KvEngine,
     EC: RegionCacheEngine,
 {
-    pub fn try_from_disk_snap(
+    pub(crate) fn try_from_disk_snap(
         snap: &EK::Snapshot,
         opts: &ReadOptions,
         cf: &str,
@@ -41,7 +41,7 @@ where
             }))
     }
 
-    pub fn try_from_cache_snap(
+    pub(crate) fn try_from_cache_snap(
         snap: &EC::Snapshot,
         opts: &ReadOptions,
         cf: &str,
diff --git a/components/hybrid_engine/src/engine.rs b/components/hybrid_engine/src/engine.rs
index 39c8cadde17..d7f5caf038e 100644
--- a/components/hybrid_engine/src/engine.rs
+++ b/components/hybrid_engine/src/engine.rs
@@ -1,13 +1,6 @@
 // Copyright 2023 TiKV Project Authors. Licensed under Apache-2.0.
 
-use engine_traits::{
-    KvEngine, Mutable, Peekable, ReadOptions, RegionCacheEngine, Result, SnapshotContext,
-    SnapshotMiscExt, SyncMutable, WriteBatch, WriteBatchExt,
-};
-use keys::DATA_PREFIX_KEY;
-use kvproto::metapb::{self, RegionEpoch};
-
-use crate::snapshot::HybridEngineSnapshot;
+use engine_traits::{KvEngine, RegionCacheEngine};
 
 /// This engine is structured with both a disk engine and an region cache
 /// engine. The disk engine houses the complete database data, whereas the
@@ -21,32 +14,11 @@ where
     EK: KvEngine,
     EC: RegionCacheEngine,
 {
+    #[allow(dead_code)]
     disk_engine: EK,
     region_cache_engine: EC,
 }
 
-impl<EK, EC> HybridEngine<EK, EC>
-where
-    EK: KvEngine,
-    EC: RegionCacheEngine,
-{
-    pub fn disk_engine(&self) -> &EK {
-        &self.disk_engine
-    }
-
-    pub fn mut_disk_engine(&mut self) -> &mut EK {
-        &mut self.disk_engine
-    }
-
-    pub fn region_cache_engine(&self) -> &EC {
-        &self.region_cache_engine
-    }
-
-    pub fn mut_region_cache_engine(&mut self) -> &mut EC {
-        &mut self.region_cache_engine
-    }
-}
-
 impl<EK, EC> HybridEngine<EK, EC>
 where
     EK: KvEngine,
@@ -59,7 +31,12 @@ where
         }
     }
 
-    pub fn new_snapshot(&self, ctx: Option<SnapshotContext>) -> HybridEngineSnapshot<EK, EC> {
+    #[cfg(test)]
+    pub(crate) fn new_snapshot(
+        &self,
+        ctx: Option<SnapshotContext>,
+    ) -> crate::HybridEngineSnapshot<EK, EC> {
+        use engine_traits::SnapshotMiscExt;
         let disk_snap = self.disk_engine.snapshot();
         let region_cache_snap = if !self.region_cache_engine.enabled() {
             None
@@ -74,127 +51,32 @@ where
         } else {
             None
         };
-        HybridEngineSnapshot::new(disk_snap, region_cache_snap)
+        crate::HybridEngineSnapshot::new(disk_snap, region_cache_snap)
     }
-}
 
-impl<EK, EC> HybridEngine<EK, EC>
-where
-    EK: KvEngine,
-    EC: RegionCacheEngine,
-    HybridEngine<EK, EC>: WriteBatchExt,
-{
-    fn sync_write<F>(&self, key: &[u8], f: F) -> Result<()>
-    where
-        F: FnOnce(&mut <Self as WriteBatchExt>::WriteBatch) -> Result<()>,
-    {
-        let mut batch = self.write_batch();
-        if let Some(cached_region) = self.region_cache_engine.get_region_for_key(key) {
-            // CacheRegion does not contains enough information for Region so the transfer
-            // is not accurate but this method is not called in production code.
-            let mut region = metapb::Region::default();
-            region.set_id(cached_region.id);
-            region.set_start_key(cached_region.start[DATA_PREFIX_KEY.len()..].to_vec());
-            region.set_end_key(cached_region.end[DATA_PREFIX_KEY.len()..].to_vec());
-            let mut epoch = RegionEpoch::default();
-            epoch.version = cached_region.epoch_version;
-            region.set_region_epoch(epoch);
-            batch.prepare_for_region(&region);
-        }
-        f(&mut batch)?;
-        let _ = batch.write()?;
-        Ok(())
-    }
-}
-
-// todo: implement KvEngine methods as well as it's super traits.
-impl<EK, EC> KvEngine for HybridEngine<EK, EC>
-where
-    EK: KvEngine,
-    EC: RegionCacheEngine,
-    HybridEngine<EK, EC>: WriteBatchExt,
-{
-    type Snapshot = HybridEngineSnapshot<EK, EC>;
-
-    fn snapshot(&self) -> Self::Snapshot {
-        unreachable!()
-    }
-
-    fn sync(&self) -> engine_traits::Result<()> {
-        self.disk_engine.sync()
-    }
-
-    fn bad_downcast<T: 'static>(&self) -> &T {
-        self.disk_engine.bad_downcast()
-    }
-
-    #[cfg(feature = "testexport")]
-    fn inner_refcount(&self) -> usize {
-        self.disk_engine.inner_refcount()
-    }
-}
-
-impl<EK, EC> Peekable for HybridEngine<EK, EC>
-where
-    EK: KvEngine,
-    EC: RegionCacheEngine,
-{
-    type DbVector = EK::DbVector;
-
-    // region cache engine only supports peekable trait in the snapshot of it
-    fn get_value_opt(&self, opts: &ReadOptions, key: &[u8]) -> Result<Option<Self::DbVector>> {
-        self.disk_engine.get_value_opt(opts, key)
+    #[cfg(test)]
+    pub(crate) fn disk_engine(&self) -> &EK {
+        &self.disk_engine
     }
 
-    // region cache engine only supports peekable trait in the snapshot of it
-    fn get_value_cf_opt(
-        &self,
-        opts: &ReadOptions,
-        cf: &str,
-        key: &[u8],
-    ) -> Result<Option<Self::DbVector>> {
-        self.disk_engine.get_value_cf_opt(opts, cf, key)
+    pub fn region_cache_engine(&self) -> &EC {
+        &self.region_cache_engine
     }
 }
 
-impl<EK, EC> SyncMutable for HybridEngine<EK, EC>
-where
-    EK: KvEngine,
-    EC: RegionCacheEngine,
-    HybridEngine<EK, EC>: WriteBatchExt,
-{
-    fn put(&self, key: &[u8], value: &[u8]) -> Result<()> {
-        self.sync_write(key, |b| b.put(key, value))
-    }
-
-    fn put_cf(&self, cf: &str, key: &[u8], value: &[u8]) -> Result<()> {
-        self.sync_write(key, |b| b.put_cf(cf, key, value))
-    }
-
-    fn delete(&self, key: &[u8]) -> Result<()> {
-        self.sync_write(key, |b| b.delete(key))
-    }
-
-    fn delete_cf(&self, cf: &str, key: &[u8]) -> Result<()> {
-        self.sync_write(key, |b| b.delete_cf(cf, key))
-    }
-
-    fn delete_range(&self, begin_key: &[u8], end_key: &[u8]) -> Result<()> {
-        self.sync_write(begin_key, |b| b.delete_range(begin_key, end_key))
-    }
-
-    fn delete_range_cf(&self, cf: &str, begin_key: &[u8], end_key: &[u8]) -> Result<()> {
-        self.sync_write(begin_key, |b| b.delete_range_cf(cf, begin_key, end_key))
-    }
+#[cfg(test)]
+#[derive(Debug, Clone)]
+pub struct SnapshotContext {
+    pub region: Option<engine_traits::CacheRegion>,
+    pub read_ts: u64,
 }
 
 #[cfg(test)]
 mod tests {
-
     use std::sync::Arc;
 
     use engine_rocks::util::new_engine;
-    use engine_traits::{CacheRegion, SnapshotContext, CF_DEFAULT, CF_LOCK, CF_WRITE};
+    use engine_traits::{CacheRegion, CF_DEFAULT, CF_LOCK, CF_WRITE};
     use in_memory_engine::{
         config::InMemoryEngineConfigManager, test_util::new_region, InMemoryEngineConfig,
         InMemoryEngineContext, RegionCacheMemoryEngine,
@@ -203,6 +85,7 @@ mod tests {
     use tempfile::Builder;
     use tikv_util::config::VersionTrack;
 
+    use super::*;
     use crate::HybridEngine;
 
     #[test]
diff --git a/components/hybrid_engine/src/engine_iterator.rs b/components/hybrid_engine/src/engine_iterator.rs
index ac3cdc77d18..6f55b394ca6 100644
--- a/components/hybrid_engine/src/engine_iterator.rs
+++ b/components/hybrid_engine/src/engine_iterator.rs
@@ -18,13 +18,13 @@ where
     EK: KvEngine,
     EC: RegionCacheEngine,
 {
-    pub fn disk_engine_iterator(iter: <EK::Snapshot as Iterable>::Iterator) -> Self {
+    pub(crate) fn disk_engine_iterator(iter: <EK::Snapshot as Iterable>::Iterator) -> Self {
         Self {
             iter: Either::Left(iter),
         }
     }
 
-    pub fn region_cache_engine_iterator(iter: <EC::Snapshot as Iterable>::Iterator) -> Self {
+    pub(crate) fn region_cache_engine_iterator(iter: <EC::Snapshot as Iterable>::Iterator) -> Self {
         Self {
             iter: Either::Right(iter),
         }
diff --git a/components/hybrid_engine/src/flow_control_factors.rs b/components/hybrid_engine/src/flow_control_factors.rs
deleted file mode 100644
index 9649671d418..00000000000
--- a/components/hybrid_engine/src/flow_control_factors.rs
+++ /dev/null
@@ -1,23 +0,0 @@
-// Copyright 2023 TiKV Project Authors. Licensed under Apache-2.0.
-
-use engine_traits::{FlowControlFactorsExt, KvEngine, RegionCacheEngine, Result};
-
-use crate::engine::HybridEngine;
-
-impl<EK, EC> FlowControlFactorsExt for HybridEngine<EK, EC>
-where
-    EK: KvEngine,
-    EC: RegionCacheEngine,
-{
-    fn get_cf_num_files_at_level(&self, cf: &str, level: usize) -> Result<Option<u64>> {
-        self.disk_engine().get_cf_num_files_at_level(cf, level)
-    }
-
-    fn get_cf_num_immutable_mem_table(&self, cf: &str) -> Result<Option<u64>> {
-        self.disk_engine().get_cf_num_immutable_mem_table(cf)
-    }
-
-    fn get_cf_pending_compaction_bytes(&self, cf: &str) -> Result<Option<u64>> {
-        self.disk_engine().get_cf_pending_compaction_bytes(cf)
-    }
-}
diff --git a/components/hybrid_engine/src/hybrid_metrics.rs b/components/hybrid_engine/src/hybrid_metrics.rs
deleted file mode 100644
index 2d49d9ad1d9..00000000000
--- a/components/hybrid_engine/src/hybrid_metrics.rs
+++ /dev/null
@@ -1,25 +0,0 @@
-// Copyright 2023 TiKV Project Authors. Licensed under Apache-2.0.
-
-use engine_traits::{KvEngine, RegionCacheEngine, StatisticsReporter};
-
-use crate::engine::HybridEngine;
-
-pub struct HybridEngineStatisticsReporter {}
-
-impl<EK, EC> StatisticsReporter<HybridEngine<EK, EC>> for HybridEngineStatisticsReporter
-where
-    EK: KvEngine,
-    EC: RegionCacheEngine,
-{
-    fn new(name: &str) -> Self {
-        unimplemented!()
-    }
-
-    fn collect(&mut self, engine: &HybridEngine<EK, EC>) {
-        unimplemented!()
-    }
-
-    fn flush(&mut self) {
-        unimplemented!()
-    }
-}
diff --git a/components/hybrid_engine/src/import.rs b/components/hybrid_engine/src/import.rs
deleted file mode 100644
index 8ce62f11cd9..00000000000
--- a/components/hybrid_engine/src/import.rs
+++ /dev/null
@@ -1,17 +0,0 @@
-// Copyright 2023 TiKV Project Authors. Licensed under Apache-2.0.
-
-use engine_traits::{ImportExt, KvEngine, RegionCacheEngine};
-
-use crate::engine::HybridEngine;
-
-impl<EK, EC> ImportExt for HybridEngine<EK, EC>
-where
-    EK: KvEngine,
-    EC: RegionCacheEngine,
-{
-    type IngestExternalFileOptions = EK::IngestExternalFileOptions;
-
-    fn ingest_external_file_cf(&self, cf: &str, files: &[&str]) -> engine_traits::Result<()> {
-        self.disk_engine().ingest_external_file_cf(cf, files)
-    }
-}
diff --git a/components/hybrid_engine/src/iterable.rs b/components/hybrid_engine/src/iterable.rs
deleted file mode 100644
index 3b45b687643..00000000000
--- a/components/hybrid_engine/src/iterable.rs
+++ /dev/null
@@ -1,19 +0,0 @@
-// Copyright 2023 TiKV Project Authors. Licensed under Apache-2.0.
-
-use engine_traits::{IterOptions, Iterable, KvEngine, RegionCacheEngine, Result};
-
-use crate::engine::HybridEngine;
-
-impl<EK, EC> Iterable for HybridEngine<EK, EC>
-where
-    EK: KvEngine,
-    EC: RegionCacheEngine,
-{
-    type Iterator = EK::Iterator;
-
-    fn iterator_opt(&self, cf: &str, opts: IterOptions) -> Result<Self::Iterator> {
-        // Iterator of region cache engine should only be created from the
-        // snapshot of it
-        self.disk_engine().iterator_opt(cf, opts)
-    }
-}
diff --git a/components/hybrid_engine/src/lib.rs b/components/hybrid_engine/src/lib.rs
index c2301f1c897..8db02de65d3 100644
--- a/components/hybrid_engine/src/lib.rs
+++ b/components/hybrid_engine/src/lib.rs
@@ -1,33 +1,17 @@
 // Copyright 2023 TiKV Project Authors. Licensed under Apache-2.0.
-#![allow(dead_code)]
-#![allow(unused_variables)]
+// TODO: HybridEngine has became a very thin shim, consider merging
+// `HybridEngineSnapshot` and `HybridEngineIterator` into in_memory_engine
+// crate.
+
 #![feature(let_chains)]
 
-mod cf_names;
-mod cf_options;
-mod checkpoint;
-mod compact;
-mod db_options;
 mod db_vector;
 mod engine;
 mod engine_iterator;
-mod flow_control_factors;
-mod hybrid_metrics;
-mod import;
-mod iterable;
 mod metrics;
-mod misc;
-mod mvcc_properties;
 pub mod observer;
-mod perf_context;
-mod range_properties;
-mod region_cache_engine;
 mod snapshot;
-mod sst;
-mod table_properties;
-mod ttl_properties;
 pub mod util;
-mod write_batch;
 
 pub use engine::HybridEngine;
 pub use snapshot::HybridEngineSnapshot;
diff --git a/components/hybrid_engine/src/metrics.rs b/components/hybrid_engine/src/metrics.rs
index d961d3d6455..32e49a223ca 100644
--- a/components/hybrid_engine/src/metrics.rs
+++ b/components/hybrid_engine/src/metrics.rs
@@ -38,7 +38,7 @@ lazy_static! {
     pub static ref IN_MEMORY_ENGINE_SNAPSHOT_ACQUIRE_FAILED_REASON_COUNT_VEC: IntCounterVec =
         register_int_counter_vec!(
             "tikv_in_memory_engine_snapshot_acquire_failed_reason_count",
-            "The reasons for why range cache snapshot is not acquired",
+            "The reasons for why region cache snapshot is not acquired",
             &["type"],
         )
         .unwrap();
diff --git a/components/hybrid_engine/src/misc.rs b/components/hybrid_engine/src/misc.rs
deleted file mode 100644
index d89a31d8f7a..00000000000
--- a/components/hybrid_engine/src/misc.rs
+++ /dev/null
@@ -1,222 +0,0 @@
-// Copyright 2023 TiKV Project Authors. Licensed under Apache-2.0.
-
-use engine_traits::{
-    CacheRegion, EvictReason, KvEngine, MiscExt, RegionCacheEngine, RegionEvent, Result,
-    WriteBatchExt,
-};
-
-use crate::{engine::HybridEngine, hybrid_metrics::HybridEngineStatisticsReporter};
-
-impl<EK, EC> MiscExt for HybridEngine<EK, EC>
-where
-    EK: KvEngine,
-    EC: RegionCacheEngine,
-    HybridEngine<EK, EC>: WriteBatchExt,
-{
-    type StatisticsReporter = HybridEngineStatisticsReporter;
-
-    fn flush_cf(&self, cf: &str, wait: bool) -> Result<()> {
-        self.disk_engine().flush_cf(cf, wait)
-    }
-
-    fn flush_cfs(&self, cfs: &[&str], wait: bool) -> Result<()> {
-        self.disk_engine().flush_cfs(cfs, wait)
-    }
-
-    fn flush_oldest_cf(
-        &self,
-        wait: bool,
-        threshold: Option<std::time::SystemTime>,
-    ) -> Result<bool> {
-        self.disk_engine().flush_oldest_cf(wait, threshold)
-    }
-
-    fn delete_ranges_cf(
-        &self,
-        wopts: &engine_traits::WriteOptions,
-        cf: &str,
-        strategy: engine_traits::DeleteStrategy,
-        ranges: &[engine_traits::Range<'_>],
-    ) -> Result<bool> {
-        for r in ranges {
-            self.region_cache_engine()
-                .on_region_event(RegionEvent::EvictByRange {
-                    range: CacheRegion::new(0, 0, r.start_key.to_vec(), r.end_key.to_vec()),
-                    reason: EvictReason::DeleteRange,
-                });
-        }
-        self.disk_engine()
-            .delete_ranges_cf(wopts, cf, strategy, ranges)
-    }
-
-    fn get_approximate_memtable_stats_cf(
-        &self,
-        cf: &str,
-        range: &engine_traits::Range<'_>,
-    ) -> Result<(u64, u64)> {
-        self.disk_engine()
-            .get_approximate_memtable_stats_cf(cf, range)
-    }
-
-    fn ingest_maybe_slowdown_writes(&self, cf: &str) -> Result<bool> {
-        self.disk_engine().ingest_maybe_slowdown_writes(cf)
-    }
-
-    fn get_sst_key_ranges(&self, cf: &str, level: usize) -> Result<Vec<(Vec<u8>, Vec<u8>)>> {
-        self.disk_engine().get_sst_key_ranges(cf, level)
-    }
-
-    fn get_engine_used_size(&self) -> Result<u64> {
-        self.disk_engine().get_engine_used_size()
-    }
-
-    fn path(&self) -> &str {
-        self.disk_engine().path()
-    }
-
-    fn sync_wal(&self) -> Result<()> {
-        self.disk_engine().sync_wal()
-    }
-
-    fn disable_manual_compaction(&self) -> Result<()> {
-        self.disk_engine().disable_manual_compaction()
-    }
-
-    fn enable_manual_compaction(&self) -> Result<()> {
-        self.disk_engine().enable_manual_compaction()
-    }
-
-    fn pause_background_work(&self) -> Result<()> {
-        self.disk_engine().pause_background_work()
-    }
-
-    fn continue_background_work(&self) -> Result<()> {
-        self.disk_engine().continue_background_work()
-    }
-
-    fn exists(path: &str) -> bool {
-        EK::exists(path)
-    }
-
-    fn locked(path: &str) -> Result<bool> {
-        EK::locked(path)
-    }
-
-    fn dump_stats(&self) -> Result<String> {
-        self.disk_engine().dump_stats()
-    }
-
-    fn get_latest_sequence_number(&self) -> u64 {
-        self.disk_engine().get_latest_sequence_number()
-    }
-
-    fn get_oldest_snapshot_sequence_number(&self) -> Option<u64> {
-        self.disk_engine().get_oldest_snapshot_sequence_number()
-    }
-
-    fn get_total_sst_files_size_cf(&self, cf: &str) -> Result<Option<u64>> {
-        self.disk_engine().get_total_sst_files_size_cf(cf)
-    }
-
-    fn get_num_keys(&self) -> Result<u64> {
-        self.disk_engine().get_num_keys()
-    }
-
-    fn get_range_stats(
-        &self,
-        cf: &str,
-        start: &[u8],
-        end: &[u8],
-    ) -> Result<Option<engine_traits::RangeStats>> {
-        self.disk_engine().get_range_stats(cf, start, end)
-    }
-
-    fn is_stalled_or_stopped(&self) -> bool {
-        self.disk_engine().is_stalled_or_stopped()
-    }
-
-    fn get_active_memtable_stats_cf(
-        &self,
-        cf: &str,
-    ) -> Result<Option<(u64, std::time::SystemTime)>> {
-        self.disk_engine().get_active_memtable_stats_cf(cf)
-    }
-
-    fn get_accumulated_flush_count_cf(cf: &str) -> Result<u64> {
-        EK::get_accumulated_flush_count_cf(cf)
-    }
-
-    type DiskEngine = EK::DiskEngine;
-    fn get_disk_engine(&self) -> &Self::DiskEngine {
-        self.disk_engine().get_disk_engine()
-    }
-}
-
-#[cfg(test)]
-pub mod tests {
-    use engine_traits::{
-        CacheRegion, DeleteStrategy, MiscExt, Mutable, Range, RegionCacheEngine, WriteBatch,
-        WriteBatchExt, WriteOptions, CF_DEFAULT,
-    };
-    use in_memory_engine::{test_util::new_region, InMemoryEngineConfig};
-
-    use crate::util::hybrid_engine_for_tests;
-
-    #[test]
-    fn test_delete_range() {
-        let r1 = new_region(1, b"k00", b"k10");
-        let r2 = new_region(2, b"k20", b"k30");
-        let r3 = new_region(3, b"k40", b"k50");
-        let r1_clone = r1.clone();
-        let r2_clone = r2.clone();
-        let r3_clone = r3.clone();
-        let (_path, hybrid_engine) = hybrid_engine_for_tests(
-            "temp",
-            InMemoryEngineConfig::config_for_test(),
-            move |memory_engine| {
-                memory_engine.new_region(r1_clone);
-                memory_engine.new_region(r2_clone);
-                memory_engine.new_region(r3_clone);
-            },
-        )
-        .unwrap();
-
-        let cache_r1 = CacheRegion::from_region(&r1);
-        let cache_r2 = CacheRegion::from_region(&r2);
-        let cache_r3 = CacheRegion::from_region(&r3);
-
-        let mut write_batch = hybrid_engine.write_batch();
-        write_batch.prepare_for_region(&r1);
-        write_batch.put(b"zk02", b"val").unwrap();
-        write_batch.put(b"zk03", b"val").unwrap();
-        write_batch.prepare_for_region(&r2);
-        write_batch.put(b"zk22", b"val").unwrap();
-        write_batch.put(b"zk23", b"val").unwrap();
-        write_batch.prepare_for_region(&r3);
-        write_batch.put(b"zk42", b"val").unwrap();
-        write_batch.put(b"zk43", b"val").unwrap();
-        write_batch.write().unwrap();
-
-        hybrid_engine
-            .delete_ranges_cf(
-                &WriteOptions::default(),
-                CF_DEFAULT,
-                DeleteStrategy::DeleteByRange,
-                &[Range::new(b"zk00", b"zk15"), Range::new(b"zk22", b"zk27")],
-            )
-            .unwrap();
-
-        hybrid_engine
-            .region_cache_engine()
-            .snapshot(cache_r1.clone(), 1000, 1000)
-            .unwrap_err();
-        hybrid_engine
-            .region_cache_engine()
-            .snapshot(cache_r2.clone(), 1000, 1000)
-            .unwrap_err();
-        hybrid_engine
-            .region_cache_engine()
-            .snapshot(cache_r3.clone(), 1000, 1000)
-            .unwrap();
-    }
-}
diff --git a/components/hybrid_engine/src/mvcc_properties.rs b/components/hybrid_engine/src/mvcc_properties.rs
deleted file mode 100644
index 0d03258d2de..00000000000
--- a/components/hybrid_engine/src/mvcc_properties.rs
+++ /dev/null
@@ -1,23 +0,0 @@
-// Copyright 2023 TiKV Project Authors. Licensed under Apache-2.0.
-
-use engine_traits::{KvEngine, MvccProperties, MvccPropertiesExt, RegionCacheEngine};
-use txn_types::TimeStamp;
-
-use crate::engine::HybridEngine;
-
-impl<EK, EC> MvccPropertiesExt for HybridEngine<EK, EC>
-where
-    EK: KvEngine,
-    EC: RegionCacheEngine,
-{
-    fn get_mvcc_properties_cf(
-        &self,
-        cf: &str,
-        safe_point: TimeStamp,
-        start_key: &[u8],
-        end_key: &[u8],
-    ) -> Option<MvccProperties> {
-        self.disk_engine()
-            .get_mvcc_properties_cf(cf, safe_point, start_key, end_key)
-    }
-}
diff --git a/components/hybrid_engine/src/observer/load_eviction.rs b/components/hybrid_engine/src/observer/load_eviction.rs
index c6388ea6b08..5acfebe1650 100644
--- a/components/hybrid_engine/src/observer/load_eviction.rs
+++ b/components/hybrid_engine/src/observer/load_eviction.rs
@@ -139,7 +139,7 @@ impl Coprocessor for LoadEvictionObserver {}
 impl QueryObserver for LoadEvictionObserver {
     fn pre_exec_query(
         &self,
-        ctx: &mut ObserverContext<'_>,
+        _: &mut ObserverContext<'_>,
         reqs: &[kvproto::raft_cmdpb::Request],
         _: u64,
         _: u64,
@@ -304,11 +304,11 @@ mod tests {
             self.region_events.lock().unwrap().push(event);
         }
 
-        fn region_cached(&self, range: &Region) -> bool {
+        fn region_cached(&self, _: &Region) -> bool {
             unreachable!()
         }
 
-        fn load_region(&self, range: &Region) {
+        fn load_region(&self, _: &Region) {
             unreachable!()
         }
     }
@@ -397,7 +397,7 @@ mod tests {
         region.set_id(1);
         region.mut_peers().push(Peer::default());
         let mut ctx = ObserverContext::new(&region);
-        let role_change = RoleChange::new(StateRole::Leader);
+        let role_change = RoleChange::new_for_test(StateRole::Leader);
         observer.on_role_change(&mut ctx, &role_change);
         let cached_region = CacheRegion::from_region(&region);
         let expected = RegionEvent::TryLoad {
diff --git a/components/hybrid_engine/src/observer/mod.rs b/components/hybrid_engine/src/observer/mod.rs
index 885b347fadd..5144395357d 100644
--- a/components/hybrid_engine/src/observer/mod.rs
+++ b/components/hybrid_engine/src/observer/mod.rs
@@ -2,6 +2,8 @@
 
 mod load_eviction;
 mod snapshot;
+#[cfg(test)]
+mod test_write_batch;
 mod write_batch;
 
 pub use load_eviction::LoadEvictionObserver;
diff --git a/components/hybrid_engine/src/observer/test_write_batch.rs b/components/hybrid_engine/src/observer/test_write_batch.rs
new file mode 100644
index 00000000000..faa1cbf30d1
--- /dev/null
+++ b/components/hybrid_engine/src/observer/test_write_batch.rs
@@ -0,0 +1,278 @@
+// Copyright 2024 TiKV Project Authors. Licensed under Apache-2.0.
+
+use std::{sync::mpsc::sync_channel, time::Duration};
+
+use crossbeam::epoch;
+use engine_traits::{CacheRegion, Mutable, Peekable, RegionCacheEngine, WriteBatch, WriteBatchExt};
+use in_memory_engine::{
+    decode_key, test_util::new_region, InMemoryEngineConfig, InternalKey, RegionCacheStatus,
+    ValueType,
+};
+use raftstore::coprocessor::{WriteBatchObserver, WriteBatchWrapper};
+
+use super::RegionCacheWriteBatchObserver;
+use crate::{engine::SnapshotContext, util::hybrid_engine_for_tests};
+
+#[test]
+fn test_sequence_number_unique() {
+    let (_path, hybrid_engine) =
+        hybrid_engine_for_tests("temp", InMemoryEngineConfig::config_for_test(), |_| {}).unwrap();
+
+    let (tx, rx) = sync_channel(0);
+    fail::cfg_callback("ime_on_completes_batch_loading", move || {
+        fail::cfg("ime_on_start_loading_region", "pause").unwrap();
+        tx.send(true).unwrap();
+    })
+    .unwrap();
+
+    let engine = hybrid_engine.region_cache_engine().clone();
+    let observer = RegionCacheWriteBatchObserver::new(engine.clone());
+
+    // first write some data, these data should be handled by batch loading.
+    let mut wb = WriteBatchWrapper::new(
+        hybrid_engine.disk_engine().write_batch(),
+        Some(observer.create_observable_write_batch()),
+    );
+
+    wb.put(b"zk5", b"val").unwrap(); // seq 1
+    wb.put(b"zk7", b"val").unwrap(); // seq 2
+
+    let r = new_region(1, b"k", b"k5");
+    engine.new_region(r.clone());
+    wb.write().unwrap();
+
+    // Mock that we have a loading range, and there are some keys written in it
+    // during the load
+    let r2 = new_region(2, b"k5", b"k7");
+    let r3 = new_region(3, b"k7", b"k9");
+    let cache_region2 = CacheRegion::from_region(&r2);
+    let cache_region3 = CacheRegion::from_region(&r3);
+    engine.load_region(cache_region2.clone()).unwrap();
+    engine.load_region(cache_region3.clone()).unwrap();
+
+    // The sequence number of write batch should be increased one by one, otherwise
+    // if a delete and a put of the same key occurs in the same write batch,
+    // the delete will be hidden by the put even the delete is performed
+    // after the put.
+    // while we block the batch loading of region3, it's new KVs are still directly
+    // written into the skiplist.
+    let mut wb = WriteBatchWrapper::new(
+        hybrid_engine.disk_engine().write_batch(),
+        Some(observer.create_observable_write_batch()),
+    );
+    wb.prepare_for_region(&r);
+    wb.put(b"zk", b"val").unwrap(); // seq 3
+    wb.delete(b"zk").unwrap(); // seq 4
+    wb.put(b"zk2", b"val").unwrap(); // seq 5
+
+    wb.prepare_for_region(&r2);
+    wb.put(b"zk6", b"val").unwrap(); // seq 6
+    wb.delete(b"zk5").unwrap(); // seq 7
+    wb.put(b"zk5", b"val2").unwrap(); // seq 8
+
+    wb.prepare_for_region(&r3);
+    wb.put(b"zk8", b"val").unwrap(); // seq 9
+    wb.put(b"zk7", b"val2").unwrap(); // seq 10
+
+    rx.recv().unwrap();
+    wb.write().unwrap();
+
+    let mut iter = engine.core().engine().cf_handle("default").iterator();
+    let guard = &epoch::pin();
+
+    let mut first = true;
+
+    for (k, sequence, v_type) in [
+        (b"zk".to_vec(), 4, ValueType::Deletion),
+        (b"zk".to_vec(), 3, ValueType::Value),
+        (b"zk2".to_vec(), 5, ValueType::Value),
+        (b"zk5".to_vec(), 8, ValueType::Value),
+        (b"zk5".to_vec(), 7, ValueType::Deletion),
+        // NOTE: for batch loading, we always use the current seq number
+        // to write all the keys.
+        (b"zk5".to_vec(), 2, ValueType::Value),
+        (b"zk6".to_vec(), 6, ValueType::Value),
+        (b"zk7".to_vec(), 10, ValueType::Value),
+        // "zk7" with seq 2 is block, so invisible here.
+        (b"zk8".to_vec(), 9, ValueType::Value),
+    ] {
+        if first {
+            iter.seek_to_first(guard);
+            first = false;
+        } else {
+            iter.next(guard);
+        }
+
+        let expected_key = InternalKey {
+            user_key: k.as_slice(),
+            v_type,
+            sequence,
+        };
+        let key = iter.key();
+        let got_key = decode_key(key.as_bytes());
+        assert_eq!(expected_key, got_key);
+    }
+}
+
+#[test]
+fn test_write_to_both_engines() {
+    let region = new_region(1, b"", b"z");
+    let region_clone = region.clone();
+    let (_path, hybrid_engine) = hybrid_engine_for_tests(
+        "temp",
+        InMemoryEngineConfig::config_for_test(),
+        move |memory_engine| {
+            let id = region_clone.id;
+            memory_engine.new_region(region_clone);
+            memory_engine.core().region_manager().set_safe_point(id, 5);
+        },
+    )
+    .unwrap();
+    let engine = hybrid_engine.region_cache_engine().clone();
+    let observer = RegionCacheWriteBatchObserver::new(engine.clone());
+
+    let cache_region = CacheRegion::from_region(&region);
+    let mut ob_wb = observer.new_observable_write_batch();
+    ob_wb.cache_write_batch.prepare_for_region(&region);
+    ob_wb
+        .cache_write_batch
+        .set_region_cache_status(RegionCacheStatus::Cached);
+    let mut write_batch = WriteBatchWrapper::new(
+        hybrid_engine.disk_engine().write_batch(),
+        Some(Box::new(ob_wb)),
+    );
+    write_batch.put(b"zhello", b"world").unwrap();
+    let seq = write_batch.write().unwrap();
+    assert!(seq > 0);
+    let actual: &[u8] = &hybrid_engine
+        .disk_engine()
+        .get_value(b"zhello")
+        .unwrap()
+        .unwrap();
+    assert_eq!(b"world", &actual);
+    let ctx = SnapshotContext {
+        region: Some(cache_region.clone()),
+        read_ts: 10,
+    };
+    let snap = hybrid_engine.new_snapshot(Some(ctx));
+    let actual: &[u8] = &snap.get_value(b"zhello").unwrap().unwrap();
+    assert_eq!(b"world", &actual);
+    let actual: &[u8] = &snap.disk_snap().get_value(b"zhello").unwrap().unwrap();
+    assert_eq!(b"world", &actual);
+    let actual: &[u8] = &snap
+        .region_cache_snap()
+        .unwrap()
+        .get_value(b"zhello")
+        .unwrap()
+        .unwrap();
+    assert_eq!(b"world", &actual);
+}
+
+#[test]
+fn test_set_sequence_number() {
+    let (_path, hybrid_engine) = hybrid_engine_for_tests(
+        "temp",
+        InMemoryEngineConfig::config_for_test(),
+        |memory_engine| {
+            let region = new_region(1, b"k00", b"k10");
+            memory_engine.new_region(region);
+            memory_engine.core().region_manager().set_safe_point(1, 10);
+        },
+    )
+    .unwrap();
+
+    let engine = hybrid_engine.region_cache_engine().clone();
+    let observer = RegionCacheWriteBatchObserver::new(engine.clone());
+    let mut write_batch = observer.new_observable_write_batch();
+
+    write_batch
+        .cache_write_batch
+        .set_sequence_number(0)
+        .unwrap(); // First call ok.
+    assert!(
+        write_batch
+            .cache_write_batch
+            .set_sequence_number(0)
+            .is_err()
+    ); // Second call err.
+}
+
+#[test]
+fn test_delete_range() {
+    let region1 = new_region(1, b"k00", b"k10");
+    let region2 = new_region(2, b"k20", b"k30");
+    let cache_region1 = CacheRegion::from_region(&region1);
+    let cache_region2 = CacheRegion::from_region(&region2);
+
+    let region1_clone = region1.clone();
+    let region2_clone = region2.clone();
+    let (_path, hybrid_engine) = hybrid_engine_for_tests(
+        "temp",
+        InMemoryEngineConfig::config_for_test(),
+        move |memory_engine| {
+            memory_engine.new_region(region1_clone);
+            memory_engine.new_region(region2_clone);
+        },
+    )
+    .unwrap();
+
+    let engine = hybrid_engine.region_cache_engine().clone();
+    let observer = RegionCacheWriteBatchObserver::new(engine.clone());
+
+    let mut wb = WriteBatchWrapper::new(
+        hybrid_engine.disk_engine().write_batch(),
+        Some(observer.create_observable_write_batch()),
+    );
+    wb.prepare_for_region(&region1);
+    wb.put(b"zk05", b"val").unwrap();
+    wb.put(b"zk08", b"val2").unwrap();
+    wb.prepare_for_region(&region2);
+    wb.put(b"zk25", b"val3").unwrap();
+    wb.put(b"zk27", b"val4").unwrap();
+    wb.write().unwrap();
+
+    hybrid_engine
+        .region_cache_engine()
+        .snapshot(cache_region1.clone(), 1000, 1000)
+        .unwrap();
+    hybrid_engine
+        .region_cache_engine()
+        .snapshot(cache_region2.clone(), 1000, 1000)
+        .unwrap();
+    assert_eq!(
+        4,
+        hybrid_engine
+            .region_cache_engine()
+            .core()
+            .engine()
+            .cf_handle("default")
+            .len()
+    );
+
+    let mut wb = WriteBatchWrapper::new(
+        hybrid_engine.disk_engine().write_batch(),
+        Some(observer.create_observable_write_batch()),
+    );
+    // all ranges overlapped with it will be evicted
+    wb.prepare_for_region(&region1);
+    wb.delete_range(b"zk05", b"zk08").unwrap();
+    wb.prepare_for_region(&region2);
+    wb.delete_range(b"zk20", b"zk21").unwrap();
+    wb.write().unwrap();
+
+    hybrid_engine
+        .region_cache_engine()
+        .snapshot(cache_region1.clone(), 1000, 1000)
+        .unwrap_err();
+    hybrid_engine
+        .region_cache_engine()
+        .snapshot(cache_region2.clone(), 1000, 1000)
+        .unwrap_err();
+    let m_engine = hybrid_engine.region_cache_engine();
+
+    test_util::eventually(
+        Duration::from_millis(100),
+        Duration::from_millis(2000),
+        || m_engine.core().engine().cf_handle("default").is_empty(),
+    );
+}
diff --git a/components/hybrid_engine/src/observer/write_batch.rs b/components/hybrid_engine/src/observer/write_batch.rs
index efb62f1c2cf..854dc6a2eb7 100644
--- a/components/hybrid_engine/src/observer/write_batch.rs
+++ b/components/hybrid_engine/src/observer/write_batch.rs
@@ -23,23 +23,30 @@ impl RegionCacheWriteBatchObserver {
             .registry
             .register_write_batch_observer(BoxWriteBatchObserver::new(self.clone()));
     }
+
+    pub(crate) fn new_observable_write_batch(&self) -> HybridObservableWriteBatch {
+        HybridObservableWriteBatch {
+            cache_write_batch: RegionCacheWriteBatch::from(&self.cache_engine),
+        }
+    }
 }
 
 impl Coprocessor for RegionCacheWriteBatchObserver {}
 
 impl WriteBatchObserver for RegionCacheWriteBatchObserver {
     fn create_observable_write_batch(&self) -> Box<dyn ObservableWriteBatch> {
-        Box::new(HybridObservableWriteBatch {
-            cache_write_batch: RegionCacheWriteBatch::from(&self.cache_engine),
-        })
+        Box::new(self.new_observable_write_batch())
     }
 }
 
-struct HybridObservableWriteBatch {
-    cache_write_batch: RegionCacheWriteBatch,
+pub(crate) struct HybridObservableWriteBatch {
+    pub(crate) cache_write_batch: RegionCacheWriteBatch,
 }
 
 impl ObservableWriteBatch for HybridObservableWriteBatch {
+    fn prepare_for_region(&mut self, region: &metapb::Region) {
+        self.cache_write_batch.prepare_for_region(region);
+    }
     fn write_opt_seq(&mut self, opts: &WriteOptions, seq_num: u64) {
         self.cache_write_batch.set_sequence_number(seq_num).unwrap();
         self.cache_write_batch.write_opt(opts).unwrap();
@@ -62,16 +69,16 @@ impl WriteBatch for HybridObservableWriteBatch {
     fn write(&mut self) -> Result<u64> {
         unimplemented!("write")
     }
-    fn write_opt(&mut self, opts: &WriteOptions) -> Result<u64> {
+    fn write_opt(&mut self, _: &WriteOptions) -> Result<u64> {
         unimplemented!("write_opt")
     }
-    fn write_callback_opt(&mut self, opts: &WriteOptions, cb: impl FnMut(u64)) -> Result<u64>
+    fn write_callback_opt(&mut self, _: &WriteOptions, _: impl FnMut(u64)) -> Result<u64>
     where
         Self: Sized,
     {
         unimplemented!("write_callback_opt")
     }
-    fn merge(&mut self, other: Self) -> Result<()>
+    fn merge(&mut self, _: Self) -> Result<()>
     where
         Self: Sized,
     {
@@ -102,9 +109,6 @@ impl WriteBatch for HybridObservableWriteBatch {
     fn rollback_to_save_point(&mut self) -> Result<()> {
         self.cache_write_batch.rollback_to_save_point()
     }
-    fn prepare_for_region(&mut self, region: &metapb::Region) {
-        self.cache_write_batch.prepare_for_region(region);
-    }
 }
 
 impl Mutable for HybridObservableWriteBatch {
@@ -124,12 +128,12 @@ impl Mutable for HybridObservableWriteBatch {
         self.cache_write_batch.delete_cf(cf, key)
     }
     fn delete_range(&mut self, begin_key: &[u8], end_key: &[u8]) -> Result<()> {
-        // delete_range in range cache engine means eviction -- all ranges overlapped
+        // delete_range in in memory engine means eviction -- all ranges overlapped
         // with [begin_key, end_key] will be evicted.
         self.cache_write_batch.delete_range(begin_key, end_key)
     }
     fn delete_range_cf(&mut self, cf: &str, begin_key: &[u8], end_key: &[u8]) -> Result<()> {
-        // delete_range in range cache engine means eviction -- all ranges overlapped
+        // delete_range in in memory engine means eviction -- all ranges overlapped
         // with [begin_key, end_key] will be evicted.
         self.cache_write_batch
             .delete_range_cf(cf, begin_key, end_key)
diff --git a/components/hybrid_engine/src/perf_context.rs b/components/hybrid_engine/src/perf_context.rs
deleted file mode 100644
index 1db4e8c9d27..00000000000
--- a/components/hybrid_engine/src/perf_context.rs
+++ /dev/null
@@ -1,20 +0,0 @@
-// Copyright 2023 TiKV Project Authors. Licensed under Apache-2.0.
-
-use engine_traits::{KvEngine, PerfContextExt, PerfContextKind, RegionCacheEngine};
-
-use crate::engine::HybridEngine;
-
-impl<EK, EC> PerfContextExt for HybridEngine<EK, EC>
-where
-    EK: KvEngine,
-    EC: RegionCacheEngine,
-{
-    type PerfContext = EK::PerfContext;
-
-    fn get_perf_context(
-        level: engine_traits::PerfLevel,
-        kind: PerfContextKind,
-    ) -> Self::PerfContext {
-        EK::get_perf_context(level, kind)
-    }
-}
diff --git a/components/hybrid_engine/src/range_properties.rs b/components/hybrid_engine/src/range_properties.rs
deleted file mode 100644
index 7f38379f36d..00000000000
--- a/components/hybrid_engine/src/range_properties.rs
+++ /dev/null
@@ -1,60 +0,0 @@
-// Copyright 2023 TiKV Project Authors. Licensed under Apache-2.0.
-
-use engine_traits::{KvEngine, Range, RangePropertiesExt, RegionCacheEngine, Result};
-
-use crate::engine::HybridEngine;
-
-impl<EK, EC> RangePropertiesExt for HybridEngine<EK, EC>
-where
-    EK: KvEngine,
-    EC: RegionCacheEngine,
-{
-    fn get_range_approximate_keys(&self, range: Range<'_>, large_threshold: u64) -> Result<u64> {
-        self.disk_engine()
-            .get_range_approximate_keys(range, large_threshold)
-    }
-
-    fn get_range_approximate_keys_cf(
-        &self,
-        cfname: &str,
-        range: Range<'_>,
-        large_threshold: u64,
-    ) -> Result<u64> {
-        self.disk_engine()
-            .get_range_approximate_keys_cf(cfname, range, large_threshold)
-    }
-
-    fn get_range_approximate_size(&self, range: Range<'_>, large_threshold: u64) -> Result<u64> {
-        self.disk_engine()
-            .get_range_approximate_size(range, large_threshold)
-    }
-
-    fn get_range_approximate_size_cf(
-        &self,
-        cfname: &str,
-        range: Range<'_>,
-        large_threshold: u64,
-    ) -> Result<u64> {
-        self.disk_engine()
-            .get_range_approximate_size_cf(cfname, range, large_threshold)
-    }
-
-    fn get_range_approximate_split_keys(
-        &self,
-        range: Range<'_>,
-        key_count: usize,
-    ) -> Result<Vec<Vec<u8>>> {
-        self.disk_engine()
-            .get_range_approximate_split_keys(range, key_count)
-    }
-
-    fn get_range_approximate_split_keys_cf(
-        &self,
-        cfname: &str,
-        range: Range<'_>,
-        key_count: usize,
-    ) -> Result<Vec<Vec<u8>>> {
-        self.disk_engine()
-            .get_range_approximate_split_keys_cf(cfname, range, key_count)
-    }
-}
diff --git a/components/hybrid_engine/src/region_cache_engine.rs b/components/hybrid_engine/src/region_cache_engine.rs
deleted file mode 100644
index 6581232289d..00000000000
--- a/components/hybrid_engine/src/region_cache_engine.rs
+++ /dev/null
@@ -1,26 +0,0 @@
-// Copyright 2024 TiKV Project Authors. Licensed under Apache-2.0.
-
-use engine_traits::{KvEngine, RegionCacheEngine, RegionCacheEngineExt, RegionEvent};
-
-use crate::HybridEngine;
-
-impl<EK, EC> RegionCacheEngineExt for HybridEngine<EK, EC>
-where
-    EK: KvEngine,
-    EC: RegionCacheEngine,
-{
-    #[inline]
-    fn on_region_event(&self, event: RegionEvent) {
-        self.region_cache_engine().on_region_event(event);
-    }
-
-    #[inline]
-    fn region_cached(&self, region: &kvproto::metapb::Region) -> bool {
-        self.region_cache_engine().region_cached(region)
-    }
-
-    #[inline]
-    fn load_region(&self, region: &kvproto::metapb::Region) {
-        self.region_cache_engine().load_region(region)
-    }
-}
diff --git a/components/hybrid_engine/src/snapshot.rs b/components/hybrid_engine/src/snapshot.rs
index 48f16c8f16b..01587de3fa9 100644
--- a/components/hybrid_engine/src/snapshot.rs
+++ b/components/hybrid_engine/src/snapshot.rs
@@ -78,7 +78,7 @@ where
     EK: KvEngine,
     EC: RegionCacheEngine,
 {
-    fn region_cache_engine_hit(&self) -> bool {
+    fn in_memory_engine_hit(&self) -> bool {
         self.region_cache_snap.is_some()
     }
 }
@@ -160,14 +160,17 @@ where
 
 #[cfg(test)]
 mod tests {
-
     use engine_traits::{
-        CacheRegion, IterOptions, Iterable, Iterator, Mutable, SnapshotContext, WriteBatch,
-        WriteBatchExt, CF_DEFAULT,
+        CacheRegion, IterOptions, Iterable, Iterator, Mutable, WriteBatch, WriteBatchExt,
+        CF_DEFAULT,
     };
     use in_memory_engine::{test_util::new_region, InMemoryEngineConfig, RegionCacheStatus};
+    use raftstore::coprocessor::WriteBatchWrapper;
 
-    use crate::util::hybrid_engine_for_tests;
+    use crate::{
+        engine::SnapshotContext, observer::RegionCacheWriteBatchObserver,
+        util::hybrid_engine_for_tests,
+    };
 
     #[test]
     fn test_iterator() {
@@ -192,11 +195,17 @@ mod tests {
             let mut iter = snap.iterator_opt(CF_DEFAULT, iter_opt.clone()).unwrap();
             assert!(!iter.seek_to_first().unwrap());
         }
-        let mut write_batch = hybrid_engine.write_batch();
-        write_batch.prepare_for_region(&region);
-        write_batch
+        let engine = hybrid_engine.region_cache_engine().clone();
+        let observer = RegionCacheWriteBatchObserver::new(engine.clone());
+        let mut ob_wb = observer.new_observable_write_batch();
+        ob_wb.cache_write_batch.prepare_for_region(&region);
+        ob_wb
             .cache_write_batch
             .set_region_cache_status(RegionCacheStatus::Cached);
+        let mut write_batch = WriteBatchWrapper::new(
+            hybrid_engine.disk_engine().write_batch(),
+            Some(Box::new(ob_wb)),
+        );
         write_batch.put(b"zhello", b"world").unwrap();
         let seq = write_batch.write().unwrap();
         assert!(seq > 0);
diff --git a/components/hybrid_engine/src/sst.rs b/components/hybrid_engine/src/sst.rs
deleted file mode 100644
index 604888ccf44..00000000000
--- a/components/hybrid_engine/src/sst.rs
+++ /dev/null
@@ -1,53 +0,0 @@
-// Copyright 2023 TiKV Project Authors. Licensed under Apache-2.0.
-
-use engine_traits::{
-    KvEngine, RegionCacheEngine, Result, SstCompressionType, SstExt, SstWriterBuilder,
-};
-
-use crate::engine::HybridEngine;
-
-pub struct HybridEngineSstWriteBuilder<EK: SstExt>(EK::SstWriterBuilder);
-
-impl<EK, EC> SstExt for HybridEngine<EK, EC>
-where
-    EK: KvEngine,
-    EC: RegionCacheEngine,
-{
-    type SstReader = EK::SstReader;
-    type SstWriter = EK::SstWriter;
-    type SstWriterBuilder = HybridEngineSstWriteBuilder<EK>;
-}
-
-impl<EK, EC> SstWriterBuilder<HybridEngine<EK, EC>> for HybridEngineSstWriteBuilder<EK>
-where
-    EK: KvEngine,
-    EC: RegionCacheEngine,
-{
-    fn new() -> Self {
-        HybridEngineSstWriteBuilder(EK::SstWriterBuilder::new())
-    }
-
-    fn set_db(self, db: &HybridEngine<EK, EC>) -> Self {
-        HybridEngineSstWriteBuilder(self.0.set_db(db.disk_engine()))
-    }
-
-    fn set_cf(self, cf: &str) -> Self {
-        HybridEngineSstWriteBuilder(self.0.set_cf(cf))
-    }
-
-    fn set_in_memory(self, in_memory: bool) -> Self {
-        HybridEngineSstWriteBuilder(self.0.set_in_memory(in_memory))
-    }
-
-    fn set_compression_type(self, compression: Option<SstCompressionType>) -> Self {
-        HybridEngineSstWriteBuilder(self.0.set_compression_type(compression))
-    }
-
-    fn set_compression_level(self, level: i32) -> Self {
-        HybridEngineSstWriteBuilder(self.0.set_compression_level(level))
-    }
-
-    fn build(self, path: &str) -> Result<<HybridEngine<EK, EC> as SstExt>::SstWriter> {
-        self.0.build(path)
-    }
-}
diff --git a/components/hybrid_engine/src/table_properties.rs b/components/hybrid_engine/src/table_properties.rs
deleted file mode 100644
index 6ad95e5931a..00000000000
--- a/components/hybrid_engine/src/table_properties.rs
+++ /dev/null
@@ -1,21 +0,0 @@
-// Copyright 2023 TiKV Project Authors. Licensed under Apache-2.0.
-
-use engine_traits::{KvEngine, Range, RegionCacheEngine, Result, TablePropertiesExt};
-
-use crate::engine::HybridEngine;
-
-impl<EK, EC> TablePropertiesExt for HybridEngine<EK, EC>
-where
-    EK: KvEngine,
-    EC: RegionCacheEngine,
-{
-    type TablePropertiesCollection = EK::TablePropertiesCollection;
-
-    fn table_properties_collection(
-        &self,
-        cf: &str,
-        ranges: &[Range<'_>],
-    ) -> Result<Self::TablePropertiesCollection> {
-        self.disk_engine().table_properties_collection(cf, ranges)
-    }
-}
diff --git a/components/hybrid_engine/src/ttl_properties.rs b/components/hybrid_engine/src/ttl_properties.rs
deleted file mode 100644
index d5b7d8578b5..00000000000
--- a/components/hybrid_engine/src/ttl_properties.rs
+++ /dev/null
@@ -1,21 +0,0 @@
-// Copyright 2023 TiKV Project Authors. Licensed under Apache-2.0.
-
-use engine_traits::{KvEngine, RegionCacheEngine, Result, TtlProperties, TtlPropertiesExt};
-
-use crate::engine::HybridEngine;
-
-impl<EK, EC> TtlPropertiesExt for HybridEngine<EK, EC>
-where
-    EK: KvEngine,
-    EC: RegionCacheEngine,
-{
-    fn get_range_ttl_properties_cf(
-        &self,
-        cf: &str,
-        start_key: &[u8],
-        end_key: &[u8],
-    ) -> Result<Vec<(String, TtlProperties)>> {
-        self.disk_engine()
-            .get_range_ttl_properties_cf(cf, start_key, end_key)
-    }
-}
diff --git a/components/hybrid_engine/src/util.rs b/components/hybrid_engine/src/util.rs
index 2b04ffe1972..2a5828a7658 100644
--- a/components/hybrid_engine/src/util.rs
+++ b/components/hybrid_engine/src/util.rs
@@ -19,7 +19,7 @@ use crate::HybridEngine;
 /// use hybrid_engine::util::hybrid_engine_for_tests;
 /// use in_memory_engine::{test_util::new_region, InMemoryEngineConfig};
 /// let mut config = InMemoryEngineConfig::default();
-/// config.enabled = true;
+/// config.enable = true;
 /// let (_path, _hybrid_engine) = hybrid_engine_for_tests("temp", config, |memory_engine| {
 ///     let region = new_region(1, b"", b"z");
 ///     memory_engine.new_region(region);
diff --git a/components/hybrid_engine/src/write_batch.rs b/components/hybrid_engine/src/write_batch.rs
deleted file mode 100644
index 7c2d08b5755..00000000000
--- a/components/hybrid_engine/src/write_batch.rs
+++ /dev/null
@@ -1,302 +0,0 @@
-// Copyright 2023 TiKV Project Authors. Licensed under Apache-2.0.
-
-use std::sync::atomic::{AtomicBool, Ordering};
-
-use engine_traits::{
-    is_data_cf, KvEngine, Mutable, Result, WriteBatch, WriteBatchExt, WriteOptions,
-};
-use in_memory_engine::{RegionCacheMemoryEngine, RegionCacheWriteBatch};
-use kvproto::metapb;
-
-use crate::engine::HybridEngine;
-
-pub struct HybridEngineWriteBatch<EK: KvEngine> {
-    disk_write_batch: EK::WriteBatch,
-    pub(crate) cache_write_batch: RegionCacheWriteBatch,
-}
-
-impl<EK> WriteBatchExt for HybridEngine<EK, RegionCacheMemoryEngine>
-where
-    EK: KvEngine,
-{
-    type WriteBatch = HybridEngineWriteBatch<EK>;
-    const WRITE_BATCH_MAX_KEYS: usize = EK::WRITE_BATCH_MAX_KEYS;
-
-    fn write_batch(&self) -> Self::WriteBatch {
-        HybridEngineWriteBatch {
-            disk_write_batch: self.disk_engine().write_batch(),
-            cache_write_batch: self.region_cache_engine().write_batch(),
-        }
-    }
-
-    fn write_batch_with_cap(&self, cap: usize) -> Self::WriteBatch {
-        HybridEngineWriteBatch {
-            disk_write_batch: self.disk_engine().write_batch_with_cap(cap),
-            cache_write_batch: self.region_cache_engine().write_batch_with_cap(cap),
-        }
-    }
-}
-
-impl<EK: KvEngine> WriteBatch for HybridEngineWriteBatch<EK> {
-    fn write_opt(&mut self, opts: &WriteOptions) -> Result<u64> {
-        self.write_callback_opt(opts, |_| ())
-    }
-
-    fn write_callback_opt(&mut self, opts: &WriteOptions, mut cb: impl FnMut(u64)) -> Result<u64> {
-        let called = AtomicBool::new(false);
-        let res = self
-            .disk_write_batch
-            .write_callback_opt(opts, |s| {
-                if !called.fetch_or(true, Ordering::SeqCst) {
-                    self.cache_write_batch.set_sequence_number(s).unwrap();
-                    self.cache_write_batch.write_opt(opts).unwrap();
-                }
-            })
-            .map(|s| {
-                cb(s);
-                s
-            });
-        self.cache_write_batch.maybe_compact_lock_cf();
-        res
-    }
-
-    fn data_size(&self) -> usize {
-        self.disk_write_batch.data_size()
-    }
-
-    fn count(&self) -> usize {
-        self.disk_write_batch.count()
-    }
-
-    fn is_empty(&self) -> bool {
-        self.disk_write_batch.is_empty()
-    }
-
-    fn should_write_to_engine(&self) -> bool {
-        self.disk_write_batch.should_write_to_engine()
-    }
-
-    fn clear(&mut self) {
-        self.disk_write_batch.clear();
-        self.cache_write_batch.clear()
-    }
-
-    fn set_save_point(&mut self) {
-        self.disk_write_batch.set_save_point();
-        self.cache_write_batch.set_save_point()
-    }
-
-    fn pop_save_point(&mut self) -> Result<()> {
-        self.disk_write_batch.pop_save_point()?;
-        self.cache_write_batch.pop_save_point()
-    }
-
-    fn rollback_to_save_point(&mut self) -> Result<()> {
-        self.disk_write_batch.rollback_to_save_point()?;
-        self.cache_write_batch.rollback_to_save_point()
-    }
-
-    fn merge(&mut self, other: Self) -> Result<()> {
-        self.disk_write_batch.merge(other.disk_write_batch)?;
-        self.cache_write_batch.merge(other.cache_write_batch)
-    }
-
-    fn prepare_for_region(&mut self, r: &metapb::Region) {
-        self.cache_write_batch.prepare_for_region(r);
-    }
-}
-
-impl<EK: KvEngine> Mutable for HybridEngineWriteBatch<EK> {
-    fn put(&mut self, key: &[u8], value: &[u8]) -> Result<()> {
-        self.disk_write_batch.put(key, value)?;
-        self.cache_write_batch.put(key, value)
-    }
-
-    fn put_cf(&mut self, cf: &str, key: &[u8], value: &[u8]) -> Result<()> {
-        self.disk_write_batch.put_cf(cf, key, value)?;
-        if is_data_cf(cf) {
-            self.cache_write_batch.put_cf(cf, key, value)?;
-        }
-        Ok(())
-    }
-
-    fn delete(&mut self, key: &[u8]) -> Result<()> {
-        self.disk_write_batch.delete(key)?;
-        self.cache_write_batch.delete(key)
-    }
-
-    fn delete_cf(&mut self, cf: &str, key: &[u8]) -> Result<()> {
-        self.disk_write_batch.delete_cf(cf, key)?;
-        self.cache_write_batch.delete_cf(cf, key)
-    }
-
-    fn delete_range(&mut self, begin_key: &[u8], end_key: &[u8]) -> Result<()> {
-        self.disk_write_batch.delete_range(begin_key, end_key)?;
-        // delete_range in range cache engine means eviction -- all ranges overlapped
-        // with [begin_key, end_key] will be evicted.
-        self.cache_write_batch.delete_range(begin_key, end_key)
-    }
-
-    fn delete_range_cf(&mut self, cf: &str, begin_key: &[u8], end_key: &[u8]) -> Result<()> {
-        self.disk_write_batch
-            .delete_range_cf(cf, begin_key, end_key)?;
-        // delete_range in range cache engine means eviction -- all ranges overlapped
-        // with [begin_key, end_key] will be evicted.
-        self.cache_write_batch
-            .delete_range_cf(cf, begin_key, end_key)
-    }
-}
-
-#[cfg(test)]
-mod tests {
-
-    use std::time::Duration;
-
-    use engine_traits::{
-        CacheRegion, Mutable, Peekable, RegionCacheEngine, SnapshotContext, WriteBatch,
-        WriteBatchExt,
-    };
-    use in_memory_engine::{test_util::new_region, InMemoryEngineConfig, RegionCacheStatus};
-
-    use crate::util::hybrid_engine_for_tests;
-
-    #[test]
-    fn test_write_to_both_engines() {
-        let region = new_region(1, b"", b"z");
-        let region_clone = region.clone();
-        let (_path, hybrid_engine) = hybrid_engine_for_tests(
-            "temp",
-            InMemoryEngineConfig::config_for_test(),
-            move |memory_engine| {
-                let id = region_clone.id;
-                memory_engine.new_region(region_clone);
-                memory_engine.core().region_manager().set_safe_point(id, 5);
-            },
-        )
-        .unwrap();
-        let cache_region = CacheRegion::from_region(&region);
-        let mut write_batch = hybrid_engine.write_batch();
-        write_batch.prepare_for_region(&region);
-        write_batch
-            .cache_write_batch
-            .set_region_cache_status(RegionCacheStatus::Cached);
-        write_batch.put(b"zhello", b"world").unwrap();
-        let seq = write_batch.write().unwrap();
-        assert!(seq > 0);
-        let actual: &[u8] = &hybrid_engine.get_value(b"zhello").unwrap().unwrap();
-        assert_eq!(b"world", &actual);
-        let ctx = SnapshotContext {
-            region: Some(cache_region.clone()),
-            read_ts: 10,
-        };
-        let snap = hybrid_engine.new_snapshot(Some(ctx));
-        let actual: &[u8] = &snap.get_value(b"zhello").unwrap().unwrap();
-        assert_eq!(b"world", &actual);
-        let actual: &[u8] = &snap.disk_snap().get_value(b"zhello").unwrap().unwrap();
-        assert_eq!(b"world", &actual);
-        let actual: &[u8] = &snap
-            .region_cache_snap()
-            .unwrap()
-            .get_value(b"zhello")
-            .unwrap()
-            .unwrap();
-        assert_eq!(b"world", &actual);
-    }
-
-    #[test]
-    fn test_in_memory_engine() {
-        let (_path, hybrid_engine) = hybrid_engine_for_tests(
-            "temp",
-            InMemoryEngineConfig::config_for_test(),
-            |memory_engine| {
-                let region = new_region(1, b"k00", b"k10");
-                memory_engine.new_region(region);
-                memory_engine.core().region_manager().set_safe_point(1, 10);
-            },
-        )
-        .unwrap();
-
-        let mut write_batch = hybrid_engine.write_batch();
-        write_batch
-            .cache_write_batch
-            .set_sequence_number(0)
-            .unwrap(); // First call ok.
-        assert!(
-            write_batch
-                .cache_write_batch
-                .set_sequence_number(0)
-                .is_err()
-        ); // Second call err.
-    }
-
-    #[test]
-    fn test_delete_range() {
-        let region1 = new_region(1, b"k00", b"k10");
-        let region2 = new_region(2, b"k20", b"k30");
-        let cache_region1 = CacheRegion::from_region(&region1);
-        let cache_region2 = CacheRegion::from_region(&region2);
-
-        let region1_clone = region1.clone();
-        let region2_clone = region2.clone();
-        let (_path, hybrid_engine) = hybrid_engine_for_tests(
-            "temp",
-            InMemoryEngineConfig::config_for_test(),
-            move |memory_engine| {
-                memory_engine.new_region(region1_clone);
-                memory_engine.new_region(region2_clone);
-            },
-        )
-        .unwrap();
-
-        let mut wb = hybrid_engine.write_batch();
-        wb.prepare_for_region(&region1);
-        wb.put(b"zk05", b"val").unwrap();
-        wb.put(b"zk08", b"val2").unwrap();
-        wb.prepare_for_region(&region2);
-        wb.put(b"zk25", b"val3").unwrap();
-        wb.put(b"zk27", b"val4").unwrap();
-        wb.write().unwrap();
-
-        hybrid_engine
-            .region_cache_engine()
-            .snapshot(cache_region1.clone(), 1000, 1000)
-            .unwrap();
-        hybrid_engine
-            .region_cache_engine()
-            .snapshot(cache_region2.clone(), 1000, 1000)
-            .unwrap();
-        assert_eq!(
-            4,
-            hybrid_engine
-                .region_cache_engine()
-                .core()
-                .engine()
-                .cf_handle("default")
-                .len()
-        );
-
-        let mut wb = hybrid_engine.write_batch();
-        // all ranges overlapped with it will be evicted
-        wb.prepare_for_region(&region1);
-        wb.delete_range(b"zk05", b"zk08").unwrap();
-        wb.prepare_for_region(&region2);
-        wb.delete_range(b"zk20", b"zk21").unwrap();
-        wb.write().unwrap();
-
-        hybrid_engine
-            .region_cache_engine()
-            .snapshot(cache_region1.clone(), 1000, 1000)
-            .unwrap_err();
-        hybrid_engine
-            .region_cache_engine()
-            .snapshot(cache_region2.clone(), 1000, 1000)
-            .unwrap_err();
-        let m_engine = hybrid_engine.region_cache_engine();
-
-        test_util::eventually(
-            Duration::from_millis(100),
-            Duration::from_millis(2000),
-            || m_engine.core().engine().cf_handle("default").is_empty(),
-        );
-    }
-}
diff --git a/components/hybrid_engine/tests/failpoints/mod.rs b/components/hybrid_engine/tests/failpoints/mod.rs
deleted file mode 100644
index 25c77e4c418..00000000000
--- a/components/hybrid_engine/tests/failpoints/mod.rs
+++ /dev/null
@@ -1,3 +0,0 @@
-// Copyright 2024 TiKV Project Authors. Licensed under Apache-2.0.
-
-mod test_write_batch;
diff --git a/components/hybrid_engine/tests/failpoints/test_write_batch.rs b/components/hybrid_engine/tests/failpoints/test_write_batch.rs
deleted file mode 100644
index 0b51f0020cc..00000000000
--- a/components/hybrid_engine/tests/failpoints/test_write_batch.rs
+++ /dev/null
@@ -1,102 +0,0 @@
-// Copyright 2024 TiKV Project Authors. Licensed under Apache-2.0.
-
-use std::sync::mpsc::sync_channel;
-
-use crossbeam::epoch;
-use engine_traits::{CacheRegion, Mutable, WriteBatch, WriteBatchExt};
-use hybrid_engine::util::hybrid_engine_for_tests;
-use in_memory_engine::{
-    decode_key, test_util::new_region, InMemoryEngineConfig, InternalKey, ValueType,
-};
-
-#[test]
-fn test_sequence_number_unique() {
-    let (_path, hybrid_engine) =
-        hybrid_engine_for_tests("temp", InMemoryEngineConfig::config_for_test(), |_| {}).unwrap();
-
-    let (tx, rx) = sync_channel(0);
-    fail::cfg_callback("ime_on_completes_batch_loading", move || {
-        fail::cfg("ime_on_start_loading_region", "pause").unwrap();
-        tx.send(true).unwrap();
-    })
-    .unwrap();
-
-    // first write some data, these data should be handled by batch loading.
-    let mut wb = hybrid_engine.write_batch();
-    wb.put(b"zk5", b"val").unwrap(); // seq 1
-    wb.put(b"zk7", b"val").unwrap(); // seq 2
-
-    let engine = hybrid_engine.region_cache_engine().clone();
-    let r = new_region(1, b"k", b"k5");
-    engine.new_region(r.clone());
-    wb.write().unwrap();
-
-    // Mock that we have a loading range, and there are some keys written in it
-    // during the load
-    let r2 = new_region(2, b"k5", b"k7");
-    let r3 = new_region(3, b"k7", b"k9");
-    let cache_region2 = CacheRegion::from_region(&r2);
-    let cache_region3 = CacheRegion::from_region(&r3);
-    engine.load_region(cache_region2.clone()).unwrap();
-    engine.load_region(cache_region3.clone()).unwrap();
-
-    // The sequence number of write batch should be increased one by one, otherwise
-    // if a delete and a put of the same key occurs in the same write batch,
-    // the delete will be hidden by the put even the delete is performed
-    // after the put.
-    // while we block the batch loading of region3, it's new KVs are still directly
-    // written into the skiplist.
-    let mut wb = hybrid_engine.write_batch();
-    wb.prepare_for_region(&r);
-    wb.put(b"zk", b"val").unwrap(); // seq 3
-    wb.delete(b"zk").unwrap(); // seq 4
-    wb.put(b"zk2", b"val").unwrap(); // seq 5
-
-    wb.prepare_for_region(&r2);
-    wb.put(b"zk6", b"val").unwrap(); // seq 6
-    wb.delete(b"zk5").unwrap(); // seq 7
-    wb.put(b"zk5", b"val2").unwrap(); // seq 8
-
-    wb.prepare_for_region(&r3);
-    wb.put(b"zk8", b"val").unwrap(); // seq 9
-    wb.put(b"zk7", b"val2").unwrap(); // seq 10
-
-    rx.recv().unwrap();
-    wb.write().unwrap();
-
-    let mut iter = engine.core().engine().cf_handle("default").iterator();
-    let guard = &epoch::pin();
-
-    let mut first = true;
-
-    for (k, sequence, v_type) in [
-        (b"zk".to_vec(), 4, ValueType::Deletion),
-        (b"zk".to_vec(), 3, ValueType::Value),
-        (b"zk2".to_vec(), 5, ValueType::Value),
-        (b"zk5".to_vec(), 8, ValueType::Value),
-        (b"zk5".to_vec(), 7, ValueType::Deletion),
-        // NOTE: for batch loading, we always use the current seq number
-        // to write all the keys.
-        (b"zk5".to_vec(), 2, ValueType::Value),
-        (b"zk6".to_vec(), 6, ValueType::Value),
-        (b"zk7".to_vec(), 10, ValueType::Value),
-        // "zk7" with seq 2 is block, so invisible here.
-        (b"zk8".to_vec(), 9, ValueType::Value),
-    ] {
-        if first {
-            iter.seek_to_first(guard);
-            first = false;
-        } else {
-            iter.next(guard);
-        }
-
-        let expected_key = InternalKey {
-            user_key: k.as_slice(),
-            v_type,
-            sequence,
-        };
-        let key = iter.key();
-        let got_key = decode_key(key.as_bytes());
-        assert_eq!(expected_key, got_key);
-    }
-}
diff --git a/components/in_memory_engine/src/write_batch.rs b/components/in_memory_engine/src/write_batch.rs
index 19346eecb3c..fd92d4a5b8a 100644
--- a/components/in_memory_engine/src/write_batch.rs
+++ b/components/in_memory_engine/src/write_batch.rs
@@ -125,6 +125,37 @@ impl RegionCacheWriteBatch {
         }
     }
 
+    pub fn prepare_for_region(&mut self, region: &metapb::Region) {
+        // If the region is already prepared for write, we do not need to prepare it
+        // again. See comments for the `prepared_regions` field for more details.
+        if let Some(current_region) = &self.current_region
+            && current_region.id == region.id
+        {
+            return;
+        }
+        let time = Instant::now();
+        // verify that the region is not prepared before
+        if self.prepared_regions.contains(&region.id) {
+            panic!(
+                "region {} is prepared for write before, but it is not the current region",
+                region.id
+            );
+        }
+        self.prepared_regions.push(region.id);
+        // record last region for clearing region in written flags.
+        self.record_last_written_region();
+
+        let cached_region = CacheRegion::from_region(region);
+        // TODO: remote range.
+        self.set_region_cache_status(
+            self.engine
+                .prepare_for_apply(&cached_region, region.is_in_flashback),
+        );
+        self.current_region = Some(cached_region);
+        self.current_region_evicted = false;
+        self.prepare_for_write_duration += time.saturating_elapsed();
+    }
+
     /// Trigger a CleanLockTombstone task if the accumulated lock cf
     /// modification exceeds the threshold (16MB).
     ///
@@ -515,37 +546,6 @@ impl WriteBatch for RegionCacheWriteBatch {
         self.buffer.append(&mut other.buffer);
         Ok(())
     }
-
-    fn prepare_for_region(&mut self, region: &metapb::Region) {
-        // If the region is already prepared for write, we do not need to prepare it
-        // again. See comments for the `prepared_regions` field for more details.
-        if let Some(current_region) = &self.current_region
-            && current_region.id == region.id
-        {
-            return;
-        }
-        let time = Instant::now();
-        // verify that the region is not prepared before
-        if self.prepared_regions.contains(&region.id) {
-            panic!(
-                "region {} is prepared for write before, but it is not the current region",
-                region.id
-            );
-        }
-        self.prepared_regions.push(region.id);
-        // record last region for clearing region in written flags.
-        self.record_last_written_region();
-
-        let cached_region = CacheRegion::from_region(region);
-        // TODO: remote range.
-        self.set_region_cache_status(
-            self.engine
-                .prepare_for_apply(&cached_region, region.is_in_flashback),
-        );
-        self.current_region = Some(cached_region);
-        self.current_region_evicted = false;
-        self.prepare_for_write_duration += time.saturating_elapsed();
-    }
 }
 
 impl Mutable for RegionCacheWriteBatch {
diff --git a/components/raftstore/src/coprocessor/dispatcher.rs b/components/raftstore/src/coprocessor/dispatcher.rs
index adc8cf3e38c..b350d971e2d 100644
--- a/components/raftstore/src/coprocessor/dispatcher.rs
+++ b/components/raftstore/src/coprocessor/dispatcher.rs
@@ -1401,7 +1401,7 @@ mod tests {
         index += ObserverIndex::PostApplyQuery as usize;
         assert_all!([&ob.called], &[index]);
 
-        host.on_role_change(&region, RoleChange::new(StateRole::Leader));
+        host.on_role_change(&region, RoleChange::new_for_test(StateRole::Leader));
         index += ObserverIndex::OnRoleChange as usize;
         assert_all!([&ob.called], &[index]);
 
diff --git a/components/raftstore/src/coprocessor/mod.rs b/components/raftstore/src/coprocessor/mod.rs
index b17f6df5ad3..b40d5294610 100644
--- a/components/raftstore/src/coprocessor/mod.rs
+++ b/components/raftstore/src/coprocessor/mod.rs
@@ -295,8 +295,7 @@ pub struct RoleChange {
 }
 
 impl RoleChange {
-    #[cfg(any(test, feature = "testexport"))]
-    pub fn new(state: StateRole) -> Self {
+    pub fn new_for_test(state: StateRole) -> Self {
         RoleChange {
             state,
             leader_id: raft::INVALID_ID,
diff --git a/components/raftstore/src/coprocessor/read_write/write_batch.rs b/components/raftstore/src/coprocessor/read_write/write_batch.rs
index bc1a334c7b3..72c98523086 100644
--- a/components/raftstore/src/coprocessor/read_write/write_batch.rs
+++ b/components/raftstore/src/coprocessor/read_write/write_batch.rs
@@ -13,11 +13,9 @@ pub trait WriteBatchObserver: Send {
 /// additional methods to specify which region the write operations belong to.
 // TODO: May be we can unified it with `CmdObserver`?
 pub trait ObservableWriteBatch: WriteBatch + Send {
-    // It declares that the following consecutive write will be within this
-    // region.
-    // TODO: move `prepare_for_region` from `WriteBatch` to here.
-    // fn prepare_for_region(&mut self, _: CacheRegion);
-
+    /// It declares that the following consecutive write will be within this
+    /// region.
+    fn prepare_for_region(&mut self, region: &metapb::Region);
     /// Commit the WriteBatch with the given options and sequence number.
     fn write_opt_seq(&mut self, opts: &WriteOptions, seq_num: u64);
     /// It is called after a write operation is finished.
@@ -39,6 +37,12 @@ impl<WB> WriteBatchWrapper<WB> {
             observable_write_batch,
         }
     }
+
+    pub fn prepare_for_region(&mut self, region: &metapb::Region) {
+        if let Some(w) = self.observable_write_batch.as_mut() {
+            w.prepare_for_region(region)
+        }
+    }
 }
 
 impl<WB: WriteBatch> WriteBatch for WriteBatchWrapper<WB> {
@@ -113,12 +117,6 @@ impl<WB: WriteBatch> WriteBatch for WriteBatchWrapper<WB> {
     fn merge(&mut self, _: Self) -> Result<()> {
         unimplemented!("WriteBatchWrapper does not support merge")
     }
-
-    fn prepare_for_region(&mut self, region: &metapb::Region) {
-        if let Some(w) = self.observable_write_batch.as_mut() {
-            w.prepare_for_region(region)
-        }
-    }
 }
 
 impl<WB: WriteBatch> Mutable for WriteBatchWrapper<WB> {
@@ -151,8 +149,6 @@ impl<WB: WriteBatch> Mutable for WriteBatchWrapper<WB> {
     }
 
     fn delete_range(&mut self, begin_key: &[u8], end_key: &[u8]) -> Result<()> {
-        // delete_range in range cache engine means eviction -- all ranges overlapped
-        // with [begin_key, end_key] will be evicted.
         if let Some(w) = self.observable_write_batch.as_mut() {
             w.delete_range_cf(CF_DEFAULT, begin_key, end_key)?;
         }
@@ -160,8 +156,6 @@ impl<WB: WriteBatch> Mutable for WriteBatchWrapper<WB> {
     }
 
     fn delete_range_cf(&mut self, cf: &str, begin_key: &[u8], end_key: &[u8]) -> Result<()> {
-        // delete_range in range cache engine means eviction -- all ranges overlapped
-        // with [begin_key, end_key] will be evicted.
         if let Some(w) = self.observable_write_batch.as_mut() {
             w.delete_range_cf(cf, begin_key, end_key)?;
         }
diff --git a/components/server/src/server.rs b/components/server/src/server.rs
index 19f47fa725f..a083ce5769a 100644
--- a/components/server/src/server.rs
+++ b/components/server/src/server.rs
@@ -1673,8 +1673,9 @@ where
             );
 
             // Hybrid engine observer.
-            let eviction_observer =
-                HybridEngineLoadEvictionObserver::new(Arc::new(in_memory_engine.clone()));
+            let eviction_observer = HybridEngineLoadEvictionObserver::new(Arc::new(
+                in_memory_engine.region_cache_engine().clone(),
+            ));
             eviction_observer.register_to(self.coprocessor_host.as_mut().unwrap());
             let write_batch_observer =
                 RegionCacheWriteBatchObserver::new(in_memory_engine.region_cache_engine().clone());
diff --git a/components/test_raftstore/src/cluster.rs b/components/test_raftstore/src/cluster.rs
index 549c0e35f41..0a165ec0528 100644
--- a/components/test_raftstore/src/cluster.rs
+++ b/components/test_raftstore/src/cluster.rs
@@ -58,7 +58,6 @@ use tikv_util::{
 };
 use txn_types::WriteBatchFlags;
 
-use self::region_cache_engine::RangCacheEngineExt;
 use super::*;
 use crate::Config;
 // We simulate 3 or 5 nodes, each has a store.
@@ -178,11 +177,6 @@ pub struct Cluster<T: Simulator> {
     pub sim: Arc<RwLock<T>>,
     pub pd_client: Arc<TestPdClient>,
     resource_manager: Option<Arc<ResourceGroupManager>>,
-
-    // When this is set, the `HybridEngineImpl` will be used as the underlying KvEngine. In
-    // addition, it atomaticaly load the whole range when start. When we want to do something
-    // specific, for example, only load ranges of some regions, we may not set this.
-    region_cache_engine_enabled_with_whole_range: bool,
 }
 
 impl<T: Simulator> Cluster<T> {
@@ -216,7 +210,6 @@ impl<T: Simulator> Cluster<T> {
             resource_manager: Some(Arc::new(ResourceGroupManager::default())),
             kv_statistics: vec![],
             raft_statistics: vec![],
-            region_cache_engine_enabled_with_whole_range: false,
         }
     }
 
@@ -346,17 +339,6 @@ impl<T: Simulator> Cluster<T> {
         self.create_engines();
         self.bootstrap_region().unwrap();
         self.start().unwrap();
-        if self.region_cache_engine_enabled_with_whole_range {
-            let pd_regions = self.pd_client.scan_regions(&[], &[], i32::MAX).unwrap();
-            let regions: Vec<_> = pd_regions
-                .into_iter()
-                .map(|mut r| r.take_region())
-                .collect();
-
-            self.engines
-                .iter()
-                .for_each(|(_, engines)| engines.kv.cache_regions(&regions));
-        }
     }
 
     // Bootstrap the store with fixed ID (like 1, 2, .. 5) and
@@ -2020,10 +2002,6 @@ impl<T: Simulator> Cluster<T> {
 
         Ok(())
     }
-
-    pub fn region_cache_engine_enabled_with_whole_range(&mut self, v: bool) {
-        self.region_cache_engine_enabled_with_whole_range = v;
-    }
 }
 
 impl<T: Simulator> Drop for Cluster<T> {
@@ -2064,27 +2042,3 @@ impl RawEngine<RocksEngine> for RocksEngine {
         self.get_msg_cf(CF_RAFT, &keys::raft_state_key(region_id))
     }
 }
-
-impl RawEngine<RocksEngine> for HybridEngineImpl {
-    fn region_cache_engine(&self) -> bool {
-        true
-    }
-
-    fn region_local_state(
-        &self,
-        region_id: u64,
-    ) -> engine_traits::Result<Option<RegionLocalState>> {
-        self.disk_engine()
-            .get_msg_cf(CF_RAFT, &keys::region_state_key(region_id))
-    }
-
-    fn raft_apply_state(&self, region_id: u64) -> engine_traits::Result<Option<RaftApplyState>> {
-        self.disk_engine()
-            .get_msg_cf(CF_RAFT, &keys::apply_state_key(region_id))
-    }
-
-    fn raft_local_state(&self, region_id: u64) -> engine_traits::Result<Option<RaftLocalState>> {
-        self.disk_engine()
-            .get_msg_cf(CF_RAFT, &keys::raft_state_key(region_id))
-    }
-}
diff --git a/components/test_raftstore/src/lib.rs b/components/test_raftstore/src/lib.rs
index 96cdccc15ec..be38155af6c 100644
--- a/components/test_raftstore/src/lib.rs
+++ b/components/test_raftstore/src/lib.rs
@@ -11,7 +11,6 @@ extern crate tikv_util;
 mod cluster;
 mod config;
 mod node;
-pub mod region_cache_engine;
 mod router;
 mod server;
 mod transport_simulate;
diff --git a/components/test_raftstore/src/region_cache_engine.rs b/components/test_raftstore/src/region_cache_engine.rs
deleted file mode 100644
index 443f1503d06..00000000000
--- a/components/test_raftstore/src/region_cache_engine.rs
+++ /dev/null
@@ -1,22 +0,0 @@
-// Copyright 2024 TiKV Project Authors. Licensed under Apache-2.0.
-
-use engine_rocks::RocksEngine;
-use kvproto::metapb::Region;
-
-use crate::HybridEngineImpl;
-
-pub trait RangCacheEngineExt {
-    fn cache_regions(&self, _regions: &[Region]);
-}
-
-impl RangCacheEngineExt for HybridEngineImpl {
-    fn cache_regions(&self, regions: &[Region]) {
-        for r in regions {
-            self.region_cache_engine().new_region(r.clone());
-        }
-    }
-}
-
-impl RangCacheEngineExt for RocksEngine {
-    fn cache_regions(&self, _regions: &[Region]) {}
-}
diff --git a/components/test_raftstore/src/server.rs b/components/test_raftstore/src/server.rs
index 06be7650ccd..d73157c51ac 100644
--- a/components/test_raftstore/src/server.rs
+++ b/components/test_raftstore/src/server.rs
@@ -346,7 +346,8 @@ impl ServerCluster {
                 Box::new(router.clone()),
             );
             // Eviction observer
-            let observer = LoadEvictionObserver::new(Arc::new(in_memory_engine.clone()));
+            let observer =
+                LoadEvictionObserver::new(Arc::new(in_memory_engine.region_cache_engine().clone()));
             observer.register_to(&mut coprocessor_host);
             // Write batch observer
             let write_batch_observer =
@@ -953,10 +954,7 @@ pub fn new_server_cluster(id: u64, count: usize) -> Cluster<ServerCluster> {
     Cluster::new(id, count, sim, pd_client, ApiVersion::V1)
 }
 
-pub fn new_server_cluster_with_hybrid_engine_with_no_region_cache(
-    id: u64,
-    count: usize,
-) -> Cluster<ServerCluster> {
+pub fn new_server_cluster_with_hybrid_engine(id: u64, count: usize) -> Cluster<ServerCluster> {
     let pd_client = Arc::new(TestPdClient::new(id, false));
     let sim = Arc::new(RwLock::new(ServerCluster::new(Arc::clone(&pd_client))));
     let mut cluster = Cluster::new(id, count, sim, pd_client, ApiVersion::V1);
diff --git a/components/tikv_kv/src/lib.rs b/components/tikv_kv/src/lib.rs
index e7cd1e4db92..96fa7c63ce3 100644
--- a/components/tikv_kv/src/lib.rs
+++ b/components/tikv_kv/src/lib.rs
@@ -541,10 +541,9 @@ pub trait SnapshotExt {
         None
     }
 
-    /// Whether the snapshot acquired hit the cached range in the range cache
-    /// engine. It always returns false if the range cahce engine is not
-    /// enabled.
-    fn region_cache_engine_hit(&self) -> bool {
+    /// Whether the snapshot acquired hit the in memory engine. It always
+    /// returns false if the in memory engine is disabled.
+    fn in_memory_engine_hit(&self) -> bool {
         false
     }
 }
diff --git a/components/tikv_kv/src/raftstore_impls.rs b/components/tikv_kv/src/raftstore_impls.rs
index 17191924642..74dffce30ca 100644
--- a/components/tikv_kv/src/raftstore_impls.rs
+++ b/components/tikv_kv/src/raftstore_impls.rs
@@ -62,8 +62,8 @@ impl<'a, S: Snapshot> SnapshotExt for RegionSnapshotExt<'a, S> {
         self.snapshot.bucket_meta.clone()
     }
 
-    fn region_cache_engine_hit(&self) -> bool {
-        self.snapshot.get_snapshot().region_cache_engine_hit()
+    fn in_memory_engine_hit(&self) -> bool {
+        self.snapshot.get_snapshot().in_memory_engine_hit()
     }
 }
 
diff --git a/metrics/grafana/tikv_details.dashboard.py b/metrics/grafana/tikv_details.dashboard.py
index b58a7f28b9f..402e7ab58b1 100644
--- a/metrics/grafana/tikv_details.dashboard.py
+++ b/metrics/grafana/tikv_details.dashboard.py
@@ -4314,7 +4314,7 @@ def CoprocessorDetail() -> RowPanel:
                     target(
                         expr=expr_sum_rate(
                             "tikv_coprocessor_scan_details",
-                            label_selectors=['req=~"index|index_by_region_cache"'],
+                            label_selectors=['req=~"index|index_by_in_memory_engine"'],
                             by_labels=["tag"],
                         ),
                         additional_groupby=True,
@@ -4348,7 +4348,7 @@ def CoprocessorDetail() -> RowPanel:
                     target(
                         expr=expr_sum_rate(
                             "tikv_coprocessor_scan_details",
-                            label_selectors=['req=~"index|index_by_region_cache"'],
+                            label_selectors=['req=~"index|index_by_in_memory_engine"'],
                             by_labels=["cf", "tag"],
                         ),
                         additional_groupby=True,
diff --git a/metrics/grafana/tikv_details.json b/metrics/grafana/tikv_details.json
index 04c80e7ddd5..584d399814e 100644
--- a/metrics/grafana/tikv_details.json
+++ b/metrics/grafana/tikv_details.json
@@ -48782,7 +48782,7 @@
           "targets": [
             {
               "datasource": "${DS_TEST-CLUSTER}",
-              "expr": "sum(rate(\n    tikv_coprocessor_scan_details\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",req=~\"index|index_by_region_cache\"}\n    [$__rate_interval]\n)) by (tag, $additional_groupby) ",
+              "expr": "sum(rate(\n    tikv_coprocessor_scan_details\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",req=~\"index|index_by_in_memory_engine\"}\n    [$__rate_interval]\n)) by (tag, $additional_groupby) ",
               "format": "time_series",
               "hide": false,
               "instant": false,
@@ -48790,7 +48790,7 @@
               "intervalFactor": 1,
               "legendFormat": "{{tag}} {{$additional_groupby}}",
               "metric": "",
-              "query": "sum(rate(\n    tikv_coprocessor_scan_details\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",req=~\"index|index_by_region_cache\"}\n    [$__rate_interval]\n)) by (tag, $additional_groupby) ",
+              "query": "sum(rate(\n    tikv_coprocessor_scan_details\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",req=~\"index|index_by_in_memory_engine\"}\n    [$__rate_interval]\n)) by (tag, $additional_groupby) ",
               "refId": "",
               "step": 10,
               "target": ""
@@ -49048,7 +49048,7 @@
           "targets": [
             {
               "datasource": "${DS_TEST-CLUSTER}",
-              "expr": "sum(rate(\n    tikv_coprocessor_scan_details\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",req=~\"index|index_by_region_cache\"}\n    [$__rate_interval]\n)) by (cf, tag, $additional_groupby) ",
+              "expr": "sum(rate(\n    tikv_coprocessor_scan_details\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",req=~\"index|index_by_in_memory_engine\"}\n    [$__rate_interval]\n)) by (cf, tag, $additional_groupby) ",
               "format": "time_series",
               "hide": false,
               "instant": false,
@@ -49056,7 +49056,7 @@
               "intervalFactor": 1,
               "legendFormat": "{{cf}}-{{tag}} {{$additional_groupby}}",
               "metric": "",
-              "query": "sum(rate(\n    tikv_coprocessor_scan_details\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",req=~\"index|index_by_region_cache\"}\n    [$__rate_interval]\n)) by (cf, tag, $additional_groupby) ",
+              "query": "sum(rate(\n    tikv_coprocessor_scan_details\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",req=~\"index|index_by_in_memory_engine\"}\n    [$__rate_interval]\n)) by (cf, tag, $additional_groupby) ",
               "refId": "",
               "step": 10,
               "target": ""
diff --git a/metrics/grafana/tikv_details.json.sha256 b/metrics/grafana/tikv_details.json.sha256
index 513411203b4..3817d376747 100644
--- a/metrics/grafana/tikv_details.json.sha256
+++ b/metrics/grafana/tikv_details.json.sha256
@@ -1 +1 @@
-a002799e26cac25ad5ffcf9c15414f3837389edb9b1559f91a297457cd22b507  ./metrics/grafana/tikv_details.json
+a0f1b4d3924faf1a0eaef40179f4b05653e449b94d2f9659cd2ca61990530d0c  ./metrics/grafana/tikv_details.json
diff --git a/src/coprocessor/endpoint.rs b/src/coprocessor/endpoint.rs
index 878e5599869..2030c92c305 100644
--- a/src/coprocessor/endpoint.rs
+++ b/src/coprocessor/endpoint.rs
@@ -442,7 +442,7 @@ impl<E: Engine> Endpoint<E> {
         .await?;
         let latest_buckets = snapshot.ext().get_buckets();
 
-        let region_cache_snap = snapshot.ext().region_cache_engine_hit();
+        let region_cache_snap = snapshot.ext().in_memory_engine_hit();
         tracker.adjust_snapshot_type(region_cache_snap);
 
         // Check if the buckets version is latest.
diff --git a/src/coprocessor/metrics.rs b/src/coprocessor/metrics.rs
index 0486b6e8b70..309c407afc4 100644
--- a/src/coprocessor/metrics.rs
+++ b/src/coprocessor/metrics.rs
@@ -20,7 +20,7 @@ make_auto_flush_static_metric! {
         select,
         select_by_in_memory_engine,
         index,
-        index_by_region_cache,
+        index_by_in_memory_engine,
         // For AnalyzeType::{TypeColumn,TypeMixed}.
         analyze_table,
         // For AnalyzeType::{TypeIndex,TypeCommonHandle}.
diff --git a/src/coprocessor/tracker.rs b/src/coprocessor/tracker.rs
index c888c5f8226..197c35ab238 100644
--- a/src/coprocessor/tracker.rs
+++ b/src/coprocessor/tracker.rs
@@ -114,7 +114,7 @@ impl<E: Engine> Tracker<E> {
             if self.req_ctx.tag == ReqTag::select {
                 self.req_ctx.tag = ReqTag::select_by_in_memory_engine;
             } else if self.req_ctx.tag == ReqTag::index {
-                self.req_ctx.tag = ReqTag::index_by_region_cache;
+                self.req_ctx.tag = ReqTag::index_by_in_memory_engine;
             }
         }
     }
@@ -378,7 +378,7 @@ impl<E: Engine> Tracker<E> {
         if self.req_ctx.tag == ReqTag::select
             || self.req_ctx.tag == ReqTag::index
             || self.req_ctx.tag == ReqTag::select_by_in_memory_engine
-            || self.req_ctx.tag == ReqTag::index_by_region_cache
+            || self.req_ctx.tag == ReqTag::index_by_in_memory_engine
         {
             tls_collect_query(
                 region_id,
@@ -406,7 +406,7 @@ impl<E: Engine> Tracker<E> {
             static SELECT: RefCell<Option<Box<dyn PerfContext>>> = RefCell::new(None);
             static SELECT_BY_IN_MEMORY_ENGINE: RefCell<Option<Box<dyn PerfContext>>> = RefCell::new(None);
             static INDEX: RefCell<Option<Box<dyn PerfContext>>> = RefCell::new(None);
-            static INDEX_BY_REGION_CACHE: RefCell<Option<Box<dyn PerfContext>>> = RefCell::new(None);
+            static INDEX_BY_IN_MEMORY_ENGINE: RefCell<Option<Box<dyn PerfContext>>> = RefCell::new(None);
             static ANALYZE_TABLE: RefCell<Option<Box<dyn PerfContext>>> = RefCell::new(None);
             static ANALYZE_INDEX: RefCell<Option<Box<dyn PerfContext>>> = RefCell::new(None);
             static ANALYZE_FULL_SAMPLING: RefCell<Option<Box<dyn PerfContext>>> = RefCell::new(None);
@@ -418,7 +418,7 @@ impl<E: Engine> Tracker<E> {
             ReqTag::select => &SELECT,
             ReqTag::select_by_in_memory_engine => &SELECT_BY_IN_MEMORY_ENGINE,
             ReqTag::index => &INDEX,
-            ReqTag::index_by_region_cache => &INDEX_BY_REGION_CACHE,
+            ReqTag::index_by_in_memory_engine => &INDEX_BY_IN_MEMORY_ENGINE,
             ReqTag::analyze_table => &ANALYZE_TABLE,
             ReqTag::analyze_index => &ANALYZE_INDEX,
             ReqTag::analyze_full_sampling => &ANALYZE_FULL_SAMPLING,
diff --git a/src/server/lock_manager/deadlock.rs b/src/server/lock_manager/deadlock.rs
index 7fc8e046ef0..d1b148d899d 100644
--- a/src/server/lock_manager/deadlock.rs
+++ b/src/server/lock_manager/deadlock.rs
@@ -1561,19 +1561,19 @@ pub mod tests {
         host.on_region_changed(&region, RegionChangeEvent::Create, StateRole::Follower);
         check_role(Role::Follower);
         for &follower_role in &follower_roles {
-            host.on_role_change(&region, RoleChange::new(follower_role));
+            host.on_role_change(&region, RoleChange::new_for_test(follower_role));
             check_role(Role::Follower);
-            host.on_role_change(&invalid, RoleChange::new(StateRole::Leader));
+            host.on_role_change(&invalid, RoleChange::new_for_test(StateRole::Leader));
             check_role(Role::Follower);
-            host.on_role_change(&other, RoleChange::new(StateRole::Leader));
+            host.on_role_change(&other, RoleChange::new_for_test(StateRole::Leader));
             check_role(Role::Follower);
-            host.on_role_change(&region, RoleChange::new(StateRole::Leader));
+            host.on_role_change(&region, RoleChange::new_for_test(StateRole::Leader));
             check_role(Role::Leader);
-            host.on_role_change(&invalid, RoleChange::new(follower_role));
+            host.on_role_change(&invalid, RoleChange::new_for_test(follower_role));
             check_role(Role::Leader);
-            host.on_role_change(&other, RoleChange::new(follower_role));
+            host.on_role_change(&other, RoleChange::new_for_test(follower_role));
             check_role(Role::Leader);
-            host.on_role_change(&region, RoleChange::new(follower_role));
+            host.on_role_change(&region, RoleChange::new_for_test(follower_role));
             check_role(Role::Follower);
         }
 
diff --git a/tests/failpoints/cases/test_in_memory_engine.rs b/tests/failpoints/cases/test_in_memory_engine.rs
index 42b23154ed3..aa9231497dd 100644
--- a/tests/failpoints/cases/test_in_memory_engine.rs
+++ b/tests/failpoints/cases/test_in_memory_engine.rs
@@ -34,8 +34,8 @@ use test_coprocessor::{
     handle_request, init_data_with_details_pd_client, DagChunkSpliter, DagSelect, ProductTable,
 };
 use test_raftstore::{
-    get_tso, new_peer, new_put_cf_cmd, new_server_cluster_with_hybrid_engine_with_no_region_cache,
-    Cluster, ServerCluster,
+    get_tso, new_peer, new_put_cf_cmd, new_server_cluster_with_hybrid_engine, Cluster,
+    ServerCluster,
 };
 use test_util::eventually;
 use tidb_query_datatype::{
@@ -144,7 +144,7 @@ fn async_put(
 
 #[test]
 fn test_put_copr_get() {
-    let mut cluster = new_server_cluster_with_hybrid_engine_with_no_region_cache(0, 1);
+    let mut cluster = new_server_cluster_with_hybrid_engine(0, 1);
     cluster.cfg.raft_store.apply_batch_system.pool_size = 1;
     cluster.run();
 
@@ -174,13 +174,13 @@ fn test_put_copr_get() {
 
     must_copr_point_get(&mut cluster, &product, 1);
 
-    // verify it's read from range cache engine
+    // verify it's read from in memory engine
     rx.try_recv().unwrap();
 }
 
 #[test]
 fn test_load() {
-    let mut cluster = new_server_cluster_with_hybrid_engine_with_no_region_cache(0, 1);
+    let mut cluster = new_server_cluster_with_hybrid_engine(0, 1);
     cluster.cfg.raft_store.apply_batch_system.pool_size = 2;
     cluster.run();
 
@@ -248,7 +248,7 @@ fn test_load() {
     for table in &tables {
         must_copr_point_get(&mut cluster, table, 1);
 
-        // verify it's read from range cache engine
+        // verify it's read from in memory engine
         assert!(rx.try_recv().unwrap());
     }
 }
@@ -257,7 +257,7 @@ fn test_load() {
 // splits.
 #[test]
 fn test_load_with_split() {
-    let mut cluster = new_server_cluster_with_hybrid_engine_with_no_region_cache(0, 1);
+    let mut cluster = new_server_cluster_with_hybrid_engine(0, 1);
     cluster.cfg.raft_store.apply_batch_system.pool_size = 2;
     cluster.run();
 
@@ -319,7 +319,7 @@ fn test_load_with_split() {
     for table in &tables {
         must_copr_point_get(&mut cluster, table, 1);
 
-        // verify it's read from range cache engine
+        // verify it's read from in memory engine
         assert!(rx.try_recv().unwrap());
     }
 }
@@ -335,7 +335,7 @@ fn test_load_with_split() {
 // table1-table2 is scheduled.
 #[test]
 fn test_load_with_split2() {
-    let mut cluster = new_server_cluster_with_hybrid_engine_with_no_region_cache(0, 1);
+    let mut cluster = new_server_cluster_with_hybrid_engine(0, 1);
     cluster.cfg.raft_store.apply_batch_system.pool_size = 4;
     cluster.run();
     let region_cache_engine = cluster.sim.rl().get_region_cache_engine(1);
@@ -426,7 +426,7 @@ fn test_load_with_split2() {
 // range, and even been evicted.
 #[test]
 fn test_load_with_eviction() {
-    let mut cluster = new_server_cluster_with_hybrid_engine_with_no_region_cache(0, 1);
+    let mut cluster = new_server_cluster_with_hybrid_engine(0, 1);
     cluster.run();
     // load range
     let region_cache_engine = cluster.sim.rl().get_region_cache_engine(1);
@@ -496,7 +496,7 @@ fn test_load_with_eviction() {
 
 #[test]
 fn test_evictions_after_transfer_leader() {
-    let mut cluster = new_server_cluster_with_hybrid_engine_with_no_region_cache(0, 2);
+    let mut cluster = new_server_cluster_with_hybrid_engine(0, 2);
     cluster.run();
 
     let r = cluster.get_region(b"");
@@ -521,7 +521,7 @@ fn test_evictions_after_transfer_leader() {
 
 #[test]
 fn test_eviction_after_merge() {
-    let mut cluster = new_server_cluster_with_hybrid_engine_with_no_region_cache(0, 1);
+    let mut cluster = new_server_cluster_with_hybrid_engine(0, 1);
     cluster.run();
     let r = cluster.get_region(b"");
     cluster.must_split(&r, b"key1");
@@ -557,7 +557,7 @@ fn test_eviction_after_merge() {
 
 #[test]
 fn test_manual_load_range_after_transfer_leader() {
-    let mut cluster = new_server_cluster_with_hybrid_engine_with_no_region_cache(0, 2);
+    let mut cluster = new_server_cluster_with_hybrid_engine(0, 2);
     cluster.run();
 
     let r = cluster.get_region(b"");
@@ -598,7 +598,7 @@ fn test_manual_load_range_after_transfer_leader() {
 
 #[test]
 fn test_eviction_after_ingest_sst() {
-    let mut cluster = new_server_cluster_with_hybrid_engine_with_no_region_cache(0, 1);
+    let mut cluster = new_server_cluster_with_hybrid_engine(0, 1);
     cluster.run();
 
     // Generate a sst file.
@@ -667,7 +667,7 @@ fn test_eviction_after_ingest_sst() {
 
 #[test]
 fn test_pre_load_when_transfer_ledaer() {
-    let mut cluster = new_server_cluster_with_hybrid_engine_with_no_region_cache(0, 3);
+    let mut cluster = new_server_cluster_with_hybrid_engine(0, 3);
     cluster.run();
 
     let (tx, rx) = unbounded();
@@ -699,7 +699,7 @@ fn test_pre_load_when_transfer_ledaer() {
 fn test_background_loading_pending_region() {
     fail::cfg("ime_background_check_load_pending_interval", "return(1000)").unwrap();
 
-    let mut cluster = new_server_cluster_with_hybrid_engine_with_no_region_cache(0, 1);
+    let mut cluster = new_server_cluster_with_hybrid_engine(0, 1);
     cluster.run();
 
     let r = cluster.get_region(b"");
@@ -722,7 +722,7 @@ fn test_background_loading_pending_region() {
 #[test]
 fn test_delete_range() {
     let delete_range = |unsafe_destroy_range| {
-        let mut cluster = new_server_cluster_with_hybrid_engine_with_no_region_cache(0, 1);
+        let mut cluster = new_server_cluster_with_hybrid_engine(0, 1);
         cluster.run();
 
         let (tx, rx) = sync_channel(0);
@@ -753,7 +753,7 @@ fn test_delete_range() {
         })
         .unwrap();
         must_copr_point_get(&mut cluster, &product, 1);
-        // verify it's read from range cache engine
+        // verify it's read from in memory engine
         rx.try_recv().unwrap();
 
         if unsafe_destroy_range {
@@ -791,7 +791,7 @@ fn test_delete_range() {
 
 #[test]
 fn test_evict_on_flashback() {
-    let mut cluster = new_server_cluster_with_hybrid_engine_with_no_region_cache(0, 1);
+    let mut cluster = new_server_cluster_with_hybrid_engine(0, 1);
     cluster.cfg.raft_store.apply_batch_system.pool_size = 1;
     cluster.run();
 
@@ -833,7 +833,7 @@ fn test_evict_on_flashback() {
 #[test]
 fn test_load_during_flashback() {
     fail::cfg("ime_background_check_load_pending_interval", "return(1000)").unwrap();
-    let mut cluster = new_server_cluster_with_hybrid_engine_with_no_region_cache(0, 1);
+    let mut cluster = new_server_cluster_with_hybrid_engine(0, 1);
     cluster.cfg.raft_store.apply_batch_system.pool_size = 1;
     cluster.run();
 
@@ -872,7 +872,7 @@ fn test_load_during_flashback() {
 // internal state of IME's write-batch may not be cleared.
 #[test]
 fn test_apply_prepared_but_not_write() {
-    let mut cluster = new_server_cluster_with_hybrid_engine_with_no_region_cache(0, 1);
+    let mut cluster = new_server_cluster_with_hybrid_engine(0, 1);
     cluster.cfg.raft_store.apply_batch_system.pool_size = 1;
     cluster.run();
 
@@ -966,7 +966,7 @@ fn test_apply_prepared_but_not_write() {
 
 #[test]
 fn test_eviction_when_destroy_peer() {
-    let mut cluster = new_server_cluster_with_hybrid_engine_with_no_region_cache(0, 1);
+    let mut cluster = new_server_cluster_with_hybrid_engine(0, 1);
     cluster.run();
 
     let t1 = ProductTable::new();
diff --git a/tests/integrations/raftstore/test_lease_read.rs b/tests/integrations/raftstore/test_lease_read.rs
index 42ebc07c6f5..013550772f9 100644
--- a/tests/integrations/raftstore/test_lease_read.rs
+++ b/tests/integrations/raftstore/test_lease_read.rs
@@ -278,9 +278,6 @@ fn test_node_lease_unsafe_during_leader_transfers() {
 }
 
 #[test_case(test_raftstore::new_node_cluster)]
-// transfer leader means eviction, so hybrid_engine with range cache memory
-// engine does not fit in.
-// #[test_case(test_raftstore::new_node_cluster_with_hybrid_engine)]
 // #[test_case(test_raftstore_v2::new_node_cluster)]
 // TODO: batch get snapshot is not supported in raftstore v2 currently.
 //       https://github.com/tikv/tikv/issues/14876

From 78e79668891a9ae5729665e365bc7cabe0c0c284 Mon Sep 17 00:00:00 2001
From: Purelind <purelind@users.noreply.github.com>
Date: Tue, 29 Oct 2024 19:48:12 +0800
Subject: [PATCH 15/16] chore: bump master version to 8.5.0-alpha (#17729)

ref tikv/tikv#16890

Signed-off-by: purelind <purelind@gmail.com>
---
 Cargo.lock | 2 +-
 Cargo.toml | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index 64e1d923080..ed4d90893a8 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -6286,7 +6286,7 @@ dependencies = [
 
 [[package]]
 name = "tikv"
-version = "8.4.0-alpha"
+version = "8.5.0-alpha"
 dependencies = [
  "anyhow",
  "api_version",
diff --git a/Cargo.toml b/Cargo.toml
index 317fe797421..7ce84c9b489 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "tikv"
-version = "8.4.0-alpha"
+version = "8.5.0-alpha"
 authors = ["The TiKV Authors"]
 description = "A distributed transactional key-value database powered by Rust and Raft"
 license = "Apache-2.0"

From 845a71bfb8c57849e18cde20d91d55fa49e9f6d9 Mon Sep 17 00:00:00 2001
From: Yang Zhang <yang.zhang@pingcap.com>
Date: Tue, 29 Oct 2024 23:48:25 -0700
Subject: [PATCH 16/16] RocksDB: Fix deprecated option (#17732)

close tikv/tikv#17731

Add deprecated RocksDB option to avoid panic upgrading from TiKV 6.1.x to 8.4+

Signed-off-by: Yang Zhang <yang.zhang@pingcap.com>
---
 Cargo.lock | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index ed4d90893a8..0fea51bf3f1 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -2883,7 +2883,7 @@ dependencies = [
 [[package]]
 name = "librocksdb_sys"
 version = "0.1.0"
-source = "git+https://github.com/tikv/rust-rocksdb.git#f728853552a1419c665c17ea6155f1c4f828c36c"
+source = "git+https://github.com/tikv/rust-rocksdb.git#2bb1e4e32b9e45cf3fd8210766a9db38eacd5e4d"
 dependencies = [
  "bindgen 0.65.1",
  "bzip2-sys",
@@ -2902,7 +2902,7 @@ dependencies = [
 [[package]]
 name = "libtitan_sys"
 version = "0.0.1"
-source = "git+https://github.com/tikv/rust-rocksdb.git#f728853552a1419c665c17ea6155f1c4f828c36c"
+source = "git+https://github.com/tikv/rust-rocksdb.git#2bb1e4e32b9e45cf3fd8210766a9db38eacd5e4d"
 dependencies = [
  "bzip2-sys",
  "cc",
@@ -4704,7 +4704,7 @@ dependencies = [
 [[package]]
 name = "rocksdb"
 version = "0.3.0"
-source = "git+https://github.com/tikv/rust-rocksdb.git#f728853552a1419c665c17ea6155f1c4f828c36c"
+source = "git+https://github.com/tikv/rust-rocksdb.git#2bb1e4e32b9e45cf3fd8210766a9db38eacd5e4d"
 dependencies = [
  "libc 0.2.151",
  "librocksdb_sys",