From 7ce5be60f7626030a2e19cab3e712348057bcb69 Mon Sep 17 00:00:00 2001
From: Croxx <mrcroxx@outlook.com>
Date: Sat, 14 Sep 2024 14:27:48 +0800
Subject: [PATCH 01/53] doc: remove details tag in change log for better
 searching (#711)

Signed-off-by: MrCroxx <mrcroxx@outlook.com>
---
 CHANGELOG.md | 243 ++++++++++++++++++---------------------------------
 1 file changed, 84 insertions(+), 159 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 17b3fa6b..af62fc78 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,5 +1,7 @@
 ## 2024-09-12
 
+### Releases
+
 | crate | version |
 | - | - |
 | foyer | 0.11.2 |
@@ -9,8 +11,6 @@
 | foyer-storage | 0.10.2 |
 | foyer-bench | 0.3.2 |
 
-<details>
-
 ### Changes
 
 - Support windows (for `foyer` only).
@@ -20,10 +20,10 @@
 - Use bytes size for `foyer-bench`.
 - Fix install deps script.
 
-</details>
-
 ## 2024-08-31
 
+### Releases
+
 | crate | version |
 | - | - |
 | foyer | 0.11.1 |
@@ -33,8 +33,6 @@
 | foyer-storage | 0.10.1 |
 | foyer-bench | 0.3.1 |
 
-<details>
-
 ### Changes
 
 - Add metrics for serde.
@@ -43,10 +41,10 @@
 - Implement `Default` for `TokioRuntimeConfig`.
 - Fix typos and format code with unstable features.
 
-</details>
-
 ## 2024-08-21
 
+### Releases
+
 | crate | version |
 | - | - |
 | foyer | 0.11.0 |
@@ -56,8 +54,6 @@
 | foyer-storage | 0.10.0 |
 | foyer-bench | 0.3.0 |
 
-<details>
-
 ### Changes
 
 - Support disk cache on raw block device.
@@ -69,42 +65,38 @@
 - Update `foyer-bench` with more fine-grained configurations.
 - Fix panics with `None` recover mode.
 
-</details>
-
 ## 2024-08-15
 
+### Releases
+
 | crate | version |
 | - | - |
 | foyer | 0.10.4 |
 | foyer-storage | 0.9.3 |
 | foyer-bench | 0.2.3 |
 
-<details>
-
 ### Changes
 
 - Support serde for recover mode configuration.
 
-</details>
-
 ## 2024-08-14
 
+### Releases
+
 | crate | version |
 | - | - |
 | foyer | 0.10.2 |
 | foyer-storage | 0.9.2 |
 | foyer-bench | 0.2.2 |
 
-<details>
-
 ### Changes
 
 - Fix panic with "none" recovery mode.
 
-</details>
-
 ## 2024-07-08
 
+### Releases
+
 | crate | version |
 | - | - |
 | foyer | 0.10.1 |
@@ -114,16 +106,14 @@
 | foyer-storage | 0.9.1 |
 | foyer-bench | 0.2.1 |
 
-<details>
-
 ### Changes
 
 - Refine write model, make flush buffer threshold configurable to mitigate memory usage spike and OOM.
 
-</details>
-
 ## 2024-07-02
 
+### Releases
+
 | crate | version |
 | - | - |
 | foyer | 0.10.0 |
@@ -133,37 +123,33 @@
 | foyer-storage | 0.9.0 |
 | foyer-bench | 0.2.0 |
 
-<details>
-
 ### Changes
 
 - Introduce tail-based tracing framework with [minitrace](https://github.com/tikv/minitrace-rust). [Tail-based Tracing Example](https://github.com/foyer-rs/foyer/tree/main/examples/tail_based_tracing.rs).
 - Fix `fetch()` disk cache refill on in-memory cache miss.
 - Publish *foyer* logo! 
 
-<img src="https://raw.githubusercontent.com/foyer-rs/foyer/main/etc/logo/slogan.min.svg"/>
-
-</details>
+<img src="https://raw.githubusercontent.com/foyer-rs/foyer/main/etc/logo/slogan.min.svg" style="width: 200px"/>
 
 ## 2024-06-14
 
+### Releases
+
 | crate | version |
 | - | - |
 | foyer | 0.9.4 |
 | foyer-storage | 0.8.5 |
 | foyer-bench | 0.1.4 |
 
-<details>
-
 ### Changes
 
 - Fix phantom entries after foyer storage recovery. [#560](https://github.com/foyer-rs/foyer/pull/560)
 - Fix hybrid cache hit metrics with `fetch()` interface. [#563](https://github.com/foyer-rs/foyer/pull/563)
 
-</details>
-
 ## 2024-06-05
 
+### Releases
+
 | crate | version |
 | - | - |
 | foyer | 0.9.3 |
@@ -173,31 +159,27 @@
 | foyer-storage | 0.8.4 |
 | foyer-bench | 0.1.3 |
 
-<details>
-
 ### Changes
 
 - Hybrid cache `fetch()` use the dedicated runtime by default if enabled.
 - Separate `fetch()` and `fetch_with_runtime()` interface for in-memory cache.
 
-</details>
-
 ## 2024-06-04
 
+### Releases
+
 | crate | version |
 | - | - |
 | foyer-storage | 0.8.3 |
 
-<details>
-
 ### Changes
 
 - Fix "invalid argument (code: 22)" on target aarch64.
 
-</details>
-
 ## 2024-06-03
 
+### Releases
+
 | crate | version |
 | - | - |
 | foyer | 0.9.2 |
@@ -207,16 +189,14 @@
 | foyer-storage | 0.8.2 |
 | foyer-bench | 0.1.2 |
 
-<details>
-
 ### Changes
 
 - Support customized cache event listener.
 
-</details>
-
 ## 2024-05-31
 
+### Releases
+
 | crate | version |
 | - | - |
 | foyer | 0.9.1 |
@@ -226,8 +206,6 @@
 | foyer-storage | 0.8.1 |
 | foyer-bench | 0.1.1 |
 
-<details>
-
 ### Changes
 
 - Fix "attempt to subtract with overflow" panic after cloning cache entry. [#543](https://github.com/foyer-rs/foyer/issues/543).
@@ -240,10 +218,10 @@
 - Remove `pop()` related interface from the in-memory cache.
 - Refine intrusive data structure implementation.
 
-</details>
-
 ## 2024-05-27
 
+### Releases
+
 | crate | version |
 | - | - |
 | foyer | 0.9.0 |
@@ -253,8 +231,6 @@
 | foyer-storage | 0.8.0 |
 | foyer-bench | 0.1.0 |
 
-<details>
-
 ### Changes
 
 - Refine the storage engine to reduce the overhead and boost the performance.
@@ -267,10 +243,10 @@
 - Reduce unnecessary dependencies.
 - More details: [foyer - Development Roadmap](https://github.com/orgs/foyer-rs/projects/2).
 
-</details>
-
 ## 2024-04-28
 
+### Releases
+
 | crate | version |
 | - | - |
 | foyer | 0.8.9 |
@@ -279,17 +255,15 @@
 | foyer-storage | 0.7.6 |
 | foyer-storage-bench | 0.7.5 |
 
-<details>
-
 ### Changes
 
 - feat: Add config to control the recover mode.
 - feat: Add config to enable/disable direct i/o. (Enabled by default for large entries optimization.)
 
-</details>
-
 ## 2024-04-28
 
+### Releases
+
 | crate | version |
 | - | - |
 | foyer | 0.8.8 |
@@ -297,60 +271,52 @@
 | foyer-storage | 0.7.5 |
 | foyer-storage-bench | 0.7.4 |
 
-<details>
-
 ### Changes
 
 - feat: Impl `Debug` for `HybridCache`.
 - feat: Impl `serde`, `Default` for eviction configs.
 - refactor: Add internal trait `EvictionConfig` to bound eviction algorithm configs.
 
-</details>
-
 ## 2024-04-27
 
+### Releases
+
 | crate | version |
 | - | - |
 | foyer | 0.8.7 |
 
-<details>
-
 ### Changes
 
 - Make `HybridCache` clonable.
 
-</details>
-
 ## 2024-04-27
 
+### Releases
+
 | crate | version |
 | - | - |
 | foyer-memory | 0.3.4 |
 
-<details>
-
 ### Changes
 
 - Fix S3FIFO ghost queue.
 
-</details>
-
 ## 2024-04-26
 
+### Releases
+
 | crate | version |
 | - | - |
 | foyer-storage | 0.7.4 |
 
-<details>
-
 ### Changes
 
 - Fix `FsDeviceBuilder` on a non-exist directory without capacity given.
 
-</details>
-
 ## 2024-04-26
 
+### Releases
+
 | crate | version |
 | - | - |
 | foyer | 0.8.6 |
@@ -360,23 +326,19 @@
 | foyer-storage | 0.7.3 |
 | foyer-storage-bench | 0.7.3 |
 
-<details>
-
 ### Changes
 
 - Remove unused dependencies.
 - Remove hakari workspace hack.
 
-</details>
-
 ## 2024-04-26
 
+### Releases
+
 | crate | version |
 | - | - |
 | foyer | 0.8.5 |
 
-<details>
-
 ### Changes
 
 - Expose `EntryState`, `HybridEntry`.
@@ -385,38 +347,34 @@
 - Re-export `ahash::RandomState`.
 - Loose `entry()` args trait bounds.
 
-</details>
-
 ## 2024-04-25
 
+### Releases
+
 | crate | version |
 | - | - |
 | foyer | 0.8.4 |
 
-<details>
-
 ### Changes
 
 - Expose `HybridCacheEntry`.
 
-</details>
-
 ## 2024-04-25
 
+### Releases
+
 | crate | version |
 | - | - |
 | foyer | 0.8.3 |
 
-<details>
-
 ### Changes
 
 - Expose `Key`, `Value`, `StorageKey`, `StorageValue` traits.
 
-</details>
-
 ## 2024-04-24
 
+### Releases
+
 | crate | version |
 | - | - |
 | foyer | 0.8.2 |
@@ -427,16 +385,14 @@
 | foyer-storage-bench | 0.7.2 |
 | foyer-workspace-hack | 0.5.2 |
 
-<details>
-
 ### Changes
 
 - Add `nightly` feature to make it compatible with night toolchain.
 
-</details>
-
 ## 2024-04-24
 
+### Releases
+
 | crate | version |
 | - | - |
 | foyer | 0.8.1 |
@@ -447,18 +403,16 @@
 | foyer-storage-bench | 0.7.1 |
 | foyer-workspace-hack | 0.5.1 |
 
-<details>
-
 ### Changes
 
 - Add `with_flush` to enable flush for each io.
 - Loose MSRV to 1.76 .
 - Flush the device on store close.
 
-</details>
-
 ## 2024-04-23
 
+### Releases
+
 | crate | version |
 | - | - |
 | foyer | 0.8.0 |
@@ -469,8 +423,6 @@
 | foyer-storage-bench | 0.7.0 |
 | foyer-workspace-hack | 0.5.0 |
 
-<details>
-
 ### Changes
 
 - Combine in-memory cache and disk cache into `HybridCache`.
@@ -482,10 +434,10 @@
 - Fix S3FIFO eviction bugs.
 - Add more examples.
 
-</details>
-
 ## 2024-04-11
 
+### Releases
+
 | crate | version |
 | - | - |
 | foyer | 0.7.0 |
@@ -496,90 +448,78 @@
 | foyer-storage-bench | 0.6.0 |
 | foyer-workspace-hack | 0.4.0 |
 
-<details>
-
 ### Changes
 
 - Make `foyer` compatible with rust stable toolchain (MSRV = 1.77.2). 🎉
 
-</details>
-
 ## 2024-04-09
 
+### Releases
+
 | crate | version |
 | - | - |
 | foyer-storage | 0.5.1 |
 | foyer-memory | 0.1.4 |
 
-<details>
-
 ### Changes
 
 - fix: Fix panics on `state()` for s3fifo entry.
 - fix: Enable `offset_of` feature for `foyer-storage`.
 
-</details>
-
 ## 2024-04-08
 
+### Releases
+
 | crate | version |
 | - | - |
 | foyer-intrusive | 0.3.1 |
 | foyer-memory | 0.1.3 |
 
-<details>
-
 ### Changes
 
 - feat: Introduce s3fifo to `foyer-memory`.
 - fix: Fix doctest for `foyer-intrusive`.
 
-</details>
-
 ## 2024-03-21
 
+### Releases
+
 | crate | version |
 | - | - |
 | foyer-memory | 0.1.2 |
 
-<details>
-
 ### Changes
 
 - fix: `foyer-memory` export `DefaultCacheEventListener`.
 
-</details>
-
 ## 2024-03-14
 
+### Releases
+
 | crate | version |
 | - | - |
 | foyer-memory | 0.1.1 |
 
-<details>
-
 ### Changes
 
 - Make eviction config clonable.
 
-</details>
-
 ## 2024-03-13
 
+### Releases
+
 | crate | version |
 | - | - |
 | foyer-storage-bench | 0.5.1 |
 
-<details>
-
 ### Changes
 
 - Fix `foyer-storage-bench` build with `trace` feature.
 
-</details>
-
 ## 2024-03-12
 
+### Releases
+
 | crate | version |
 | - | - |
 | foyer | 0.6.0 |
@@ -590,17 +530,15 @@
 | foyer-storage-bench | 0.5.0 |
 | foyer-workspace-hack | 0.3.0 |
 
-<details>
-
 ### Changes
 
 - Release foyer in-memory cache as crate `foyer-memory`.
 - Bump other components with changes.
 
-</details>
-
 ## 2023-12-28
 
+### Releases
+
 | crate | version |
 | - | - |
 | foyer | 0.5.0 |
@@ -610,21 +548,16 @@
 | foyer-storage-bench | 0.4.0 |
 | foyer-workspace-hack | 0.2.0 |
 
-<details>
-
 ### Changes
 
 - Bump rust-toolchain to "nightly-2023-12-26".
 - Introduce time-series distribution args to bench tool. [#253](https://github.com/foyer-rs/foyer/pull/253)
-
-### Fixes
-
 - Fix duplicated insert drop metrics.
 
-</details>
-
 ## 2023-12-22
 
+### Releases
+
 | crate | version |
 | - | - |
 | foyer | 0.4.0 |
@@ -632,45 +565,40 @@
 | foyer-storage-bench | 0.3.0 |
 | foyer-workspace-hack | 0.1.1 |
 
-<details>
-
 ### Changes
 
 - Remove config `flusher_buffer_capacity`.
-
-### Fixes
-
 - Fix benchmark tool cache miss ratio.
 
-</details>
-
 ## 2023-12-20
 
+### Releases
+
 | crate | version |
 | - | - |
 | foyer-storage | 0.2.2 |
 
-<details>
+### Changes
 
 - Fix metrics for writer dropping.
 - Add interface `insert_async_with_callback` and `insert_if_not_exists_async_with_callback` for callers to get the insert result.
 
-</details>
-
 ## 2023-12-18
 
+### Releases
+
 | crate | version |
 | - | - |
 | foyer-storage | 0.2.1 |
 
-<details>
+### Changes
 
 - Introduce the entry size histogram, update metrics.
 
-</details>
-
 ## 2023-12-18
 
+### Releases
+
 | crate | version |
 | - | - |
 | foyer | 0.3.0 |
@@ -678,16 +606,16 @@
 | foyer-storage | 0.2.0 |
 | foyer-storage-bench | 0.2.0 |
 
-<details>
+### Changes
 
 - Introduce the associated type `Cursor` for trait `Key` and `Value` to reduce unnecessary buffer copy if possible.
 - Remove the ring buffer and continuum tracker for they are no longer needed.
 - Update the configuration of the storage engine and the benchmark tool.
 
-</details>
-
 ## 2023-11-29
 
+### Releases
+
 | crate | version |
 | - | - |
 | foyer | 0.2.0 |
@@ -697,7 +625,7 @@
 | foyer-storage-bench | 0.1.0 |
 | foyer-workspace-hack | 0.1.0 |
 
-<details>
+### Changes
 
 The first version that can be used as file cache.
 
@@ -715,17 +643,14 @@ Brief description about the subcrates:
 - foyer-storage-bench: Runnable benchmark tool for the file cache storage engine.
 - foyer-workspace-hack: Generated by [hakari](https://crates.io/crates/hakari) to prevent building each crate from triggering building from scratch.
 
-</details>
-
-
 ## 2023-05-12
 
+### Releases
+
 | crate | version |
 | - | - |
 | foyer | 0.1.0 |
 
-<details>
+### Changes
 
 Initial version with just basic interfaces.
-
-</details>

From 4878ef25eb4d6cfe42802e26d82096ba1174658c Mon Sep 17 00:00:00 2001
From: Croxx <mrcroxx@outlook.com>
Date: Sat, 14 Sep 2024 22:57:06 +0800
Subject: [PATCH 02/53] chore: add some metadata for changelog (#712)

* chore: add some metadata for changelog

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

* chore: add more meta

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

* chore: just more

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

---------

Signed-off-by: MrCroxx <mrcroxx@outlook.com>
---
 CHANGELOG.md | 21 +++++++++++++++++++++
 1 file changed, 21 insertions(+)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index af62fc78..327c191f 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,3 +1,24 @@
+---
+title: Changelog
+description: Changelog for foyer.
+authors:
+  - name: Croxx
+    title: Author of foyer
+    url: https://github.com/mrcroxx
+    image_url: https://github.com/mrcroxx.png
+    socials:
+      x: CroxxMr
+      github: mrcroxx
+      linkedin: mrcroxx
+      newsletter: https://blog.mrcroxx.com
+tags: [changelog]
+date: 2023-05-12T11:02:09+08:00
+---
+
+# Changelog
+
+<!-- truncate -->
+
 ## 2024-09-12
 
 ### Releases

From 714b3c3d64ff1104b94384a0bfcaf986b3c60978 Mon Sep 17 00:00:00 2001
From: Croxx <mrcroxx@outlook.com>
Date: Sat, 14 Sep 2024 23:18:11 +0800
Subject: [PATCH 03/53] chore: ignore changelog author details (#713)

Signed-off-by: MrCroxx <mrcroxx@outlook.com>
---
 CHANGELOG.md | 11 +----------
 1 file changed, 1 insertion(+), 10 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 327c191f..47a2dd24 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,16 +1,7 @@
 ---
 title: Changelog
 description: Changelog for foyer.
-authors:
-  - name: Croxx
-    title: Author of foyer
-    url: https://github.com/mrcroxx
-    image_url: https://github.com/mrcroxx.png
-    socials:
-      x: CroxxMr
-      github: mrcroxx
-      linkedin: mrcroxx
-      newsletter: https://blog.mrcroxx.com
+authors: mrcroxx
 tags: [changelog]
 date: 2023-05-12T11:02:09+08:00
 ---

From c28c6c1a81c793ce3d20c4ba73943a01df47f0c9 Mon Sep 17 00:00:00 2001
From: Croxx <mrcroxx@outlook.com>
Date: Thu, 19 Sep 2024 14:24:15 +0800
Subject: [PATCH 04/53] chore: update readme, add website (#719)

Signed-off-by: MrCroxx <mrcroxx@outlook.com>
---
 README.md        | 24 +++++++++++++++++++-----
 foyer/src/lib.rs |  4 ++++
 2 files changed, 23 insertions(+), 5 deletions(-)

diff --git a/README.md b/README.md
index 6ea75be1..a878d205 100644
--- a/README.md
+++ b/README.md
@@ -2,12 +2,18 @@
     <img src="https://raw.githubusercontent.com/foyer-rs/foyer/main/etc/logo/slogan.min.svg" />
 </p>
 
+<div style="text-align: center;">
+
+![Website](https://img.shields.io/website?url=https%3A%2F%2Ffoyer.rs&up_message=foyer.rs&style=for-the-badge&logo=rust&labelColor=555555)
+![Crates.io Version](https://img.shields.io/crates/v/foyer?style=for-the-badge&logo=docs.rs&labelColor=555555)
+![docs.rs](https://img.shields.io/docsrs/foyer?style=for-the-badge&logo=docs.rs&labelColor=555555)
+
+</div>
+
 # foyer
 
-![Crates.io Version](https://img.shields.io/crates/v/foyer)
-![Crates.io MSRV](https://img.shields.io/crates/msrv/foyer)
 ![GitHub License](https://img.shields.io/github/license/foyer-rs/foyer)
-
+![Crates.io MSRV](https://img.shields.io/crates/msrv/foyer)
 [![CI](https://github.com/foyer-rs/foyer/actions/workflows/ci.yml/badge.svg)](https://github.com/foyer-rs/foyer/actions/workflows/ci.yml)
 [![License Checker](https://github.com/foyer-rs/foyer/actions/workflows/license_check.yml/badge.svg)](https://github.com/foyer-rs/foyer/actions/workflows/license_check.yml)
 [![codecov](https://codecov.io/github/foyer-rs/foyer/branch/main/graph/badge.svg?token=YO33YQCB70)](https://codecov.io/github/foyer-rs/foyer)
@@ -18,9 +24,11 @@ foyer draws inspiration from [Facebook/CacheLib](https://github.com/facebook/cac
 
 However, *foyer* is more than just a *rewrite in Rust* effort; it introduces a variety of new and optimized features.
 
+For more details, please visit foyer's website: https://foyer.rs 🥰
+
 ## Features
 
-- **Hybrid Cache**: Seamlessly integrates both in-memory and disk-based caching for optimal performance and flexibility.
+- **Hybrid Cache**: Seamlessly integrates both in-memory and disk cache for optimal performance and flexibility.
 - **Plug-and-Play Algorithms**: Empowers users with easily replaceable caching algorithms, ensuring adaptability to diverse use cases.
 - **Fearless Concurrency**: Built to handle high concurrency with robust thread-safe mechanisms, guaranteeing reliable performance under heavy loads.
 - **Zero-Copy In-Memory Cache Abstraction**: Leveraging Rust's robust type system, the in-memory cache in foyer achieves a better performance with zero-copy abstraction.
@@ -34,7 +42,13 @@ Feel free to open a PR and add your projects here:
 - [RisingWave](https://github.com/risingwavelabs/risingwave): SQL stream processing, analytics, and management.
 - [Chroma](https://github.com/chroma-core/chroma): Embedding database for LLM apps.
 
-## Usage
+## Document
+
+Tutorial & Document: https://foyer.rs
+
+API References: https://docs.rs/foyer
+
+## Quick Start
 
 To use *foyer* in your project, add this line to the `dependencies` section of `Cargo.toml`.
 
diff --git a/foyer/src/lib.rs b/foyer/src/lib.rs
index dc0710cd..27a1f61f 100644
--- a/foyer/src/lib.rs
+++ b/foyer/src/lib.rs
@@ -17,6 +17,10 @@
 #![warn(clippy::allow_attributes)]
 
 //! A hybrid cache library that supports plug-and-play cache algorithms, in-memory cache and disk cache.
+//!
+//! ![Crates.io Version](https://img.shields.io/crates/v/foyer?style=for-the-badge&logo=docs.rs&labelColor=555555)
+//! ![docs.rs](https://img.shields.io/docsrs/foyer?style=for-the-badge&logo=docs.rs&labelColor=555555)
+//! ![Website](https://img.shields.io/website?url=https%3A%2F%2Ffoyer.rs&up_message=foyer.rs&style=for-the-badge&logo=rust&labelColor=555555)
 
 use foyer_common as common;
 use foyer_memory as memory;

From 474fd2fe7da36e77b859bd59cc5ea60ef1598cef Mon Sep 17 00:00:00 2001
From: Croxx <mrcroxx@outlook.com>
Date: Thu, 19 Sep 2024 17:33:51 +0800
Subject: [PATCH 05/53] chore: update badge with link (#721)

* chore: update badge with link

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

* chore: center the badges

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

* chore: add website center

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

* chore: fix link

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

* chore: add more guides

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

* chore: add links in rust doc

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

* chore: update

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

---------

Signed-off-by: MrCroxx <mrcroxx@outlook.com>
---
 README.md        | 34 ++++++++++++++++++++++------------
 foyer/src/lib.rs | 11 ++++++++---
 2 files changed, 30 insertions(+), 15 deletions(-)

diff --git a/README.md b/README.md
index a878d205..eb639ae5 100644
--- a/README.md
+++ b/README.md
@@ -2,13 +2,22 @@
     <img src="https://raw.githubusercontent.com/foyer-rs/foyer/main/etc/logo/slogan.min.svg" />
 </p>
 
-<div style="text-align: center;">
-
-![Website](https://img.shields.io/website?url=https%3A%2F%2Ffoyer.rs&up_message=foyer.rs&style=for-the-badge&logo=rust&labelColor=555555)
-![Crates.io Version](https://img.shields.io/crates/v/foyer?style=for-the-badge&logo=docs.rs&labelColor=555555)
-![docs.rs](https://img.shields.io/docsrs/foyer?style=for-the-badge&logo=docs.rs&labelColor=555555)
+<p align="center">
+    <a href="https://foyer.rs">
+        <img src="https://img.shields.io/website?url=https%3A%2F%2Ffoyer.rs&up_message=foyer.rs&down_message=website&style=for-the-badge&logo=htmx" alt="docs.rs" />
+    </a>
+    <a href="https://crates.io/crates/foyer">
+        <img src="https://img.shields.io/crates/v/foyer?style=for-the-badge&logo=crates.io&labelColor=555555" alt="crates.io" />
+    </a>
+    <a href="https://docs.rs/foyer">
+        <img src="https://img.shields.io/docsrs/foyer?style=for-the-badge&logo=rust&label=docs.rs&labelColor=555555" alt="docs.rs" />
+    </a>
+</p>
 
-</div>
+<p align="center">
+    <b>Tutorial & Document:</b>
+    <a href="https://foyer.rs"><b>https://foyer.rs</b></a>
+</p>
 
 # foyer
 
@@ -26,6 +35,11 @@ However, *foyer* is more than just a *rewrite in Rust* effort; it introduces a v
 
 For more details, please visit foyer's website: https://foyer.rs 🥰
 
+[Website](https://foyer.rs) |
+[Tutorial](https://foyer.rs/docs/overview) |
+[API Docs](https://docs.rs/foyer) |
+[Crate](https://crates.io/crates/foyer)
+
 ## Features
 
 - **Hybrid Cache**: Seamlessly integrates both in-memory and disk cache for optimal performance and flexibility.
@@ -42,14 +56,10 @@ Feel free to open a PR and add your projects here:
 - [RisingWave](https://github.com/risingwavelabs/risingwave): SQL stream processing, analytics, and management.
 - [Chroma](https://github.com/chroma-core/chroma): Embedding database for LLM apps.
 
-## Document
-
-Tutorial & Document: https://foyer.rs
-
-API References: https://docs.rs/foyer
-
 ## Quick Start
 
+**This section only shows briefs. Please visit https://foyer.rs for more details.**
+
 To use *foyer* in your project, add this line to the `dependencies` section of `Cargo.toml`.
 
 ```toml
diff --git a/foyer/src/lib.rs b/foyer/src/lib.rs
index 27a1f61f..2a636cd5 100644
--- a/foyer/src/lib.rs
+++ b/foyer/src/lib.rs
@@ -18,9 +18,14 @@
 
 //! A hybrid cache library that supports plug-and-play cache algorithms, in-memory cache and disk cache.
 //!
-//! ![Crates.io Version](https://img.shields.io/crates/v/foyer?style=for-the-badge&logo=docs.rs&labelColor=555555)
-//! ![docs.rs](https://img.shields.io/docsrs/foyer?style=for-the-badge&logo=docs.rs&labelColor=555555)
-//! ![Website](https://img.shields.io/website?url=https%3A%2F%2Ffoyer.rs&up_message=foyer.rs&style=for-the-badge&logo=rust&labelColor=555555)
+//! ![Website](https://img.shields.io/website?url=https%3A%2F%2Ffoyer.rs&up_message=foyer.rs&down_message=website&style=for-the-badge&logo=htmx&link=https%3A%2F%2Ffoyer.rs)
+//! ![Crates.io Version](https://img.shields.io/crates/v/foyer?style=for-the-badge&logo=crates.io&labelColor=555555&link=https%3A%2F%2Fcrates.io%2Fcrates%2Ffoyer)
+//! ![docs.rs](https://img.shields.io/docsrs/foyer?style=for-the-badge&logo=rust&label=docs.rs&labelColor=555555&link=https%3A%2F%2Fdocs.rs%2Ffoyer)
+//!
+//! [Website](https://foyer.rs) |
+//! [Tutorial](https://foyer.rs/docs/overview) |
+//! [API Docs](https://docs.rs/foyer) |
+//! [Crate](https://crates.io/crates/foyer)
 
 use foyer_common as common;
 use foyer_memory as memory;

From 7ebc971d2c1542382e7ea2c2008c5083b3fede41 Mon Sep 17 00:00:00 2001
From: Croxx <mrcroxx@outlook.com>
Date: Fri, 20 Sep 2024 12:26:09 +0800
Subject: [PATCH 06/53] fix: fix io buffer pool alignment (#724)

Signed-off-by: MrCroxx <mrcroxx@outlook.com>
---
 foyer-storage/src/io_buffer_pool.rs | 5 ++++-
 foyer-storage/src/large/batch.rs    | 1 +
 2 files changed, 5 insertions(+), 1 deletion(-)

diff --git a/foyer-storage/src/io_buffer_pool.rs b/foyer-storage/src/io_buffer_pool.rs
index 4a136fdf..83e42503 100644
--- a/foyer-storage/src/io_buffer_pool.rs
+++ b/foyer-storage/src/io_buffer_pool.rs
@@ -14,7 +14,9 @@
 
 use std::collections::VecDeque;
 
-use crate::{IoBuffer, IoBytes};
+use foyer_common::bits;
+
+use crate::{device::ALIGN, IoBuffer, IoBytes};
 
 pub enum Buffer {
     IoBuffer(IoBuffer),
@@ -41,6 +43,7 @@ pub struct IoBufferPool {
 
 impl IoBufferPool {
     pub fn new(buffer_size: usize, capacity: usize) -> Self {
+        bits::assert_aligned(ALIGN, buffer_size);
         Self {
             capacity,
             buffer_size,
diff --git a/foyer-storage/src/large/batch.rs b/foyer-storage/src/large/batch.rs
index ed7e771d..a4e46f87 100644
--- a/foyer-storage/src/large/batch.rs
+++ b/foyer-storage/src/large/batch.rs
@@ -117,6 +117,7 @@ where
     S: HashBuilder + Debug,
 {
     pub fn new(capacity: usize, region_manager: RegionManager, device: MonitoredDevice, indexer: Indexer) -> Self {
+        let capacity = bits::align_up(device.align(), capacity);
         let mut batch = Self {
             buffer: IoBuffer::new(capacity),
             len: 0,

From a52da3a523dcaccbad30d7e513aa69d2ab3fc7d8 Mon Sep 17 00:00:00 2001
From: Croxx <mrcroxx@outlook.com>
Date: Fri, 20 Sep 2024 13:07:40 +0800
Subject: [PATCH 07/53] chore: relase foyer 0.11.3 (#725)

Signed-off-by: MrCroxx <mrcroxx@outlook.com>
---
 CHANGELOG.md               | 18 +++++++++++++++++-
 foyer-bench/Cargo.toml     |  4 ++--
 foyer-common/Cargo.toml    |  2 +-
 foyer-intrusive/Cargo.toml |  4 ++--
 foyer-memory/Cargo.toml    |  6 +++---
 foyer-storage/Cargo.toml   |  6 +++---
 foyer/Cargo.toml           |  8 ++++----
 7 files changed, 32 insertions(+), 16 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 47a2dd24..a989f5f9 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -2,7 +2,6 @@
 title: Changelog
 description: Changelog for foyer.
 authors: mrcroxx
-tags: [changelog]
 date: 2023-05-12T11:02:09+08:00
 ---
 
@@ -10,6 +9,23 @@ date: 2023-05-12T11:02:09+08:00
 
 <!-- truncate -->
 
+## 2024-09-20
+
+### Releases
+
+| crate | version |
+| - | - |
+| foyer | 0.11.3 |
+| foyer-common | 0.9.3 |
+| foyer-intrusive | 0.9.3 |
+| foyer-memory | 0.7.3 |
+| foyer-storage | 0.10.3 |
+| foyer-bench | 0.3.3 |
+
+### Changes
+
+- Fix panicked by io buffer pool alignment issue.
+
 ## 2024-09-12
 
 ### Releases
diff --git a/foyer-bench/Cargo.toml b/foyer-bench/Cargo.toml
index c6bd15bc..9ca1c6bb 100644
--- a/foyer-bench/Cargo.toml
+++ b/foyer-bench/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "foyer-bench"
-version = "0.3.2"
+version = "0.3.3"
 edition = "2021"
 authors = ["MrCroxx <mrcroxx@outlook.com>"]
 description = "bench tool for foyer - the hybrid cache for Rust"
@@ -17,7 +17,7 @@ clap = { workspace = true }
 console-subscriber = { version = "0.4", optional = true }
 fastrace = { workspace = true, optional = true }
 fastrace-jaeger = { workspace = true, optional = true }
-foyer = { version = "0.11.2", path = "../foyer" }
+foyer = { version = "0.11.3", path = "../foyer" }
 futures = "0.3"
 hdrhistogram = "7"
 itertools = { workspace = true }
diff --git a/foyer-common/Cargo.toml b/foyer-common/Cargo.toml
index bd9d5f37..16ef110d 100644
--- a/foyer-common/Cargo.toml
+++ b/foyer-common/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "foyer-common"
-version = "0.9.2"
+version = "0.9.3"
 edition = "2021"
 authors = ["MrCroxx <mrcroxx@outlook.com>"]
 description = "common components for foyer - the hybrid cache for Rust"
diff --git a/foyer-intrusive/Cargo.toml b/foyer-intrusive/Cargo.toml
index 598e35c5..abdf950d 100644
--- a/foyer-intrusive/Cargo.toml
+++ b/foyer-intrusive/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "foyer-intrusive"
-version = "0.9.2"
+version = "0.9.3"
 edition = "2021"
 authors = ["MrCroxx <mrcroxx@outlook.com>"]
 description = "intrusive data structures for foyer - the hybrid cache for Rust"
@@ -11,7 +11,7 @@ readme = "../README.md"
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
-foyer-common = { version = "0.9.2", path = "../foyer-common" }
+foyer-common = { version = "0.9.3", path = "../foyer-common" }
 itertools = { workspace = true }
 
 [features]
diff --git a/foyer-memory/Cargo.toml b/foyer-memory/Cargo.toml
index ba2ee1c0..f971ad0e 100644
--- a/foyer-memory/Cargo.toml
+++ b/foyer-memory/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "foyer-memory"
-version = "0.7.2"
+version = "0.7.3"
 edition = "2021"
 authors = ["MrCroxx <mrcroxx@outlook.com>"]
 description = "memory cache for foyer - the hybrid cache for Rust"
@@ -15,8 +15,8 @@ ahash = "0.8"
 bitflags = "2"
 cmsketch = "0.2.1"
 fastrace = { workspace = true }
-foyer-common = { version = "0.9.2", path = "../foyer-common" }
-foyer-intrusive = { version = "0.9.2", path = "../foyer-intrusive" }
+foyer-common = { version = "0.9.3", path = "../foyer-common" }
+foyer-intrusive = { version = "0.9.3", path = "../foyer-intrusive" }
 futures = "0.3"
 hashbrown = "0.14"
 itertools = { workspace = true }
diff --git a/foyer-storage/Cargo.toml b/foyer-storage/Cargo.toml
index 1a2c365c..3812d80b 100644
--- a/foyer-storage/Cargo.toml
+++ b/foyer-storage/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "foyer-storage"
-version = "0.10.2"
+version = "0.10.3"
 edition = "2021"
 authors = ["MrCroxx <mrcroxx@outlook.com>"]
 description = "storage engine for foyer - the hybrid cache for Rust"
@@ -24,8 +24,8 @@ bytes = "1"
 clap = { workspace = true }
 either = "1"
 fastrace = { workspace = true }
-foyer-common = { version = "0.9.2", path = "../foyer-common" }
-foyer-memory = { version = "0.7.2", path = "../foyer-memory" }
+foyer-common = { version = "0.9.3", path = "../foyer-common" }
+foyer-memory = { version = "0.7.3", path = "../foyer-memory" }
 fs4 = "0.9.1"
 futures = "0.3"
 itertools = { workspace = true }
diff --git a/foyer/Cargo.toml b/foyer/Cargo.toml
index 6c925596..25a4e837 100644
--- a/foyer/Cargo.toml
+++ b/foyer/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "foyer"
-version = "0.11.2"
+version = "0.11.3"
 edition = "2021"
 authors = ["MrCroxx <mrcroxx@outlook.com>"]
 description = "Hybrid cache for Rust"
@@ -15,9 +15,9 @@ rust-version = "1.81.0"
 ahash = "0.8"
 anyhow = "1"
 fastrace = { workspace = true }
-foyer-common = { version = "0.9.2", path = "../foyer-common" }
-foyer-memory = { version = "0.7.2", path = "../foyer-memory" }
-foyer-storage = { version = "0.10.2", path = "../foyer-storage" }
+foyer-common = { version = "0.9.3", path = "../foyer-common" }
+foyer-memory = { version = "0.7.3", path = "../foyer-memory" }
+foyer-storage = { version = "0.10.3", path = "../foyer-storage" }
 futures = "0.3"
 pin-project = "1"
 tokio = { workspace = true }

From fd68f1b33903f30f3f93d5e20708ebea70d24272 Mon Sep 17 00:00:00 2001
From: Croxx <mrcroxx@outlook.com>
Date: Fri, 20 Sep 2024 13:48:29 +0800
Subject: [PATCH 08/53] doc: fix a tiny broken link (#728)

Signed-off-by: MrCroxx <mrcroxx@outlook.com>
---
 foyer-storage/src/device/direct_file.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/foyer-storage/src/device/direct_file.rs b/foyer-storage/src/device/direct_file.rs
index b56cbd59..6b66c020 100644
--- a/foyer-storage/src/device/direct_file.rs
+++ b/foyer-storage/src/device/direct_file.rs
@@ -251,7 +251,7 @@ impl Dev for DirectFileDevice {
     }
 }
 
-/// [`DirectFiDeviceOptionsBuilder`] is used to build the options for the direct fs device.
+/// [`DirectFileDeviceOptionsBuilder`] is used to build the options for the direct fs device.
 ///
 /// The direct fs device uses a directory in a file system to store the data of disk cache.
 ///

From f1490da1078c54359ee7896a607a245cf08bb9ff Mon Sep 17 00:00:00 2001
From: Croxx <mrcroxx@outlook.com>
Date: Fri, 20 Sep 2024 15:18:11 +0800
Subject: [PATCH 09/53] refactor: rename with_buffer_threshold to
 with_buffer_pool_size (#729)

Signed-off-by: MrCroxx <mrcroxx@outlook.com>
---
 examples/hybrid_full.rs     |  2 +-
 foyer-storage/src/store.rs  | 28 ++++++++++++++++++++++------
 foyer/src/hybrid/builder.rs | 24 +++++++++++++++++++++++-
 3 files changed, 46 insertions(+), 8 deletions(-)

diff --git a/examples/hybrid_full.rs b/examples/hybrid_full.rs
index 14ca0ff1..97f98500 100644
--- a/examples/hybrid_full.rs
+++ b/examples/hybrid_full.rs
@@ -48,7 +48,7 @@ async fn main() -> Result<()> {
         .with_recover_concurrency(8)
         .with_flushers(2)
         .with_reclaimers(2)
-        .with_buffer_threshold(256 * 1024 * 1024)
+        .with_buffer_pool_size(256 * 1024 * 1024)
         .with_clean_region_threshold(4)
         .with_eviction_pickers(vec![Box::<FifoPicker>::default()])
         .with_admission_picker(Arc::new(RateLimitPicker::new(100 * 1024 * 1024)))
diff --git a/foyer-storage/src/store.rs b/foyer-storage/src/store.rs
index 5ed47948..663f9870 100644
--- a/foyer-storage/src/store.rs
+++ b/foyer-storage/src/store.rs
@@ -357,8 +357,7 @@ where
     recover_concurrency: usize,
     flushers: usize,
     reclaimers: usize,
-    // FIXME(MrCroxx): rename the field and builder fn.
-    buffer_threshold: usize,
+    buffer_pool_size: usize,
     clean_region_threshold: Option<usize>,
     eviction_pickers: Vec<Box<dyn EvictionPicker>>,
     admission_picker: Arc<dyn AdmissionPicker<Key = K>>,
@@ -387,7 +386,7 @@ where
             recover_concurrency: 8,
             flushers: 1,
             reclaimers: 1,
-            buffer_threshold: 16 * 1024 * 1024, // 16 MiB
+            buffer_pool_size: 16 * 1024 * 1024, // 16 MiB
             clean_region_threshold: None,
             eviction_pickers: vec![Box::new(InvalidRatioPicker::new(0.8)), Box::<FifoPicker>::default()],
             admission_picker: Arc::<AdmitAllPicker<K>>::default(),
@@ -469,6 +468,7 @@ where
         self
     }
 
+    // FIXME(MrCroxx): remove it after 0.12
     /// Set the total flush buffer threshold.
     ///
     /// Each flusher shares a volume at `threshold / flushers`.
@@ -476,8 +476,24 @@ where
     /// If the buffer of the flush queue exceeds the threshold, the further entries will be ignored.
     ///
     /// Default: 16 MiB.
+    #[deprecated(
+        since = "0.11.4",
+        note = "The function will be renamed to \"with_buffer_pool_size()\", use it instead."
+    )]
     pub fn with_buffer_threshold(mut self, threshold: usize) -> Self {
-        self.buffer_threshold = threshold;
+        self.buffer_pool_size = threshold;
+        self
+    }
+
+    /// Set the total flush buffer pool size.
+    ///
+    /// Each flusher shares a volume at `threshold / flushers`.
+    ///
+    /// If the buffer of the flush queue exceeds the threshold, the further entries will be ignored.
+    ///
+    /// Default: 16 MiB.
+    pub fn with_buffer_pool_size(mut self, buffer_pool_size: usize) -> Self {
+        self.buffer_pool_size = buffer_pool_size;
         self
     }
 
@@ -659,7 +675,7 @@ where
                                     eviction_pickers: self.eviction_pickers,
                                     reinsertion_picker: self.reinsertion_picker,
                                     tombstone_log_config: self.tombstone_log_config,
-                                    buffer_threshold: self.buffer_threshold,
+                                    buffer_threshold: self.buffer_pool_size,
                                     statistics: statistics.clone(),
                                     write_runtime_handle,
                                     read_runtime_handle,
@@ -702,7 +718,7 @@ where
                                         eviction_pickers: self.eviction_pickers,
                                         reinsertion_picker: self.reinsertion_picker,
                                         tombstone_log_config: self.tombstone_log_config,
-                                        buffer_threshold: self.buffer_threshold,
+                                        buffer_threshold: self.buffer_pool_size,
                                         statistics: statistics.clone(),
                                         write_runtime_handle,
                                         read_runtime_handle,
diff --git a/foyer/src/hybrid/builder.rs b/foyer/src/hybrid/builder.rs
index 3ee2a450..34eb69be 100644
--- a/foyer/src/hybrid/builder.rs
+++ b/foyer/src/hybrid/builder.rs
@@ -299,6 +299,7 @@ where
         }
     }
 
+    // FIXME(MrCroxx): remove it after 0.12
     /// Set the total flush buffer threshold.
     ///
     /// Each flusher shares a volume at `threshold / flushers`.
@@ -306,8 +307,29 @@ where
     /// If the buffer of the flush queue exceeds the threshold, the further entries will be ignored.
     ///
     /// Default: 16 MiB.
+    #[deprecated(
+        since = "0.11.4",
+        note = "The function will be renamed to \"with_buffer_pool_size()\", use it instead."
+    )]
     pub fn with_buffer_threshold(self, threshold: usize) -> Self {
-        let builder = self.builder.with_buffer_threshold(threshold);
+        let builder = self.builder.with_buffer_pool_size(threshold);
+        Self {
+            name: self.name,
+            tracing_config: self.tracing_config,
+            memory: self.memory,
+            builder,
+        }
+    }
+
+    /// Set the total flush buffer pool size.
+    ///
+    /// Each flusher shares a volume at `threshold / flushers`.
+    ///
+    /// If the buffer of the flush queue exceeds the threshold, the further entries will be ignored.
+    ///
+    /// Default: 16 MiB.
+    pub fn with_buffer_pool_size(self, buffer_pool_size: usize) -> Self {
+        let builder = self.builder.with_buffer_pool_size(buffer_pool_size);
         Self {
             name: self.name,
             tracing_config: self.tracing_config,

From 553a9cbdd8a392d1aa071eeb9c00b49555f60162 Mon Sep 17 00:00:00 2001
From: Croxx <mrcroxx@outlook.com>
Date: Fri, 20 Sep 2024 15:59:28 +0800
Subject: [PATCH 10/53] chore: warn if DirectFileDevice is used in a fs (#730)

Signed-off-by: MrCroxx <mrcroxx@outlook.com>
---
 foyer-storage/src/device/direct_file.rs | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/foyer-storage/src/device/direct_file.rs b/foyer-storage/src/device/direct_file.rs
index 6b66c020..6b8170f3 100644
--- a/foyer-storage/src/device/direct_file.rs
+++ b/foyer-storage/src/device/direct_file.rs
@@ -199,6 +199,12 @@ impl Dev for DirectFileDevice {
         let file = opts.open(&options.path)?;
 
         if file.metadata().unwrap().is_file() {
+            tracing::warn!(
+                "{}\n{}\n{}",
+                "It seems a `DirectFileDevice` is used within a normal file system, which is inefficient.",
+                "Please use `DirectFileDevice` directly on a raw block device.",
+                "Or use `DirectFsDevice` within a normal file system.",
+            );
             file.set_len(options.capacity as _)?;
         }
 

From 60839ee7e399f3ee32b57fe548a03881ab540531 Mon Sep 17 00:00:00 2001
From: Croxx <mrcroxx@outlook.com>
Date: Tue, 24 Sep 2024 14:21:15 +0800
Subject: [PATCH 11/53] refactor: revert the pre-serialization and parallel
 buffer (#717)

* refactor: use entry uncompressed serialized size for selection

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

* refactor: refactor entry serde

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

* refactor: revert the parallelism batching design

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

* refactor: remove spawn on write

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

* chore: pass ffmt test

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

* fix: fix build on nightly with nightly feature

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

* chore: fix

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

---------

Signed-off-by: MrCroxx <mrcroxx@outlook.com>
---
 examples/Cargo.toml                 |   8 +-
 foyer-bench/Cargo.toml              |   2 +-
 foyer-storage/Cargo.toml            |   1 +
 foyer-storage/src/engine.rs         |  21 +--
 foyer-storage/src/large/batch.rs    | 154 +++++++++++-----------
 foyer-storage/src/large/flusher.rs  | 190 ++++++++++++++--------------
 foyer-storage/src/large/generic.rs  |  29 ++---
 foyer-storage/src/lib.rs            |   1 +
 foyer-storage/src/serde.rs          | 105 +++++++++------
 foyer-storage/src/small/generic.rs  |   4 +-
 foyer-storage/src/storage/either.rs |  13 +-
 foyer-storage/src/storage/mod.rs    |   4 +-
 foyer-storage/src/storage/noop.rs   |  13 +-
 foyer-storage/src/store.rs          |  29 +----
 14 files changed, 286 insertions(+), 288 deletions(-)

diff --git a/examples/Cargo.toml b/examples/Cargo.toml
index 74b49b7f..2584ca5f 100644
--- a/examples/Cargo.toml
+++ b/examples/Cargo.toml
@@ -19,10 +19,10 @@ fastrace = { workspace = true }
 fastrace-jaeger = { workspace = true, optional = true }
 fastrace-opentelemetry = { workspace = true, optional = true }
 foyer = { version = "*", path = "../foyer" }
-opentelemetry = { version = "0.24", optional = true }
-opentelemetry-otlp = { version = "0.17", optional = true }
-opentelemetry-semantic-conventions = { version = "0.16", optional = true }
-opentelemetry_sdk = { version = "0.24", features = [
+opentelemetry = { version = "0.25", optional = true }
+opentelemetry-otlp = { version = "0.25", optional = true }
+opentelemetry-semantic-conventions = { version = "0.25", optional = true }
+opentelemetry_sdk = { version = "0.25", features = [
     "rt-tokio",
     "trace",
 ], optional = true }
diff --git a/foyer-bench/Cargo.toml b/foyer-bench/Cargo.toml
index 9ca1c6bb..8afb448f 100644
--- a/foyer-bench/Cargo.toml
+++ b/foyer-bench/Cargo.toml
@@ -29,7 +29,7 @@ serde = { workspace = true }
 serde_bytes = "0.11.15"
 tokio = { workspace = true }
 tracing = "0.1"
-tracing-opentelemetry = { version = "0.25", optional = true }
+tracing-opentelemetry = { version = "0.26", optional = true }
 tracing-subscriber = { version = "0.3", features = ["env-filter"] }
 zipf = "7"
 
diff --git a/foyer-storage/Cargo.toml b/foyer-storage/Cargo.toml
index 3812d80b..e14de4a2 100644
--- a/foyer-storage/Cargo.toml
+++ b/foyer-storage/Cargo.toml
@@ -24,6 +24,7 @@ bytes = "1"
 clap = { workspace = true }
 either = "1"
 fastrace = { workspace = true }
+flume = "0.11"
 foyer-common = { version = "0.9.3", path = "../foyer-common" }
 foyer-memory = { version = "0.7.3", path = "../foyer-memory" }
 fs4 = "0.9.1"
diff --git a/foyer-storage/src/engine.rs b/foyer-storage/src/engine.rs
index 51357acd..33629b3e 100644
--- a/foyer-storage/src/engine.rs
+++ b/foyer-storage/src/engine.rs
@@ -28,13 +28,12 @@ use futures::Future;
 use crate::{
     error::Result,
     large::generic::{GenericLargeStorage, GenericLargeStorageConfig},
-    serde::KvInfo,
     small::generic::{GenericSmallStorage, GenericSmallStorageConfig},
     storage::{
         either::{Either, EitherConfig, Selection, Selector},
         noop::Noop,
     },
-    DeviceStats, IoBytes, Storage,
+    DeviceStats, Storage,
 };
 
 pub struct SizeSelector<K, V, S>
@@ -84,8 +83,12 @@ where
     type Value = V;
     type BuildHasher = S;
 
-    fn select(&self, _entry: &CacheEntry<Self::Key, Self::Value, Self::BuildHasher>, buffer: &IoBytes) -> Selection {
-        if buffer.len() < self.threshold {
+    fn select(
+        &self,
+        _entry: &CacheEntry<Self::Key, Self::Value, Self::BuildHasher>,
+        estimated_size: usize,
+    ) -> Selection {
+        if estimated_size < self.threshold {
             Selection::Left
         } else {
             Selection::Right
@@ -239,12 +242,12 @@ where
         }
     }
 
-    fn enqueue(&self, entry: CacheEntry<Self::Key, Self::Value, Self::BuildHasher>, buffer: IoBytes, info: KvInfo) {
+    fn enqueue(&self, entry: CacheEntry<Self::Key, Self::Value, Self::BuildHasher>, estimated_size: usize) {
         match self {
-            Engine::Noop(storage) => storage.enqueue(entry, buffer, info),
-            Engine::Large(storage) => storage.enqueue(entry, buffer, info),
-            Engine::Small(storage) => storage.enqueue(entry, buffer, info),
-            Engine::Combined(storage) => storage.enqueue(entry, buffer, info),
+            Engine::Noop(storage) => storage.enqueue(entry, estimated_size),
+            Engine::Large(storage) => storage.enqueue(entry, estimated_size),
+            Engine::Small(storage) => storage.enqueue(entry, estimated_size),
+            Engine::Combined(storage) => storage.enqueue(entry, estimated_size),
         }
     }
 
diff --git a/foyer-storage/src/large/batch.rs b/foyer-storage/src/large/batch.rs
index a4e46f87..ba5a8b04 100644
--- a/foyer-storage/src/large/batch.rs
+++ b/foyer-storage/src/large/batch.rs
@@ -12,19 +12,14 @@
 //  See the License for the specific language governing permissions and
 //  limitations under the License.
 
-use std::{
-    fmt::Debug,
-    mem::ManuallyDrop,
-    ops::{Deref, DerefMut, Range},
-    time::Instant,
-};
+use std::{fmt::Debug, ops::Range, sync::Arc, time::Instant};
 
 use foyer_common::{
     bits,
     code::{HashBuilder, StorageKey, StorageValue},
+    metrics::Metrics,
     range::RangeBoundsExt,
     strict_assert_eq,
-    wait_group::{WaitGroup, WaitGroupFuture, WaitGroupGuard},
 };
 use foyer_memory::CacheEntry;
 use itertools::Itertools;
@@ -39,38 +34,12 @@ use super::{
 use crate::{
     device::{bytes::IoBytes, MonitoredDevice, RegionId},
     io_buffer_pool::IoBufferPool,
-    large::indexer::HashedEntryAddress,
+    large::{indexer::HashedEntryAddress, serde::EntryHeader},
     region::{GetCleanRegionHandle, RegionManager},
-    Dev, DevExt, IoBuffer,
+    serde::{Checksummer, EntrySerializer},
+    Compression, Dev, DevExt, IoBuffer,
 };
 
-pub struct Allocation {
-    _guard: WaitGroupGuard,
-    slice: ManuallyDrop<Box<[u8]>>,
-}
-
-impl Deref for Allocation {
-    type Target = [u8];
-
-    fn deref(&self) -> &Self::Target {
-        self.slice.as_ref()
-    }
-}
-
-impl DerefMut for Allocation {
-    fn deref_mut(&mut self) -> &mut Self::Target {
-        self.slice.as_mut()
-    }
-}
-
-impl Allocation {
-    unsafe fn new(buffer: &mut [u8], guard: WaitGroupGuard) -> Self {
-        let fake = Vec::from_raw_parts(buffer.as_mut_ptr(), buffer.len(), buffer.len());
-        let slice = ManuallyDrop::new(fake.into_boxed_slice());
-        Self { _guard: guard, slice }
-    }
-}
-
 pub struct BatchMut<K, V, S>
 where
     K: StorageKey,
@@ -83,7 +52,6 @@ where
     tombstones: Vec<TombstoneInfo>,
     waiters: Vec<oneshot::Sender<()>>,
     init: Option<Instant>,
-    wait: WaitGroup,
 
     /// Cache write buffer between rotation to reduce page fault.
     buffer_pool: IoBufferPool,
@@ -91,6 +59,7 @@ where
     region_manager: RegionManager,
     device: MonitoredDevice,
     indexer: Indexer,
+    metrics: Arc<Metrics>,
 }
 
 impl<K, V, S> Debug for BatchMut<K, V, S>
@@ -116,7 +85,13 @@ where
     V: StorageValue,
     S: HashBuilder + Debug,
 {
-    pub fn new(capacity: usize, region_manager: RegionManager, device: MonitoredDevice, indexer: Indexer) -> Self {
+    pub fn new(
+        capacity: usize,
+        region_manager: RegionManager,
+        device: MonitoredDevice,
+        indexer: Indexer,
+        metrics: Arc<Metrics>,
+    ) -> Self {
         let capacity = bits::align_up(device.align(), capacity);
         let mut batch = Self {
             buffer: IoBuffer::new(capacity),
@@ -125,26 +100,56 @@ where
             tombstones: vec![],
             waiters: vec![],
             init: None,
-            wait: WaitGroup::default(),
             buffer_pool: IoBufferPool::new(capacity, 1),
             region_manager,
             device,
             indexer,
+            metrics,
         };
         batch.append_group();
         batch
     }
 
-    pub fn entry(&mut self, size: usize, entry: CacheEntry<K, V, S>, sequence: Sequence) -> Option<Allocation> {
+    pub fn entry(&mut self, entry: CacheEntry<K, V, S>, compression: &Compression, sequence: Sequence) -> bool {
         tracing::trace!("[batch]: append entry with sequence: {sequence}");
 
-        let aligned = bits::align_up(self.device.align(), size);
+        self.may_init();
 
-        if entry.is_outdated() || self.len + aligned > self.buffer.len() {
-            return None;
+        if entry.is_outdated() {
+            return false;
         }
 
-        let allocation = self.allocate(aligned);
+        let pos = self.len;
+
+        let info = match EntrySerializer::serialize(
+            entry.key(),
+            entry.value(),
+            compression,
+            &mut self.buffer[pos + EntryHeader::serialized_len()..],
+            &self.metrics,
+        ) {
+            Ok(info) => info,
+            Err(e) => {
+                tracing::warn!("[batch]: serialize entry error: {e}");
+                return false;
+            }
+        };
+
+        let header = EntryHeader {
+            key_len: info.key_len as _,
+            value_len: info.value_len as _,
+            hash: entry.hash(),
+            sequence,
+            checksum: Checksummer::checksum(
+                &self.buffer[pos + EntryHeader::serialized_len()
+                    ..pos + EntryHeader::serialized_len() + info.key_len + info.value_len],
+            ),
+            compression: *compression,
+        };
+        header.write(&mut self.buffer[pos..pos + EntryHeader::serialized_len()]);
+
+        let aligned = bits::align_up(self.device.align(), header.entry_len());
+        self.advance(aligned);
 
         let group = self.groups.last_mut().unwrap();
         group.indices.push(HashedEntryAddress {
@@ -152,7 +157,7 @@ where
             address: EntryAddress {
                 region: RegionId::MAX,
                 offset: group.region.offset as u32 + group.region.len as u32,
-                len: size as _,
+                len: header.entry_len() as _,
                 sequence,
             },
         });
@@ -160,27 +165,35 @@ where
         group.region.len += aligned;
         group.range.end += aligned;
 
-        Some(allocation)
+        true
     }
 
     pub fn tombstone(&mut self, tombstone: Tombstone, stats: Option<InvalidStats>) {
         tracing::trace!("[batch]: append tombstone");
+
         self.may_init();
+
         self.tombstones.push(TombstoneInfo { tombstone, stats });
     }
 
-    pub fn reinsertion(&mut self, reinsertion: &Reinsertion) -> Option<Allocation> {
+    pub fn reinsertion(&mut self, reinsertion: &Reinsertion) -> bool {
         tracing::trace!("[batch]: submit reinsertion");
 
+        self.may_init();
+
         let aligned = bits::align_up(self.device.align(), reinsertion.buffer.len());
 
         // Skip if the entry is no longer in the indexer.
         // Skip if the batch buffer size exceeds the threshold.
         if self.indexer.get(reinsertion.hash).is_none() || self.len + aligned > self.buffer.len() {
-            return None;
+            return false;
         }
 
-        let allocation = self.allocate(aligned);
+        let pos = self.len;
+
+        self.buffer[pos..pos + reinsertion.buffer.len()].copy_from_slice(&reinsertion.buffer);
+
+        self.advance(aligned);
 
         let group = self.groups.last_mut().unwrap();
         // Reserve buffer space for entry.
@@ -196,22 +209,17 @@ where
         group.region.len += aligned;
         group.range.end += aligned;
 
-        Some(allocation)
+        true
     }
 
     /// Register a waiter to be notified after the batch is finished.
-    pub fn wait(&mut self) -> oneshot::Receiver<()> {
+    pub fn wait(&mut self, tx: oneshot::Sender<()>) {
         tracing::trace!("[batch]: register waiter");
         self.may_init();
-        let (tx, rx) = oneshot::channel();
         self.waiters.push(tx);
-        rx
     }
 
-    // Note: Make sure `rotate` is called after all buffer from the last batch are dropped.
-    //
-    // Otherwise, the page fault caused by the buffer pool will hurt the performance.
-    pub fn rotate(&mut self) -> Option<(Batch<K, V, S>, WaitGroupFuture)> {
+    pub fn rotate(&mut self) -> Option<Batch<K, V, S>> {
         if self.is_empty() {
             return None;
         }
@@ -222,8 +230,6 @@ where
         let buffer = IoBytes::from(buffer);
         self.buffer_pool.release(buffer.clone());
 
-        let wait = std::mem::take(&mut self.wait);
-
         let init = self.init.take();
 
         let tombstones = std::mem::take(&mut self.tombstones);
@@ -269,20 +275,16 @@ where
             None => self.append_group(),
         }
 
-        Some((
-            Batch {
-                groups,
-                tombstones,
-                waiters,
-                init,
-            },
-            wait.wait(),
-        ))
+        Some(Batch {
+            groups,
+            tombstones,
+            waiters,
+            init,
+        })
     }
 
-    fn allocate(&mut self, len: usize) -> Allocation {
+    fn advance(&mut self, len: usize) {
         assert!(bits::is_aligned(self.device.align(), len));
-        self.may_init();
         assert!(bits::is_aligned(self.device.align(), self.len));
 
         // Rotate group if the current one is full.
@@ -292,24 +294,22 @@ where
             self.append_group();
         }
 
-        // Reserve buffer space for entry.
-        let start = self.len;
-        let end = start + len;
-        self.len = end;
-
-        unsafe { Allocation::new(&mut self.buffer[start..end], self.wait.acquire()) }
+        self.len += len;
     }
 
-    fn is_empty(&self) -> bool {
+    #[inline]
+    pub fn is_empty(&self) -> bool {
         self.tombstones.is_empty() && self.groups.iter().all(|group| group.range.is_empty()) && self.waiters.is_empty()
     }
 
+    #[inline]
     fn may_init(&mut self) {
         if self.init.is_none() {
             self.init = Some(Instant::now());
         }
     }
 
+    #[inline]
     fn append_group(&mut self) {
         self.groups.push(GroupMut {
             region: RegionHandle {
diff --git a/foyer-storage/src/large/flusher.rs b/foyer-storage/src/large/flusher.rs
index 4572bc14..5bff8c7f 100644
--- a/foyer-storage/src/large/flusher.rs
+++ b/foyer-storage/src/large/flusher.rs
@@ -21,12 +21,13 @@ use std::{
 use foyer_common::{
     code::{HashBuilder, StorageKey, StorageValue},
     metrics::Metrics,
-    strict_assert,
 };
 use foyer_memory::CacheEntry;
 use futures::future::{try_join, try_join_all};
-use parking_lot::Mutex;
-use tokio::{runtime::Handle, sync::Notify};
+use tokio::{
+    runtime::Handle,
+    sync::{oneshot, OwnedSemaphorePermit, Semaphore},
+};
 
 use super::{
     batch::{Batch, BatchMut, InvalidStats, TombstoneInfo},
@@ -39,13 +40,10 @@ use super::{
 use crate::{
     device::MonitoredDevice,
     error::{Error, Result},
-    large::serde::EntryHeader,
     region::RegionManager,
-    serde::{Checksummer, KvInfo},
-    Compression, IoBytes, Statistics,
+    Compression, Statistics,
 };
 
-#[derive(Debug)]
 pub enum Submission<K, V, S>
 where
     K: StorageKey,
@@ -54,8 +52,7 @@ where
 {
     CacheEntry {
         entry: CacheEntry<K, V, S>,
-        buffer: IoBytes,
-        info: KvInfo,
+        estimated_size: usize,
         sequence: Sequence,
     },
     Tombstone {
@@ -65,6 +62,39 @@ where
     Reinsertion {
         reinsertion: Reinsertion,
     },
+    Wait {
+        tx: oneshot::Sender<()>,
+    },
+}
+
+impl<K, V, S> Debug for Submission<K, V, S>
+where
+    K: StorageKey,
+    V: StorageValue,
+    S: HashBuilder + Debug,
+{
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        match self {
+            Self::CacheEntry {
+                entry: _,
+                estimated_size,
+                sequence,
+            } => f
+                .debug_struct("CacheEntry")
+                .field("estimated_size", estimated_size)
+                .field("sequence", sequence)
+                .finish(),
+            Self::Tombstone { tombstone, stats } => f
+                .debug_struct("Tombstone")
+                .field("tombstone", tombstone)
+                .field("stats", stats)
+                .finish(),
+            Self::Reinsertion { reinsertion } => {
+                f.debug_struct("Reinsertion").field("reinsertion", reinsertion).finish()
+            }
+            Self::Wait { .. } => f.debug_struct("Wait").finish(),
+        }
+    }
 }
 
 #[derive(Debug)]
@@ -74,11 +104,8 @@ where
     V: StorageValue,
     S: HashBuilder + Debug,
 {
-    batch: Arc<Mutex<BatchMut<K, V, S>>>,
+    tx: flume::Sender<Submission<K, V, S>>,
 
-    notify: Arc<Notify>,
-
-    compression: Compression,
     metrics: Arc<Metrics>,
 }
 
@@ -90,9 +117,7 @@ where
 {
     fn clone(&self) -> Self {
         Self {
-            batch: self.batch.clone(),
-            notify: self.notify.clone(),
-            compression: self.compression,
+            tx: self.tx.clone(),
             metrics: self.metrics.clone(),
         }
     }
@@ -115,22 +140,25 @@ where
         metrics: Arc<Metrics>,
         runtime: Handle,
     ) -> Result<Self> {
-        let notify = Arc::new(Notify::new());
+        let (tx, rx) = flume::unbounded();
 
         let buffer_size = config.buffer_threshold / config.flushers;
-        let batch = Arc::new(Mutex::new(BatchMut::new(
+        let batch = BatchMut::new(
             buffer_size,
             region_manager.clone(),
             device.clone(),
             indexer.clone(),
-        )));
+            metrics.clone(),
+        );
 
         let runner = Runner {
-            batch: batch.clone(),
-            notify: notify.clone(),
+            rx,
+            batch,
+            flight: Arc::new(Semaphore::new(1)),
             region_manager,
             indexer,
             tombstone_log,
+            compression: config.compression,
             flush: config.flush,
             stats,
             metrics: metrics.clone(),
@@ -142,74 +170,23 @@ where
             }
         });
 
-        Ok(Self {
-            batch,
-            notify,
-            compression: config.compression,
-            metrics,
-        })
+        Ok(Self { tx, metrics })
     }
 
     pub fn submit(&self, submission: Submission<K, V, S>) {
-        match submission {
-            Submission::CacheEntry {
-                entry,
-                buffer,
-                info,
-                sequence,
-            } => self.entry(entry, buffer, info, sequence),
-            Submission::Tombstone { tombstone, stats } => self.tombstone(tombstone, stats),
-            Submission::Reinsertion { reinsertion } => self.reinsertion(reinsertion),
+        tracing::trace!("[lodc flusher]: submit task: {submission:?}");
+        if let Err(e) = self.tx.send(submission) {
+            tracing::error!("[lodc flusher]: error raised when submitting task, error: {e}");
         }
-        self.notify.notify_one();
     }
 
     pub fn wait(&self) -> impl Future<Output = ()> + Send + 'static {
-        let waiter = self.batch.lock().wait();
-        self.notify.notify_one();
+        let (tx, rx) = oneshot::channel();
+        self.submit(Submission::Wait { tx });
         async move {
-            let _ = waiter.await;
+            let _ = rx.await;
         }
     }
-
-    fn entry(&self, entry: CacheEntry<K, V, S>, buffer: IoBytes, info: KvInfo, sequence: u64) {
-        let header = EntryHeader {
-            key_len: info.key_len as _,
-            value_len: info.value_len as _,
-            hash: entry.hash(),
-            sequence,
-            checksum: Checksummer::checksum(&buffer),
-            compression: self.compression,
-        };
-
-        let mut allocation = match self.batch.lock().entry(header.entry_len(), entry, sequence) {
-            Some(allocation) => allocation,
-            None => {
-                self.metrics.storage_queue_drop.increment(1);
-                return;
-            }
-        };
-        strict_assert!(allocation.len() >= header.entry_len());
-
-        header.write(&mut allocation[0..EntryHeader::serialized_len()]);
-        allocation[EntryHeader::serialized_len()..header.entry_len()].copy_from_slice(&buffer);
-    }
-
-    fn tombstone(&self, tombstone: Tombstone, stats: Option<InvalidStats>) {
-        self.batch.lock().tombstone(tombstone, stats);
-    }
-
-    fn reinsertion(&self, reinsertion: Reinsertion) {
-        let mut allocation = match self.batch.lock().reinsertion(&reinsertion) {
-            Some(allocation) => allocation,
-            None => {
-                self.metrics.storage_queue_drop.increment(1);
-                return;
-            }
-        };
-        strict_assert!(allocation.len() > reinsertion.buffer.len());
-        allocation[0..reinsertion.buffer.len()].copy_from_slice(&reinsertion.buffer);
-    }
 }
 
 struct Runner<K, V, S>
@@ -218,14 +195,15 @@ where
     V: StorageValue,
     S: HashBuilder + Debug,
 {
-    batch: Arc<Mutex<BatchMut<K, V, S>>>,
-
-    notify: Arc<Notify>,
+    rx: flume::Receiver<Submission<K, V, S>>,
+    batch: BatchMut<K, V, S>,
+    flight: Arc<Semaphore>,
 
     region_manager: RegionManager,
     indexer: Indexer,
     tombstone_log: Option<TombstoneLog>,
 
+    compression: Compression,
     flush: bool,
 
     stats: Arc<Statistics>,
@@ -238,25 +216,47 @@ where
     V: StorageValue,
     S: HashBuilder + Debug,
 {
-    pub async fn run(self) -> Result<()> {
-        // TODO(MrCroxx): Graceful shutdown.
+    pub async fn run(mut self) -> Result<()> {
         loop {
-            let rotation = self.batch.lock().rotate();
-            let (batch, wait) = match rotation {
-                Some(rotation) => rotation,
-                None => {
-                    self.notify.notified().await;
-                    continue;
+            let flight = self.flight.clone();
+            tokio::select! {
+                biased;
+                Ok(permit) = flight.acquire_owned(), if !self.batch.is_empty() => {
+                    // TODO(MrCroxx): `rotate()` should always return a `Some(..)` here.
+                    if let Some(batch) = self.batch.rotate() {
+                        self.commit(batch, permit).await;
+                    }
+                }
+                Ok(submission) = self.rx.recv_async() => {
+                    self.submit(submission);
                 }
-            };
+                // Graceful shutdown.
+                else => break,
+            }
+        }
+        Ok(())
+    }
 
-            wait.await;
+    fn submit(&mut self, submission: Submission<K, V, S>) {
+        let report = |enqueued: bool| {
+            if !enqueued {
+                self.metrics.storage_queue_drop.increment(1);
+            }
+        };
 
-            self.commit(batch).await
+        match submission {
+            Submission::CacheEntry {
+                entry,
+                estimated_size: _,
+                sequence,
+            } => report(self.batch.entry(entry, &self.compression, sequence)),
+            Submission::Tombstone { tombstone, stats } => self.batch.tombstone(tombstone, stats),
+            Submission::Reinsertion { reinsertion } => report(self.batch.reinsertion(&reinsertion)),
+            Submission::Wait { tx } => self.batch.wait(tx),
         }
     }
 
-    async fn commit(&self, batch: Batch<K, V, S>) {
+    async fn commit(&mut self, batch: Batch<K, V, S>, permit: OwnedSemaphorePermit) {
         tracing::trace!("[flusher] commit batch: {batch:?}");
 
         // Write regions concurrently.
@@ -331,5 +331,7 @@ where
             self.metrics.storage_queue_rotate.increment(1);
             self.metrics.storage_queue_rotate_duration.record(init.elapsed());
         }
+
+        drop(permit);
     }
 }
diff --git a/foyer-storage/src/large/generic.rs b/foyer-storage/src/large/generic.rs
index fd307d9a..ae2d710d 100644
--- a/foyer-storage/src/large/generic.rs
+++ b/foyer-storage/src/large/generic.rs
@@ -52,10 +52,9 @@ use crate::{
     },
     picker::{EvictionPicker, ReinsertionPicker},
     region::RegionManager,
-    serde::{EntryDeserializer, KvInfo},
+    serde::EntryDeserializer,
     statistics::Statistics,
     storage::Storage,
-    IoBytes,
 };
 
 pub struct GenericLargeStorageConfig<K, V, S>
@@ -296,7 +295,7 @@ where
     }
 
     #[fastrace::trace(name = "foyer::storage::large::generic::enqueue")]
-    fn enqueue(&self, entry: CacheEntry<K, V, S>, buffer: IoBytes, info: KvInfo) {
+    fn enqueue(&self, entry: CacheEntry<K, V, S>, estimated_size: usize) {
         if !self.inner.active.load(Ordering::Relaxed) {
             tracing::warn!("cannot enqueue new entry after closed");
             return;
@@ -305,8 +304,7 @@ where
         let sequence = self.inner.sequence.fetch_add(1, Ordering::Relaxed);
         self.inner.flushers[sequence as usize % self.inner.flushers.len()].submit(Submission::CacheEntry {
             entry,
-            buffer,
-            info,
+            estimated_size,
             sequence,
         });
     }
@@ -462,8 +460,8 @@ where
         self.close().await
     }
 
-    fn enqueue(&self, entry: CacheEntry<Self::Key, Self::Value, Self::BuildHasher>, buffer: IoBytes, info: KvInfo) {
-        self.enqueue(entry, buffer, info)
+    fn enqueue(&self, entry: CacheEntry<Self::Key, Self::Value, Self::BuildHasher>, estimated_size: usize) {
+        self.enqueue(entry, estimated_size)
     }
 
     fn load(&self, hash: u64) -> impl Future<Output = Result<Option<(Self::Key, Self::Value)>>> + Send + 'static {
@@ -508,8 +506,8 @@ mod tests {
         },
         picker::utils::{FifoPicker, RejectAllPicker},
         serde::EntrySerializer,
-        test_utils::{metrics_for_test, BiasedPicker},
-        IoBytesMut, TombstoneLogConfigBuilder,
+        test_utils::BiasedPicker,
+        TombstoneLogConfigBuilder,
     };
 
     const KB: usize = 1024;
@@ -602,17 +600,8 @@ mod tests {
     }
 
     fn enqueue(store: &GenericLargeStorage<u64, Vec<u8>, RandomState>, entry: CacheEntry<u64, Vec<u8>, RandomState>) {
-        let mut buffer = IoBytesMut::new();
-        let info = EntrySerializer::serialize(
-            entry.key(),
-            entry.value(),
-            &Compression::None,
-            &mut buffer,
-            metrics_for_test(),
-        )
-        .unwrap();
-        let buffer = buffer.freeze();
-        store.enqueue(entry, buffer, info);
+        let estimated_size = EntrySerializer::estimated_size(entry.key(), entry.value());
+        store.enqueue(entry, estimated_size);
     }
 
     #[test_log::test(tokio::test)]
diff --git a/foyer-storage/src/lib.rs b/foyer-storage/src/lib.rs
index 14d081e7..2aac3b26 100644
--- a/foyer-storage/src/lib.rs
+++ b/foyer-storage/src/lib.rs
@@ -15,6 +15,7 @@
 //! A disk cache engine that serves as the disk cache backend of `foyer`.
 
 #![cfg_attr(feature = "nightly", feature(allocator_api))]
+#![cfg_attr(feature = "nightly", feature(write_all_vectored))]
 #![warn(missing_docs)]
 #![warn(clippy::allow_attributes)]
 
diff --git a/foyer-storage/src/serde.rs b/foyer-storage/src/serde.rs
index 0d19d5b0..c3ec570e 100644
--- a/foyer-storage/src/serde.rs
+++ b/foyer-storage/src/serde.rs
@@ -12,10 +12,9 @@
 //  See the License for the specific language governing permissions and
 //  limitations under the License.
 
-use std::{fmt::Debug, hash::Hasher, time::Instant};
+use std::{fmt::Debug, hash::Hasher, io::Write, time::Instant};
 
 use foyer_common::{
-    bits,
     code::{StorageKey, StorageValue},
     metrics::Metrics,
 };
@@ -23,9 +22,7 @@ use twox_hash::XxHash64;
 
 use crate::{
     compress::Compression,
-    device::ALIGN,
     error::{Error, Result},
-    IoBytesMut,
 };
 
 #[derive(Debug)]
@@ -45,68 +42,115 @@ pub struct KvInfo {
     pub value_len: usize,
 }
 
+#[derive(Debug)]
+pub struct TrackedWriter<W> {
+    inner: W,
+    written: usize,
+}
+
+impl<W> TrackedWriter<W> {
+    pub fn new(inner: W) -> Self {
+        Self { inner, written: 0 }
+    }
+
+    pub fn written(&self) -> usize {
+        self.written
+    }
+
+    pub fn recount(&mut self) {
+        self.written = 0;
+    }
+}
+
+impl<W> Write for TrackedWriter<W>
+where
+    W: Write,
+{
+    fn write(&mut self, buf: &[u8]) -> std::io::Result<usize> {
+        self.inner.write(buf).inspect(|len| self.written += len)
+    }
+
+    fn flush(&mut self) -> std::io::Result<()> {
+        self.inner.flush()
+    }
+
+    fn write_vectored(&mut self, bufs: &[std::io::IoSlice<'_>]) -> std::io::Result<usize> {
+        self.inner.write_vectored(bufs).inspect(|len| self.written += len)
+    }
+
+    fn write_all(&mut self, buf: &[u8]) -> std::io::Result<()> {
+        self.inner.write_all(buf).inspect(|_| self.written += buf.len())
+    }
+
+    #[cfg(feature = "nightly")]
+    fn write_all_vectored(&mut self, bufs: &mut [std::io::IoSlice<'_>]) -> std::io::Result<()> {
+        self.inner
+            .write_all_vectored(bufs)
+            .inspect(|_| self.written += bufs.iter().map(|slice| slice.len()).sum::<usize>())
+    }
+}
+
 #[derive(Debug)]
 pub struct EntrySerializer;
 
 impl EntrySerializer {
     #[fastrace::trace(name = "foyer::storage::serde::serialize")]
-    pub fn serialize<'a, K, V>(
+    pub fn serialize<'a, K, V, W>(
         key: &'a K,
         value: &'a V,
         compression: &'a Compression,
-        mut buffer: &'a mut IoBytesMut,
+        writer: W,
         metrics: &Metrics,
     ) -> Result<KvInfo>
     where
         K: StorageKey,
         V: StorageValue,
+        W: Write,
     {
         let now = Instant::now();
 
-        let mut cursor = buffer.len();
+        let mut writer = TrackedWriter::new(writer);
 
         // serialize value
         match compression {
             Compression::None => {
-                bincode::serialize_into(&mut buffer, &value).map_err(Error::from)?;
+                bincode::serialize_into(&mut writer, &value).map_err(Error::from)?;
             }
             Compression::Zstd => {
-                let encoder = zstd::Encoder::new(&mut buffer, 0).map_err(Error::from)?.auto_finish();
-                bincode::serialize_into(encoder, &value).map_err(Error::from)?;
+                // Do not use `auto_finish()` here, for we will lost `ZeroWrite` error.
+                let mut encoder = zstd::Encoder::new(&mut writer, 0).map_err(Error::from)?;
+                bincode::serialize_into(&mut encoder, &value).map_err(Error::from)?;
+                encoder.finish().map_err(Error::from)?;
             }
-
             Compression::Lz4 => {
                 let encoder = lz4::EncoderBuilder::new()
                     .checksum(lz4::ContentChecksum::NoChecksum)
                     .auto_flush(true)
-                    .build(&mut buffer)
+                    .build(&mut writer)
                     .map_err(Error::from)?;
                 bincode::serialize_into(encoder, &value).map_err(Error::from)?;
             }
         }
 
-        let value_len = buffer.len() - cursor;
-        cursor = buffer.len();
+        let value_len = writer.written();
+        writer.recount();
 
         // serialize key
-        bincode::serialize_into(&mut buffer, &key).map_err(Error::from)?;
-        let key_len = buffer.len() - cursor;
+        bincode::serialize_into(&mut writer, &key).map_err(Error::from)?;
+        let key_len = writer.written();
 
         metrics.storage_entry_serialize_duration.record(now.elapsed());
 
         Ok(KvInfo { key_len, value_len })
     }
 
-    pub fn size_hint<'a, K, V>(key: &'a K, value: &'a V) -> usize
+    pub fn estimated_size<'a, K, V>(key: &'a K, value: &'a V) -> usize
     where
         K: StorageKey,
         V: StorageValue,
     {
-        let hint = match (bincode::serialized_size(key), bincode::serialized_size(value)) {
-            (Ok(k), Ok(v)) => (k + v) as usize,
-            _ => 0,
-        };
-        bits::align_up(ALIGN, hint)
+        // `serialized_size` should always return `Ok(..)` without a hard size limit.
+        (bincode::serialized_size(key).unwrap() + bincode::serialized_size(value).unwrap()) as usize
     }
 }
 
@@ -176,20 +220,3 @@ impl EntryDeserializer {
         }
     }
 }
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-    use crate::test_utils::metrics_for_test;
-
-    #[test]
-    fn test_serde_size_hint() {
-        let key = 42u64;
-        let value = vec![b'x'; 114514];
-        let hint = EntrySerializer::size_hint(&key, &value);
-        let mut buf = IoBytesMut::new();
-        EntrySerializer::serialize(&key, &value, &Compression::None, &mut buf, metrics_for_test()).unwrap();
-        assert!(hint >= buf.len());
-        assert!(hint.abs_diff(buf.len()) < ALIGN);
-    }
-}
diff --git a/foyer-storage/src/small/generic.rs b/foyer-storage/src/small/generic.rs
index cd086bc1..dd8ae8e2 100644
--- a/foyer-storage/src/small/generic.rs
+++ b/foyer-storage/src/small/generic.rs
@@ -18,7 +18,7 @@ use foyer_common::code::{HashBuilder, StorageKey, StorageValue};
 use foyer_memory::CacheEntry;
 use futures::Future;
 
-use crate::{error::Result, serde::KvInfo, storage::Storage, DeviceStats, IoBytes};
+use crate::{error::Result, storage::Storage, DeviceStats};
 
 pub struct GenericSmallStorageConfig<K, V, S>
 where
@@ -90,7 +90,7 @@ where
         todo!()
     }
 
-    fn enqueue(&self, _entry: CacheEntry<Self::Key, Self::Value, Self::BuildHasher>, _buffer: IoBytes, _info: KvInfo) {
+    fn enqueue(&self, _entry: CacheEntry<Self::Key, Self::Value, Self::BuildHasher>, _estimated_size: usize) {
         todo!()
     }
 
diff --git a/foyer-storage/src/storage/either.rs b/foyer-storage/src/storage/either.rs
index 8708f453..97d945ec 100644
--- a/foyer-storage/src/storage/either.rs
+++ b/foyer-storage/src/storage/either.rs
@@ -26,7 +26,7 @@ use futures::{
     pin_mut, Future, FutureExt,
 };
 
-use crate::{error::Result, serde::KvInfo, storage::Storage, DeviceStats, IoBytes};
+use crate::{error::Result, storage::Storage, DeviceStats};
 
 enum OrderFuture<F1, F2, F3> {
     LeftFirst(F1),
@@ -123,7 +123,8 @@ pub trait Selector: Send + Sync + 'static + Debug {
     type Value: StorageValue;
     type BuildHasher: HashBuilder;
 
-    fn select(&self, entry: &CacheEntry<Self::Key, Self::Value, Self::BuildHasher>, buffer: &IoBytes) -> Selection;
+    fn select(&self, entry: &CacheEntry<Self::Key, Self::Value, Self::BuildHasher>, estimated_size: usize)
+        -> Selection;
 }
 
 pub struct Either<K, V, S, SL, SR, SE>
@@ -212,15 +213,15 @@ where
         Ok(())
     }
 
-    fn enqueue(&self, entry: CacheEntry<Self::Key, Self::Value, Self::BuildHasher>, buffer: IoBytes, info: KvInfo) {
-        match self.selector.select(&entry, &buffer) {
+    fn enqueue(&self, entry: CacheEntry<Self::Key, Self::Value, Self::BuildHasher>, estimated_size: usize) {
+        match self.selector.select(&entry, estimated_size) {
             Selection::Left => {
                 self.right.delete(entry.hash());
-                self.left.enqueue(entry, buffer, info);
+                self.left.enqueue(entry, estimated_size);
             }
             Selection::Right => {
                 self.right.delete(entry.hash());
-                self.right.enqueue(entry, buffer, info);
+                self.right.enqueue(entry, estimated_size);
             }
         }
     }
diff --git a/foyer-storage/src/storage/mod.rs b/foyer-storage/src/storage/mod.rs
index f3f56f49..b82e08da 100644
--- a/foyer-storage/src/storage/mod.rs
+++ b/foyer-storage/src/storage/mod.rs
@@ -20,7 +20,7 @@ use std::{fmt::Debug, future::Future, sync::Arc};
 use foyer_common::code::{HashBuilder, StorageKey, StorageValue};
 use foyer_memory::CacheEntry;
 
-use crate::{device::monitor::DeviceStats, error::Result, serde::KvInfo, IoBytes};
+use crate::{device::monitor::DeviceStats, error::Result};
 
 /// The storage trait for the disk cache storage engine.
 pub trait Storage: Send + Sync + 'static + Clone + Debug {
@@ -44,7 +44,7 @@ pub trait Storage: Send + Sync + 'static + Clone + Debug {
     fn close(&self) -> impl Future<Output = Result<()>> + Send;
 
     /// Push a in-memory cache entry to the disk cache write queue.
-    fn enqueue(&self, entry: CacheEntry<Self::Key, Self::Value, Self::BuildHasher>, buffer: IoBytes, info: KvInfo);
+    fn enqueue(&self, entry: CacheEntry<Self::Key, Self::Value, Self::BuildHasher>, estimated_size: usize);
 
     /// Load a cache entry from the disk cache.
     ///
diff --git a/foyer-storage/src/storage/noop.rs b/foyer-storage/src/storage/noop.rs
index dc9a9f25..b8daf6b5 100644
--- a/foyer-storage/src/storage/noop.rs
+++ b/foyer-storage/src/storage/noop.rs
@@ -18,7 +18,7 @@ use foyer_common::code::{HashBuilder, StorageKey, StorageValue};
 use foyer_memory::CacheEntry;
 use futures::future::ready;
 
-use crate::{device::monitor::DeviceStats, error::Result, serde::KvInfo, storage::Storage, IoBytes};
+use crate::{device::monitor::DeviceStats, error::Result, storage::Storage};
 
 pub struct Noop<K, V, S>
 where
@@ -70,7 +70,7 @@ where
         Ok(())
     }
 
-    fn enqueue(&self, _entry: CacheEntry<Self::Key, Self::Value, Self::BuildHasher>, _buffer: IoBytes, _info: KvInfo) {}
+    fn enqueue(&self, _entry: CacheEntry<Self::Key, Self::Value, Self::BuildHasher>, _estimated_size: usize) {}
 
     fn load(&self, _: u64) -> impl Future<Output = Result<Option<(Self::Key, Self::Value)>>> + Send + 'static {
         ready(Ok(None))
@@ -112,14 +112,7 @@ mod tests {
         let memory = cache_for_test();
         let store = Noop::open(()).await.unwrap();
 
-        store.enqueue(
-            memory.insert(0, vec![b'x'; 16384]),
-            IoBytes::new(),
-            KvInfo {
-                key_len: 0,
-                value_len: 0,
-            },
-        );
+        store.enqueue(memory.insert(0, vec![b'x'; 16384]), 16384);
         store.wait().await;
         assert!(store.load(memory.hash(&0)).await.unwrap().is_none());
         store.delete(memory.hash(&0));
diff --git a/foyer-storage/src/store.rs b/foyer-storage/src/store.rs
index 663f9870..e9ecb23e 100644
--- a/foyer-storage/src/store.rs
+++ b/foyer-storage/src/store.rs
@@ -45,7 +45,7 @@ use crate::{
         either::{EitherConfig, Order},
         Storage,
     },
-    Dev, DevExt, DirectFileDeviceOptions, IoBytesMut,
+    Dev, DevExt, DirectFileDeviceOptions,
 };
 
 /// The disk cache engine that serves as the storage backend of `foyer`.
@@ -139,29 +139,10 @@ where
     pub fn enqueue(&self, entry: CacheEntry<K, V, S>, force: bool) {
         let now = Instant::now();
 
-        let compression = self.inner.compression;
-        let this = self.clone();
-
-        self.inner.write_runtime_handle.spawn(async move {
-            if force || this.pick(entry.key()) {
-                let mut buffer = IoBytesMut::with_capacity(EntrySerializer::size_hint(entry.key(), entry.value()));
-                match EntrySerializer::serialize(
-                    entry.key(),
-                    entry.value(),
-                    &compression,
-                    &mut buffer,
-                    &this.inner.metrics,
-                ) {
-                    Ok(info) => {
-                        let buffer = buffer.freeze();
-                        this.inner.engine.enqueue(entry, buffer, info);
-                    }
-                    Err(e) => {
-                        tracing::warn!("[store]: serialize kv error: {e}");
-                    }
-                }
-            }
-        });
+        if force || self.pick(entry.key()) {
+            let estimated_size = EntrySerializer::estimated_size(entry.key(), entry.value());
+            self.inner.engine.enqueue(entry, estimated_size);
+        }
 
         self.inner.metrics.storage_enqueue.increment(1);
         self.inner.metrics.storage_enqueue_duration.record(now.elapsed());

From af065891abee3d7b8bf05b23d06fd1313c3caccd Mon Sep 17 00:00:00 2001
From: Croxx <mrcroxx@outlook.com>
Date: Tue, 24 Sep 2024 15:06:14 +0800
Subject: [PATCH 12/53] refactor: replace manual future enum with auto_enum
 (#732)

Signed-off-by: MrCroxx <mrcroxx@outlook.com>
---
 foyer-storage/Cargo.toml            |  1 +
 foyer-storage/src/engine.rs         | 66 ++++++-----------------------
 foyer-storage/src/storage/either.rs | 56 ++++--------------------
 3 files changed, 22 insertions(+), 101 deletions(-)

diff --git a/foyer-storage/Cargo.toml b/foyer-storage/Cargo.toml
index e14de4a2..dcb1b2ca 100644
--- a/foyer-storage/Cargo.toml
+++ b/foyer-storage/Cargo.toml
@@ -18,6 +18,7 @@ anyhow = "1.0"
 # TODO(MrCroxx): use `array_chunks` after `#![feature(array_chunks)]` is stable.
 array-util = "1"
 async-channel = "2"
+auto_enums = { version = "0.8", features = ["futures03"] }
 bincode = "1"
 bitflags = "2.3.1"
 bytes = "1"
diff --git a/foyer-storage/src/engine.rs b/foyer-storage/src/engine.rs
index 33629b3e..eb16abe7 100644
--- a/foyer-storage/src/engine.rs
+++ b/foyer-storage/src/engine.rs
@@ -12,15 +12,10 @@
 //  See the License for the specific language governing permissions and
 //  limitations under the License.
 
-use std::{
-    fmt::Debug,
-    marker::PhantomData,
-    pin::Pin,
-    sync::Arc,
-    task::{Context, Poll},
-};
+use std::{fmt::Debug, marker::PhantomData, sync::Arc};
 
 use ahash::RandomState;
+use auto_enums::auto_enum;
 use foyer_common::code::{HashBuilder, StorageKey, StorageValue};
 use foyer_memory::CacheEntry;
 use futures::Future;
@@ -96,45 +91,6 @@ where
     }
 }
 
-enum StoreFuture<F1, F2, F3, F4> {
-    Noop(F1),
-    Large(F2),
-    Small(F3),
-    Combined(F4),
-}
-
-impl<F1, F2, F3, F4> StoreFuture<F1, F2, F3, F4> {
-    pub fn as_pin_mut(self: Pin<&mut Self>) -> StoreFuture<Pin<&mut F1>, Pin<&mut F2>, Pin<&mut F3>, Pin<&mut F4>> {
-        unsafe {
-            match *Pin::get_unchecked_mut(self) {
-                StoreFuture::Noop(ref mut inner) => StoreFuture::Noop(Pin::new_unchecked(inner)),
-                StoreFuture::Large(ref mut inner) => StoreFuture::Large(Pin::new_unchecked(inner)),
-                StoreFuture::Small(ref mut inner) => StoreFuture::Small(Pin::new_unchecked(inner)),
-                StoreFuture::Combined(ref mut inner) => StoreFuture::Combined(Pin::new_unchecked(inner)),
-            }
-        }
-    }
-}
-
-impl<F1, F2, F3, F4> Future for StoreFuture<F1, F2, F3, F4>
-where
-    F1: Future,
-    F2: Future<Output = F1::Output>,
-    F3: Future<Output = F1::Output>,
-    F4: Future<Output = F1::Output>,
-{
-    type Output = F1::Output;
-
-    fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Self::Output> {
-        match self.as_pin_mut() {
-            StoreFuture::Noop(future) => future.poll(cx),
-            StoreFuture::Large(future) => future.poll(cx),
-            StoreFuture::Small(future) => future.poll(cx),
-            StoreFuture::Combined(future) => future.poll(cx),
-        }
-    }
-}
-
 #[expect(clippy::type_complexity)]
 pub enum EngineConfig<K, V, S = RandomState>
 where
@@ -251,12 +207,13 @@ where
         }
     }
 
+    #[auto_enum(Future)]
     fn load(&self, hash: u64) -> impl Future<Output = Result<Option<(Self::Key, Self::Value)>>> + Send + 'static {
         match self {
-            Engine::Noop(storage) => StoreFuture::Noop(storage.load(hash)),
-            Engine::Large(storage) => StoreFuture::Large(storage.load(hash)),
-            Engine::Small(storage) => StoreFuture::Small(storage.load(hash)),
-            Engine::Combined(storage) => StoreFuture::Combined(storage.load(hash)),
+            Engine::Noop(storage) => storage.load(hash),
+            Engine::Large(storage) => storage.load(hash),
+            Engine::Small(storage) => storage.load(hash),
+            Engine::Combined(storage) => storage.load(hash),
         }
     }
 
@@ -296,12 +253,13 @@ where
         }
     }
 
+    #[auto_enum(Future)]
     fn wait(&self) -> impl Future<Output = ()> + Send + 'static {
         match self {
-            Engine::Noop(storage) => StoreFuture::Noop(storage.wait()),
-            Engine::Large(storage) => StoreFuture::Large(storage.wait()),
-            Engine::Small(storage) => StoreFuture::Small(storage.wait()),
-            Engine::Combined(storage) => StoreFuture::Combined(storage.wait()),
+            Engine::Noop(storage) => storage.wait(),
+            Engine::Large(storage) => storage.wait(),
+            Engine::Small(storage) => storage.wait(),
+            Engine::Combined(storage) => storage.wait(),
         }
     }
 }
diff --git a/foyer-storage/src/storage/either.rs b/foyer-storage/src/storage/either.rs
index 97d945ec..c6c34c16 100644
--- a/foyer-storage/src/storage/either.rs
+++ b/foyer-storage/src/storage/either.rs
@@ -12,13 +12,9 @@
 //  See the License for the specific language governing permissions and
 //  limitations under the License.
 
-use std::{
-    fmt::Debug,
-    pin::Pin,
-    sync::Arc,
-    task::{Context, Poll},
-};
+use std::{fmt::Debug, sync::Arc};
 
+use auto_enums::auto_enum;
 use foyer_common::code::{HashBuilder, StorageKey, StorageValue};
 use foyer_memory::CacheEntry;
 use futures::{
@@ -28,41 +24,6 @@ use futures::{
 
 use crate::{error::Result, storage::Storage, DeviceStats};
 
-enum OrderFuture<F1, F2, F3> {
-    LeftFirst(F1),
-    RightFirst(F2),
-    Parallel(F3),
-}
-
-impl<F1, F2, F3> OrderFuture<F1, F2, F3> {
-    pub fn as_pin_mut(self: Pin<&mut Self>) -> OrderFuture<Pin<&mut F1>, Pin<&mut F2>, Pin<&mut F3>> {
-        unsafe {
-            match *Pin::get_unchecked_mut(self) {
-                OrderFuture::LeftFirst(ref mut inner) => OrderFuture::LeftFirst(Pin::new_unchecked(inner)),
-                OrderFuture::RightFirst(ref mut inner) => OrderFuture::RightFirst(Pin::new_unchecked(inner)),
-                OrderFuture::Parallel(ref mut inner) => OrderFuture::Parallel(Pin::new_unchecked(inner)),
-            }
-        }
-    }
-}
-
-impl<F1, F2, F3> Future for OrderFuture<F1, F2, F3>
-where
-    F1: Future,
-    F2: Future<Output = F1::Output>,
-    F3: Future<Output = F1::Output>,
-{
-    type Output = F1::Output;
-
-    fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Self::Output> {
-        match self.as_pin_mut() {
-            OrderFuture::LeftFirst(future) => future.poll(cx),
-            OrderFuture::RightFirst(future) => future.poll(cx),
-            OrderFuture::Parallel(future) => future.poll(cx),
-        }
-    }
-}
-
 /// Order of ops.
 #[derive(Debug, Clone, Copy)]
 pub enum Order {
@@ -226,24 +187,25 @@ where
         }
     }
 
+    #[auto_enum(Future)]
     fn load(&self, hash: u64) -> impl Future<Output = Result<Option<(Self::Key, Self::Value)>>> + Send + 'static {
         let fleft = self.left.load(hash);
         let fright = self.right.load(hash);
         match self.load_order {
             // FIXME(MrCroxx): false-positive on hash collision.
-            Order::LeftFirst => OrderFuture::LeftFirst(fleft.then(|res| match res {
+            Order::LeftFirst => fleft.then(|res| match res {
                 Ok(Some(kv)) => ready(Ok(Some(kv))).left_future(),
                 Err(e) => ready(Err(e)).left_future(),
                 Ok(None) => fright.right_future(),
-            })),
+            }),
             // FIXME(MrCroxx): false-positive on hash collision.
-            Order::RightFirst => OrderFuture::RightFirst(fright.then(|res| match res {
+            Order::RightFirst => fright.then(|res| match res {
                 Ok(Some(kv)) => ready(Ok(Some(kv))).left_future(),
                 Err(e) => ready(Err(e)).left_future(),
                 Ok(None) => fleft.right_future(),
-            })),
+            }),
             Order::Parallel => {
-                OrderFuture::Parallel(async move {
+                async move {
                     pin_mut!(fleft);
                     pin_mut!(fright);
                     // Returns a 4-way `Either` by nesting `Either` in `Either`.
@@ -261,7 +223,7 @@ where
                             },
                         })
                         .await
-                })
+                }
             }
         }
     }

From 5fe395746b10ed7e656e73109f067ce520a165ff Mon Sep 17 00:00:00 2001
From: Croxx <mrcroxx@outlook.com>
Date: Tue, 24 Sep 2024 15:28:58 +0800
Subject: [PATCH 13/53] chore: release foyer 0.11.4 (#733)

Signed-off-by: MrCroxx <mrcroxx@outlook.com>
---
 CHANGELOG.md               | 19 +++++++++++++++++++
 foyer-bench/Cargo.toml     |  4 ++--
 foyer-common/Cargo.toml    |  2 +-
 foyer-intrusive/Cargo.toml |  4 ++--
 foyer-memory/Cargo.toml    |  6 +++---
 foyer-storage/Cargo.toml   |  6 +++---
 foyer/Cargo.toml           |  8 ++++----
 7 files changed, 34 insertions(+), 15 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index a989f5f9..9ffc44c3 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -9,6 +9,25 @@ date: 2023-05-12T11:02:09+08:00
 
 <!-- truncate -->
 
+## 2024-09-24
+
+### Releases
+
+| crate | version |
+| - | - |
+| foyer | 0.11.4 |
+| foyer-common | 0.9.4 |
+| foyer-intrusive | 0.9.4 |
+| foyer-memory | 0.7.4 |
+| foyer-storage | 0.10.4 |
+| foyer-bench | 0.3.4 |
+
+### Changes
+
+- Revert pre-serialization design. The insert latency and memory usage would be better for most cases.
+- Rename `with_buffer_threshold` to `with_buffer_pool_size`. The old method is kept but marked as deprecated.
+- Raise a warn when using `DirectFileDevice` on within a file system.
+
 ## 2024-09-20
 
 ### Releases
diff --git a/foyer-bench/Cargo.toml b/foyer-bench/Cargo.toml
index 8afb448f..d34cfcf6 100644
--- a/foyer-bench/Cargo.toml
+++ b/foyer-bench/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "foyer-bench"
-version = "0.3.3"
+version = "0.3.4"
 edition = "2021"
 authors = ["MrCroxx <mrcroxx@outlook.com>"]
 description = "bench tool for foyer - the hybrid cache for Rust"
@@ -17,7 +17,7 @@ clap = { workspace = true }
 console-subscriber = { version = "0.4", optional = true }
 fastrace = { workspace = true, optional = true }
 fastrace-jaeger = { workspace = true, optional = true }
-foyer = { version = "0.11.3", path = "../foyer" }
+foyer = { version = "0.11.4", path = "../foyer" }
 futures = "0.3"
 hdrhistogram = "7"
 itertools = { workspace = true }
diff --git a/foyer-common/Cargo.toml b/foyer-common/Cargo.toml
index 16ef110d..b8da41c3 100644
--- a/foyer-common/Cargo.toml
+++ b/foyer-common/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "foyer-common"
-version = "0.9.3"
+version = "0.9.4"
 edition = "2021"
 authors = ["MrCroxx <mrcroxx@outlook.com>"]
 description = "common components for foyer - the hybrid cache for Rust"
diff --git a/foyer-intrusive/Cargo.toml b/foyer-intrusive/Cargo.toml
index abdf950d..7963f58e 100644
--- a/foyer-intrusive/Cargo.toml
+++ b/foyer-intrusive/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "foyer-intrusive"
-version = "0.9.3"
+version = "0.9.4"
 edition = "2021"
 authors = ["MrCroxx <mrcroxx@outlook.com>"]
 description = "intrusive data structures for foyer - the hybrid cache for Rust"
@@ -11,7 +11,7 @@ readme = "../README.md"
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
-foyer-common = { version = "0.9.3", path = "../foyer-common" }
+foyer-common = { version = "0.9.4", path = "../foyer-common" }
 itertools = { workspace = true }
 
 [features]
diff --git a/foyer-memory/Cargo.toml b/foyer-memory/Cargo.toml
index f971ad0e..2aad532c 100644
--- a/foyer-memory/Cargo.toml
+++ b/foyer-memory/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "foyer-memory"
-version = "0.7.3"
+version = "0.7.4"
 edition = "2021"
 authors = ["MrCroxx <mrcroxx@outlook.com>"]
 description = "memory cache for foyer - the hybrid cache for Rust"
@@ -15,8 +15,8 @@ ahash = "0.8"
 bitflags = "2"
 cmsketch = "0.2.1"
 fastrace = { workspace = true }
-foyer-common = { version = "0.9.3", path = "../foyer-common" }
-foyer-intrusive = { version = "0.9.3", path = "../foyer-intrusive" }
+foyer-common = { version = "0.9.4", path = "../foyer-common" }
+foyer-intrusive = { version = "0.9.4", path = "../foyer-intrusive" }
 futures = "0.3"
 hashbrown = "0.14"
 itertools = { workspace = true }
diff --git a/foyer-storage/Cargo.toml b/foyer-storage/Cargo.toml
index dcb1b2ca..7c910ebb 100644
--- a/foyer-storage/Cargo.toml
+++ b/foyer-storage/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "foyer-storage"
-version = "0.10.3"
+version = "0.10.4"
 edition = "2021"
 authors = ["MrCroxx <mrcroxx@outlook.com>"]
 description = "storage engine for foyer - the hybrid cache for Rust"
@@ -26,8 +26,8 @@ clap = { workspace = true }
 either = "1"
 fastrace = { workspace = true }
 flume = "0.11"
-foyer-common = { version = "0.9.3", path = "../foyer-common" }
-foyer-memory = { version = "0.7.3", path = "../foyer-memory" }
+foyer-common = { version = "0.9.4", path = "../foyer-common" }
+foyer-memory = { version = "0.7.4", path = "../foyer-memory" }
 fs4 = "0.9.1"
 futures = "0.3"
 itertools = { workspace = true }
diff --git a/foyer/Cargo.toml b/foyer/Cargo.toml
index 25a4e837..e4f4904f 100644
--- a/foyer/Cargo.toml
+++ b/foyer/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "foyer"
-version = "0.11.3"
+version = "0.11.4"
 edition = "2021"
 authors = ["MrCroxx <mrcroxx@outlook.com>"]
 description = "Hybrid cache for Rust"
@@ -15,9 +15,9 @@ rust-version = "1.81.0"
 ahash = "0.8"
 anyhow = "1"
 fastrace = { workspace = true }
-foyer-common = { version = "0.9.3", path = "../foyer-common" }
-foyer-memory = { version = "0.7.3", path = "../foyer-memory" }
-foyer-storage = { version = "0.10.3", path = "../foyer-storage" }
+foyer-common = { version = "0.9.4", path = "../foyer-common" }
+foyer-memory = { version = "0.7.4", path = "../foyer-memory" }
+foyer-storage = { version = "0.10.4", path = "../foyer-storage" }
 futures = "0.3"
 pin-project = "1"
 tokio = { workspace = true }

From b49e2753c0d2055090ebf0b9633a0de0d094565c Mon Sep 17 00:00:00 2001
From: Croxx <mrcroxx@outlook.com>
Date: Wed, 25 Sep 2024 00:18:03 +0800
Subject: [PATCH 14/53] fix: hold runtime ref and handle to prevent spawn after
 shutdown (#736)

* fix: hold runtime ref and handle to prevent spawn after shutdown

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

* fix: make ffmt happy

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

* fix: make device hold runtime, too

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

---------

Signed-off-by: MrCroxx <mrcroxx@outlook.com>
---
 foyer-common/src/asyncify.rs            |  8 +--
 foyer-common/src/runtime.rs             | 84 ++++++++++++++++++++++-
 foyer-memory/src/cache.rs               |  3 +-
 foyer-memory/src/generic.rs             | 13 +++-
 foyer-storage/src/device/direct_file.rs | 20 +++---
 foyer-storage/src/device/direct_fs.rs   | 25 ++++---
 foyer-storage/src/device/mod.rs         | 11 +--
 foyer-storage/src/device/monitor.rs     | 10 +--
 foyer-storage/src/large/flusher.rs      | 10 ++-
 foyer-storage/src/large/generic.rs      | 64 +++++++++---------
 foyer-storage/src/large/reclaimer.rs    | 14 ++--
 foyer-storage/src/large/recover.rs      |  7 +-
 foyer-storage/src/large/scanner.rs      | 24 ++++---
 foyer-storage/src/large/tombstone.rs    | 25 ++++---
 foyer-storage/src/lib.rs                |  1 +
 foyer-storage/src/prelude.rs            |  3 +-
 foyer-storage/src/runtime.rs            | 89 +++++++++++++++++++++++++
 foyer-storage/src/store.rs              | 86 ++++++++----------------
 foyer/src/hybrid/cache.rs               |  7 +-
 foyer/src/prelude.rs                    |  4 +-
 20 files changed, 330 insertions(+), 178 deletions(-)
 create mode 100644 foyer-storage/src/runtime.rs

diff --git a/foyer-common/src/asyncify.rs b/foyer-common/src/asyncify.rs
index 9dacfb5b..ba441b49 100644
--- a/foyer-common/src/asyncify.rs
+++ b/foyer-common/src/asyncify.rs
@@ -12,7 +12,7 @@
 //  See the License for the specific language governing permissions and
 //  limitations under the License.
 
-use tokio::runtime::Handle;
+use crate::runtime::SingletonHandle;
 
 /// Convert the block call to async call.
 #[cfg(not(madsim))]
@@ -36,9 +36,9 @@ where
     f()
 }
 
-/// Convert the block call to async call with given runtime.
+/// Convert the block call to async call with given runtime handle.
 #[cfg(not(madsim))]
-pub async fn asyncify_with_runtime<F, T>(runtime: &Handle, f: F) -> T
+pub async fn asyncify_with_runtime<F, T>(runtime: &SingletonHandle, f: F) -> T
 where
     F: FnOnce() -> T + Send + 'static,
     T: Send + 'static,
@@ -50,7 +50,7 @@ where
 /// Convert the block call to async call with given runtime.
 ///
 /// madsim compatible mode.
-pub async fn asyncify_with_runtime<F, T>(_: &Handle, f: F) -> T
+pub async fn asyncify_with_runtime<F, T>(_: &SingletonHandle, f: F) -> T
 where
     F: FnOnce() -> T + Send + 'static,
     T: Send + 'static,
diff --git a/foyer-common/src/runtime.rs b/foyer-common/src/runtime.rs
index 703171b2..d2b269c2 100644
--- a/foyer-common/src/runtime.rs
+++ b/foyer-common/src/runtime.rs
@@ -14,11 +14,15 @@
 
 use std::{
     fmt::Debug,
+    future::Future,
     mem::ManuallyDrop,
     ops::{Deref, DerefMut},
 };
 
-use tokio::runtime::Runtime;
+use tokio::{
+    runtime::{Handle, Runtime},
+    task::JoinHandle,
+};
 
 /// A wrapper around [`Runtime`] that shuts down the runtime in the background when dropped.
 ///
@@ -62,3 +66,81 @@ impl From<Runtime> for BackgroundShutdownRuntime {
         Self(ManuallyDrop::new(runtime))
     }
 }
+
+/// A non-clonable runtime handle.
+#[derive(Debug)]
+pub struct SingletonHandle(Handle);
+
+impl From<Handle> for SingletonHandle {
+    fn from(handle: Handle) -> Self {
+        Self(handle)
+    }
+}
+
+impl SingletonHandle {
+    /// Spawns a future onto the Tokio runtime.
+    ///
+    /// This spawns the given future onto the runtime's executor, usually a
+    /// thread pool. The thread pool is then responsible for polling the future
+    /// until it completes.
+    ///
+    /// The provided future will start running in the background immediately
+    /// when `spawn` is called, even if you don't await the returned
+    /// `JoinHandle`.
+    ///
+    /// See [module level][mod] documentation for more details.
+    ///
+    /// [mod]: index.html
+    ///
+    /// # Examples
+    ///
+    /// ```
+    /// use tokio::runtime::Runtime;
+    ///
+    /// # fn dox() {
+    /// // Create the runtime
+    /// let rt = Runtime::new().unwrap();
+    /// // Get a handle from this runtime
+    /// let handle = rt.handle();
+    ///
+    /// // Spawn a future onto the runtime using the handle
+    /// handle.spawn(async {
+    ///     println!("now running on a worker thread");
+    /// });
+    /// # }
+    /// ```
+    pub fn spawn<F>(&self, future: F) -> JoinHandle<F::Output>
+    where
+        F: Future + Send + 'static,
+        F::Output: Send + 'static,
+    {
+        self.0.spawn(future)
+    }
+
+    /// Runs the provided function on an executor dedicated to blocking
+    /// operations.
+    ///
+    /// # Examples
+    ///
+    /// ```
+    /// use tokio::runtime::Runtime;
+    ///
+    /// # fn dox() {
+    /// // Create the runtime
+    /// let rt = Runtime::new().unwrap();
+    /// // Get a handle from this runtime
+    /// let handle = rt.handle();
+    ///
+    /// // Spawn a blocking function onto the runtime using the handle
+    /// handle.spawn_blocking(|| {
+    ///     println!("now running on a worker thread");
+    /// });
+    /// # }
+    pub fn spawn_blocking<F, R>(&self, func: F) -> JoinHandle<R>
+    where
+        F: FnOnce() -> R + Send + 'static,
+        R: Send + 'static,
+    {
+        self.0.spawn_blocking(func)
+    }
+}
diff --git a/foyer-memory/src/cache.rs b/foyer-memory/src/cache.rs
index 8e50ca02..6d8fc3c1 100644
--- a/foyer-memory/src/cache.rs
+++ b/foyer-memory/src/cache.rs
@@ -19,6 +19,7 @@ use foyer_common::{
     code::{HashBuilder, Key, Value},
     event::EventListener,
     future::Diversion,
+    runtime::SingletonHandle,
 };
 use futures::Future;
 use pin_project::pin_project;
@@ -834,7 +835,7 @@ where
         key: K,
         context: CacheContext,
         fetch: F,
-        runtime: &tokio::runtime::Handle,
+        runtime: &SingletonHandle,
     ) -> Fetch<K, V, ER, S>
     where
         F: FnOnce() -> FU,
diff --git a/foyer-memory/src/generic.rs b/foyer-memory/src/generic.rs
index 8f20b089..8d8d7094 100644
--- a/foyer-memory/src/generic.rs
+++ b/foyer-memory/src/generic.rs
@@ -35,6 +35,7 @@ use foyer_common::{
     future::{Diversion, DiversionFuture},
     metrics::Metrics,
     object_pool::ObjectPool,
+    runtime::SingletonHandle,
     strict_assert, strict_assert_eq,
 };
 use hashbrown::hash_map::{Entry as HashMapEntry, HashMap};
@@ -739,7 +740,12 @@ where
         FU: Future<Output = std::result::Result<V, ER>> + Send + 'static,
         ER: Send + 'static + Debug,
     {
-        self.fetch_inner(key, CacheContext::default(), fetch, &tokio::runtime::Handle::current())
+        self.fetch_inner(
+            key,
+            CacheContext::default(),
+            fetch,
+            &tokio::runtime::Handle::current().into(),
+        )
     }
 
     pub fn fetch_with_context<F, FU, ER>(
@@ -753,15 +759,16 @@ where
         FU: Future<Output = std::result::Result<V, ER>> + Send + 'static,
         ER: Send + 'static + Debug,
     {
-        self.fetch_inner(key, context, fetch, &tokio::runtime::Handle::current())
+        self.fetch_inner(key, context, fetch, &tokio::runtime::Handle::current().into())
     }
 
+    #[doc(hidden)]
     pub fn fetch_inner<F, FU, ER, ID>(
         self: &Arc<Self>,
         key: K,
         context: CacheContext,
         fetch: F,
-        runtime: &tokio::runtime::Handle,
+        runtime: &SingletonHandle,
     ) -> GenericFetch<K, V, E, I, S, ER>
     where
         F: FnOnce() -> FU,
diff --git a/foyer-storage/src/device/direct_file.rs b/foyer-storage/src/device/direct_file.rs
index 6b8170f3..ec02e2c2 100644
--- a/foyer-storage/src/device/direct_file.rs
+++ b/foyer-storage/src/device/direct_file.rs
@@ -21,13 +21,12 @@ use std::{
 use foyer_common::{asyncify::asyncify_with_runtime, bits};
 use fs4::free_space;
 use serde::{Deserialize, Serialize};
-use tokio::runtime::Handle;
 
 use super::{Dev, DevExt, DevOptions, RegionId};
 use crate::{
     device::ALIGN,
     error::{Error, Result},
-    IoBytes, IoBytesMut,
+    IoBytes, IoBytesMut, Runtime,
 };
 
 /// Options for the direct file device.
@@ -49,7 +48,7 @@ pub struct DirectFileDevice {
     capacity: usize,
     region_size: usize,
 
-    runtime: Handle,
+    runtime: Runtime,
 }
 
 impl DevOptions for DirectFileDeviceOptions {
@@ -90,7 +89,7 @@ impl DirectFileDevice {
 
         let file = self.file.clone();
 
-        asyncify_with_runtime(&self.runtime, move || {
+        asyncify_with_runtime(self.runtime.write(), move || {
             #[cfg(target_family = "windows")]
             let written = {
                 use std::os::windows::fs::FileExt;
@@ -133,7 +132,7 @@ impl DirectFileDevice {
 
         let file = self.file.clone();
 
-        let mut buffer = asyncify_with_runtime(&self.runtime, move || {
+        let mut buffer = asyncify_with_runtime(self.runtime.read(), move || {
             #[cfg(target_family = "windows")]
             let read = {
                 use std::os::windows::fs::FileExt;
@@ -172,9 +171,7 @@ impl Dev for DirectFileDevice {
     }
 
     #[fastrace::trace(name = "foyer::storage::device::direct_file::open")]
-    async fn open(options: Self::Options) -> Result<Self> {
-        let runtime = Handle::current();
-
+    async fn open(options: Self::Options, runtime: Runtime) -> Result<Self> {
         options.verify()?;
 
         let dir = options
@@ -253,7 +250,7 @@ impl Dev for DirectFileDevice {
     #[fastrace::trace(name = "foyer::storage::device::direct_file::flush")]
     async fn flush(&self, _: Option<RegionId>) -> Result<()> {
         let file = self.file.clone();
-        asyncify_with_runtime(&self.runtime, move || file.sync_all().map_err(Error::from)).await
+        asyncify_with_runtime(self.runtime.write(), move || file.sync_all().map_err(Error::from)).await
     }
 }
 
@@ -360,6 +357,7 @@ mod tests {
     #[test_log::test(tokio::test)]
     async fn test_direct_file_device_io() {
         let dir = tempfile::tempdir().unwrap();
+        let runtime = Runtime::current();
 
         let options = DirectFileDeviceOptionsBuilder::new(dir.path().join("test-direct-file"))
             .with_capacity(4 * 1024 * 1024)
@@ -368,7 +366,7 @@ mod tests {
 
         tracing::debug!("{options:?}");
 
-        let device = DirectFileDevice::open(options.clone()).await.unwrap();
+        let device = DirectFileDevice::open(options.clone(), runtime.clone()).await.unwrap();
 
         let mut buf = IoBytesMut::with_capacity(64 * 1024);
         buf.extend(repeat_n(b'x', 64 * 1024 - 100));
@@ -383,7 +381,7 @@ mod tests {
 
         drop(device);
 
-        let device = DirectFileDevice::open(options).await.unwrap();
+        let device = DirectFileDevice::open(options, runtime).await.unwrap();
 
         let b = device.read(0, 4096, 64 * 1024 - 100).await.unwrap().freeze();
         assert_eq!(buf, b);
diff --git a/foyer-storage/src/device/direct_fs.rs b/foyer-storage/src/device/direct_fs.rs
index 307e8e7b..94fc3f11 100644
--- a/foyer-storage/src/device/direct_fs.rs
+++ b/foyer-storage/src/device/direct_fs.rs
@@ -23,13 +23,12 @@ use fs4::free_space;
 use futures::future::try_join_all;
 use itertools::Itertools;
 use serde::{Deserialize, Serialize};
-use tokio::runtime::Handle;
 
 use super::{Dev, DevExt, DevOptions, RegionId};
 use crate::{
     device::ALIGN,
     error::{Error, Result},
-    IoBytes, IoBytesMut,
+    IoBytes, IoBytesMut, Runtime,
 };
 
 /// Options for the direct fs device.
@@ -56,7 +55,7 @@ struct DirectFsDeviceInner {
     capacity: usize,
     file_size: usize,
 
-    runtime: Handle,
+    runtime: Runtime,
 }
 
 impl DevOptions for DirectFsDeviceOptions {
@@ -106,17 +105,16 @@ impl Dev for DirectFsDevice {
     }
 
     #[fastrace::trace(name = "foyer::storage::device::direct_fs::open")]
-    async fn open(options: Self::Options) -> Result<Self> {
-        let runtime = Handle::current();
-
+    async fn open(options: Self::Options, runtime: Runtime) -> Result<Self> {
         options.verify()?;
 
         // TODO(MrCroxx): write and read options to a manifest file for pinning
 
         let regions = options.capacity / options.file_size;
 
-        let path = options.dir.clone();
-        asyncify_with_runtime(&runtime, move || create_dir_all(path)).await?;
+        if !options.dir.exists() {
+            create_dir_all(&options.dir)?;
+        }
 
         let futures = (0..regions)
             .map(|i| {
@@ -165,7 +163,7 @@ impl Dev for DirectFsDevice {
 
         let file = self.file(region).clone();
 
-        asyncify_with_runtime(&self.inner.runtime, move || {
+        asyncify_with_runtime(self.inner.runtime.write(), move || {
             #[cfg(target_family = "windows")]
             let written = {
                 use std::os::windows::fs::FileExt;
@@ -207,7 +205,7 @@ impl Dev for DirectFsDevice {
 
         let file = self.file(region).clone();
 
-        let mut buffer = asyncify_with_runtime(&self.inner.runtime, move || {
+        let mut buffer = asyncify_with_runtime(self.inner.runtime.read(), move || {
             #[cfg(target_family = "unix")]
             let read = {
                 use std::os::unix::fs::FileExt;
@@ -237,7 +235,7 @@ impl Dev for DirectFsDevice {
     async fn flush(&self, region: Option<super::RegionId>) -> Result<()> {
         let flush = |region: RegionId| {
             let file = self.file(region).clone();
-            asyncify_with_runtime(&self.inner.runtime, move || file.sync_all().map_err(Error::from))
+            asyncify_with_runtime(self.inner.runtime.write(), move || file.sync_all().map_err(Error::from))
         };
 
         if let Some(region) = region {
@@ -352,6 +350,7 @@ mod tests {
     #[test_log::test(tokio::test)]
     async fn test_direct_fd_device_io() {
         let dir = tempfile::tempdir().unwrap();
+        let runtime = Runtime::current();
 
         let options = DirectFsDeviceOptionsBuilder::new(dir.path())
             .with_capacity(4 * 1024 * 1024)
@@ -360,7 +359,7 @@ mod tests {
 
         tracing::debug!("{options:?}");
 
-        let device = DirectFsDevice::open(options.clone()).await.unwrap();
+        let device = DirectFsDevice::open(options.clone(), runtime.clone()).await.unwrap();
 
         let mut buf = IoBytesMut::with_capacity(64 * 1024);
         buf.extend(repeat_n(b'x', 64 * 1024 - 100));
@@ -375,7 +374,7 @@ mod tests {
 
         drop(device);
 
-        let device = DirectFsDevice::open(options).await.unwrap();
+        let device = DirectFsDevice::open(options, runtime).await.unwrap();
 
         let b = device.read(0, 4096, 64 * 1024 - 100).await.unwrap().freeze();
         assert_eq!(buf, b);
diff --git a/foyer-storage/src/device/mod.rs b/foyer-storage/src/device/mod.rs
index 23faa656..26dfe5f7 100644
--- a/foyer-storage/src/device/mod.rs
+++ b/foyer-storage/src/device/mod.rs
@@ -25,7 +25,7 @@ use monitor::Monitored;
 
 use crate::{
     error::Result, DirectFileDevice, DirectFileDeviceOptions, DirectFsDevice, DirectFsDeviceOptions, IoBytes,
-    IoBytesMut,
+    IoBytesMut, Runtime,
 };
 
 pub const ALIGN: usize = 4096;
@@ -52,9 +52,10 @@ pub trait Dev: Send + Sync + 'static + Sized + Clone + Debug {
     /// The region size of the device, must be 4K aligned.
     fn region_size(&self) -> usize;
 
+    // TODO(MrCroxx): Refactor the builder.
     /// Open the device with the given options.
     #[must_use]
-    fn open(options: Self::Options) -> impl Future<Output = Result<Self>> + Send;
+    fn open(options: Self::Options, runtime: Runtime) -> impl Future<Output = Result<Self>> + Send;
 
     /// Write API for the device.
     #[must_use]
@@ -134,10 +135,10 @@ impl Dev for Device {
         }
     }
 
-    async fn open(options: Self::Options) -> Result<Self> {
+    async fn open(options: Self::Options, runtime: Runtime) -> Result<Self> {
         match options {
-            DeviceOptions::DirectFile(opts) => Ok(Self::DirectFile(DirectFileDevice::open(opts).await?)),
-            DeviceOptions::DirectFs(opts) => Ok(Self::DirectFs(DirectFsDevice::open(opts).await?)),
+            DeviceOptions::DirectFile(opts) => Ok(Self::DirectFile(DirectFileDevice::open(opts, runtime).await?)),
+            DeviceOptions::DirectFs(opts) => Ok(Self::DirectFs(DirectFsDevice::open(opts, runtime).await?)),
         }
     }
 
diff --git a/foyer-storage/src/device/monitor.rs b/foyer-storage/src/device/monitor.rs
index 35354ace..3e338d1d 100644
--- a/foyer-storage/src/device/monitor.rs
+++ b/foyer-storage/src/device/monitor.rs
@@ -24,7 +24,7 @@ use std::{
 use foyer_common::{bits, metrics::Metrics};
 
 use super::RegionId;
-use crate::{error::Result, Dev, DevExt, DevOptions, DirectFileDevice, IoBytes, IoBytesMut};
+use crate::{error::Result, Dev, DevExt, DevOptions, DirectFileDevice, IoBytes, IoBytesMut, Runtime};
 
 /// The statistics information of the device.
 #[derive(Debug, Default)]
@@ -87,8 +87,8 @@ impl<D> Monitored<D>
 where
     D: Dev,
 {
-    async fn open(options: MonitoredOptions<D>) -> Result<Self> {
-        let device = D::open(options.options).await?;
+    async fn open(options: MonitoredOptions<D>, runtime: Runtime) -> Result<Self> {
+        let device = D::open(options.options, runtime).await?;
         Ok(Self {
             device,
             stats: Arc::default(),
@@ -159,8 +159,8 @@ where
         self.device.region_size()
     }
 
-    async fn open(options: Self::Options) -> Result<Self> {
-        Self::open(options).await
+    async fn open(options: Self::Options, runtime: Runtime) -> Result<Self> {
+        Self::open(options, runtime).await
     }
 
     async fn write(&self, buf: IoBytes, region: RegionId, offset: u64) -> Result<()> {
diff --git a/foyer-storage/src/large/flusher.rs b/foyer-storage/src/large/flusher.rs
index 5bff8c7f..7a179701 100644
--- a/foyer-storage/src/large/flusher.rs
+++ b/foyer-storage/src/large/flusher.rs
@@ -24,10 +24,7 @@ use foyer_common::{
 };
 use foyer_memory::CacheEntry;
 use futures::future::{try_join, try_join_all};
-use tokio::{
-    runtime::Handle,
-    sync::{oneshot, OwnedSemaphorePermit, Semaphore},
-};
+use tokio::sync::{oneshot, OwnedSemaphorePermit, Semaphore};
 
 use super::{
     batch::{Batch, BatchMut, InvalidStats, TombstoneInfo},
@@ -41,6 +38,7 @@ use crate::{
     device::MonitoredDevice,
     error::{Error, Result},
     region::RegionManager,
+    runtime::Runtime,
     Compression, Statistics,
 };
 
@@ -138,7 +136,7 @@ where
         tombstone_log: Option<TombstoneLog>,
         stats: Arc<Statistics>,
         metrics: Arc<Metrics>,
-        runtime: Handle,
+        runtime: &Runtime,
     ) -> Result<Self> {
         let (tx, rx) = flume::unbounded();
 
@@ -164,7 +162,7 @@ where
             metrics: metrics.clone(),
         };
 
-        runtime.spawn(async move {
+        runtime.write().spawn(async move {
             if let Err(e) = runner.run().await {
                 tracing::error!("[flusher]: flusher exit with error: {e}");
             }
diff --git a/foyer-storage/src/large/generic.rs b/foyer-storage/src/large/generic.rs
index ae2d710d..489c6be0 100644
--- a/foyer-storage/src/large/generic.rs
+++ b/foyer-storage/src/large/generic.rs
@@ -32,7 +32,7 @@ use foyer_common::{
 };
 use foyer_memory::CacheEntry;
 use futures::future::{join_all, try_join_all};
-use tokio::{runtime::Handle, sync::Semaphore};
+use tokio::sync::Semaphore;
 
 use super::{
     batch::InvalidStats,
@@ -52,6 +52,7 @@ use crate::{
     },
     picker::{EvictionPicker, ReinsertionPicker},
     region::RegionManager,
+    runtime::Runtime,
     serde::EntryDeserializer,
     statistics::Statistics,
     storage::Storage,
@@ -79,9 +80,7 @@ where
     pub reinsertion_picker: Arc<dyn ReinsertionPicker<Key = K>>,
     pub tombstone_log_config: Option<TombstoneLogConfig>,
     pub statistics: Arc<Statistics>,
-    pub read_runtime_handle: Handle,
-    pub write_runtime_handle: Handle,
-    pub user_runtime_handle: Handle,
+    pub runtime: Runtime,
     pub marker: PhantomData<(V, S)>,
 }
 
@@ -108,8 +107,7 @@ where
             .field("reinsertion_pickers", &self.reinsertion_picker)
             .field("tombstone_log_config", &self.tombstone_log_config)
             .field("statistics", &self.statistics)
-            .field("read_runtime_handle", &self.read_runtime_handle)
-            .field("write_runtime_handle", &self.write_runtime_handle)
+            .field("runtime", &self.runtime)
             .finish()
     }
 }
@@ -153,9 +151,7 @@ where
 
     sequence: AtomicSequence,
 
-    _read_runtime_handle: Handle,
-    write_runtime_handle: Handle,
-    _user_runtime_handle: Handle,
+    runtime: Runtime,
 
     active: AtomicBool,
 
@@ -190,13 +186,14 @@ where
         let mut tombstones = vec![];
         let tombstone_log = match &config.tombstone_log_config {
             None => None,
-            Some(config) => {
+            Some(tombstone_log_config) => {
                 let log = TombstoneLog::open(
-                    &config.path,
+                    &tombstone_log_config.path,
                     device.clone(),
-                    config.flush,
+                    tombstone_log_config.flush,
                     &mut tombstones,
                     metrics.clone(),
+                    config.runtime.clone(),
                 )
                 .await?;
                 Some(log)
@@ -225,7 +222,7 @@ where
             &region_manager,
             &tombstones,
             metrics.clone(),
-            config.user_runtime_handle.clone(),
+            config.runtime.clone(),
         )
         .await?;
 
@@ -238,7 +235,7 @@ where
                 tombstone_log.clone(),
                 stats.clone(),
                 metrics.clone(),
-                config.write_runtime_handle.clone(),
+                &config.runtime,
             )
             .await
         }))
@@ -254,7 +251,7 @@ where
                 stats.clone(),
                 config.flush,
                 metrics.clone(),
-                config.write_runtime_handle.clone(),
+                &config.runtime,
             )
             .await
         }))
@@ -270,9 +267,7 @@ where
                 statistics: stats,
                 flush: config.flush,
                 sequence,
-                _read_runtime_handle: config.read_runtime_handle,
-                write_runtime_handle: config.write_runtime_handle,
-                _user_runtime_handle: config.user_runtime_handle,
+                runtime: config.runtime,
                 active: AtomicBool::new(true),
                 metrics,
             }),
@@ -390,7 +385,7 @@ where
         });
 
         let this = self.clone();
-        self.inner.write_runtime_handle.spawn(async move {
+        self.inner.runtime.write().spawn(async move {
             let sequence = this.inner.sequence.fetch_add(1, Ordering::Relaxed);
             this.inner.flushers[sequence as usize % this.inner.flushers.len()].submit(Submission::Tombstone {
                 tombstone: Tombstone { hash, sequence },
@@ -497,6 +492,7 @@ mod tests {
     use ahash::RandomState;
     use foyer_memory::{Cache, CacheBuilder, FifoConfig};
     use itertools::Itertools;
+    use tokio::runtime::Handle;
 
     use super::*;
     use crate::{
@@ -519,15 +515,19 @@ mod tests {
     }
 
     async fn device_for_test(dir: impl AsRef<Path>) -> MonitoredDevice {
-        Monitored::open(MonitoredOptions {
-            options: DirectFsDeviceOptions {
-                dir: dir.as_ref().into(),
-                capacity: 64 * KB,
-                file_size: 16 * KB,
-            }
-            .into(),
-            metrics: Arc::new(Metrics::new("test")),
-        })
+        let runtime = Runtime::current();
+        Monitored::open(
+            MonitoredOptions {
+                options: DirectFsDeviceOptions {
+                    dir: dir.as_ref().into(),
+                    capacity: 64 * KB,
+                    file_size: 16 * KB,
+                }
+                .into(),
+                metrics: Arc::new(Metrics::new("test")),
+            },
+            runtime,
+        )
         .await
         .unwrap()
     }
@@ -560,9 +560,7 @@ mod tests {
             tombstone_log_config: None,
             buffer_threshold: 16 * 1024 * 1024,
             statistics: Arc::<Statistics>::default(),
-            read_runtime_handle: Handle::current(),
-            write_runtime_handle: Handle::current(),
-            user_runtime_handle: Handle::current(),
+            runtime: Runtime::new(None, None, Handle::current()),
             marker: PhantomData,
         };
         GenericLargeStorage::open(config).await.unwrap()
@@ -591,9 +589,7 @@ mod tests {
             tombstone_log_config: Some(TombstoneLogConfigBuilder::new(path).with_flush(true).build()),
             buffer_threshold: 16 * 1024 * 1024,
             statistics: Arc::<Statistics>::default(),
-            read_runtime_handle: Handle::current(),
-            write_runtime_handle: Handle::current(),
-            user_runtime_handle: Handle::current(),
+            runtime: Runtime::new(None, None, Handle::current()),
             marker: PhantomData,
         };
         GenericLargeStorage::open(config).await.unwrap()
diff --git a/foyer-storage/src/large/reclaimer.rs b/foyer-storage/src/large/reclaimer.rs
index b7a6b249..dafbff26 100644
--- a/foyer-storage/src/large/reclaimer.rs
+++ b/foyer-storage/src/large/reclaimer.rs
@@ -20,10 +20,7 @@ use foyer_common::{
 };
 use futures::future::join_all;
 use itertools::Itertools;
-use tokio::{
-    runtime::Handle,
-    sync::{mpsc, oneshot, Semaphore, SemaphorePermit},
-};
+use tokio::sync::{mpsc, oneshot, Semaphore, SemaphorePermit};
 
 use crate::{
     device::IO_BUFFER_ALLOCATOR,
@@ -36,6 +33,7 @@ use crate::{
     },
     picker::ReinsertionPicker,
     region::{Region, RegionManager},
+    runtime::Runtime,
     statistics::Statistics,
     IoBytes,
 };
@@ -56,7 +54,7 @@ impl Reclaimer {
         stats: Arc<Statistics>,
         flush: bool,
         metrics: Arc<Metrics>,
-        runtime: Handle,
+        runtime: &Runtime,
     ) -> Self
     where
         K: StorageKey,
@@ -78,7 +76,7 @@ impl Reclaimer {
             runtime: runtime.clone(),
         };
 
-        let _handle = runtime.spawn(async move { runner.run().await });
+        let _handle = runtime.write().spawn(async move { runner.run().await });
 
         Self { wait_tx }
     }
@@ -116,7 +114,7 @@ where
 
     wait_rx: mpsc::UnboundedReceiver<oneshot::Sender<()>>,
 
-    runtime: Handle,
+    runtime: Runtime,
 }
 
 impl<K, V, S> ReclaimRunner<K, V, S>
@@ -223,7 +221,7 @@ where
         let unpicked_count = unpicked.len();
 
         let waits = self.flushers.iter().map(|flusher| flusher.wait()).collect_vec();
-        self.runtime.spawn(async move {
+        self.runtime.write().spawn(async move {
             join_all(waits).await;
         });
         self.indexer.remove_batch(&unpicked);
diff --git a/foyer-storage/src/large/recover.rs b/foyer-storage/src/large/recover.rs
index 74ed3bcb..007258a0 100644
--- a/foyer-storage/src/large/recover.rs
+++ b/foyer-storage/src/large/recover.rs
@@ -27,7 +27,7 @@ use foyer_common::{
 use futures::future::try_join_all;
 use itertools::Itertools;
 use serde::{Deserialize, Serialize};
-use tokio::{runtime::Handle, sync::Semaphore};
+use tokio::sync::Semaphore;
 
 use super::{
     generic::GenericLargeStorageConfig,
@@ -43,6 +43,7 @@ use crate::{
         tombstone::Tombstone,
     },
     region::{Region, RegionManager},
+    runtime::Runtime,
 };
 
 /// The recover mode of the disk cache.
@@ -72,7 +73,7 @@ impl RecoverRunner {
         region_manager: &RegionManager,
         tombstones: &[Tombstone],
         metrics: Arc<Metrics>,
-        runtime: Handle,
+        runtime: Runtime,
     ) -> Result<()>
     where
         K: StorageKey,
@@ -86,7 +87,7 @@ impl RecoverRunner {
             let semaphore = semaphore.clone();
             let region = region_manager.region(id).clone();
             let metrics = metrics.clone();
-            runtime.spawn(async move {
+            runtime.user().spawn(async move {
                 let permit = semaphore.acquire().await;
                 let res = RegionRecoverRunner::run(mode, region, metrics).await;
                 drop(permit);
diff --git a/foyer-storage/src/large/scanner.rs b/foyer-storage/src/large/scanner.rs
index ce5eeb08..20f43309 100644
--- a/foyer-storage/src/large/scanner.rs
+++ b/foyer-storage/src/large/scanner.rs
@@ -248,21 +248,25 @@ mod tests {
             Dev, MonitoredDevice,
         },
         region::RegionStats,
-        DirectFsDeviceOptions,
+        DirectFsDeviceOptions, Runtime,
     };
 
     const KB: usize = 1024;
 
     async fn device_for_test(dir: impl AsRef<Path>) -> MonitoredDevice {
-        Monitored::open(MonitoredOptions {
-            options: DirectFsDeviceOptions {
-                dir: dir.as_ref().into(),
-                capacity: 64 * KB,
-                file_size: 16 * KB,
-            }
-            .into(),
-            metrics: Arc::new(Metrics::new("test")),
-        })
+        let runtime = Runtime::current();
+        Monitored::open(
+            MonitoredOptions {
+                options: DirectFsDeviceOptions {
+                    dir: dir.as_ref().into(),
+                    capacity: 64 * KB,
+                    file_size: 16 * KB,
+                }
+                .into(),
+                metrics: Arc::new(Metrics::new("test")),
+            },
+            runtime,
+        )
         .await
         .unwrap()
     }
diff --git a/foyer-storage/src/large/tombstone.rs b/foyer-storage/src/large/tombstone.rs
index b44a9a6d..09a9ee27 100644
--- a/foyer-storage/src/large/tombstone.rs
+++ b/foyer-storage/src/large/tombstone.rs
@@ -30,7 +30,7 @@ use crate::{
         Dev, DevExt, RegionId,
     },
     error::{Error, Result},
-    IoBytesMut,
+    IoBytesMut, Runtime,
 };
 
 /// The configurations for the tombstone log.
@@ -121,6 +121,7 @@ impl TombstoneLog {
         flush: bool,
         tombstones: &mut Vec<Tombstone>,
         metrics: Arc<Metrics>,
+        runtime: Runtime,
     ) -> Result<Self>
     where
         D: Dev,
@@ -134,13 +135,16 @@ impl TombstoneLog {
         // For the alignment is 4K and the slot size is 16B, tombstone log requires 1/256 of the cache device size.
         let capacity = bits::align_up(align, (cache_device.capacity() / align) * Tombstone::serialized_len());
 
-        let device = Monitored::open(MonitoredOptions {
-            options: DirectFileDeviceOptionsBuilder::new(path)
-                .with_region_size(align)
-                .with_capacity(capacity)
-                .build(),
-            metrics,
-        })
+        let device = Monitored::open(
+            MonitoredOptions {
+                options: DirectFileDeviceOptionsBuilder::new(path)
+                    .with_region_size(align)
+                    .with_capacity(capacity)
+                    .build(),
+                metrics,
+            },
+            runtime,
+        )
         .await?;
 
         let tasks = bits::align_up(Self::RECOVER_IO_SIZE, capacity) / Self::RECOVER_IO_SIZE;
@@ -312,6 +316,8 @@ mod tests {
 
     #[test_log::test(tokio::test)]
     async fn test_tombstone_log() {
+        let runtime = Runtime::current();
+
         let dir = tempdir().unwrap();
 
         // 4 MB cache device => 16 KB tombstone log => 1K tombstones
@@ -319,6 +325,7 @@ mod tests {
             DirectFsDeviceOptionsBuilder::new(dir.path())
                 .with_capacity(4 * 1024 * 1024)
                 .build(),
+            runtime.clone(),
         )
         .await
         .unwrap();
@@ -329,6 +336,7 @@ mod tests {
             true,
             &mut vec![],
             Arc::new(Metrics::new("test")),
+            runtime.clone(),
         )
         .await
         .unwrap();
@@ -358,6 +366,7 @@ mod tests {
             true,
             &mut vec![],
             Arc::new(Metrics::new("test")),
+            runtime,
         )
         .await
         .unwrap();
diff --git a/foyer-storage/src/lib.rs b/foyer-storage/src/lib.rs
index 2aac3b26..ceb30ebb 100644
--- a/foyer-storage/src/lib.rs
+++ b/foyer-storage/src/lib.rs
@@ -27,6 +27,7 @@ mod io_buffer_pool;
 mod large;
 mod picker;
 mod region;
+mod runtime;
 mod serde;
 mod small;
 mod statistics;
diff --git a/foyer-storage/src/prelude.rs b/foyer-storage/src/prelude.rs
index 2744dee1..c2564896 100644
--- a/foyer-storage/src/prelude.rs
+++ b/foyer-storage/src/prelude.rs
@@ -30,7 +30,8 @@ pub use crate::{
         utils::{AdmitAllPicker, FifoPicker, InvalidRatioPicker, RateLimitPicker, RejectAllPicker},
         AdmissionPicker, EvictionPicker, ReinsertionPicker,
     },
+    runtime::Runtime,
     statistics::Statistics,
     storage::{either::Order, Storage},
-    store::{CombinedConfig, DeviceConfig, RuntimeConfig, RuntimeHandles, Store, StoreBuilder, TokioRuntimeConfig},
+    store::{CombinedConfig, DeviceConfig, RuntimeConfig, Store, StoreBuilder, TokioRuntimeConfig},
 };
diff --git a/foyer-storage/src/runtime.rs b/foyer-storage/src/runtime.rs
new file mode 100644
index 00000000..fc361cb5
--- /dev/null
+++ b/foyer-storage/src/runtime.rs
@@ -0,0 +1,89 @@
+//  Copyright 2024 Foyer Project Authors
+//
+//  Licensed under the Apache License, Version 2.0 (the "License");
+//  you may not use this file except in compliance with the License.
+//  You may obtain a copy of the License at
+//
+//  http://www.apache.org/licenses/LICENSE-2.0
+//
+//  Unless required by applicable law or agreed to in writing, software
+//  distributed under the License is distributed on an "AS IS" BASIS,
+//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//  See the License for the specific language governing permissions and
+//  limitations under the License.
+
+use std::sync::Arc;
+
+use foyer_common::runtime::{BackgroundShutdownRuntime, SingletonHandle};
+use tokio::runtime::Handle;
+
+#[derive(Debug)]
+struct RuntimeInner {
+    _read_runtime: Option<Arc<BackgroundShutdownRuntime>>,
+    _write_runtime: Option<Arc<BackgroundShutdownRuntime>>,
+
+    read_runtime_handle: SingletonHandle,
+    write_runtime_handle: SingletonHandle,
+    user_runtime_handle: SingletonHandle,
+}
+
+/// [`Runtime`] holds the runtime reference and non-clonable handles to prevent handle usage after runtime shutdown.
+#[derive(Debug, Clone)]
+pub struct Runtime {
+    inner: Arc<RuntimeInner>,
+}
+
+impl Runtime {
+    /// Create a new runtime with runtimes if given.
+    pub fn new(
+        read_runtime: Option<Arc<BackgroundShutdownRuntime>>,
+        write_runtime: Option<Arc<BackgroundShutdownRuntime>>,
+        user_runtime_handle: Handle,
+    ) -> Self {
+        let read_runtime_handle = read_runtime
+            .as_ref()
+            .map(|rt| rt.handle().clone())
+            .unwrap_or(user_runtime_handle.clone());
+        let write_runtime_handle = write_runtime
+            .as_ref()
+            .map(|rt| rt.handle().clone())
+            .unwrap_or(user_runtime_handle.clone());
+        Self {
+            inner: Arc::new(RuntimeInner {
+                _read_runtime: read_runtime,
+                _write_runtime: write_runtime,
+                read_runtime_handle: read_runtime_handle.into(),
+                write_runtime_handle: write_runtime_handle.into(),
+                user_runtime_handle: user_runtime_handle.into(),
+            }),
+        }
+    }
+
+    /// Create a new runtime with current runtime env only.
+    pub fn current() -> Self {
+        Self {
+            inner: Arc::new(RuntimeInner {
+                _read_runtime: None,
+                _write_runtime: None,
+                read_runtime_handle: Handle::current().into(),
+                write_runtime_handle: Handle::current().into(),
+                user_runtime_handle: Handle::current().into(),
+            }),
+        }
+    }
+
+    /// Get the non-clonable read runtime handle.
+    pub fn read(&self) -> &SingletonHandle {
+        &self.inner.read_runtime_handle
+    }
+
+    /// Get the non-clonable write runtime handle.
+    pub fn write(&self) -> &SingletonHandle {
+        &self.inner.write_runtime_handle
+    }
+
+    /// Get the non-clonable user runtime handle.
+    pub fn user(&self) -> &SingletonHandle {
+        &self.inner.user_runtime_handle
+    }
+}
diff --git a/foyer-storage/src/store.rs b/foyer-storage/src/store.rs
index e9ecb23e..88a04d00 100644
--- a/foyer-storage/src/store.rs
+++ b/foyer-storage/src/store.rs
@@ -38,6 +38,7 @@ use crate::{
         utils::{AdmitAllPicker, FifoPicker, InvalidRatioPicker, RejectAllPicker},
         AdmissionPicker, EvictionPicker, ReinsertionPicker,
     },
+    runtime::Runtime,
     serde::EntrySerializer,
     small::generic::GenericSmallStorageConfig,
     statistics::Statistics,
@@ -72,12 +73,7 @@ where
 
     compression: Compression,
 
-    read_runtime: Option<Arc<BackgroundShutdownRuntime>>,
-    write_runtime: Option<Arc<BackgroundShutdownRuntime>>,
-
-    read_runtime_handle: Handle,
-    write_runtime_handle: Handle,
-    user_runtime_handle: Handle,
+    runtime: Runtime,
 
     statistics: Arc<Statistics>,
     metrics: Arc<Metrics>,
@@ -95,11 +91,7 @@ where
             .field("engine", &self.inner.engine)
             .field("admission_picker", &self.inner.admission_picker)
             .field("compression", &self.inner.compression)
-            .field("read_runtime", &self.inner.read_runtime)
-            .field("write_runtime", &self.inner.write_runtime)
-            .field("read_runtime_handle", &self.inner.read_runtime_handle)
-            .field("write_runtime_handle", &self.inner.write_runtime_handle)
-            .field("user_runtime_handle", &self.inner.user_runtime_handle)
+            .field("runtimes", &self.inner.runtime)
             .finish()
     }
 }
@@ -156,7 +148,7 @@ where
     {
         let hash = self.inner.memory.hash(key);
         let future = self.inner.engine.load(hash);
-        match self.inner.read_runtime_handle.spawn(future).await.unwrap() {
+        match self.inner.runtime.read().spawn(future).await.unwrap() {
             Ok(Some((k, v))) if k.borrow() == key => Ok(Some((k, v))),
             Ok(_) => Ok(None),
             Err(e) => Err(e),
@@ -196,12 +188,17 @@ where
     }
 
     /// Get the runtime handles.
-    pub fn runtimes(&self) -> RuntimeHandles<'_> {
-        RuntimeHandles {
-            read_runtime_handle: &self.inner.read_runtime_handle,
-            write_runtime_handle: &self.inner.write_runtime_handle,
-            user_runtime_handle: &self.inner.user_runtime_handle,
-        }
+    #[deprecated(
+        since = "0.11.5",
+        note = "The function will be renamed to \"runtime()\", use it instead."
+    )]
+    pub fn runtimes(&self) -> &Runtime {
+        &self.inner.runtime
+    }
+
+    /// Get the runtime.
+    pub fn runtime(&self) -> &Runtime {
+        &self.inner.runtime
     }
 }
 
@@ -311,16 +308,6 @@ pub enum RuntimeConfig {
     },
 }
 
-/// Runtime handles.
-pub struct RuntimeHandles<'a> {
-    /// Runtime handle for reads.
-    pub read_runtime_handle: &'a Handle,
-    /// Runtime handle for writes.
-    pub write_runtime_handle: &'a Handle,
-    /// User runtime handle.
-    pub user_runtime_handle: &'a Handle,
-}
-
 /// The builder of the disk cache.
 pub struct StoreBuilder<K, V, S = RandomState>
 where
@@ -582,19 +569,15 @@ where
             Ok::<_, Error>(Arc::new(runtime))
         };
 
-        let (read_runtime, write_runtime, read_runtime_handle, write_runtime_handle) = match self.runtime_config {
+        let user_runtime_handle = Handle::current();
+        let (read_runtime, write_runtime) = match self.runtime_config {
             RuntimeConfig::Disabled => {
                 tracing::warn!("[store]: Dedicated runtime is disabled");
-                (None, None, Handle::current(), Handle::current())
+                (None, None)
             }
             RuntimeConfig::Unified(runtime_config) => {
                 let runtime = build_runtime(&runtime_config, "unified")?;
-                (
-                    Some(runtime.clone()),
-                    Some(runtime.clone()),
-                    runtime.handle().clone(),
-                    runtime.handle().clone(),
-                )
+                (Some(runtime.clone()), Some(runtime.clone()))
             }
             RuntimeConfig::Separated {
                 read_runtime_config,
@@ -602,24 +585,15 @@ where
             } => {
                 let read_runtime = build_runtime(&read_runtime_config, "read")?;
                 let write_runtime = build_runtime(&write_runtime_config, "write")?;
-                let read_runtime_handle = read_runtime.handle().clone();
-                let write_runtime_handle = write_runtime.handle().clone();
-                (
-                    Some(read_runtime),
-                    Some(write_runtime),
-                    read_runtime_handle,
-                    write_runtime_handle,
-                )
+                (Some(read_runtime), Some(write_runtime))
             }
         };
-        let user_runtime_handle = Handle::current();
+        let runtime = Runtime::new(read_runtime, write_runtime, user_runtime_handle);
 
         let engine = {
             let statistics = statistics.clone();
             let metrics = metrics.clone();
-            let write_runtime_handle = write_runtime_handle.clone();
-            let read_runtime_handle = read_runtime_handle.clone();
-            let user_runtime_handle = user_runtime_handle.clone();
+            let runtime = runtime.clone();
             // Use the user runtime to open engine.
             tokio::spawn(async move {
                 match self.device_config {
@@ -633,7 +607,7 @@ where
                         let device = match Monitored::open(MonitoredOptions {
                             options,
                             metrics: metrics.clone(),
-                        })
+                        }, runtime.clone())
                         .await {
                             Ok(device) => device,
                             Err(e) =>return Err(e),
@@ -658,9 +632,7 @@ where
                                     tombstone_log_config: self.tombstone_log_config,
                                     buffer_threshold: self.buffer_pool_size,
                                     statistics: statistics.clone(),
-                                    write_runtime_handle,
-                                    read_runtime_handle,
-                                    user_runtime_handle,
+                                    runtime,
                                     marker: PhantomData,
                                 }))
                                 .await
@@ -701,9 +673,7 @@ where
                                         tombstone_log_config: self.tombstone_log_config,
                                         buffer_threshold: self.buffer_pool_size,
                                         statistics: statistics.clone(),
-                                        write_runtime_handle,
-                                        read_runtime_handle,
-                                        user_runtime_handle,
+                                        runtime,
                                         marker: PhantomData,
                                     },
                                     load_order,
@@ -721,11 +691,7 @@ where
             engine,
             admission_picker,
             compression,
-            read_runtime,
-            write_runtime,
-            read_runtime_handle,
-            write_runtime_handle,
-            user_runtime_handle,
+            runtime,
             statistics,
             metrics,
         };
diff --git a/foyer/src/hybrid/cache.rs b/foyer/src/hybrid/cache.rs
index 9baf52c1..5dbc3e20 100644
--- a/foyer/src/hybrid/cache.rs
+++ b/foyer/src/hybrid/cache.rs
@@ -485,7 +485,7 @@ where
             context,
             || {
                 let metrics = self.metrics.clone();
-                let user_runtime_handle = self.storage().runtimes().user_runtime_handle.clone();
+                let runtime = self.storage().runtime().clone();
 
                 async move {
                     match store.load(&key).await.map_err(anyhow::Error::from) {
@@ -502,7 +502,8 @@ where
                     metrics.hybrid_miss.increment(1);
                     metrics.hybrid_miss_duration.record(now.elapsed());
 
-                    user_runtime_handle
+                    runtime
+                        .user()
                         .spawn(
                             future
                                 .map(|res| Diversion {
@@ -515,7 +516,7 @@ where
                         .unwrap()
                 }
             },
-            self.storage().runtimes().read_runtime_handle,
+            self.storage().runtime().read(),
         );
 
         if inner.state() == FetchState::Hit {
diff --git a/foyer/src/prelude.rs b/foyer/src/prelude.rs
index fd470ffd..d8af4d38 100644
--- a/foyer/src/prelude.rs
+++ b/foyer/src/prelude.rs
@@ -27,8 +27,8 @@ pub use storage::{
     AdmissionPicker, AdmitAllPicker, Compression, Dev, DevExt, DevOptions, DeviceStats, DirectFileDevice,
     DirectFileDeviceOptions, DirectFileDeviceOptionsBuilder, DirectFsDevice, DirectFsDeviceOptions,
     DirectFsDeviceOptionsBuilder, EvictionPicker, FifoPicker, InvalidRatioPicker, RateLimitPicker, RecoverMode,
-    ReinsertionPicker, RejectAllPicker, RuntimeConfig, RuntimeHandles, Storage, Store, StoreBuilder,
-    TokioRuntimeConfig, TombstoneLogConfigBuilder,
+    ReinsertionPicker, RejectAllPicker, Runtime, RuntimeConfig, Storage, Store, StoreBuilder, TokioRuntimeConfig,
+    TombstoneLogConfigBuilder,
 };
 
 pub use crate::hybrid::{

From 6ac0e837c6309ae596049d66d6cfad1d338eccf6 Mon Sep 17 00:00:00 2001
From: Croxx <mrcroxx@outlook.com>
Date: Wed, 25 Sep 2024 00:41:46 +0800
Subject: [PATCH 15/53] chore: release foyer 0.11.5 (#737)

Signed-off-by: MrCroxx <mrcroxx@outlook.com>
---
 CHANGELOG.md               | 17 +++++++++++++++++
 foyer-bench/Cargo.toml     |  4 ++--
 foyer-common/Cargo.toml    |  2 +-
 foyer-intrusive/Cargo.toml |  4 ++--
 foyer-memory/Cargo.toml    |  6 +++---
 foyer-storage/Cargo.toml   |  6 +++---
 foyer/Cargo.toml           |  8 ++++----
 7 files changed, 32 insertions(+), 15 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 9ffc44c3..d42ef499 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -9,6 +9,23 @@ date: 2023-05-12T11:02:09+08:00
 
 <!-- truncate -->
 
+## 2024-09-25
+
+### Releases
+
+| crate | version |
+| - | - |
+| foyer | 0.11.5 |
+| foyer-common | 0.9.5 |
+| foyer-intrusive | 0.9.5 |
+| foyer-memory | 0.7.5 |
+| foyer-storage | 0.10.5 |
+| foyer-bench | 0.3.5 |
+
+### Changes
+
+- Fix panic on dropping the hybrid cache. #736
+
 ## 2024-09-24
 
 ### Releases
diff --git a/foyer-bench/Cargo.toml b/foyer-bench/Cargo.toml
index d34cfcf6..22d397fc 100644
--- a/foyer-bench/Cargo.toml
+++ b/foyer-bench/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "foyer-bench"
-version = "0.3.4"
+version = "0.3.5"
 edition = "2021"
 authors = ["MrCroxx <mrcroxx@outlook.com>"]
 description = "bench tool for foyer - the hybrid cache for Rust"
@@ -17,7 +17,7 @@ clap = { workspace = true }
 console-subscriber = { version = "0.4", optional = true }
 fastrace = { workspace = true, optional = true }
 fastrace-jaeger = { workspace = true, optional = true }
-foyer = { version = "0.11.4", path = "../foyer" }
+foyer = { version = "0.11.5", path = "../foyer" }
 futures = "0.3"
 hdrhistogram = "7"
 itertools = { workspace = true }
diff --git a/foyer-common/Cargo.toml b/foyer-common/Cargo.toml
index b8da41c3..3e64b1f5 100644
--- a/foyer-common/Cargo.toml
+++ b/foyer-common/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "foyer-common"
-version = "0.9.4"
+version = "0.9.5"
 edition = "2021"
 authors = ["MrCroxx <mrcroxx@outlook.com>"]
 description = "common components for foyer - the hybrid cache for Rust"
diff --git a/foyer-intrusive/Cargo.toml b/foyer-intrusive/Cargo.toml
index 7963f58e..a39927dd 100644
--- a/foyer-intrusive/Cargo.toml
+++ b/foyer-intrusive/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "foyer-intrusive"
-version = "0.9.4"
+version = "0.9.5"
 edition = "2021"
 authors = ["MrCroxx <mrcroxx@outlook.com>"]
 description = "intrusive data structures for foyer - the hybrid cache for Rust"
@@ -11,7 +11,7 @@ readme = "../README.md"
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
-foyer-common = { version = "0.9.4", path = "../foyer-common" }
+foyer-common = { version = "0.9.5", path = "../foyer-common" }
 itertools = { workspace = true }
 
 [features]
diff --git a/foyer-memory/Cargo.toml b/foyer-memory/Cargo.toml
index 2aad532c..dbb2a3a5 100644
--- a/foyer-memory/Cargo.toml
+++ b/foyer-memory/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "foyer-memory"
-version = "0.7.4"
+version = "0.7.5"
 edition = "2021"
 authors = ["MrCroxx <mrcroxx@outlook.com>"]
 description = "memory cache for foyer - the hybrid cache for Rust"
@@ -15,8 +15,8 @@ ahash = "0.8"
 bitflags = "2"
 cmsketch = "0.2.1"
 fastrace = { workspace = true }
-foyer-common = { version = "0.9.4", path = "../foyer-common" }
-foyer-intrusive = { version = "0.9.4", path = "../foyer-intrusive" }
+foyer-common = { version = "0.9.5", path = "../foyer-common" }
+foyer-intrusive = { version = "0.9.5", path = "../foyer-intrusive" }
 futures = "0.3"
 hashbrown = "0.14"
 itertools = { workspace = true }
diff --git a/foyer-storage/Cargo.toml b/foyer-storage/Cargo.toml
index 7c910ebb..b5863a39 100644
--- a/foyer-storage/Cargo.toml
+++ b/foyer-storage/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "foyer-storage"
-version = "0.10.4"
+version = "0.10.5"
 edition = "2021"
 authors = ["MrCroxx <mrcroxx@outlook.com>"]
 description = "storage engine for foyer - the hybrid cache for Rust"
@@ -26,8 +26,8 @@ clap = { workspace = true }
 either = "1"
 fastrace = { workspace = true }
 flume = "0.11"
-foyer-common = { version = "0.9.4", path = "../foyer-common" }
-foyer-memory = { version = "0.7.4", path = "../foyer-memory" }
+foyer-common = { version = "0.9.5", path = "../foyer-common" }
+foyer-memory = { version = "0.7.5", path = "../foyer-memory" }
 fs4 = "0.9.1"
 futures = "0.3"
 itertools = { workspace = true }
diff --git a/foyer/Cargo.toml b/foyer/Cargo.toml
index e4f4904f..33a2d4f0 100644
--- a/foyer/Cargo.toml
+++ b/foyer/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "foyer"
-version = "0.11.4"
+version = "0.11.5"
 edition = "2021"
 authors = ["MrCroxx <mrcroxx@outlook.com>"]
 description = "Hybrid cache for Rust"
@@ -15,9 +15,9 @@ rust-version = "1.81.0"
 ahash = "0.8"
 anyhow = "1"
 fastrace = { workspace = true }
-foyer-common = { version = "0.9.4", path = "../foyer-common" }
-foyer-memory = { version = "0.7.4", path = "../foyer-memory" }
-foyer-storage = { version = "0.10.4", path = "../foyer-storage" }
+foyer-common = { version = "0.9.5", path = "../foyer-common" }
+foyer-memory = { version = "0.7.5", path = "../foyer-memory" }
+foyer-storage = { version = "0.10.5", path = "../foyer-storage" }
 futures = "0.3"
 pin-project = "1"
 tokio = { workspace = true }

From b6524e28e5b5faaab4a7943480d65ac49fd397df Mon Sep 17 00:00:00 2001
From: Croxx <mrcroxx@outlook.com>
Date: Wed, 25 Sep 2024 11:07:35 +0800
Subject: [PATCH 16/53] chore: update license checker version (#739)

Signed-off-by: MrCroxx <mrcroxx@outlook.com>
---
 .github/workflows/license_check.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/license_check.yml b/.github/workflows/license_check.yml
index 762f4294..bf30028c 100644
--- a/.github/workflows/license_check.yml
+++ b/.github/workflows/license_check.yml
@@ -16,4 +16,4 @@ jobs:
     steps:
     - uses: actions/checkout@v3
     - name: Check License Header
-      uses: apache/skywalking-eyes/header@df70871af1a8109c9a5b1dc824faaf65246c5236
+      uses: apache/skywalking-eyes/header@v0.6.0

From 656055ab1fd1ac60e0c8be323bfdfb446863a22d Mon Sep 17 00:00:00 2001
From: Croxx <mrcroxx@outlook.com>
Date: Thu, 26 Sep 2024 15:10:40 +0800
Subject: [PATCH 17/53] feat: introduce small object disk cache (#658)

* stash, todo: rebase me

* fix: fix sodc checksum and test

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

* feat: make store builder support mixed disk cache engine

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

* feat: impl the new builder, update bench and examples

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

* fix: make clippy happy

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

* fix: fix engine clap

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

* refactor: update eviction picker API

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

* fix: fix small batch copy

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

* fix: upgrade deps

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

* fix: fix set panic on invalid len

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

* feat: impl better bloom filter

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

* feat: add options for foyer-bench to control small engine

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

* refactor: rename a const

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

* test: try add test for small batch

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

* chore: make ffmt happy

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

* refactor: refine the small engine to fit the new design

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

* chore: make ffmt happy

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

* refactor: use 12B entry header for small entry

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

* fix: fix stats for small object disk cache

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

* test: add ci for mixed engine

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

* chore: fix ci

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

---------

Signed-off-by: MrCroxx <mrcroxx@outlook.com>
---
 .github/workflows/ci.yml                |  16 +-
 examples/hybrid.rs                      |   4 +-
 examples/hybrid_full.rs                 |  41 +-
 examples/tail_based_tracing.rs          |   4 +-
 foyer-bench/src/main.rs                 |  54 +-
 foyer-common/src/runtime.rs             |  74 +++
 foyer-storage/Cargo.toml                |   2 +
 foyer-storage/src/device/direct_file.rs |   2 +-
 foyer-storage/src/engine.rs             |  86 +--
 foyer-storage/src/io_buffer_pool.rs     |   2 +
 foyer-storage/src/large/batch.rs        |   8 +-
 foyer-storage/src/large/flusher.rs      |   4 +-
 foyer-storage/src/large/generic.rs      |  12 +-
 foyer-storage/src/large/reclaimer.rs    |   2 +-
 foyer-storage/src/picker/mod.rs         |   4 +-
 foyer-storage/src/picker/utils.rs       |   5 +-
 foyer-storage/src/prelude.rs            |   5 +-
 foyer-storage/src/serde.rs              |  12 +-
 foyer-storage/src/small/batch.rs        | 329 +++++++++++
 foyer-storage/src/small/bloom_filter.rs | 201 +++++++
 foyer-storage/src/small/flusher.rs      | 225 ++++++++
 foyer-storage/src/small/generic.rs      | 203 ++++++-
 foyer-storage/src/small/mod.rs          |   6 +
 foyer-storage/src/small/serde.rs        |  97 ++++
 foyer-storage/src/small/set.rs          | 537 ++++++++++++++++++
 foyer-storage/src/small/set_manager.rs  | 253 +++++++++
 foyer-storage/src/storage/either.rs     |   3 +-
 foyer-storage/src/store.rs              | 703 +++++++++++++++---------
 foyer-storage/tests/storage_test.rs     |  16 +-
 foyer/src/hybrid/builder.rs             | 179 +-----
 foyer/src/hybrid/cache.rs               |   6 +-
 foyer/src/prelude.rs                    |   6 +-
 32 files changed, 2544 insertions(+), 557 deletions(-)
 create mode 100644 foyer-storage/src/small/batch.rs
 create mode 100644 foyer-storage/src/small/bloom_filter.rs
 create mode 100644 foyer-storage/src/small/flusher.rs
 create mode 100644 foyer-storage/src/small/serde.rs
 create mode 100644 foyer-storage/src/small/set.rs
 create mode 100644 foyer-storage/src/small/set_manager.rs

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index c14a51b7..fe8163a4 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -200,7 +200,9 @@ jobs:
           CI: true
         run: |
           mkdir -p $GITHUB_WORKSPACE/foyer-data/foyer-bench/codecov
-          cargo llvm-cov --no-report run --package foyer-bench --bin foyer-bench --features "strict_assertions,sanity" -- --dir $GITHUB_WORKSPACE/foyer-data/foyer-bench/codecov --mem 16MiB --disk 256MiB --region-size 16MiB --get-range 1000 --w-rate 1MiB --r-rate 1MiB --admission-rate-limit 10MiB --time 60
+          cargo llvm-cov --no-report run --package foyer-bench --bin foyer-bench --features "strict_assertions,sanity" -- --dir $GITHUB_WORKSPACE/foyer-data/foyer-bench/codecov --engine large --mem 16MiB --disk 256MiB --region-size 16MiB --get-range 1000 --w-rate 1MiB --r-rate 1MiB --admission-rate-limit 10MiB --entry-size-min 2KiB --entry-size-max 128KiB --time 60
+          cargo llvm-cov --no-report run --package foyer-bench --bin foyer-bench --features "strict_assertions,sanity" -- --dir $GITHUB_WORKSPACE/foyer-data/foyer-bench/codecov --engine small --mem 4MiB --disk 256MiB --region-size 16MiB --get-range 1000 --w-rate 1MiB --r-rate 1MiB --admission-rate-limit 1MiB --entry-size-min 1KiB --entry-size-max 24KiB --time 60
+          cargo llvm-cov --no-report run --package foyer-bench --bin foyer-bench --features "strict_assertions,sanity" -- --dir $GITHUB_WORKSPACE/foyer-data/foyer-bench/codecov --engine mixed=0.1 --mem 16MiB --disk 256MiB --region-size 16MiB --get-range 1000 --w-rate 1MiB --r-rate 1MiB --admission-rate-limit 10MiB --entry-size-min 1KiB --entry-size-max 128KiB --time 60
       - name: Generate codecov report
         run: |
           cargo llvm-cov report --lcov --output-path lcov.info
@@ -240,7 +242,9 @@ jobs:
         run: |-
           cargo build --all --features deadlock
           mkdir -p $GITHUB_WORKSPACE/foyer-data/foyer-storage/deadlock
-          timeout 2m ./target/debug/foyer-bench --dir $GITHUB_WORKSPACE/foyer-data/foyer-bench/deadlock --mem 16MiB --disk 256MiB --region-size 16MiB --get-range 1000 --w-rate 1MiB --r-rate 1MiB --admission-rate-limit 10MiB --time 60
+          timeout 2m ./target/debug/foyer-bench --dir $GITHUB_WORKSPACE/foyer-data/foyer-bench/deadlock --engine large --mem 16MiB --disk 256MiB --region-size 16MiB --get-range 1000 --w-rate 1MiB --r-rate 1MiB --admission-rate-limit 10MiB --entry-size-min 2KiB --entry-size-max 128KiB --time 60
+          timeout 2m ./target/debug/foyer-bench --dir $GITHUB_WORKSPACE/foyer-data/foyer-bench/deadlock --engine small --mem 4MiB --disk 256MiB --region-size 16MiB --get-range 1000 --w-rate 1MiB --r-rate 1MiB --admission-rate-limit 1MiB --entry-size-min 1KiB --entry-size-max 24KiB --time 60
+          timeout 2m ./target/debug/foyer-bench --dir $GITHUB_WORKSPACE/foyer-data/foyer-bench/deadlock --engine mixed=0.1 --mem 16MiB --disk 256MiB --region-size 16MiB --get-range 1000 --w-rate 1MiB --r-rate 1MiB --admission-rate-limit 10MiB --entry-size-min 1KiB --entry-size-max 128KiB --time 60
   asan:
     name: run with address saniziter
     runs-on: ubuntu-latest
@@ -278,7 +282,9 @@ jobs:
         run: |-
           cargo +${{ env.RUST_TOOLCHAIN_NIGHTLY }} build --all --target x86_64-unknown-linux-gnu
           mkdir -p $GITHUB_WORKSPACE/foyer-data/foyer-bench/asan
-          timeout 2m ./target/x86_64-unknown-linux-gnu/debug/foyer-bench --dir $GITHUB_WORKSPACE/foyer-data/foyer-bench/asan --mem 16MiB --disk 256MiB --region-size 16MiB --get-range 1000 --w-rate 1MiB --r-rate 1MiB --admission-rate-limit 10MiB --time 60
+          timeout 2m ./target/x86_64-unknown-linux-gnu/debug/foyer-bench --dir $GITHUB_WORKSPACE/foyer-data/foyer-bench/asan --engine large --mem 16MiB --disk 256MiB --region-size 16MiB --get-range 1000 --w-rate 1MiB --r-rate 1MiB --admission-rate-limit 10MiB --entry-size-min 2KiB --entry-size-max 128KiB --time 60
+          timeout 2m ./target/x86_64-unknown-linux-gnu/debug/foyer-bench --dir $GITHUB_WORKSPACE/foyer-data/foyer-bench/asan --engine small --mem 4MiB --disk 256MiB --region-size 16MiB --get-range 1000 --w-rate 1MiB --r-rate 1MiB --admission-rate-limit 1MiB --entry-size-min 1KiB --entry-size-max 24KiB --time 60
+          timeout 2m ./target/x86_64-unknown-linux-gnu/debug/foyer-bench --dir $GITHUB_WORKSPACE/foyer-data/foyer-bench/asan --engine mixed=0.1 --mem 16MiB --disk 256MiB --region-size 16MiB --get-range 1000 --w-rate 1MiB --r-rate 1MiB --admission-rate-limit 10MiB --entry-size-min 1KiB --entry-size-max 128KiB --time 60
       - name: Prepare Artifacts on Failure
         if: ${{ failure() }}
         run: |-
@@ -326,7 +332,9 @@ jobs:
         run: |-
           cargo +${{ env.RUST_TOOLCHAIN_NIGHTLY }} build --all --target x86_64-unknown-linux-gnu
           mkdir -p $GITHUB_WORKSPACE/foyer-data/foyer-bench/lsan
-          timeout 2m ./target/x86_64-unknown-linux-gnu/debug/foyer-bench --dir $GITHUB_WORKSPACE/foyer-data/foyer-bench/lsan --mem 16MiB --disk 256MiB --region-size 16MiB --get-range 1000 --w-rate 1MiB --r-rate 1MiB --admission-rate-limit 10MiB --time 60
+          timeout 2m ./target/x86_64-unknown-linux-gnu/debug/foyer-bench --dir $GITHUB_WORKSPACE/foyer-data/foyer-bench/lsan --engine large --mem 16MiB --disk 256MiB --region-size 16MiB --get-range 1000 --w-rate 1MiB --r-rate 1MiB --admission-rate-limit 10MiB --entry-size-min 2KiB --entry-size-max 128KiB --time 60
+          timeout 2m ./target/x86_64-unknown-linux-gnu/debug/foyer-bench --dir $GITHUB_WORKSPACE/foyer-data/foyer-bench/lsan --engine small --mem 4MiB --disk 256MiB --region-size 16MiB --get-range 1000 --w-rate 1MiB --r-rate 1MiB --admission-rate-limit 1MiB --entry-size-min 1KiB --entry-size-max 24KiB --time 60
+          timeout 2m ./target/x86_64-unknown-linux-gnu/debug/foyer-bench --dir $GITHUB_WORKSPACE/foyer-data/foyer-bench/lsan --engine mixed=0.1 --mem 16MiB --disk 256MiB --region-size 16MiB --get-range 1000 --w-rate 1MiB --r-rate 1MiB --admission-rate-limit 10MiB --entry-size-min 1KiB --entry-size-max 128KiB --time 60
       - name: Prepare Artifacts on Failure
         if: ${{ failure() }}
         run: |-
diff --git a/examples/hybrid.rs b/examples/hybrid.rs
index e8411295..2bf7a450 100644
--- a/examples/hybrid.rs
+++ b/examples/hybrid.rs
@@ -12,7 +12,7 @@
 //  See the License for the specific language governing permissions and
 //  limitations under the License.
 
-use foyer::{DirectFsDeviceOptionsBuilder, HybridCache, HybridCacheBuilder};
+use foyer::{DirectFsDeviceOptionsBuilder, Engine, HybridCache, HybridCacheBuilder};
 
 #[tokio::main]
 async fn main() -> anyhow::Result<()> {
@@ -20,7 +20,7 @@ async fn main() -> anyhow::Result<()> {
 
     let hybrid: HybridCache<u64, String> = HybridCacheBuilder::new()
         .memory(64 * 1024 * 1024)
-        .storage()
+        .storage(Engine::Large) // use large object disk cache engine only
         .with_device_config(
             DirectFsDeviceOptionsBuilder::new(dir.path())
                 .with_capacity(256 * 1024 * 1024)
diff --git a/examples/hybrid_full.rs b/examples/hybrid_full.rs
index 97f98500..60cafb8e 100644
--- a/examples/hybrid_full.rs
+++ b/examples/hybrid_full.rs
@@ -17,8 +17,8 @@ use std::sync::Arc;
 use anyhow::Result;
 use chrono::Datelike;
 use foyer::{
-    DirectFsDeviceOptionsBuilder, FifoPicker, HybridCache, HybridCacheBuilder, LruConfig, RateLimitPicker, RecoverMode,
-    RuntimeConfig, TokioRuntimeConfig, TombstoneLogConfigBuilder,
+    DirectFsDeviceOptionsBuilder, Engine, FifoPicker, HybridCache, HybridCacheBuilder, LargeEngineOptions, LruConfig,
+    RateLimitPicker, RecoverMode, RuntimeConfig, SmallEngineOptions, TokioRuntimeConfig, TombstoneLogConfigBuilder,
 };
 use tempfile::tempdir;
 
@@ -35,7 +35,7 @@ async fn main() -> Result<()> {
         .with_object_pool_capacity(1024)
         .with_hash_builder(ahash::RandomState::default())
         .with_weighter(|_key, value: &String| value.len())
-        .storage()
+        .storage(Engine::Mixed(0.1))
         .with_device_config(
             DirectFsDeviceOptionsBuilder::new(dir.path())
                 .with_capacity(64 * 1024 * 1024)
@@ -43,22 +43,9 @@ async fn main() -> Result<()> {
                 .build(),
         )
         .with_flush(true)
-        .with_indexer_shards(64)
         .with_recover_mode(RecoverMode::Quiet)
-        .with_recover_concurrency(8)
-        .with_flushers(2)
-        .with_reclaimers(2)
-        .with_buffer_pool_size(256 * 1024 * 1024)
-        .with_clean_region_threshold(4)
-        .with_eviction_pickers(vec![Box::<FifoPicker>::default()])
         .with_admission_picker(Arc::new(RateLimitPicker::new(100 * 1024 * 1024)))
-        .with_reinsertion_picker(Arc::new(RateLimitPicker::new(10 * 1024 * 1024)))
         .with_compression(foyer::Compression::Lz4)
-        .with_tombstone_log_config(
-            TombstoneLogConfigBuilder::new(dir.path().join("tombstone-log-file"))
-                .with_flush(true)
-                .build(),
-        )
         .with_runtime_config(RuntimeConfig::Separated {
             read_runtime_config: TokioRuntimeConfig {
                 worker_threads: 4,
@@ -69,6 +56,28 @@ async fn main() -> Result<()> {
                 max_blocking_threads: 8,
             },
         })
+        .with_large_object_disk_cache_options(
+            LargeEngineOptions::new()
+                .with_indexer_shards(64)
+                .with_recover_concurrency(8)
+                .with_flushers(2)
+                .with_reclaimers(2)
+                .with_buffer_pool_size(256 * 1024 * 1024)
+                .with_clean_region_threshold(4)
+                .with_eviction_pickers(vec![Box::<FifoPicker>::default()])
+                .with_reinsertion_picker(Arc::new(RateLimitPicker::new(10 * 1024 * 1024)))
+                .with_tombstone_log_config(
+                    TombstoneLogConfigBuilder::new(dir.path().join("tombstone-log-file"))
+                        .with_flush(true)
+                        .build(),
+                ),
+        )
+        .with_small_object_disk_cache_options(
+            SmallEngineOptions::new()
+                .with_set_size(16 * 1024)
+                .with_set_cache_capacity(64)
+                .with_flushers(2),
+        )
         .build()
         .await?;
 
diff --git a/examples/tail_based_tracing.rs b/examples/tail_based_tracing.rs
index e74a0cef..171aebea 100644
--- a/examples/tail_based_tracing.rs
+++ b/examples/tail_based_tracing.rs
@@ -14,7 +14,7 @@
 
 use std::time::Duration;
 
-use foyer::{DirectFsDeviceOptionsBuilder, HybridCache, HybridCacheBuilder};
+use foyer::{DirectFsDeviceOptionsBuilder, Engine, HybridCache, HybridCacheBuilder};
 
 #[cfg(feature = "jaeger")]
 fn init_jaeger_exporter() {
@@ -70,7 +70,7 @@ async fn main() -> anyhow::Result<()> {
 
     let hybrid: HybridCache<u64, String> = HybridCacheBuilder::new()
         .memory(64 * 1024 * 1024)
-        .storage()
+        .storage(Engine::Large)
         .with_device_config(
             DirectFsDeviceOptionsBuilder::new(dir.path())
                 .with_capacity(256 * 1024 * 1024)
diff --git a/foyer-bench/src/main.rs b/foyer-bench/src/main.rs
index f72c30ea..87587bb5 100644
--- a/foyer-bench/src/main.rs
+++ b/foyer-bench/src/main.rs
@@ -34,9 +34,9 @@ use analyze::{analyze, monitor, Metrics};
 use bytesize::ByteSize;
 use clap::{builder::PossibleValuesParser, ArgGroup, Parser};
 use foyer::{
-    Compression, DirectFileDeviceOptionsBuilder, DirectFsDeviceOptionsBuilder, FifoConfig, FifoPicker, HybridCache,
-    HybridCacheBuilder, InvalidRatioPicker, LfuConfig, LruConfig, RateLimitPicker, RecoverMode, RuntimeConfig,
-    S3FifoConfig, TokioRuntimeConfig, TracingConfig,
+    Compression, DirectFileDeviceOptionsBuilder, DirectFsDeviceOptionsBuilder, Engine, FifoConfig, FifoPicker,
+    HybridCache, HybridCacheBuilder, InvalidRatioPicker, LargeEngineOptions, LfuConfig, LruConfig, RateLimitPicker,
+    RecoverMode, RuntimeConfig, S3FifoConfig, SmallEngineOptions, TokioRuntimeConfig, TracingConfig,
 };
 use futures::future::join_all;
 use itertools::Itertools;
@@ -204,6 +204,11 @@ pub struct Args {
     #[arg(long, value_enum, default_value_t = Compression::None)]
     compression: Compression,
 
+    // TODO(MrCroxx): use mixed engine by default.
+    /// Disk cache engine.
+    #[arg(long, default_value_t = Engine::Large)]
+    engine: Engine,
+
     /// Time-series operation distribution.
     ///
     /// Available values: "none", "uniform", "zipf".
@@ -233,6 +238,12 @@ pub struct Args {
     #[arg(long, value_parser = PossibleValuesParser::new(["lru", "lfu", "fifo", "s3fifo"]), default_value = "lru")]
     eviction: String,
 
+    #[arg(long, default_value_t = ByteSize::kib(16))]
+    set_size: ByteSize,
+
+    #[arg(long, default_value_t = 64)]
+    set_cache_capacity: usize,
+
     /// Record insert trace threshold. Only effective with "mtrace" feature.
     #[arg(long, default_value_t = 1000 * 1000)]
     trace_insert_us: usize,
@@ -448,7 +459,7 @@ async fn benchmark(args: Args) {
 
     let mut builder = builder
         .with_weighter(|_: &u64, value: &Value| u64::BITS as usize / 8 + value.len())
-        .storage();
+        .storage(args.engine);
 
     builder = match (args.file.as_ref(), args.dir.as_ref()) {
         (Some(file), None) => builder.with_device_config(
@@ -468,15 +479,7 @@ async fn benchmark(args: Args) {
 
     builder = builder
         .with_flush(args.flush)
-        .with_indexer_shards(args.shards)
         .with_recover_mode(args.recover_mode)
-        .with_recover_concurrency(args.recover_concurrency)
-        .with_flushers(args.flushers)
-        .with_reclaimers(args.reclaimers)
-        .with_eviction_pickers(vec![
-            Box::new(InvalidRatioPicker::new(args.invalid_ratio)),
-            Box::<FifoPicker>::default(),
-        ])
         .with_compression(args.compression)
         .with_runtime_config(match args.runtime.as_str() {
             "disabled" => RuntimeConfig::Disabled,
@@ -497,20 +500,39 @@ async fn benchmark(args: Args) {
             _ => unreachable!(),
         });
 
+    let mut large = LargeEngineOptions::new()
+        .with_indexer_shards(args.shards)
+        .with_recover_concurrency(args.recover_concurrency)
+        .with_flushers(args.flushers)
+        .with_reclaimers(args.reclaimers)
+        .with_eviction_pickers(vec![
+            Box::new(InvalidRatioPicker::new(args.invalid_ratio)),
+            Box::<FifoPicker>::default(),
+        ]);
+
+    let small = SmallEngineOptions::new()
+        .with_flushers(args.flushers)
+        .with_set_size(args.set_size.as_u64() as _)
+        .with_set_cache_capacity(args.set_cache_capacity);
+
     if args.admission_rate_limit.as_u64() > 0 {
         builder =
             builder.with_admission_picker(Arc::new(RateLimitPicker::new(args.admission_rate_limit.as_u64() as _)));
     }
     if args.reinsertion_rate_limit.as_u64() > 0 {
-        builder =
-            builder.with_reinsertion_picker(Arc::new(RateLimitPicker::new(args.admission_rate_limit.as_u64() as _)));
+        large = large.with_reinsertion_picker(Arc::new(RateLimitPicker::new(args.admission_rate_limit.as_u64() as _)));
     }
 
     if args.clean_region_threshold > 0 {
-        builder = builder.with_clean_region_threshold(args.clean_region_threshold);
+        large = large.with_clean_region_threshold(args.clean_region_threshold);
     }
 
-    let hybrid = builder.build().await.unwrap();
+    let hybrid = builder
+        .with_large_object_disk_cache_options(large)
+        .with_small_object_disk_cache_options(small)
+        .build()
+        .await
+        .unwrap();
 
     #[cfg(feature = "mtrace")]
     hybrid.enable_tracing();
diff --git a/foyer-common/src/runtime.rs b/foyer-common/src/runtime.rs
index d2b269c2..a3727090 100644
--- a/foyer-common/src/runtime.rs
+++ b/foyer-common/src/runtime.rs
@@ -143,4 +143,78 @@ impl SingletonHandle {
     {
         self.0.spawn_blocking(func)
     }
+
+    /// Runs a future to completion on this `Handle`'s associated `Runtime`.
+    ///
+    /// This runs the given future on the current thread, blocking until it is
+    /// complete, and yielding its resolved result. Any tasks or timers which
+    /// the future spawns internally will be executed on the runtime.
+    ///
+    /// When this is used on a `current_thread` runtime, only the
+    /// [`Runtime::block_on`] method can drive the IO and timer drivers, but the
+    /// `Handle::block_on` method cannot drive them. This means that, when using
+    /// this method on a `current_thread` runtime, anything that relies on IO or
+    /// timers will not work unless there is another thread currently calling
+    /// [`Runtime::block_on`] on the same runtime.
+    ///
+    /// # If the runtime has been shut down
+    ///
+    /// If the `Handle`'s associated `Runtime` has been shut down (through
+    /// [`Runtime::shutdown_background`], [`Runtime::shutdown_timeout`], or by
+    /// dropping it) and `Handle::block_on` is used it might return an error or
+    /// panic. Specifically IO resources will return an error and timers will
+    /// panic. Runtime independent futures will run as normal.
+    ///
+    /// # Panics
+    ///
+    /// This function panics if the provided future panics, if called within an
+    /// asynchronous execution context, or if a timer future is executed on a
+    /// runtime that has been shut down.
+    ///
+    /// # Examples
+    ///
+    /// ```
+    /// use tokio::runtime::Runtime;
+    ///
+    /// // Create the runtime
+    /// let rt  = Runtime::new().unwrap();
+    ///
+    /// // Get a handle from this runtime
+    /// let handle = rt.handle();
+    ///
+    /// // Execute the future, blocking the current thread until completion
+    /// handle.block_on(async {
+    ///     println!("hello");
+    /// });
+    /// ```
+    ///
+    /// Or using `Handle::current`:
+    ///
+    /// ```
+    /// use tokio::runtime::Handle;
+    ///
+    /// #[tokio::main]
+    /// async fn main () {
+    ///     let handle = Handle::current();
+    ///     std::thread::spawn(move || {
+    ///         // Using Handle::block_on to run async code in the new thread.
+    ///         handle.block_on(async {
+    ///             println!("hello");
+    ///         });
+    ///     });
+    /// }
+    /// ```
+    ///
+    /// [`JoinError`]: struct@crate::task::JoinError
+    /// [`JoinHandle`]: struct@crate::task::JoinHandle
+    /// [`Runtime::block_on`]: fn@crate::runtime::Runtime::block_on
+    /// [`Runtime::shutdown_background`]: fn@crate::runtime::Runtime::shutdown_background
+    /// [`Runtime::shutdown_timeout`]: fn@crate::runtime::Runtime::shutdown_timeout
+    /// [`spawn_blocking`]: crate::task::spawn_blocking
+    /// [`tokio::fs`]: crate::fs
+    /// [`tokio::net`]: crate::net
+    /// [`tokio::time`]: crate::time
+    pub fn block_on<F: Future>(&self, future: F) -> F::Output {
+        self.0.block_on(future)
+    }
 }
diff --git a/foyer-storage/Cargo.toml b/foyer-storage/Cargo.toml
index b5863a39..d77630d3 100644
--- a/foyer-storage/Cargo.toml
+++ b/foyer-storage/Cargo.toml
@@ -33,7 +33,9 @@ futures = "0.3"
 itertools = { workspace = true }
 libc = "0.2"
 lz4 = "1.24"
+ordered_hash_map = "0.4"
 parking_lot = { version = "0.12", features = ["arc_lock"] }
+paste = "1"
 pin-project = "1"
 rand = "0.8"
 serde = { workspace = true }
diff --git a/foyer-storage/src/device/direct_file.rs b/foyer-storage/src/device/direct_file.rs
index ec02e2c2..cd6845a1 100644
--- a/foyer-storage/src/device/direct_file.rs
+++ b/foyer-storage/src/device/direct_file.rs
@@ -197,7 +197,7 @@ impl Dev for DirectFileDevice {
 
         if file.metadata().unwrap().is_file() {
             tracing::warn!(
-                "{}\n{}\n{}",
+                "{} {} {}",
                 "It seems a `DirectFileDevice` is used within a normal file system, which is inefficient.",
                 "Please use `DirectFileDevice` directly on a raw block device.",
                 "Or use `DirectFsDevice` within a normal file system.",
diff --git a/foyer-storage/src/engine.rs b/foyer-storage/src/engine.rs
index eb16abe7..6735315e 100644
--- a/foyer-storage/src/engine.rs
+++ b/foyer-storage/src/engine.rs
@@ -101,7 +101,7 @@ where
     Noop,
     Large(GenericLargeStorageConfig<K, V, S>),
     Small(GenericSmallStorageConfig<K, V, S>),
-    Combined(EitherConfig<K, V, S, GenericSmallStorage<K, V, S>, GenericLargeStorage<K, V, S>, SizeSelector<K, V, S>>),
+    Mixed(EitherConfig<K, V, S, GenericSmallStorage<K, V, S>, GenericLargeStorage<K, V, S>, SizeSelector<K, V, S>>),
 }
 
 impl<K, V, S> Debug for EngineConfig<K, V, S>
@@ -115,13 +115,13 @@ where
             Self::Noop => write!(f, "Noop"),
             Self::Large(config) => f.debug_tuple("Large").field(config).finish(),
             Self::Small(config) => f.debug_tuple("Small").field(config).finish(),
-            Self::Combined(config) => f.debug_tuple("Combined").field(config).finish(),
+            Self::Mixed(config) => f.debug_tuple("Mixed").field(config).finish(),
         }
     }
 }
 
 #[expect(clippy::type_complexity)]
-pub enum Engine<K, V, S = RandomState>
+pub enum EngineEnum<K, V, S = RandomState>
 where
     K: StorageKey,
     V: StorageValue,
@@ -133,11 +133,11 @@ where
     Large(GenericLargeStorage<K, V, S>),
     /// Small object disk cache.
     Small(GenericSmallStorage<K, V, S>),
-    /// Combined large and small object disk cache.
-    Combined(Either<K, V, S, GenericSmallStorage<K, V, S>, GenericLargeStorage<K, V, S>, SizeSelector<K, V, S>>),
+    /// Mixed large and small object disk cache.
+    Mixed(Either<K, V, S, GenericSmallStorage<K, V, S>, GenericLargeStorage<K, V, S>, SizeSelector<K, V, S>>),
 }
 
-impl<K, V, S> Debug for Engine<K, V, S>
+impl<K, V, S> Debug for EngineEnum<K, V, S>
 where
     K: StorageKey,
     V: StorageValue,
@@ -148,12 +148,12 @@ where
             Self::Noop(storage) => f.debug_tuple("Noop").field(storage).finish(),
             Self::Large(storage) => f.debug_tuple("Large").field(storage).finish(),
             Self::Small(storage) => f.debug_tuple("Small").field(storage).finish(),
-            Self::Combined(storage) => f.debug_tuple("Combined").field(storage).finish(),
+            Self::Mixed(storage) => f.debug_tuple("Mixed").field(storage).finish(),
         }
     }
 }
 
-impl<K, V, S> Clone for Engine<K, V, S>
+impl<K, V, S> Clone for EngineEnum<K, V, S>
 where
     K: StorageKey,
     V: StorageValue,
@@ -164,12 +164,12 @@ where
             Self::Noop(storage) => Self::Noop(storage.clone()),
             Self::Large(storage) => Self::Large(storage.clone()),
             Self::Small(storage) => Self::Small(storage.clone()),
-            Self::Combined(storage) => Self::Combined(storage.clone()),
+            Self::Mixed(storage) => Self::Mixed(storage.clone()),
         }
     }
 }
 
-impl<K, V, S> Storage for Engine<K, V, S>
+impl<K, V, S> Storage for EngineEnum<K, V, S>
 where
     K: StorageKey,
     V: StorageValue,
@@ -185,81 +185,81 @@ where
             EngineConfig::Noop => Ok(Self::Noop(Noop::open(()).await?)),
             EngineConfig::Large(config) => Ok(Self::Large(GenericLargeStorage::open(config).await?)),
             EngineConfig::Small(config) => Ok(Self::Small(GenericSmallStorage::open(config).await?)),
-            EngineConfig::Combined(config) => Ok(Self::Combined(Either::open(config).await?)),
+            EngineConfig::Mixed(config) => Ok(Self::Mixed(Either::open(config).await?)),
         }
     }
 
     async fn close(&self) -> Result<()> {
         match self {
-            Engine::Noop(storage) => storage.close().await,
-            Engine::Large(storage) => storage.close().await,
-            Engine::Small(storage) => storage.close().await,
-            Engine::Combined(storage) => storage.close().await,
+            EngineEnum::Noop(storage) => storage.close().await,
+            EngineEnum::Large(storage) => storage.close().await,
+            EngineEnum::Small(storage) => storage.close().await,
+            EngineEnum::Mixed(storage) => storage.close().await,
         }
     }
 
     fn enqueue(&self, entry: CacheEntry<Self::Key, Self::Value, Self::BuildHasher>, estimated_size: usize) {
         match self {
-            Engine::Noop(storage) => storage.enqueue(entry, estimated_size),
-            Engine::Large(storage) => storage.enqueue(entry, estimated_size),
-            Engine::Small(storage) => storage.enqueue(entry, estimated_size),
-            Engine::Combined(storage) => storage.enqueue(entry, estimated_size),
+            EngineEnum::Noop(storage) => storage.enqueue(entry, estimated_size),
+            EngineEnum::Large(storage) => storage.enqueue(entry, estimated_size),
+            EngineEnum::Small(storage) => storage.enqueue(entry, estimated_size),
+            EngineEnum::Mixed(storage) => storage.enqueue(entry, estimated_size),
         }
     }
 
     #[auto_enum(Future)]
     fn load(&self, hash: u64) -> impl Future<Output = Result<Option<(Self::Key, Self::Value)>>> + Send + 'static {
         match self {
-            Engine::Noop(storage) => storage.load(hash),
-            Engine::Large(storage) => storage.load(hash),
-            Engine::Small(storage) => storage.load(hash),
-            Engine::Combined(storage) => storage.load(hash),
+            EngineEnum::Noop(storage) => storage.load(hash),
+            EngineEnum::Large(storage) => storage.load(hash),
+            EngineEnum::Small(storage) => storage.load(hash),
+            EngineEnum::Mixed(storage) => storage.load(hash),
         }
     }
 
     fn delete(&self, hash: u64) {
         match self {
-            Engine::Noop(storage) => storage.delete(hash),
-            Engine::Large(storage) => storage.delete(hash),
-            Engine::Small(storage) => storage.delete(hash),
-            Engine::Combined(storage) => storage.delete(hash),
+            EngineEnum::Noop(storage) => storage.delete(hash),
+            EngineEnum::Large(storage) => storage.delete(hash),
+            EngineEnum::Small(storage) => storage.delete(hash),
+            EngineEnum::Mixed(storage) => storage.delete(hash),
         }
     }
 
     fn may_contains(&self, hash: u64) -> bool {
         match self {
-            Engine::Noop(storage) => storage.may_contains(hash),
-            Engine::Large(storage) => storage.may_contains(hash),
-            Engine::Small(storage) => storage.may_contains(hash),
-            Engine::Combined(storage) => storage.may_contains(hash),
+            EngineEnum::Noop(storage) => storage.may_contains(hash),
+            EngineEnum::Large(storage) => storage.may_contains(hash),
+            EngineEnum::Small(storage) => storage.may_contains(hash),
+            EngineEnum::Mixed(storage) => storage.may_contains(hash),
         }
     }
 
     async fn destroy(&self) -> Result<()> {
         match self {
-            Engine::Noop(storage) => storage.destroy().await,
-            Engine::Large(storage) => storage.destroy().await,
-            Engine::Small(storage) => storage.destroy().await,
-            Engine::Combined(storage) => storage.destroy().await,
+            EngineEnum::Noop(storage) => storage.destroy().await,
+            EngineEnum::Large(storage) => storage.destroy().await,
+            EngineEnum::Small(storage) => storage.destroy().await,
+            EngineEnum::Mixed(storage) => storage.destroy().await,
         }
     }
 
     fn stats(&self) -> Arc<DeviceStats> {
         match self {
-            Engine::Noop(storage) => storage.stats(),
-            Engine::Large(storage) => storage.stats(),
-            Engine::Small(storage) => storage.stats(),
-            Engine::Combined(storage) => storage.stats(),
+            EngineEnum::Noop(storage) => storage.stats(),
+            EngineEnum::Large(storage) => storage.stats(),
+            EngineEnum::Small(storage) => storage.stats(),
+            EngineEnum::Mixed(storage) => storage.stats(),
         }
     }
 
     #[auto_enum(Future)]
     fn wait(&self) -> impl Future<Output = ()> + Send + 'static {
         match self {
-            Engine::Noop(storage) => storage.wait(),
-            Engine::Large(storage) => storage.wait(),
-            Engine::Small(storage) => storage.wait(),
-            Engine::Combined(storage) => storage.wait(),
+            EngineEnum::Noop(storage) => storage.wait(),
+            EngineEnum::Large(storage) => storage.wait(),
+            EngineEnum::Small(storage) => storage.wait(),
+            EngineEnum::Mixed(storage) => storage.wait(),
         }
     }
 }
diff --git a/foyer-storage/src/io_buffer_pool.rs b/foyer-storage/src/io_buffer_pool.rs
index 83e42503..cc6ceff2 100644
--- a/foyer-storage/src/io_buffer_pool.rs
+++ b/foyer-storage/src/io_buffer_pool.rs
@@ -18,6 +18,7 @@ use foyer_common::bits;
 
 use crate::{device::ALIGN, IoBuffer, IoBytes};
 
+#[derive(Debug)]
 pub enum Buffer {
     IoBuffer(IoBuffer),
     IoBytes(IoBytes),
@@ -35,6 +36,7 @@ impl From<IoBytes> for Buffer {
     }
 }
 
+#[derive(Debug)]
 pub struct IoBufferPool {
     capacity: usize,
     buffer_size: usize,
diff --git a/foyer-storage/src/large/batch.rs b/foyer-storage/src/large/batch.rs
index ba5a8b04..cacccfe6 100644
--- a/foyer-storage/src/large/batch.rs
+++ b/foyer-storage/src/large/batch.rs
@@ -86,13 +86,13 @@ where
     S: HashBuilder + Debug,
 {
     pub fn new(
-        capacity: usize,
+        buffer_size: usize,
         region_manager: RegionManager,
         device: MonitoredDevice,
         indexer: Indexer,
         metrics: Arc<Metrics>,
     ) -> Self {
-        let capacity = bits::align_up(device.align(), capacity);
+        let capacity = bits::align_up(device.align(), buffer_size);
         let mut batch = Self {
             buffer: IoBuffer::new(capacity),
             len: 0,
@@ -130,7 +130,7 @@ where
         ) {
             Ok(info) => info,
             Err(e) => {
-                tracing::warn!("[batch]: serialize entry error: {e}");
+                tracing::warn!("[lodc batch]: serialize entry error: {e}");
                 return false;
             }
         };
@@ -140,7 +140,7 @@ where
             value_len: info.value_len as _,
             hash: entry.hash(),
             sequence,
-            checksum: Checksummer::checksum(
+            checksum: Checksummer::checksum64(
                 &self.buffer[pos + EntryHeader::serialized_len()
                     ..pos + EntryHeader::serialized_len() + info.key_len + info.value_len],
             ),
diff --git a/foyer-storage/src/large/flusher.rs b/foyer-storage/src/large/flusher.rs
index 7a179701..1bcd9d95 100644
--- a/foyer-storage/src/large/flusher.rs
+++ b/foyer-storage/src/large/flusher.rs
@@ -128,7 +128,7 @@ where
     S: HashBuilder + Debug,
 {
     #[expect(clippy::too_many_arguments)]
-    pub async fn open(
+    pub fn open(
         config: &GenericLargeStorageConfig<K, V, S>,
         indexer: Indexer,
         region_manager: RegionManager,
@@ -140,7 +140,7 @@ where
     ) -> Result<Self> {
         let (tx, rx) = flume::unbounded();
 
-        let buffer_size = config.buffer_threshold / config.flushers;
+        let buffer_size = config.buffer_pool_size / config.flushers;
         let batch = BatchMut::new(
             buffer_size,
             region_manager.clone(),
diff --git a/foyer-storage/src/large/generic.rs b/foyer-storage/src/large/generic.rs
index 489c6be0..e08057c7 100644
--- a/foyer-storage/src/large/generic.rs
+++ b/foyer-storage/src/large/generic.rs
@@ -74,7 +74,7 @@ where
     pub recover_concurrency: usize,
     pub flushers: usize,
     pub reclaimers: usize,
-    pub buffer_threshold: usize,
+    pub buffer_pool_size: usize,
     pub clean_region_threshold: usize,
     pub eviction_pickers: Vec<Box<dyn EvictionPicker>>,
     pub reinsertion_picker: Arc<dyn ReinsertionPicker<Key = K>>,
@@ -101,7 +101,7 @@ where
             .field("recover_concurrency", &self.recover_concurrency)
             .field("flushers", &self.flushers)
             .field("reclaimers", &self.reclaimers)
-            .field("buffer_threshold", &self.buffer_threshold)
+            .field("buffer_pool_size", &self.buffer_pool_size)
             .field("clean_region_threshold", &self.clean_region_threshold)
             .field("eviction_pickers", &self.eviction_pickers)
             .field("reinsertion_pickers", &self.reinsertion_picker)
@@ -203,7 +203,7 @@ where
         let indexer = Indexer::new(config.indexer_shards);
         let mut eviction_pickers = std::mem::take(&mut config.eviction_pickers);
         for picker in eviction_pickers.iter_mut() {
-            picker.init(device.regions(), device.region_size());
+            picker.init(0..device.regions() as RegionId, device.region_size());
         }
         let reclaim_semaphore = Arc::new(Semaphore::new(0));
         let region_manager = RegionManager::new(
@@ -237,7 +237,6 @@ where
                 metrics.clone(),
                 &config.runtime,
             )
-            .await
         }))
         .await?;
 
@@ -253,7 +252,6 @@ where
                 metrics.clone(),
                 &config.runtime,
             )
-            .await
         }))
         .await;
 
@@ -558,7 +556,7 @@ mod tests {
             eviction_pickers: vec![Box::<FifoPicker>::default()],
             reinsertion_picker,
             tombstone_log_config: None,
-            buffer_threshold: 16 * 1024 * 1024,
+            buffer_pool_size: 16 * 1024 * 1024,
             statistics: Arc::<Statistics>::default(),
             runtime: Runtime::new(None, None, Handle::current()),
             marker: PhantomData,
@@ -587,7 +585,7 @@ mod tests {
             eviction_pickers: vec![Box::<FifoPicker>::default()],
             reinsertion_picker: Arc::<RejectAllPicker<u64>>::default(),
             tombstone_log_config: Some(TombstoneLogConfigBuilder::new(path).with_flush(true).build()),
-            buffer_threshold: 16 * 1024 * 1024,
+            buffer_pool_size: 16 * 1024 * 1024,
             statistics: Arc::<Statistics>::default(),
             runtime: Runtime::new(None, None, Handle::current()),
             marker: PhantomData,
diff --git a/foyer-storage/src/large/reclaimer.rs b/foyer-storage/src/large/reclaimer.rs
index dafbff26..e49f4b43 100644
--- a/foyer-storage/src/large/reclaimer.rs
+++ b/foyer-storage/src/large/reclaimer.rs
@@ -45,7 +45,7 @@ pub struct Reclaimer {
 
 impl Reclaimer {
     #[expect(clippy::too_many_arguments)]
-    pub async fn open<K, V, S>(
+    pub fn open<K, V, S>(
         region_manager: RegionManager,
         reclaim_semaphore: Arc<Semaphore>,
         reinsertion_picker: Arc<dyn ReinsertionPicker<Key = K>>,
diff --git a/foyer-storage/src/picker/mod.rs b/foyer-storage/src/picker/mod.rs
index 8cec22df..2657bf48 100644
--- a/foyer-storage/src/picker/mod.rs
+++ b/foyer-storage/src/picker/mod.rs
@@ -12,7 +12,7 @@
 //  See the License for the specific language governing permissions and
 //  limitations under the License.
 
-use std::{collections::HashMap, fmt::Debug, sync::Arc};
+use std::{collections::HashMap, fmt::Debug, ops::Range, sync::Arc};
 
 use crate::{device::RegionId, region::RegionStats, statistics::Statistics};
 
@@ -38,7 +38,7 @@ pub trait ReinsertionPicker: Send + Sync + 'static + Debug {
 pub trait EvictionPicker: Send + Sync + 'static + Debug {
     /// Init the eviction picker with information.
     #[expect(unused_variables)]
-    fn init(&mut self, regions: usize, region_size: usize) {}
+    fn init(&mut self, regions: Range<RegionId>, region_size: usize) {}
 
     /// Pick a region to evict.
     ///
diff --git a/foyer-storage/src/picker/utils.rs b/foyer-storage/src/picker/utils.rs
index a095ed36..fb793d5c 100644
--- a/foyer-storage/src/picker/utils.rs
+++ b/foyer-storage/src/picker/utils.rs
@@ -16,6 +16,7 @@ use std::{
     collections::{HashMap, VecDeque},
     fmt::Debug,
     marker::PhantomData,
+    ops::Range,
     sync::{
         atomic::{AtomicUsize, Ordering},
         Arc,
@@ -257,7 +258,7 @@ impl InvalidRatioPicker {
 }
 
 impl EvictionPicker for InvalidRatioPicker {
-    fn init(&mut self, _: usize, region_size: usize) {
+    fn init(&mut self, _: Range<RegionId>, region_size: usize) {
         self.region_size = region_size;
     }
 
@@ -321,7 +322,7 @@ mod tests {
     #[test]
     fn test_invalid_ratio_picker() {
         let mut picker = InvalidRatioPicker::new(0.5);
-        picker.init(10, 10);
+        picker.init(0..10, 10);
 
         let mut m = HashMap::new();
 
diff --git a/foyer-storage/src/prelude.rs b/foyer-storage/src/prelude.rs
index c2564896..ec0b94c8 100644
--- a/foyer-storage/src/prelude.rs
+++ b/foyer-storage/src/prelude.rs
@@ -33,5 +33,8 @@ pub use crate::{
     runtime::Runtime,
     statistics::Statistics,
     storage::{either::Order, Storage},
-    store::{CombinedConfig, DeviceConfig, RuntimeConfig, Store, StoreBuilder, TokioRuntimeConfig},
+    store::{
+        DeviceConfig, Engine, LargeEngineOptions, RuntimeConfig, SmallEngineOptions, Store, StoreBuilder,
+        TokioRuntimeConfig,
+    },
 };
diff --git a/foyer-storage/src/serde.rs b/foyer-storage/src/serde.rs
index c3ec570e..169e5377 100644
--- a/foyer-storage/src/serde.rs
+++ b/foyer-storage/src/serde.rs
@@ -18,7 +18,7 @@ use foyer_common::{
     code::{StorageKey, StorageValue},
     metrics::Metrics,
 };
-use twox_hash::XxHash64;
+use twox_hash::{XxHash32, XxHash64};
 
 use crate::{
     compress::Compression,
@@ -29,11 +29,17 @@ use crate::{
 pub struct Checksummer;
 
 impl Checksummer {
-    pub fn checksum(buf: &[u8]) -> u64 {
+    pub fn checksum64(buf: &[u8]) -> u64 {
         let mut hasher = XxHash64::with_seed(0);
         hasher.write(buf);
         hasher.finish()
     }
+
+    pub fn checksum32(buf: &[u8]) -> u32 {
+        let mut hasher = XxHash32::with_seed(0);
+        hasher.write(buf);
+        hasher.finish() as u32
+    }
 }
 
 #[derive(Debug)]
@@ -183,7 +189,7 @@ impl EntryDeserializer {
 
         // calculate checksum if needed
         if let Some(expected) = checksum {
-            let get = Checksummer::checksum(&buffer[..value_len + ken_len]);
+            let get = Checksummer::checksum64(&buffer[..value_len + ken_len]);
             if expected != get {
                 return Err(Error::ChecksumMismatch { expected, get });
             }
diff --git a/foyer-storage/src/small/batch.rs b/foyer-storage/src/small/batch.rs
new file mode 100644
index 00000000..310aa156
--- /dev/null
+++ b/foyer-storage/src/small/batch.rs
@@ -0,0 +1,329 @@
+//  Copyright 2024 Foyer Project Authors
+//
+//  Licensed under the Apache License, Version 2.0 (the "License");
+//  you may not use this file except in compliance with the License.
+//  You may obtain a copy of the License at
+//
+//  http://www.apache.org/licenses/LICENSE-2.0
+//
+//  Unless required by applicable law or agreed to in writing, software
+//  distributed under the License is distributed on an "AS IS" BASIS,
+//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//  See the License for the specific language governing permissions and
+//  limitations under the License.
+
+use std::{
+    collections::{HashMap, HashSet},
+    fmt::Debug,
+    ops::Range,
+    sync::Arc,
+    time::Instant,
+};
+
+use foyer_common::{
+    bits,
+    code::{HashBuilder, StorageKey, StorageValue},
+    metrics::Metrics,
+};
+use foyer_memory::CacheEntry;
+use itertools::Itertools;
+use tokio::sync::oneshot;
+
+use crate::{
+    device::ALIGN,
+    io_buffer_pool::IoBufferPool,
+    serde::EntrySerializer,
+    small::{serde::EntryHeader, set::SetId},
+    Compression, IoBuffer, IoBytes,
+};
+
+type Sequence = usize;
+
+#[derive(Debug)]
+struct ItemMut<K, V, S>
+where
+    K: StorageKey,
+    V: StorageValue,
+    S: HashBuilder + Debug,
+{
+    range: Range<usize>,
+    entry: CacheEntry<K, V, S>,
+    sequence: Sequence,
+}
+
+#[derive(Debug)]
+struct SetBatchMut<K, V, S>
+where
+    K: StorageKey,
+    V: StorageValue,
+    S: HashBuilder + Debug,
+{
+    items: Vec<ItemMut<K, V, S>>,
+    deletes: HashMap<u64, Sequence>,
+}
+
+impl<K, V, S> Default for SetBatchMut<K, V, S>
+where
+    K: StorageKey,
+    V: StorageValue,
+    S: HashBuilder + Debug,
+{
+    fn default() -> Self {
+        Self {
+            items: vec![],
+            deletes: HashMap::new(),
+        }
+    }
+}
+
+#[derive(Debug)]
+pub struct BatchMut<K, V, S>
+where
+    K: StorageKey,
+    V: StorageValue,
+    S: HashBuilder + Debug,
+{
+    /// Total set count.
+    total: SetId,
+
+    sets: HashMap<SetId, SetBatchMut<K, V, S>>,
+    buffer: IoBuffer,
+    len: usize,
+    sequence: Sequence,
+
+    /// Cache write buffer between rotation to reduce page fault.
+    buffer_pool: IoBufferPool,
+
+    waiters: Vec<oneshot::Sender<()>>,
+
+    init: Option<Instant>,
+
+    metrics: Arc<Metrics>,
+}
+
+impl<K, V, S> BatchMut<K, V, S>
+where
+    K: StorageKey,
+    V: StorageValue,
+    S: HashBuilder + Debug,
+{
+    pub fn new(total: SetId, buffer_size: usize, metrics: Arc<Metrics>) -> Self {
+        let buffer_size = bits::align_up(ALIGN, buffer_size);
+
+        Self {
+            total,
+            sets: HashMap::new(),
+            buffer: IoBuffer::new(buffer_size),
+            len: 0,
+            sequence: 0,
+            buffer_pool: IoBufferPool::new(buffer_size, 1),
+            waiters: vec![],
+            init: None,
+            metrics,
+        }
+    }
+
+    pub fn insert(&mut self, entry: CacheEntry<K, V, S>, estimated_size: usize) -> bool {
+        // For the small object disk cache does NOT compress entries, `estimated_size` is actually `exact_size`.
+        tracing::trace!("[sodc batch]: insert entry");
+
+        if self.init.is_none() {
+            self.init = Some(Instant::now());
+        }
+        self.sequence += 1;
+
+        let sid = self.sid(entry.hash());
+        let len = EntryHeader::ENTRY_HEADER_SIZE + estimated_size;
+
+        let set = &mut self.sets.entry(sid).or_default();
+
+        set.deletes.insert(entry.hash(), self.sequence);
+
+        if entry.is_outdated() || self.len + len > self.buffer.len() {
+            return false;
+        }
+
+        let info = match EntrySerializer::serialize(
+            entry.key(),
+            entry.value(),
+            &Compression::None,
+            &mut self.buffer[self.len + EntryHeader::ENTRY_HEADER_SIZE..self.len + len],
+            &self.metrics,
+        ) {
+            Ok(info) => info,
+            Err(e) => {
+                tracing::warn!("[sodc batch]: serialize entry error: {e}");
+                return false;
+            }
+        };
+        assert_eq!(info.key_len + info.value_len + EntryHeader::ENTRY_HEADER_SIZE, len);
+        let header = EntryHeader::new(entry.hash(), info.key_len, info.value_len);
+        header.write(&mut self.buffer[self.len..self.len + EntryHeader::ENTRY_HEADER_SIZE]);
+
+        set.items.push(ItemMut {
+            range: self.len..self.len + len,
+            entry,
+            sequence: self.sequence,
+        });
+        self.len += len;
+
+        true
+    }
+
+    pub fn delete(&mut self, hash: u64) {
+        tracing::trace!("[sodc batch]: delete entry");
+
+        if self.init.is_none() {
+            self.init = Some(Instant::now());
+        }
+        self.sequence += 1;
+
+        let sid = self.sid(hash);
+        self.sets.entry(sid).or_default().deletes.insert(hash, self.sequence);
+    }
+
+    /// Register a waiter to be notified after the batch is finished.
+    pub fn wait(&mut self, tx: oneshot::Sender<()>) {
+        tracing::trace!("[sodc batch]: register waiter");
+        if self.init.is_none() {
+            self.init = Some(Instant::now());
+        }
+        self.waiters.push(tx);
+    }
+
+    fn sid(&self, hash: u64) -> SetId {
+        hash % self.total
+    }
+
+    pub fn is_empty(&self) -> bool {
+        self.init.is_none()
+    }
+
+    pub fn rotate(&mut self) -> Option<Batch<K, V, S>> {
+        if self.is_empty() {
+            return None;
+        }
+
+        let mut buffer = self.buffer_pool.acquire();
+        std::mem::swap(&mut self.buffer, &mut buffer);
+        self.len = 0;
+        self.sequence = 0;
+        let buffer = IoBytes::from(buffer);
+        self.buffer_pool.release(buffer.clone());
+
+        let sets = self
+            .sets
+            .drain()
+            .map(|(sid, batch)| {
+                let items = batch
+                    .items
+                    .into_iter()
+                    .filter(|item| item.sequence >= batch.deletes.get(&item.entry.hash()).copied().unwrap_or_default())
+                    .map(|item| Item {
+                        buffer: buffer.slice(item.range),
+                        entry: item.entry,
+                    })
+                    .collect_vec();
+                let deletes = batch.deletes.keys().copied().collect();
+                (
+                    sid,
+                    SetBatch {
+                        deletions: deletes,
+                        items,
+                    },
+                )
+            })
+            .collect();
+
+        let waiters = std::mem::take(&mut self.waiters);
+        let init = self.init.take();
+
+        Some(Batch { sets, waiters, init })
+    }
+}
+
+pub struct Item<K, V, S>
+where
+    K: StorageKey,
+    V: StorageValue,
+    S: HashBuilder + Debug,
+{
+    pub buffer: IoBytes,
+    pub entry: CacheEntry<K, V, S>,
+}
+
+impl<K, V, S> Debug for Item<K, V, S>
+where
+    K: StorageKey,
+    V: StorageValue,
+    S: HashBuilder + Debug,
+{
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        f.debug_struct("Item").field("hash", &self.entry.hash()).finish()
+    }
+}
+
+pub struct SetBatch<K, V, S>
+where
+    K: StorageKey,
+    V: StorageValue,
+    S: HashBuilder + Debug,
+{
+    pub deletions: HashSet<u64>,
+    pub items: Vec<Item<K, V, S>>,
+}
+
+impl<K, V, S> Debug for SetBatch<K, V, S>
+where
+    K: StorageKey,
+    V: StorageValue,
+    S: HashBuilder + Debug,
+{
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        f.debug_struct("SetBatch")
+            .field("deletes", &self.deletions)
+            .field("items", &self.items)
+            .finish()
+    }
+}
+
+pub struct Batch<K, V, S>
+where
+    K: StorageKey,
+    V: StorageValue,
+    S: HashBuilder + Debug,
+{
+    pub sets: HashMap<SetId, SetBatch<K, V, S>>,
+    pub waiters: Vec<oneshot::Sender<()>>,
+    pub init: Option<Instant>,
+}
+
+impl<K, V, S> Default for Batch<K, V, S>
+where
+    K: StorageKey,
+    V: StorageValue,
+    S: HashBuilder + Debug,
+{
+    fn default() -> Self {
+        Self {
+            sets: HashMap::new(),
+            waiters: vec![],
+            init: None,
+        }
+    }
+}
+
+impl<K, V, S> Debug for Batch<K, V, S>
+where
+    K: StorageKey,
+    V: StorageValue,
+    S: HashBuilder + Debug,
+{
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        f.debug_struct("Batch")
+            .field("sets", &self.sets)
+            .field("waiters", &self.waiters)
+            .field("init", &self.init)
+            .finish()
+    }
+}
diff --git a/foyer-storage/src/small/bloom_filter.rs b/foyer-storage/src/small/bloom_filter.rs
new file mode 100644
index 00000000..4cf98689
--- /dev/null
+++ b/foyer-storage/src/small/bloom_filter.rs
@@ -0,0 +1,201 @@
+//  Copyright 2024 Foyer Project Authors
+//
+//  Licensed under the Apache License, Version 2.0 (the "License");
+//  you may not use this file except in compliance with the License.
+//  You may obtain a copy of the License at
+//
+//  http://www.apache.org/licenses/LICENSE-2.0
+//
+//  Unless required by applicable law or agreed to in writing, software
+//  distributed under the License is distributed on an "AS IS" BASIS,
+//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//  See the License for the specific language governing permissions and
+//  limitations under the License.
+
+#![cfg_attr(not(test), expect(dead_code))]
+
+use paste::paste;
+
+macro_rules! bloom_filter {
+    ($( {$type:ty, $suffix:ident}, )*) => {
+        paste! {
+            $(
+                /// A [<$type>] bloom filter with N hash hashers.
+                #[derive(Debug, Clone, PartialEq, Eq)]
+                pub struct [<BloomFilter $suffix>]<const N: usize> {
+                    data: [$type; N],
+                }
+
+                impl<const N: usize> Default for [<BloomFilter $suffix>]<N> {
+                    fn default() -> Self {
+                        Self::new()
+                    }
+                }
+
+                impl<const N: usize> [<BloomFilter $suffix>]<N> {
+                    const BYTES: usize = $type::BITS as usize / u8::BITS as usize * N;
+
+                    pub fn new() -> Self {
+                        Self {
+                            data: [0; N],
+                        }
+                    }
+
+                    pub fn read(raw: &[u8]) -> Self {
+                        let mut data = [0; N];
+                        data.copy_from_slice(unsafe { std::slice::from_raw_parts(raw.as_ptr() as *const $type, N) });
+                        Self { data }
+                    }
+
+                    pub fn write(&self, raw: &mut [u8]) {
+                        raw[..Self::BYTES].copy_from_slice(unsafe { std::slice::from_raw_parts(self.data.as_ptr() as *const u8, Self::BYTES) })
+                    }
+
+                    pub fn insert(&mut self, hash: u64) {
+                        for i in 0..N {
+                            let seed = twang_mix64(i as _);
+                            let hash = combine_hashes(hash, seed);
+                            let bit = hash as usize % $type::BITS as usize;
+                            self.data[i] |= 1 << bit;
+                        }
+                    }
+
+                    pub fn lookup(&self, hash: u64) -> bool {
+                        for i in 0..N {
+                            let seed = twang_mix64(i as _);
+                            let hash = combine_hashes(hash, seed) as $type;
+                            let bit = hash as usize % $type::BITS as usize;
+                            if self.data[i] & (1 << bit) == 0 {
+                                return false;
+                            }
+                        }
+                        true
+                    }
+
+                    pub fn clear(&mut self) {
+                        self.data = [0; N];
+                    }
+                }
+            )*
+        }
+    };
+}
+
+macro_rules! for_all_uint_types {
+    ($macro:ident) => {
+        $macro! {
+            {u8, U8},
+            {u16, U16},
+            {u32, U32},
+            {u64, U64},
+            {usize, Usize},
+        }
+    };
+}
+
+for_all_uint_types! { bloom_filter }
+
+/// Reduce two 64-bit hashes into one.
+///
+/// Ported from CacheLib, which uses the `Hash128to64` function from Google's city hash.
+#[inline(always)]
+fn combine_hashes(upper: u64, lower: u64) -> u64 {
+    const MUL: u64 = 0x9ddfea08eb382d69;
+
+    let mut a = (lower ^ upper).wrapping_mul(MUL);
+    a ^= a >> 47;
+    let mut b = (upper ^ a).wrapping_mul(MUL);
+    b ^= b >> 47;
+    b = b.wrapping_mul(MUL);
+    b
+}
+
+#[inline(always)]
+fn twang_mix64(val: u64) -> u64 {
+    let mut val = (!val).wrapping_add(val << 21); // val *= (1 << 21); val -= 1
+    val = val ^ (val >> 24);
+    val = val.wrapping_add(val << 3).wrapping_add(val << 8); // val *= 1 + (1 << 3) + (1 << 8)
+    val = val ^ (val >> 14);
+    val = val.wrapping_add(val << 2).wrapping_add(val << 4); // va; *= 1 + (1 << 2) + (1 << 4)
+    val = val ^ (val >> 28);
+    val = val.wrapping_add(val << 31); // val *= 1 + (1 << 31)
+    val
+}
+
+macro_rules! test_bloom_filter {
+    ($( {$type:ty, $suffix:ident}, )*) => {
+        #[cfg(test)]
+        mod tests {
+            use super::*;
+
+            const N: usize = 4;
+
+            paste! {
+                $(
+                    #[test]
+                    fn [<test_bloom_filter_ $suffix:lower _insert_lookup_clear>]() {
+                        let mut bf = [<BloomFilter $suffix>]::<N>::new();
+
+                        bf.insert(42);
+                        assert!(bf.lookup(42));
+                        assert!(!bf.lookup(114514));
+                        bf.clear();
+                        assert!(!bf.lookup(42));
+                    }
+
+                    #[test]
+                    fn [<test_bloom_filter_ $suffix:lower _multiple_inserts>]() {
+                        let mut bf = [<BloomFilter $suffix>]::<N>::new();
+                        bf.insert(1);
+                        bf.insert(2);
+                        bf.insert(3);
+                        assert!(bf.lookup(1));
+                        assert!(bf.lookup(2));
+                        assert!(bf.lookup(3));
+                        assert!(!bf.lookup(4));
+                    }
+
+                    #[test]
+                    fn [<test_bloom_filter_ $suffix:lower _false_positives>]() {
+                        const INSERTS: usize = [<BloomFilter $suffix>]::<N>::BYTES;
+                        const LOOKUPS: usize = [<BloomFilter $suffix>]::<N>::BYTES * 100;
+                        const THRESHOLD: f64 = 0.1;
+                        let mut bf = [<BloomFilter $suffix>]::<N>::new();
+                        // Insert a bunch of values
+                        for i in 0..INSERTS {
+                            bf.insert(i as _);
+                            println!("{i}: {:X?}", bf.data);
+                        }
+                        // Check for false positives
+                        let mut false_positives = 0;
+                        for i in INSERTS..INSERTS + LOOKUPS {
+                            if bf.lookup(i as _) {
+                                false_positives += 1;
+                            }
+                        }
+                        let ratio = false_positives as f64 / LOOKUPS as f64;
+                        println!("ratio: {ratio}");
+                        assert!(
+                            ratio < THRESHOLD,
+                            "false positive ratio {ratio} > threshold {THRESHOLD}, inserts: {INSERTS}, lookups: {LOOKUPS}"
+                        );
+                    }
+
+                    #[test]
+                    fn [<test_bloom_filter_ $suffix:lower _read_write>]() {
+                        let mut buf = [0; [<BloomFilter $suffix>]::<N>::BYTES];
+                        let mut bf = [<BloomFilter $suffix>]::<N>::new();
+                        bf.insert(42);
+                        bf.write(&mut buf);
+                        let bf2 = [<BloomFilter $suffix>]::<N>::read(&buf);
+                        assert_eq!(bf, bf2);
+                    }
+                )*
+            }
+
+        }
+
+    };
+}
+
+for_all_uint_types! { test_bloom_filter }
diff --git a/foyer-storage/src/small/flusher.rs b/foyer-storage/src/small/flusher.rs
new file mode 100644
index 00000000..43e720a3
--- /dev/null
+++ b/foyer-storage/src/small/flusher.rs
@@ -0,0 +1,225 @@
+//  Copyright 2024 Foyer Project Authors
+//
+//  Licensed under the Apache License, Version 2.0 (the "License");
+//  you may not use this file except in compliance with the License.
+//  You may obtain a copy of the License at
+//
+//  http://www.apache.org/licenses/LICENSE-2.0
+//
+//  Unless required by applicable law or agreed to in writing, software
+//  distributed under the License is distributed on an "AS IS" BASIS,
+//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//  See the License for the specific language governing permissions and
+//  limitations under the License.
+
+use std::{
+    fmt::Debug,
+    future::Future,
+    sync::{atomic::Ordering, Arc},
+};
+
+use foyer_common::{
+    code::{HashBuilder, StorageKey, StorageValue},
+    metrics::Metrics,
+};
+use foyer_memory::CacheEntry;
+use futures::future::try_join_all;
+use tokio::sync::{oneshot, OwnedSemaphorePermit, Semaphore};
+
+use super::{
+    batch::{Batch, BatchMut, SetBatch},
+    generic::GenericSmallStorageConfig,
+    set_manager::SetManager,
+};
+use crate::{
+    error::{Error, Result},
+    Statistics,
+};
+
+pub enum Submission<K, V, S>
+where
+    K: StorageKey,
+    V: StorageValue,
+    S: HashBuilder + Debug,
+{
+    Insertion {
+        entry: CacheEntry<K, V, S>,
+        estimated_size: usize,
+    },
+    Deletion {
+        hash: u64,
+    },
+    Wait {
+        tx: oneshot::Sender<()>,
+    },
+}
+
+impl<K, V, S> Debug for Submission<K, V, S>
+where
+    K: StorageKey,
+    V: StorageValue,
+    S: HashBuilder + Debug,
+{
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        match self {
+            Self::Insertion {
+                entry: _,
+                estimated_size,
+            } => f
+                .debug_struct("Insertion")
+                .field("estimated_size", estimated_size)
+                .finish(),
+            Self::Deletion { hash } => f.debug_struct("Deletion").field("hash", hash).finish(),
+            Self::Wait { .. } => f.debug_struct("Wait").finish(),
+        }
+    }
+}
+
+pub struct Flusher<K, V, S>
+where
+    K: StorageKey,
+    V: StorageValue,
+    S: HashBuilder + Debug,
+{
+    tx: flume::Sender<Submission<K, V, S>>,
+}
+
+impl<K, V, S> Flusher<K, V, S>
+where
+    K: StorageKey,
+    V: StorageValue,
+    S: HashBuilder + Debug,
+{
+    pub fn open(
+        config: &GenericSmallStorageConfig<K, V, S>,
+        set_manager: SetManager,
+        stats: Arc<Statistics>,
+        metrics: Arc<Metrics>,
+    ) -> Self {
+        let (tx, rx) = flume::unbounded();
+
+        let buffer_size = config.buffer_pool_size / config.flushers;
+
+        let batch = BatchMut::new(set_manager.sets() as _, buffer_size, metrics.clone());
+
+        let runner = Runner {
+            rx,
+            batch,
+            flight: Arc::new(Semaphore::new(1)),
+            set_manager,
+            stats,
+            metrics,
+        };
+
+        config.runtime.write().spawn(async move {
+            if let Err(e) = runner.run().await {
+                tracing::error!("[sodc flusher]: flusher exit with error: {e}");
+            }
+        });
+
+        Self { tx }
+    }
+
+    pub fn submit(&self, submission: Submission<K, V, S>) {
+        tracing::trace!("[sodc flusher]: submit task: {submission:?}");
+        if let Err(e) = self.tx.send(submission) {
+            tracing::error!("[sodc flusher]: error raised when submitting task, error: {e}");
+        }
+    }
+
+    pub fn wait(&self) -> impl Future<Output = ()> + Send + 'static {
+        let (tx, rx) = oneshot::channel();
+        self.submit(Submission::Wait { tx });
+        async move {
+            let _ = rx.await;
+        }
+    }
+}
+
+struct Runner<K, V, S>
+where
+    K: StorageKey,
+    V: StorageValue,
+    S: HashBuilder + Debug,
+{
+    rx: flume::Receiver<Submission<K, V, S>>,
+    batch: BatchMut<K, V, S>,
+    flight: Arc<Semaphore>,
+
+    set_manager: SetManager,
+
+    stats: Arc<Statistics>,
+    metrics: Arc<Metrics>,
+}
+
+impl<K, V, S> Runner<K, V, S>
+where
+    K: StorageKey,
+    V: StorageValue,
+    S: HashBuilder + Debug,
+{
+    pub async fn run(mut self) -> Result<()> {
+        loop {
+            let flight = self.flight.clone();
+            tokio::select! {
+                biased;
+                Ok(permit) = flight.acquire_owned(), if !self.batch.is_empty() => {
+                    // TODO(MrCroxx): `rotate()` should always return a `Some(..)` here.
+                    if let Some(batch) = self.batch.rotate() {
+                        self.commit(batch, permit).await;
+                    }
+                }
+                Ok(submission) = self.rx.recv_async() => {
+                    self.submit(submission);
+                }
+                // Graceful shutdown.
+                else => break,
+            }
+        }
+        Ok(())
+    }
+
+    fn submit(&mut self, submission: Submission<K, V, S>) {
+        let report = |enqueued: bool| {
+            if !enqueued {
+                self.metrics.storage_queue_drop.increment(1);
+            }
+        };
+
+        match submission {
+            Submission::Insertion { entry, estimated_size } => report(self.batch.insert(entry, estimated_size)),
+            Submission::Deletion { hash } => self.batch.delete(hash),
+            Submission::Wait { tx } => self.batch.wait(tx),
+        }
+    }
+
+    pub async fn commit(&self, batch: Batch<K, V, S>, permit: OwnedSemaphorePermit) {
+        tracing::trace!("[sodc flusher] commit batch: {batch:?}");
+
+        let futures = batch.sets.into_iter().map(|(sid, SetBatch { deletions, items })| {
+            let set_manager = self.set_manager.clone();
+            let stats = self.stats.clone();
+            async move {
+                let mut set = set_manager.write(sid).await?;
+                set.apply(&deletions, items);
+                set_manager.apply(set).await?;
+
+                stats
+                    .cache_write_bytes
+                    .fetch_add(set_manager.set_size(), Ordering::Relaxed);
+
+                Ok::<_, Error>(())
+            }
+        });
+
+        if let Err(e) = try_join_all(futures).await {
+            tracing::error!("[sodc flusher]: error raised when committing batch, error: {e}");
+        }
+
+        for waiter in batch.waiters {
+            let _ = waiter.send(());
+        }
+
+        drop(permit);
+    }
+}
diff --git a/foyer-storage/src/small/generic.rs b/foyer-storage/src/small/generic.rs
index dd8ae8e2..3b088204 100644
--- a/foyer-storage/src/small/generic.rs
+++ b/foyer-storage/src/small/generic.rs
@@ -12,13 +12,29 @@
 //  See the License for the specific language governing permissions and
 //  limitations under the License.
 
-use std::{fmt::Debug, marker::PhantomData, sync::Arc};
+use std::{
+    fmt::Debug,
+    marker::PhantomData,
+    ops::Range,
+    sync::{
+        atomic::{AtomicBool, Ordering},
+        Arc,
+    },
+};
 
 use foyer_common::code::{HashBuilder, StorageKey, StorageValue};
 use foyer_memory::CacheEntry;
-use futures::Future;
+use futures::{future::join_all, Future};
+use itertools::Itertools;
 
-use crate::{error::Result, storage::Storage, DeviceStats};
+use super::flusher::Submission;
+use crate::{
+    device::{MonitoredDevice, RegionId},
+    error::Result,
+    small::{flusher::Flusher, set::SetId, set_manager::SetManager},
+    storage::Storage,
+    DeviceStats, Runtime, Statistics,
+};
 
 pub struct GenericSmallStorageConfig<K, V, S>
 where
@@ -26,7 +42,16 @@ where
     V: StorageValue,
     S: HashBuilder + Debug,
 {
-    pub placeholder: PhantomData<(K, V, S)>,
+    pub set_size: usize,
+    pub set_cache_capacity: usize,
+    pub device: MonitoredDevice,
+    pub regions: Range<RegionId>,
+    pub flush: bool,
+    pub flushers: usize,
+    pub buffer_pool_size: usize,
+    pub statistics: Arc<Statistics>,
+    pub runtime: Runtime,
+    pub marker: PhantomData<(K, V, S)>,
 }
 
 impl<K, V, S> Debug for GenericSmallStorageConfig<K, V, S>
@@ -36,17 +61,45 @@ where
     S: HashBuilder + Debug,
 {
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-        f.debug_struct("GenericSmallStorageConfig").finish()
+        f.debug_struct("GenericSmallStorageConfig")
+            .field("set_size", &self.set_size)
+            .field("set_cache_capacity", &self.set_cache_capacity)
+            .field("device", &self.device)
+            .field("regions", &self.regions)
+            .field("flush", &self.flush)
+            .field("flushers", &self.flushers)
+            .field("buffer_pool_size", &self.buffer_pool_size)
+            .field("statistics", &self.statistics)
+            .field("runtime", &self.runtime)
+            .field("marker", &self.marker)
+            .finish()
     }
 }
 
+struct GenericSmallStorageInner<K, V, S>
+where
+    K: StorageKey,
+    V: StorageValue,
+    S: HashBuilder + Debug,
+{
+    flushers: Vec<Flusher<K, V, S>>,
+
+    device: MonitoredDevice,
+    set_manager: SetManager,
+
+    active: AtomicBool,
+
+    stats: Arc<Statistics>,
+    runtime: Runtime,
+}
+
 pub struct GenericSmallStorage<K, V, S>
 where
     K: StorageKey,
     V: StorageValue,
     S: HashBuilder + Debug,
 {
-    _marker: PhantomData<(K, V, S)>,
+    inner: Arc<GenericSmallStorageInner<K, V, S>>,
 }
 
 impl<K, V, S> Debug for GenericSmallStorage<K, V, S>
@@ -67,7 +120,114 @@ where
     S: HashBuilder + Debug,
 {
     fn clone(&self) -> Self {
-        Self { _marker: PhantomData }
+        Self {
+            inner: self.inner.clone(),
+        }
+    }
+}
+
+impl<K, V, S> GenericSmallStorage<K, V, S>
+where
+    K: StorageKey,
+    V: StorageValue,
+    S: HashBuilder + Debug,
+{
+    async fn open(config: GenericSmallStorageConfig<K, V, S>) -> Result<Self> {
+        let stats = config.statistics.clone();
+        let metrics = config.device.metrics().clone();
+
+        let set_manager = SetManager::new(
+            config.set_size,
+            config.set_cache_capacity,
+            config.device.clone(),
+            config.regions.clone(),
+            config.flush,
+        );
+
+        let flushers = (0..config.flushers)
+            .map(|_| Flusher::open(&config, set_manager.clone(), stats.clone(), metrics.clone()))
+            .collect_vec();
+
+        let inner = GenericSmallStorageInner {
+            flushers,
+            device: config.device,
+            set_manager,
+            active: AtomicBool::new(true),
+            stats,
+            runtime: config.runtime,
+        };
+        let inner = Arc::new(inner);
+
+        Ok(Self { inner })
+    }
+
+    fn wait(&self) -> impl Future<Output = ()> + Send + 'static {
+        let wait_flushers = join_all(self.inner.flushers.iter().map(|flusher| flusher.wait()));
+        async move {
+            wait_flushers.await;
+        }
+    }
+
+    async fn close(&self) -> Result<()> {
+        self.inner.active.store(false, Ordering::Relaxed);
+        self.wait().await;
+        Ok(())
+    }
+
+    fn enqueue(&self, entry: CacheEntry<K, V, S>, estimated_size: usize) {
+        if !self.inner.active.load(Ordering::Relaxed) {
+            tracing::warn!("cannot enqueue new entry after closed");
+            return;
+        }
+
+        // Entries with the same hash must be grouped in the batch.
+        let id = entry.hash() as usize % self.inner.flushers.len();
+        self.inner.flushers[id].submit(Submission::Insertion { entry, estimated_size });
+    }
+
+    fn load(&self, hash: u64) -> impl Future<Output = Result<Option<(K, V)>>> + Send + 'static {
+        let set_manager = self.inner.set_manager.clone();
+        let sid = hash % set_manager.sets() as SetId;
+        let stats = self.inner.stats.clone();
+
+        async move {
+            stats
+                .cache_read_bytes
+                .fetch_add(set_manager.set_size(), Ordering::Relaxed);
+
+            match set_manager.read(sid, hash).await? {
+                Some(set) => {
+                    let kv = set.get(hash)?;
+                    Ok(kv)
+                }
+                None => Ok(None),
+            }
+        }
+    }
+
+    fn delete(&self, hash: u64) {
+        if !self.inner.active.load(Ordering::Relaxed) {
+            tracing::warn!("cannot enqueue new entry after closed");
+            return;
+        }
+
+        // Entries with the same hash MUST be grouped in the same batch.
+        let id = hash as usize % self.inner.flushers.len();
+        self.inner.flushers[id].submit(Submission::Deletion { hash });
+    }
+
+    fn may_contains(&self, hash: u64) -> bool {
+        let set_manager = self.inner.set_manager.clone();
+        let sid = hash % set_manager.sets() as SetId;
+        // FIXME: Anyway without blocking? Use atomic?
+        self.inner
+            .runtime
+            .read()
+            .block_on(async move { set_manager.contains(sid, hash).await })
+    }
+
+    fn stats(&self) -> Arc<DeviceStats> {
+        self.inner.device.stat().clone()
     }
 }
 
@@ -82,27 +242,29 @@ where
     type BuildHasher = S;
     type Config = GenericSmallStorageConfig<K, V, S>;
 
-    async fn open(_config: Self::Config) -> Result<Self> {
-        todo!()
+    async fn open(config: Self::Config) -> Result<Self> {
+        Self::open(config).await
     }
 
     async fn close(&self) -> Result<()> {
-        todo!()
+        self.close().await?;
+        Ok(())
     }
 
-    fn enqueue(&self, _entry: CacheEntry<Self::Key, Self::Value, Self::BuildHasher>, _estimated_size: usize) {
-        todo!()
+    fn enqueue(&self, entry: CacheEntry<Self::Key, Self::Value, Self::BuildHasher>, estimated_size: usize) {
+        self.enqueue(entry, estimated_size);
     }
 
-    #[expect(clippy::manual_async_fn)]
-    fn load(&self, _hash: u64) -> impl Future<Output = Result<Option<(Self::Key, Self::Value)>>> + Send + 'static {
-        async { todo!() }
+    fn load(&self, hash: u64) -> impl Future<Output = Result<Option<(Self::Key, Self::Value)>>> + Send + 'static {
+        self.load(hash)
     }
 
-    fn delete(&self, _hash: u64) {}
+    fn delete(&self, hash: u64) {
+        self.delete(hash)
+    }
 
-    fn may_contains(&self, _hash: u64) -> bool {
-        todo!()
+    fn may_contains(&self, hash: u64) -> bool {
+        self.may_contains(hash)
     }
 
     async fn destroy(&self) -> Result<()> {
@@ -110,11 +272,10 @@ where
     }
 
     fn stats(&self) -> Arc<DeviceStats> {
-        todo!()
+        self.stats()
     }
 
-    #[expect(clippy::manual_async_fn)]
     fn wait(&self) -> impl Future<Output = ()> + Send + 'static {
-        async { todo!() }
+        self.wait()
     }
 }
diff --git a/foyer-storage/src/small/mod.rs b/foyer-storage/src/small/mod.rs
index 01e617c2..44c6f579 100644
--- a/foyer-storage/src/small/mod.rs
+++ b/foyer-storage/src/small/mod.rs
@@ -12,4 +12,10 @@
 //  See the License for the specific language governing permissions and
 //  limitations under the License.
 
+pub mod batch;
+pub mod bloom_filter;
+pub mod flusher;
 pub mod generic;
+pub mod serde;
+pub mod set;
+pub mod set_manager;
diff --git a/foyer-storage/src/small/serde.rs b/foyer-storage/src/small/serde.rs
new file mode 100644
index 00000000..2648ad48
--- /dev/null
+++ b/foyer-storage/src/small/serde.rs
@@ -0,0 +1,97 @@
+//  Copyright 2024 Foyer Project Authors
+//
+//  Licensed under the Apache License, Version 2.0 (the "License");
+//  you may not use this file except in compliance with the License.
+//  You may obtain a copy of the License at
+//
+//  http://www.apache.org/licenses/LICENSE-2.0
+//
+//  Unless required by applicable law or agreed to in writing, software
+//  distributed under the License is distributed on an "AS IS" BASIS,
+//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//  See the License for the specific language governing permissions and
+//  limitations under the License.
+
+use bytes::{Buf, BufMut};
+
+/// max key/value len: `64 KiB - 1`
+///
+/// # Format
+///
+/// ```plain
+/// | hash 64b | key len 16b | value len 16b |
+/// ```
+#[derive(Debug, PartialEq, Eq)]
+pub struct EntryHeader {
+    hash: u64,
+    key_len: u16,
+    value_len: u16,
+}
+
+impl EntryHeader {
+    pub const ENTRY_HEADER_SIZE: usize = (16 + 16 + 64) / 8;
+
+    pub fn new(hash: u64, key_len: usize, value_len: usize) -> Self {
+        Self {
+            hash,
+            key_len: key_len as _,
+            value_len: value_len as _,
+        }
+    }
+
+    #[inline]
+    pub fn hash(&self) -> u64 {
+        self.hash
+    }
+
+    #[inline]
+    pub fn key_len(&self) -> usize {
+        self.key_len as _
+    }
+
+    #[inline]
+    pub fn value_len(&self) -> usize {
+        self.value_len as _
+    }
+
+    #[inline]
+    pub fn entry_len(&self) -> usize {
+        Self::ENTRY_HEADER_SIZE + self.key_len() + self.value_len()
+    }
+
+    pub fn write(&self, mut buf: impl BufMut) {
+        buf.put_u64(self.hash);
+        buf.put_u16(self.key_len);
+        buf.put_u16(self.value_len);
+    }
+
+    pub fn read(mut buf: impl Buf) -> Self {
+        let hash = buf.get_u64();
+        let key_len = buf.get_u16();
+        let value_len = buf.get_u16();
+        Self {
+            hash,
+            key_len,
+            value_len,
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::IoBytesMut;
+
+    #[test]
+    fn test_entry_header_serde() {
+        let header = EntryHeader {
+            hash: 114514,
+            key_len: 114,
+            value_len: 514,
+        };
+        let mut buf = IoBytesMut::new();
+        header.write(&mut buf);
+        let h = EntryHeader::read(&buf[..]);
+        assert_eq!(header, h);
+    }
+}
diff --git a/foyer-storage/src/small/set.rs b/foyer-storage/src/small/set.rs
new file mode 100644
index 00000000..02e94adb
--- /dev/null
+++ b/foyer-storage/src/small/set.rs
@@ -0,0 +1,537 @@
+//  Copyright 2024 Foyer Project Authors
+//
+//  Licensed under the Apache License, Version 2.0 (the "License");
+//  you may not use this file except in compliance with the License.
+//  You may obtain a copy of the License at
+//
+//  http://www.apache.org/licenses/LICENSE-2.0
+//
+//  Unless required by applicable law or agreed to in writing, software
+//  distributed under the License is distributed on an "AS IS" BASIS,
+//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//  See the License for the specific language governing permissions and
+//  limitations under the License.
+
+use std::{
+    collections::HashSet,
+    fmt::Debug,
+    ops::{Deref, DerefMut, Range},
+    sync::Arc,
+    time::{SystemTime, UNIX_EPOCH},
+};
+
+use bytes::{Buf, BufMut};
+use foyer_common::code::{HashBuilder, StorageKey, StorageValue};
+
+use super::{batch::Item, bloom_filter::BloomFilterU64, serde::EntryHeader};
+use crate::{
+    error::Result,
+    serde::{Checksummer, EntryDeserializer},
+    IoBytes, IoBytesMut,
+};
+
+pub type SetId = u64;
+
+#[derive(Debug)]
+pub struct Set {
+    storage: Arc<SetStorage>,
+}
+
+impl Deref for Set {
+    type Target = SetStorage;
+
+    fn deref(&self) -> &Self::Target {
+        &self.storage
+    }
+}
+
+impl Set {
+    pub fn new(storage: Arc<SetStorage>) -> Self {
+        Self { storage }
+    }
+}
+
+#[derive(Debug)]
+pub struct SetMut {
+    storage: SetStorage,
+}
+
+impl Deref for SetMut {
+    type Target = SetStorage;
+
+    fn deref(&self) -> &Self::Target {
+        &self.storage
+    }
+}
+
+impl DerefMut for SetMut {
+    fn deref_mut(&mut self) -> &mut Self::Target {
+        &mut self.storage
+    }
+}
+
+impl SetMut {
+    pub fn new(storage: SetStorage) -> Self {
+        Self { storage }
+    }
+
+    pub fn into_storage(self) -> SetStorage {
+        self.storage
+    }
+}
+
+/// # Format
+///
+/// ```plain
+/// | checksum (4B) | timestamp (8B) | len (4B) |
+/// | bloom filter (4 * 8B = 32B) |
+/// ```
+pub struct SetStorage {
+    /// Set checksum.
+    checksum: u32,
+
+    /// Set written data length.
+    len: usize,
+    /// Set data length capacity.
+    capacity: usize,
+    /// Set size.
+    size: usize,
+    /// Set last updated timestamp.
+    timestamp: u64,
+    /// Set bloom filter.
+    bloom_filter: BloomFilterU64<4>,
+
+    buffer: IoBytesMut,
+}
+
+impl Debug for SetStorage {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        f.debug_struct("SetStorage")
+            .field("checksum", &self.checksum)
+            .field("len", &self.len)
+            .field("capacity", &self.capacity)
+            .field("size", &self.size)
+            .field("timestamp", &self.timestamp)
+            .field("bloom_filter", &self.bloom_filter)
+            .finish()
+    }
+}
+
+impl SetStorage {
+    pub const SET_HEADER_SIZE: usize = 48;
+
+    pub fn load(buffer: IoBytesMut) -> Self {
+        assert!(buffer.len() >= Self::SET_HEADER_SIZE);
+
+        let checksum = (&buffer[0..4]).get_u32();
+        let timestamp = (&buffer[4..12]).get_u64();
+        let len = (&buffer[12..16]).get_u32() as usize;
+        let bloom_filter = BloomFilterU64::read(&buffer[16..48]);
+
+        let mut this = Self {
+            checksum,
+            len,
+            capacity: buffer.len() - Self::SET_HEADER_SIZE,
+            size: buffer.len(),
+            timestamp,
+            bloom_filter,
+            buffer,
+        };
+
+        if Self::SET_HEADER_SIZE + this.len >= this.buffer.len() {
+            // invalid len
+            this.clear();
+        } else {
+            let c = Checksummer::checksum32(&this.buffer[4..Self::SET_HEADER_SIZE + this.len]);
+            if c != checksum {
+                // checksum mismatch
+                this.clear();
+            }
+        }
+
+        this
+    }
+
+    pub fn update(&mut self) {
+        self.bloom_filter.write(&mut self.buffer[16..48]);
+        (&mut self.buffer[12..16]).put_u32(self.len as _);
+        self.timestamp = SystemTime::now().duration_since(UNIX_EPOCH).unwrap().as_millis() as u64;
+        (&mut self.buffer[4..12]).put_u64(self.timestamp);
+        self.checksum = Checksummer::checksum32(&self.buffer[4..Self::SET_HEADER_SIZE + self.len]);
+        (&mut self.buffer[0..4]).put_u32(self.checksum);
+    }
+
+    pub fn bloom_filter(&self) -> &BloomFilterU64<4> {
+        &self.bloom_filter
+    }
+
+    #[cfg_attr(not(test), expect(dead_code))]
+    pub fn len(&self) -> usize {
+        self.len
+    }
+
+    #[cfg_attr(not(test), expect(dead_code))]
+    pub fn is_empty(&self) -> bool {
+        self.len == 0
+    }
+
+    pub fn clear(&mut self) {
+        self.len = 0;
+        self.bloom_filter.clear();
+    }
+
+    pub fn freeze(self) -> IoBytes {
+        self.buffer.freeze()
+    }
+
+    pub fn apply<K, V, S>(&mut self, deletions: &HashSet<u64>, items: Vec<Item<K, V, S>>)
+    where
+        K: StorageKey,
+        V: StorageValue,
+        S: HashBuilder + Debug,
+    {
+        self.deletes(deletions);
+        self.append(items);
+    }
+
+    fn deletes(&mut self, deletes: &HashSet<u64>) {
+        if deletes.is_empty() {
+            return;
+        }
+
+        let mut rcursor = 0;
+        let mut wcursor = 0;
+        // Rebuild bloom filter.
+        self.bloom_filter.clear();
+
+        while rcursor < self.len {
+            let header = EntryHeader::read(
+                &self.buffer
+                    [Self::SET_HEADER_SIZE + rcursor..Self::SET_HEADER_SIZE + rcursor + EntryHeader::ENTRY_HEADER_SIZE],
+            );
+
+            if !deletes.contains(&header.hash()) {
+                if rcursor != wcursor {
+                    self.buffer.copy_within(
+                        Self::SET_HEADER_SIZE + rcursor..Self::SET_HEADER_SIZE + header.entry_len(),
+                        wcursor,
+                    );
+                }
+                wcursor += header.entry_len();
+                self.bloom_filter.insert(header.hash());
+            }
+
+            rcursor += header.entry_len();
+        }
+
+        self.len = wcursor;
+    }
+
+    fn append<K, V, S>(&mut self, items: Vec<Item<K, V, S>>)
+    where
+        K: StorageKey,
+        V: StorageValue,
+        S: HashBuilder + Debug,
+    {
+        let (skip, size, _) = items
+            .iter()
+            .rev()
+            .fold((items.len(), 0, true), |(skip, size, proceed), item| {
+                let proceed = proceed && size + item.buffer.len() <= self.size - Self::SET_HEADER_SIZE;
+                if proceed {
+                    (skip - 1, size + item.buffer.len(), proceed)
+                } else {
+                    (skip, size, proceed)
+                }
+            });
+
+        self.reserve(size);
+        let mut cursor = Self::SET_HEADER_SIZE + self.len;
+        for item in items.iter().skip(skip) {
+            self.buffer[cursor..cursor + item.buffer.len()].copy_from_slice(&item.buffer);
+            self.bloom_filter.insert(item.entry.hash());
+            cursor += item.buffer.len();
+        }
+        self.len = cursor - Self::SET_HEADER_SIZE;
+    }
+
+    pub fn get<K, V>(&self, hash: u64) -> Result<Option<(K, V)>>
+    where
+        K: StorageKey,
+        V: StorageValue,
+    {
+        if !self.bloom_filter.lookup(hash) {
+            return Ok(None);
+        }
+        for entry in self.iter() {
+            if hash == entry.hash {
+                let k = EntryDeserializer::deserialize_key::<K>(entry.key)?;
+                let v = EntryDeserializer::deserialize_value::<V>(entry.value, crate::Compression::None)?;
+                return Ok(Some((k, v)));
+            }
+        }
+        Ok(None)
+    }
+
+    /// from:
+    ///
+    /// ```plain
+    /// 0        wipe          len       capacity
+    /// |_________|ooooooooooooo|___________|
+    /// ```
+    ///
+    /// to:
+    ///
+    /// ```plain
+    /// 0     new len = len - wipe       capacity
+    /// |ooooooooooooo|_____________________|
+    /// ```
+    fn reserve(&mut self, required: usize) {
+        let remains = self.capacity - self.len;
+        if remains >= required {
+            return;
+        }
+
+        let mut wipe = 0;
+        for entry in self.iter() {
+            wipe += entry.len();
+            if remains + wipe >= required {
+                break;
+            }
+        }
+        self.buffer.copy_within(
+            Self::SET_HEADER_SIZE + wipe..Self::SET_HEADER_SIZE + self.len,
+            Self::SET_HEADER_SIZE,
+        );
+        self.len -= wipe;
+        assert!(self.capacity - self.len >= required);
+        let mut bloom_filter = BloomFilterU64::default();
+        for entry in self.iter() {
+            bloom_filter.insert(entry.hash);
+        }
+        self.bloom_filter = bloom_filter;
+    }
+
+    fn iter(&self) -> SetIter<'_> {
+        SetIter::open(self)
+    }
+
+    fn data(&self) -> &[u8] {
+        &self.buffer[Self::SET_HEADER_SIZE..self.size]
+    }
+}
+
+pub struct SetEntry<'a> {
+    offset: usize,
+    pub hash: u64,
+    pub key: &'a [u8],
+    pub value: &'a [u8],
+}
+
+impl<'a> SetEntry<'a> {
+    /// Length of the entry with header, key and value included.
+    pub fn len(&self) -> usize {
+        EntryHeader::ENTRY_HEADER_SIZE + self.key.len() + self.value.len()
+    }
+
+    /// Range of the entry in the set data.
+    #[expect(unused)]
+    pub fn range(&self) -> Range<usize> {
+        self.offset..self.offset + self.len()
+    }
+}
+
+pub struct SetIter<'a> {
+    set: &'a SetStorage,
+    offset: usize,
+}
+
+impl<'a> SetIter<'a> {
+    fn open(set: &'a SetStorage) -> Self {
+        Self { set, offset: 0 }
+    }
+
+    fn is_valid(&self) -> bool {
+        self.offset < self.set.len
+    }
+
+    fn next(&mut self) -> Option<SetEntry<'a>> {
+        if !self.is_valid() {
+            return None;
+        }
+        let mut cursor = self.offset;
+        let header = EntryHeader::read(&self.set.data()[cursor..cursor + EntryHeader::ENTRY_HEADER_SIZE]);
+        cursor += EntryHeader::ENTRY_HEADER_SIZE;
+        let value = &self.set.data()[cursor..cursor + header.value_len()];
+        cursor += header.value_len();
+        let key = &self.set.data()[cursor..cursor + header.key_len()];
+        let entry = SetEntry {
+            offset: self.offset,
+            hash: header.hash(),
+            key,
+            value,
+        };
+        self.offset += entry.len();
+        Some(entry)
+    }
+}
+
+impl<'a> Iterator for SetIter<'a> {
+    type Item = SetEntry<'a>;
+
+    fn next(&mut self) -> Option<Self::Item> {
+        self.next()
+    }
+}
+
+#[cfg(test)]
+mod tests {
+
+    use foyer_memory::{Cache, CacheBuilder, CacheEntry};
+
+    use super::*;
+    use crate::{serde::EntrySerializer, test_utils::metrics_for_test, Compression};
+
+    const PAGE: usize = 4096;
+
+    fn buffer(entry: &CacheEntry<u64, Vec<u8>>) -> IoBytes {
+        let mut buf = IoBytesMut::new();
+
+        // reserve header
+        let header = EntryHeader::new(0, 0, 0);
+        header.write(&mut buf);
+
+        let info = EntrySerializer::serialize(
+            entry.key(),
+            entry.value(),
+            &Compression::None,
+            &mut buf,
+            metrics_for_test(),
+        )
+        .unwrap();
+
+        let header = EntryHeader::new(entry.hash(), info.key_len, info.value_len);
+        header.write(&mut buf[0..EntryHeader::ENTRY_HEADER_SIZE]);
+
+        buf.freeze()
+    }
+
+    fn assert_some(storage: &SetStorage, entry: &CacheEntry<u64, Vec<u8>>) {
+        let ret = storage.get::<u64, Vec<u8>>(entry.hash()).unwrap();
+        let (k, v) = ret.unwrap();
+        assert_eq!(&k, entry.key());
+        assert_eq!(&v, entry.value());
+    }
+
+    fn assert_none(storage: &SetStorage, hash: u64) {
+        let ret = storage.get::<u64, Vec<u8>>(hash).unwrap();
+        assert!(ret.is_none());
+    }
+
+    fn memory_for_test() -> Cache<u64, Vec<u8>> {
+        CacheBuilder::new(100).build()
+    }
+
+    #[test]
+    #[should_panic]
+    fn test_set_storage_empty() {
+        let buffer = IoBytesMut::new();
+        SetStorage::load(buffer);
+    }
+
+    #[test]
+    fn test_set_storage_basic() {
+        let memory = memory_for_test();
+
+        let mut buf = IoBytesMut::with_capacity(PAGE);
+        unsafe { buf.set_len(PAGE) };
+
+        // load will result in an empty set
+        let mut storage = SetStorage::load(buf);
+        assert!(storage.is_empty());
+
+        let e1 = memory.insert(1, vec![b'1'; 42]);
+        let b1 = buffer(&e1);
+        storage.apply(
+            &HashSet::from_iter([2, 4]),
+            vec![Item {
+                buffer: b1.clone(),
+                entry: e1.clone(),
+            }],
+        );
+        assert_eq!(storage.len(), b1.len());
+        assert_some(&storage, &e1);
+
+        let e2 = memory.insert(2, vec![b'2'; 97]);
+        let b2 = buffer(&e2);
+        storage.apply(
+            &HashSet::from_iter([e1.hash(), 3, 5]),
+            vec![Item {
+                buffer: b2.clone(),
+                entry: e2.clone(),
+            }],
+        );
+        assert_eq!(storage.len(), b2.len());
+        assert_none(&storage, e1.hash());
+        assert_some(&storage, &e2);
+
+        let e3 = memory.insert(3, vec![b'3'; 211]);
+        let b3 = buffer(&e3);
+        storage.apply(
+            &HashSet::from_iter([e1.hash()]),
+            vec![Item {
+                buffer: b3.clone(),
+                entry: e3.clone(),
+            }],
+        );
+        assert_eq!(storage.len(), b2.len() + b3.len());
+        assert_none(&storage, e1.hash());
+        assert_some(&storage, &e2);
+        assert_some(&storage, &e3);
+
+        let e4 = memory.insert(4, vec![b'4'; 3800]);
+        let b4 = buffer(&e4);
+        storage.apply(
+            &HashSet::from_iter([e1.hash()]),
+            vec![Item {
+                buffer: b4.clone(),
+                entry: e4.clone(),
+            }],
+        );
+        assert_eq!(storage.len(), b4.len());
+        assert_none(&storage, e1.hash());
+        assert_none(&storage, e2.hash());
+        assert_none(&storage, e3.hash());
+        assert_some(&storage, &e4);
+
+        // test recovery
+        storage.update();
+        let bytes = storage.freeze();
+        let mut buf = IoBytesMut::with_capacity(PAGE);
+        unsafe { buf.set_len(PAGE) };
+        buf[0..bytes.len()].copy_from_slice(&bytes);
+        let mut storage = SetStorage::load(buf);
+
+        assert_eq!(storage.len(), b4.len());
+        assert_none(&storage, e1.hash());
+        assert_none(&storage, e2.hash());
+        assert_none(&storage, e3.hash());
+        assert_some(&storage, &e4);
+
+        // test oversize entry
+        let e5 = memory.insert(5, vec![b'5'; 20 * 1024]);
+        let b5 = buffer(&e5);
+        storage.apply(
+            &HashSet::new(),
+            vec![Item {
+                buffer: b5.clone(),
+                entry: e5.clone(),
+            }],
+        );
+        assert_eq!(storage.len(), b4.len());
+        assert_none(&storage, e1.hash());
+        assert_none(&storage, e2.hash());
+        assert_none(&storage, e3.hash());
+        assert_some(&storage, &e4);
+    }
+}
diff --git a/foyer-storage/src/small/set_manager.rs b/foyer-storage/src/small/set_manager.rs
new file mode 100644
index 00000000..441f0253
--- /dev/null
+++ b/foyer-storage/src/small/set_manager.rs
@@ -0,0 +1,253 @@
+//  Copyright 2024 Foyer Project Authors
+//
+//  Licensed under the Apache License, Version 2.0 (the "License");
+//  you may not use this file except in compliance with the License.
+//  You may obtain a copy of the License at
+//
+//  http://www.apache.org/licenses/LICENSE-2.0
+//
+//  Unless required by applicable law or agreed to in writing, software
+//  distributed under the License is distributed on an "AS IS" BASIS,
+//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//  See the License for the specific language governing permissions and
+//  limitations under the License.
+
+use std::{
+    fmt::Debug,
+    ops::{Deref, DerefMut, Range},
+    sync::Arc,
+};
+
+use foyer_common::strict_assert;
+use itertools::Itertools;
+use ordered_hash_map::OrderedHashMap;
+use tokio::sync::{Mutex, RwLock, RwLockReadGuard, RwLockWriteGuard};
+
+use super::{
+    bloom_filter::BloomFilterU64,
+    set::{Set, SetId, SetMut, SetStorage},
+};
+use crate::{
+    device::{MonitoredDevice, RegionId},
+    error::Result,
+    Dev,
+};
+
+struct SetManagerInner {
+    /// A phantom rwlock to prevent set storage operations on disk.
+    ///
+    /// All set disk operations must be prevented by the lock.
+    ///
+    /// In addition, the rwlock also serves as the lock of the in-memory bloom filter.
+    sets: Vec<RwLock<BloomFilterU64<4>>>,
+    cache: Mutex<OrderedHashMap<SetId, Arc<SetStorage>>>,
+    set_cache_capacity: usize,
+
+    set_size: usize,
+    device: MonitoredDevice,
+    regions: Range<RegionId>,
+    flush: bool,
+}
+
+#[derive(Clone)]
+pub struct SetManager {
+    inner: Arc<SetManagerInner>,
+}
+
+impl Debug for SetManager {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        f.debug_struct("SetManager")
+            .field("sets", &self.inner.sets)
+            .field("cache", &self.inner.cache)
+            .field("set_cache_capacity", &self.inner.set_cache_capacity)
+            .field("set_size", &self.inner.set_size)
+            .field("device", &self.inner.device)
+            .field("regions", &self.inner.regions)
+            .field("flush", &self.inner.flush)
+            .finish()
+    }
+}
+
+impl SetManager {
+    pub fn new(
+        set_size: usize,
+        set_cache_capacity: usize,
+        device: MonitoredDevice,
+        regions: Range<RegionId>,
+        flush: bool,
+    ) -> Self {
+        let sets = (device.region_size() / set_size) * (regions.end - regions.start) as usize;
+        assert!(sets > 0);
+
+        let sets = (0..sets).map(|_| RwLock::default()).collect_vec();
+        let cache = Mutex::new(OrderedHashMap::with_capacity(set_cache_capacity));
+
+        let inner = SetManagerInner {
+            sets,
+            cache,
+            set_cache_capacity,
+            set_size,
+            device,
+            regions,
+            flush,
+        };
+        let inner = Arc::new(inner);
+        Self { inner }
+    }
+
+    pub async fn write(&self, id: SetId) -> Result<SetWriteGuard<'_>> {
+        let guard = self.inner.sets[id as usize].write().await;
+
+        let invalid = self.inner.cache.lock().await.remove(&id);
+        let storage = match invalid {
+            // `guard` already guarantees that there is only one storage reference left.
+            Some(storage) => Arc::into_inner(storage).unwrap(),
+            None => self.storage(id).await?,
+        };
+
+        Ok(SetWriteGuard {
+            bloom_filter: guard,
+            id,
+            set: SetMut::new(storage),
+            drop: DropPanicGuard::default(),
+        })
+    }
+
+    pub async fn read(&self, id: SetId, hash: u64) -> Result<Option<SetReadGuard<'_>>> {
+        let guard = self.inner.sets[id as usize].read().await;
+        if !guard.lookup(hash) {
+            return Ok(None);
+        }
+
+        let mut cache = self.inner.cache.lock().await;
+        let cached = cache.get(&id).cloned();
+        let storage = match cached {
+            Some(storage) => storage,
+            None => {
+                let storage = self.storage(id).await?;
+                let storage = Arc::new(storage);
+                cache.insert(id, storage.clone());
+                if cache.len() > self.inner.set_cache_capacity {
+                    cache.pop_front();
+                    strict_assert!(cache.len() <= self.inner.set_cache_capacity);
+                }
+                storage
+            }
+        };
+        drop(cache);
+
+        Ok(Some(SetReadGuard {
+            _bloom_filter: guard,
+            _id: id,
+            set: Set::new(storage),
+        }))
+    }
+
+    pub async fn apply(&self, mut guard: SetWriteGuard<'_>) -> Result<()> {
+        let mut storage = guard.set.into_storage();
+
+        // Update in-memory bloom filter.
+        storage.update();
+        *guard.bloom_filter = storage.bloom_filter().clone();
+
+        let buffer = storage.freeze();
+
+        let (region, offset) = self.locate(guard.id);
+        self.inner.device.write(buffer, region, offset).await?;
+        if self.inner.flush {
+            self.inner.device.flush(Some(region)).await?;
+        }
+        guard.drop.disable();
+        drop(guard.bloom_filter);
+        Ok(())
+    }
+
+    pub async fn contains(&self, id: SetId, hash: u64) -> bool {
+        let guard = self.inner.sets[id as usize].read().await;
+        guard.lookup(hash)
+    }
+
+    pub fn sets(&self) -> usize {
+        self.inner.sets.len()
+    }
+
+    pub fn set_size(&self) -> usize {
+        self.inner.set_size
+    }
+
+    async fn storage(&self, id: SetId) -> Result<SetStorage> {
+        let (region, offset) = self.locate(id);
+        let buffer = self.inner.device.read(region, offset, self.inner.set_size).await?;
+        let storage = SetStorage::load(buffer);
+        Ok(storage)
+    }
+
+    #[inline]
+    fn region_sets(&self) -> usize {
+        self.inner.device.region_size() / self.inner.set_size
+    }
+
+    #[inline]
+    fn locate(&self, id: SetId) -> (RegionId, u64) {
+        let region_sets = self.region_sets();
+        let region = id as RegionId / region_sets as RegionId;
+        let offset = ((id as usize % region_sets) * self.inner.set_size) as u64;
+        (region, offset)
+    }
+}
+
+#[derive(Debug, Default)]
+struct DropPanicGuard {
+    disabled: bool,
+}
+
+impl Drop for DropPanicGuard {
+    fn drop(&mut self) {
+        if !self.disabled {
+            panic!("unexpected drop panic guard drop");
+        }
+    }
+}
+
+impl DropPanicGuard {
+    fn disable(&mut self) {
+        self.disabled = true;
+    }
+}
+
+#[derive(Debug)]
+pub struct SetWriteGuard<'a> {
+    bloom_filter: RwLockWriteGuard<'a, BloomFilterU64<4>>,
+    id: SetId,
+    set: SetMut,
+    drop: DropPanicGuard,
+}
+
+impl<'a> Deref for SetWriteGuard<'a> {
+    type Target = SetMut;
+
+    fn deref(&self) -> &Self::Target {
+        &self.set
+    }
+}
+
+impl<'a> DerefMut for SetWriteGuard<'a> {
+    fn deref_mut(&mut self) -> &mut Self::Target {
+        &mut self.set
+    }
+}
+
+#[derive(Debug)]
+pub struct SetReadGuard<'a> {
+    _bloom_filter: RwLockReadGuard<'a, BloomFilterU64<4>>,
+    _id: SetId,
+    set: Set,
+}
+
+impl<'a> Deref for SetReadGuard<'a> {
+    type Target = Set;
+
+    fn deref(&self) -> &Self::Target {
+        &self.set
+    }
+}
diff --git a/foyer-storage/src/storage/either.rs b/foyer-storage/src/storage/either.rs
index c6c34c16..30788549 100644
--- a/foyer-storage/src/storage/either.rs
+++ b/foyer-storage/src/storage/either.rs
@@ -21,11 +21,12 @@ use futures::{
     future::{join, ready, select, try_join, Either as EitherFuture},
     pin_mut, Future, FutureExt,
 };
+use serde::{Deserialize, Serialize};
 
 use crate::{error::Result, storage::Storage, DeviceStats};
 
 /// Order of ops.
-#[derive(Debug, Clone, Copy)]
+#[derive(Debug, Clone, Copy, Serialize, Deserialize)]
 pub enum Order {
     /// Use the left engine first.
     ///
diff --git a/foyer-storage/src/store.rs b/foyer-storage/src/store.rs
index 88a04d00..b4d5ae16 100644
--- a/foyer-storage/src/store.rs
+++ b/foyer-storage/src/store.rs
@@ -12,10 +12,19 @@
 //  See the License for the specific language governing permissions and
 //  limitations under the License.
 
-use std::{borrow::Borrow, fmt::Debug, hash::Hash, marker::PhantomData, sync::Arc, time::Instant};
+use std::{
+    borrow::Borrow,
+    fmt::{Debug, Display},
+    hash::Hash,
+    marker::PhantomData,
+    str::FromStr,
+    sync::Arc,
+    time::Instant,
+};
 
 use ahash::RandomState;
 use foyer_common::{
+    bits,
     code::{HashBuilder, StorageKey, StorageValue},
     metrics::Metrics,
     runtime::BackgroundShutdownRuntime,
@@ -29,9 +38,9 @@ use crate::{
     device::{
         direct_fs::DirectFsDeviceOptions,
         monitor::{DeviceStats, Monitored, MonitoredOptions},
-        DeviceOptions, RegionId,
+        DeviceOptions, RegionId, ALIGN,
     },
-    engine::{Engine, EngineConfig, SizeSelector},
+    engine::{EngineConfig, EngineEnum, SizeSelector},
     error::{Error, Result},
     large::{generic::GenericLargeStorageConfig, recover::RecoverMode, tombstone::TombstoneLogConfig},
     picker::{
@@ -67,7 +76,7 @@ where
 {
     memory: Cache<K, V, S>,
 
-    engine: Engine<K, V, S>,
+    engine: EngineEnum<K, V, S>,
 
     admission_picker: Arc<dyn AdmissionPicker<Key = K>>,
 
@@ -187,15 +196,6 @@ where
         self.inner.engine.stats()
     }
 
-    /// Get the runtime handles.
-    #[deprecated(
-        since = "0.11.5",
-        note = "The function will be renamed to \"runtime()\", use it instead."
-    )]
-    pub fn runtimes(&self) -> &Runtime {
-        &self.inner.runtime
-    }
-
     /// Get the runtime.
     pub fn runtime(&self) -> &Runtime {
         &self.inner.runtime
@@ -223,36 +223,38 @@ impl From<DirectFsDeviceOptions> for DeviceConfig {
     }
 }
 
-/// [`CombinedConfig`] controls the ratio of the large object disk cache and the small object disk cache.
+/// [`Engine`] controls the ratio of the large object disk cache and the small object disk cache.
 ///
-/// If [`CombinedConfig::Combined`] is used, it will use the `Either` engine
+/// If [`Engine::Mixed`] is used, it will use the `Either` engine
 /// with the small object disk cache as the left engine,
 /// and the large object disk cache as the right engine.
-#[derive(Debug, Clone)]
-pub enum CombinedConfig {
+#[derive(Debug, Clone, Copy, Serialize, Deserialize)]
+pub enum Engine {
     /// All space are used as the large object disk cache.
     Large,
     /// All space are used as the small object disk cache.
     Small,
-    /// Combined the large object disk cache and the small object disk cache.
-    Combined {
-        /// The ratio of the large object disk cache.
-        large_object_cache_ratio: f64,
-        /// The serialized entry size threshold to use the large object disk cache.
-        large_object_threshold: usize,
-        /// Load order.
-        load_order: Order,
-    },
+    /// Mixed the large object disk cache and the small object disk cache.
+    ///
+    /// The argument controls the ratio of the small object disk cache.
+    ///
+    /// Range: [0 ~ 1]
+    Mixed(f64),
 }
 
-impl Default for CombinedConfig {
+impl Default for Engine {
     fn default() -> Self {
-        // TODO(MrCroxx): Use combined cache after small object disk cache is ready.
+        // TODO(MrCroxx): Use Mixed cache after small object disk cache is ready.
         Self::Large
     }
 }
 
-impl CombinedConfig {
+impl Engine {
+    /// Threshold for distinguishing small and large objects.
+    pub const OBJECT_SIZE_THRESHOLD: usize = 2048;
+    /// Check the large object disk cache first, for checking it does NOT involve disk ops.
+    pub const MIXED_LOAD_ORDER: Order = Order::RightFirst;
+
     /// Default large object disk cache only config.
     pub fn large() -> Self {
         Self::Large
@@ -263,16 +265,44 @@ impl CombinedConfig {
         Self::Small
     }
 
-    /// Default combined large object disk cache and small object disk cache only config.
-    pub fn combined() -> Self {
-        Self::Combined {
-            large_object_cache_ratio: 0.5,
-            large_object_threshold: 4096,
-            load_order: Order::RightFirst,
+    /// Default mixed large object disk cache and small object disk cache config.
+    pub fn mixed() -> Self {
+        Self::Mixed(0.1)
+    }
+}
+
+impl Display for Engine {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        match self {
+            Engine::Large => write!(f, "large"),
+            Engine::Small => write!(f, "small"),
+            Engine::Mixed(ratio) => write!(f, "mixed({ratio})"),
         }
     }
 }
 
+impl FromStr for Engine {
+    type Err = String;
+
+    fn from_str(s: &str) -> std::result::Result<Self, Self::Err> {
+        const MIXED_PREFIX: &str = "mixed=";
+
+        match s {
+            "large" => return Ok(Engine::Large),
+            "small" => return Ok(Engine::Small),
+            _ => {}
+        }
+
+        if s.starts_with(MIXED_PREFIX) {
+            if let Ok(ratio) = s[MIXED_PREFIX.len()..s.len()].parse::<f64>() {
+                return Ok(Engine::Mixed(ratio));
+            }
+        }
+
+        Err(format!("invalid input: {s}"))
+    }
+}
+
 /// Tokio runtime configuration.
 #[derive(Debug, Clone, Default, Serialize, Deserialize)]
 pub struct TokioRuntimeConfig {
@@ -315,25 +345,20 @@ where
     V: StorageValue,
     S: HashBuilder + Debug,
 {
+    name: String,
     memory: Cache<K, V, S>,
 
-    name: String,
     device_config: DeviceConfig,
-    flush: bool,
-    indexer_shards: usize,
-    recover_mode: RecoverMode,
-    recover_concurrency: usize,
-    flushers: usize,
-    reclaimers: usize,
-    buffer_pool_size: usize,
-    clean_region_threshold: Option<usize>,
-    eviction_pickers: Vec<Box<dyn EvictionPicker>>,
+    engine: Engine,
+    runtime_config: RuntimeConfig,
+
     admission_picker: Arc<dyn AdmissionPicker<Key = K>>,
-    reinsertion_picker: Arc<dyn ReinsertionPicker<Key = K>>,
     compression: Compression,
-    tombstone_log_config: Option<TombstoneLogConfig>,
-    combined_config: CombinedConfig,
-    runtime_config: RuntimeConfig,
+    recover_mode: RecoverMode,
+    flush: bool,
+
+    large: LargeEngineOptions<K, V, S>,
+    small: SmallEngineOptions<K, V, S>,
 }
 
 impl<K, V, S> StoreBuilder<K, V, S>
@@ -343,26 +368,26 @@ where
     S: HashBuilder + Debug,
 {
     /// Setup disk cache store for the given in-memory cache.
-    pub fn new(memory: Cache<K, V, S>) -> Self {
+    pub fn new(memory: Cache<K, V, S>, engine: Engine) -> Self {
+        if matches!(engine, Engine::Mixed(ratio) if !(0.0..=1.0).contains(&ratio)) {
+            panic!("mixed engine small object disk cache ratio must be a f64 in range [0.0, 1.0]");
+        }
+
         Self {
-            memory,
             name: "foyer".to_string(),
+            memory,
+
             device_config: DeviceConfig::None,
-            flush: false,
-            indexer_shards: 64,
-            recover_mode: RecoverMode::Quiet,
-            recover_concurrency: 8,
-            flushers: 1,
-            reclaimers: 1,
-            buffer_pool_size: 16 * 1024 * 1024, // 16 MiB
-            clean_region_threshold: None,
-            eviction_pickers: vec![Box::new(InvalidRatioPicker::new(0.8)), Box::<FifoPicker>::default()],
+            engine,
+            runtime_config: RuntimeConfig::Disabled,
+
             admission_picker: Arc::<AdmitAllPicker<K>>::default(),
-            reinsertion_picker: Arc::<RejectAllPicker<K>>::default(),
             compression: Compression::None,
-            tombstone_log_config: None,
-            combined_config: CombinedConfig::default(),
-            runtime_config: RuntimeConfig::Disabled,
+            recover_mode: RecoverMode::Quiet,
+            flush: false,
+
+            large: LargeEngineOptions::new(),
+            small: SmallEngineOptions::new(),
         }
     }
 
@@ -390,11 +415,11 @@ where
         self
     }
 
-    /// Set the shard num of the indexer. Each shard has its own lock.
+    /// Set the compression algorithm of the disk cache store.
     ///
-    /// Default: `64`.
-    pub fn with_indexer_shards(mut self, indexer_shards: usize) -> Self {
-        self.indexer_shards = indexer_shards;
+    /// Default: [`Compression::None`].
+    pub fn with_compression(mut self, compression: Compression) -> Self {
+        self.compression = compression;
         self
     }
 
@@ -408,88 +433,6 @@ where
         self
     }
 
-    /// Set the recover concurrency for the disk cache store.
-    ///
-    /// Default: `8`.
-    pub fn with_recover_concurrency(mut self, recover_concurrency: usize) -> Self {
-        self.recover_concurrency = recover_concurrency;
-        self
-    }
-
-    /// Set the flusher count for the disk cache store.
-    ///
-    /// The flusher count limits how many regions can be concurrently written.
-    ///
-    /// Default: `1`.
-    pub fn with_flushers(mut self, flushers: usize) -> Self {
-        self.flushers = flushers;
-        self
-    }
-
-    /// Set the reclaimer count for the disk cache store.
-    ///
-    /// The reclaimer count limits how many regions can be concurrently reclaimed.
-    ///
-    /// Default: `1`.
-    pub fn with_reclaimers(mut self, reclaimers: usize) -> Self {
-        self.reclaimers = reclaimers;
-        self
-    }
-
-    // FIXME(MrCroxx): remove it after 0.12
-    /// Set the total flush buffer threshold.
-    ///
-    /// Each flusher shares a volume at `threshold / flushers`.
-    ///
-    /// If the buffer of the flush queue exceeds the threshold, the further entries will be ignored.
-    ///
-    /// Default: 16 MiB.
-    #[deprecated(
-        since = "0.11.4",
-        note = "The function will be renamed to \"with_buffer_pool_size()\", use it instead."
-    )]
-    pub fn with_buffer_threshold(mut self, threshold: usize) -> Self {
-        self.buffer_pool_size = threshold;
-        self
-    }
-
-    /// Set the total flush buffer pool size.
-    ///
-    /// Each flusher shares a volume at `threshold / flushers`.
-    ///
-    /// If the buffer of the flush queue exceeds the threshold, the further entries will be ignored.
-    ///
-    /// Default: 16 MiB.
-    pub fn with_buffer_pool_size(mut self, buffer_pool_size: usize) -> Self {
-        self.buffer_pool_size = buffer_pool_size;
-        self
-    }
-
-    /// Set the clean region threshold for the disk cache store.
-    ///
-    /// The reclaimers only work when the clean region count is equal to or lower than the clean region threshold.
-    ///
-    /// Default: the same value as the `reclaimers`.
-    pub fn with_clean_region_threshold(mut self, clean_region_threshold: usize) -> Self {
-        self.clean_region_threshold = Some(clean_region_threshold);
-        self
-    }
-
-    /// Set the eviction pickers for th disk cache store.
-    ///
-    /// The eviction picker is used to pick the region to reclaim.
-    ///
-    /// The eviction pickers are applied in order. If the previous eviction picker doesn't pick any region, the next one
-    /// will be applied.
-    ///
-    /// If no eviction picker picks a region, a region will be picked randomly.
-    ///
-    /// Default: [ invalid ratio picker { threshold = 0.8 }, fifo picker ]
-    pub fn with_eviction_pickers(mut self, eviction_pickers: Vec<Box<dyn EvictionPicker>>) -> Self {
-        self.eviction_pickers = eviction_pickers;
-        self
-    }
-
     /// Set the admission pickers for th disk cache store.
     ///
     /// The admission picker is used to pick the entries that can be inserted into the disk cache store.
@@ -500,57 +443,42 @@ where
         self
     }
 
-    /// Set the reinsertion pickers for th disk cache store.
-    ///
-    /// The reinsertion picker is used to pick the entries that can be reinsertion into the disk cache store while
-    /// reclaiming.
-    ///
-    /// Note: Only extremely important entries should be picked. If too many entries are picked, both insertion and
-    /// reinsertion will be stuck.
-    ///
-    /// Default: [`RejectAllPicker`].
-    pub fn with_reinsertion_picker(mut self, reinsertion_picker: Arc<dyn ReinsertionPicker<Key = K>>) -> Self {
-        self.reinsertion_picker = reinsertion_picker;
+    /// Configure the dedicated runtime for the disk cache store.
+    pub fn with_runtime_config(mut self, runtime_config: RuntimeConfig) -> Self {
+        self.runtime_config = runtime_config;
         self
     }
 
-    /// Set the compression algorithm of the disk cache store.
+    /// Setup the large object disk cache engine with the given options.
     ///
-    /// Default: [`Compression::None`].
-    pub fn with_compression(mut self, compression: Compression) -> Self {
-        self.compression = compression;
+    /// Otherwise, the default options will be used. See [`LargeEngineOptions`].
+    pub fn with_large_object_disk_cache_options(mut self, options: LargeEngineOptions<K, V, S>) -> Self {
+        if matches!(self.engine, Engine::Small { .. }) {
+            tracing::warn!("[store builder]: Setting up large object disk cache options, but only small object disk cache is enabled.");
+        }
+        self.large = options;
         self
     }
 
-    /// Enable the tombstone log with the given config.
+    /// Setup the small object disk cache engine with the given options.
     ///
-    /// For updatable cache, either the tombstone log or [`RecoverMode::None`] must be enabled to prevent from the
-    /// phantom entries after reopen.
-    pub fn with_tombstone_log_config(mut self, tombstone_log_config: TombstoneLogConfig) -> Self {
-        self.tombstone_log_config = Some(tombstone_log_config);
-        self
-    }
-
-    /// Set the ratio of the large object disk cache and the small object disk cache.
-    pub fn with_combined_config(mut self, combined_config: CombinedConfig) -> Self {
-        self.combined_config = combined_config;
-        self
-    }
-
-    /// Configure the dedicated runtime for the disk cache store.
-    pub fn with_runtime_config(mut self, runtime_config: RuntimeConfig) -> Self {
-        self.runtime_config = runtime_config;
+    /// Otherwise, the default options will be used. See [`SmallEngineOptions`].
+    pub fn with_small_object_disk_cache_options(mut self, options: SmallEngineOptions<K, V, S>) -> Self {
+        if matches!(self.engine, Engine::Large { .. }) {
+            tracing::warn!("[store builder]: Setting up small object disk cache options, but only large object disk cache is enabled.");
+        }
+        self.small = options;
         self
     }
 
     /// Build the disk cache store with the given configuration.
     pub async fn build(self) -> Result<Store<K, V, S>> {
-        let clean_region_threshold = self.clean_region_threshold.unwrap_or(self.reclaimers);
-
         let memory = self.memory.clone();
         let admission_picker = self.admission_picker.clone();
+
         let metrics = Arc::new(Metrics::new(&self.name));
         let statistics = Arc::<Statistics>::default();
+
         let compression = self.compression;
 
         let build_runtime = |config: &TokioRuntimeConfig, suffix: &str| {
@@ -596,94 +524,109 @@ where
             let runtime = runtime.clone();
             // Use the user runtime to open engine.
             tokio::spawn(async move {
-                match self.device_config {
-                    DeviceConfig::None => {
-                        tracing::warn!(
-                            "[store builder]: No device config set. Use `NoneStore` which always returns `None` for queries."
-                        );
-                        Engine::open(EngineConfig::Noop).await
-                    }
-                    DeviceConfig::DeviceOptions(options) => {
-                        let device = match Monitored::open(MonitoredOptions {
-                            options,
-                            metrics: metrics.clone(),
-                        }, runtime.clone())
-                        .await {
-                            Ok(device) => device,
-                            Err(e) =>return Err(e),
-                        };
-                        match self.combined_config {
-                            CombinedConfig::Large => {
-                                let regions = 0..device.regions() as RegionId;
-                                Engine::open(EngineConfig::Large(GenericLargeStorageConfig {
+            match self.device_config {
+                DeviceConfig::None => {
+                    tracing::warn!(
+                        "[store builder]: No device config set. Use `NoneStore` which always returns `None` for queries."
+                    );
+                    EngineEnum::open(EngineConfig::Noop).await
+                }
+                DeviceConfig::DeviceOptions(options) => {
+                    let device = match Monitored::open(MonitoredOptions {
+                        options,
+                        metrics: metrics.clone(),
+                    }, runtime.clone())
+                    .await {
+                        Ok(device) => device,
+                        Err(e) =>return Err(e),
+                    };
+                    match self.engine {
+                        Engine::Large => {
+                            let regions = 0..device.regions() as RegionId;
+                            EngineEnum::open(EngineConfig::Large(GenericLargeStorageConfig {
+                                name: self.name,
+                                device,
+                                regions,
+                                compression: self.compression,
+                                flush: self.flush,
+                                indexer_shards: self.large.indexer_shards,
+                                recover_mode: self.recover_mode,
+                                recover_concurrency: self.large.recover_concurrency,
+                                flushers: self.large.flushers,
+                                reclaimers: self.large.reclaimers,
+                                clean_region_threshold: self.large.clean_region_threshold.unwrap_or(self.large.reclaimers),
+                                eviction_pickers: self.large.eviction_pickers,
+                                reinsertion_picker: self.large.reinsertion_picker,
+                                tombstone_log_config: self.large.tombstone_log_config,
+                                buffer_pool_size: self.large.buffer_pool_size,
+                                statistics: statistics.clone(),
+                                runtime,
+                                marker: PhantomData,
+                            }))
+                            .await
+                        }
+                        Engine::Small => {
+                            let regions = 0..device.regions() as RegionId;
+                            EngineEnum::open(EngineConfig::Small(GenericSmallStorageConfig {
+                                set_size: self.small.set_size,
+                                set_cache_capacity: self.small.set_cache_capacity,
+                                device,
+                                regions,
+                                flush: self.flush,
+                                flushers: self.small.flushers,
+                                buffer_pool_size: self.small.buffer_pool_size,
+                                statistics: statistics.clone(),
+                                runtime,
+                                marker: PhantomData,
+                            }))
+                            .await
+                        }
+                        Engine::Mixed(ratio) => {
+                            let small_region_count = std::cmp::max((device.regions() as f64 * ratio) as usize,1);
+                            let small_regions = 0..small_region_count as RegionId;
+                            let large_regions = small_region_count as RegionId..device.regions() as RegionId;
+                            EngineEnum::open(EngineConfig::Mixed(EitherConfig {
+                                selector: SizeSelector::new(Engine::OBJECT_SIZE_THRESHOLD),
+                                left: GenericSmallStorageConfig {
+                                    set_size: self.small.set_size,
+                                    set_cache_capacity: self.small.set_cache_capacity,
+                                    device: device.clone(),
+                                    regions: small_regions,
+                                    flush: self.flush,
+                                    flushers: self.small.flushers,
+                                    buffer_pool_size: self.small.buffer_pool_size,
+                                    statistics: statistics.clone(),
+                                    runtime: runtime.clone(),
+                                    marker: PhantomData,
+                                },
+                                right: GenericLargeStorageConfig {
                                     name: self.name,
                                     device,
-                                    regions,
+                                    regions: large_regions,
                                     compression: self.compression,
                                     flush: self.flush,
-                                    indexer_shards: self.indexer_shards,
+                                    indexer_shards: self.large.indexer_shards,
                                     recover_mode: self.recover_mode,
-                                    recover_concurrency: self.recover_concurrency,
-                                    flushers: self.flushers,
-                                    reclaimers: self.reclaimers,
-                                    clean_region_threshold,
-                                    eviction_pickers: self.eviction_pickers,
-                                    reinsertion_picker: self.reinsertion_picker,
-                                    tombstone_log_config: self.tombstone_log_config,
-                                    buffer_threshold: self.buffer_pool_size,
+                                    recover_concurrency: self.large.recover_concurrency,
+                                    flushers: self.large.flushers,
+                                    reclaimers: self.large.reclaimers,
+                                    clean_region_threshold: self.large.clean_region_threshold.unwrap_or(self.large.reclaimers),
+                                    eviction_pickers: self.large.eviction_pickers,
+                                    reinsertion_picker: self.large.reinsertion_picker,
+                                    tombstone_log_config: self.large.tombstone_log_config,
+                                    buffer_pool_size: self.large.buffer_pool_size,
                                     statistics: statistics.clone(),
                                     runtime,
                                     marker: PhantomData,
-                                }))
-                                .await
-                            }
-                            CombinedConfig::Small => {
-                                Engine::open(EngineConfig::Small(GenericSmallStorageConfig {
-                                    placeholder: PhantomData,
-                                }))
-                                .await
-                            }
-                            CombinedConfig::Combined {
-                                large_object_cache_ratio,
-                                large_object_threshold,
-                                load_order,
-                            } => {
-                                let large_region_count = (device.regions() as f64 * large_object_cache_ratio) as usize;
-                                let large_regions =
-                                    (device.regions() - large_region_count) as RegionId..device.regions() as RegionId;
-                                Engine::open(EngineConfig::Combined(EitherConfig {
-                                    selector: SizeSelector::new(large_object_threshold),
-                                    left: GenericSmallStorageConfig {
-                                        placeholder: PhantomData,
-                                    },
-                                    right: GenericLargeStorageConfig {
-                                        name: self.name,
-                                        device,
-                                        regions: large_regions,
-                                        compression: self.compression,
-                                        flush: self.flush,
-                                        indexer_shards: self.indexer_shards,
-                                        recover_mode: self.recover_mode,
-                                        recover_concurrency: self.recover_concurrency,
-                                        flushers: self.flushers,
-                                        reclaimers: self.reclaimers,
-                                        clean_region_threshold,
-                                        eviction_pickers: self.eviction_pickers,
-                                        reinsertion_picker: self.reinsertion_picker,
-                                        tombstone_log_config: self.tombstone_log_config,
-                                        buffer_threshold: self.buffer_pool_size,
-                                        statistics: statistics.clone(),
-                                        runtime,
-                                        marker: PhantomData,
-                                    },
-                                    load_order,
-                                }))
-                                .await
-                            }
+                                },
+                                load_order: Engine::MIXED_LOAD_ORDER,
+                            }))
+                            .await
                         }
                     }
                 }
-            }).await.unwrap()?
+            }
+        }).await.unwrap()?
         };
 
         let inner = StoreInner {
@@ -701,3 +644,241 @@ where
         Ok(store)
     }
 }
+
+/// Large object disk cache engine default options.
+pub struct LargeEngineOptions<K, V, S = RandomState>
+where
+    K: StorageKey,
+    V: StorageValue,
+    S: HashBuilder + Debug,
+{
+    indexer_shards: usize,
+    recover_concurrency: usize,
+    flushers: usize,
+    reclaimers: usize,
+    buffer_pool_size: usize,
+    clean_region_threshold: Option<usize>,
+    eviction_pickers: Vec<Box<dyn EvictionPicker>>,
+    reinsertion_picker: Arc<dyn ReinsertionPicker<Key = K>>,
+    tombstone_log_config: Option<TombstoneLogConfig>,
+
+    _marker: PhantomData<(K, V, S)>,
+}
+
+impl<K, V, S> Default for LargeEngineOptions<K, V, S>
+where
+    K: StorageKey,
+    V: StorageValue,
+    S: HashBuilder + Debug,
+{
+    fn default() -> Self {
+        Self::new()
+    }
+}
+
+impl<K, V, S> LargeEngineOptions<K, V, S>
+where
+    K: StorageKey,
+    V: StorageValue,
+    S: HashBuilder + Debug,
+{
+    /// Create large object disk cache engine default options.
+    pub fn new() -> Self {
+        Self {
+            indexer_shards: 64,
+            recover_concurrency: 8,
+            flushers: 1,
+            reclaimers: 1,
+            buffer_pool_size: 16 * 1024 * 1024, // 16 MiB
+            clean_region_threshold: None,
+            eviction_pickers: vec![Box::new(InvalidRatioPicker::new(0.8)), Box::<FifoPicker>::default()],
+            reinsertion_picker: Arc::<RejectAllPicker<K>>::default(),
+            tombstone_log_config: None,
+            _marker: PhantomData,
+        }
+    }
+
+    /// Set the shard num of the indexer. Each shard has its own lock.
+    ///
+    /// Default: `64`.
+    pub fn with_indexer_shards(mut self, indexer_shards: usize) -> Self {
+        self.indexer_shards = indexer_shards;
+        self
+    }
+
+    /// Set the recover concurrency for the disk cache store.
+    ///
+    /// Default: `8`.
+    pub fn with_recover_concurrency(mut self, recover_concurrency: usize) -> Self {
+        self.recover_concurrency = recover_concurrency;
+        self
+    }
+
+    /// Set the flusher count for the disk cache store.
+    ///
+    /// The flusher count limits how many regions can be concurrently written.
+    ///
+    /// Default: `1`.
+    pub fn with_flushers(mut self, flushers: usize) -> Self {
+        self.flushers = flushers;
+        self
+    }
+
+    /// Set the reclaimer count for the disk cache store.
+    ///
+    /// The reclaimer count limits how many regions can be concurrently reclaimed.
+    ///
+    /// Default: `1`.
+    pub fn with_reclaimers(mut self, reclaimers: usize) -> Self {
+        self.reclaimers = reclaimers;
+        self
+    }
+
+    /// Set the total flush buffer pool size.
+    ///
+    /// Each flusher shares a volume at `threshold / flushers`.
+    ///
+    /// If the buffer of the flush queue exceeds the threshold, the further entries will be ignored.
+    ///
+    /// Default: 16 MiB.
+    pub fn with_buffer_pool_size(mut self, buffer_pool_size: usize) -> Self {
+        self.buffer_pool_size = buffer_pool_size;
+        self
+    }
+
+    /// Set the clean region threshold for the disk cache store.
+    ///
+    /// The reclaimers only work when the clean region count is equal to or lower than the clean region threshold.
+    ///
+    /// Default: the same value as the `reclaimers`.
+    pub fn with_clean_region_threshold(mut self, clean_region_threshold: usize) -> Self {
+        self.clean_region_threshold = Some(clean_region_threshold);
+        self
+    }
+
+    /// Set the eviction pickers for th disk cache store.
+    ///
+    /// The eviction picker is used to pick the region to reclaim.
+    ///
+    /// The eviction pickers are applied in order. If the previous eviction picker doesn't pick any region, the next one
+    /// will be applied.
+    ///
+    /// If no eviction picker picks a region, a region will be picked randomly.
+    ///
+    /// Default: [ invalid ratio picker { threshold = 0.8 }, fifo picker ]
+    pub fn with_eviction_pickers(mut self, eviction_pickers: Vec<Box<dyn EvictionPicker>>) -> Self {
+        self.eviction_pickers = eviction_pickers;
+        self
+    }
+
+    /// Set the reinsertion pickers for th disk cache store.
+    ///
+    /// The reinsertion picker is used to pick the entries that can be reinsertion into the disk cache store while
+    /// reclaiming.
+    ///
+    /// Note: Only extremely important entries should be picked. If too many entries are picked, both insertion and
+    /// reinsertion will be stuck.
+    ///
+    /// Default: [`RejectAllPicker`].
+    pub fn with_reinsertion_picker(mut self, reinsertion_picker: Arc<dyn ReinsertionPicker<Key = K>>) -> Self {
+        self.reinsertion_picker = reinsertion_picker;
+        self
+    }
+
+    /// Enable the tombstone log with the given config.
+    ///
+    /// For updatable cache, either the tombstone log or [`RecoverMode::None`] must be enabled to prevent from the
+    /// phantom entries after reopen.
+    pub fn with_tombstone_log_config(mut self, tombstone_log_config: TombstoneLogConfig) -> Self {
+        self.tombstone_log_config = Some(tombstone_log_config);
+        self
+    }
+}
+
+/// Small object disk cache engine default options.
+pub struct SmallEngineOptions<K, V, S = RandomState>
+where
+    K: StorageKey,
+    V: StorageValue,
+    S: HashBuilder + Debug,
+{
+    set_size: usize,
+    set_cache_capacity: usize,
+    buffer_pool_size: usize,
+    flushers: usize,
+
+    _marker: PhantomData<(K, V, S)>,
+}
+
+impl<K, V, S> Default for SmallEngineOptions<K, V, S>
+where
+    K: StorageKey,
+    V: StorageValue,
+    S: HashBuilder + Debug,
+{
+    fn default() -> Self {
+        Self::new()
+    }
+}
+
+/// Create small object disk cache engine default options.
+impl<K, V, S> SmallEngineOptions<K, V, S>
+where
+    K: StorageKey,
+    V: StorageValue,
+    S: HashBuilder + Debug,
+{
+    /// Create small object disk cache engine default options.
+    pub fn new() -> Self {
+        Self {
+            set_size: 16 * 1024,    // 16 KiB
+            set_cache_capacity: 64, // 64 sets
+            flushers: 1,
+            buffer_pool_size: 4 * 1024 * 1024, // 4 MiB
+            _marker: PhantomData,
+        }
+    }
+
+    /// Set the set size of the set-associated cache.
+    ///
+    /// The set size will be 4K aligned.
+    ///
+    /// Default: 16 KiB
+    pub fn with_set_size(mut self, set_size: usize) -> Self {
+        bits::assert_aligned(ALIGN, set_size);
+        self.set_size = set_size;
+        self
+    }
+
+    /// Set the capacity of the set cache.
+    ///
+    /// Count by set amount.
+    ///
+    /// Default: 64
+    pub fn with_set_cache_capacity(mut self, set_cache_capacity: usize) -> Self {
+        self.set_cache_capacity = set_cache_capacity;
+        self
+    }
+
+    /// Set the total flush buffer pool size.
+    ///
+    /// Each flusher shares a volume at `threshold / flushers`.
+    ///
+    /// If the buffer of the flush queue exceeds the threshold, the further entries will be ignored.
+    ///
+    /// Default: 4 MiB.
+    pub fn with_buffer_pool_size(mut self, buffer_pool_size: usize) -> Self {
+        self.buffer_pool_size = buffer_pool_size;
+        self
+    }
+
+    /// Set the flusher count for the disk cache store.
+    ///
+    /// The flusher count limits how many regions can be concurrently written.
+    ///
+    /// Default: `1`.
+    pub fn with_flushers(mut self, flushers: usize) -> Self {
+        self.flushers = flushers;
+        self
+    }
+}
diff --git a/foyer-storage/tests/storage_test.rs b/foyer-storage/tests/storage_test.rs
index e7983577..e57bbf9c 100644
--- a/foyer-storage/tests/storage_test.rs
+++ b/foyer-storage/tests/storage_test.rs
@@ -18,7 +18,9 @@ use std::{path::Path, sync::Arc, time::Duration};
 
 use ahash::RandomState;
 use foyer_memory::{Cache, CacheBuilder, CacheEntry, FifoConfig};
-use foyer_storage::{test_utils::Recorder, Compression, DirectFsDeviceOptionsBuilder, StoreBuilder};
+use foyer_storage::{
+    test_utils::Recorder, Compression, DirectFsDeviceOptionsBuilder, Engine, LargeEngineOptions, StoreBuilder,
+};
 
 const KB: usize = 1024;
 const MB: usize = 1024 * 1024;
@@ -106,18 +108,22 @@ fn basic(
     path: impl AsRef<Path>,
     recorder: &Arc<Recorder<u64>>,
 ) -> StoreBuilder<u64, Vec<u8>> {
-    StoreBuilder::new(memory.clone())
+    // TODO(MrCroxx): Test mixed engine here.
+    StoreBuilder::new(memory.clone(), Engine::Large)
         .with_device_config(
             DirectFsDeviceOptionsBuilder::new(path)
                 .with_capacity(4 * MB)
                 .with_file_size(MB)
                 .build(),
         )
-        .with_indexer_shards(4)
         .with_admission_picker(recorder.clone())
-        .with_reinsertion_picker(recorder.clone())
-        .with_recover_concurrency(2)
         .with_flush(true)
+        .with_large_object_disk_cache_options(
+            LargeEngineOptions::new()
+                .with_recover_concurrency(2)
+                .with_indexer_shards(4)
+                .with_reinsertion_picker(recorder.clone()),
+        )
 }
 
 #[test_log::test(tokio::test)]
diff --git a/foyer/src/hybrid/builder.rs b/foyer/src/hybrid/builder.rs
index 34eb69be..ab01c3c1 100644
--- a/foyer/src/hybrid/builder.rs
+++ b/foyer/src/hybrid/builder.rs
@@ -22,8 +22,8 @@ use foyer_common::{
 };
 use foyer_memory::{Cache, CacheBuilder, EvictionConfig, Weighter};
 use foyer_storage::{
-    AdmissionPicker, Compression, DeviceConfig, EvictionPicker, RecoverMode, ReinsertionPicker, RuntimeConfig,
-    StoreBuilder, TombstoneLogConfig,
+    AdmissionPicker, Compression, DeviceConfig, Engine, LargeEngineOptions, RecoverMode, RuntimeConfig,
+    SmallEngineOptions, StoreBuilder,
 };
 
 use crate::HybridCache;
@@ -173,11 +173,11 @@ where
         }
     }
 
-    /// Continue to modify the in-memory cache configurations.
-    pub fn storage(self) -> HybridCacheBuilderPhaseStorage<K, V, S> {
+    /// Continue to modify the disk cache configurations.
+    pub fn storage(self, engine: Engine) -> HybridCacheBuilderPhaseStorage<K, V, S> {
         let memory = self.builder.build();
         HybridCacheBuilderPhaseStorage {
-            builder: StoreBuilder::new(memory.clone()).with_name(&self.name),
+            builder: StoreBuilder::new(memory.clone(), engine).with_name(&self.name),
             name: self.name,
             tracing_config: self.tracing_config,
             memory,
@@ -228,19 +228,6 @@ where
         }
     }
 
-    /// Set the shard num of the indexer. Each shard has its own lock.
-    ///
-    /// Default: `64`.
-    pub fn with_indexer_shards(self, indexer_shards: usize) -> Self {
-        let builder = self.builder.with_indexer_shards(indexer_shards);
-        Self {
-            name: self.name,
-            tracing_config: self.tracing_config,
-            memory: self.memory,
-            builder,
-        }
-    }
-
     /// Set the recover mode for the disk cache store.
     ///
     /// See more in [`RecoverMode`].
@@ -256,123 +243,6 @@ where
         }
     }
 
-    /// Set the recover concurrency for the disk cache store.
-    ///
-    /// Default: `8`.
-    pub fn with_recover_concurrency(self, recover_concurrency: usize) -> Self {
-        let builder = self.builder.with_recover_concurrency(recover_concurrency);
-        Self {
-            name: self.name,
-            tracing_config: self.tracing_config,
-            memory: self.memory,
-            builder,
-        }
-    }
-
-    /// Set the flusher count for the disk cache store.
-    ///
-    /// The flusher count limits how many regions can be concurrently written.
-    ///
-    /// Default: `1`.
-    pub fn with_flushers(self, flushers: usize) -> Self {
-        let builder = self.builder.with_flushers(flushers);
-        Self {
-            name: self.name,
-            tracing_config: self.tracing_config,
-            memory: self.memory,
-            builder,
-        }
-    }
-
-    /// Set the reclaimer count for the disk cache store.
-    ///
-    /// The reclaimer count limits how many regions can be concurrently reclaimed.
-    ///
-    /// Default: `1`.
-    pub fn with_reclaimers(self, reclaimers: usize) -> Self {
-        let builder = self.builder.with_reclaimers(reclaimers);
-        Self {
-            name: self.name,
-            tracing_config: self.tracing_config,
-            memory: self.memory,
-            builder,
-        }
-    }
-
-    // FIXME(MrCroxx): remove it after 0.12
-    /// Set the total flush buffer threshold.
-    ///
-    /// Each flusher shares a volume at `threshold / flushers`.
-    ///
-    /// If the buffer of the flush queue exceeds the threshold, the further entries will be ignored.
-    ///
-    /// Default: 16 MiB.
-    #[deprecated(
-        since = "0.11.4",
-        note = "The function will be renamed to \"with_buffer_pool_size()\", use it instead."
-    )]
-    pub fn with_buffer_threshold(self, threshold: usize) -> Self {
-        let builder = self.builder.with_buffer_pool_size(threshold);
-        Self {
-            name: self.name,
-            tracing_config: self.tracing_config,
-            memory: self.memory,
-            builder,
-        }
-    }
-
-    /// Set the total flush buffer pool size.
-    ///
-    /// Each flusher shares a volume at `threshold / flushers`.
-    ///
-    /// If the buffer of the flush queue exceeds the threshold, the further entries will be ignored.
-    ///
-    /// Default: 16 MiB.
-    pub fn with_buffer_pool_size(self, buffer_pool_size: usize) -> Self {
-        let builder = self.builder.with_buffer_pool_size(buffer_pool_size);
-        Self {
-            name: self.name,
-            tracing_config: self.tracing_config,
-            memory: self.memory,
-            builder,
-        }
-    }
-
-    /// Set the clean region threshold for the disk cache store.
-    ///
-    /// The reclaimers only work when the clean region count is equal to or lower than the clean region threshold.
-    ///
-    /// Default: the same value as the `reclaimers`.
-    pub fn with_clean_region_threshold(self, clean_region_threshold: usize) -> Self {
-        let builder = self.builder.with_clean_region_threshold(clean_region_threshold);
-        Self {
-            name: self.name,
-            tracing_config: self.tracing_config,
-            memory: self.memory,
-            builder,
-        }
-    }
-
-    /// Set the eviction pickers for th disk cache store.
-    ///
-    /// The eviction picker is used to pick the region to reclaim.
-    ///
-    /// The eviction pickers are applied in order. If the previous eviction picker doesn't pick any region, the next one
-    /// will be applied.
-    ///
-    /// If no eviction picker picks a region, a region will be picked randomly.
-    ///
-    /// Default: [ invalid ratio picker { threshold = 0.8 }, fifo picker ]
-    pub fn with_eviction_pickers(self, eviction_pickers: Vec<Box<dyn EvictionPicker>>) -> Self {
-        let builder = self.builder.with_eviction_pickers(eviction_pickers);
-        Self {
-            name: self.name,
-            tracing_config: self.tracing_config,
-            memory: self.memory,
-            builder,
-        }
-    }
-
     /// Set the admission pickers for th disk cache store.
     ///
     /// The admission picker is used to pick the entries that can be inserted into the disk cache store.
@@ -388,17 +258,11 @@ where
         }
     }
 
-    /// Set the reinsertion pickers for th disk cache store.
-    ///
-    /// The reinsertion picker is used to pick the entries that can be reinsertion into the disk cache store while
-    /// reclaiming.
-    ///
-    /// Note: Only extremely important entries should be picked. If too many entries are picked, both insertion and
-    /// reinsertion will be stuck.
+    /// Set the compression algorithm of the disk cache store.
     ///
-    /// Default: [`RejectAllPicker`].
-    pub fn with_reinsertion_picker(self, reinsertion_picker: Arc<dyn ReinsertionPicker<Key = K>>) -> Self {
-        let builder = self.builder.with_reinsertion_picker(reinsertion_picker);
+    /// Default: [`Compression::None`].
+    pub fn with_compression(self, compression: Compression) -> Self {
+        let builder = self.builder.with_compression(compression);
         Self {
             name: self.name,
             tracing_config: self.tracing_config,
@@ -407,11 +271,9 @@ where
         }
     }
 
-    /// Set the compression algorithm of the disk cache store.
-    ///
-    /// Default: [`Compression::None`].
-    pub fn with_compression(self, compression: Compression) -> Self {
-        let builder = self.builder.with_compression(compression);
+    /// Configure the dedicated runtime for the disk cache store.
+    pub fn with_runtime_config(self, runtime_config: RuntimeConfig) -> Self {
+        let builder = self.builder.with_runtime_config(runtime_config);
         Self {
             name: self.name,
             tracing_config: self.tracing_config,
@@ -420,12 +282,11 @@ where
         }
     }
 
-    /// Enable the tombstone log with the given config.
+    /// Setup the large object disk cache engine with the given options.
     ///
-    /// For updatable cache, either the tombstone log or [`RecoverMode::None`] must be enabled to prevent from the
-    /// phantom entries after reopen.
-    pub fn with_tombstone_log_config(self, tombstone_log_config: TombstoneLogConfig) -> Self {
-        let builder = self.builder.with_tombstone_log_config(tombstone_log_config);
+    /// Otherwise, the default options will be used. See [`LargeEngineOptions`].
+    pub fn with_large_object_disk_cache_options(self, options: LargeEngineOptions<K, V, S>) -> Self {
+        let builder = self.builder.with_large_object_disk_cache_options(options);
         Self {
             name: self.name,
             tracing_config: self.tracing_config,
@@ -434,9 +295,11 @@ where
         }
     }
 
-    /// Configure the dedicated runtime for the disk cache store.
-    pub fn with_runtime_config(self, runtime_config: RuntimeConfig) -> Self {
-        let builder = self.builder.with_runtime_config(runtime_config);
+    /// Setup the small object disk cache engine with the given options.
+    ///
+    /// Otherwise, the default options will be used. See [`SmallEngineOptions`].
+    pub fn with_small_object_disk_cache_options(self, options: SmallEngineOptions<K, V, S>) -> Self {
+        let builder = self.builder.with_small_object_disk_cache_options(options);
         Self {
             name: self.name,
             tracing_config: self.tracing_config,
diff --git a/foyer/src/hybrid/cache.rs b/foyer/src/hybrid/cache.rs
index 5dbc3e20..8e726832 100644
--- a/foyer/src/hybrid/cache.rs
+++ b/foyer/src/hybrid/cache.rs
@@ -549,7 +549,8 @@ mod tests {
         HybridCacheBuilder::new()
             .with_name("test")
             .memory(4 * MB)
-            .storage()
+            // TODO(MrCroxx): Test with `Engine::Mixed`.
+            .storage(Engine::Large)
             .with_device_config(
                 DirectFsDeviceOptionsBuilder::new(dir)
                     .with_capacity(16 * MB)
@@ -572,7 +573,8 @@ mod tests {
         HybridCacheBuilder::new()
             .with_name("test")
             .memory(4 * MB)
-            .storage()
+            // TODO(MrCroxx): Test with `Engine::Mixed`.
+            .storage(Engine::Large)
             .with_device_config(
                 DirectFsDeviceOptionsBuilder::new(dir)
                     .with_capacity(16 * MB)
diff --git a/foyer/src/prelude.rs b/foyer/src/prelude.rs
index d8af4d38..a9c426fa 100644
--- a/foyer/src/prelude.rs
+++ b/foyer/src/prelude.rs
@@ -26,9 +26,9 @@ pub use memory::{
 pub use storage::{
     AdmissionPicker, AdmitAllPicker, Compression, Dev, DevExt, DevOptions, DeviceStats, DirectFileDevice,
     DirectFileDeviceOptions, DirectFileDeviceOptionsBuilder, DirectFsDevice, DirectFsDeviceOptions,
-    DirectFsDeviceOptionsBuilder, EvictionPicker, FifoPicker, InvalidRatioPicker, RateLimitPicker, RecoverMode,
-    ReinsertionPicker, RejectAllPicker, Runtime, RuntimeConfig, Storage, Store, StoreBuilder, TokioRuntimeConfig,
-    TombstoneLogConfigBuilder,
+    DirectFsDeviceOptionsBuilder, Engine, EvictionPicker, FifoPicker, InvalidRatioPicker, LargeEngineOptions,
+    RateLimitPicker, RecoverMode, ReinsertionPicker, RejectAllPicker, Runtime, RuntimeConfig, SmallEngineOptions,
+    Storage, Store, StoreBuilder, TokioRuntimeConfig, TombstoneLogConfigBuilder,
 };
 
 pub use crate::hybrid::{

From 88089d7f2b81509ce979eb2b25b5e47a7677ea35 Mon Sep 17 00:00:00 2001
From: Croxx <mrcroxx@outlook.com>
Date: Thu, 26 Sep 2024 16:03:44 +0800
Subject: [PATCH 18/53] chore: use workspace.package to manage crate meta
 (#740)

Besides, replace all "Foyer" with "foyer".

Signed-off-by: MrCroxx <mrcroxx@outlook.com>
---
 .licenserc.yaml                               |  2 +-
 Cargo.toml                                    | 17 +++++++++++++
 LICENSE                                       |  2 +-
 examples/Cargo.toml                           | 22 ++++++++--------
 examples/event_listener.rs                    |  2 +-
 examples/hybrid.rs                            |  2 +-
 examples/hybrid_full.rs                       |  2 +-
 examples/memory.rs                            |  2 +-
 examples/tail_based_tracing.rs                |  2 +-
 foyer-bench/Cargo.toml                        | 20 ++++++++-------
 foyer-bench/src/analyze.rs                    |  2 +-
 foyer-bench/src/main.rs                       |  2 +-
 foyer-bench/src/rate.rs                       |  2 +-
 foyer-bench/src/text.rs                       |  2 +-
 foyer-cli/Cargo.toml                          | 19 ++++++++------
 foyer-cli/src/args/error.rs                   |  2 +-
 foyer-cli/src/args/fio.rs                     |  2 +-
 foyer-cli/src/args/mod.rs                     |  2 +-
 foyer-cli/src/main.rs                         |  2 +-
 foyer-common/Cargo.toml                       | 18 +++++++------
 foyer-common/src/assert.rs                    |  2 +-
 foyer-common/src/asyncify.rs                  |  2 +-
 foyer-common/src/bits.rs                      |  2 +-
 foyer-common/src/buf.rs                       |  2 +-
 foyer-common/src/code.rs                      |  2 +-
 foyer-common/src/countdown.rs                 |  2 +-
 foyer-common/src/event.rs                     |  2 +-
 foyer-common/src/future.rs                    |  2 +-
 foyer-common/src/lib.rs                       |  2 +-
 foyer-common/src/metrics.rs                   |  2 +-
 foyer-common/src/object_pool.rs               |  2 +-
 foyer-common/src/range.rs                     |  2 +-
 foyer-common/src/rate.rs                      |  2 +-
 foyer-common/src/rated_ticket.rs              |  2 +-
 foyer-common/src/runtime.rs                   |  2 +-
 foyer-common/src/tracing.rs                   |  2 +-
 foyer-common/src/wait_group.rs                |  2 +-
 foyer-intrusive/Cargo.toml                    | 20 ++++++++-------
 foyer-intrusive/src/adapter.rs                |  2 +-
 foyer-intrusive/src/dlist.rs                  |  2 +-
 foyer-intrusive/src/lib.rs                    |  2 +-
 foyer-memory/Cargo.toml                       | 22 ++++++++--------
 .../benches/bench_dynamic_dispatch.rs         |  2 +-
 foyer-memory/benches/bench_hit_ratio.rs       |  2 +-
 foyer-memory/src/cache.rs                     |  4 +--
 foyer-memory/src/context.rs                   |  2 +-
 foyer-memory/src/eviction/fifo.rs             |  2 +-
 foyer-memory/src/eviction/lfu.rs              |  2 +-
 foyer-memory/src/eviction/lru.rs              |  2 +-
 foyer-memory/src/eviction/mod.rs              |  2 +-
 foyer-memory/src/eviction/s3fifo.rs           |  2 +-
 foyer-memory/src/eviction/sanity.rs           |  2 +-
 foyer-memory/src/eviction/test_utils.rs       |  2 +-
 foyer-memory/src/generic.rs                   |  2 +-
 foyer-memory/src/handle.rs                    |  2 +-
 foyer-memory/src/indexer/hash_table.rs        |  2 +-
 foyer-memory/src/indexer/mod.rs               |  2 +-
 foyer-memory/src/indexer/sanity.rs            |  2 +-
 foyer-memory/src/lib.rs                       |  2 +-
 foyer-memory/src/prelude.rs                   |  2 +-
 foyer-storage/Cargo.toml                      | 22 ++++++++--------
 foyer-storage/src/compress.rs                 |  2 +-
 foyer-storage/src/device/allocator.rs         |  2 +-
 foyer-storage/src/device/bytes.rs             |  2 +-
 foyer-storage/src/device/direct_file.rs       |  2 +-
 foyer-storage/src/device/direct_fs.rs         |  2 +-
 foyer-storage/src/device/mod.rs               |  2 +-
 foyer-storage/src/device/monitor.rs           |  2 +-
 foyer-storage/src/engine.rs                   |  2 +-
 foyer-storage/src/error.rs                    |  2 +-
 foyer-storage/src/io_buffer_pool.rs           |  2 +-
 foyer-storage/src/large/batch.rs              |  2 +-
 foyer-storage/src/large/flusher.rs            |  2 +-
 foyer-storage/src/large/generic.rs            |  2 +-
 foyer-storage/src/large/indexer.rs            |  2 +-
 foyer-storage/src/large/mod.rs                |  2 +-
 foyer-storage/src/large/reclaimer.rs          |  2 +-
 foyer-storage/src/large/recover.rs            |  2 +-
 foyer-storage/src/large/scanner.rs            |  2 +-
 foyer-storage/src/large/serde.rs              |  2 +-
 foyer-storage/src/large/tombstone.rs          |  2 +-
 foyer-storage/src/lib.rs                      |  2 +-
 foyer-storage/src/picker/mod.rs               |  2 +-
 foyer-storage/src/picker/utils.rs             |  2 +-
 foyer-storage/src/prelude.rs                  |  2 +-
 foyer-storage/src/region.rs                   |  2 +-
 foyer-storage/src/runtime.rs                  |  2 +-
 foyer-storage/src/serde.rs                    |  2 +-
 foyer-storage/src/small/batch.rs              |  2 +-
 foyer-storage/src/small/bloom_filter.rs       |  2 +-
 foyer-storage/src/small/flusher.rs            |  2 +-
 foyer-storage/src/small/generic.rs            |  2 +-
 foyer-storage/src/small/mod.rs                |  2 +-
 foyer-storage/src/small/serde.rs              |  2 +-
 foyer-storage/src/small/set.rs                |  2 +-
 foyer-storage/src/small/set_manager.rs        |  2 +-
 foyer-storage/src/statistics.rs               |  2 +-
 foyer-storage/src/storage/either.rs           |  2 +-
 foyer-storage/src/storage/mod.rs              |  2 +-
 foyer-storage/src/storage/noop.rs             |  2 +-
 foyer-storage/src/store.rs                    |  4 +--
 foyer-storage/src/test_utils.rs               |  2 +-
 foyer-storage/tests/storage_test.rs           |  2 +-
 foyer-util/Cargo.toml                         | 20 ++++++++-------
 foyer-util/src/async_batch_pipeline.rs        |  2 +-
 foyer-util/src/batch.rs                       |  2 +-
 foyer-util/src/compact_bloom_filter.rs        |  2 +-
 foyer-util/src/continuum.rs                   |  2 +-
 foyer-util/src/erwlock.rs                     |  2 +-
 foyer-util/src/iostat.rs                      |  2 +-
 foyer-util/src/judge.rs                       |  2 +-
 foyer-util/src/lib.rs                         |  2 +-
 foyer-util/src/slab/mod.rs                    |  2 +-
 foyer-util/src/slab/slab_linked_list/mod.rs   |  2 +-
 foyer-util/src/slab/slab_linked_list/tests.rs |  2 +-
 foyer-util/src/slab/tests.rs                  |  2 +-
 foyer/Cargo.toml                              | 25 ++++++++++---------
 foyer/src/hybrid/builder.rs                   |  4 +--
 foyer/src/hybrid/cache.rs                     |  2 +-
 foyer/src/hybrid/mod.rs                       |  2 +-
 foyer/src/hybrid/writer.rs                    |  2 +-
 foyer/src/lib.rs                              |  2 +-
 foyer/src/prelude.rs                          |  2 +-
 123 files changed, 236 insertions(+), 201 deletions(-)

diff --git a/.licenserc.yaml b/.licenserc.yaml
index bfb4ceb2..0bdd2646 100644
--- a/.licenserc.yaml
+++ b/.licenserc.yaml
@@ -1,7 +1,7 @@
 header:
   license:
     spdx-id: Apache-2.0
-    copyright-owner: Foyer Project Authors
+    copyright-owner: foyer Project Authors
 
   paths:
     - "**/*.rs"
diff --git a/Cargo.toml b/Cargo.toml
index 367bb99b..f917c84b 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -12,6 +12,17 @@ members = [
     "foyer-util",
 ]
 
+[workspace.package]
+version = "0.12.0-dev"
+edition = "2021"
+rust-version = "1.81.0"
+repository = "https://github.com/foyer-rs/foyer"
+homepage = "https://foyer.rs"
+keywords = ["cache", "hybrid"]
+authors = ["MrCroxx <mrcroxx@outlook.com>"]
+license = "Apache-2.0"
+readme = "README.md"
+
 [workspace.dependencies]
 tokio = { package = "madsim-tokio", version = "0.2", features = [
     "rt",
@@ -34,6 +45,12 @@ fastrace-jaeger = "0.7"
 fastrace-opentelemetry = "0.7"
 clap = { version = "4", features = ["derive"] }
 bytesize = { package = "foyer-bytesize", version = "2" }
+# foyer components
+foyer-common = { version = "0.12.0-dev", path = "foyer-common" }
+foyer-intrusive = { version = "0.12.0-dev", path = "foyer-intrusive" }
+foyer-memory = { version = "0.12.0-dev", path = "foyer-memory" }
+foyer-storage = { version = "0.12.0-dev", path = "foyer-storage" }
+foyer = { version = "0.12.0-dev", path = "foyer" }
 
 [profile.release]
 debug = true
diff --git a/LICENSE b/LICENSE
index 7e01351a..48b5e9ab 100644
--- a/LICENSE
+++ b/LICENSE
@@ -187,7 +187,7 @@
       same "printed page" as the copyright notice for easier
       identification within third-party archives.
 
-   Copyright 2024 Foyer Project Authors
+   Copyright 2024 foyer Project Authors
 
    Licensed under the Apache License, Version 2.0 (the "License");
    you may not use this file except in compliance with the License.
diff --git a/examples/Cargo.toml b/examples/Cargo.toml
index 2584ca5f..a0b38593 100644
--- a/examples/Cargo.toml
+++ b/examples/Cargo.toml
@@ -1,15 +1,17 @@
 [package]
 name = "examples"
-version = "0.0.0"
-edition = "2021"
-authors = ["MrCroxx <mrcroxx@outlook.com>"]
-description = "Hybrid cache for Rust"
-license = "Apache-2.0"
-repository = "https://github.com/foyer-rs/foyer"
-homepage = "https://github.com/foyer-rs/foyer"
-readme = "../README.md"
-# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+description = "examples for foyer - Hybrid cache for Rust"
+version = { workspace = true }
+edition = { workspace = true }
+rust-version = { workspace = true }
+repository = { workspace = true }
+homepage = { workspace = true }
+keywords = { workspace = true }
+authors = { workspace = true }
+license = { workspace = true }
+readme = { workspace = true }
 publish = false
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
 ahash = "0.8"
@@ -18,7 +20,7 @@ chrono = "0.4"
 fastrace = { workspace = true }
 fastrace-jaeger = { workspace = true, optional = true }
 fastrace-opentelemetry = { workspace = true, optional = true }
-foyer = { version = "*", path = "../foyer" }
+foyer = { workspace = true }
 opentelemetry = { version = "0.25", optional = true }
 opentelemetry-otlp = { version = "0.25", optional = true }
 opentelemetry-semantic-conventions = { version = "0.25", optional = true }
diff --git a/examples/event_listener.rs b/examples/event_listener.rs
index 4b905b8e..f7de1674 100644
--- a/examples/event_listener.rs
+++ b/examples/event_listener.rs
@@ -1,4 +1,4 @@
-//  Copyright 2024 Foyer Project Authors
+//  Copyright 2024 foyer Project Authors
 //
 //  Licensed under the Apache License, Version 2.0 (the "License");
 //  you may not use this file except in compliance with the License.
diff --git a/examples/hybrid.rs b/examples/hybrid.rs
index 2bf7a450..fe15af3f 100644
--- a/examples/hybrid.rs
+++ b/examples/hybrid.rs
@@ -1,4 +1,4 @@
-//  Copyright 2024 Foyer Project Authors
+//  Copyright 2024 foyer Project Authors
 //
 //  Licensed under the Apache License, Version 2.0 (the "License");
 //  you may not use this file except in compliance with the License.
diff --git a/examples/hybrid_full.rs b/examples/hybrid_full.rs
index 60cafb8e..acdf3d40 100644
--- a/examples/hybrid_full.rs
+++ b/examples/hybrid_full.rs
@@ -1,4 +1,4 @@
-//  Copyright 2024 Foyer Project Authors
+//  Copyright 2024 foyer Project Authors
 //
 //  Licensed under the Apache License, Version 2.0 (the "License");
 //  you may not use this file except in compliance with the License.
diff --git a/examples/memory.rs b/examples/memory.rs
index e18d2892..389c1ae2 100644
--- a/examples/memory.rs
+++ b/examples/memory.rs
@@ -1,4 +1,4 @@
-//  Copyright 2024 Foyer Project Authors
+//  Copyright 2024 foyer Project Authors
 //
 //  Licensed under the Apache License, Version 2.0 (the "License");
 //  you may not use this file except in compliance with the License.
diff --git a/examples/tail_based_tracing.rs b/examples/tail_based_tracing.rs
index 171aebea..254e4377 100644
--- a/examples/tail_based_tracing.rs
+++ b/examples/tail_based_tracing.rs
@@ -1,4 +1,4 @@
-//  Copyright 2024 Foyer Project Authors
+//  Copyright 2024 foyer Project Authors
 //
 //  Licensed under the Apache License, Version 2.0 (the "License");
 //  you may not use this file except in compliance with the License.
diff --git a/foyer-bench/Cargo.toml b/foyer-bench/Cargo.toml
index 22d397fc..6c641de7 100644
--- a/foyer-bench/Cargo.toml
+++ b/foyer-bench/Cargo.toml
@@ -1,13 +1,15 @@
 [package]
 name = "foyer-bench"
-version = "0.3.5"
-edition = "2021"
-authors = ["MrCroxx <mrcroxx@outlook.com>"]
-description = "bench tool for foyer - the hybrid cache for Rust"
-license = "Apache-2.0"
-repository = "https://github.com/foyer-rs/foyer"
-homepage = "https://github.com/foyer-rs/foyer"
-readme = "../README.md"
+description = "bench tool for foyer - Hybrid cache for Rust"
+version = { workspace = true }
+edition = { workspace = true }
+rust-version = { workspace = true }
+repository = { workspace = true }
+homepage = { workspace = true }
+keywords = { workspace = true }
+authors = { workspace = true }
+license = { workspace = true }
+readme = { workspace = true }
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
@@ -17,7 +19,7 @@ clap = { workspace = true }
 console-subscriber = { version = "0.4", optional = true }
 fastrace = { workspace = true, optional = true }
 fastrace-jaeger = { workspace = true, optional = true }
-foyer = { version = "0.11.5", path = "../foyer" }
+foyer = { workspace = true }
 futures = "0.3"
 hdrhistogram = "7"
 itertools = { workspace = true }
diff --git a/foyer-bench/src/analyze.rs b/foyer-bench/src/analyze.rs
index 395ad949..50ecc7a6 100644
--- a/foyer-bench/src/analyze.rs
+++ b/foyer-bench/src/analyze.rs
@@ -1,4 +1,4 @@
-//  Copyright 2024 Foyer Project Authors
+//  Copyright 2024 foyer Project Authors
 //
 //  Licensed under the Apache License, Version 2.0 (the "License");
 //  you may not use this file except in compliance with the License.
diff --git a/foyer-bench/src/main.rs b/foyer-bench/src/main.rs
index 87587bb5..0a0ef321 100644
--- a/foyer-bench/src/main.rs
+++ b/foyer-bench/src/main.rs
@@ -1,4 +1,4 @@
-//  Copyright 2024 Foyer Project Authors
+//  Copyright 2024 foyer Project Authors
 //
 //  Licensed under the Apache License, Version 2.0 (the "License");
 //  you may not use this file except in compliance with the License.
diff --git a/foyer-bench/src/rate.rs b/foyer-bench/src/rate.rs
index e7d86a34..7cfe284e 100644
--- a/foyer-bench/src/rate.rs
+++ b/foyer-bench/src/rate.rs
@@ -1,4 +1,4 @@
-//  Copyright 2024 Foyer Project Authors
+//  Copyright 2024 foyer Project Authors
 //
 //  Licensed under the Apache License, Version 2.0 (the "License");
 //  you may not use this file except in compliance with the License.
diff --git a/foyer-bench/src/text.rs b/foyer-bench/src/text.rs
index 2ec8f8f2..1b53a2e0 100644
--- a/foyer-bench/src/text.rs
+++ b/foyer-bench/src/text.rs
@@ -1,4 +1,4 @@
-//  Copyright 2024 Foyer Project Authors
+//  Copyright 2024 foyer Project Authors
 //
 //  Licensed under the Apache License, Version 2.0 (the "License");
 //  you may not use this file except in compliance with the License.
diff --git a/foyer-cli/Cargo.toml b/foyer-cli/Cargo.toml
index 707677ff..7112d357 100644
--- a/foyer-cli/Cargo.toml
+++ b/foyer-cli/Cargo.toml
@@ -1,13 +1,16 @@
 [package]
 name = "foyer-cli"
-version = "0.0.0"
-edition = "2021"
-authors = ["MrCroxx <mrcroxx@outlook.com>"]
-description = "Hybrid cache for Rust"
-license = "Apache-2.0"
-repository = "https://github.com/foyer-rs/foyer"
-homepage = "https://github.com/foyer-rs/foyer"
-readme = "../README.md"
+description = "cli tool for foyer - Hybrid cache for Rust"
+version = { workspace = true }
+edition = { workspace = true }
+rust-version = { workspace = true }
+repository = { workspace = true }
+homepage = { workspace = true }
+keywords = { workspace = true }
+authors = { workspace = true }
+license = { workspace = true }
+readme = { workspace = true }
+publish = false
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
diff --git a/foyer-cli/src/args/error.rs b/foyer-cli/src/args/error.rs
index 4bd96da4..09bd03e3 100644
--- a/foyer-cli/src/args/error.rs
+++ b/foyer-cli/src/args/error.rs
@@ -1,4 +1,4 @@
-//  Copyright 2024 Foyer Project Authors
+//  Copyright 2024 foyer Project Authors
 //
 //  Licensed under the Apache License, Version 2.0 (the "License");
 //  you may not use this file except in compliance with the License.
diff --git a/foyer-cli/src/args/fio.rs b/foyer-cli/src/args/fio.rs
index 5a087bdd..2a687ea8 100644
--- a/foyer-cli/src/args/fio.rs
+++ b/foyer-cli/src/args/fio.rs
@@ -1,4 +1,4 @@
-//  Copyright 2024 Foyer Project Authors
+//  Copyright 2024 foyer Project Authors
 //
 //  Licensed under the Apache License, Version 2.0 (the "License");
 //  you may not use this file except in compliance with the License.
diff --git a/foyer-cli/src/args/mod.rs b/foyer-cli/src/args/mod.rs
index 5c5e4dd1..bb8e3862 100644
--- a/foyer-cli/src/args/mod.rs
+++ b/foyer-cli/src/args/mod.rs
@@ -1,4 +1,4 @@
-//  Copyright 2024 Foyer Project Authors
+//  Copyright 2024 foyer Project Authors
 //
 //  Licensed under the Apache License, Version 2.0 (the "License");
 //  you may not use this file except in compliance with the License.
diff --git a/foyer-cli/src/main.rs b/foyer-cli/src/main.rs
index da1dee74..081289f8 100644
--- a/foyer-cli/src/main.rs
+++ b/foyer-cli/src/main.rs
@@ -1,4 +1,4 @@
-//  Copyright 2024 Foyer Project Authors
+//  Copyright 2024 foyer Project Authors
 //
 //  Licensed under the Apache License, Version 2.0 (the "License");
 //  you may not use this file except in compliance with the License.
diff --git a/foyer-common/Cargo.toml b/foyer-common/Cargo.toml
index 3e64b1f5..c416d6cd 100644
--- a/foyer-common/Cargo.toml
+++ b/foyer-common/Cargo.toml
@@ -1,13 +1,15 @@
 [package]
 name = "foyer-common"
-version = "0.9.5"
-edition = "2021"
-authors = ["MrCroxx <mrcroxx@outlook.com>"]
-description = "common components for foyer - the hybrid cache for Rust"
-license = "Apache-2.0"
-repository = "https://github.com/foyer-rs/foyer"
-homepage = "https://github.com/foyer-rs/foyer"
-readme = "../README.md"
+description = "common components for foyer - Hybrid cache for Rust"
+version = { workspace = true }
+edition = { workspace = true }
+rust-version = { workspace = true }
+repository = { workspace = true }
+homepage = { workspace = true }
+keywords = { workspace = true }
+authors = { workspace = true }
+license = { workspace = true }
+readme = { workspace = true }
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
diff --git a/foyer-common/src/assert.rs b/foyer-common/src/assert.rs
index 8f15f3da..8e2a744f 100644
--- a/foyer-common/src/assert.rs
+++ b/foyer-common/src/assert.rs
@@ -1,4 +1,4 @@
-//  Copyright 2024 Foyer Project Authors
+//  Copyright 2024 foyer Project Authors
 //
 //  Licensed under the Apache License, Version 2.0 (the "License");
 //  you may not use this file except in compliance with the License.
diff --git a/foyer-common/src/asyncify.rs b/foyer-common/src/asyncify.rs
index ba441b49..8b62f302 100644
--- a/foyer-common/src/asyncify.rs
+++ b/foyer-common/src/asyncify.rs
@@ -1,4 +1,4 @@
-//  Copyright 2024 Foyer Project Authors
+//  Copyright 2024 foyer Project Authors
 //
 //  Licensed under the Apache License, Version 2.0 (the "License");
 //  you may not use this file except in compliance with the License.
diff --git a/foyer-common/src/bits.rs b/foyer-common/src/bits.rs
index 0cc7c86b..f8ed230e 100644
--- a/foyer-common/src/bits.rs
+++ b/foyer-common/src/bits.rs
@@ -1,4 +1,4 @@
-//  Copyright 2024 Foyer Project Authors
+//  Copyright 2024 foyer Project Authors
 //
 //  Licensed under the Apache License, Version 2.0 (the "License");
 //  you may not use this file except in compliance with the License.
diff --git a/foyer-common/src/buf.rs b/foyer-common/src/buf.rs
index da006a93..7e62228a 100644
--- a/foyer-common/src/buf.rs
+++ b/foyer-common/src/buf.rs
@@ -1,4 +1,4 @@
-//  Copyright 2024 Foyer Project Authors
+//  Copyright 2024 foyer Project Authors
 //
 //  Licensed under the Apache License, Version 2.0 (the "License");
 //  you may not use this file except in compliance with the License.
diff --git a/foyer-common/src/code.rs b/foyer-common/src/code.rs
index ae4826e6..f1fbf3eb 100644
--- a/foyer-common/src/code.rs
+++ b/foyer-common/src/code.rs
@@ -1,4 +1,4 @@
-//  Copyright 2024 Foyer Project Authors
+//  Copyright 2024 foyer Project Authors
 //
 //  Licensed under the Apache License, Version 2.0 (the "License");
 //  you may not use this file except in compliance with the License.
diff --git a/foyer-common/src/countdown.rs b/foyer-common/src/countdown.rs
index 1d80e319..c12b80f9 100644
--- a/foyer-common/src/countdown.rs
+++ b/foyer-common/src/countdown.rs
@@ -1,4 +1,4 @@
-//  Copyright 2024 Foyer Project Authors
+//  Copyright 2024 foyer Project Authors
 //
 //  Licensed under the Apache License, Version 2.0 (the "License");
 //  you may not use this file except in compliance with the License.
diff --git a/foyer-common/src/event.rs b/foyer-common/src/event.rs
index 0c76edd9..f6d17070 100644
--- a/foyer-common/src/event.rs
+++ b/foyer-common/src/event.rs
@@ -1,4 +1,4 @@
-//  Copyright 2024 Foyer Project Authors
+//  Copyright 2024 foyer Project Authors
 //
 //  Licensed under the Apache License, Version 2.0 (the "License");
 //  you may not use this file except in compliance with the License.
diff --git a/foyer-common/src/future.rs b/foyer-common/src/future.rs
index 5e9ec127..498fa853 100644
--- a/foyer-common/src/future.rs
+++ b/foyer-common/src/future.rs
@@ -1,4 +1,4 @@
-//  Copyright 2024 Foyer Project Authors
+//  Copyright 2024 foyer Project Authors
 //
 //  Licensed under the Apache License, Version 2.0 (the "License");
 //  you may not use this file except in compliance with the License.
diff --git a/foyer-common/src/lib.rs b/foyer-common/src/lib.rs
index 208d9ee1..bff5d147 100644
--- a/foyer-common/src/lib.rs
+++ b/foyer-common/src/lib.rs
@@ -1,4 +1,4 @@
-//  Copyright 2024 Foyer Project Authors
+//  Copyright 2024 foyer Project Authors
 //
 //  Licensed under the Apache License, Version 2.0 (the "License");
 //  you may not use this file except in compliance with the License.
diff --git a/foyer-common/src/metrics.rs b/foyer-common/src/metrics.rs
index 0544551d..d8ea3adc 100644
--- a/foyer-common/src/metrics.rs
+++ b/foyer-common/src/metrics.rs
@@ -1,4 +1,4 @@
-//  Copyright 2024 Foyer Project Authors
+//  Copyright 2024 foyer Project Authors
 //
 //  Licensed under the Apache License, Version 2.0 (the "License");
 //  you may not use this file except in compliance with the License.
diff --git a/foyer-common/src/object_pool.rs b/foyer-common/src/object_pool.rs
index 673030e9..9e4d6048 100644
--- a/foyer-common/src/object_pool.rs
+++ b/foyer-common/src/object_pool.rs
@@ -1,4 +1,4 @@
-//  Copyright 2024 Foyer Project Authors
+//  Copyright 2024 foyer Project Authors
 //
 //  Licensed under the Apache License, Version 2.0 (the "License");
 //  you may not use this file except in compliance with the License.
diff --git a/foyer-common/src/range.rs b/foyer-common/src/range.rs
index 34d23aa4..0fae4d77 100644
--- a/foyer-common/src/range.rs
+++ b/foyer-common/src/range.rs
@@ -1,4 +1,4 @@
-//  Copyright 2024 Foyer Project Authors
+//  Copyright 2024 foyer Project Authors
 //
 //  Licensed under the Apache License, Version 2.0 (the "License");
 //  you may not use this file except in compliance with the License.
diff --git a/foyer-common/src/rate.rs b/foyer-common/src/rate.rs
index 75fef3fc..ef9af776 100644
--- a/foyer-common/src/rate.rs
+++ b/foyer-common/src/rate.rs
@@ -1,4 +1,4 @@
-//  Copyright 2024 Foyer Project Authors
+//  Copyright 2024 foyer Project Authors
 //
 //  Licensed under the Apache License, Version 2.0 (the "License");
 //  you may not use this file except in compliance with the License.
diff --git a/foyer-common/src/rated_ticket.rs b/foyer-common/src/rated_ticket.rs
index 34852930..026cdf2d 100644
--- a/foyer-common/src/rated_ticket.rs
+++ b/foyer-common/src/rated_ticket.rs
@@ -1,4 +1,4 @@
-//  Copyright 2024 Foyer Project Authors
+//  Copyright 2024 foyer Project Authors
 //
 //  Licensed under the Apache License, Version 2.0 (the "License");
 //  you may not use this file except in compliance with the License.
diff --git a/foyer-common/src/runtime.rs b/foyer-common/src/runtime.rs
index a3727090..14be9c18 100644
--- a/foyer-common/src/runtime.rs
+++ b/foyer-common/src/runtime.rs
@@ -1,4 +1,4 @@
-//  Copyright 2024 Foyer Project Authors
+//  Copyright 2024 foyer Project Authors
 //
 //  Licensed under the Apache License, Version 2.0 (the "License");
 //  you may not use this file except in compliance with the License.
diff --git a/foyer-common/src/tracing.rs b/foyer-common/src/tracing.rs
index 708b5208..057e9c50 100644
--- a/foyer-common/src/tracing.rs
+++ b/foyer-common/src/tracing.rs
@@ -1,4 +1,4 @@
-//  Copyright 2024 Foyer Project Authors
+//  Copyright 2024 foyer Project Authors
 //
 //  Licensed under the Apache License, Version 2.0 (the "License");
 //  you may not use this file except in compliance with the License.
diff --git a/foyer-common/src/wait_group.rs b/foyer-common/src/wait_group.rs
index 855dee63..ae4f71bd 100644
--- a/foyer-common/src/wait_group.rs
+++ b/foyer-common/src/wait_group.rs
@@ -1,4 +1,4 @@
-//  Copyright 2024 Foyer Project Authors
+//  Copyright 2024 foyer Project Authors
 //
 //  Licensed under the Apache License, Version 2.0 (the "License");
 //  you may not use this file except in compliance with the License.
diff --git a/foyer-intrusive/Cargo.toml b/foyer-intrusive/Cargo.toml
index a39927dd..85ce93b5 100644
--- a/foyer-intrusive/Cargo.toml
+++ b/foyer-intrusive/Cargo.toml
@@ -1,17 +1,19 @@
 [package]
 name = "foyer-intrusive"
-version = "0.9.5"
-edition = "2021"
-authors = ["MrCroxx <mrcroxx@outlook.com>"]
-description = "intrusive data structures for foyer - the hybrid cache for Rust"
-license = "Apache-2.0"
-repository = "https://github.com/foyer-rs/foyer"
-homepage = "https://github.com/foyer-rs/foyer"
-readme = "../README.md"
+description = "intrusive data structures for foyer - Hybrid cache for Rust"
+version = { workspace = true }
+edition = { workspace = true }
+rust-version = { workspace = true }
+repository = { workspace = true }
+homepage = { workspace = true }
+keywords = { workspace = true }
+authors = { workspace = true }
+license = { workspace = true }
+readme = { workspace = true }
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
-foyer-common = { version = "0.9.5", path = "../foyer-common" }
+foyer-common = { workspace = true }
 itertools = { workspace = true }
 
 [features]
diff --git a/foyer-intrusive/src/adapter.rs b/foyer-intrusive/src/adapter.rs
index 25063921..7a6a957a 100644
--- a/foyer-intrusive/src/adapter.rs
+++ b/foyer-intrusive/src/adapter.rs
@@ -1,4 +1,4 @@
-//  Copyright 2024 Foyer Project Authors
+//  Copyright 2024 foyer Project Authors
 //
 //  Licensed under the Apache License, Version 2.0 (the "License");
 //  you may not use this file except in compliance with the License.
diff --git a/foyer-intrusive/src/dlist.rs b/foyer-intrusive/src/dlist.rs
index 71b20da6..04ecea48 100644
--- a/foyer-intrusive/src/dlist.rs
+++ b/foyer-intrusive/src/dlist.rs
@@ -1,4 +1,4 @@
-//  Copyright 2024 Foyer Project Authors
+//  Copyright 2024 foyer Project Authors
 //
 //  Licensed under the Apache License, Version 2.0 (the "License");
 //  you may not use this file except in compliance with the License.
diff --git a/foyer-intrusive/src/lib.rs b/foyer-intrusive/src/lib.rs
index bf64ea7f..4ec6cca3 100644
--- a/foyer-intrusive/src/lib.rs
+++ b/foyer-intrusive/src/lib.rs
@@ -1,4 +1,4 @@
-//  Copyright 2024 Foyer Project Authors
+//  Copyright 2024 foyer Project Authors
 //
 //  Licensed under the Apache License, Version 2.0 (the "License");
 //  you may not use this file except in compliance with the License.
diff --git a/foyer-memory/Cargo.toml b/foyer-memory/Cargo.toml
index dbb2a3a5..078fbe72 100644
--- a/foyer-memory/Cargo.toml
+++ b/foyer-memory/Cargo.toml
@@ -1,13 +1,15 @@
 [package]
 name = "foyer-memory"
-version = "0.7.5"
-edition = "2021"
-authors = ["MrCroxx <mrcroxx@outlook.com>"]
-description = "memory cache for foyer - the hybrid cache for Rust"
-license = "Apache-2.0"
-repository = "https://github.com/foyer-rs/foyer"
-homepage = "https://github.com/foyer-rs/foyer"
-readme = "../README.md"
+description = "memory cache for foyer - Hybrid cache for Rust"
+version = { workspace = true }
+edition = { workspace = true }
+rust-version = { workspace = true }
+repository = { workspace = true }
+homepage = { workspace = true }
+keywords = { workspace = true }
+authors = { workspace = true }
+license = { workspace = true }
+readme = { workspace = true }
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
@@ -15,8 +17,8 @@ ahash = "0.8"
 bitflags = "2"
 cmsketch = "0.2.1"
 fastrace = { workspace = true }
-foyer-common = { version = "0.9.5", path = "../foyer-common" }
-foyer-intrusive = { version = "0.9.5", path = "../foyer-intrusive" }
+foyer-common = { workspace = true }
+foyer-intrusive = { workspace = true }
 futures = "0.3"
 hashbrown = "0.14"
 itertools = { workspace = true }
diff --git a/foyer-memory/benches/bench_dynamic_dispatch.rs b/foyer-memory/benches/bench_dynamic_dispatch.rs
index 144676b6..826643c5 100644
--- a/foyer-memory/benches/bench_dynamic_dispatch.rs
+++ b/foyer-memory/benches/bench_dynamic_dispatch.rs
@@ -1,4 +1,4 @@
-//  Copyright 2024 Foyer Project Authors
+//  Copyright 2024 foyer Project Authors
 //
 //  Licensed under the Apache License, Version 2.0 (the "License");
 //  you may not use this file except in compliance with the License.
diff --git a/foyer-memory/benches/bench_hit_ratio.rs b/foyer-memory/benches/bench_hit_ratio.rs
index fd49fd0f..be1fdb13 100644
--- a/foyer-memory/benches/bench_hit_ratio.rs
+++ b/foyer-memory/benches/bench_hit_ratio.rs
@@ -1,4 +1,4 @@
-//  Copyright 2024 Foyer Project Authors
+//  Copyright 2024 foyer Project Authors
 //
 //  Licensed under the Apache License, Version 2.0 (the "License");
 //  you may not use this file except in compliance with the License.
diff --git a/foyer-memory/src/cache.rs b/foyer-memory/src/cache.rs
index 6d8fc3c1..ab072a48 100644
--- a/foyer-memory/src/cache.rs
+++ b/foyer-memory/src/cache.rs
@@ -1,4 +1,4 @@
-//  Copyright 2024 Foyer Project Authors
+//  Copyright 2024 foyer Project Authors
 //
 //  Licensed under the Apache License, Version 2.0 (the "License");
 //  you may not use this file except in compliance with the License.
@@ -338,7 +338,7 @@ where
 {
     /// Set the name of the foyer in-memory cache instance.
     ///
-    /// Foyer will use the name as the prefix of the metric names.
+    /// foyer will use the name as the prefix of the metric names.
     ///
     /// Default: `foyer`.
     pub fn with_name(mut self, name: &str) -> Self {
diff --git a/foyer-memory/src/context.rs b/foyer-memory/src/context.rs
index eba7eff7..3af8ab18 100644
--- a/foyer-memory/src/context.rs
+++ b/foyer-memory/src/context.rs
@@ -1,4 +1,4 @@
-//  Copyright 2024 Foyer Project Authors
+//  Copyright 2024 foyer Project Authors
 //
 //  Licensed under the Apache License, Version 2.0 (the "License");
 //  you may not use this file except in compliance with the License.
diff --git a/foyer-memory/src/eviction/fifo.rs b/foyer-memory/src/eviction/fifo.rs
index 606e6065..a1e0d974 100644
--- a/foyer-memory/src/eviction/fifo.rs
+++ b/foyer-memory/src/eviction/fifo.rs
@@ -1,4 +1,4 @@
-//  Copyright 2024 Foyer Project Authors
+//  Copyright 2024 foyer Project Authors
 //
 //  Licensed under the Apache License, Version 2.0 (the "License");
 //  you may not use this file except in compliance with the License.
diff --git a/foyer-memory/src/eviction/lfu.rs b/foyer-memory/src/eviction/lfu.rs
index f17aaade..5ed7f21c 100644
--- a/foyer-memory/src/eviction/lfu.rs
+++ b/foyer-memory/src/eviction/lfu.rs
@@ -1,4 +1,4 @@
-//  Copyright 2024 Foyer Project Authors
+//  Copyright 2024 foyer Project Authors
 //
 //  Licensed under the Apache License, Version 2.0 (the "License");
 //  you may not use this file except in compliance with the License.
diff --git a/foyer-memory/src/eviction/lru.rs b/foyer-memory/src/eviction/lru.rs
index 54cdf2af..132df3c6 100644
--- a/foyer-memory/src/eviction/lru.rs
+++ b/foyer-memory/src/eviction/lru.rs
@@ -1,4 +1,4 @@
-//  Copyright 2024 Foyer Project Authors
+//  Copyright 2024 foyer Project Authors
 //
 //  Licensed under the Apache License, Version 2.0 (the "License");
 //  you may not use this file except in compliance with the License.
diff --git a/foyer-memory/src/eviction/mod.rs b/foyer-memory/src/eviction/mod.rs
index e8eaa5e9..fa7297be 100644
--- a/foyer-memory/src/eviction/mod.rs
+++ b/foyer-memory/src/eviction/mod.rs
@@ -1,4 +1,4 @@
-//  Copyright 2024 Foyer Project Authors
+//  Copyright 2024 foyer Project Authors
 //
 //  Licensed under the Apache License, Version 2.0 (the "License");
 //  you may not use this file except in compliance with the License.
diff --git a/foyer-memory/src/eviction/s3fifo.rs b/foyer-memory/src/eviction/s3fifo.rs
index 1ae5d601..fcee8588 100644
--- a/foyer-memory/src/eviction/s3fifo.rs
+++ b/foyer-memory/src/eviction/s3fifo.rs
@@ -1,4 +1,4 @@
-//  Copyright 2024 Foyer Project Authors
+//  Copyright 2024 foyer Project Authors
 //
 //  Licensed under the Apache License, Version 2.0 (the "License");
 //  you may not use this file except in compliance with the License.
diff --git a/foyer-memory/src/eviction/sanity.rs b/foyer-memory/src/eviction/sanity.rs
index 176ede26..4d965a13 100644
--- a/foyer-memory/src/eviction/sanity.rs
+++ b/foyer-memory/src/eviction/sanity.rs
@@ -1,4 +1,4 @@
-//  Copyright 2024 Foyer Project Authors
+//  Copyright 2024 foyer Project Authors
 //
 //  Licensed under the Apache License, Version 2.0 (the "License");
 //  you may not use this file except in compliance with the License.
diff --git a/foyer-memory/src/eviction/test_utils.rs b/foyer-memory/src/eviction/test_utils.rs
index dd3f1f6f..be965799 100644
--- a/foyer-memory/src/eviction/test_utils.rs
+++ b/foyer-memory/src/eviction/test_utils.rs
@@ -1,4 +1,4 @@
-//  Copyright 2024 Foyer Project Authors
+//  Copyright 2024 foyer Project Authors
 //
 //  Licensed under the Apache License, Version 2.0 (the "License");
 //  you may not use this file except in compliance with the License.
diff --git a/foyer-memory/src/generic.rs b/foyer-memory/src/generic.rs
index 8d8d7094..60f57a6c 100644
--- a/foyer-memory/src/generic.rs
+++ b/foyer-memory/src/generic.rs
@@ -1,4 +1,4 @@
-//  Copyright 2024 Foyer Project Authors
+//  Copyright 2024 foyer Project Authors
 //
 //  Licensed under the Apache License, Version 2.0 (the "License");
 //  you may not use this file except in compliance with the License.
diff --git a/foyer-memory/src/handle.rs b/foyer-memory/src/handle.rs
index 4cca7464..9476cda2 100644
--- a/foyer-memory/src/handle.rs
+++ b/foyer-memory/src/handle.rs
@@ -1,4 +1,4 @@
-//  Copyright 2024 Foyer Project Authors
+//  Copyright 2024 foyer Project Authors
 //
 //  Licensed under the Apache License, Version 2.0 (the "License");
 //  you may not use this file except in compliance with the License.
diff --git a/foyer-memory/src/indexer/hash_table.rs b/foyer-memory/src/indexer/hash_table.rs
index b0837d72..19dc7d29 100644
--- a/foyer-memory/src/indexer/hash_table.rs
+++ b/foyer-memory/src/indexer/hash_table.rs
@@ -1,4 +1,4 @@
-//  Copyright 2024 Foyer Project Authors
+//  Copyright 2024 foyer Project Authors
 //
 //  Licensed under the Apache License, Version 2.0 (the "License");
 //  you may not use this file except in compliance with the License.
diff --git a/foyer-memory/src/indexer/mod.rs b/foyer-memory/src/indexer/mod.rs
index 0b6cf795..552ded1a 100644
--- a/foyer-memory/src/indexer/mod.rs
+++ b/foyer-memory/src/indexer/mod.rs
@@ -1,4 +1,4 @@
-//  Copyright 2024 Foyer Project Authors
+//  Copyright 2024 foyer Project Authors
 //
 //  Licensed under the Apache License, Version 2.0 (the "License");
 //  you may not use this file except in compliance with the License.
diff --git a/foyer-memory/src/indexer/sanity.rs b/foyer-memory/src/indexer/sanity.rs
index d84b59ed..fcde9a5d 100644
--- a/foyer-memory/src/indexer/sanity.rs
+++ b/foyer-memory/src/indexer/sanity.rs
@@ -1,4 +1,4 @@
-//  Copyright 2024 Foyer Project Authors
+//  Copyright 2024 foyer Project Authors
 //
 //  Licensed under the Apache License, Version 2.0 (the "License");
 //  you may not use this file except in compliance with the License.
diff --git a/foyer-memory/src/lib.rs b/foyer-memory/src/lib.rs
index cf3e2fe3..afcb3d14 100644
--- a/foyer-memory/src/lib.rs
+++ b/foyer-memory/src/lib.rs
@@ -1,4 +1,4 @@
-//  Copyright 2024 Foyer Project Authors
+//  Copyright 2024 foyer Project Authors
 //
 //  Licensed under the Apache License, Version 2.0 (the "License");
 //  you may not use this file except in compliance with the License.
diff --git a/foyer-memory/src/prelude.rs b/foyer-memory/src/prelude.rs
index d1cd10ed..ee2925da 100644
--- a/foyer-memory/src/prelude.rs
+++ b/foyer-memory/src/prelude.rs
@@ -1,4 +1,4 @@
-//  Copyright 2024 Foyer Project Authors
+//  Copyright 2024 foyer Project Authors
 //
 //  Licensed under the Apache License, Version 2.0 (the "License");
 //  you may not use this file except in compliance with the License.
diff --git a/foyer-storage/Cargo.toml b/foyer-storage/Cargo.toml
index d77630d3..fab85a03 100644
--- a/foyer-storage/Cargo.toml
+++ b/foyer-storage/Cargo.toml
@@ -1,13 +1,15 @@
 [package]
 name = "foyer-storage"
-version = "0.10.5"
-edition = "2021"
-authors = ["MrCroxx <mrcroxx@outlook.com>"]
-description = "storage engine for foyer - the hybrid cache for Rust"
-license = "Apache-2.0"
-repository = "https://github.com/foyer-rs/foyer"
-homepage = "https://github.com/foyer-rs/foyer"
-readme = "../README.md"
+description = "storage engine for foyer - Hybrid cache for Rust"
+version = { workspace = true }
+edition = { workspace = true }
+rust-version = { workspace = true }
+repository = { workspace = true }
+homepage = { workspace = true }
+keywords = { workspace = true }
+authors = { workspace = true }
+license = { workspace = true }
+readme = { workspace = true }
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
@@ -26,8 +28,8 @@ clap = { workspace = true }
 either = "1"
 fastrace = { workspace = true }
 flume = "0.11"
-foyer-common = { version = "0.9.5", path = "../foyer-common" }
-foyer-memory = { version = "0.7.5", path = "../foyer-memory" }
+foyer-common = { workspace = true }
+foyer-memory = { workspace = true }
 fs4 = "0.9.1"
 futures = "0.3"
 itertools = { workspace = true }
diff --git a/foyer-storage/src/compress.rs b/foyer-storage/src/compress.rs
index 94cc3281..04a43b0b 100644
--- a/foyer-storage/src/compress.rs
+++ b/foyer-storage/src/compress.rs
@@ -1,4 +1,4 @@
-//  Copyright 2024 Foyer Project Authors
+//  Copyright 2024 foyer Project Authors
 //
 //  Licensed under the Apache License, Version 2.0 (the "License");
 //  you may not use this file except in compliance with the License.
diff --git a/foyer-storage/src/device/allocator.rs b/foyer-storage/src/device/allocator.rs
index f2f2f6a1..52089125 100644
--- a/foyer-storage/src/device/allocator.rs
+++ b/foyer-storage/src/device/allocator.rs
@@ -1,4 +1,4 @@
-//  Copyright 2024 Foyer Project Authors
+//  Copyright 2024 foyer Project Authors
 //
 //  Licensed under the Apache License, Version 2.0 (the "License");
 //  you may not use this file except in compliance with the License.
diff --git a/foyer-storage/src/device/bytes.rs b/foyer-storage/src/device/bytes.rs
index 6a53e83a..d7dece75 100644
--- a/foyer-storage/src/device/bytes.rs
+++ b/foyer-storage/src/device/bytes.rs
@@ -1,4 +1,4 @@
-//  Copyright 2024 Foyer Project Authors
+//  Copyright 2024 foyer Project Authors
 //
 //  Licensed under the Apache License, Version 2.0 (the "License");
 //  you may not use this file except in compliance with the License.
diff --git a/foyer-storage/src/device/direct_file.rs b/foyer-storage/src/device/direct_file.rs
index cd6845a1..8c4ab1ff 100644
--- a/foyer-storage/src/device/direct_file.rs
+++ b/foyer-storage/src/device/direct_file.rs
@@ -1,4 +1,4 @@
-//  Copyright 2024 Foyer Project Authors
+//  Copyright 2024 foyer Project Authors
 //
 //  Licensed under the Apache License, Version 2.0 (the "License");
 //  you may not use this file except in compliance with the License.
diff --git a/foyer-storage/src/device/direct_fs.rs b/foyer-storage/src/device/direct_fs.rs
index 94fc3f11..4e1deb95 100644
--- a/foyer-storage/src/device/direct_fs.rs
+++ b/foyer-storage/src/device/direct_fs.rs
@@ -1,4 +1,4 @@
-//  Copyright 2024 Foyer Project Authors
+//  Copyright 2024 foyer Project Authors
 //
 //  Licensed under the Apache License, Version 2.0 (the "License");
 //  you may not use this file except in compliance with the License.
diff --git a/foyer-storage/src/device/mod.rs b/foyer-storage/src/device/mod.rs
index 26dfe5f7..998dbf25 100644
--- a/foyer-storage/src/device/mod.rs
+++ b/foyer-storage/src/device/mod.rs
@@ -1,4 +1,4 @@
-//  Copyright 2024 Foyer Project Authors
+//  Copyright 2024 foyer Project Authors
 //
 //  Licensed under the Apache License, Version 2.0 (the "License");
 //  you may not use this file except in compliance with the License.
diff --git a/foyer-storage/src/device/monitor.rs b/foyer-storage/src/device/monitor.rs
index 3e338d1d..bf29227f 100644
--- a/foyer-storage/src/device/monitor.rs
+++ b/foyer-storage/src/device/monitor.rs
@@ -1,4 +1,4 @@
-//  Copyright 2024 Foyer Project Authors
+//  Copyright 2024 foyer Project Authors
 //
 //  Licensed under the Apache License, Version 2.0 (the "License");
 //  you may not use this file except in compliance with the License.
diff --git a/foyer-storage/src/engine.rs b/foyer-storage/src/engine.rs
index 6735315e..fc1a40bf 100644
--- a/foyer-storage/src/engine.rs
+++ b/foyer-storage/src/engine.rs
@@ -1,4 +1,4 @@
-//  Copyright 2024 Foyer Project Authors
+//  Copyright 2024 foyer Project Authors
 //
 //  Licensed under the Apache License, Version 2.0 (the "License");
 //  you may not use this file except in compliance with the License.
diff --git a/foyer-storage/src/error.rs b/foyer-storage/src/error.rs
index 87442e15..0ef2eb3c 100644
--- a/foyer-storage/src/error.rs
+++ b/foyer-storage/src/error.rs
@@ -1,4 +1,4 @@
-//  Copyright 2024 Foyer Project Authors
+//  Copyright 2024 foyer Project Authors
 //
 //  Licensed under the Apache License, Version 2.0 (the "License");
 //  you may not use this file except in compliance with the License.
diff --git a/foyer-storage/src/io_buffer_pool.rs b/foyer-storage/src/io_buffer_pool.rs
index cc6ceff2..d0796fa7 100644
--- a/foyer-storage/src/io_buffer_pool.rs
+++ b/foyer-storage/src/io_buffer_pool.rs
@@ -1,4 +1,4 @@
-//  Copyright 2024 Foyer Project Authors
+//  Copyright 2024 foyer Project Authors
 //
 //  Licensed under the Apache License, Version 2.0 (the "License");
 //  you may not use this file except in compliance with the License.
diff --git a/foyer-storage/src/large/batch.rs b/foyer-storage/src/large/batch.rs
index cacccfe6..602828fd 100644
--- a/foyer-storage/src/large/batch.rs
+++ b/foyer-storage/src/large/batch.rs
@@ -1,4 +1,4 @@
-//  Copyright 2024 Foyer Project Authors
+//  Copyright 2024 foyer Project Authors
 //
 //  Licensed under the Apache License, Version 2.0 (the "License");
 //  you may not use this file except in compliance with the License.
diff --git a/foyer-storage/src/large/flusher.rs b/foyer-storage/src/large/flusher.rs
index 1bcd9d95..4a789540 100644
--- a/foyer-storage/src/large/flusher.rs
+++ b/foyer-storage/src/large/flusher.rs
@@ -1,4 +1,4 @@
-//  Copyright 2024 Foyer Project Authors
+//  Copyright 2024 foyer Project Authors
 //
 //  Licensed under the Apache License, Version 2.0 (the "License");
 //  you may not use this file except in compliance with the License.
diff --git a/foyer-storage/src/large/generic.rs b/foyer-storage/src/large/generic.rs
index e08057c7..c501714a 100644
--- a/foyer-storage/src/large/generic.rs
+++ b/foyer-storage/src/large/generic.rs
@@ -1,4 +1,4 @@
-//  Copyright 2024 Foyer Project Authors
+//  Copyright 2024 foyer Project Authors
 //
 //  Licensed under the Apache License, Version 2.0 (the "License");
 //  you may not use this file except in compliance with the License.
diff --git a/foyer-storage/src/large/indexer.rs b/foyer-storage/src/large/indexer.rs
index 7ba536c9..db32810d 100644
--- a/foyer-storage/src/large/indexer.rs
+++ b/foyer-storage/src/large/indexer.rs
@@ -1,4 +1,4 @@
-//  Copyright 2024 Foyer Project Authors
+//  Copyright 2024 foyer Project Authors
 //
 //  Licensed under the Apache License, Version 2.0 (the "License");
 //  you may not use this file except in compliance with the License.
diff --git a/foyer-storage/src/large/mod.rs b/foyer-storage/src/large/mod.rs
index 4804f133..83d314c6 100644
--- a/foyer-storage/src/large/mod.rs
+++ b/foyer-storage/src/large/mod.rs
@@ -1,4 +1,4 @@
-//  Copyright 2024 Foyer Project Authors
+//  Copyright 2024 foyer Project Authors
 //
 //  Licensed under the Apache License, Version 2.0 (the "License");
 //  you may not use this file except in compliance with the License.
diff --git a/foyer-storage/src/large/reclaimer.rs b/foyer-storage/src/large/reclaimer.rs
index e49f4b43..ac2ffb64 100644
--- a/foyer-storage/src/large/reclaimer.rs
+++ b/foyer-storage/src/large/reclaimer.rs
@@ -1,4 +1,4 @@
-//  Copyright 2024 Foyer Project Authors
+//  Copyright 2024 foyer Project Authors
 //
 //  Licensed under the Apache License, Version 2.0 (the "License");
 //  you may not use this file except in compliance with the License.
diff --git a/foyer-storage/src/large/recover.rs b/foyer-storage/src/large/recover.rs
index 007258a0..cc4bf827 100644
--- a/foyer-storage/src/large/recover.rs
+++ b/foyer-storage/src/large/recover.rs
@@ -1,4 +1,4 @@
-//  Copyright 2024 Foyer Project Authors
+//  Copyright 2024 foyer Project Authors
 //
 //  Licensed under the Apache License, Version 2.0 (the "License");
 //  you may not use this file except in compliance with the License.
diff --git a/foyer-storage/src/large/scanner.rs b/foyer-storage/src/large/scanner.rs
index 20f43309..bd4bb77d 100644
--- a/foyer-storage/src/large/scanner.rs
+++ b/foyer-storage/src/large/scanner.rs
@@ -1,4 +1,4 @@
-//  Copyright 2024 Foyer Project Authors
+//  Copyright 2024 foyer Project Authors
 //
 //  Licensed under the Apache License, Version 2.0 (the "License");
 //  you may not use this file except in compliance with the License.
diff --git a/foyer-storage/src/large/serde.rs b/foyer-storage/src/large/serde.rs
index 03ea69e1..51768dcf 100644
--- a/foyer-storage/src/large/serde.rs
+++ b/foyer-storage/src/large/serde.rs
@@ -1,4 +1,4 @@
-//  Copyright 2024 Foyer Project Authors
+//  Copyright 2024 foyer Project Authors
 //
 //  Licensed under the Apache License, Version 2.0 (the "License");
 //  you may not use this file except in compliance with the License.
diff --git a/foyer-storage/src/large/tombstone.rs b/foyer-storage/src/large/tombstone.rs
index 09a9ee27..3d61e803 100644
--- a/foyer-storage/src/large/tombstone.rs
+++ b/foyer-storage/src/large/tombstone.rs
@@ -1,4 +1,4 @@
-//  Copyright 2024 Foyer Project Authors
+//  Copyright 2024 foyer Project Authors
 //
 //  Licensed under the Apache License, Version 2.0 (the "License");
 //  you may not use this file except in compliance with the License.
diff --git a/foyer-storage/src/lib.rs b/foyer-storage/src/lib.rs
index ceb30ebb..7d5b2b6a 100644
--- a/foyer-storage/src/lib.rs
+++ b/foyer-storage/src/lib.rs
@@ -1,4 +1,4 @@
-//  Copyright 2024 Foyer Project Authors
+//  Copyright 2024 foyer Project Authors
 //
 //  Licensed under the Apache License, Version 2.0 (the "License");
 //  you may not use this file except in compliance with the License.
diff --git a/foyer-storage/src/picker/mod.rs b/foyer-storage/src/picker/mod.rs
index 2657bf48..3e79c491 100644
--- a/foyer-storage/src/picker/mod.rs
+++ b/foyer-storage/src/picker/mod.rs
@@ -1,4 +1,4 @@
-//  Copyright 2024 Foyer Project Authors
+//  Copyright 2024 foyer Project Authors
 //
 //  Licensed under the Apache License, Version 2.0 (the "License");
 //  you may not use this file except in compliance with the License.
diff --git a/foyer-storage/src/picker/utils.rs b/foyer-storage/src/picker/utils.rs
index fb793d5c..525328c5 100644
--- a/foyer-storage/src/picker/utils.rs
+++ b/foyer-storage/src/picker/utils.rs
@@ -1,4 +1,4 @@
-//  Copyright 2024 Foyer Project Authors
+//  Copyright 2024 foyer Project Authors
 //
 //  Licensed under the Apache License, Version 2.0 (the "License");
 //  you may not use this file except in compliance with the License.
diff --git a/foyer-storage/src/prelude.rs b/foyer-storage/src/prelude.rs
index ec0b94c8..ac221496 100644
--- a/foyer-storage/src/prelude.rs
+++ b/foyer-storage/src/prelude.rs
@@ -1,4 +1,4 @@
-//  Copyright 2024 Foyer Project Authors
+//  Copyright 2024 foyer Project Authors
 //
 //  Licensed under the Apache License, Version 2.0 (the "License");
 //  you may not use this file except in compliance with the License.
diff --git a/foyer-storage/src/region.rs b/foyer-storage/src/region.rs
index 6046bab6..fad51781 100644
--- a/foyer-storage/src/region.rs
+++ b/foyer-storage/src/region.rs
@@ -1,4 +1,4 @@
-//  Copyright 2024 Foyer Project Authors
+//  Copyright 2024 foyer Project Authors
 //
 //  Licensed under the Apache License, Version 2.0 (the "License");
 //  you may not use this file except in compliance with the License.
diff --git a/foyer-storage/src/runtime.rs b/foyer-storage/src/runtime.rs
index fc361cb5..a2266e90 100644
--- a/foyer-storage/src/runtime.rs
+++ b/foyer-storage/src/runtime.rs
@@ -1,4 +1,4 @@
-//  Copyright 2024 Foyer Project Authors
+//  Copyright 2024 foyer Project Authors
 //
 //  Licensed under the Apache License, Version 2.0 (the "License");
 //  you may not use this file except in compliance with the License.
diff --git a/foyer-storage/src/serde.rs b/foyer-storage/src/serde.rs
index 169e5377..a466cbba 100644
--- a/foyer-storage/src/serde.rs
+++ b/foyer-storage/src/serde.rs
@@ -1,4 +1,4 @@
-//  Copyright 2024 Foyer Project Authors
+//  Copyright 2024 foyer Project Authors
 //
 //  Licensed under the Apache License, Version 2.0 (the "License");
 //  you may not use this file except in compliance with the License.
diff --git a/foyer-storage/src/small/batch.rs b/foyer-storage/src/small/batch.rs
index 310aa156..8ca25f8d 100644
--- a/foyer-storage/src/small/batch.rs
+++ b/foyer-storage/src/small/batch.rs
@@ -1,4 +1,4 @@
-//  Copyright 2024 Foyer Project Authors
+//  Copyright 2024 foyer Project Authors
 //
 //  Licensed under the Apache License, Version 2.0 (the "License");
 //  you may not use this file except in compliance with the License.
diff --git a/foyer-storage/src/small/bloom_filter.rs b/foyer-storage/src/small/bloom_filter.rs
index 4cf98689..a2707ebe 100644
--- a/foyer-storage/src/small/bloom_filter.rs
+++ b/foyer-storage/src/small/bloom_filter.rs
@@ -1,4 +1,4 @@
-//  Copyright 2024 Foyer Project Authors
+//  Copyright 2024 foyer Project Authors
 //
 //  Licensed under the Apache License, Version 2.0 (the "License");
 //  you may not use this file except in compliance with the License.
diff --git a/foyer-storage/src/small/flusher.rs b/foyer-storage/src/small/flusher.rs
index 43e720a3..f95a440e 100644
--- a/foyer-storage/src/small/flusher.rs
+++ b/foyer-storage/src/small/flusher.rs
@@ -1,4 +1,4 @@
-//  Copyright 2024 Foyer Project Authors
+//  Copyright 2024 foyer Project Authors
 //
 //  Licensed under the Apache License, Version 2.0 (the "License");
 //  you may not use this file except in compliance with the License.
diff --git a/foyer-storage/src/small/generic.rs b/foyer-storage/src/small/generic.rs
index 3b088204..bbec7b45 100644
--- a/foyer-storage/src/small/generic.rs
+++ b/foyer-storage/src/small/generic.rs
@@ -1,4 +1,4 @@
-//  Copyright 2024 Foyer Project Authors
+//  Copyright 2024 foyer Project Authors
 //
 //  Licensed under the Apache License, Version 2.0 (the "License");
 //  you may not use this file except in compliance with the License.
diff --git a/foyer-storage/src/small/mod.rs b/foyer-storage/src/small/mod.rs
index 44c6f579..96921064 100644
--- a/foyer-storage/src/small/mod.rs
+++ b/foyer-storage/src/small/mod.rs
@@ -1,4 +1,4 @@
-//  Copyright 2024 Foyer Project Authors
+//  Copyright 2024 foyer Project Authors
 //
 //  Licensed under the Apache License, Version 2.0 (the "License");
 //  you may not use this file except in compliance with the License.
diff --git a/foyer-storage/src/small/serde.rs b/foyer-storage/src/small/serde.rs
index 2648ad48..b704946a 100644
--- a/foyer-storage/src/small/serde.rs
+++ b/foyer-storage/src/small/serde.rs
@@ -1,4 +1,4 @@
-//  Copyright 2024 Foyer Project Authors
+//  Copyright 2024 foyer Project Authors
 //
 //  Licensed under the Apache License, Version 2.0 (the "License");
 //  you may not use this file except in compliance with the License.
diff --git a/foyer-storage/src/small/set.rs b/foyer-storage/src/small/set.rs
index 02e94adb..62234e21 100644
--- a/foyer-storage/src/small/set.rs
+++ b/foyer-storage/src/small/set.rs
@@ -1,4 +1,4 @@
-//  Copyright 2024 Foyer Project Authors
+//  Copyright 2024 foyer Project Authors
 //
 //  Licensed under the Apache License, Version 2.0 (the "License");
 //  you may not use this file except in compliance with the License.
diff --git a/foyer-storage/src/small/set_manager.rs b/foyer-storage/src/small/set_manager.rs
index 441f0253..98dd3c22 100644
--- a/foyer-storage/src/small/set_manager.rs
+++ b/foyer-storage/src/small/set_manager.rs
@@ -1,4 +1,4 @@
-//  Copyright 2024 Foyer Project Authors
+//  Copyright 2024 foyer Project Authors
 //
 //  Licensed under the Apache License, Version 2.0 (the "License");
 //  you may not use this file except in compliance with the License.
diff --git a/foyer-storage/src/statistics.rs b/foyer-storage/src/statistics.rs
index 5fcc285d..bac5f618 100644
--- a/foyer-storage/src/statistics.rs
+++ b/foyer-storage/src/statistics.rs
@@ -1,4 +1,4 @@
-//  Copyright 2024 Foyer Project Authors
+//  Copyright 2024 foyer Project Authors
 //
 //  Licensed under the Apache License, Version 2.0 (the "License");
 //  you may not use this file except in compliance with the License.
diff --git a/foyer-storage/src/storage/either.rs b/foyer-storage/src/storage/either.rs
index 30788549..60f1b1db 100644
--- a/foyer-storage/src/storage/either.rs
+++ b/foyer-storage/src/storage/either.rs
@@ -1,4 +1,4 @@
-//  Copyright 2024 Foyer Project Authors
+//  Copyright 2024 foyer Project Authors
 //
 //  Licensed under the Apache License, Version 2.0 (the "License");
 //  you may not use this file except in compliance with the License.
diff --git a/foyer-storage/src/storage/mod.rs b/foyer-storage/src/storage/mod.rs
index b82e08da..c9861e5e 100644
--- a/foyer-storage/src/storage/mod.rs
+++ b/foyer-storage/src/storage/mod.rs
@@ -1,4 +1,4 @@
-//  Copyright 2024 Foyer Project Authors
+//  Copyright 2024 foyer Project Authors
 //
 //  Licensed under the Apache License, Version 2.0 (the "License");
 //  you may not use this file except in compliance with the License.
diff --git a/foyer-storage/src/storage/noop.rs b/foyer-storage/src/storage/noop.rs
index b8daf6b5..2930d165 100644
--- a/foyer-storage/src/storage/noop.rs
+++ b/foyer-storage/src/storage/noop.rs
@@ -1,4 +1,4 @@
-//  Copyright 2024 Foyer Project Authors
+//  Copyright 2024 foyer Project Authors
 //
 //  Licensed under the Apache License, Version 2.0 (the "License");
 //  you may not use this file except in compliance with the License.
diff --git a/foyer-storage/src/store.rs b/foyer-storage/src/store.rs
index b4d5ae16..adea7039 100644
--- a/foyer-storage/src/store.rs
+++ b/foyer-storage/src/store.rs
@@ -1,4 +1,4 @@
-//  Copyright 2024 Foyer Project Authors
+//  Copyright 2024 foyer Project Authors
 //
 //  Licensed under the Apache License, Version 2.0 (the "License");
 //  you may not use this file except in compliance with the License.
@@ -393,7 +393,7 @@ where
 
     /// Set the name of the foyer disk cache instance.
     ///
-    /// Foyer will use the name as the prefix of the metric names.
+    /// foyer will use the name as the prefix of the metric names.
     ///
     /// Default: `foyer`.
     pub fn with_name(mut self, name: &str) -> Self {
diff --git a/foyer-storage/src/test_utils.rs b/foyer-storage/src/test_utils.rs
index 317a85e7..cfef7dbc 100644
--- a/foyer-storage/src/test_utils.rs
+++ b/foyer-storage/src/test_utils.rs
@@ -1,4 +1,4 @@
-//  Copyright 2024 Foyer Project Authors
+//  Copyright 2024 foyer Project Authors
 //
 //  Licensed under the Apache License, Version 2.0 (the "License");
 //  you may not use this file except in compliance with the License.
diff --git a/foyer-storage/tests/storage_test.rs b/foyer-storage/tests/storage_test.rs
index e57bbf9c..e20b75ef 100644
--- a/foyer-storage/tests/storage_test.rs
+++ b/foyer-storage/tests/storage_test.rs
@@ -1,4 +1,4 @@
-//  Copyright 2024 Foyer Project Authors
+//  Copyright 2024 foyer Project Authors
 //
 //  Licensed under the Apache License, Version 2.0 (the "License");
 //  you may not use this file except in compliance with the License.
diff --git a/foyer-util/Cargo.toml b/foyer-util/Cargo.toml
index 05374840..ab381677 100644
--- a/foyer-util/Cargo.toml
+++ b/foyer-util/Cargo.toml
@@ -1,13 +1,15 @@
 [package]
 name = "foyer-utils"
-version = "0.0.0"
-edition = "2021"
-authors = ["MrCroxx <mrcroxx@outlook.com>"]
-description = "utils for foyer - the hybrid cache for Rust"
-license = "Apache-2.0"
-repository = "https://github.com/foyer-rs/foyer"
-homepage = "https://github.com/foyer-rs/foyer"
-readme = "../README.md"
+description = "utils for foyer - Hybrid cache for Rust"
+version = { workspace = true }
+edition = { workspace = true }
+rust-version = { workspace = true }
+repository = { workspace = true }
+homepage = { workspace = true }
+keywords = { workspace = true }
+authors = { workspace = true }
+license = { workspace = true }
+readme = { workspace = true }
 publish = false
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
@@ -16,7 +18,7 @@ bitmaps = "3"
 bitvec = "1"
 bytes = "1"
 cfg-if = "1"
-foyer-common = { path = "../foyer-common" }
+foyer-common = { workspace = true }
 futures = "0.3"
 hashbrown = "0.14"
 itertools = { workspace = true }
diff --git a/foyer-util/src/async_batch_pipeline.rs b/foyer-util/src/async_batch_pipeline.rs
index dd15875d..6278f919 100644
--- a/foyer-util/src/async_batch_pipeline.rs
+++ b/foyer-util/src/async_batch_pipeline.rs
@@ -1,4 +1,4 @@
-//  Copyright 2024 Foyer Project Authors
+//  Copyright 2024 foyer Project Authors
 //
 //  Licensed under the Apache License, Version 2.0 (the "License");
 //  you may not use this file except in compliance with the License.
diff --git a/foyer-util/src/batch.rs b/foyer-util/src/batch.rs
index a9a72ea6..016e3591 100644
--- a/foyer-util/src/batch.rs
+++ b/foyer-util/src/batch.rs
@@ -1,4 +1,4 @@
-//  Copyright 2024 Foyer Project Authors
+//  Copyright 2024 foyer Project Authors
 //
 //  Licensed under the Apache License, Version 2.0 (the "License");
 //  you may not use this file except in compliance with the License.
diff --git a/foyer-util/src/compact_bloom_filter.rs b/foyer-util/src/compact_bloom_filter.rs
index 103244f5..9c7505d2 100644
--- a/foyer-util/src/compact_bloom_filter.rs
+++ b/foyer-util/src/compact_bloom_filter.rs
@@ -1,4 +1,4 @@
-//  Copyright 2024 Foyer Project Authors
+//  Copyright 2024 foyer Project Authors
 //
 //  Licensed under the Apache License, Version 2.0 (the "License");
 //  you may not use this file except in compliance with the License.
diff --git a/foyer-util/src/continuum.rs b/foyer-util/src/continuum.rs
index 4efac1d4..1ee8a089 100644
--- a/foyer-util/src/continuum.rs
+++ b/foyer-util/src/continuum.rs
@@ -1,4 +1,4 @@
-//  Copyright 2024 Foyer Project Authors
+//  Copyright 2024 foyer Project Authors
 //
 //  Licensed under the Apache License, Version 2.0 (the "License");
 //  you may not use this file except in compliance with the License.
diff --git a/foyer-util/src/erwlock.rs b/foyer-util/src/erwlock.rs
index 9bd48f6f..1c95264e 100644
--- a/foyer-util/src/erwlock.rs
+++ b/foyer-util/src/erwlock.rs
@@ -1,4 +1,4 @@
-//  Copyright 2024 Foyer Project Authors
+//  Copyright 2024 foyer Project Authors
 //
 //  Licensed under the Apache License, Version 2.0 (the "License");
 //  you may not use this file except in compliance with the License.
diff --git a/foyer-util/src/iostat.rs b/foyer-util/src/iostat.rs
index cc88e8e1..7a02e12d 100644
--- a/foyer-util/src/iostat.rs
+++ b/foyer-util/src/iostat.rs
@@ -1,4 +1,4 @@
-//  Copyright 2024 Foyer Project Authors
+//  Copyright 2024 foyer Project Authors
 //
 //  Licensed under the Apache License, Version 2.0 (the "License");
 //  you may not use this file except in compliance with the License.
diff --git a/foyer-util/src/judge.rs b/foyer-util/src/judge.rs
index dad658fd..d654429c 100644
--- a/foyer-util/src/judge.rs
+++ b/foyer-util/src/judge.rs
@@ -1,4 +1,4 @@
-//  Copyright 2024 Foyer Project Authors
+//  Copyright 2024 foyer Project Authors
 //
 //  Licensed under the Apache License, Version 2.0 (the "License");
 //  you may not use this file except in compliance with the License.
diff --git a/foyer-util/src/lib.rs b/foyer-util/src/lib.rs
index 05d4f02d..7136494c 100644
--- a/foyer-util/src/lib.rs
+++ b/foyer-util/src/lib.rs
@@ -1,4 +1,4 @@
-//  Copyright 2024 Foyer Project Authors
+//  Copyright 2024 foyer Project Authors
 //
 //  Licensed under the Apache License, Version 2.0 (the "License");
 //  you may not use this file except in compliance with the License.
diff --git a/foyer-util/src/slab/mod.rs b/foyer-util/src/slab/mod.rs
index efb72834..c1b889ab 100644
--- a/foyer-util/src/slab/mod.rs
+++ b/foyer-util/src/slab/mod.rs
@@ -1,4 +1,4 @@
-//  Copyright 2024 Foyer Project Authors
+//  Copyright 2024 foyer Project Authors
 //
 //  Licensed under the Apache License, Version 2.0 (the "License");
 //  you may not use this file except in compliance with the License.
diff --git a/foyer-util/src/slab/slab_linked_list/mod.rs b/foyer-util/src/slab/slab_linked_list/mod.rs
index 99c8c390..5f7dc928 100644
--- a/foyer-util/src/slab/slab_linked_list/mod.rs
+++ b/foyer-util/src/slab/slab_linked_list/mod.rs
@@ -1,4 +1,4 @@
-//  Copyright 2024 Foyer Project Authors
+//  Copyright 2024 foyer Project Authors
 //
 //  Licensed under the Apache License, Version 2.0 (the "License");
 //  you may not use this file except in compliance with the License.
diff --git a/foyer-util/src/slab/slab_linked_list/tests.rs b/foyer-util/src/slab/slab_linked_list/tests.rs
index 037be82e..0189d777 100644
--- a/foyer-util/src/slab/slab_linked_list/tests.rs
+++ b/foyer-util/src/slab/slab_linked_list/tests.rs
@@ -1,4 +1,4 @@
-//  Copyright 2024 Foyer Project Authors
+//  Copyright 2024 foyer Project Authors
 //
 //  Licensed under the Apache License, Version 2.0 (the "License");
 //  you may not use this file except in compliance with the License.
diff --git a/foyer-util/src/slab/tests.rs b/foyer-util/src/slab/tests.rs
index dd12ba1d..75a0abbe 100644
--- a/foyer-util/src/slab/tests.rs
+++ b/foyer-util/src/slab/tests.rs
@@ -1,4 +1,4 @@
-//  Copyright 2024 Foyer Project Authors
+//  Copyright 2024 foyer Project Authors
 //
 //  Licensed under the Apache License, Version 2.0 (the "License");
 //  you may not use this file except in compliance with the License.
diff --git a/foyer/Cargo.toml b/foyer/Cargo.toml
index 33a2d4f0..c9eb989b 100644
--- a/foyer/Cargo.toml
+++ b/foyer/Cargo.toml
@@ -1,23 +1,24 @@
 [package]
 name = "foyer"
-version = "0.11.5"
-edition = "2021"
-authors = ["MrCroxx <mrcroxx@outlook.com>"]
-description = "Hybrid cache for Rust"
-license = "Apache-2.0"
-repository = "https://github.com/foyer-rs/foyer"
-homepage = "https://github.com/foyer-rs/foyer"
-readme = "../README.md"
-rust-version = "1.81.0"
+description = "foyer - Hybrid cache for Rust"
+version = { workspace = true }
+edition = { workspace = true }
+rust-version = { workspace = true }
+repository = { workspace = true }
+homepage = { workspace = true }
+keywords = { workspace = true }
+authors = { workspace = true }
+license = { workspace = true }
+readme = { workspace = true }
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
 ahash = "0.8"
 anyhow = "1"
 fastrace = { workspace = true }
-foyer-common = { version = "0.9.5", path = "../foyer-common" }
-foyer-memory = { version = "0.7.5", path = "../foyer-memory" }
-foyer-storage = { version = "0.10.5", path = "../foyer-storage" }
+foyer-common = { workspace = true }
+foyer-memory = { workspace = true }
+foyer-storage = { workspace = true }
 futures = "0.3"
 pin-project = "1"
 tokio = { workspace = true }
diff --git a/foyer/src/hybrid/builder.rs b/foyer/src/hybrid/builder.rs
index ab01c3c1..892cf0b8 100644
--- a/foyer/src/hybrid/builder.rs
+++ b/foyer/src/hybrid/builder.rs
@@ -1,4 +1,4 @@
-//  Copyright 2024 Foyer Project Authors
+//  Copyright 2024 foyer Project Authors
 //
 //  Licensed under the Apache License, Version 2.0 (the "License");
 //  you may not use this file except in compliance with the License.
@@ -53,7 +53,7 @@ impl<K, V> HybridCacheBuilder<K, V> {
 
     /// Set the name of the foyer hybrid cache instance.
     ///
-    /// Foyer will use the name as the prefix of the metric names.
+    /// foyer will use the name as the prefix of the metric names.
     ///
     /// Default: `foyer`.
     pub fn with_name(mut self, name: &str) -> Self {
diff --git a/foyer/src/hybrid/cache.rs b/foyer/src/hybrid/cache.rs
index 8e726832..49948fc2 100644
--- a/foyer/src/hybrid/cache.rs
+++ b/foyer/src/hybrid/cache.rs
@@ -1,4 +1,4 @@
-//  Copyright 2024 Foyer Project Authors
+//  Copyright 2024 foyer Project Authors
 //
 //  Licensed under the Apache License, Version 2.0 (the "License");
 //  you may not use this file except in compliance with the License.
diff --git a/foyer/src/hybrid/mod.rs b/foyer/src/hybrid/mod.rs
index dcd12ead..08d4ed07 100644
--- a/foyer/src/hybrid/mod.rs
+++ b/foyer/src/hybrid/mod.rs
@@ -1,4 +1,4 @@
-//  Copyright 2024 Foyer Project Authors
+//  Copyright 2024 foyer Project Authors
 //
 //  Licensed under the Apache License, Version 2.0 (the "License");
 //  you may not use this file except in compliance with the License.
diff --git a/foyer/src/hybrid/writer.rs b/foyer/src/hybrid/writer.rs
index 3d040594..5d4e72ae 100644
--- a/foyer/src/hybrid/writer.rs
+++ b/foyer/src/hybrid/writer.rs
@@ -1,4 +1,4 @@
-//  Copyright 2024 Foyer Project Authors
+//  Copyright 2024 foyer Project Authors
 //
 //  Licensed under the Apache License, Version 2.0 (the "License");
 //  you may not use this file except in compliance with the License.
diff --git a/foyer/src/lib.rs b/foyer/src/lib.rs
index 2a636cd5..d116d7e1 100644
--- a/foyer/src/lib.rs
+++ b/foyer/src/lib.rs
@@ -1,4 +1,4 @@
-//  Copyright 2024 Foyer Project Authors
+//  Copyright 2024 foyer Project Authors
 //
 //  Licensed under the Apache License, Version 2.0 (the "License");
 //  you may not use this file except in compliance with the License.
diff --git a/foyer/src/prelude.rs b/foyer/src/prelude.rs
index a9c426fa..4746626e 100644
--- a/foyer/src/prelude.rs
+++ b/foyer/src/prelude.rs
@@ -1,4 +1,4 @@
-//  Copyright 2024 Foyer Project Authors
+//  Copyright 2024 foyer Project Authors
 //
 //  Licensed under the Apache License, Version 2.0 (the "License");
 //  you may not use this file except in compliance with the License.

From fa8f011caaf7053fb8f281565cbbc833d2fa539f Mon Sep 17 00:00:00 2001
From: Croxx <mrcroxx@outlook.com>
Date: Fri, 27 Sep 2024 17:44:52 +0800
Subject: [PATCH 19/53] feat: impl destroy for sodc (#744)

* feat: impl destroy for sodc

Changes:

- Use set 0 as meta set.
- Use watermark in metaset for destroy.
- sodc must start from region 0

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

* chore: fix ffmt

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

* fix: clear cache after destroy, use u128 timestamp

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

---------

Signed-off-by: MrCroxx <mrcroxx@outlook.com>
---
 foyer-storage/Cargo.toml               |   1 +
 foyer-storage/src/small/batch.rs       |  12 +--
 foyer-storage/src/small/generic.rs     | 144 +++++++++++++++++++++++--
 foyer-storage/src/small/set.rs         |  57 ++++++----
 foyer-storage/src/small/set_manager.rs | 123 +++++++++++++++++++--
 5 files changed, 295 insertions(+), 42 deletions(-)

diff --git a/foyer-storage/Cargo.toml b/foyer-storage/Cargo.toml
index fab85a03..e0b85011 100644
--- a/foyer-storage/Cargo.toml
+++ b/foyer-storage/Cargo.toml
@@ -48,6 +48,7 @@ twox-hash = "1"
 zstd = "0.13"
 
 [dev-dependencies]
+bytesize = { workspace = true }
 tempfile = "3"
 test-log = { workspace = true }
 
diff --git a/foyer-storage/src/small/batch.rs b/foyer-storage/src/small/batch.rs
index 8ca25f8d..30fb3385 100644
--- a/foyer-storage/src/small/batch.rs
+++ b/foyer-storage/src/small/batch.rs
@@ -33,7 +33,7 @@ use crate::{
     device::ALIGN,
     io_buffer_pool::IoBufferPool,
     serde::EntrySerializer,
-    small::{serde::EntryHeader, set::SetId},
+    small::{serde::EntryHeader, set::SetId, set_manager::SetPicker},
     Compression, IoBuffer, IoBytes,
 };
 
@@ -83,9 +83,6 @@ where
     V: StorageValue,
     S: HashBuilder + Debug,
 {
-    /// Total set count.
-    total: SetId,
-
     sets: HashMap<SetId, SetBatchMut<K, V, S>>,
     buffer: IoBuffer,
     len: usize,
@@ -93,6 +90,7 @@ where
 
     /// Cache write buffer between rotation to reduce page fault.
     buffer_pool: IoBufferPool,
+    set_picker: SetPicker,
 
     waiters: Vec<oneshot::Sender<()>>,
 
@@ -107,16 +105,16 @@ where
     V: StorageValue,
     S: HashBuilder + Debug,
 {
-    pub fn new(total: SetId, buffer_size: usize, metrics: Arc<Metrics>) -> Self {
+    pub fn new(sets: usize, buffer_size: usize, metrics: Arc<Metrics>) -> Self {
         let buffer_size = bits::align_up(ALIGN, buffer_size);
 
         Self {
-            total,
             sets: HashMap::new(),
             buffer: IoBuffer::new(buffer_size),
             len: 0,
             sequence: 0,
             buffer_pool: IoBufferPool::new(buffer_size, 1),
+            set_picker: SetPicker::new(sets),
             waiters: vec![],
             init: None,
             metrics,
@@ -192,7 +190,7 @@ where
     }
 
     fn sid(&self, hash: u64) -> SetId {
-        hash % self.total
+        self.set_picker.sid(hash)
     }
 
     pub fn is_empty(&self) -> bool {
diff --git a/foyer-storage/src/small/generic.rs b/foyer-storage/src/small/generic.rs
index bbec7b45..d1f6b753 100644
--- a/foyer-storage/src/small/generic.rs
+++ b/foyer-storage/src/small/generic.rs
@@ -27,11 +27,13 @@ use foyer_memory::CacheEntry;
 use futures::{future::join_all, Future};
 use itertools::Itertools;
 
-use super::flusher::Submission;
 use crate::{
     device::{MonitoredDevice, RegionId},
     error::Result,
-    small::{flusher::Flusher, set::SetId, set_manager::SetManager},
+    small::{
+        flusher::{Flusher, Submission},
+        set_manager::SetManager,
+    },
     storage::Storage,
     DeviceStats, Runtime, Statistics,
 };
@@ -136,13 +138,20 @@ where
         let stats = config.statistics.clone();
         let metrics = config.device.metrics().clone();
 
-        let set_manager = SetManager::new(
+        assert_eq!(
+            config.regions.start, 0,
+            "small object disk cache must start with region 0, current: {:?}",
+            config.regions
+        );
+
+        let set_manager = SetManager::open(
             config.set_size,
             config.set_cache_capacity,
             config.device.clone(),
             config.regions.clone(),
             config.flush,
-        );
+        )
+        .await?;
 
         let flushers = (0..config.flushers)
             .map(|_| Flusher::open(&config, set_manager.clone(), stats.clone(), metrics.clone()))
@@ -187,7 +196,7 @@ where
 
     fn load(&self, hash: u64) -> impl Future<Output = Result<Option<(K, V)>>> + Send + 'static {
         let set_manager = self.inner.set_manager.clone();
-        let sid = hash % set_manager.sets() as SetId;
+        let sid = set_manager.set_picker().sid(hash);
         let stats = self.inner.stats.clone();
 
         async move {
@@ -216,9 +225,14 @@ where
         self.inner.flushers[id].submit(Submission::Deletion { hash });
     }
 
+    async fn destroy(&self) -> Result<()> {
+        // TODO(MrCroxx): reset bloom filters
+        self.inner.set_manager.destroy().await
+    }
+
     fn may_contains(&self, hash: u64) -> bool {
         let set_manager = self.inner.set_manager.clone();
-        let sid = hash % set_manager.sets() as SetId;
+        let sid = set_manager.set_picker().sid(hash);
         // FIXME: Anyway without blocking? Use atomic?
         self.inner
             .runtime
@@ -268,7 +282,7 @@ where
     }
 
     async fn destroy(&self) -> Result<()> {
-        todo!()
+        self.destroy().await
     }
 
     fn stats(&self) -> Arc<DeviceStats> {
@@ -279,3 +293,119 @@ where
         self.wait()
     }
 }
+
+#[cfg(test)]
+mod tests {
+    use std::path::Path;
+
+    use ahash::RandomState;
+    use bytesize::ByteSize;
+    use foyer_common::metrics::Metrics;
+    use foyer_memory::{Cache, CacheBuilder, FifoConfig};
+    use tokio::runtime::Handle;
+
+    use super::*;
+    use crate::{
+        device::{
+            monitor::{Monitored, MonitoredOptions},
+            Dev,
+        },
+        serde::EntrySerializer,
+        DevExt, DirectFsDeviceOptions,
+    };
+
+    fn cache_for_test() -> Cache<u64, Vec<u8>> {
+        CacheBuilder::new(10)
+            .with_eviction_config(FifoConfig::default())
+            .build()
+    }
+
+    async fn device_for_test(dir: impl AsRef<Path>) -> MonitoredDevice {
+        let runtime = Runtime::current();
+        Monitored::open(
+            MonitoredOptions {
+                options: DirectFsDeviceOptions {
+                    dir: dir.as_ref().into(),
+                    capacity: ByteSize::kib(64).as_u64() as _,
+                    file_size: ByteSize::kib(16).as_u64() as _,
+                }
+                .into(),
+                metrics: Arc::new(Metrics::new("test")),
+            },
+            runtime,
+        )
+        .await
+        .unwrap()
+    }
+
+    async fn store_for_test(dir: impl AsRef<Path>) -> GenericSmallStorage<u64, Vec<u8>, RandomState> {
+        let device = device_for_test(dir).await;
+        let regions = 0..device.regions() as RegionId;
+        let config = GenericSmallStorageConfig {
+            set_size: ByteSize::kib(4).as_u64() as _,
+            set_cache_capacity: 4,
+            device,
+            regions,
+            flush: false,
+            flushers: 1,
+            buffer_pool_size: ByteSize::kib(64).as_u64() as _,
+            statistics: Arc::<Statistics>::default(),
+            runtime: Runtime::new(None, None, Handle::current()),
+            marker: PhantomData,
+        };
+        GenericSmallStorage::open(config).await.unwrap()
+    }
+
+    fn enqueue(store: &GenericSmallStorage<u64, Vec<u8>, RandomState>, entry: &CacheEntry<u64, Vec<u8>>) {
+        let estimated_size = EntrySerializer::estimated_size(entry.key(), entry.value());
+        store.enqueue(entry.clone(), estimated_size);
+    }
+
+    async fn assert_some(store: &GenericSmallStorage<u64, Vec<u8>, RandomState>, entry: &CacheEntry<u64, Vec<u8>>) {
+        assert_eq!(
+            store.load(entry.hash()).await.unwrap().unwrap(),
+            (*entry.key(), entry.value().clone())
+        );
+    }
+
+    async fn assert_none(store: &GenericSmallStorage<u64, Vec<u8>, RandomState>, entry: &CacheEntry<u64, Vec<u8>>) {
+        assert!(store.load(entry.hash()).await.unwrap().is_none());
+    }
+
+    #[test_log::test(tokio::test)]
+    async fn test_store_enqueue_lookup_destroy_recovery() {
+        let dir = tempfile::tempdir().unwrap();
+
+        let memory = cache_for_test();
+        let store = store_for_test(dir.path()).await;
+
+        let e1 = memory.insert(1, vec![1; 42]);
+        enqueue(&store, &e1);
+        store.wait().await;
+
+        assert_some(&store, &e1).await;
+
+        store.delete(e1.hash());
+        store.wait().await;
+
+        assert_none(&store, &e1).await;
+
+        let e2 = memory.insert(2, vec![2; 192]);
+        let e3 = memory.insert(3, vec![3; 168]);
+
+        enqueue(&store, &e1);
+        enqueue(&store, &e2);
+        enqueue(&store, &e3);
+        store.wait().await;
+
+        assert_some(&store, &e1).await;
+        assert_some(&store, &e2).await;
+        assert_some(&store, &e3).await;
+
+        store.destroy().await.unwrap();
+
+        assert_none(&store, &e1).await;
+        assert_none(&store, &e2).await;
+        assert_none(&store, &e3).await;
+    }
+}
diff --git a/foyer-storage/src/small/set.rs b/foyer-storage/src/small/set.rs
index 62234e21..2af18480 100644
--- a/foyer-storage/src/small/set.rs
+++ b/foyer-storage/src/small/set.rs
@@ -83,7 +83,7 @@ impl SetMut {
 /// # Format
 ///
 /// ```plain
-/// | checksum (4B) | timestamp (8B) | len (4B) |
+/// | checksum (4B) | ns timestamp (16B) | len (4B) |
 /// | bloom filter (4 * 8B = 32B) |
 /// ```
 pub struct SetStorage {
@@ -97,7 +97,7 @@ pub struct SetStorage {
     /// Set size.
     size: usize,
     /// Set last updated timestamp.
-    timestamp: u64,
+    timestamp: u128,
     /// Set bloom filter.
     bloom_filter: BloomFilterU64<4>,
 
@@ -118,15 +118,18 @@ impl Debug for SetStorage {
 }
 
 impl SetStorage {
-    pub const SET_HEADER_SIZE: usize = 48;
+    pub const SET_HEADER_SIZE: usize = 56;
 
-    pub fn load(buffer: IoBytesMut) -> Self {
+    /// Load the set storage from buffer.
+    ///
+    /// If `after` is set and the set storage is before the timestamp, load an empty set storage.
+    pub fn load(buffer: IoBytesMut, watermark: u128) -> Self {
         assert!(buffer.len() >= Self::SET_HEADER_SIZE);
 
         let checksum = (&buffer[0..4]).get_u32();
-        let timestamp = (&buffer[4..12]).get_u64();
-        let len = (&buffer[12..16]).get_u32() as usize;
-        let bloom_filter = BloomFilterU64::read(&buffer[16..48]);
+        let timestamp = (&buffer[4..20]).get_u128();
+        let len = (&buffer[20..24]).get_u32() as usize;
+        let bloom_filter = BloomFilterU64::read(&buffer[24..56]);
 
         let mut this = Self {
             checksum,
@@ -138,25 +141,29 @@ impl SetStorage {
             buffer,
         };
 
-        if Self::SET_HEADER_SIZE + this.len >= this.buffer.len() {
+        this.verify(watermark);
+
+        this
+    }
+
+    fn verify(&mut self, watermark: u128) {
+        if Self::SET_HEADER_SIZE + self.len >= self.buffer.len() || self.timestamp < watermark {
             // invalid len
-            this.clear();
+            self.clear();
         } else {
-            let c = Checksummer::checksum32(&this.buffer[4..Self::SET_HEADER_SIZE + this.len]);
-            if c != checksum {
+            let c = Checksummer::checksum32(&self.buffer[4..Self::SET_HEADER_SIZE + self.len]);
+            if c != self.checksum {
                 // checksum mismatch
-                this.clear();
+                self.clear();
             }
         }
-
-        this
     }
 
     pub fn update(&mut self) {
-        self.bloom_filter.write(&mut self.buffer[16..48]);
-        (&mut self.buffer[12..16]).put_u32(self.len as _);
-        self.timestamp = SystemTime::now().duration_since(UNIX_EPOCH).unwrap().as_millis() as u64;
-        (&mut self.buffer[4..12]).put_u64(self.timestamp);
+        self.bloom_filter.write(&mut self.buffer[24..56]);
+        (&mut self.buffer[20..24]).put_u32(self.len as _);
+        self.timestamp = SetTimestamp::current();
+        (&mut self.buffer[4..20]).put_u128(self.timestamp);
         self.checksum = Checksummer::checksum32(&self.buffer[4..Self::SET_HEADER_SIZE + self.len]);
         (&mut self.buffer[0..4]).put_u32(self.checksum);
     }
@@ -384,6 +391,14 @@ impl<'a> Iterator for SetIter<'a> {
     }
 }
 
+pub struct SetTimestamp;
+
+impl SetTimestamp {
+    pub fn current() -> u128 {
+        SystemTime::now().duration_since(UNIX_EPOCH).unwrap().as_nanos()
+    }
+}
+
 #[cfg(test)]
 mod tests {
 
@@ -436,7 +451,7 @@ mod tests {
     #[should_panic]
     fn test_set_storage_empty() {
         let buffer = IoBytesMut::new();
-        SetStorage::load(buffer);
+        SetStorage::load(buffer, 0);
     }
 
     #[test]
@@ -447,7 +462,7 @@ mod tests {
         unsafe { buf.set_len(PAGE) };
 
         // load will result in an empty set
-        let mut storage = SetStorage::load(buf);
+        let mut storage = SetStorage::load(buf, 0);
         assert!(storage.is_empty());
 
         let e1 = memory.insert(1, vec![b'1'; 42]);
@@ -510,7 +525,7 @@ mod tests {
         let mut buf = IoBytesMut::with_capacity(PAGE);
         unsafe { buf.set_len(PAGE) };
         buf[0..bytes.len()].copy_from_slice(&bytes);
-        let mut storage = SetStorage::load(buf);
+        let mut storage = SetStorage::load(buf, 0);
 
         assert_eq!(storage.len(), b4.len());
         assert_none(&storage, e1.hash());
diff --git a/foyer-storage/src/small/set_manager.rs b/foyer-storage/src/small/set_manager.rs
index 98dd3c22..9a1974cc 100644
--- a/foyer-storage/src/small/set_manager.rs
+++ b/foyer-storage/src/small/set_manager.rs
@@ -18,6 +18,7 @@ use std::{
     sync::Arc,
 };
 
+use bytes::{Buf, BufMut};
 use foyer_common::strict_assert;
 use itertools::Itertools;
 use ordered_hash_map::OrderedHashMap;
@@ -25,12 +26,12 @@ use tokio::sync::{Mutex, RwLock, RwLockReadGuard, RwLockWriteGuard};
 
 use super::{
     bloom_filter::BloomFilterU64,
-    set::{Set, SetId, SetMut, SetStorage},
+    set::{Set, SetId, SetMut, SetStorage, SetTimestamp},
 };
 use crate::{
-    device::{MonitoredDevice, RegionId},
+    device::{Dev, MonitoredDevice, RegionId},
     error::Result,
-    Dev,
+    IoBytesMut,
 };
 
 struct SetManagerInner {
@@ -42,6 +43,9 @@ struct SetManagerInner {
     sets: Vec<RwLock<BloomFilterU64<4>>>,
     cache: Mutex<OrderedHashMap<SetId, Arc<SetStorage>>>,
     set_cache_capacity: usize,
+    set_picker: SetPicker,
+
+    metadata: RwLock<Metadata>,
 
     set_size: usize,
     device: MonitoredDevice,
@@ -60,6 +64,8 @@ impl Debug for SetManager {
             .field("sets", &self.inner.sets)
             .field("cache", &self.inner.cache)
             .field("set_cache_capacity", &self.inner.set_cache_capacity)
+            .field("set_picker", &self.inner.set_picker)
+            .field("metadata", &self.inner.metadata)
             .field("set_size", &self.inner.set_size)
             .field("device", &self.inner.device)
             .field("regions", &self.inner.regions)
@@ -69,16 +75,23 @@ impl Debug for SetManager {
 }
 
 impl SetManager {
-    pub fn new(
+    pub async fn open(
         set_size: usize,
         set_cache_capacity: usize,
         device: MonitoredDevice,
         regions: Range<RegionId>,
         flush: bool,
-    ) -> Self {
+    ) -> Result<Self> {
         let sets = (device.region_size() / set_size) * (regions.end - regions.start) as usize;
         assert!(sets > 0);
 
+        let set_picker = SetPicker::new(sets);
+
+        // load & flush metadata
+        let metadata = Metadata::load(&device).await?;
+        metadata.flush(&device).await?;
+        let metadata = RwLock::new(metadata);
+
         let sets = (0..sets).map(|_| RwLock::default()).collect_vec();
         let cache = Mutex::new(OrderedHashMap::with_capacity(set_cache_capacity));
 
@@ -86,13 +99,15 @@ impl SetManager {
             sets,
             cache,
             set_cache_capacity,
+            set_picker,
+            metadata,
             set_size,
             device,
             regions,
             flush,
         };
         let inner = Arc::new(inner);
-        Self { inner }
+        Ok(Self { inner })
     }
 
     pub async fn write(&self, id: SetId) -> Result<SetWriteGuard<'_>> {
@@ -175,10 +190,32 @@ impl SetManager {
         self.inner.set_size
     }
 
+    pub fn set_picker(&self) -> &SetPicker {
+        &self.inner.set_picker
+    }
+
+    pub async fn watermark(&self) -> u128 {
+        self.inner.metadata.read().await.watermark
+    }
+
+    pub async fn destroy(&self) -> Result<()> {
+        self.update_watermark().await?;
+        self.inner.cache.lock().await.clear();
+        Ok(())
+    }
+
+    async fn update_watermark(&self) -> Result<()> {
+        let mut metadata = self.inner.metadata.write().await;
+
+        let watermark = SetTimestamp::current();
+        metadata.watermark = watermark;
+        metadata.flush(&self.inner.device).await
+    }
+
     async fn storage(&self, id: SetId) -> Result<SetStorage> {
         let (region, offset) = self.locate(id);
         let buffer = self.inner.device.read(region, offset, self.inner.set_size).await?;
-        let storage = SetStorage::load(buffer);
+        let storage = SetStorage::load(buffer, self.watermark().await);
         Ok(storage)
     }
 
@@ -251,3 +288,75 @@ impl<'a> Deref for SetReadGuard<'a> {
         &self.set
     }
 }
+
+#[derive(Debug, Clone)]
+pub struct SetPicker {
+    sets: usize,
+}
+
+impl SetPicker {
+    /// Create a [`SetPicker`] with a total size count.
+    ///
+    /// The `sets` should be the count of all sets.
+    ///
+    /// Note:
+    ///
+    /// The 0th set will be used as the meta set.
+    pub fn new(sets: usize) -> Self {
+        Self { sets }
+    }
+
+    pub fn sid(&self, hash: u64) -> SetId {
+        // skip the meta set
+        hash % (self.sets as SetId - 1) + 1
+    }
+}
+
+#[derive(Debug)]
+struct Metadata {
+    /// watermark timestamp
+    watermark: u128,
+}
+
+impl Default for Metadata {
+    fn default() -> Self {
+        Self {
+            watermark: SetTimestamp::current(),
+        }
+    }
+}
+
+impl Metadata {
+    const MAGIC: u64 = 0x20230512deadbeef;
+    const SIZE: usize = 8 + 16;
+
+    fn write(&self, mut buf: impl BufMut) {
+        buf.put_u64(Self::MAGIC);
+        buf.put_u128(self.watermark);
+    }
+
+    fn read(mut buf: impl Buf) -> Self {
+        let magic = buf.get_u64();
+        let watermark = buf.get_u128();
+
+        if magic != Self::MAGIC || watermark > SetTimestamp::current() {
+            return Self::default();
+        }
+
+        Self { watermark }
+    }
+
+    async fn flush(&self, device: &MonitoredDevice) -> Result<()> {
+        let mut buf = IoBytesMut::with_capacity(Self::SIZE);
+        self.write(&mut buf);
+        let buf = buf.freeze();
+        device.write(buf, 0, 0).await?;
+        Ok(())
+    }
+
+    async fn load(device: &MonitoredDevice) -> Result<Self> {
+        let buf = device.read(0, 0, Metadata::SIZE).await?;
+        let metadata = Metadata::read(&buf[..Metadata::SIZE]);
+        Ok(metadata)
+    }
+}

From 7b2b9cefd9b5737261025dfe4fe18c52df250183 Mon Sep 17 00:00:00 2001
From: Croxx <mrcroxx@outlook.com>
Date: Sun, 29 Sep 2024 00:02:02 +0800
Subject: [PATCH 20/53] refactor: refine device build (#745)

* refactor: refine device build

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

* refactor: restrict device config visibility

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

* refactor: remove verify from dev config trait, unnecessary

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

---------

Signed-off-by: MrCroxx <mrcroxx@outlook.com>
---
 examples/hybrid.rs                      |  8 +--
 examples/hybrid_full.rs                 |  9 ++-
 examples/tail_based_tracing.rs          |  8 +--
 foyer-bench/src/main.rs                 | 20 +++---
 foyer-storage/src/device/direct_file.rs | 83 +++++++++++-----------
 foyer-storage/src/device/direct_fs.rs   | 92 ++++++++++++-------------
 foyer-storage/src/device/mod.rs         | 53 ++++++--------
 foyer-storage/src/device/monitor.rs     | 29 +++-----
 foyer-storage/src/large/generic.rs      | 20 +++---
 foyer-storage/src/large/scanner.rs      | 18 +++--
 foyer-storage/src/large/tombstone.rs    | 18 ++---
 foyer-storage/src/prelude.rs            |  8 +--
 foyer-storage/src/small/generic.rs      | 14 ++--
 foyer-storage/src/store.rs              | 43 ++++++------
 foyer-storage/tests/storage_test.rs     |  9 ++-
 foyer/src/hybrid/builder.rs             |  8 +--
 foyer/src/hybrid/cache.rs               | 14 ++--
 foyer/src/prelude.rs                    |  9 ++-
 18 files changed, 211 insertions(+), 252 deletions(-)

diff --git a/examples/hybrid.rs b/examples/hybrid.rs
index fe15af3f..898cd31d 100644
--- a/examples/hybrid.rs
+++ b/examples/hybrid.rs
@@ -12,7 +12,7 @@
 //  See the License for the specific language governing permissions and
 //  limitations under the License.
 
-use foyer::{DirectFsDeviceOptionsBuilder, Engine, HybridCache, HybridCacheBuilder};
+use foyer::{DirectFsDeviceOptions, Engine, HybridCache, HybridCacheBuilder};
 
 #[tokio::main]
 async fn main() -> anyhow::Result<()> {
@@ -21,11 +21,7 @@ async fn main() -> anyhow::Result<()> {
     let hybrid: HybridCache<u64, String> = HybridCacheBuilder::new()
         .memory(64 * 1024 * 1024)
         .storage(Engine::Large) // use large object disk cache engine only
-        .with_device_config(
-            DirectFsDeviceOptionsBuilder::new(dir.path())
-                .with_capacity(256 * 1024 * 1024)
-                .build(),
-        )
+        .with_device_options(DirectFsDeviceOptions::new(dir.path()).with_capacity(256 * 1024 * 1024))
         .build()
         .await?;
 
diff --git a/examples/hybrid_full.rs b/examples/hybrid_full.rs
index acdf3d40..aca38a03 100644
--- a/examples/hybrid_full.rs
+++ b/examples/hybrid_full.rs
@@ -17,7 +17,7 @@ use std::sync::Arc;
 use anyhow::Result;
 use chrono::Datelike;
 use foyer::{
-    DirectFsDeviceOptionsBuilder, Engine, FifoPicker, HybridCache, HybridCacheBuilder, LargeEngineOptions, LruConfig,
+    DirectFsDeviceOptions, Engine, FifoPicker, HybridCache, HybridCacheBuilder, LargeEngineOptions, LruConfig,
     RateLimitPicker, RecoverMode, RuntimeConfig, SmallEngineOptions, TokioRuntimeConfig, TombstoneLogConfigBuilder,
 };
 use tempfile::tempdir;
@@ -36,11 +36,10 @@ async fn main() -> Result<()> {
         .with_hash_builder(ahash::RandomState::default())
         .with_weighter(|_key, value: &String| value.len())
         .storage(Engine::Mixed(0.1))
-        .with_device_config(
-            DirectFsDeviceOptionsBuilder::new(dir.path())
+        .with_device_options(
+            DirectFsDeviceOptions::new(dir.path())
                 .with_capacity(64 * 1024 * 1024)
-                .with_file_size(4 * 1024 * 1024)
-                .build(),
+                .with_file_size(4 * 1024 * 1024),
         )
         .with_flush(true)
         .with_recover_mode(RecoverMode::Quiet)
diff --git a/examples/tail_based_tracing.rs b/examples/tail_based_tracing.rs
index 254e4377..6bfbba98 100644
--- a/examples/tail_based_tracing.rs
+++ b/examples/tail_based_tracing.rs
@@ -14,7 +14,7 @@
 
 use std::time::Duration;
 
-use foyer::{DirectFsDeviceOptionsBuilder, Engine, HybridCache, HybridCacheBuilder};
+use foyer::{DirectFsDeviceOptions, Engine, HybridCache, HybridCacheBuilder};
 
 #[cfg(feature = "jaeger")]
 fn init_jaeger_exporter() {
@@ -71,11 +71,7 @@ async fn main() -> anyhow::Result<()> {
     let hybrid: HybridCache<u64, String> = HybridCacheBuilder::new()
         .memory(64 * 1024 * 1024)
         .storage(Engine::Large)
-        .with_device_config(
-            DirectFsDeviceOptionsBuilder::new(dir.path())
-                .with_capacity(256 * 1024 * 1024)
-                .build(),
-        )
+        .with_device_options(DirectFsDeviceOptions::new(dir.path()).with_capacity(256 * 1024 * 1024))
         .build()
         .await?;
 
diff --git a/foyer-bench/src/main.rs b/foyer-bench/src/main.rs
index 0a0ef321..5cd639e1 100644
--- a/foyer-bench/src/main.rs
+++ b/foyer-bench/src/main.rs
@@ -34,9 +34,9 @@ use analyze::{analyze, monitor, Metrics};
 use bytesize::ByteSize;
 use clap::{builder::PossibleValuesParser, ArgGroup, Parser};
 use foyer::{
-    Compression, DirectFileDeviceOptionsBuilder, DirectFsDeviceOptionsBuilder, Engine, FifoConfig, FifoPicker,
-    HybridCache, HybridCacheBuilder, InvalidRatioPicker, LargeEngineOptions, LfuConfig, LruConfig, RateLimitPicker,
-    RecoverMode, RuntimeConfig, S3FifoConfig, SmallEngineOptions, TokioRuntimeConfig, TracingConfig,
+    Compression, DirectFileDeviceOptions, DirectFsDeviceOptions, Engine, FifoConfig, FifoPicker, HybridCache,
+    HybridCacheBuilder, InvalidRatioPicker, LargeEngineOptions, LfuConfig, LruConfig, RateLimitPicker, RecoverMode,
+    RuntimeConfig, S3FifoConfig, SmallEngineOptions, TokioRuntimeConfig, TracingConfig,
 };
 use futures::future::join_all;
 use itertools::Itertools;
@@ -462,17 +462,15 @@ async fn benchmark(args: Args) {
         .storage(args.engine);
 
     builder = match (args.file.as_ref(), args.dir.as_ref()) {
-        (Some(file), None) => builder.with_device_config(
-            DirectFileDeviceOptionsBuilder::new(file)
+        (Some(file), None) => builder.with_device_options(
+            DirectFileDeviceOptions::new(file)
                 .with_capacity(args.disk.as_u64() as _)
-                .with_region_size(args.region_size.as_u64() as _)
-                .build(),
+                .with_region_size(args.region_size.as_u64() as _),
         ),
-        (None, Some(dir)) => builder.with_device_config(
-            DirectFsDeviceOptionsBuilder::new(dir)
+        (None, Some(dir)) => builder.with_device_options(
+            DirectFsDeviceOptions::new(dir)
                 .with_capacity(args.disk.as_u64() as _)
-                .with_file_size(args.region_size.as_u64() as _)
-                .build(),
+                .with_file_size(args.region_size.as_u64() as _),
         ),
         _ => unreachable!(),
     };
diff --git a/foyer-storage/src/device/direct_file.rs b/foyer-storage/src/device/direct_file.rs
index 8c4ab1ff..88bba196 100644
--- a/foyer-storage/src/device/direct_file.rs
+++ b/foyer-storage/src/device/direct_file.rs
@@ -22,36 +22,21 @@ use foyer_common::{asyncify::asyncify_with_runtime, bits};
 use fs4::free_space;
 use serde::{Deserialize, Serialize};
 
-use super::{Dev, DevExt, DevOptions, RegionId};
+use super::{Dev, DevExt, RegionId};
 use crate::{
     device::ALIGN,
     error::{Error, Result},
     IoBytes, IoBytesMut, Runtime,
 };
 
-/// Options for the direct file device.
 #[derive(Debug, Clone, Serialize, Deserialize)]
-pub struct DirectFileDeviceOptions {
-    /// Path of the direct file device.
-    pub path: PathBuf,
-    /// Capacity of the direct file device.
-    pub capacity: usize,
-    /// Region size of the direct file device.
-    pub region_size: usize,
-}
-
-/// A device that uses a single direct i/o file.
-#[derive(Debug, Clone)]
-pub struct DirectFileDevice {
-    file: Arc<File>,
-
+pub struct DirectFileDeviceConfig {
+    path: PathBuf,
     capacity: usize,
     region_size: usize,
-
-    runtime: Runtime,
 }
 
-impl DevOptions for DirectFileDeviceOptions {
+impl DirectFileDeviceConfig {
     fn verify(&self) -> Result<()> {
         if self.region_size == 0 || self.region_size % ALIGN != 0 {
             return Err(anyhow::anyhow!(
@@ -74,6 +59,17 @@ impl DevOptions for DirectFileDeviceOptions {
     }
 }
 
+/// A device that uses a single direct i/o file.
+#[derive(Debug, Clone)]
+pub struct DirectFileDevice {
+    file: Arc<File>,
+
+    capacity: usize,
+    region_size: usize,
+
+    runtime: Runtime,
+}
+
 impl DirectFileDevice {
     /// Positioned write API for the direct file device.
     #[fastrace::trace(name = "foyer::storage::device::direct_file::pwrite")]
@@ -160,7 +156,7 @@ impl DirectFileDevice {
 }
 
 impl Dev for DirectFileDevice {
-    type Options = DirectFileDeviceOptions;
+    type Config = DirectFileDeviceConfig;
 
     fn capacity(&self) -> usize {
         self.capacity
@@ -171,7 +167,7 @@ impl Dev for DirectFileDevice {
     }
 
     #[fastrace::trace(name = "foyer::storage::device::direct_file::open")]
-    async fn open(options: Self::Options, runtime: Runtime) -> Result<Self> {
+    async fn open(options: Self::Config, runtime: Runtime) -> Result<Self> {
         options.verify()?;
 
         let dir = options
@@ -254,19 +250,19 @@ impl Dev for DirectFileDevice {
     }
 }
 
-/// [`DirectFileDeviceOptionsBuilder`] is used to build the options for the direct fs device.
+/// [`DirectFileDeviceOptions`] is used to build the options for the direct fs device.
 ///
 /// The direct fs device uses a directory in a file system to store the data of disk cache.
 ///
 /// It uses direct I/O to reduce buffer copy and page cache pollution if supported.
 #[derive(Debug)]
-pub struct DirectFileDeviceOptionsBuilder {
+pub struct DirectFileDeviceOptions {
     path: PathBuf,
     capacity: Option<usize>,
     region_size: Option<usize>,
 }
 
-impl DirectFileDeviceOptionsBuilder {
+impl DirectFileDeviceOptions {
     const DEFAULT_FILE_SIZE: usize = 64 * 1024 * 1024;
 
     /// Use the given file path as the direct file device path.
@@ -297,14 +293,15 @@ impl DirectFileDeviceOptionsBuilder {
         self.region_size = Some(region_size);
         self
     }
+}
 
-    /// Build the options of the direct file device with the given arguments.
-    pub fn build(self) -> DirectFileDeviceOptions {
-        let path = self.path;
+impl From<DirectFileDeviceOptions> for DirectFileDeviceConfig {
+    fn from(options: DirectFileDeviceOptions) -> Self {
+        let path = options.path;
 
         let align_v = |value: usize, align: usize| value - value % align;
 
-        let capacity = self.capacity.unwrap_or({
+        let capacity = options.capacity.unwrap_or({
             // Create an empty directory before to get free space.
             let dir = path.parent().expect("path must point to a file").to_path_buf();
             create_dir_all(&dir).unwrap();
@@ -312,12 +309,15 @@ impl DirectFileDeviceOptionsBuilder {
         });
         let capacity = align_v(capacity, ALIGN);
 
-        let region_size = self.region_size.unwrap_or(Self::DEFAULT_FILE_SIZE).min(capacity);
+        let region_size = options
+            .region_size
+            .unwrap_or(DirectFileDeviceOptions::DEFAULT_FILE_SIZE)
+            .min(capacity);
         let region_size = align_v(region_size, ALIGN);
 
         let capacity = align_v(capacity, region_size);
 
-        DirectFileDeviceOptions {
+        DirectFileDeviceConfig {
             path,
             capacity,
             region_size,
@@ -335,11 +335,11 @@ mod tests {
     fn test_options_builder() {
         let dir = tempfile::tempdir().unwrap();
 
-        let options = DirectFileDeviceOptionsBuilder::new(dir.path().join("test-direct-file")).build();
+        let config: DirectFileDeviceConfig = DirectFileDeviceOptions::new(dir.path().join("test-direct-file")).into();
 
-        tracing::debug!("{options:?}");
+        tracing::debug!("{config:?}");
 
-        options.verify().unwrap();
+        config.verify().unwrap();
     }
 
     #[test_log::test]
@@ -347,11 +347,12 @@ mod tests {
     fn test_options_builder_noent() {
         let dir = tempfile::tempdir().unwrap();
 
-        let options = DirectFileDeviceOptionsBuilder::new(dir.path().join("noent").join("test-direct-file")).build();
+        let config: DirectFileDeviceConfig =
+            DirectFileDeviceOptions::new(dir.path().join("noent").join("test-direct-file")).into();
 
-        tracing::debug!("{options:?}");
+        tracing::debug!("{config:?}");
 
-        options.verify().unwrap();
+        config.verify().unwrap();
     }
 
     #[test_log::test(tokio::test)]
@@ -359,14 +360,14 @@ mod tests {
         let dir = tempfile::tempdir().unwrap();
         let runtime = Runtime::current();
 
-        let options = DirectFileDeviceOptionsBuilder::new(dir.path().join("test-direct-file"))
+        let config: DirectFileDeviceConfig = DirectFileDeviceOptions::new(dir.path().join("test-direct-file"))
             .with_capacity(4 * 1024 * 1024)
             .with_region_size(1024 * 1024)
-            .build();
+            .into();
 
-        tracing::debug!("{options:?}");
+        tracing::debug!("{config:?}");
 
-        let device = DirectFileDevice::open(options.clone(), runtime.clone()).await.unwrap();
+        let device = DirectFileDevice::open(config.clone(), runtime.clone()).await.unwrap();
 
         let mut buf = IoBytesMut::with_capacity(64 * 1024);
         buf.extend(repeat_n(b'x', 64 * 1024 - 100));
@@ -381,7 +382,7 @@ mod tests {
 
         drop(device);
 
-        let device = DirectFileDevice::open(options, runtime).await.unwrap();
+        let device = DirectFileDevice::open(config, runtime).await.unwrap();
 
         let b = device.read(0, 4096, 64 * 1024 - 100).await.unwrap().freeze();
         assert_eq!(buf, b);
diff --git a/foyer-storage/src/device/direct_fs.rs b/foyer-storage/src/device/direct_fs.rs
index 4e1deb95..30310a06 100644
--- a/foyer-storage/src/device/direct_fs.rs
+++ b/foyer-storage/src/device/direct_fs.rs
@@ -24,41 +24,21 @@ use futures::future::try_join_all;
 use itertools::Itertools;
 use serde::{Deserialize, Serialize};
 
-use super::{Dev, DevExt, DevOptions, RegionId};
+use super::{Dev, DevExt, RegionId};
 use crate::{
     device::ALIGN,
     error::{Error, Result},
     IoBytes, IoBytesMut, Runtime,
 };
 
-/// Options for the direct fs device.
 #[derive(Debug, Clone, Serialize, Deserialize)]
-pub struct DirectFsDeviceOptions {
-    /// Directory of the direct fs device.
-    pub dir: PathBuf,
-    /// Capacity of the direct fs device.
-    pub capacity: usize,
-    /// Direct i/o file size of the direct fs device.
-    pub file_size: usize,
-}
-
-/// A device that uses direct i/o files in a directory of a file system.
-#[derive(Debug, Clone)]
-pub struct DirectFsDevice {
-    inner: Arc<DirectFsDeviceInner>,
-}
-
-#[derive(Debug)]
-struct DirectFsDeviceInner {
-    files: Vec<Arc<File>>,
-
+pub struct DirectFsDeviceConfig {
+    dir: PathBuf,
     capacity: usize,
     file_size: usize,
-
-    runtime: Runtime,
 }
 
-impl DevOptions for DirectFsDeviceOptions {
+impl DirectFsDeviceConfig {
     fn verify(&self) -> Result<()> {
         if self.file_size == 0 || self.file_size % ALIGN != 0 {
             return Err(anyhow::anyhow!(
@@ -81,6 +61,22 @@ impl DevOptions for DirectFsDeviceOptions {
     }
 }
 
+/// A device that uses direct i/o files in a directory of a file system.
+#[derive(Debug, Clone)]
+pub struct DirectFsDevice {
+    inner: Arc<DirectFsDeviceInner>,
+}
+
+#[derive(Debug)]
+struct DirectFsDeviceInner {
+    files: Vec<Arc<File>>,
+
+    capacity: usize,
+    file_size: usize,
+
+    runtime: Runtime,
+}
+
 impl DirectFsDevice {
     const PREFIX: &'static str = "foyer-storage-direct-fs-";
 
@@ -94,7 +90,7 @@ impl DirectFsDevice {
 }
 
 impl Dev for DirectFsDevice {
-    type Options = DirectFsDeviceOptions;
+    type Config = DirectFsDeviceConfig;
 
     fn capacity(&self) -> usize {
         self.inner.capacity
@@ -105,7 +101,7 @@ impl Dev for DirectFsDevice {
     }
 
     #[fastrace::trace(name = "foyer::storage::device::direct_fs::open")]
-    async fn open(options: Self::Options, runtime: Runtime) -> Result<Self> {
+    async fn open(options: Self::Config, runtime: Runtime) -> Result<Self> {
         options.verify()?;
 
         // TODO(MrCroxx): write and read options to a manifest file for pinning
@@ -248,19 +244,19 @@ impl Dev for DirectFsDevice {
     }
 }
 
-/// [`DirectFsDeviceOptionsBuilder`] is used to build the options for the direct fs device.
+/// [`DirectFsDeviceOptions`] is used to build the options for the direct fs device.
 ///
 /// The direct fs device uses a directory in a file system to store the data of disk cache.
 ///
 /// It uses direct I/O to reduce buffer copy and page cache pollution if supported.
 #[derive(Debug)]
-pub struct DirectFsDeviceOptionsBuilder {
+pub struct DirectFsDeviceOptions {
     dir: PathBuf,
     capacity: Option<usize>,
     file_size: Option<usize>,
 }
 
-impl DirectFsDeviceOptionsBuilder {
+impl DirectFsDeviceOptions {
     const DEFAULT_FILE_SIZE: usize = 64 * 1024 * 1024;
 
     /// Use the given `dir` as the direct fs device.
@@ -291,26 +287,30 @@ impl DirectFsDeviceOptionsBuilder {
         self.file_size = Some(file_size);
         self
     }
+}
 
-    /// Build the options of the direct fs device with the given arguments.
-    pub fn build(self) -> DirectFsDeviceOptions {
-        let dir = self.dir;
+impl From<DirectFsDeviceOptions> for DirectFsDeviceConfig {
+    fn from(options: DirectFsDeviceOptions) -> Self {
+        let dir = options.dir;
 
         let align_v = |value: usize, align: usize| value - value % align;
 
-        let capacity = self.capacity.unwrap_or({
+        let capacity = options.capacity.unwrap_or({
             // Create an empty directory before to get free space.
             create_dir_all(&dir).unwrap();
             free_space(&dir).unwrap() as usize / 10 * 8
         });
         let capacity = align_v(capacity, ALIGN);
 
-        let file_size = self.file_size.unwrap_or(Self::DEFAULT_FILE_SIZE).min(capacity);
+        let file_size = options
+            .file_size
+            .unwrap_or(DirectFsDeviceOptions::DEFAULT_FILE_SIZE)
+            .min(capacity);
         let file_size = align_v(file_size, ALIGN);
 
         let capacity = align_v(capacity, file_size);
 
-        DirectFsDeviceOptions {
+        DirectFsDeviceConfig {
             dir,
             capacity,
             file_size,
@@ -328,11 +328,11 @@ mod tests {
     fn test_options_builder() {
         let dir = tempfile::tempdir().unwrap();
 
-        let options = DirectFsDeviceOptionsBuilder::new(dir.path()).build();
+        let config: DirectFsDeviceConfig = DirectFsDeviceOptions::new(dir.path()).into();
 
-        tracing::debug!("{options:?}");
+        tracing::debug!("{config:?}");
 
-        options.verify().unwrap();
+        config.verify().unwrap();
     }
 
     #[test_log::test]
@@ -340,11 +340,11 @@ mod tests {
     fn test_options_builder_noent() {
         let dir = tempfile::tempdir().unwrap();
 
-        let options = DirectFsDeviceOptionsBuilder::new(dir.path().join("noent")).build();
+        let config: DirectFsDeviceConfig = DirectFsDeviceOptions::new(dir.path().join("noent")).into();
 
-        tracing::debug!("{options:?}");
+        tracing::debug!("{config:?}");
 
-        options.verify().unwrap();
+        config.verify().unwrap();
     }
 
     #[test_log::test(tokio::test)]
@@ -352,14 +352,14 @@ mod tests {
         let dir = tempfile::tempdir().unwrap();
         let runtime = Runtime::current();
 
-        let options = DirectFsDeviceOptionsBuilder::new(dir.path())
+        let config: DirectFsDeviceConfig = DirectFsDeviceOptions::new(dir.path())
             .with_capacity(4 * 1024 * 1024)
             .with_file_size(1024 * 1024)
-            .build();
+            .into();
 
-        tracing::debug!("{options:?}");
+        tracing::debug!("{config:?}");
 
-        let device = DirectFsDevice::open(options.clone(), runtime.clone()).await.unwrap();
+        let device = DirectFsDevice::open(config.clone(), runtime.clone()).await.unwrap();
 
         let mut buf = IoBytesMut::with_capacity(64 * 1024);
         buf.extend(repeat_n(b'x', 64 * 1024 - 100));
@@ -374,7 +374,7 @@ mod tests {
 
         drop(device);
 
-        let device = DirectFsDevice::open(options, runtime).await.unwrap();
+        let device = DirectFsDevice::open(config, runtime).await.unwrap();
 
         let b = device.read(0, 4096, 64 * 1024 - 100).await.unwrap().freeze();
         assert_eq!(buf, b);
diff --git a/foyer-storage/src/device/mod.rs b/foyer-storage/src/device/mod.rs
index 998dbf25..15127518 100644
--- a/foyer-storage/src/device/mod.rs
+++ b/foyer-storage/src/device/mod.rs
@@ -21,6 +21,8 @@ pub mod monitor;
 use std::{fmt::Debug, future::Future};
 
 use allocator::AlignedAllocator;
+use direct_file::DirectFileDeviceConfig;
+use direct_fs::DirectFsDeviceConfig;
 use monitor::Monitored;
 
 use crate::{
@@ -33,18 +35,16 @@ pub const IO_BUFFER_ALLOCATOR: AlignedAllocator<ALIGN> = AlignedAllocator::new()
 
 pub type RegionId = u32;
 
-/// Options for the device.
-pub trait DevOptions: Send + Sync + 'static + Debug + Clone {
-    /// Verify the correctness of the options.
-    fn verify(&self) -> Result<()>;
-}
+/// Config for the device.
+pub trait DevConfig: Send + Sync + 'static + Debug {}
+impl<T: Send + Sync + 'static + Debug> DevConfig for T {}
 
 /// [`Dev`] represents 4K aligned block device.
 ///
 /// Both i/o block and i/o buffer must be aligned to 4K.
 pub trait Dev: Send + Sync + 'static + Sized + Clone + Debug {
-    /// Options for the device.
-    type Options: DevOptions;
+    /// Config for the device.
+    type Config: DevConfig;
 
     /// The capacity of the device, must be 4K aligned.
     fn capacity(&self) -> usize;
@@ -53,9 +53,9 @@ pub trait Dev: Send + Sync + 'static + Sized + Clone + Debug {
     fn region_size(&self) -> usize;
 
     // TODO(MrCroxx): Refactor the builder.
-    /// Open the device with the given options.
+    /// Open the device with the given config.
     #[must_use]
-    fn open(options: Self::Options, runtime: Runtime) -> impl Future<Output = Result<Self>> + Send;
+    fn open(config: Self::Config, runtime: Runtime) -> impl Future<Output = Result<Self>> + Send;
 
     /// Write API for the device.
     #[must_use]
@@ -86,29 +86,20 @@ pub trait DevExt: Dev {
 impl<T> DevExt for T where T: Dev {}
 
 #[derive(Debug, Clone)]
-pub enum DeviceOptions {
-    DirectFile(DirectFileDeviceOptions),
-    DirectFs(DirectFsDeviceOptions),
-}
-
-impl From<DirectFileDeviceOptions> for DeviceOptions {
-    fn from(value: DirectFileDeviceOptions) -> Self {
-        Self::DirectFile(value)
-    }
+pub enum DeviceConfig {
+    DirectFile(DirectFileDeviceConfig),
+    DirectFs(DirectFsDeviceConfig),
 }
 
-impl From<DirectFsDeviceOptions> for DeviceOptions {
-    fn from(value: DirectFsDeviceOptions) -> Self {
-        Self::DirectFs(value)
+impl From<DirectFileDeviceOptions> for DeviceConfig {
+    fn from(options: DirectFileDeviceOptions) -> Self {
+        Self::DirectFile(options.into())
     }
 }
 
-impl DevOptions for DeviceOptions {
-    fn verify(&self) -> Result<()> {
-        match self {
-            DeviceOptions::DirectFile(dev) => dev.verify(),
-            DeviceOptions::DirectFs(dev) => dev.verify(),
-        }
+impl From<DirectFsDeviceOptions> for DeviceConfig {
+    fn from(options: DirectFsDeviceOptions) -> Self {
+        Self::DirectFs(options.into())
     }
 }
 
@@ -119,7 +110,7 @@ pub enum Device {
 }
 
 impl Dev for Device {
-    type Options = DeviceOptions;
+    type Config = DeviceConfig;
 
     fn capacity(&self) -> usize {
         match self {
@@ -135,10 +126,10 @@ impl Dev for Device {
         }
     }
 
-    async fn open(options: Self::Options, runtime: Runtime) -> Result<Self> {
+    async fn open(options: Self::Config, runtime: Runtime) -> Result<Self> {
         match options {
-            DeviceOptions::DirectFile(opts) => Ok(Self::DirectFile(DirectFileDevice::open(opts, runtime).await?)),
-            DeviceOptions::DirectFs(opts) => Ok(Self::DirectFs(DirectFsDevice::open(opts, runtime).await?)),
+            DeviceConfig::DirectFile(opts) => Ok(Self::DirectFile(DirectFileDevice::open(opts, runtime).await?)),
+            DeviceConfig::DirectFs(opts) => Ok(Self::DirectFs(DirectFsDevice::open(opts, runtime).await?)),
         }
     }
 
diff --git a/foyer-storage/src/device/monitor.rs b/foyer-storage/src/device/monitor.rs
index bf29227f..1e897b51 100644
--- a/foyer-storage/src/device/monitor.rs
+++ b/foyer-storage/src/device/monitor.rs
@@ -24,7 +24,7 @@ use std::{
 use foyer_common::{bits, metrics::Metrics};
 
 use super::RegionId;
-use crate::{error::Result, Dev, DevExt, DevOptions, DirectFileDevice, IoBytes, IoBytesMut, Runtime};
+use crate::{error::Result, Dev, DevExt, DirectFileDevice, IoBytes, IoBytesMut, Runtime};
 
 /// The statistics information of the device.
 #[derive(Debug, Default)]
@@ -44,35 +44,26 @@ pub struct DeviceStats {
 }
 
 #[derive(Clone)]
-pub struct MonitoredOptions<D>
+pub struct MonitoredConfig<D>
 where
     D: Dev,
 {
-    pub options: D::Options,
+    pub config: D::Config,
     pub metrics: Arc<Metrics>,
 }
 
-impl<D> Debug for MonitoredOptions<D>
+impl<D> Debug for MonitoredConfig<D>
 where
     D: Dev,
 {
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
         f.debug_struct("MonitoredOptions")
-            .field("options", &self.options)
+            .field("options", &self.config)
             .field("metrics", &self.metrics)
             .finish()
     }
 }
 
-impl<D> DevOptions for MonitoredOptions<D>
-where
-    D: Dev,
-{
-    fn verify(&self) -> Result<()> {
-        self.options.verify()
-    }
-}
-
 #[derive(Debug, Clone)]
 pub struct Monitored<D>
 where
@@ -87,8 +78,8 @@ impl<D> Monitored<D>
 where
     D: Dev,
 {
-    async fn open(options: MonitoredOptions<D>, runtime: Runtime) -> Result<Self> {
-        let device = D::open(options.options, runtime).await?;
+    async fn open(options: MonitoredConfig<D>, runtime: Runtime) -> Result<Self> {
+        let device = D::open(options.config, runtime).await?;
         Ok(Self {
             device,
             stats: Arc::default(),
@@ -149,7 +140,7 @@ impl<D> Dev for Monitored<D>
 where
     D: Dev,
 {
-    type Options = MonitoredOptions<D>;
+    type Config = MonitoredConfig<D>;
 
     fn capacity(&self) -> usize {
         self.device.capacity()
@@ -159,8 +150,8 @@ where
         self.device.region_size()
     }
 
-    async fn open(options: Self::Options, runtime: Runtime) -> Result<Self> {
-        Self::open(options, runtime).await
+    async fn open(config: Self::Config, runtime: Runtime) -> Result<Self> {
+        Self::open(config, runtime).await
     }
 
     async fn write(&self, buf: IoBytes, region: RegionId, offset: u64) -> Result<()> {
diff --git a/foyer-storage/src/large/generic.rs b/foyer-storage/src/large/generic.rs
index c501714a..5c8fce90 100644
--- a/foyer-storage/src/large/generic.rs
+++ b/foyer-storage/src/large/generic.rs
@@ -488,20 +488,18 @@ mod tests {
     use std::{fs::File, path::Path};
 
     use ahash::RandomState;
+    use bytesize::ByteSize;
     use foyer_memory::{Cache, CacheBuilder, FifoConfig};
     use itertools::Itertools;
     use tokio::runtime::Handle;
 
     use super::*;
     use crate::{
-        device::{
-            direct_fs::DirectFsDeviceOptions,
-            monitor::{Monitored, MonitoredOptions},
-        },
+        device::monitor::{Monitored, MonitoredConfig},
         picker::utils::{FifoPicker, RejectAllPicker},
         serde::EntrySerializer,
         test_utils::BiasedPicker,
-        TombstoneLogConfigBuilder,
+        DirectFsDeviceOptions, TombstoneLogConfigBuilder,
     };
 
     const KB: usize = 1024;
@@ -515,13 +513,11 @@ mod tests {
     async fn device_for_test(dir: impl AsRef<Path>) -> MonitoredDevice {
         let runtime = Runtime::current();
         Monitored::open(
-            MonitoredOptions {
-                options: DirectFsDeviceOptions {
-                    dir: dir.as_ref().into(),
-                    capacity: 64 * KB,
-                    file_size: 16 * KB,
-                }
-                .into(),
+            MonitoredConfig {
+                config: DirectFsDeviceOptions::new(dir)
+                    .with_capacity(ByteSize::kib(64).as_u64() as _)
+                    .with_file_size(ByteSize::kib(16).as_u64() as _)
+                    .into(),
                 metrics: Arc::new(Metrics::new("test")),
             },
             runtime,
diff --git a/foyer-storage/src/large/scanner.rs b/foyer-storage/src/large/scanner.rs
index bd4bb77d..9704f6f0 100644
--- a/foyer-storage/src/large/scanner.rs
+++ b/foyer-storage/src/large/scanner.rs
@@ -241,28 +241,26 @@ impl RegionScanner {
 mod tests {
     use std::path::Path;
 
+    use bytesize::ByteSize;
+
     use super::*;
     use crate::{
         device::{
-            monitor::{Monitored, MonitoredOptions},
+            monitor::{Monitored, MonitoredConfig},
             Dev, MonitoredDevice,
         },
         region::RegionStats,
         DirectFsDeviceOptions, Runtime,
     };
 
-    const KB: usize = 1024;
-
     async fn device_for_test(dir: impl AsRef<Path>) -> MonitoredDevice {
         let runtime = Runtime::current();
         Monitored::open(
-            MonitoredOptions {
-                options: DirectFsDeviceOptions {
-                    dir: dir.as_ref().into(),
-                    capacity: 64 * KB,
-                    file_size: 16 * KB,
-                }
-                .into(),
+            MonitoredConfig {
+                config: DirectFsDeviceOptions::new(dir)
+                    .with_capacity(ByteSize::kib(64).as_u64() as _)
+                    .with_file_size(ByteSize::kib(16).as_u64() as _)
+                    .into(),
                 metrics: Arc::new(Metrics::new("test")),
             },
             runtime,
diff --git a/foyer-storage/src/large/tombstone.rs b/foyer-storage/src/large/tombstone.rs
index 3d61e803..c5da6a5a 100644
--- a/foyer-storage/src/large/tombstone.rs
+++ b/foyer-storage/src/large/tombstone.rs
@@ -25,12 +25,12 @@ use tokio::sync::Mutex;
 
 use crate::{
     device::{
-        direct_file::{DirectFileDevice, DirectFileDeviceOptionsBuilder},
-        monitor::{Monitored, MonitoredOptions},
+        direct_file::DirectFileDevice,
+        monitor::{Monitored, MonitoredConfig},
         Dev, DevExt, RegionId,
     },
     error::{Error, Result},
-    IoBytesMut, Runtime,
+    DirectFileDeviceOptions, IoBytesMut, Runtime,
 };
 
 /// The configurations for the tombstone log.
@@ -136,11 +136,11 @@ impl TombstoneLog {
         let capacity = bits::align_up(align, (cache_device.capacity() / align) * Tombstone::serialized_len());
 
         let device = Monitored::open(
-            MonitoredOptions {
-                options: DirectFileDeviceOptionsBuilder::new(path)
+            MonitoredConfig {
+                config: DirectFileDeviceOptions::new(path)
                     .with_region_size(align)
                     .with_capacity(capacity)
-                    .build(),
+                    .into(),
                 metrics,
             },
             runtime,
@@ -312,7 +312,7 @@ mod tests {
     use tempfile::tempdir;
 
     use super::*;
-    use crate::device::direct_fs::{DirectFsDevice, DirectFsDeviceOptionsBuilder};
+    use crate::device::direct_fs::{DirectFsDevice, DirectFsDeviceOptions};
 
     #[test_log::test(tokio::test)]
     async fn test_tombstone_log() {
@@ -322,9 +322,9 @@ mod tests {
 
         // 4 MB cache device => 16 KB tombstone log => 1K tombstones
         let device = DirectFsDevice::open(
-            DirectFsDeviceOptionsBuilder::new(dir.path())
+            DirectFsDeviceOptions::new(dir.path())
                 .with_capacity(4 * 1024 * 1024)
-                .build(),
+                .into(),
             runtime.clone(),
         )
         .await
diff --git a/foyer-storage/src/prelude.rs b/foyer-storage/src/prelude.rs
index ac221496..1cd942b4 100644
--- a/foyer-storage/src/prelude.rs
+++ b/foyer-storage/src/prelude.rs
@@ -16,10 +16,10 @@ pub use crate::{
     compress::Compression,
     device::{
         bytes::{IoBuffer, IoBytes, IoBytesMut},
-        direct_file::{DirectFileDevice, DirectFileDeviceOptions, DirectFileDeviceOptionsBuilder},
-        direct_fs::{DirectFsDevice, DirectFsDeviceOptions, DirectFsDeviceOptionsBuilder},
+        direct_file::{DirectFileDevice, DirectFileDeviceOptions},
+        direct_fs::{DirectFsDevice, DirectFsDeviceOptions},
         monitor::DeviceStats,
-        Dev, DevExt, DevOptions,
+        Dev, DevConfig, DevExt,
     },
     error::{Error, Result},
     large::{
@@ -34,7 +34,7 @@ pub use crate::{
     statistics::Statistics,
     storage::{either::Order, Storage},
     store::{
-        DeviceConfig, Engine, LargeEngineOptions, RuntimeConfig, SmallEngineOptions, Store, StoreBuilder,
+        DeviceOptions, Engine, LargeEngineOptions, RuntimeConfig, SmallEngineOptions, Store, StoreBuilder,
         TokioRuntimeConfig,
     },
 };
diff --git a/foyer-storage/src/small/generic.rs b/foyer-storage/src/small/generic.rs
index d1f6b753..07fa414f 100644
--- a/foyer-storage/src/small/generic.rs
+++ b/foyer-storage/src/small/generic.rs
@@ -307,7 +307,7 @@ mod tests {
     use super::*;
     use crate::{
         device::{
-            monitor::{Monitored, MonitoredOptions},
+            monitor::{Monitored, MonitoredConfig},
             Dev,
         },
         serde::EntrySerializer,
@@ -323,13 +323,11 @@ mod tests {
     async fn device_for_test(dir: impl AsRef<Path>) -> MonitoredDevice {
         let runtime = Runtime::current();
         Monitored::open(
-            MonitoredOptions {
-                options: DirectFsDeviceOptions {
-                    dir: dir.as_ref().into(),
-                    capacity: ByteSize::kib(64).as_u64() as _,
-                    file_size: ByteSize::kib(16).as_u64() as _,
-                }
-                .into(),
+            MonitoredConfig {
+                config: DirectFsDeviceOptions::new(dir)
+                    .with_capacity(ByteSize::kib(64).as_u64() as _)
+                    .with_file_size(ByteSize::kib(16).as_u64() as _)
+                    .into(),
                 metrics: Arc::new(Metrics::new("test")),
             },
             runtime,
diff --git a/foyer-storage/src/store.rs b/foyer-storage/src/store.rs
index adea7039..ac49db59 100644
--- a/foyer-storage/src/store.rs
+++ b/foyer-storage/src/store.rs
@@ -36,9 +36,8 @@ use tokio::runtime::Handle;
 use crate::{
     compress::Compression,
     device::{
-        direct_fs::DirectFsDeviceOptions,
-        monitor::{DeviceStats, Monitored, MonitoredOptions},
-        DeviceOptions, RegionId, ALIGN,
+        monitor::{DeviceStats, Monitored, MonitoredConfig},
+        DeviceConfig, RegionId, ALIGN,
     },
     engine::{EngineConfig, EngineEnum, SizeSelector},
     error::{Error, Result},
@@ -55,7 +54,7 @@ use crate::{
         either::{EitherConfig, Order},
         Storage,
     },
-    Dev, DevExt, DirectFileDeviceOptions,
+    Dev, DevExt, DirectFileDeviceOptions, DirectFsDeviceOptions,
 };
 
 /// The disk cache engine that serves as the storage backend of `foyer`.
@@ -204,22 +203,22 @@ where
 
 /// The configurations for the device.
 #[derive(Debug, Clone)]
-pub enum DeviceConfig {
+pub enum DeviceOptions {
     /// No device.
     None,
     /// With device options.
-    DeviceOptions(DeviceOptions),
+    DeviceConfig(DeviceConfig),
 }
 
-impl From<DirectFileDeviceOptions> for DeviceConfig {
-    fn from(value: DirectFileDeviceOptions) -> Self {
-        Self::DeviceOptions(value.into())
+impl From<DirectFileDeviceOptions> for DeviceOptions {
+    fn from(options: DirectFileDeviceOptions) -> Self {
+        Self::DeviceConfig(options.into())
     }
 }
 
-impl From<DirectFsDeviceOptions> for DeviceConfig {
-    fn from(value: DirectFsDeviceOptions) -> Self {
-        Self::DeviceOptions(value.into())
+impl From<DirectFsDeviceOptions> for DeviceOptions {
+    fn from(options: DirectFsDeviceOptions) -> Self {
+        Self::DeviceConfig(options.into())
     }
 }
 
@@ -348,7 +347,7 @@ where
     name: String,
     memory: Cache<K, V, S>,
 
-    device_config: DeviceConfig,
+    device_options: DeviceOptions,
     engine: Engine,
     runtime_config: RuntimeConfig,
 
@@ -377,7 +376,7 @@ where
             name: "foyer".to_string(),
             memory,
 
-            device_config: DeviceConfig::None,
+            device_options: DeviceOptions::None,
             engine,
             runtime_config: RuntimeConfig::Disabled,
 
@@ -401,9 +400,9 @@ where
         self
     }
 
-    /// Set device config for the disk cache store.
-    pub fn with_device_config(mut self, device_config: impl Into<DeviceConfig>) -> Self {
-        self.device_config = device_config.into();
+    /// Set device options for the disk cache store.
+    pub fn with_device_options(mut self, device_options: impl Into<DeviceOptions>) -> Self {
+        self.device_options = device_options.into();
         self
     }
 
@@ -524,16 +523,16 @@ where
             let runtime = runtime.clone();
             // Use the user runtime to open engine.
             tokio::spawn(async move {
-            match self.device_config {
-                DeviceConfig::None => {
+            match self.device_options {
+                DeviceOptions::None => {
                     tracing::warn!(
                         "[store builder]: No device config set. Use `NoneStore` which always returns `None` for queries."
                     );
                     EngineEnum::open(EngineConfig::Noop).await
                 }
-                DeviceConfig::DeviceOptions(options) => {
-                    let device = match Monitored::open(MonitoredOptions {
-                        options,
+                DeviceOptions::DeviceConfig(options) => {
+                    let device = match Monitored::open(MonitoredConfig {
+                        config: options,
                         metrics: metrics.clone(),
                     }, runtime.clone())
                     .await {
diff --git a/foyer-storage/tests/storage_test.rs b/foyer-storage/tests/storage_test.rs
index e20b75ef..a2304352 100644
--- a/foyer-storage/tests/storage_test.rs
+++ b/foyer-storage/tests/storage_test.rs
@@ -19,7 +19,7 @@ use std::{path::Path, sync::Arc, time::Duration};
 use ahash::RandomState;
 use foyer_memory::{Cache, CacheBuilder, CacheEntry, FifoConfig};
 use foyer_storage::{
-    test_utils::Recorder, Compression, DirectFsDeviceOptionsBuilder, Engine, LargeEngineOptions, StoreBuilder,
+    test_utils::Recorder, Compression, DirectFsDeviceOptions, Engine, LargeEngineOptions, StoreBuilder,
 };
 
 const KB: usize = 1024;
@@ -110,11 +110,10 @@ fn basic(
 ) -> StoreBuilder<u64, Vec<u8>> {
     // TODO(MrCroxx): Test mixed engine here.
     StoreBuilder::new(memory.clone(), Engine::Large)
-        .with_device_config(
-            DirectFsDeviceOptionsBuilder::new(path)
+        .with_device_options(
+            DirectFsDeviceOptions::new(path)
                 .with_capacity(4 * MB)
-                .with_file_size(MB)
-                .build(),
+                .with_file_size(MB),
         )
         .with_admission_picker(recorder.clone())
         .with_flush(true)
diff --git a/foyer/src/hybrid/builder.rs b/foyer/src/hybrid/builder.rs
index 892cf0b8..f5c3683e 100644
--- a/foyer/src/hybrid/builder.rs
+++ b/foyer/src/hybrid/builder.rs
@@ -22,7 +22,7 @@ use foyer_common::{
 };
 use foyer_memory::{Cache, CacheBuilder, EvictionConfig, Weighter};
 use foyer_storage::{
-    AdmissionPicker, Compression, DeviceConfig, Engine, LargeEngineOptions, RecoverMode, RuntimeConfig,
+    AdmissionPicker, Compression, DeviceOptions, Engine, LargeEngineOptions, RecoverMode, RuntimeConfig,
     SmallEngineOptions, StoreBuilder,
 };
 
@@ -204,9 +204,9 @@ where
     V: StorageValue,
     S: HashBuilder + Debug,
 {
-    /// Set device config for the disk cache store.
-    pub fn with_device_config(self, device_config: impl Into<DeviceConfig>) -> Self {
-        let builder = self.builder.with_device_config(device_config);
+    /// Set device options for the disk cache store.
+    pub fn with_device_options(self, device_options: impl Into<DeviceOptions>) -> Self {
+        let builder = self.builder.with_device_options(device_options);
         Self {
             name: self.name,
             tracing_config: self.tracing_config,
diff --git a/foyer/src/hybrid/cache.rs b/foyer/src/hybrid/cache.rs
index 49948fc2..5a728018 100644
--- a/foyer/src/hybrid/cache.rs
+++ b/foyer/src/hybrid/cache.rs
@@ -551,11 +551,10 @@ mod tests {
             .memory(4 * MB)
             // TODO(MrCroxx): Test with `Engine::Mixed`.
             .storage(Engine::Large)
-            .with_device_config(
-                DirectFsDeviceOptionsBuilder::new(dir)
+            .with_device_options(
+                DirectFsDeviceOptions::new(dir)
                     .with_capacity(16 * MB)
-                    .with_file_size(MB)
-                    .build(),
+                    .with_file_size(MB),
             )
             .build()
             .await
@@ -575,11 +574,10 @@ mod tests {
             .memory(4 * MB)
             // TODO(MrCroxx): Test with `Engine::Mixed`.
             .storage(Engine::Large)
-            .with_device_config(
-                DirectFsDeviceOptionsBuilder::new(dir)
+            .with_device_options(
+                DirectFsDeviceOptions::new(dir)
                     .with_capacity(16 * MB)
-                    .with_file_size(MB)
-                    .build(),
+                    .with_file_size(MB),
             )
             .with_admission_picker(Arc::new(BiasedPicker::new(admits)))
             .build()
diff --git a/foyer/src/prelude.rs b/foyer/src/prelude.rs
index 4746626e..937c1f5f 100644
--- a/foyer/src/prelude.rs
+++ b/foyer/src/prelude.rs
@@ -24,11 +24,10 @@ pub use memory::{
     S3FifoConfig, Weighter,
 };
 pub use storage::{
-    AdmissionPicker, AdmitAllPicker, Compression, Dev, DevExt, DevOptions, DeviceStats, DirectFileDevice,
-    DirectFileDeviceOptions, DirectFileDeviceOptionsBuilder, DirectFsDevice, DirectFsDeviceOptions,
-    DirectFsDeviceOptionsBuilder, Engine, EvictionPicker, FifoPicker, InvalidRatioPicker, LargeEngineOptions,
-    RateLimitPicker, RecoverMode, ReinsertionPicker, RejectAllPicker, Runtime, RuntimeConfig, SmallEngineOptions,
-    Storage, Store, StoreBuilder, TokioRuntimeConfig, TombstoneLogConfigBuilder,
+    AdmissionPicker, AdmitAllPicker, Compression, Dev, DevConfig, DevExt, DeviceStats, DirectFileDevice,
+    DirectFileDeviceOptions, DirectFsDevice, DirectFsDeviceOptions, Engine, EvictionPicker, FifoPicker,
+    InvalidRatioPicker, LargeEngineOptions, RateLimitPicker, RecoverMode, ReinsertionPicker, RejectAllPicker, Runtime,
+    RuntimeConfig, SmallEngineOptions, Storage, Store, StoreBuilder, TokioRuntimeConfig, TombstoneLogConfigBuilder,
 };
 
 pub use crate::hybrid::{

From 7a0a35fe449460a735d79b3bd53232d4926ce39a Mon Sep 17 00:00:00 2001
From: Croxx <mrcroxx@outlook.com>
Date: Sun, 29 Sep 2024 12:29:55 +0800
Subject: [PATCH 21/53] feat: introduce submit queue size threshold (#749)

* feat: introduce submit queue size threshold

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

* chore: update hint

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

---------

Signed-off-by: MrCroxx <mrcroxx@outlook.com>
---
 foyer-storage/src/large/flusher.rs | 27 +++++++++++++++++++++++----
 foyer-storage/src/large/generic.rs | 23 ++++++++++++++++++++++-
 foyer-storage/src/store.rs         | 15 +++++++++++++++
 3 files changed, 60 insertions(+), 5 deletions(-)

diff --git a/foyer-storage/src/large/flusher.rs b/foyer-storage/src/large/flusher.rs
index 4a789540..c3e1053d 100644
--- a/foyer-storage/src/large/flusher.rs
+++ b/foyer-storage/src/large/flusher.rs
@@ -15,7 +15,10 @@
 use std::{
     fmt::Debug,
     future::Future,
-    sync::{atomic::Ordering, Arc},
+    sync::{
+        atomic::{AtomicUsize, Ordering},
+        Arc,
+    },
 };
 
 use foyer_common::{
@@ -103,6 +106,7 @@ where
     S: HashBuilder + Debug,
 {
     tx: flume::Sender<Submission<K, V, S>>,
+    submit_queue_size: Arc<AtomicUsize>,
 
     metrics: Arc<Metrics>,
 }
@@ -116,6 +120,7 @@ where
     fn clone(&self) -> Self {
         Self {
             tx: self.tx.clone(),
+            submit_queue_size: self.submit_queue_size.clone(),
             metrics: self.metrics.clone(),
         }
     }
@@ -133,6 +138,7 @@ where
         indexer: Indexer,
         region_manager: RegionManager,
         device: MonitoredDevice,
+        submit_queue_size: Arc<AtomicUsize>,
         tombstone_log: Option<TombstoneLog>,
         stats: Arc<Statistics>,
         metrics: Arc<Metrics>,
@@ -153,6 +159,7 @@ where
             rx,
             batch,
             flight: Arc::new(Semaphore::new(1)),
+            submit_queue_size: submit_queue_size.clone(),
             region_manager,
             indexer,
             tombstone_log,
@@ -168,11 +175,18 @@ where
             }
         });
 
-        Ok(Self { tx, metrics })
+        Ok(Self {
+            tx,
+            submit_queue_size,
+            metrics,
+        })
     }
 
     pub fn submit(&self, submission: Submission<K, V, S>) {
         tracing::trace!("[lodc flusher]: submit task: {submission:?}");
+        if let Submission::CacheEntry { estimated_size, .. } = &submission {
+            self.submit_queue_size.fetch_add(*estimated_size, Ordering::Relaxed);
+        }
         if let Err(e) = self.tx.send(submission) {
             tracing::error!("[lodc flusher]: error raised when submitting task, error: {e}");
         }
@@ -196,6 +210,7 @@ where
     rx: flume::Receiver<Submission<K, V, S>>,
     batch: BatchMut<K, V, S>,
     flight: Arc<Semaphore>,
+    submit_queue_size: Arc<AtomicUsize>,
 
     region_manager: RegionManager,
     indexer: Indexer,
@@ -245,9 +260,13 @@ where
         match submission {
             Submission::CacheEntry {
                 entry,
-                estimated_size: _,
+                estimated_size,
                 sequence,
-            } => report(self.batch.entry(entry, &self.compression, sequence)),
+            } => {
+                report(self.batch.entry(entry, &self.compression, sequence));
+                self.submit_queue_size.fetch_sub(estimated_size, Ordering::Relaxed);
+            }
+
             Submission::Tombstone { tombstone, stats } => self.batch.tombstone(tombstone, stats),
             Submission::Reinsertion { reinsertion } => report(self.batch.reinsertion(&reinsertion)),
             Submission::Wait { tx } => self.batch.wait(tx),
diff --git a/foyer-storage/src/large/generic.rs b/foyer-storage/src/large/generic.rs
index 5c8fce90..047d96a7 100644
--- a/foyer-storage/src/large/generic.rs
+++ b/foyer-storage/src/large/generic.rs
@@ -18,7 +18,7 @@ use std::{
     marker::PhantomData,
     ops::Range,
     sync::{
-        atomic::{AtomicBool, Ordering},
+        atomic::{AtomicBool, AtomicUsize, Ordering},
         Arc,
     },
     time::Instant,
@@ -75,6 +75,7 @@ where
     pub flushers: usize,
     pub reclaimers: usize,
     pub buffer_pool_size: usize,
+    pub submit_queue_size_threshold: usize,
     pub clean_region_threshold: usize,
     pub eviction_pickers: Vec<Box<dyn EvictionPicker>>,
     pub reinsertion_picker: Arc<dyn ReinsertionPicker<Key = K>>,
@@ -102,6 +103,7 @@ where
             .field("flushers", &self.flushers)
             .field("reclaimers", &self.reclaimers)
             .field("buffer_pool_size", &self.buffer_pool_size)
+            .field("submit_queue_size_threshold", &self.submit_queue_size_threshold)
             .field("clean_region_threshold", &self.clean_region_threshold)
             .field("eviction_pickers", &self.eviction_pickers)
             .field("reinsertion_pickers", &self.reinsertion_picker)
@@ -145,6 +147,9 @@ where
     flushers: Vec<Flusher<K, V, S>>,
     reclaimers: Vec<Reclaimer>,
 
+    submit_queue_size: Arc<AtomicUsize>,
+    submit_queue_size_threshold: usize,
+
     statistics: Arc<Statistics>,
 
     flush: bool,
@@ -213,6 +218,7 @@ where
             metrics.clone(),
         );
         let sequence = AtomicSequence::default();
+        let submit_queue_size = Arc::<AtomicUsize>::default();
 
         RecoverRunner::run(
             &config,
@@ -232,6 +238,7 @@ where
                 indexer.clone(),
                 region_manager.clone(),
                 device.clone(),
+                submit_queue_size.clone(),
                 tombstone_log.clone(),
                 stats.clone(),
                 metrics.clone(),
@@ -262,6 +269,8 @@ where
                 region_manager,
                 flushers,
                 reclaimers,
+                submit_queue_size,
+                submit_queue_size_threshold: config.submit_queue_size_threshold,
                 statistics: stats,
                 flush: config.flush,
                 sequence,
@@ -294,7 +303,17 @@ where
             return;
         }
 
+        if self.inner.submit_queue_size.load(Ordering::Relaxed) > self.inner.submit_queue_size_threshold {
+            tracing::warn!(
+                "[lodc] {} {}",
+                "submit queue overflow, new entry ignored.",
+                "Hint: set an appropriate rate limiter as the admission picker or scale out flushers."
+            );
+            return;
+        }
+
         let sequence = self.inner.sequence.fetch_add(1, Ordering::Relaxed);
+
         self.inner.flushers[sequence as usize % self.inner.flushers.len()].submit(Submission::CacheEntry {
             entry,
             estimated_size,
@@ -553,6 +572,7 @@ mod tests {
             reinsertion_picker,
             tombstone_log_config: None,
             buffer_pool_size: 16 * 1024 * 1024,
+            submit_queue_size_threshold: 16 * 1024 * 1024 * 2,
             statistics: Arc::<Statistics>::default(),
             runtime: Runtime::new(None, None, Handle::current()),
             marker: PhantomData,
@@ -582,6 +602,7 @@ mod tests {
             reinsertion_picker: Arc::<RejectAllPicker<u64>>::default(),
             tombstone_log_config: Some(TombstoneLogConfigBuilder::new(path).with_flush(true).build()),
             buffer_pool_size: 16 * 1024 * 1024,
+            submit_queue_size_threshold: 16 * 1024 * 1024 * 2,
             statistics: Arc::<Statistics>::default(),
             runtime: Runtime::new(None, None, Handle::current()),
             marker: PhantomData,
diff --git a/foyer-storage/src/store.rs b/foyer-storage/src/store.rs
index ac49db59..741dd9c6 100644
--- a/foyer-storage/src/store.rs
+++ b/foyer-storage/src/store.rs
@@ -558,6 +558,7 @@ where
                                 reinsertion_picker: self.large.reinsertion_picker,
                                 tombstone_log_config: self.large.tombstone_log_config,
                                 buffer_pool_size: self.large.buffer_pool_size,
+                                submit_queue_size_threshold: self.large.submit_queue_size_threshold.unwrap_or(self.large.buffer_pool_size * 2),
                                 statistics: statistics.clone(),
                                 runtime,
                                 marker: PhantomData,
@@ -614,6 +615,7 @@ where
                                     reinsertion_picker: self.large.reinsertion_picker,
                                     tombstone_log_config: self.large.tombstone_log_config,
                                     buffer_pool_size: self.large.buffer_pool_size,
+                                    submit_queue_size_threshold: self.large.submit_queue_size_threshold.unwrap_or(self.large.buffer_pool_size * 2),
                                     statistics: statistics.clone(),
                                     runtime,
                                     marker: PhantomData,
@@ -656,6 +658,7 @@ where
     flushers: usize,
     reclaimers: usize,
     buffer_pool_size: usize,
+    submit_queue_size_threshold: Option<usize>,
     clean_region_threshold: Option<usize>,
     eviction_pickers: Vec<Box<dyn EvictionPicker>>,
     reinsertion_picker: Arc<dyn ReinsertionPicker<Key = K>>,
@@ -689,6 +692,7 @@ where
             flushers: 1,
             reclaimers: 1,
             buffer_pool_size: 16 * 1024 * 1024, // 16 MiB
+            submit_queue_size_threshold: None,
             clean_region_threshold: None,
             eviction_pickers: vec![Box::new(InvalidRatioPicker::new(0.8)), Box::<FifoPicker>::default()],
             reinsertion_picker: Arc::<RejectAllPicker<K>>::default(),
@@ -745,6 +749,17 @@ where
         self
     }
 
+    /// Set the submit queue size threshold.
+    ///
+    /// If the total entry estimated size in the submit queue exceeds the threshold, the further entries will be
+    /// ignored.
+    ///
+    /// Default: `buffer_pool_size`` * 2.
+    pub fn with_submit_queue_size_threshold(mut self, buffer_pool_size: usize) -> Self {
+        self.buffer_pool_size = buffer_pool_size;
+        self
+    }
+
     /// Set the clean region threshold for the disk cache store.
     ///
     /// The reclaimers only work when the clean region count is equal to or lower than the clean region threshold.

From d49c4800e264f88c53ecca038b011af99a7c82da Mon Sep 17 00:00:00 2001
From: Croxx <mrcroxx@outlook.com>
Date: Sun, 29 Sep 2024 22:30:24 +0800
Subject: [PATCH 22/53] bug: fix panic when serde as batching (#750)

Signed-off-by: MrCroxx <mrcroxx@outlook.com>
---
 foyer-storage/src/large/batch.rs | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/foyer-storage/src/large/batch.rs b/foyer-storage/src/large/batch.rs
index 602828fd..5adcea96 100644
--- a/foyer-storage/src/large/batch.rs
+++ b/foyer-storage/src/large/batch.rs
@@ -121,6 +121,11 @@ where
 
         let pos = self.len;
 
+        if pos + EntryHeader::serialized_len() >= self.buffer.len() {
+            // Only handle start position overflow. End position overflow will be handled by serde.
+            return false;
+        }
+
         let info = match EntrySerializer::serialize(
             entry.key(),
             entry.value(),

From 8d9b23f3b0b58485368b173ca582823493abe76e Mon Sep 17 00:00:00 2001
From: Croxx <mrcroxx@outlook.com>
Date: Tue, 8 Oct 2024 11:43:35 +0800
Subject: [PATCH 23/53] chore: fix typos with the lates typos checker (#752)

Signed-off-by: MrCroxx <mrcroxx@outlook.com>
---
 CHANGELOG.md                 | 4 ++--
 foyer-common/src/runtime.rs  | 2 +-
 foyer-storage/src/runtime.rs | 8 ++++----
 3 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index d42ef499..8b2741ad 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -351,7 +351,7 @@ date: 2023-05-12T11:02:09+08:00
 
 ### Changes
 
-- Make `HybridCache` clonable.
+- Make `HybridCache` cloneable.
 
 ## 2024-04-27
 
@@ -566,7 +566,7 @@ date: 2023-05-12T11:02:09+08:00
 
 ### Changes
 
-- Make eviction config clonable.
+- Make eviction config cloneable.
 
 ## 2024-03-13
 
diff --git a/foyer-common/src/runtime.rs b/foyer-common/src/runtime.rs
index 14be9c18..1a149741 100644
--- a/foyer-common/src/runtime.rs
+++ b/foyer-common/src/runtime.rs
@@ -67,7 +67,7 @@ impl From<Runtime> for BackgroundShutdownRuntime {
     }
 }
 
-/// A non-clonable runtime handle.
+/// A non-cloneable runtime handle.
 #[derive(Debug)]
 pub struct SingletonHandle(Handle);
 
diff --git a/foyer-storage/src/runtime.rs b/foyer-storage/src/runtime.rs
index a2266e90..4b000e61 100644
--- a/foyer-storage/src/runtime.rs
+++ b/foyer-storage/src/runtime.rs
@@ -27,7 +27,7 @@ struct RuntimeInner {
     user_runtime_handle: SingletonHandle,
 }
 
-/// [`Runtime`] holds the runtime reference and non-clonable handles to prevent handle usage after runtime shutdown.
+/// [`Runtime`] holds the runtime reference and non-cloneable handles to prevent handle usage after runtime shutdown.
 #[derive(Debug, Clone)]
 pub struct Runtime {
     inner: Arc<RuntimeInner>,
@@ -72,17 +72,17 @@ impl Runtime {
         }
     }
 
-    /// Get the non-clonable read runtime handle.
+    /// Get the non-cloneable read runtime handle.
     pub fn read(&self) -> &SingletonHandle {
         &self.inner.read_runtime_handle
     }
 
-    /// Get the non-clonable write runtime handle.
+    /// Get the non-cloneable write runtime handle.
     pub fn write(&self) -> &SingletonHandle {
         &self.inner.write_runtime_handle
     }
 
-    /// Get the non-clonable user runtime handle.
+    /// Get the non-cloneable user runtime handle.
     pub fn user(&self) -> &SingletonHandle {
         &self.inner.user_runtime_handle
     }

From 16bdb54bb39474e4c1bdc3b9216125d1186a3ce4 Mon Sep 17 00:00:00 2001
From: Croxx <mrcroxx@outlook.com>
Date: Tue, 8 Oct 2024 14:15:38 +0800
Subject: [PATCH 24/53] feat: support jeprof for foyer-bench (#748)

* feat: support jeprof for foyer-bench

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

* chore: update features and defauts for foyer-bench

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

* fix: remove mtrace from foyer-bench default feature

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

* refactor: remove jeprof as default for foyer-bench

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

---------

Signed-off-by: MrCroxx <mrcroxx@outlook.com>
---
 .gitignore             | 4 +++-
 Cargo.toml             | 2 +-
 foyer-bench/Cargo.toml | 2 ++
 3 files changed, 6 insertions(+), 2 deletions(-)

diff --git a/.gitignore b/.gitignore
index e53cc65c..5c844a32 100644
--- a/.gitignore
+++ b/.gitignore
@@ -11,4 +11,6 @@ docker-compose.override.yaml
 perf.data*
 flamegraph.svg
 
-trace.txt
\ No newline at end of file
+trace.txt
+jeprof.out.*
+*.collapsed
\ No newline at end of file
diff --git a/Cargo.toml b/Cargo.toml
index f917c84b..9878fe54 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -53,4 +53,4 @@ foyer-storage = { version = "0.12.0-dev", path = "foyer-storage" }
 foyer = { version = "0.12.0-dev", path = "foyer" }
 
 [profile.release]
-debug = true
+debug = "full"
diff --git a/foyer-bench/Cargo.toml b/foyer-bench/Cargo.toml
index 6c641de7..1f535370 100644
--- a/foyer-bench/Cargo.toml
+++ b/foyer-bench/Cargo.toml
@@ -39,9 +39,11 @@ zipf = "7"
 tikv-jemallocator = { version = "0.6", optional = true }
 
 [features]
+default = ["jemalloc"]
 deadlock = ["parking_lot/deadlock_detection", "foyer/deadlock"]
 tokio-console = ["console-subscriber"]
 strict_assertions = ["foyer/strict_assertions"]
 sanity = ["foyer/sanity"]
 jemalloc = ["tikv-jemallocator"]
+jeprof = ["jemalloc", "tikv-jemallocator/profiling"]
 mtrace = ["foyer/mtrace", "fastrace-jaeger", "fastrace"]

From d326b1d1d5680dc81b99bc5debd4d09427bc9159 Mon Sep 17 00:00:00 2001
From: Croxx <mrcroxx@outlook.com>
Date: Tue, 8 Oct 2024 14:47:49 +0800
Subject: [PATCH 25/53] chore: rename feature mtrace to tracing, refine
 manifest (#754)

Signed-off-by: MrCroxx <mrcroxx@outlook.com>
---
 .github/workflows/ci.yml       |  6 +++---
 Cargo.toml                     |  4 ++++
 Makefile                       |  6 +++---
 examples/tail_based_tracing.rs |  2 +-
 foyer-bench/Cargo.toml         |  8 ++++----
 foyer-bench/src/main.rs        | 20 ++++++++++----------
 foyer-common/Cargo.toml        |  8 ++++----
 foyer-memory/Cargo.toml        |  4 ++--
 foyer-storage/Cargo.toml       |  7 ++++---
 foyer-util/Cargo.toml          |  2 +-
 foyer/Cargo.toml               |  8 ++++----
 11 files changed, 40 insertions(+), 35 deletions(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index fe8163a4..e93a9ce8 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -164,7 +164,7 @@ jobs:
         run: |
           cargo clippy --all-targets --features tokio-console -- -D warnings
           cargo clippy --all-targets --features deadlock -- -D warnings
-          cargo clippy --all-targets --features mtrace -- -D warnings
+          cargo clippy --all-targets --features tracing -- -D warnings
           cargo clippy --all-targets -- -D warnings
       - if: steps.cache.outputs.cache-hit != 'true'
         uses: taiki-e/install-action@cargo-llvm-cov
@@ -191,8 +191,8 @@ jobs:
           cargo llvm-cov --no-report run --example hybrid
           cargo llvm-cov --no-report run --example hybrid_full
           cargo llvm-cov --no-report run --example event_listener
-          cargo llvm-cov --no-report run --features "mtrace,jaeger" --example tail_based_tracing
-          cargo llvm-cov --no-report run --features "mtrace,ot" --example tail_based_tracing
+          cargo llvm-cov --no-report run --features "tracing,jaeger" --example tail_based_tracing
+          cargo llvm-cov --no-report run --features "tracing,ot" --example tail_based_tracing
       - name: Run foyer-bench with coverage
         if: runner.os == 'Linux'
         env:
diff --git a/Cargo.toml b/Cargo.toml
index 9878fe54..90be98e0 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -45,6 +45,7 @@ fastrace-jaeger = "0.7"
 fastrace-opentelemetry = "0.7"
 clap = { version = "4", features = ["derive"] }
 bytesize = { package = "foyer-bytesize", version = "2" }
+hashbrown = "0.15"
 # foyer components
 foyer-common = { version = "0.12.0-dev", path = "foyer-common" }
 foyer-intrusive = { version = "0.12.0-dev", path = "foyer-intrusive" }
@@ -52,5 +53,8 @@ foyer-memory = { version = "0.12.0-dev", path = "foyer-memory" }
 foyer-storage = { version = "0.12.0-dev", path = "foyer-storage" }
 foyer = { version = "0.12.0-dev", path = "foyer" }
 
+[workspace.lints.rust]
+unexpected_cfgs = { level = "warn", check-cfg = ['cfg(madsim)'] }
+
 [profile.release]
 debug = "full"
diff --git a/Makefile b/Makefile
index 0c9c6fd6..d536f2ec 100644
--- a/Makefile
+++ b/Makefile
@@ -20,7 +20,7 @@ check-all:
 	cargo clippy --all-targets --features deadlock
 	cargo clippy --all-targets --features tokio-console
 	cargo clippy --all-targets --features sanity
-	cargo clippy --all-targets --features mtrace
+	cargo clippy --all-targets --features tracing
 	cargo clippy --all-targets
 
 test:
@@ -42,8 +42,8 @@ example:
 	cargo run --example hybrid
 	cargo run --example hybrid_full
 	cargo run --example event_listener
-	cargo run --features "mtrace,jaeger" --example tail_based_tracing
-	cargo run --features "mtrace,ot" --example tail_based_tracing
+	cargo run --features "tracing,jaeger" --example tail_based_tracing
+	cargo run --features "tracing,ot" --example tail_based_tracing
 
 full: check-all test-all example udeps
 
diff --git a/examples/tail_based_tracing.rs b/examples/tail_based_tracing.rs
index 6bfbba98..0d515091 100644
--- a/examples/tail_based_tracing.rs
+++ b/examples/tail_based_tracing.rs
@@ -61,7 +61,7 @@ fn init_exporter() {
     panic!("Either jaeger or opentelemetry feature must be enabled!");
 }
 
-/// NOTE: To run this example, please enable feature "mtrace" and either "jaeger" or "ot".
+/// NOTE: To run this example, please enable feature "tracing" and either "jaeger" or "ot".
 #[tokio::main]
 async fn main() -> anyhow::Result<()> {
     init_exporter();
diff --git a/foyer-bench/Cargo.toml b/foyer-bench/Cargo.toml
index 1f535370..d942e473 100644
--- a/foyer-bench/Cargo.toml
+++ b/foyer-bench/Cargo.toml
@@ -41,9 +41,9 @@ tikv-jemallocator = { version = "0.6", optional = true }
 [features]
 default = ["jemalloc"]
 deadlock = ["parking_lot/deadlock_detection", "foyer/deadlock"]
-tokio-console = ["console-subscriber"]
+tokio-console = ["dep:console-subscriber"]
 strict_assertions = ["foyer/strict_assertions"]
 sanity = ["foyer/sanity"]
-jemalloc = ["tikv-jemallocator"]
-jeprof = ["jemalloc", "tikv-jemallocator/profiling"]
-mtrace = ["foyer/mtrace", "fastrace-jaeger", "fastrace"]
+jemalloc = ["dep:tikv-jemallocator"]
+jeprof = ["jemalloc", "tikv-jemallocator?/profiling"]
+tracing = ["foyer/tracing", "dep:fastrace-jaeger", "dep:fastrace"]
diff --git a/foyer-bench/src/main.rs b/foyer-bench/src/main.rs
index 5cd639e1..bbe52904 100644
--- a/foyer-bench/src/main.rs
+++ b/foyer-bench/src/main.rs
@@ -244,19 +244,19 @@ pub struct Args {
     #[arg(long, default_value_t = 64)]
     set_cache_capacity: usize,
 
-    /// Record insert trace threshold. Only effective with "mtrace" feature.
+    /// Record insert trace threshold. Only effective with "tracing" feature.
     #[arg(long, default_value_t = 1000 * 1000)]
     trace_insert_us: usize,
-    /// Record get trace threshold. Only effective with "mtrace" feature.
+    /// Record get trace threshold. Only effective with "tracing" feature.
     #[arg(long, default_value_t = 1000 * 1000)]
     trace_get_us: usize,
-    /// Record obtain trace threshold. Only effective with "mtrace" feature.
+    /// Record obtain trace threshold. Only effective with "tracing" feature.
     #[arg(long, default_value_t = 1000 * 1000)]
     trace_obtain_us: usize,
-    /// Record remove trace threshold. Only effective with "mtrace" feature.
+    /// Record remove trace threshold. Only effective with "tracing" feature.
     #[arg(long, default_value_t = 1000 * 1000)]
     trace_remove_us: usize,
-    /// Record fetch trace threshold. Only effective with "mtrace" feature.
+    /// Record fetch trace threshold. Only effective with "tracing" feature.
     #[arg(long, default_value_t = 1000 * 1000)]
     trace_fetch_us: usize,
 }
@@ -350,14 +350,14 @@ fn setup() {
     console_subscriber::init();
 }
 
-#[cfg(feature = "mtrace")]
+#[cfg(feature = "tracing")]
 fn setup() {
     use fastrace::collector::Config;
     let reporter = fastrace_jaeger::JaegerReporter::new("127.0.0.1:6831".parse().unwrap(), "foyer-bench").unwrap();
     fastrace::set_reporter(reporter, Config::default().report_interval(Duration::from_millis(1)));
 }
 
-#[cfg(not(any(feature = "tokio-console", feature = "mtrace")))]
+#[cfg(not(any(feature = "tokio-console", feature = "tracing")))]
 fn setup() {
     use tracing_subscriber::{prelude::*, EnvFilter};
 
@@ -371,10 +371,10 @@ fn setup() {
         .init();
 }
 
-#[cfg(not(any(feature = "mtrace")))]
+#[cfg(not(any(feature = "tracing")))]
 fn teardown() {}
 
-#[cfg(feature = "mtrace")]
+#[cfg(feature = "tracing")]
 fn teardown() {
     fastrace::flush();
 }
@@ -532,7 +532,7 @@ async fn benchmark(args: Args) {
         .await
         .unwrap();
 
-    #[cfg(feature = "mtrace")]
+    #[cfg(feature = "tracing")]
     hybrid.enable_tracing();
 
     let stats = hybrid.stats();
diff --git a/foyer-common/Cargo.toml b/foyer-common/Cargo.toml
index c416d6cd..5eee4d06 100644
--- a/foyer-common/Cargo.toml
+++ b/foyer-common/Cargo.toml
@@ -18,7 +18,7 @@ cfg-if = "1"
 crossbeam = "0.8"
 fastrace = { workspace = true }
 futures = "0.3"
-hashbrown = "0.14"
+hashbrown = { workspace = true }
 itertools = { workspace = true }
 metrics = { workspace = true }
 parking_lot = { version = "0.12", features = ["arc_lock"] }
@@ -32,7 +32,7 @@ rand = "0.8.5"
 
 [features]
 strict_assertions = []
-mtrace = ["fastrace/enable"]
+tracing = ["fastrace/enable"]
 
-[lints.rust]
-unexpected_cfgs = { level = "warn", check-cfg = ['cfg(madsim)'] }
+[lints]
+workspace = true
diff --git a/foyer-memory/Cargo.toml b/foyer-memory/Cargo.toml
index 078fbe72..914d315c 100644
--- a/foyer-memory/Cargo.toml
+++ b/foyer-memory/Cargo.toml
@@ -20,7 +20,7 @@ fastrace = { workspace = true }
 foyer-common = { workspace = true }
 foyer-intrusive = { workspace = true }
 futures = "0.3"
-hashbrown = "0.14"
+hashbrown = { workspace = true }
 itertools = { workspace = true }
 parking_lot = "0.12"
 pin-project = "1"
@@ -43,7 +43,7 @@ strict_assertions = [
     "foyer-intrusive/strict_assertions",
 ]
 sanity = ["strict_assertions"]
-mtrace = ["fastrace/enable", "foyer-common/mtrace"]
+tracing = ["fastrace/enable", "foyer-common/tracing"]
 
 [[bench]]
 name = "bench_hit_ratio"
diff --git a/foyer-storage/Cargo.toml b/foyer-storage/Cargo.toml
index e0b85011..326fb17c 100644
--- a/foyer-storage/Cargo.toml
+++ b/foyer-storage/Cargo.toml
@@ -32,6 +32,7 @@ foyer-common = { workspace = true }
 foyer-memory = { workspace = true }
 fs4 = "0.9.1"
 futures = "0.3"
+hashbrown = { workspace = true }
 itertools = { workspace = true }
 libc = "0.2"
 lz4 = "1.24"
@@ -60,7 +61,7 @@ strict_assertions = [
     "foyer-common/strict_assertions",
     "foyer-memory/strict_assertions",
 ]
-mtrace = ["fastrace/enable", "foyer-common/mtrace", "foyer-memory/mtrace"]
+tracing = ["fastrace/enable", "foyer-common/tracing", "foyer-memory/tracing"]
 
-[lints.rust]
-unexpected_cfgs = { level = "warn", check-cfg = ['cfg(madsim)'] }
+[lints]
+workspace = true
diff --git a/foyer-util/Cargo.toml b/foyer-util/Cargo.toml
index ab381677..1b485ee4 100644
--- a/foyer-util/Cargo.toml
+++ b/foyer-util/Cargo.toml
@@ -20,7 +20,7 @@ bytes = "1"
 cfg-if = "1"
 foyer-common = { workspace = true }
 futures = "0.3"
-hashbrown = "0.14"
+hashbrown = { workspace = true }
 itertools = { workspace = true }
 parking_lot = { version = "0.12", features = ["arc_lock"] }
 serde = { workspace = true }
diff --git a/foyer/Cargo.toml b/foyer/Cargo.toml
index c9eb989b..aad21ea8 100644
--- a/foyer/Cargo.toml
+++ b/foyer/Cargo.toml
@@ -38,9 +38,9 @@ strict_assertions = [
     "foyer-storage/strict_assertions",
 ]
 sanity = ["strict_assertions", "foyer-memory/sanity"]
-mtrace = [
+tracing = [
     "fastrace/enable",
-    "foyer-common/mtrace",
-    "foyer-memory/mtrace",
-    "foyer-storage/mtrace",
+    "foyer-common/tracing",
+    "foyer-memory/tracing",
+    "foyer-storage/tracing",
 ]

From 726a5f5e37bf5d6c25194a114f36b7f8756a1278 Mon Sep 17 00:00:00 2001
From: Croxx <mrcroxx@outlook.com>
Date: Wed, 9 Oct 2024 00:04:03 +0800
Subject: [PATCH 26/53] refactor: refine runtime and tracing options (#756)

* refactor: refine runtime and tracing options

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

* refactor: rename API

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

* refactor: tiny refines

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

* chore: pass ffmt check

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

---------

Signed-off-by: MrCroxx <mrcroxx@outlook.com>
---
 examples/hybrid_full.rs        |   8 +-
 examples/tail_based_tracing.rs |   6 +-
 foyer-bench/Cargo.toml         |   1 +
 foyer-bench/src/main.rs        |  48 +++++------
 foyer-common/src/tracing.rs    | 151 ++++++++++++++++++++++-----------
 foyer-storage/src/prelude.rs   |   4 +-
 foyer-storage/src/store.rs     |  32 +++----
 foyer/src/hybrid/builder.rs    |  54 ++++++------
 foyer/src/hybrid/cache.rs      |  17 ++--
 foyer/src/prelude.rs           |  47 +++++-----
 10 files changed, 209 insertions(+), 159 deletions(-)

diff --git a/examples/hybrid_full.rs b/examples/hybrid_full.rs
index aca38a03..aba9510f 100644
--- a/examples/hybrid_full.rs
+++ b/examples/hybrid_full.rs
@@ -18,7 +18,7 @@ use anyhow::Result;
 use chrono::Datelike;
 use foyer::{
     DirectFsDeviceOptions, Engine, FifoPicker, HybridCache, HybridCacheBuilder, LargeEngineOptions, LruConfig,
-    RateLimitPicker, RecoverMode, RuntimeConfig, SmallEngineOptions, TokioRuntimeConfig, TombstoneLogConfigBuilder,
+    RateLimitPicker, RecoverMode, RuntimeOptions, SmallEngineOptions, TokioRuntimeOptions, TombstoneLogConfigBuilder,
 };
 use tempfile::tempdir;
 
@@ -45,12 +45,12 @@ async fn main() -> Result<()> {
         .with_recover_mode(RecoverMode::Quiet)
         .with_admission_picker(Arc::new(RateLimitPicker::new(100 * 1024 * 1024)))
         .with_compression(foyer::Compression::Lz4)
-        .with_runtime_config(RuntimeConfig::Separated {
-            read_runtime_config: TokioRuntimeConfig {
+        .with_runtime_options(RuntimeOptions::Separated {
+            read_runtime_options: TokioRuntimeOptions {
                 worker_threads: 4,
                 max_blocking_threads: 8,
             },
-            write_runtime_config: TokioRuntimeConfig {
+            write_runtime_options: TokioRuntimeOptions {
                 worker_threads: 4,
                 max_blocking_threads: 8,
             },
diff --git a/examples/tail_based_tracing.rs b/examples/tail_based_tracing.rs
index 0d515091..4c20e91f 100644
--- a/examples/tail_based_tracing.rs
+++ b/examples/tail_based_tracing.rs
@@ -14,7 +14,7 @@
 
 use std::time::Duration;
 
-use foyer::{DirectFsDeviceOptions, Engine, HybridCache, HybridCacheBuilder};
+use foyer::{DirectFsDeviceOptions, Engine, HybridCache, HybridCacheBuilder, TracingOptions};
 
 #[cfg(feature = "jaeger")]
 fn init_jaeger_exporter() {
@@ -76,9 +76,7 @@ async fn main() -> anyhow::Result<()> {
         .await?;
 
     hybrid.enable_tracing();
-    hybrid
-        .tracing_config()
-        .set_record_hybrid_get_threshold(Duration::from_millis(10));
+    hybrid.update_tracing_options(TracingOptions::new().with_record_hybrid_get_threshold(Duration::from_millis(10)));
 
     hybrid.insert(42, "The answer to life, the universe, and everything.".to_string());
     assert_eq!(
diff --git a/foyer-bench/Cargo.toml b/foyer-bench/Cargo.toml
index d942e473..966c2e91 100644
--- a/foyer-bench/Cargo.toml
+++ b/foyer-bench/Cargo.toml
@@ -22,6 +22,7 @@ fastrace-jaeger = { workspace = true, optional = true }
 foyer = { workspace = true }
 futures = "0.3"
 hdrhistogram = "7"
+humantime = "2"
 itertools = { workspace = true }
 metrics = { workspace = true }
 metrics-exporter-prometheus = "0.15"
diff --git a/foyer-bench/src/main.rs b/foyer-bench/src/main.rs
index bbe52904..bd3541b9 100644
--- a/foyer-bench/src/main.rs
+++ b/foyer-bench/src/main.rs
@@ -36,7 +36,7 @@ use clap::{builder::PossibleValuesParser, ArgGroup, Parser};
 use foyer::{
     Compression, DirectFileDeviceOptions, DirectFsDeviceOptions, Engine, FifoConfig, FifoPicker, HybridCache,
     HybridCacheBuilder, InvalidRatioPicker, LargeEngineOptions, LfuConfig, LruConfig, RateLimitPicker, RecoverMode,
-    RuntimeConfig, S3FifoConfig, SmallEngineOptions, TokioRuntimeConfig, TracingConfig,
+    RuntimeOptions, S3FifoConfig, SmallEngineOptions, TokioRuntimeOptions, TracingOptions,
 };
 use futures::future::join_all;
 use itertools::Itertools;
@@ -245,20 +245,20 @@ pub struct Args {
     set_cache_capacity: usize,
 
     /// Record insert trace threshold. Only effective with "tracing" feature.
-    #[arg(long, default_value_t = 1000 * 1000)]
-    trace_insert_us: usize,
+    #[arg(long, default_value = "1s")]
+    trace_insert: humantime::Duration,
     /// Record get trace threshold. Only effective with "tracing" feature.
-    #[arg(long, default_value_t = 1000 * 1000)]
-    trace_get_us: usize,
+    #[arg(long, default_value = "1s")]
+    trace_get: humantime::Duration,
     /// Record obtain trace threshold. Only effective with "tracing" feature.
-    #[arg(long, default_value_t = 1000 * 1000)]
-    trace_obtain_us: usize,
+    #[arg(long, default_value = "1s")]
+    trace_obtain: humantime::Duration,
     /// Record remove trace threshold. Only effective with "tracing" feature.
-    #[arg(long, default_value_t = 1000 * 1000)]
-    trace_remove_us: usize,
+    #[arg(long, default_value = "1s")]
+    trace_remove: humantime::Duration,
     /// Record fetch trace threshold. Only effective with "tracing" feature.
-    #[arg(long, default_value_t = 1000 * 1000)]
-    trace_fetch_us: usize,
+    #[arg(long, default_value = "1s")]
+    trace_fetch: humantime::Duration,
 }
 
 #[derive(Debug)]
@@ -433,15 +433,15 @@ async fn benchmark(args: Args) {
             .unwrap();
     }
 
-    let tracing_config = TracingConfig::default();
-    tracing_config.set_record_hybrid_insert_threshold(Duration::from_micros(args.trace_insert_us as _));
-    tracing_config.set_record_hybrid_get_threshold(Duration::from_micros(args.trace_get_us as _));
-    tracing_config.set_record_hybrid_obtain_threshold(Duration::from_micros(args.trace_obtain_us as _));
-    tracing_config.set_record_hybrid_remove_threshold(Duration::from_micros(args.trace_remove_us as _));
-    tracing_config.set_record_hybrid_fetch_threshold(Duration::from_micros(args.trace_fetch_us as _));
+    let tracing_options = TracingOptions::new()
+        .with_record_hybrid_insert_threshold(args.trace_insert.into())
+        .with_record_hybrid_get_threshold(args.trace_get.into())
+        .with_record_hybrid_obtain_threshold(args.trace_obtain.into())
+        .with_record_hybrid_remove_threshold(args.trace_remove.into())
+        .with_record_hybrid_fetch_threshold(args.trace_fetch.into());
 
     let builder = HybridCacheBuilder::new()
-        .with_tracing_config(tracing_config)
+        .with_tracing_options(tracing_options)
         .memory(args.mem.as_u64() as _)
         .with_shards(args.shards);
 
@@ -479,18 +479,18 @@ async fn benchmark(args: Args) {
         .with_flush(args.flush)
         .with_recover_mode(args.recover_mode)
         .with_compression(args.compression)
-        .with_runtime_config(match args.runtime.as_str() {
-            "disabled" => RuntimeConfig::Disabled,
-            "unified" => RuntimeConfig::Unified(TokioRuntimeConfig {
+        .with_runtime_options(match args.runtime.as_str() {
+            "disabled" => RuntimeOptions::Disabled,
+            "unified" => RuntimeOptions::Unified(TokioRuntimeOptions {
                 worker_threads: args.runtime_worker_threads,
                 max_blocking_threads: args.runtime_max_blocking_threads,
             }),
-            "separated" => RuntimeConfig::Separated {
-                read_runtime_config: TokioRuntimeConfig {
+            "separated" => RuntimeOptions::Separated {
+                read_runtime_options: TokioRuntimeOptions {
                     worker_threads: args.read_runtime_worker_threads,
                     max_blocking_threads: args.read_runtime_max_blocking_threads,
                 },
-                write_runtime_config: TokioRuntimeConfig {
+                write_runtime_options: TokioRuntimeOptions {
                     worker_threads: args.write_runtime_worker_threads,
                     max_blocking_threads: args.write_runtime_max_blocking_threads,
                 },
diff --git a/foyer-common/src/tracing.rs b/foyer-common/src/tracing.rs
index 057e9c50..1c3e9f20 100644
--- a/foyer-common/src/tracing.rs
+++ b/foyer-common/src/tracing.rs
@@ -15,7 +15,7 @@
 use std::{
     ops::Deref,
     pin::Pin,
-    sync::atomic::{AtomicUsize, Ordering},
+    sync::atomic::{AtomicU64, Ordering},
     task::{Context, Poll},
     time::Duration,
 };
@@ -26,87 +26,136 @@ use pin_project::pin_project;
 use serde::{Deserialize, Serialize};
 
 /// Configurations for tracing.
-#[derive(Debug, Serialize, Deserialize)]
+#[derive(Debug, Default)]
 pub struct TracingConfig {
     /// Threshold for recording the hybrid cache `insert` and `insert_with_context` operation in us.
-    record_hybrid_insert_threshold_us: AtomicUsize,
+    record_hybrid_insert_threshold_us: AtomicU64,
     /// Threshold for recording the hybrid cache `get` operation in us.
-    record_hybrid_get_threshold_us: AtomicUsize,
+    record_hybrid_get_threshold_us: AtomicU64,
     /// Threshold for recording the hybrid cache `obtain` operation in us.
-    record_hybrid_obtain_threshold_us: AtomicUsize,
+    record_hybrid_obtain_threshold_us: AtomicU64,
     /// Threshold for recording the hybrid cache `remove` operation in us.
-    record_hybrid_remove_threshold_us: AtomicUsize,
+    record_hybrid_remove_threshold_us: AtomicU64,
     /// Threshold for recording the hybrid cache `fetch` operation in us.
-    record_hybrid_fetch_threshold_us: AtomicUsize,
+    record_hybrid_fetch_threshold_us: AtomicU64,
 }
 
-impl Default for TracingConfig {
-    /// All thresholds are set to `1s`.
-    fn default() -> Self {
-        Self {
-            record_hybrid_insert_threshold_us: AtomicUsize::from(1000 * 1000),
-            record_hybrid_get_threshold_us: AtomicUsize::from(1000 * 1000),
-            record_hybrid_obtain_threshold_us: AtomicUsize::from(1000 * 1000),
-            record_hybrid_remove_threshold_us: AtomicUsize::from(1000 * 1000),
-            record_hybrid_fetch_threshold_us: AtomicUsize::from(1000 * 1000),
+impl TracingConfig {
+    /// Update tracing config with options.
+    pub fn update(&self, options: TracingOptions) {
+        if let Some(threshold) = options.record_hybrid_insert_threshold {
+            self.record_hybrid_insert_threshold_us
+                .store(threshold.as_micros() as _, Ordering::Relaxed);
         }
-    }
-}
 
-impl TracingConfig {
-    /// Set the threshold for recording the hybrid cache `insert` and `insert_with_context` operation.
-    pub fn set_record_hybrid_insert_threshold(&self, threshold: Duration) {
-        self.record_hybrid_insert_threshold_us
-            .store(threshold.as_micros() as _, Ordering::Relaxed);
+        if let Some(threshold) = options.record_hybrid_get_threshold {
+            self.record_hybrid_get_threshold_us
+                .store(threshold.as_micros() as _, Ordering::Relaxed);
+        }
+
+        if let Some(threshold) = options.record_hybrid_obtain_threshold {
+            self.record_hybrid_obtain_threshold_us
+                .store(threshold.as_micros() as _, Ordering::Relaxed);
+        }
+
+        if let Some(threshold) = options.record_hybrid_remove_threshold {
+            self.record_hybrid_remove_threshold_us
+                .store(threshold.as_micros() as _, Ordering::Relaxed);
+        }
+
+        if let Some(threshold) = options.record_hybrid_fetch_threshold {
+            self.record_hybrid_fetch_threshold_us
+                .store(threshold.as_micros() as _, Ordering::Relaxed);
+        }
     }
 
     /// Threshold for recording the hybrid cache `insert` and `insert_with_context` operation.
     pub fn record_hybrid_insert_threshold(&self) -> Duration {
-        Duration::from_micros(self.record_hybrid_insert_threshold_us.load(Ordering::Relaxed) as _)
-    }
-
-    /// Set the threshold for recording the hybrid cache `get` operation.
-    pub fn set_record_hybrid_get_threshold(&self, threshold: Duration) {
-        self.record_hybrid_get_threshold_us
-            .store(threshold.as_micros() as _, Ordering::Relaxed);
+        Duration::from_micros(self.record_hybrid_insert_threshold_us.load(Ordering::Relaxed))
     }
 
     /// Threshold for recording the hybrid cache `get` operation.
     pub fn record_hybrid_get_threshold(&self) -> Duration {
-        Duration::from_micros(self.record_hybrid_get_threshold_us.load(Ordering::Relaxed) as _)
-    }
-
-    /// Set the threshold for recording the hybrid cache `obtain` operation.
-    pub fn set_record_hybrid_obtain_threshold(&self, threshold: Duration) {
-        self.record_hybrid_obtain_threshold_us
-            .store(threshold.as_micros() as _, Ordering::Relaxed);
+        Duration::from_micros(self.record_hybrid_get_threshold_us.load(Ordering::Relaxed))
     }
 
     /// Threshold for recording the hybrid cache `obtain` operation.
     pub fn record_hybrid_obtain_threshold(&self) -> Duration {
-        Duration::from_micros(self.record_hybrid_obtain_threshold_us.load(Ordering::Relaxed) as _)
-    }
-
-    /// Set the threshold for recording the hybrid cache `remove` operation.
-    pub fn set_record_hybrid_remove_threshold(&self, threshold: Duration) {
-        self.record_hybrid_remove_threshold_us
-            .store(threshold.as_micros() as _, Ordering::Relaxed);
+        Duration::from_micros(self.record_hybrid_obtain_threshold_us.load(Ordering::Relaxed))
     }
 
     /// Threshold for recording the hybrid cache `remove` operation.
     pub fn record_hybrid_remove_threshold(&self) -> Duration {
-        Duration::from_micros(self.record_hybrid_remove_threshold_us.load(Ordering::Relaxed) as _)
+        Duration::from_micros(self.record_hybrid_remove_threshold_us.load(Ordering::Relaxed))
     }
 
-    /// Set the threshold for recording the hybrid cache `fetch` operation.
-    pub fn set_record_hybrid_fetch_threshold(&self, threshold: Duration) {
-        self.record_hybrid_fetch_threshold_us
-            .store(threshold.as_micros() as _, Ordering::Relaxed);
+    /// Threshold for recording the hybrid cache `fetch` operation.
+    pub fn record_hybrid_fetch_threshold(&self) -> Duration {
+        Duration::from_micros(self.record_hybrid_fetch_threshold_us.load(Ordering::Relaxed))
     }
+}
 
+/// Options for tracing.
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct TracingOptions {
+    /// Threshold for recording the hybrid cache `insert` and `insert_with_context` operation.
+    record_hybrid_insert_threshold: Option<Duration>,
+    /// Threshold for recording the hybrid cache `get` operation.
+    record_hybrid_get_threshold: Option<Duration>,
+    /// Threshold for recording the hybrid cache `obtain` operation.
+    record_hybrid_obtain_threshold: Option<Duration>,
+    /// Threshold for recording the hybrid cache `remove` operation.
+    record_hybrid_remove_threshold: Option<Duration>,
     /// Threshold for recording the hybrid cache `fetch` operation.
-    pub fn record_hybrid_fetch_threshold(&self) -> Duration {
-        Duration::from_micros(self.record_hybrid_fetch_threshold_us.load(Ordering::Relaxed) as _)
+    record_hybrid_fetch_threshold: Option<Duration>,
+}
+
+impl Default for TracingOptions {
+    fn default() -> Self {
+        Self::new()
+    }
+}
+
+impl TracingOptions {
+    /// Create an empty tracing options.
+    pub fn new() -> Self {
+        Self {
+            record_hybrid_insert_threshold: None,
+            record_hybrid_get_threshold: None,
+            record_hybrid_obtain_threshold: None,
+            record_hybrid_remove_threshold: None,
+            record_hybrid_fetch_threshold: None,
+        }
+    }
+
+    /// Set the threshold for recording the hybrid cache `insert` and `insert_with_context` operation.
+    pub fn with_record_hybrid_insert_threshold(mut self, threshold: Duration) -> Self {
+        self.record_hybrid_insert_threshold = Some(threshold);
+        self
+    }
+
+    /// Set the threshold for recording the hybrid cache `get` operation.
+    pub fn with_record_hybrid_get_threshold(mut self, threshold: Duration) -> Self {
+        self.record_hybrid_get_threshold = Some(threshold);
+        self
+    }
+
+    /// Set the threshold for recording the hybrid cache `obtain` operation.
+    pub fn with_record_hybrid_obtain_threshold(mut self, threshold: Duration) -> Self {
+        self.record_hybrid_obtain_threshold = Some(threshold);
+        self
+    }
+
+    /// Set the threshold for recording the hybrid cache `remove` operation.
+    pub fn with_record_hybrid_remove_threshold(mut self, threshold: Duration) -> Self {
+        self.record_hybrid_remove_threshold = Some(threshold);
+        self
+    }
+
+    /// Set the threshold for recording the hybrid cache `fetch` operation.
+    pub fn with_record_hybrid_fetch_threshold(mut self, threshold: Duration) -> Self {
+        self.record_hybrid_fetch_threshold = Some(threshold);
+        self
     }
 }
 
diff --git a/foyer-storage/src/prelude.rs b/foyer-storage/src/prelude.rs
index 1cd942b4..75954c60 100644
--- a/foyer-storage/src/prelude.rs
+++ b/foyer-storage/src/prelude.rs
@@ -34,7 +34,7 @@ pub use crate::{
     statistics::Statistics,
     storage::{either::Order, Storage},
     store::{
-        DeviceOptions, Engine, LargeEngineOptions, RuntimeConfig, SmallEngineOptions, Store, StoreBuilder,
-        TokioRuntimeConfig,
+        DeviceOptions, Engine, LargeEngineOptions, RuntimeOptions, SmallEngineOptions, Store, StoreBuilder,
+        TokioRuntimeOptions,
     },
 };
diff --git a/foyer-storage/src/store.rs b/foyer-storage/src/store.rs
index 741dd9c6..39428ae2 100644
--- a/foyer-storage/src/store.rs
+++ b/foyer-storage/src/store.rs
@@ -304,7 +304,7 @@ impl FromStr for Engine {
 
 /// Tokio runtime configuration.
 #[derive(Debug, Clone, Default, Serialize, Deserialize)]
-pub struct TokioRuntimeConfig {
+pub struct TokioRuntimeOptions {
     /// Dedicated runtime worker threads.
     ///
     /// If the value is set to `0`, the dedicated will use the default worker threads of tokio.
@@ -321,19 +321,19 @@ pub struct TokioRuntimeConfig {
     pub max_blocking_threads: usize,
 }
 
-/// Configuration for the dedicated runtime.
+/// Options for the dedicated runtime.
 #[derive(Debug, Clone, Serialize, Deserialize)]
-pub enum RuntimeConfig {
+pub enum RuntimeOptions {
     /// Disable dedicated runtime. The runtime which foyer is built on will be used.
     Disabled,
     /// Use unified dedicated runtime for both reads and writes.
-    Unified(TokioRuntimeConfig),
+    Unified(TokioRuntimeOptions),
     /// Use separated dedicated runtime for reads or writes.
     Separated {
         /// Dedicated runtime for reads.
-        read_runtime_config: TokioRuntimeConfig,
+        read_runtime_options: TokioRuntimeOptions,
         /// Dedicated runtime for both foreground and background writes
-        write_runtime_config: TokioRuntimeConfig,
+        write_runtime_options: TokioRuntimeOptions,
     },
 }
 
@@ -349,7 +349,7 @@ where
 
     device_options: DeviceOptions,
     engine: Engine,
-    runtime_config: RuntimeConfig,
+    runtime_config: RuntimeOptions,
 
     admission_picker: Arc<dyn AdmissionPicker<Key = K>>,
     compression: Compression,
@@ -378,7 +378,7 @@ where
 
             device_options: DeviceOptions::None,
             engine,
-            runtime_config: RuntimeConfig::Disabled,
+            runtime_config: RuntimeOptions::Disabled,
 
             admission_picker: Arc::<AdmitAllPicker<K>>::default(),
             compression: Compression::None,
@@ -443,8 +443,8 @@ where
     }
 
     /// Configure the dedicated runtime for the disk cache store.
-    pub fn with_runtime_config(mut self, runtime_config: RuntimeConfig) -> Self {
-        self.runtime_config = runtime_config;
+    pub fn with_runtime_options(mut self, runtime_options: RuntimeOptions) -> Self {
+        self.runtime_config = runtime_options;
         self
     }
 
@@ -480,7 +480,7 @@ where
 
         let compression = self.compression;
 
-        let build_runtime = |config: &TokioRuntimeConfig, suffix: &str| {
+        let build_runtime = |config: &TokioRuntimeOptions, suffix: &str| {
             let mut builder = tokio::runtime::Builder::new_multi_thread();
             #[cfg(not(madsim))]
             if config.worker_threads != 0 {
@@ -498,17 +498,17 @@ where
 
         let user_runtime_handle = Handle::current();
         let (read_runtime, write_runtime) = match self.runtime_config {
-            RuntimeConfig::Disabled => {
+            RuntimeOptions::Disabled => {
                 tracing::warn!("[store]: Dedicated runtime is disabled");
                 (None, None)
             }
-            RuntimeConfig::Unified(runtime_config) => {
+            RuntimeOptions::Unified(runtime_config) => {
                 let runtime = build_runtime(&runtime_config, "unified")?;
                 (Some(runtime.clone()), Some(runtime.clone()))
             }
-            RuntimeConfig::Separated {
-                read_runtime_config,
-                write_runtime_config,
+            RuntimeOptions::Separated {
+                read_runtime_options: read_runtime_config,
+                write_runtime_options: write_runtime_config,
             } => {
                 let read_runtime = build_runtime(&read_runtime_config, "read")?;
                 let write_runtime = build_runtime(&write_runtime_config, "write")?;
diff --git a/foyer/src/hybrid/builder.rs b/foyer/src/hybrid/builder.rs
index f5c3683e..826c091a 100644
--- a/foyer/src/hybrid/builder.rs
+++ b/foyer/src/hybrid/builder.rs
@@ -18,11 +18,11 @@ use ahash::RandomState;
 use foyer_common::{
     code::{HashBuilder, StorageKey, StorageValue},
     event::EventListener,
-    tracing::TracingConfig,
+    tracing::TracingOptions,
 };
 use foyer_memory::{Cache, CacheBuilder, EvictionConfig, Weighter};
 use foyer_storage::{
-    AdmissionPicker, Compression, DeviceOptions, Engine, LargeEngineOptions, RecoverMode, RuntimeConfig,
+    AdmissionPicker, Compression, DeviceOptions, Engine, LargeEngineOptions, RecoverMode, RuntimeOptions,
     SmallEngineOptions, StoreBuilder,
 };
 
@@ -32,7 +32,7 @@ use crate::HybridCache;
 pub struct HybridCacheBuilder<K, V> {
     name: String,
     event_listener: Option<Arc<dyn EventListener<Key = K, Value = V>>>,
-    tracing_config: TracingConfig,
+    tracing_options: TracingOptions,
 }
 
 impl<K, V> Default for HybridCacheBuilder<K, V> {
@@ -47,7 +47,7 @@ impl<K, V> HybridCacheBuilder<K, V> {
         Self {
             name: "foyer".to_string(),
             event_listener: None,
-            tracing_config: TracingConfig::default(),
+            tracing_options: TracingOptions::default(),
         }
     }
 
@@ -69,11 +69,11 @@ impl<K, V> HybridCacheBuilder<K, V> {
         self
     }
 
-    /// Set tracing config.
+    /// Set tracing options.
     ///
     /// Default: Only operations over 1s will be recorded.
-    pub fn with_tracing_config(mut self, tracing_config: TracingConfig) -> Self {
-        self.tracing_config = tracing_config;
+    pub fn with_tracing_options(mut self, tracing_options: TracingOptions) -> Self {
+        self.tracing_options = tracing_options;
         self
     }
 
@@ -90,7 +90,7 @@ impl<K, V> HybridCacheBuilder<K, V> {
         HybridCacheBuilderPhaseMemory {
             builder,
             name: self.name,
-            tracing_config: self.tracing_config,
+            tracing_options: self.tracing_options,
         }
     }
 }
@@ -103,7 +103,7 @@ where
     S: HashBuilder + Debug,
 {
     name: String,
-    tracing_config: TracingConfig,
+    tracing_options: TracingOptions,
     builder: CacheBuilder<K, V, S>,
 }
 
@@ -119,7 +119,7 @@ where
         let builder = self.builder.with_shards(shards);
         HybridCacheBuilderPhaseMemory {
             name: self.name,
-            tracing_config: self.tracing_config,
+            tracing_options: self.tracing_options,
             builder,
         }
     }
@@ -131,7 +131,7 @@ where
         let builder = self.builder.with_eviction_config(eviction_config.into());
         HybridCacheBuilderPhaseMemory {
             name: self.name,
-            tracing_config: self.tracing_config,
+            tracing_options: self.tracing_options,
             builder,
         }
     }
@@ -145,7 +145,7 @@ where
         let builder = self.builder.with_object_pool_capacity(object_pool_capacity);
         HybridCacheBuilderPhaseMemory {
             name: self.name,
-            tracing_config: self.tracing_config,
+            tracing_options: self.tracing_options,
             builder,
         }
     }
@@ -158,7 +158,7 @@ where
         let builder = self.builder.with_hash_builder(hash_builder);
         HybridCacheBuilderPhaseMemory {
             name: self.name,
-            tracing_config: self.tracing_config,
+            tracing_options: self.tracing_options,
             builder,
         }
     }
@@ -168,7 +168,7 @@ where
         let builder = self.builder.with_weighter(weighter);
         HybridCacheBuilderPhaseMemory {
             name: self.name,
-            tracing_config: self.tracing_config,
+            tracing_options: self.tracing_options,
             builder,
         }
     }
@@ -179,7 +179,7 @@ where
         HybridCacheBuilderPhaseStorage {
             builder: StoreBuilder::new(memory.clone(), engine).with_name(&self.name),
             name: self.name,
-            tracing_config: self.tracing_config,
+            tracing_options: self.tracing_options,
             memory,
         }
     }
@@ -193,7 +193,7 @@ where
     S: HashBuilder + Debug,
 {
     name: String,
-    tracing_config: TracingConfig,
+    tracing_options: TracingOptions,
     memory: Cache<K, V, S>,
     builder: StoreBuilder<K, V, S>,
 }
@@ -209,7 +209,7 @@ where
         let builder = self.builder.with_device_options(device_options);
         Self {
             name: self.name,
-            tracing_config: self.tracing_config,
+            tracing_options: self.tracing_options,
             memory: self.memory,
             builder,
         }
@@ -222,7 +222,7 @@ where
         let builder = self.builder.with_flush(flush);
         Self {
             name: self.name,
-            tracing_config: self.tracing_config,
+            tracing_options: self.tracing_options,
             memory: self.memory,
             builder,
         }
@@ -237,7 +237,7 @@ where
         let builder = self.builder.with_recover_mode(recover_mode);
         Self {
             name: self.name,
-            tracing_config: self.tracing_config,
+            tracing_options: self.tracing_options,
             memory: self.memory,
             builder,
         }
@@ -252,7 +252,7 @@ where
         let builder = self.builder.with_admission_picker(admission_picker);
         Self {
             name: self.name,
-            tracing_config: self.tracing_config,
+            tracing_options: self.tracing_options,
             memory: self.memory,
             builder,
         }
@@ -265,18 +265,18 @@ where
         let builder = self.builder.with_compression(compression);
         Self {
             name: self.name,
-            tracing_config: self.tracing_config,
+            tracing_options: self.tracing_options,
             memory: self.memory,
             builder,
         }
     }
 
     /// Configure the dedicated runtime for the disk cache store.
-    pub fn with_runtime_config(self, runtime_config: RuntimeConfig) -> Self {
-        let builder = self.builder.with_runtime_config(runtime_config);
+    pub fn with_runtime_options(self, runtime_options: RuntimeOptions) -> Self {
+        let builder = self.builder.with_runtime_options(runtime_options);
         Self {
             name: self.name,
-            tracing_config: self.tracing_config,
+            tracing_options: self.tracing_options,
             memory: self.memory,
             builder,
         }
@@ -289,7 +289,7 @@ where
         let builder = self.builder.with_large_object_disk_cache_options(options);
         Self {
             name: self.name,
-            tracing_config: self.tracing_config,
+            tracing_options: self.tracing_options,
             memory: self.memory,
             builder,
         }
@@ -302,7 +302,7 @@ where
         let builder = self.builder.with_small_object_disk_cache_options(options);
         Self {
             name: self.name,
-            tracing_config: self.tracing_config,
+            tracing_options: self.tracing_options,
             memory: self.memory,
             builder,
         }
@@ -311,6 +311,6 @@ where
     /// Build and open the hybrid cache with the given configurations.
     pub async fn build(self) -> anyhow::Result<HybridCache<K, V, S>> {
         let storage = self.builder.build().await?;
-        Ok(HybridCache::new(self.name, self.memory, storage, self.tracing_config))
+        Ok(HybridCache::new(self.name, self.memory, storage, self.tracing_options))
     }
 }
diff --git a/foyer/src/hybrid/cache.rs b/foyer/src/hybrid/cache.rs
index 5a728018..6c57a0ef 100644
--- a/foyer/src/hybrid/cache.rs
+++ b/foyer/src/hybrid/cache.rs
@@ -33,7 +33,7 @@ use foyer_common::{
     code::{HashBuilder, StorageKey, StorageValue},
     future::Diversion,
     metrics::Metrics,
-    tracing::{InRootSpan, TracingConfig},
+    tracing::{InRootSpan, TracingConfig, TracingOptions},
 };
 use foyer_memory::{Cache, CacheContext, CacheEntry, Fetch, FetchMark, FetchState};
 use foyer_storage::{DeviceStats, Store};
@@ -130,23 +130,24 @@ where
         name: String,
         memory: Cache<K, V, S>,
         storage: Store<K, V, S>,
-        tracing_config: TracingConfig,
+        tracing_options: TracingOptions,
     ) -> Self {
         let metrics = Arc::new(Metrics::new(&name));
-        let tracing_config = Arc::new(tracing_config);
-        let trace = Arc::new(AtomicBool::new(false));
+        let tracing_config = Arc::<TracingConfig>::default();
+        tracing_config.update(tracing_options);
+        let tracing = Arc::new(AtomicBool::new(false));
         Self {
             memory,
             storage,
             metrics,
             tracing_config,
-            tracing: trace,
+            tracing,
         }
     }
 
-    /// Access the trace config.
-    pub fn tracing_config(&self) -> &TracingConfig {
-        &self.tracing_config
+    /// Access the trace config with options.
+    pub fn update_tracing_options(&self, options: TracingOptions) {
+        self.tracing_config.update(options);
     }
 
     /// Access the in-memory cache.
diff --git a/foyer/src/prelude.rs b/foyer/src/prelude.rs
index 937c1f5f..9ca7aec4 100644
--- a/foyer/src/prelude.rs
+++ b/foyer/src/prelude.rs
@@ -12,27 +12,28 @@
 //  See the License for the specific language governing permissions and
 //  limitations under the License.
 
-pub use common::{
-    buf::{BufExt, BufMutExt},
-    code::{Key, StorageKey, StorageValue, Value},
-    event::EventListener,
-    range::RangeBoundsExt,
-    tracing::TracingConfig,
+pub use crate::{
+    common::{
+        buf::{BufExt, BufMutExt},
+        code::{Key, StorageKey, StorageValue, Value},
+        event::EventListener,
+        range::RangeBoundsExt,
+        tracing::TracingOptions,
+    },
+    hybrid::{
+        builder::{HybridCacheBuilder, HybridCacheBuilderPhaseMemory, HybridCacheBuilderPhaseStorage},
+        cache::{HybridCache, HybridCacheEntry, HybridFetch, HybridFetchInner},
+        writer::{HybridCacheStorageWriter, HybridCacheWriter},
+    },
+    memory::{
+        Cache, CacheBuilder, CacheContext, CacheEntry, EvictionConfig, FetchState, FifoConfig, LfuConfig, LruConfig,
+        S3FifoConfig, Weighter,
+    },
+    storage::{
+        AdmissionPicker, AdmitAllPicker, Compression, Dev, DevConfig, DevExt, DeviceStats, DirectFileDevice,
+        DirectFileDeviceOptions, DirectFsDevice, DirectFsDeviceOptions, Engine, EvictionPicker, FifoPicker,
+        InvalidRatioPicker, LargeEngineOptions, RateLimitPicker, RecoverMode, ReinsertionPicker, RejectAllPicker,
+        Runtime, RuntimeOptions, SmallEngineOptions, Storage, Store, StoreBuilder, TokioRuntimeOptions,
+        TombstoneLogConfigBuilder,
+    },
 };
-pub use memory::{
-    Cache, CacheBuilder, CacheContext, CacheEntry, EvictionConfig, FetchState, FifoConfig, LfuConfig, LruConfig,
-    S3FifoConfig, Weighter,
-};
-pub use storage::{
-    AdmissionPicker, AdmitAllPicker, Compression, Dev, DevConfig, DevExt, DeviceStats, DirectFileDevice,
-    DirectFileDeviceOptions, DirectFsDevice, DirectFsDeviceOptions, Engine, EvictionPicker, FifoPicker,
-    InvalidRatioPicker, LargeEngineOptions, RateLimitPicker, RecoverMode, ReinsertionPicker, RejectAllPicker, Runtime,
-    RuntimeConfig, SmallEngineOptions, Storage, Store, StoreBuilder, TokioRuntimeConfig, TombstoneLogConfigBuilder,
-};
-
-pub use crate::hybrid::{
-    builder::{HybridCacheBuilder, HybridCacheBuilderPhaseMemory, HybridCacheBuilderPhaseStorage},
-    cache::{HybridCache, HybridCacheEntry, HybridFetch, HybridFetchInner},
-    writer::{HybridCacheStorageWriter, HybridCacheWriter},
-};
-use crate::{common, memory, storage};

From cf53a604c7c97fb3cbdd311edc2794d0738bdc96 Mon Sep 17 00:00:00 2001
From: Croxx <mrcroxx@outlook.com>
Date: Wed, 9 Oct 2024 14:11:58 +0800
Subject: [PATCH 27/53] chore: release 0.12.0 (#755)

* chore: release 0.12.0

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

* doc: update change log

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

* doc: update examples in readme

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

* chore: update changelog

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

* chore: update changelog

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

---------

Signed-off-by: MrCroxx <mrcroxx@outlook.com>
---
 CHANGELOG.md | 23 ++++++++++++++++++
 Cargo.toml   | 12 +++++-----
 README.md    | 68 +++++++++++++++++++++++++++-------------------------
 3 files changed, 65 insertions(+), 38 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 8b2741ad..51c3d447 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -9,6 +9,29 @@ date: 2023-05-12T11:02:09+08:00
 
 <!-- truncate -->
 
+## 2024-10-09
+
+### Releases
+
+| crate | version |
+| - | - |
+| foyer | 0.12.0 |
+| foyer-common | 0.12.0 |
+| foyer-intrusive | 0.12.0 |
+| foyer-memory | 0.12.0 |
+| foyer-storage | 0.12.0 |
+| foyer-bench | 0.12.0 |
+
+### Changes
+
+- Align the versions of all components to the same. 📣
+- Introduce small object disk cache. 🎉
+- Introduce mixed/large/small storage engine.
+- Refactor builders for the hybrid cache.
+- Introduce submit queue size threshold to prevent from channel piling up.
+- Support `jeprof` for foyer-bench.
+- Rename feature "mtrace" to "tracing".
+
 ## 2024-09-25
 
 ### Releases
diff --git a/Cargo.toml b/Cargo.toml
index 90be98e0..cddc89a6 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -13,7 +13,7 @@ members = [
 ]
 
 [workspace.package]
-version = "0.12.0-dev"
+version = "0.12.0"
 edition = "2021"
 rust-version = "1.81.0"
 repository = "https://github.com/foyer-rs/foyer"
@@ -47,11 +47,11 @@ clap = { version = "4", features = ["derive"] }
 bytesize = { package = "foyer-bytesize", version = "2" }
 hashbrown = "0.15"
 # foyer components
-foyer-common = { version = "0.12.0-dev", path = "foyer-common" }
-foyer-intrusive = { version = "0.12.0-dev", path = "foyer-intrusive" }
-foyer-memory = { version = "0.12.0-dev", path = "foyer-memory" }
-foyer-storage = { version = "0.12.0-dev", path = "foyer-storage" }
-foyer = { version = "0.12.0-dev", path = "foyer" }
+foyer-common = { version = "0.12.0", path = "foyer-common" }
+foyer-intrusive = { version = "0.12.0", path = "foyer-intrusive" }
+foyer-memory = { version = "0.12.0", path = "foyer-memory" }
+foyer-storage = { version = "0.12.0", path = "foyer-storage" }
+foyer = { version = "0.12.0", path = "foyer" }
 
 [workspace.lints.rust]
 unexpected_cfgs = { level = "warn", check-cfg = ['cfg(madsim)'] }
diff --git a/README.md b/README.md
index eb639ae5..5d97d0ed 100644
--- a/README.md
+++ b/README.md
@@ -63,13 +63,13 @@ Feel free to open a PR and add your projects here:
 To use *foyer* in your project, add this line to the `dependencies` section of `Cargo.toml`.
 
 ```toml
-foyer = "0.11"
+foyer = "0.12"
 ```
 
 If your project is using the nightly rust toolchain, the `nightly` feature needs to be enabled.
 
 ```toml
-foyer = { version = "0.11", features = ["nightly"] }
+foyer = { version = "0.12", features = ["nightly"] }
 ```
 
 ### Out-of-the-box In-memory Cache
@@ -90,7 +90,7 @@ fn main() {
 ### Easy-to-use Hybrid Cache
 
 ```rust
-use foyer::{DirectFsDeviceOptionsBuilder, HybridCache, HybridCacheBuilder};
+use foyer::{DirectFsDeviceOptions, Engine, HybridCache, HybridCacheBuilder};
 
 #[tokio::main]
 async fn main() -> anyhow::Result<()> {
@@ -98,12 +98,8 @@ async fn main() -> anyhow::Result<()> {
 
     let hybrid: HybridCache<u64, String> = HybridCacheBuilder::new()
         .memory(64 * 1024 * 1024)
-        .storage()
-        .with_device_config(
-            DirectFsDeviceOptionsBuilder::new(dir.path())
-                .with_capacity(256 * 1024 * 1024)
-                .build(),
-        )
+        .storage(Engine::Large) // use large object disk cache engine only
+        .with_device_options(DirectFsDeviceOptions::new(dir.path()).with_capacity(256 * 1024 * 1024))
         .build()
         .await?;
 
@@ -125,8 +121,8 @@ use std::sync::Arc;
 use anyhow::Result;
 use chrono::Datelike;
 use foyer::{
-    DirectFsDeviceOptionsBuilder, FifoPicker, HybridCache, HybridCacheBuilder, LruConfig, RateLimitPicker, RecoverMode,
-    RuntimeConfig, TokioRuntimeConfig, TombstoneLogConfigBuilder,
+    DirectFsDeviceOptions, Engine, FifoPicker, HybridCache, HybridCacheBuilder, LargeEngineOptions, LruConfig,
+    RateLimitPicker, RecoverMode, RuntimeOptions, SmallEngineOptions, TokioRuntimeOptions, TombstoneLogConfigBuilder,
 };
 use tempfile::tempdir;
 
@@ -143,40 +139,48 @@ async fn main() -> Result<()> {
         .with_object_pool_capacity(1024)
         .with_hash_builder(ahash::RandomState::default())
         .with_weighter(|_key, value: &String| value.len())
-        .storage()
-        .with_device_config(
-            DirectFsDeviceOptionsBuilder::new(dir.path())
+        .storage(Engine::Mixed(0.1))
+        .with_device_options(
+            DirectFsDeviceOptions::new(dir.path())
                 .with_capacity(64 * 1024 * 1024)
-                .with_file_size(4 * 1024 * 1024)
-                .build(),
+                .with_file_size(4 * 1024 * 1024),
         )
         .with_flush(true)
-        .with_indexer_shards(64)
         .with_recover_mode(RecoverMode::Quiet)
-        .with_recover_concurrency(8)
-        .with_flushers(2)
-        .with_reclaimers(2)
-        .with_buffer_threshold(256 * 1024 * 1024)
-        .with_clean_region_threshold(4)
-        .with_eviction_pickers(vec![Box::<FifoPicker>::default()])
         .with_admission_picker(Arc::new(RateLimitPicker::new(100 * 1024 * 1024)))
-        .with_reinsertion_picker(Arc::new(RateLimitPicker::new(10 * 1024 * 1024)))
         .with_compression(foyer::Compression::Lz4)
-        .with_tombstone_log_config(
-            TombstoneLogConfigBuilder::new(dir.path().join("tombstone-log-file"))
-                .with_flush(true)
-                .build(),
-        )
-        .with_runtime_config(RuntimeConfig::Separated {
-            read_runtime_config: TokioRuntimeConfig {
+        .with_runtime_options(RuntimeOptions::Separated {
+            read_runtime_options: TokioRuntimeOptions {
                 worker_threads: 4,
                 max_blocking_threads: 8,
             },
-            write_runtime_config: TokioRuntimeConfig {
+            write_runtime_options: TokioRuntimeOptions {
                 worker_threads: 4,
                 max_blocking_threads: 8,
             },
         })
+        .with_large_object_disk_cache_options(
+            LargeEngineOptions::new()
+                .with_indexer_shards(64)
+                .with_recover_concurrency(8)
+                .with_flushers(2)
+                .with_reclaimers(2)
+                .with_buffer_pool_size(256 * 1024 * 1024)
+                .with_clean_region_threshold(4)
+                .with_eviction_pickers(vec![Box::<FifoPicker>::default()])
+                .with_reinsertion_picker(Arc::new(RateLimitPicker::new(10 * 1024 * 1024)))
+                .with_tombstone_log_config(
+                    TombstoneLogConfigBuilder::new(dir.path().join("tombstone-log-file"))
+                        .with_flush(true)
+                        .build(),
+                ),
+        )
+        .with_small_object_disk_cache_options(
+            SmallEngineOptions::new()
+                .with_set_size(16 * 1024)
+                .with_set_cache_capacity(64)
+                .with_flushers(2),
+        )
         .build()
         .await?;
 

From d2245e34ed9b8517e375f564224aa778f38acaca Mon Sep 17 00:00:00 2001
From: Croxx <mrcroxx@outlook.com>
Date: Wed, 9 Oct 2024 17:57:40 +0800
Subject: [PATCH 28/53] chore: downgrade hashbrown to 0.14 (#758)

Signed-off-by: MrCroxx <mrcroxx@outlook.com>
---
 Cargo.toml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Cargo.toml b/Cargo.toml
index cddc89a6..64ae2d8b 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -45,7 +45,7 @@ fastrace-jaeger = "0.7"
 fastrace-opentelemetry = "0.7"
 clap = { version = "4", features = ["derive"] }
 bytesize = { package = "foyer-bytesize", version = "2" }
-hashbrown = "0.15"
+hashbrown = "0.14"
 # foyer components
 foyer-common = { version = "0.12.0", path = "foyer-common" }
 foyer-intrusive = { version = "0.12.0", path = "foyer-intrusive" }

From 0f87d64f2ae0bbfe67d2e79e4dc917fc9484aaba Mon Sep 17 00:00:00 2001
From: Croxx <mrcroxx@outlook.com>
Date: Wed, 9 Oct 2024 22:48:45 +0800
Subject: [PATCH 29/53] fix: fix build with madsim (#762)

* fix: fix build with madsim

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

* test: update ci and makefile

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

---------

Signed-off-by: MrCroxx <mrcroxx@outlook.com>
---
 .github/workflows/ci.yml    | 80 ++++++++++++++++++-------------------
 Makefile                    |  3 +-
 foyer-common/src/runtime.rs |  6 +++
 3 files changed, 47 insertions(+), 42 deletions(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index e93a9ce8..863ad4f1 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -345,43 +345,43 @@ jobs:
         with:
           name: artifacts.lsan.tgz
           path: artifacts.lsan.tgz
-  # deterministic-test:
-  #   name: run deterministic test
-  #   runs-on: ubuntu-latest
-  #   steps:
-  #     - name: Checkout
-  #       uses: actions/checkout@v4
-  #     - name: Install rust toolchain
-  #       uses: dtolnay/rust-toolchain@master
-  #       with:
-  #         toolchain: ${{ env.RUST_TOOLCHAIN }}
-  #         components: rustfmt, clippy
-  #     - name: Cache Cargo home
-  #       uses: actions/cache@v4
-  #       id: cache
-  #       with:
-  #         path: |
-  #           ~/.cargo/bin/
-  #           ~/.cargo/registry/index/
-  #           ~/.cargo/registry/cache/
-  #           ~/.cargo/git/db/
-  #         key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.toml') }}-${{ env.CACHE_KEY_SUFFIX }}-deterministic-test
-  #     - if: steps.cache.outputs.cache-hit != 'true'
-  #       uses: taiki-e/install-action@nextest
-  #     - name: Run rust clippy check (madsim)
-  #       env:
-  #         RUST_BACKTRACE: 1
-  #         RUSTFLAGS: "--cfg tokio_unstable --cfg madsim"
-  #         RUST_LOG: info
-  #         TOKIO_WORKER_THREADS: 1
-  #         CI: true
-  #       run: |-
-  #         cargo clippy --all-targets
-  #     - name: Run nextest (madsim)
-  #       env:
-  #         RUST_BACKTRACE: 1
-  #         RUSTFLAGS: "--cfg tokio_unstable --cfg madsim"
-  #         RUST_LOG: info
-  #         TOKIO_WORKER_THREADS: 1
-  #       run: |-
-  #         cargo nextest run --all
+  madsim:
+    name: check build with madsim
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+      - name: Install rust toolchain
+        uses: dtolnay/rust-toolchain@master
+        with:
+          toolchain: ${{ env.RUST_TOOLCHAIN_NIGHTLY }}
+          components: rustfmt, clippy
+      - name: Cache Cargo home
+        uses: actions/cache@v4
+        id: cache
+        with:
+          path: |
+            ~/.cargo/bin/
+            ~/.cargo/registry/index/
+            ~/.cargo/registry/cache/
+            ~/.cargo/git/db/
+          key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.toml') }}-${{ env.CACHE_KEY_SUFFIX }}-madsim
+      - if: steps.cache.outputs.cache-hit != 'true'
+        uses: taiki-e/install-action@nextest
+      - name: Run rust clippy check (madsim, check only)
+        env:
+          RUST_BACKTRACE: 1
+          RUSTFLAGS: "--cfg tokio_unstable --cfg madsim"
+          RUST_LOG: info
+          TOKIO_WORKER_THREADS: 1
+          CI: true
+        run: |-
+          cargo clippy --all-targets
+      - name: Run nextest (madsim)
+        env:
+          RUST_BACKTRACE: 1
+          RUSTFLAGS: "--cfg tokio_unstable --cfg madsim"
+          RUST_LOG: info
+          TOKIO_WORKER_THREADS: 1
+        run: |-
+          cargo nextest run --all --features "strict_assertions,sanity"
diff --git a/Makefile b/Makefile
index d536f2ec..c40e0b3a 100644
--- a/Makefile
+++ b/Makefile
@@ -34,8 +34,7 @@ test-all: test test-ignored
 
 madsim:
 	RUSTFLAGS="--cfg madsim --cfg tokio_unstable" cargo clippy --all-targets
-	RUSTFLAGS="--cfg madsim --cfg tokio_unstable" RUST_BACKTRACE=1 cargo nextest run --all
-	RUSTFLAGS="--cfg madsim --cfg tokio_unstable" RUST_BACKTRACE=1 cargo test --doc
+	RUSTFLAGS="--cfg madsim --cfg tokio_unstable" RUST_BACKTRACE=1 cargo nextest run --all --features "strict_assertions,sanity"
 
 example:
 	cargo run --example memory
diff --git a/foyer-common/src/runtime.rs b/foyer-common/src/runtime.rs
index 1a149741..62806a2a 100644
--- a/foyer-common/src/runtime.rs
+++ b/foyer-common/src/runtime.rs
@@ -214,7 +214,13 @@ impl SingletonHandle {
     /// [`tokio::fs`]: crate::fs
     /// [`tokio::net`]: crate::net
     /// [`tokio::time`]: crate::time
+    #[cfg(not(madsim))]
     pub fn block_on<F: Future>(&self, future: F) -> F::Output {
         self.0.block_on(future)
     }
+
+    #[cfg(madsim)]
+    pub fn block_on<F: Future>(&self, future: F) -> F::Output {
+        unimplemented!("`block_on()` is not supported with madsim")
+    }
 }

From b414bc55e08565b7607706d10d076e1fcb9b023e Mon Sep 17 00:00:00 2001
From: Croxx <mrcroxx@outlook.com>
Date: Thu, 10 Oct 2024 14:15:06 +0800
Subject: [PATCH 30/53] refactor: refine small object disk cache (#761)

* refactor: refine set cache

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

* refactor: refine small object disk cache

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

* chore: pass ffmt

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

---------

Signed-off-by: MrCroxx <mrcroxx@outlook.com>
---
 foyer-storage/src/small/flusher.rs     |   4 +-
 foyer-storage/src/small/generic.rs     |  33 +---
 foyer-storage/src/small/mod.rs         |   1 +
 foyer-storage/src/small/set.rs         |  51 +----
 foyer-storage/src/small/set_cache.rs   |  66 +++++++
 foyer-storage/src/small/set_manager.rs | 258 ++++++++++---------------
 foyer-storage/src/store.rs             |  12 ++
 7 files changed, 196 insertions(+), 229 deletions(-)
 create mode 100644 foyer-storage/src/small/set_cache.rs

diff --git a/foyer-storage/src/small/flusher.rs b/foyer-storage/src/small/flusher.rs
index f95a440e..902c1ee4 100644
--- a/foyer-storage/src/small/flusher.rs
+++ b/foyer-storage/src/small/flusher.rs
@@ -200,9 +200,7 @@ where
             let set_manager = self.set_manager.clone();
             let stats = self.stats.clone();
             async move {
-                let mut set = set_manager.write(sid).await?;
-                set.apply(&deletions, items);
-                set_manager.apply(set).await?;
+                set_manager.update(sid, &deletions, items).await?;
 
                 stats
                     .cache_write_bytes
diff --git a/foyer-storage/src/small/generic.rs b/foyer-storage/src/small/generic.rs
index 07fa414f..2f94e6b8 100644
--- a/foyer-storage/src/small/generic.rs
+++ b/foyer-storage/src/small/generic.rs
@@ -46,6 +46,7 @@ where
 {
     pub set_size: usize,
     pub set_cache_capacity: usize,
+    pub set_cache_shards: usize,
     pub device: MonitoredDevice,
     pub regions: Range<RegionId>,
     pub flush: bool,
@@ -66,6 +67,7 @@ where
         f.debug_struct("GenericSmallStorageConfig")
             .field("set_size", &self.set_size)
             .field("set_cache_capacity", &self.set_cache_capacity)
+            .field("set_cache_shards", &self.set_cache_shards)
             .field("device", &self.device)
             .field("regions", &self.regions)
             .field("flush", &self.flush)
@@ -92,7 +94,7 @@ where
     active: AtomicBool,
 
     stats: Arc<Statistics>,
-    runtime: Runtime,
+    _runtime: Runtime,
 }
 
 pub struct GenericSmallStorage<K, V, S>
@@ -144,14 +146,7 @@ where
             config.regions
         );
 
-        let set_manager = SetManager::open(
-            config.set_size,
-            config.set_cache_capacity,
-            config.device.clone(),
-            config.regions.clone(),
-            config.flush,
-        )
-        .await?;
+        let set_manager = SetManager::open(&config).await?;
 
         let flushers = (0..config.flushers)
             .map(|_| Flusher::open(&config, set_manager.clone(), stats.clone(), metrics.clone()))
@@ -163,7 +158,7 @@ where
             set_manager,
             active: AtomicBool::new(true),
             stats,
-            runtime: config.runtime,
+            _runtime: config.runtime,
         };
         let inner = Arc::new(inner);
 
@@ -196,7 +191,6 @@ where
 
     fn load(&self, hash: u64) -> impl Future<Output = Result<Option<(K, V)>>> + Send + 'static {
         let set_manager = self.inner.set_manager.clone();
-        let sid = set_manager.set_picker().sid(hash);
         let stats = self.inner.stats.clone();
 
         async move {
@@ -204,13 +198,7 @@ where
                 .cache_read_bytes
                 .fetch_add(set_manager.set_size(), Ordering::Relaxed);
 
-            match set_manager.read(sid, hash).await? {
-                Some(set) => {
-                    let kv = set.get(hash)?;
-                    Ok(kv)
-                }
-                None => Ok(None),
-            }
+            set_manager.load(hash).await
         }
     }
 
@@ -231,13 +219,7 @@ where
     }
 
     fn may_contains(&self, hash: u64) -> bool {
-        let set_manager = self.inner.set_manager.clone();
-        let sid = set_manager.set_picker().sid(hash);
-        // FIXME: Anyway without blocking? Use atomic?
-        self.inner
-            .runtime
-            .read()
-            .block_on(async move { set_manager.contains(sid, hash).await })
+        self.inner.set_manager.may_contains(hash)
     }
 
     fn stats(&self) -> Arc<DeviceStats> {
@@ -342,6 +324,7 @@ mod tests {
         let config = GenericSmallStorageConfig {
             set_size: ByteSize::kib(4).as_u64() as _,
             set_cache_capacity: 4,
+            set_cache_shards: 1,
             device,
             regions,
             flush: false,
diff --git a/foyer-storage/src/small/mod.rs b/foyer-storage/src/small/mod.rs
index 96921064..02862fd0 100644
--- a/foyer-storage/src/small/mod.rs
+++ b/foyer-storage/src/small/mod.rs
@@ -18,4 +18,5 @@ pub mod flusher;
 pub mod generic;
 pub mod serde;
 pub mod set;
+pub mod set_cache;
 pub mod set_manager;
diff --git a/foyer-storage/src/small/set.rs b/foyer-storage/src/small/set.rs
index 2af18480..aa24fc0a 100644
--- a/foyer-storage/src/small/set.rs
+++ b/foyer-storage/src/small/set.rs
@@ -15,8 +15,7 @@
 use std::{
     collections::HashSet,
     fmt::Debug,
-    ops::{Deref, DerefMut, Range},
-    sync::Arc,
+    ops::Range,
     time::{SystemTime, UNIX_EPOCH},
 };
 
@@ -32,54 +31,6 @@ use crate::{
 
 pub type SetId = u64;
 
-#[derive(Debug)]
-pub struct Set {
-    storage: Arc<SetStorage>,
-}
-
-impl Deref for Set {
-    type Target = SetStorage;
-
-    fn deref(&self) -> &Self::Target {
-        &self.storage
-    }
-}
-
-impl Set {
-    pub fn new(storage: Arc<SetStorage>) -> Self {
-        Self { storage }
-    }
-}
-
-#[derive(Debug)]
-pub struct SetMut {
-    storage: SetStorage,
-}
-
-impl Deref for SetMut {
-    type Target = SetStorage;
-
-    fn deref(&self) -> &Self::Target {
-        &self.storage
-    }
-}
-
-impl DerefMut for SetMut {
-    fn deref_mut(&mut self) -> &mut Self::Target {
-        &mut self.storage
-    }
-}
-
-impl SetMut {
-    pub fn new(storage: SetStorage) -> Self {
-        Self { storage }
-    }
-
-    pub fn into_storage(self) -> SetStorage {
-        self.storage
-    }
-}
-
 /// # Format
 ///
 /// ```plain
diff --git a/foyer-storage/src/small/set_cache.rs b/foyer-storage/src/small/set_cache.rs
new file mode 100644
index 00000000..89769f00
--- /dev/null
+++ b/foyer-storage/src/small/set_cache.rs
@@ -0,0 +1,66 @@
+//  Copyright 2024 foyer Project Authors
+//
+//  Licensed under the Apache License, Version 2.0 (the "License");
+//  you may not use this file except in compliance with the License.
+//  You may obtain a copy of the License at
+//
+//  http://www.apache.org/licenses/LICENSE-2.0
+//
+//  Unless required by applicable law or agreed to in writing, software
+//  distributed under the License is distributed on an "AS IS" BASIS,
+//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//  See the License for the specific language governing permissions and
+//  limitations under the License.
+
+use itertools::Itertools;
+use ordered_hash_map::OrderedHashMap;
+use parking_lot::{MappedRwLockReadGuard, RwLock, RwLockReadGuard};
+
+use super::set::{SetId, SetStorage};
+
+/// In-memory set cache to reduce disk io.
+///
+/// Simple FIFO cache.
+#[derive(Debug)]
+pub struct SetCache {
+    shards: Vec<RwLock<OrderedHashMap<SetId, SetStorage>>>,
+    shard_capacity: usize,
+}
+
+impl SetCache {
+    pub fn new(capacity: usize, shards: usize) -> Self {
+        let shard_capacity = capacity / shards;
+        let shards = (0..shards)
+            .map(|_| RwLock::new(OrderedHashMap::with_capacity(shard_capacity)))
+            .collect_vec();
+        Self { shards, shard_capacity }
+    }
+
+    pub fn insert(&self, id: SetId, storage: SetStorage) {
+        let mut shard = self.shards[self.shard(&id)].write();
+        if shard.len() == self.shard_capacity {
+            shard.pop_front();
+        }
+
+        assert!(shard.len() < self.shard_capacity);
+
+        shard.insert(id, storage);
+    }
+
+    pub fn invalid(&self, id: &SetId) {
+        let mut shard = self.shards[self.shard(id)].write();
+        shard.remove(id);
+    }
+
+    pub fn lookup(&self, id: &SetId) -> Option<MappedRwLockReadGuard<'_, SetStorage>> {
+        RwLockReadGuard::try_map(self.shards[self.shard(id)].read(), |shard| shard.get(id)).ok()
+    }
+
+    pub fn clear(&self) {
+        self.shards.iter().for_each(|shard| shard.write().clear());
+    }
+
+    fn shard(&self, id: &SetId) -> usize {
+        *id as usize % self.shards.len()
+    }
+}
diff --git a/foyer-storage/src/small/set_manager.rs b/foyer-storage/src/small/set_manager.rs
index 9a1974cc..46d366b3 100644
--- a/foyer-storage/src/small/set_manager.rs
+++ b/foyer-storage/src/small/set_manager.rs
@@ -12,21 +12,20 @@
 //  See the License for the specific language governing permissions and
 //  limitations under the License.
 
-use std::{
-    fmt::Debug,
-    ops::{Deref, DerefMut, Range},
-    sync::Arc,
-};
+use std::{collections::HashSet, fmt::Debug, ops::Range, sync::Arc};
 
 use bytes::{Buf, BufMut};
-use foyer_common::strict_assert;
+use foyer_common::code::{HashBuilder, StorageKey, StorageValue};
 use itertools::Itertools;
-use ordered_hash_map::OrderedHashMap;
-use tokio::sync::{Mutex, RwLock, RwLockReadGuard, RwLockWriteGuard};
+use parking_lot::RwLock;
+use tokio::sync::RwLock as AsyncRwLock;
 
 use super::{
+    batch::Item,
     bloom_filter::BloomFilterU64,
-    set::{Set, SetId, SetMut, SetStorage, SetTimestamp},
+    generic::GenericSmallStorageConfig,
+    set::{SetId, SetStorage, SetTimestamp},
+    set_cache::SetCache,
 };
 use crate::{
     device::{Dev, MonitoredDevice, RegionId},
@@ -34,19 +33,36 @@ use crate::{
     IoBytesMut,
 };
 
+/// # Lock Order
+///
+/// load (async set cache, not good):
+///
+/// ```plain
+///                                                                  |------------ requires async mutex -------------|
+/// lock(R) bloom filter => unlock(R) bloom filter => lock(R) set => lock(e) set cache => load => unlock(e) set cache => unlock(r) set
+/// ```
+///
+/// load (sync set cache, good):
+///
+/// ```plain
+/// lock(R) bloom filter => unlock(R) bloom filter => lock(R) set => lock(e) set cache => unlock(e) set cache => load => lock(e) set cache => unlock(e) set cache => unlock(r) set
+/// ```
+///
+/// update:
+///
+/// ```plain
+/// lock(W) set => lock(e) set cache => invalid set cache => unlock(e) set cache => update set => lock(w) bloom filter => unlock(w) bloom filter => unlock(w) set
+/// ```
 struct SetManagerInner {
-    /// A phantom rwlock to prevent set storage operations on disk.
-    ///
-    /// All set disk operations must be prevented by the lock.
-    ///
-    /// In addition, the rwlock also serves as the lock of the in-memory bloom filter.
-    sets: Vec<RwLock<BloomFilterU64<4>>>,
-    cache: Mutex<OrderedHashMap<SetId, Arc<SetStorage>>>,
-    set_cache_capacity: usize,
+    // TODO(MrCroxx): Refine this!!! Make `Set` a RAII type.
+    sets: Vec<AsyncRwLock<()>>,
+    /// As a cache, it is okay that the bloom filter returns a false-negative result, which doesn't break the
+    /// correctness.
+    loose_bloom_filters: Vec<RwLock<BloomFilterU64<4>>>,
+    set_cache: SetCache,
+    metadata: AsyncRwLock<Metadata>,
     set_picker: SetPicker,
 
-    metadata: RwLock<Metadata>,
-
     set_size: usize,
     device: MonitoredDevice,
     regions: Range<RegionId>,
@@ -62,9 +78,9 @@ impl Debug for SetManager {
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
         f.debug_struct("SetManager")
             .field("sets", &self.inner.sets)
-            .field("cache", &self.inner.cache)
-            .field("set_cache_capacity", &self.inner.set_cache_capacity)
+            .field("loose_bloom_filters", &self.inner.loose_bloom_filters)
             .field("set_picker", &self.inner.set_picker)
+            .field("set_cache", &self.inner.set_cache)
             .field("metadata", &self.inner.metadata)
             .field("set_size", &self.inner.set_size)
             .field("device", &self.inner.device)
@@ -75,111 +91,111 @@ impl Debug for SetManager {
 }
 
 impl SetManager {
-    pub async fn open(
-        set_size: usize,
-        set_cache_capacity: usize,
-        device: MonitoredDevice,
-        regions: Range<RegionId>,
-        flush: bool,
-    ) -> Result<Self> {
-        let sets = (device.region_size() / set_size) * (regions.end - regions.start) as usize;
-        assert!(sets > 0);
+    pub async fn open<K, V, S>(config: &GenericSmallStorageConfig<K, V, S>) -> Result<Self>
+    where
+        K: StorageKey,
+        V: StorageValue,
+        S: HashBuilder + Debug,
+    {
+        let device = config.device.clone();
+        let regions = config.regions.clone();
+
+        let sets = (device.region_size() / config.set_size) * (regions.end - regions.start) as usize;
+        assert!(sets > 0); // TODO: assert > 1? Set with id = 0 is used as metadata.
 
         let set_picker = SetPicker::new(sets);
 
         // load & flush metadata
         let metadata = Metadata::load(&device).await?;
         metadata.flush(&device).await?;
-        let metadata = RwLock::new(metadata);
+        let metadata = AsyncRwLock::new(metadata);
 
-        let sets = (0..sets).map(|_| RwLock::default()).collect_vec();
-        let cache = Mutex::new(OrderedHashMap::with_capacity(set_cache_capacity));
+        let set_cache = SetCache::new(config.set_cache_capacity, config.set_cache_shards);
+        let loose_bloom_filters = (0..sets).map(|_| RwLock::new(BloomFilterU64::new())).collect_vec();
+
+        let sets = (0..sets).map(|_| AsyncRwLock::default()).collect_vec();
 
         let inner = SetManagerInner {
             sets,
-            cache,
-            set_cache_capacity,
+            loose_bloom_filters,
+            set_cache,
             set_picker,
             metadata,
-            set_size,
+            set_size: config.set_size,
             device,
             regions,
-            flush,
+            flush: config.flush,
         };
         let inner = Arc::new(inner);
         Ok(Self { inner })
     }
 
-    pub async fn write(&self, id: SetId) -> Result<SetWriteGuard<'_>> {
-        let guard = self.inner.sets[id as usize].write().await;
-
-        let invalid = self.inner.cache.lock().await.remove(&id);
-        let storage = match invalid {
-            // `guard` already guarantees that there is only one storage reference left.
-            Some(storage) => Arc::into_inner(storage).unwrap(),
-            None => self.storage(id).await?,
-        };
-
-        Ok(SetWriteGuard {
-            bloom_filter: guard,
-            id,
-            set: SetMut::new(storage),
-            drop: DropPanicGuard::default(),
-        })
+    pub fn may_contains(&self, hash: u64) -> bool {
+        let sid = self.inner.set_picker.sid(hash);
+        self.inner.loose_bloom_filters[sid as usize].read().lookup(hash)
     }
 
-    pub async fn read(&self, id: SetId, hash: u64) -> Result<Option<SetReadGuard<'_>>> {
-        let guard = self.inner.sets[id as usize].read().await;
-        if !guard.lookup(hash) {
+    pub async fn load<K, V>(&self, hash: u64) -> Result<Option<(K, V)>>
+    where
+        K: StorageKey,
+        V: StorageValue,
+    {
+        let sid = self.inner.set_picker.sid(hash);
+
+        // Query bloom filter.
+        if !self.inner.loose_bloom_filters[sid as usize].read().lookup(hash) {
             return Ok(None);
         }
 
-        let mut cache = self.inner.cache.lock().await;
-        let cached = cache.get(&id).cloned();
-        let storage = match cached {
-            Some(storage) => storage,
-            None => {
-                let storage = self.storage(id).await?;
-                let storage = Arc::new(storage);
-                cache.insert(id, storage.clone());
-                if cache.len() > self.inner.set_cache_capacity {
-                    cache.pop_front();
-                    strict_assert!(cache.len() <= self.inner.set_cache_capacity);
-                }
-                storage
-            }
-        };
-        drop(cache);
+        // Acquire set lock.
+        let set = self.inner.sets[sid as usize].read().await;
+
+        // Query form set cache.
+        if let Some(cached) = self.inner.set_cache.lookup(&sid) {
+            return cached.get(hash);
+        }
+
+        // Set cache miss, load from disk.
+        let storage = self.storage(sid).await?;
+        let res = storage.get(hash);
 
-        Ok(Some(SetReadGuard {
-            _bloom_filter: guard,
-            _id: id,
-            set: Set::new(storage),
-        }))
+        // Update set cache on cache miss.
+        self.inner.set_cache.insert(sid, storage);
+
+        // Release set lock.
+        drop(set);
+
+        res
     }
 
-    pub async fn apply(&self, mut guard: SetWriteGuard<'_>) -> Result<()> {
-        let mut storage = guard.set.into_storage();
+    pub async fn update<K, V, S>(&self, sid: SetId, deletions: &HashSet<u64>, items: Vec<Item<K, V, S>>) -> Result<()>
+    where
+        K: StorageKey,
+        V: StorageValue,
+        S: HashBuilder + Debug,
+    {
+        // Acquire set lock.
+        let set = self.inner.sets[sid as usize].write().await;
+
+        self.inner.set_cache.invalid(&sid);
 
-        // Update in-memory bloom filter.
+        let mut storage = self.storage(sid).await?;
+        storage.apply(deletions, items);
         storage.update();
-        *guard.bloom_filter = storage.bloom_filter().clone();
 
-        let buffer = storage.freeze();
+        *self.inner.loose_bloom_filters[sid as usize].write() = storage.bloom_filter().clone();
 
-        let (region, offset) = self.locate(guard.id);
+        let buffer = storage.freeze();
+        let (region, offset) = self.locate(sid);
         self.inner.device.write(buffer, region, offset).await?;
         if self.inner.flush {
             self.inner.device.flush(Some(region)).await?;
         }
-        guard.drop.disable();
-        drop(guard.bloom_filter);
-        Ok(())
-    }
 
-    pub async fn contains(&self, id: SetId, hash: u64) -> bool {
-        let guard = self.inner.sets[id as usize].read().await;
-        guard.lookup(hash)
+        // Release set lock.
+        drop(set);
+
+        Ok(())
     }
 
     pub fn sets(&self) -> usize {
@@ -190,17 +206,13 @@ impl SetManager {
         self.inner.set_size
     }
 
-    pub fn set_picker(&self) -> &SetPicker {
-        &self.inner.set_picker
-    }
-
     pub async fn watermark(&self) -> u128 {
         self.inner.metadata.read().await.watermark
     }
 
     pub async fn destroy(&self) -> Result<()> {
         self.update_watermark().await?;
-        self.inner.cache.lock().await.clear();
+        self.inner.set_cache.clear();
         Ok(())
     }
 
@@ -233,62 +245,6 @@ impl SetManager {
     }
 }
 
-#[derive(Debug, Default)]
-struct DropPanicGuard {
-    disabled: bool,
-}
-
-impl Drop for DropPanicGuard {
-    fn drop(&mut self) {
-        if !self.disabled {
-            panic!("unexpected drop panic guard drop");
-        }
-    }
-}
-
-impl DropPanicGuard {
-    fn disable(&mut self) {
-        self.disabled = true;
-    }
-}
-
-#[derive(Debug)]
-pub struct SetWriteGuard<'a> {
-    bloom_filter: RwLockWriteGuard<'a, BloomFilterU64<4>>,
-    id: SetId,
-    set: SetMut,
-    drop: DropPanicGuard,
-}
-
-impl<'a> Deref for SetWriteGuard<'a> {
-    type Target = SetMut;
-
-    fn deref(&self) -> &Self::Target {
-        &self.set
-    }
-}
-
-impl<'a> DerefMut for SetWriteGuard<'a> {
-    fn deref_mut(&mut self) -> &mut Self::Target {
-        &mut self.set
-    }
-}
-
-#[derive(Debug)]
-pub struct SetReadGuard<'a> {
-    _bloom_filter: RwLockReadGuard<'a, BloomFilterU64<4>>,
-    _id: SetId,
-    set: Set,
-}
-
-impl<'a> Deref for SetReadGuard<'a> {
-    type Target = Set;
-
-    fn deref(&self) -> &Self::Target {
-        &self.set
-    }
-}
-
 #[derive(Debug, Clone)]
 pub struct SetPicker {
     sets: usize,
diff --git a/foyer-storage/src/store.rs b/foyer-storage/src/store.rs
index 39428ae2..f1634938 100644
--- a/foyer-storage/src/store.rs
+++ b/foyer-storage/src/store.rs
@@ -570,6 +570,7 @@ where
                             EngineEnum::open(EngineConfig::Small(GenericSmallStorageConfig {
                                 set_size: self.small.set_size,
                                 set_cache_capacity: self.small.set_cache_capacity,
+                                set_cache_shards: self.small.set_cache_shards,
                                 device,
                                 regions,
                                 flush: self.flush,
@@ -590,6 +591,7 @@ where
                                 left: GenericSmallStorageConfig {
                                     set_size: self.small.set_size,
                                     set_cache_capacity: self.small.set_cache_capacity,
+                                    set_cache_shards: self.small.set_cache_shards,
                                     device: device.clone(),
                                     regions: small_regions,
                                     flush: self.flush,
@@ -818,6 +820,7 @@ where
 {
     set_size: usize,
     set_cache_capacity: usize,
+    set_cache_shards: usize,
     buffer_pool_size: usize,
     flushers: usize,
 
@@ -847,6 +850,7 @@ where
         Self {
             set_size: 16 * 1024,    // 16 KiB
             set_cache_capacity: 64, // 64 sets
+            set_cache_shards: 4,
             flushers: 1,
             buffer_pool_size: 4 * 1024 * 1024, // 4 MiB
             _marker: PhantomData,
@@ -874,6 +878,14 @@ where
         self
     }
 
+    /// Set the shards of the set cache.
+    ///
+    /// Default: 4
+    pub fn with_set_cache_shards(mut self, set_cache_shards: usize) -> Self {
+        self.set_cache_shards = set_cache_shards;
+        self
+    }
+
     /// Set the total flush buffer pool size.
     ///
     /// Each flusher shares a volume at `threshold / flushers`.

From 42cdce26caeb1864eff5eb107b0d2ea4100a8f3d Mon Sep 17 00:00:00 2001
From: Croxx <mrcroxx@outlook.com>
Date: Thu, 10 Oct 2024 15:31:27 +0800
Subject: [PATCH 31/53] feat: scale shards to 1 when there is not enough
 capacity (#765)

Signed-off-by: MrCroxx <mrcroxx@outlook.com>
---
 foyer-memory/src/cache.rs   | 29 +++++++++++++++++++++++++++--
 foyer-memory/src/generic.rs |  6 ++++++
 2 files changed, 33 insertions(+), 2 deletions(-)

diff --git a/foyer-memory/src/cache.rs b/foyer-memory/src/cache.rs
index ab072a48..c44818d5 100644
--- a/foyer-memory/src/cache.rs
+++ b/foyer-memory/src/cache.rs
@@ -401,7 +401,16 @@ where
     }
 
     /// Build in-memory cache with the given configuration.
-    pub fn build(self) -> Cache<K, V, S> {
+    pub fn build(mut self) -> Cache<K, V, S> {
+        if self.capacity < self.shards {
+            tracing::warn!(
+                "The in-memory cache capacity({}) < shards({})",
+                self.capacity,
+                self.shards
+            );
+            self.shards = 1;
+        }
+
         match self.eviction_config {
             EvictionConfig::Fifo(eviction_config) => Cache::Fifo(Arc::new(GenericCache::new(GenericCacheConfig {
                 name: self.name,
@@ -657,7 +666,7 @@ where
     }
 
     /// Get the hash builder of the in-memory cache.
-    fn hash_builder(&self) -> &S {
+    pub fn hash_builder(&self) -> &S {
         match self {
             Cache::Fifo(cache) => cache.hash_builder(),
             Cache::Lru(cache) => cache.hash_builder(),
@@ -665,6 +674,16 @@ where
             Cache::S3Fifo(cache) => cache.hash_builder(),
         }
     }
+
+    /// Get the shards of the in-memory cache.
+    pub fn shards(&self) -> usize {
+        match self {
+            Cache::Fifo(cache) => cache.shards(),
+            Cache::Lru(cache) => cache.shards(),
+            Cache::Lfu(cache) => cache.shards(),
+            Cache::S3Fifo(cache) => cache.shards(),
+        }
+    }
 }
 
 /// A future that is used to get entry value from the remote storage for the in-memory cache.
@@ -870,6 +889,12 @@ mod tests {
     const OPS: usize = 10000;
     const CONCURRENCY: usize = 8;
 
+    #[test]
+    fn test_not_enough_capacity() {
+        let cache: Cache<u64, u64> = CacheBuilder::new(4).with_shards(64).build();
+        assert_eq!(cache.shards(), 1);
+    }
+
     fn fifo() -> Cache<u64, u64> {
         CacheBuilder::new(CAPACITY)
             .with_shards(SHARDS)
diff --git a/foyer-memory/src/generic.rs b/foyer-memory/src/generic.rs
index 60f57a6c..d2625836 100644
--- a/foyer-memory/src/generic.rs
+++ b/foyer-memory/src/generic.rs
@@ -497,6 +497,8 @@ where
     S: HashBuilder,
 {
     pub fn new(config: GenericCacheConfig<K, V, E, S>) -> Self {
+        assert!(config.capacity >= config.shards);
+
         let metrics = Arc::new(Metrics::new(&config.name));
 
         let usages = (0..config.shards).map(|_| Arc::new(AtomicUsize::new(0))).collect_vec();
@@ -697,6 +699,10 @@ where
         &self.hash_builder
     }
 
+    pub fn shards(&self) -> usize {
+        self.shards.len()
+    }
+
     unsafe fn try_release_external_handle(&self, ptr: NonNull<E::Handle>) {
         let entry = {
             let base = ptr.as_ref().base();

From 192d5b802b27c127cc8d1b83a18eaf76c5624614 Mon Sep 17 00:00:00 2001
From: Croxx <mrcroxx@outlook.com>
Date: Thu, 10 Oct 2024 17:10:59 +0800
Subject: [PATCH 32/53] chore: release 0.12.1 (#766)

Signed-off-by: MrCroxx <mrcroxx@outlook.com>
---
 CHANGELOG.md | 20 ++++++++++++++++++++
 Cargo.toml   | 12 ++++++------
 2 files changed, 26 insertions(+), 6 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 51c3d447..5f8bdceb 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -9,6 +9,26 @@ date: 2023-05-12T11:02:09+08:00
 
 <!-- truncate -->
 
+## 2024-10-10
+
+### Releases
+
+| crate | version |
+| - | - |
+| foyer | 0.12.1 |
+| foyer-common | 0.12.1 |
+| foyer-intrusive | 0.12.1 |
+| foyer-memory | 0.12.1 |
+| foyer-storage | 0.12.1 |
+| foyer-bench | 0.12.1 |
+
+### Changes
+
+- Downgrade hashbrown to 0.14 to fix build on nightly for projects using hashbrown < 0.15.
+- Fix build with madsim.
+- Refine small object disk cache.
+- Scale shards to 1 when there is not enough capacity.
+
 ## 2024-10-09
 
 ### Releases
diff --git a/Cargo.toml b/Cargo.toml
index 64ae2d8b..b345e9fc 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -13,7 +13,7 @@ members = [
 ]
 
 [workspace.package]
-version = "0.12.0"
+version = "0.12.1"
 edition = "2021"
 rust-version = "1.81.0"
 repository = "https://github.com/foyer-rs/foyer"
@@ -47,11 +47,11 @@ clap = { version = "4", features = ["derive"] }
 bytesize = { package = "foyer-bytesize", version = "2" }
 hashbrown = "0.14"
 # foyer components
-foyer-common = { version = "0.12.0", path = "foyer-common" }
-foyer-intrusive = { version = "0.12.0", path = "foyer-intrusive" }
-foyer-memory = { version = "0.12.0", path = "foyer-memory" }
-foyer-storage = { version = "0.12.0", path = "foyer-storage" }
-foyer = { version = "0.12.0", path = "foyer" }
+foyer-common = { version = "0.12.1", path = "foyer-common" }
+foyer-intrusive = { version = "0.12.1", path = "foyer-intrusive" }
+foyer-memory = { version = "0.12.1", path = "foyer-memory" }
+foyer-storage = { version = "0.12.1", path = "foyer-storage" }
+foyer = { version = "0.12.1", path = "foyer" }
 
 [workspace.lints.rust]
 unexpected_cfgs = { level = "warn", check-cfg = ['cfg(madsim)'] }

From 96581eb6569aba13875812e55143a4b327b55d67 Mon Sep 17 00:00:00 2001
From: Croxx <mrcroxx@outlook.com>
Date: Thu, 10 Oct 2024 17:58:14 +0800
Subject: [PATCH 33/53] fix: don't scale in shards when capacity is not enough,
 just warn (#769)

Signed-off-by: MrCroxx <mrcroxx@outlook.com>
---
 foyer-memory/src/cache.rs   | 11 ++---------
 foyer-memory/src/generic.rs |  2 --
 2 files changed, 2 insertions(+), 11 deletions(-)

diff --git a/foyer-memory/src/cache.rs b/foyer-memory/src/cache.rs
index c44818d5..47faf6be 100644
--- a/foyer-memory/src/cache.rs
+++ b/foyer-memory/src/cache.rs
@@ -401,14 +401,13 @@ where
     }
 
     /// Build in-memory cache with the given configuration.
-    pub fn build(mut self) -> Cache<K, V, S> {
+    pub fn build(self) -> Cache<K, V, S> {
         if self.capacity < self.shards {
             tracing::warn!(
-                "The in-memory cache capacity({}) < shards({})",
+                "The in-memory cache capacity({}) < shards({}).",
                 self.capacity,
                 self.shards
             );
-            self.shards = 1;
         }
 
         match self.eviction_config {
@@ -889,12 +888,6 @@ mod tests {
     const OPS: usize = 10000;
     const CONCURRENCY: usize = 8;
 
-    #[test]
-    fn test_not_enough_capacity() {
-        let cache: Cache<u64, u64> = CacheBuilder::new(4).with_shards(64).build();
-        assert_eq!(cache.shards(), 1);
-    }
-
     fn fifo() -> Cache<u64, u64> {
         CacheBuilder::new(CAPACITY)
             .with_shards(SHARDS)
diff --git a/foyer-memory/src/generic.rs b/foyer-memory/src/generic.rs
index d2625836..21fdef60 100644
--- a/foyer-memory/src/generic.rs
+++ b/foyer-memory/src/generic.rs
@@ -497,8 +497,6 @@ where
     S: HashBuilder,
 {
     pub fn new(config: GenericCacheConfig<K, V, E, S>) -> Self {
-        assert!(config.capacity >= config.shards);
-
         let metrics = Arc::new(Metrics::new(&config.name));
 
         let usages = (0..config.shards).map(|_| Arc::new(AtomicUsize::new(0))).collect_vec();

From 069aa05c4b969d3ee2d83bbf4697c0197ae8d66f Mon Sep 17 00:00:00 2001
From: Croxx <mrcroxx@outlook.com>
Date: Thu, 10 Oct 2024 18:37:11 +0800
Subject: [PATCH 34/53] chore: disable madsim test, just test build (#770)

* chore: disable madsim test, just test build

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

* chore: modify ci concurrency

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

* chore: modify concurrency

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

* chore: modify ci concurrrency

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

* chore: modify ci concurrency

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

* chore: restore concurrency

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

---------

Signed-off-by: MrCroxx <mrcroxx@outlook.com>
---
 .github/workflows/ci.yml | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 863ad4f1..3f9cf6bc 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -377,11 +377,11 @@ jobs:
           CI: true
         run: |-
           cargo clippy --all-targets
-      - name: Run nextest (madsim)
-        env:
-          RUST_BACKTRACE: 1
-          RUSTFLAGS: "--cfg tokio_unstable --cfg madsim"
-          RUST_LOG: info
-          TOKIO_WORKER_THREADS: 1
-        run: |-
-          cargo nextest run --all --features "strict_assertions,sanity"
+      # - name: Run nextest (madsim)
+      #   env:
+      #     RUST_BACKTRACE: 1
+      #     RUSTFLAGS: "--cfg tokio_unstable --cfg madsim"
+      #     RUST_LOG: info
+      #     TOKIO_WORKER_THREADS: 1
+      #   run: |-
+      #     cargo nextest run --all --features "strict_assertions,sanity"

From 0244d599a72473f8b8793ce3c58eca522e372516 Mon Sep 17 00:00:00 2001
From: Croxx <mrcroxx@outlook.com>
Date: Fri, 11 Oct 2024 10:54:05 +0800
Subject: [PATCH 35/53] chore: release foyer v0.12.2 (#771)

Signed-off-by: MrCroxx <mrcroxx@outlook.com>
---
 CHANGELOG.md | 17 +++++++++++++++++
 Cargo.toml   | 12 ++++++------
 2 files changed, 23 insertions(+), 6 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 5f8bdceb..31b8ee31 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -9,6 +9,23 @@ date: 2023-05-12T11:02:09+08:00
 
 <!-- truncate -->
 
+## 2024-10-11
+
+### Releases
+
+| crate | version |
+| - | - |
+| foyer | 0.12.2|
+| foyer-common | 0.12.2 |
+| foyer-intrusive | 0.12.2 |
+| foyer-memory | 0.12.2 |
+| foyer-storage | 0.12.2 |
+| foyer-bench | 0.12.2 |
+
+### Changes
+
+- Revert "Scale shards to 1 when there is not enough capacity". It would be useful sometimes. Just raise the warning.
+
 ## 2024-10-10
 
 ### Releases
diff --git a/Cargo.toml b/Cargo.toml
index b345e9fc..0d024a5e 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -13,7 +13,7 @@ members = [
 ]
 
 [workspace.package]
-version = "0.12.1"
+version = "0.12.2"
 edition = "2021"
 rust-version = "1.81.0"
 repository = "https://github.com/foyer-rs/foyer"
@@ -47,11 +47,11 @@ clap = { version = "4", features = ["derive"] }
 bytesize = { package = "foyer-bytesize", version = "2" }
 hashbrown = "0.14"
 # foyer components
-foyer-common = { version = "0.12.1", path = "foyer-common" }
-foyer-intrusive = { version = "0.12.1", path = "foyer-intrusive" }
-foyer-memory = { version = "0.12.1", path = "foyer-memory" }
-foyer-storage = { version = "0.12.1", path = "foyer-storage" }
-foyer = { version = "0.12.1", path = "foyer" }
+foyer-common = { version = "0.12.2", path = "foyer-common" }
+foyer-intrusive = { version = "0.12.2", path = "foyer-intrusive" }
+foyer-memory = { version = "0.12.2", path = "foyer-memory" }
+foyer-storage = { version = "0.12.2", path = "foyer-storage" }
+foyer = { version = "0.12.2", path = "foyer" }
 
 [workspace.lints.rust]
 unexpected_cfgs = { level = "warn", check-cfg = ['cfg(madsim)'] }

From 50de58c58c5f573d283c3a81e5e8f94e956d2b4d Mon Sep 17 00:00:00 2001
From: Croxx <mrcroxx@outlook.com>
Date: Fri, 11 Oct 2024 14:12:52 +0800
Subject: [PATCH 36/53] feat: support no disk (mem only) for foyer bench (#773)

Signed-off-by: MrCroxx <mrcroxx@outlook.com>
---
 foyer-bench/src/main.rs | 13 ++++++++++---
 1 file changed, 10 insertions(+), 3 deletions(-)

diff --git a/foyer-bench/src/main.rs b/foyer-bench/src/main.rs
index bd3541b9..148756e8 100644
--- a/foyer-bench/src/main.rs
+++ b/foyer-bench/src/main.rs
@@ -59,17 +59,23 @@ static GLOBAL: tikv_jemallocator::Jemalloc = tikv_jemallocator::Jemalloc;
 
 #[derive(Parser, Debug, Clone)]
 #[command(author, version, about)]
-#[command(group = ArgGroup::new("exclusive").required(true).args(&["file", "dir"]))]
+#[command(group = ArgGroup::new("exclusive").required(true).args(&["file", "dir", "no_disk"]))]
 pub struct Args {
+    /// Run with in-memory cache compatible mode.
+    ///
+    /// One of `no_disk`, `file`, `dir` must be set.
+    #[arg(long)]
+    no_disk: bool,
+
     /// File for disk cache data. Use `DirectFile` as device.
     ///
-    /// Either `file` or `dir` must be set.
+    /// One of `no_disk`, `file`, `dir` must be set.
     #[arg(short, long)]
     file: Option<String>,
 
     /// Directory for disk cache data. Use `DirectFs` as device.
     ///
-    /// Either `file` or `dir` must be set.
+    /// One of `no_disk`, `file`, `dir` must be set.
     #[arg(short, long)]
     dir: Option<String>,
 
@@ -472,6 +478,7 @@ async fn benchmark(args: Args) {
                 .with_capacity(args.disk.as_u64() as _)
                 .with_file_size(args.region_size.as_u64() as _),
         ),
+        (None, None) => builder,
         _ => unreachable!(),
     };
 

From 388bcab7b2132330d8aea48bec33306edaaa0ab2 Mon Sep 17 00:00:00 2001
From: Croxx <mrcroxx@outlook.com>
Date: Fri, 11 Oct 2024 17:25:05 +0800
Subject: [PATCH 37/53] feat: use equivalent to loose the key trait bounds
 (#774)

* refactor: use equivalent to loose the key trait bounds

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

* doc: add example for equivalent

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

---------

Signed-off-by: MrCroxx <mrcroxx@outlook.com>
---
 .github/workflows/ci.yml               |  1 +
 Cargo.toml                             | 28 +++++++++--------
 Makefile                               |  1 +
 examples/Cargo.toml                    |  5 +++
 examples/equivalent.rs                 | 43 ++++++++++++++++++++++++++
 foyer-common/src/code.rs               |  2 +-
 foyer-memory/Cargo.toml                |  1 +
 foyer-memory/src/cache.rs              | 16 ++++------
 foyer-memory/src/generic.rs            | 26 ++++++----------
 foyer-memory/src/indexer/hash_table.rs | 13 ++++----
 foyer-memory/src/indexer/mod.rs        |  9 +++---
 foyer-memory/src/indexer/sanity.rs     | 14 ++++-----
 foyer-storage/Cargo.toml               |  1 +
 foyer-storage/src/store.rs             | 13 +++-----
 foyer-storage/src/test_utils.rs        | 29 +++++++----------
 foyer/Cargo.toml                       |  1 +
 foyer/src/hybrid/cache.rs              | 23 +++++---------
 17 files changed, 125 insertions(+), 101 deletions(-)
 create mode 100644 examples/equivalent.rs

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 3f9cf6bc..46dde79f 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -193,6 +193,7 @@ jobs:
           cargo llvm-cov --no-report run --example event_listener
           cargo llvm-cov --no-report run --features "tracing,jaeger" --example tail_based_tracing
           cargo llvm-cov --no-report run --features "tracing,ot" --example tail_based_tracing
+          cargo llvm-cov --no-report run --example equivalent
       - name: Run foyer-bench with coverage
         if: runner.os == 'Linux'
         env:
diff --git a/Cargo.toml b/Cargo.toml
index 0d024a5e..b728a02c 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -24,6 +24,20 @@ license = "Apache-2.0"
 readme = "README.md"
 
 [workspace.dependencies]
+bytesize = { package = "foyer-bytesize", version = "2" }
+clap = { version = "4", features = ["derive"] }
+equivalent = "1"
+fastrace = "0.7"
+fastrace-jaeger = "0.7"
+fastrace-opentelemetry = "0.7"
+hashbrown = "0.14"
+itertools = "0.13"
+metrics = "0.23"
+serde = { version = "1", features = ["derive", "rc"] }
+test-log = { version = "0.2", default-features = false, features = [
+    "trace",
+    "color",
+] }
 tokio = { package = "madsim-tokio", version = "0.2", features = [
     "rt",
     "rt-multi-thread",
@@ -33,19 +47,7 @@ tokio = { package = "madsim-tokio", version = "0.2", features = [
     "signal",
     "fs",
 ] }
-serde = { version = "1", features = ["derive", "rc"] }
-test-log = { version = "0.2", default-features = false, features = [
-    "trace",
-    "color",
-] }
-itertools = "0.13"
-metrics = "0.23"
-fastrace = "0.7"
-fastrace-jaeger = "0.7"
-fastrace-opentelemetry = "0.7"
-clap = { version = "4", features = ["derive"] }
-bytesize = { package = "foyer-bytesize", version = "2" }
-hashbrown = "0.14"
+
 # foyer components
 foyer-common = { version = "0.12.2", path = "foyer-common" }
 foyer-intrusive = { version = "0.12.2", path = "foyer-intrusive" }
diff --git a/Makefile b/Makefile
index c40e0b3a..c33e2b34 100644
--- a/Makefile
+++ b/Makefile
@@ -43,6 +43,7 @@ example:
 	cargo run --example event_listener
 	cargo run --features "tracing,jaeger" --example tail_based_tracing
 	cargo run --features "tracing,ot" --example tail_based_tracing
+	cargo run --example equivalent
 
 full: check-all test-all example udeps
 
diff --git a/examples/Cargo.toml b/examples/Cargo.toml
index a0b38593..3f732f81 100644
--- a/examples/Cargo.toml
+++ b/examples/Cargo.toml
@@ -17,6 +17,7 @@ publish = false
 ahash = "0.8"
 anyhow = "1"
 chrono = "0.4"
+equivalent = { workspace = true }
 fastrace = { workspace = true }
 fastrace-jaeger = { workspace = true, optional = true }
 fastrace-opentelemetry = { workspace = true, optional = true }
@@ -60,3 +61,7 @@ path = "event_listener.rs"
 [[example]]
 name = "tail_based_tracing"
 path = "tail_based_tracing.rs"
+
+[[example]]
+name = "equivalent"
+path = "equivalent.rs"
diff --git a/examples/equivalent.rs b/examples/equivalent.rs
new file mode 100644
index 00000000..c8e107b5
--- /dev/null
+++ b/examples/equivalent.rs
@@ -0,0 +1,43 @@
+//  Copyright 2024 foyer Project Authors
+//
+//  Licensed under the Apache License, Version 2.0 (the "License");
+//  you may not use this file except in compliance with the License.
+//  You may obtain a copy of the License at
+//
+//  http://www.apache.org/licenses/LICENSE-2.0
+//
+//  Unless required by applicable law or agreed to in writing, software
+//  distributed under the License is distributed on an "AS IS" BASIS,
+//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//  See the License for the specific language governing permissions and
+//  limitations under the License.
+
+use equivalent::Equivalent;
+use foyer::{Cache, CacheBuilder};
+
+#[derive(Hash, PartialEq, Eq)]
+pub struct Pair<A, B>(pub A, pub B);
+
+// FYI: https://docs.rs/equivalent
+impl<'a, A: ?Sized, B: ?Sized, C, D> Equivalent<(C, D)> for Pair<&'a A, &'a B>
+where
+    A: Equivalent<C>,
+    B: Equivalent<D>,
+{
+    fn equivalent(&self, key: &(C, D)) -> bool {
+        self.0.equivalent(&key.0) && self.1.equivalent(&key.1)
+    }
+}
+
+fn main() {
+    let cache: Cache<(String, String), String> = CacheBuilder::new(16).build();
+
+    let entry = cache.insert(
+        ("hello".to_string(), "world".to_string()),
+        "This is a string tuple pair.".to_string(),
+    );
+    // With `Equivalent`, `Pair(&str, &str)` can be used to compared `(String, String)`!
+    let e = cache.get(&Pair("hello", "world")).unwrap();
+
+    assert_eq!(entry.value(), e.value());
+}
diff --git a/foyer-common/src/code.rs b/foyer-common/src/code.rs
index f1fbf3eb..fe2d34bf 100644
--- a/foyer-common/src/code.rs
+++ b/foyer-common/src/code.rs
@@ -17,7 +17,7 @@ use std::hash::{BuildHasher, Hash};
 use serde::{de::DeserializeOwned, Serialize};
 
 /// Key trait for the in-memory cache.
-pub trait Key: Send + Sync + 'static + Hash + Eq + PartialEq {}
+pub trait Key: Send + Sync + 'static + Hash + Eq {}
 /// Value trait for the in-memory cache.
 pub trait Value: Send + Sync + 'static {}
 
diff --git a/foyer-memory/Cargo.toml b/foyer-memory/Cargo.toml
index 914d315c..2d116f3a 100644
--- a/foyer-memory/Cargo.toml
+++ b/foyer-memory/Cargo.toml
@@ -16,6 +16,7 @@ readme = { workspace = true }
 ahash = "0.8"
 bitflags = "2"
 cmsketch = "0.2.1"
+equivalent = { workspace = true }
 fastrace = { workspace = true }
 foyer-common = { workspace = true }
 foyer-intrusive = { workspace = true }
diff --git a/foyer-memory/src/cache.rs b/foyer-memory/src/cache.rs
index 47faf6be..e3554e48 100644
--- a/foyer-memory/src/cache.rs
+++ b/foyer-memory/src/cache.rs
@@ -12,9 +12,10 @@
 //  See the License for the specific language governing permissions and
 //  limitations under the License.
 
-use std::{borrow::Borrow, fmt::Debug, hash::Hash, ops::Deref, sync::Arc};
+use std::{fmt::Debug, hash::Hash, ops::Deref, sync::Arc};
 
 use ahash::RandomState;
+use equivalent::Equivalent;
 use foyer_common::{
     code::{HashBuilder, Key, Value},
     event::EventListener,
@@ -566,8 +567,7 @@ where
     #[fastrace::trace(name = "foyer::memory::cache::remove")]
     pub fn remove<Q>(&self, key: &Q) -> Option<CacheEntry<K, V, S>>
     where
-        K: Borrow<Q>,
-        Q: Hash + Eq + ?Sized,
+        Q: Hash + Equivalent<K> + ?Sized,
     {
         match self {
             Cache::Fifo(cache) => cache.remove(key).map(CacheEntry::from),
@@ -581,8 +581,7 @@ where
     #[fastrace::trace(name = "foyer::memory::cache::get")]
     pub fn get<Q>(&self, key: &Q) -> Option<CacheEntry<K, V, S>>
     where
-        K: Borrow<Q>,
-        Q: Hash + Eq + ?Sized,
+        Q: Hash + Equivalent<K> + ?Sized,
     {
         match self {
             Cache::Fifo(cache) => cache.get(key).map(CacheEntry::from),
@@ -596,8 +595,7 @@ where
     #[fastrace::trace(name = "foyer::memory::cache::contains")]
     pub fn contains<Q>(&self, key: &Q) -> bool
     where
-        K: Borrow<Q>,
-        Q: Hash + Eq + ?Sized,
+        Q: Hash + Equivalent<K> + ?Sized,
     {
         match self {
             Cache::Fifo(cache) => cache.contains(key),
@@ -613,8 +611,7 @@ where
     #[fastrace::trace(name = "foyer::memory::cache::touch")]
     pub fn touch<Q>(&self, key: &Q) -> bool
     where
-        K: Borrow<Q>,
-        Q: Hash + Eq + ?Sized,
+        Q: Hash + Equivalent<K> + ?Sized,
     {
         match self {
             Cache::Fifo(cache) => cache.touch(key),
@@ -658,7 +655,6 @@ where
     /// Hash the given key with the hash builder of the cache.
     pub fn hash<Q>(&self, key: &Q) -> u64
     where
-        K: Borrow<Q>,
         Q: Hash + ?Sized,
     {
         self.hash_builder().hash_one(key)
diff --git a/foyer-memory/src/generic.rs b/foyer-memory/src/generic.rs
index 21fdef60..6d4504fe 100644
--- a/foyer-memory/src/generic.rs
+++ b/foyer-memory/src/generic.rs
@@ -13,7 +13,6 @@
 //  limitations under the License.
 
 use std::{
-    borrow::Borrow,
     fmt::Debug,
     future::Future,
     hash::Hash,
@@ -28,6 +27,7 @@ use std::{
 };
 
 use ahash::RandomState;
+use equivalent::Equivalent;
 use fastrace::{future::InSpan, prelude::*};
 use foyer_common::{
     code::{HashBuilder, Key, Value},
@@ -171,8 +171,7 @@ where
 
     unsafe fn get<Q>(&mut self, hash: u64, key: &Q) -> Option<NonNull<E::Handle>>
     where
-        K: Borrow<Q>,
-        Q: Hash + Eq + ?Sized,
+        Q: Hash + Equivalent<K> + ?Sized,
     {
         let mut ptr = match self.indexer.get(hash, key) {
             Some(ptr) => {
@@ -196,16 +195,14 @@ where
 
     unsafe fn contains<Q>(&mut self, hash: u64, key: &Q) -> bool
     where
-        K: Borrow<Q>,
-        Q: Hash + Eq + ?Sized,
+        Q: Hash + Equivalent<K> + ?Sized,
     {
         self.indexer.get(hash, key).is_some()
     }
 
     unsafe fn touch<Q>(&mut self, hash: u64, key: &Q) -> bool
     where
-        K: Borrow<Q>,
-        Q: Hash + Eq + ?Sized,
+        Q: Hash + Equivalent<K> + ?Sized,
     {
         let res = self.indexer.get(hash, key);
         if let Some(ptr) = res {
@@ -219,8 +216,7 @@ where
     /// Return `Some(..)` if the handle is released, or `None` if the handle is still in use.
     unsafe fn remove<Q>(&mut self, hash: u64, key: &Q) -> Option<NonNull<E::Handle>>
     where
-        K: Borrow<Q>,
-        Q: Hash + Eq + ?Sized,
+        Q: Hash + Equivalent<K> + ?Sized,
     {
         let mut ptr = self.indexer.remove(hash, key)?;
         let handle = ptr.as_mut();
@@ -608,8 +604,7 @@ where
     #[fastrace::trace(name = "foyer::memory::generic::remove")]
     pub fn remove<Q>(self: &Arc<Self>, key: &Q) -> Option<GenericCacheEntry<K, V, E, I, S>>
     where
-        K: Borrow<Q>,
-        Q: Hash + Eq + ?Sized,
+        Q: Hash + Equivalent<K> + ?Sized,
     {
         let hash = self.hash_builder.hash_one(key);
 
@@ -625,8 +620,7 @@ where
     #[fastrace::trace(name = "foyer::memory::generic::get")]
     pub fn get<Q>(self: &Arc<Self>, key: &Q) -> Option<GenericCacheEntry<K, V, E, I, S>>
     where
-        K: Borrow<Q>,
-        Q: Hash + Eq + ?Sized,
+        Q: Hash + Equivalent<K> + ?Sized,
     {
         let hash = self.hash_builder.hash_one(key);
 
@@ -641,8 +635,7 @@ where
 
     pub fn contains<Q>(self: &Arc<Self>, key: &Q) -> bool
     where
-        K: Borrow<Q>,
-        Q: Hash + Eq + ?Sized,
+        Q: Hash + Equivalent<K> + ?Sized,
     {
         let hash = self.hash_builder.hash_one(key);
 
@@ -654,8 +647,7 @@ where
 
     pub fn touch<Q>(&self, key: &Q) -> bool
     where
-        K: Borrow<Q>,
-        Q: Hash + Eq + ?Sized,
+        Q: Hash + Equivalent<K> + ?Sized,
     {
         let hash = self.hash_builder.hash_one(key);
 
diff --git a/foyer-memory/src/indexer/hash_table.rs b/foyer-memory/src/indexer/hash_table.rs
index 19dc7d29..47ed6dd6 100644
--- a/foyer-memory/src/indexer/hash_table.rs
+++ b/foyer-memory/src/indexer/hash_table.rs
@@ -12,8 +12,9 @@
 //  See the License for the specific language governing permissions and
 //  limitations under the License.
 
-use std::{borrow::Borrow, hash::Hash, ptr::NonNull};
+use std::{hash::Hash, ptr::NonNull};
 
+use equivalent::Equivalent;
 use foyer_common::{code::Key, strict_assert};
 use hashbrown::hash_table::{Entry as HashTableEntry, HashTable};
 
@@ -83,20 +84,18 @@ where
 
     unsafe fn get<Q>(&self, hash: u64, key: &Q) -> Option<NonNull<Self::Handle>>
     where
-        Self::Key: Borrow<Q>,
-        Q: Hash + Eq + ?Sized,
+        Q: Hash + Equivalent<Self::Key> + ?Sized,
     {
-        self.table.find(hash, |p| p.as_ref().key().borrow() == key).copied()
+        self.table.find(hash, |p| key.equivalent(p.as_ref().key())).copied()
     }
 
     unsafe fn remove<Q>(&mut self, hash: u64, key: &Q) -> Option<NonNull<Self::Handle>>
     where
-        Self::Key: Borrow<Q>,
-        Q: Hash + Eq + ?Sized,
+        Q: Hash + Equivalent<Self::Key> + ?Sized,
     {
         match self
             .table
-            .entry(hash, |p| p.as_ref().key().borrow() == key, |p| p.as_ref().base().hash())
+            .entry(hash, |p| key.equivalent(p.as_ref().key()), |p| p.as_ref().base().hash())
         {
             HashTableEntry::Occupied(o) => {
                 let (mut p, _) = o.remove();
diff --git a/foyer-memory/src/indexer/mod.rs b/foyer-memory/src/indexer/mod.rs
index 552ded1a..bacfd2c2 100644
--- a/foyer-memory/src/indexer/mod.rs
+++ b/foyer-memory/src/indexer/mod.rs
@@ -12,8 +12,9 @@
 //  See the License for the specific language governing permissions and
 //  limitations under the License.
 
-use std::{borrow::Borrow, hash::Hash, ptr::NonNull};
+use std::{hash::Hash, ptr::NonNull};
 
+use equivalent::Equivalent;
 use foyer_common::code::Key;
 
 use crate::handle::KeyedHandle;
@@ -26,12 +27,10 @@ pub trait Indexer: Send + Sync + 'static {
     unsafe fn insert(&mut self, ptr: NonNull<Self::Handle>) -> Option<NonNull<Self::Handle>>;
     unsafe fn get<Q>(&self, hash: u64, key: &Q) -> Option<NonNull<Self::Handle>>
     where
-        Self::Key: Borrow<Q>,
-        Q: Hash + Eq + ?Sized;
+        Q: Hash + Equivalent<Self::Key> + ?Sized;
     unsafe fn remove<Q>(&mut self, hash: u64, key: &Q) -> Option<NonNull<Self::Handle>>
     where
-        Self::Key: Borrow<Q>,
-        Q: Hash + Eq + ?Sized;
+        Q: Hash + Equivalent<Self::Key> + ?Sized;
     unsafe fn drain(&mut self) -> impl Iterator<Item = NonNull<Self::Handle>>;
 }
 
diff --git a/foyer-memory/src/indexer/sanity.rs b/foyer-memory/src/indexer/sanity.rs
index fcde9a5d..a0962758 100644
--- a/foyer-memory/src/indexer/sanity.rs
+++ b/foyer-memory/src/indexer/sanity.rs
@@ -14,6 +14,8 @@
 
 use std::ptr::NonNull;
 
+use equivalent::Equivalent;
+
 use super::Indexer;
 #[cfg(feature = "sanity")]
 use crate::handle::Handle;
@@ -49,8 +51,7 @@ where
 
     unsafe fn get<Q>(&self, hash: u64, key: &Q) -> Option<NonNull<Self::Handle>>
     where
-        Self::Key: std::borrow::Borrow<Q>,
-        Q: std::hash::Hash + Eq + ?Sized,
+        Q: std::hash::Hash + Equivalent<Self::Key> + ?Sized,
     {
         self.indexer
             .get(hash, key)
@@ -59,8 +60,7 @@ where
 
     unsafe fn remove<Q>(&mut self, hash: u64, key: &Q) -> Option<NonNull<Self::Handle>>
     where
-        Self::Key: std::borrow::Borrow<Q>,
-        Q: std::hash::Hash + Eq + ?Sized,
+        Q: std::hash::Hash + Equivalent<Self::Key> + ?Sized,
     {
         self.indexer
             .remove(hash, key)
@@ -90,16 +90,14 @@ where
 
     unsafe fn get<Q>(&self, hash: u64, key: &Q) -> Option<NonNull<Self::Handle>>
     where
-        Self::Key: std::borrow::Borrow<Q>,
-        Q: std::hash::Hash + Eq + ?Sized,
+        Q: std::hash::Hash + Equivalent<Self::Key> + ?Sized,
     {
         self.indexer.get(hash, key)
     }
 
     unsafe fn remove<Q>(&mut self, hash: u64, key: &Q) -> Option<NonNull<Self::Handle>>
     where
-        Self::Key: std::borrow::Borrow<Q>,
-        Q: std::hash::Hash + Eq + ?Sized,
+        Q: std::hash::Hash + Equivalent<Self::Key> + ?Sized,
     {
         self.indexer.remove(hash, key)
     }
diff --git a/foyer-storage/Cargo.toml b/foyer-storage/Cargo.toml
index 326fb17c..5563db83 100644
--- a/foyer-storage/Cargo.toml
+++ b/foyer-storage/Cargo.toml
@@ -26,6 +26,7 @@ bitflags = "2.3.1"
 bytes = "1"
 clap = { workspace = true }
 either = "1"
+equivalent = { workspace = true }
 fastrace = { workspace = true }
 flume = "0.11"
 foyer-common = { workspace = true }
diff --git a/foyer-storage/src/store.rs b/foyer-storage/src/store.rs
index f1634938..f2379c02 100644
--- a/foyer-storage/src/store.rs
+++ b/foyer-storage/src/store.rs
@@ -13,7 +13,6 @@
 //  limitations under the License.
 
 use std::{
-    borrow::Borrow,
     fmt::{Debug, Display},
     hash::Hash,
     marker::PhantomData,
@@ -23,6 +22,7 @@ use std::{
 };
 
 use ahash::RandomState;
+use equivalent::Equivalent;
 use foyer_common::{
     bits,
     code::{HashBuilder, StorageKey, StorageValue},
@@ -151,13 +151,12 @@ where
     /// Load a cache entry from the disk cache.
     pub async fn load<Q>(&self, key: &Q) -> Result<Option<(K, V)>>
     where
-        K: Borrow<Q>,
-        Q: Hash + Eq + ?Sized + Send + Sync + 'static,
+        Q: Hash + Equivalent<K> + ?Sized + Send + Sync + 'static,
     {
         let hash = self.inner.memory.hash(key);
         let future = self.inner.engine.load(hash);
         match self.inner.runtime.read().spawn(future).await.unwrap() {
-            Ok(Some((k, v))) if k.borrow() == key => Ok(Some((k, v))),
+            Ok(Some((k, v))) if key.equivalent(&k) => Ok(Some((k, v))),
             Ok(_) => Ok(None),
             Err(e) => Err(e),
         }
@@ -166,8 +165,7 @@ where
     /// Delete the cache entry with the given key from the disk cache.
     pub fn delete<'a, Q>(&'a self, key: &'a Q)
     where
-        K: Borrow<Q>,
-        Q: Hash + Eq + ?Sized,
+        Q: Hash + Equivalent<K> + ?Sized,
     {
         let hash = self.inner.memory.hash(key);
         self.inner.engine.delete(hash)
@@ -178,8 +176,7 @@ where
     /// `contains` may return a false-positive result if there is a hash collision with the given key.
     pub fn may_contains<Q>(&self, key: &Q) -> bool
     where
-        K: Borrow<Q>,
-        Q: Hash + Eq + ?Sized,
+        Q: Hash + Equivalent<K> + ?Sized,
     {
         let hash = self.inner.memory.hash(key);
         self.inner.engine.may_contains(hash)
diff --git a/foyer-storage/src/test_utils.rs b/foyer-storage/src/test_utils.rs
index cfef7dbc..115c7f53 100644
--- a/foyer-storage/src/test_utils.rs
+++ b/foyer-storage/src/test_utils.rs
@@ -19,7 +19,6 @@ use std::{
     collections::HashSet,
     fmt::Debug,
     hash::Hash,
-    marker::PhantomData,
     sync::{Arc, OnceLock},
 };
 
@@ -40,49 +39,45 @@ pub fn metrics_for_test() -> &'static Metrics {
 }
 
 /// A picker that only admits key from the given list.
-pub struct BiasedPicker<K, Q> {
-    admits: HashSet<Q>,
-    _marker: PhantomData<K>,
+pub struct BiasedPicker<K> {
+    admits: HashSet<K>,
 }
 
-impl<K, Q> Debug for BiasedPicker<K, Q>
+impl<K> Debug for BiasedPicker<K>
 where
-    Q: Debug,
+    K: Debug,
 {
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
         f.debug_struct("BiasedPicker").field("admits", &self.admits).finish()
     }
 }
 
-impl<K, Q> BiasedPicker<K, Q> {
+impl<K> BiasedPicker<K> {
     /// Create a biased picker with the given admit list.
-    pub fn new(admits: impl IntoIterator<Item = Q>) -> Self
+    pub fn new(admits: impl IntoIterator<Item = K>) -> Self
     where
-        Q: Hash + Eq,
+        K: Hash + Eq,
     {
         Self {
             admits: admits.into_iter().collect(),
-            _marker: PhantomData,
         }
     }
 }
 
-impl<K, Q> AdmissionPicker for BiasedPicker<K, Q>
+impl<K> AdmissionPicker for BiasedPicker<K>
 where
-    K: Send + Sync + 'static + Borrow<Q>,
-    Q: Hash + Eq + Send + Sync + 'static + Debug,
+    K: Send + Sync + 'static + Hash + Eq + Debug,
 {
     type Key = K;
 
     fn pick(&self, _: &Arc<Statistics>, key: &Self::Key) -> bool {
-        self.admits.contains(key.borrow())
+        self.admits.contains(key)
     }
 }
 
-impl<K, Q> ReinsertionPicker for BiasedPicker<K, Q>
+impl<K> ReinsertionPicker for BiasedPicker<K>
 where
-    K: Send + Sync + 'static + Borrow<Q>,
-    Q: Hash + Eq + Send + Sync + 'static + Debug,
+    K: Send + Sync + 'static + Hash + Eq + Debug,
 {
     type Key = K;
 
diff --git a/foyer/Cargo.toml b/foyer/Cargo.toml
index aad21ea8..3352170b 100644
--- a/foyer/Cargo.toml
+++ b/foyer/Cargo.toml
@@ -15,6 +15,7 @@ readme = { workspace = true }
 [dependencies]
 ahash = "0.8"
 anyhow = "1"
+equivalent = { workspace = true }
 fastrace = { workspace = true }
 foyer-common = { workspace = true }
 foyer-memory = { workspace = true }
diff --git a/foyer/src/hybrid/cache.rs b/foyer/src/hybrid/cache.rs
index 6c57a0ef..e784d85f 100644
--- a/foyer/src/hybrid/cache.rs
+++ b/foyer/src/hybrid/cache.rs
@@ -13,7 +13,6 @@
 //  limitations under the License.
 
 use std::{
-    borrow::Borrow,
     fmt::Debug,
     future::Future,
     hash::Hash,
@@ -28,6 +27,7 @@ use std::{
 };
 
 use ahash::RandomState;
+use equivalent::Equivalent;
 use fastrace::prelude::*;
 use foyer_common::{
     code::{HashBuilder, StorageKey, StorageValue},
@@ -211,8 +211,7 @@ where
     /// Get cached entry with the given key from the hybrid cache.
     pub async fn get<Q>(&self, key: &Q) -> anyhow::Result<Option<HybridCacheEntry<K, V, S>>>
     where
-        K: Borrow<Q>,
-        Q: Hash + Eq + Send + Sync + 'static + Clone,
+        Q: Hash + Equivalent<K> + Send + Sync + 'static + Clone,
     {
         root_span!(self, mut span, "foyer::hybrid::cache::get");
 
@@ -312,8 +311,7 @@ where
     /// Remove a cached entry with the given key from the hybrid cache.
     pub fn remove<Q>(&self, key: &Q)
     where
-        K: Borrow<Q>,
-        Q: Hash + Eq + ?Sized + Send + Sync + 'static,
+        Q: Hash + Equivalent<K> + ?Sized + Send + Sync + 'static,
     {
         root_span!(self, mut span, "foyer::hybrid::cache::remove");
 
@@ -335,8 +333,7 @@ where
     /// `contains` may return a false-positive result if there is a hash collision with the given key.
     pub fn contains<Q>(&self, key: &Q) -> bool
     where
-        K: Borrow<Q>,
-        Q: Hash + Eq + ?Sized,
+        Q: Hash + Equivalent<K> + ?Sized,
     {
         self.memory.contains(key) || self.storage.may_contains(key)
     }
@@ -537,7 +534,7 @@ where
 #[cfg(test)]
 mod tests {
 
-    use std::{borrow::Borrow, fmt::Debug, hash::Hash, path::Path, sync::Arc};
+    use std::{path::Path, sync::Arc};
 
     use storage::test_utils::BiasedPicker;
 
@@ -562,14 +559,10 @@ mod tests {
             .unwrap()
     }
 
-    async fn open_with_biased_admission_picker<Q>(
+    async fn open_with_biased_admission_picker(
         dir: impl AsRef<Path>,
-        admits: impl IntoIterator<Item = Q>,
-    ) -> HybridCache<u64, Vec<u8>>
-    where
-        u64: Borrow<Q>,
-        Q: Hash + Eq + Send + Sync + 'static + Debug,
-    {
+        admits: impl IntoIterator<Item = u64>,
+    ) -> HybridCache<u64, Vec<u8>> {
         HybridCacheBuilder::new()
             .with_name("test")
             .memory(4 * MB)

From 2b841e5bc177325bb5bb1490e1fbd7be1af16a0a Mon Sep 17 00:00:00 2001
From: Croxx <mrcroxx@outlook.com>
Date: Fri, 18 Oct 2024 15:06:27 +0800
Subject: [PATCH 38/53] refactor: use workspace lint rules for active crates
 (#776)

Signed-off-by: MrCroxx <mrcroxx@outlook.com>
---
 Cargo.toml                                     | 4 ++++
 foyer-bench/Cargo.toml                         | 3 +++
 foyer-bench/src/main.rs                        | 4 +++-
 foyer-cli/Cargo.toml                           | 3 +++
 foyer-cli/src/main.rs                          | 4 ++--
 foyer-common/src/lib.rs                        | 3 ---
 foyer-intrusive/Cargo.toml                     | 3 +++
 foyer-intrusive/src/lib.rs                     | 2 --
 foyer-memory/Cargo.toml                        | 3 +++
 foyer-memory/benches/bench_dynamic_dispatch.rs | 2 ++
 foyer-memory/benches/bench_hit_ratio.rs        | 2 ++
 foyer-memory/src/lib.rs                        | 3 ---
 foyer-storage/src/lib.rs                       | 2 --
 foyer/Cargo.toml                               | 3 +++
 foyer/src/lib.rs                               | 2 --
 15 files changed, 28 insertions(+), 15 deletions(-)

diff --git a/Cargo.toml b/Cargo.toml
index b728a02c..954908aa 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -56,7 +56,11 @@ foyer-storage = { version = "0.12.2", path = "foyer-storage" }
 foyer = { version = "0.12.2", path = "foyer" }
 
 [workspace.lints.rust]
+missing_docs = "warn"
 unexpected_cfgs = { level = "warn", check-cfg = ['cfg(madsim)'] }
 
+[workspace.lints.clippy]
+allow_attributes = "warn"
+
 [profile.release]
 debug = "full"
diff --git a/foyer-bench/Cargo.toml b/foyer-bench/Cargo.toml
index 966c2e91..be44aba6 100644
--- a/foyer-bench/Cargo.toml
+++ b/foyer-bench/Cargo.toml
@@ -48,3 +48,6 @@ sanity = ["foyer/sanity"]
 jemalloc = ["dep:tikv-jemallocator"]
 jeprof = ["jemalloc", "tikv-jemallocator?/profiling"]
 tracing = ["foyer/tracing", "dep:fastrace-jaeger", "dep:fastrace"]
+
+[lints]
+workspace = true
diff --git a/foyer-bench/src/main.rs b/foyer-bench/src/main.rs
index 148756e8..251084ae 100644
--- a/foyer-bench/src/main.rs
+++ b/foyer-bench/src/main.rs
@@ -12,6 +12,8 @@
 //  See the License for the specific language governing permissions and
 //  limitations under the License.
 
+//! foyer benchmark tools.
+
 #![warn(clippy::allow_attributes)]
 
 mod analyze;
@@ -60,7 +62,7 @@ static GLOBAL: tikv_jemallocator::Jemalloc = tikv_jemallocator::Jemalloc;
 #[derive(Parser, Debug, Clone)]
 #[command(author, version, about)]
 #[command(group = ArgGroup::new("exclusive").required(true).args(&["file", "dir", "no_disk"]))]
-pub struct Args {
+struct Args {
     /// Run with in-memory cache compatible mode.
     ///
     /// One of `no_disk`, `file`, `dir` must be set.
diff --git a/foyer-cli/Cargo.toml b/foyer-cli/Cargo.toml
index 7112d357..ca230e84 100644
--- a/foyer-cli/Cargo.toml
+++ b/foyer-cli/Cargo.toml
@@ -24,3 +24,6 @@ thiserror = "1"
 [[bin]]
 name = "foyer"
 path = "src/main.rs"
+
+[lints]
+workspace = true
diff --git a/foyer-cli/src/main.rs b/foyer-cli/src/main.rs
index 081289f8..d0a8a71b 100644
--- a/foyer-cli/src/main.rs
+++ b/foyer-cli/src/main.rs
@@ -21,13 +21,13 @@ use clap::{Parser, Subcommand};
 
 #[derive(Debug, Parser)]
 #[command(author, version, about)]
-pub struct Cli {
+struct Cli {
     #[command(subcommand)]
     command: Command,
 }
 
 #[derive(Debug, Subcommand)]
-pub enum Command {
+enum Command {
     /// Automatic arguments detector.
     Args(ArgsArgs),
 }
diff --git a/foyer-common/src/lib.rs b/foyer-common/src/lib.rs
index bff5d147..98356144 100644
--- a/foyer-common/src/lib.rs
+++ b/foyer-common/src/lib.rs
@@ -12,9 +12,6 @@
 //  See the License for the specific language governing permissions and
 //  limitations under the License.
 
-#![warn(missing_docs)]
-#![warn(clippy::allow_attributes)]
-
 //! Shared components and utils for foyer.
 
 /// Allow enable debug assertions in release profile with feature "strict_assertion".
diff --git a/foyer-intrusive/Cargo.toml b/foyer-intrusive/Cargo.toml
index 85ce93b5..6c6d0eff 100644
--- a/foyer-intrusive/Cargo.toml
+++ b/foyer-intrusive/Cargo.toml
@@ -18,3 +18,6 @@ itertools = { workspace = true }
 
 [features]
 strict_assertions = ["foyer-common/strict_assertions"]
+
+[lints]
+workspace = true
diff --git a/foyer-intrusive/src/lib.rs b/foyer-intrusive/src/lib.rs
index 4ec6cca3..b176003e 100644
--- a/foyer-intrusive/src/lib.rs
+++ b/foyer-intrusive/src/lib.rs
@@ -13,8 +13,6 @@
 //  limitations under the License.
 
 #![expect(clippy::new_without_default)]
-#![warn(missing_docs)]
-#![warn(clippy::allow_attributes)]
 
 //! Intrusive data structures and utils for foyer.
 
diff --git a/foyer-memory/Cargo.toml b/foyer-memory/Cargo.toml
index 2d116f3a..eab9cc62 100644
--- a/foyer-memory/Cargo.toml
+++ b/foyer-memory/Cargo.toml
@@ -53,3 +53,6 @@ harness = false
 [[bench]]
 name = "bench_dynamic_dispatch"
 harness = false
+
+[lints]
+workspace = true
diff --git a/foyer-memory/benches/bench_dynamic_dispatch.rs b/foyer-memory/benches/bench_dynamic_dispatch.rs
index 826643c5..3594effa 100644
--- a/foyer-memory/benches/bench_dynamic_dispatch.rs
+++ b/foyer-memory/benches/bench_dynamic_dispatch.rs
@@ -12,6 +12,8 @@
 //  See the License for the specific language governing permissions and
 //  limitations under the License.
 
+//! micro benchmark for dynamic dispatch
+
 use std::{
     sync::Arc,
     time::{Duration, Instant},
diff --git a/foyer-memory/benches/bench_hit_ratio.rs b/foyer-memory/benches/bench_hit_ratio.rs
index be1fdb13..42dbfb38 100644
--- a/foyer-memory/benches/bench_hit_ratio.rs
+++ b/foyer-memory/benches/bench_hit_ratio.rs
@@ -12,6 +12,8 @@
 //  See the License for the specific language governing permissions and
 //  limitations under the License.
 
+//! micro benchmark for foyer in-memory cache hit ratio
+
 use std::sync::Arc;
 
 use csv::Reader;
diff --git a/foyer-memory/src/lib.rs b/foyer-memory/src/lib.rs
index afcb3d14..b92cf831 100644
--- a/foyer-memory/src/lib.rs
+++ b/foyer-memory/src/lib.rs
@@ -62,9 +62,6 @@
 //! The handle that does not appear in either the indexer or the eviction container, and has no external owner, will be
 //! destroyed.
 
-#![warn(missing_docs)]
-#![warn(clippy::allow_attributes)]
-
 mod cache;
 mod context;
 mod eviction;
diff --git a/foyer-storage/src/lib.rs b/foyer-storage/src/lib.rs
index 7d5b2b6a..4d853fba 100644
--- a/foyer-storage/src/lib.rs
+++ b/foyer-storage/src/lib.rs
@@ -16,8 +16,6 @@
 
 #![cfg_attr(feature = "nightly", feature(allocator_api))]
 #![cfg_attr(feature = "nightly", feature(write_all_vectored))]
-#![warn(missing_docs)]
-#![warn(clippy::allow_attributes)]
 
 mod compress;
 mod device;
diff --git a/foyer/Cargo.toml b/foyer/Cargo.toml
index 3352170b..2d932cae 100644
--- a/foyer/Cargo.toml
+++ b/foyer/Cargo.toml
@@ -45,3 +45,6 @@ tracing = [
     "foyer-memory/tracing",
     "foyer-storage/tracing",
 ]
+
+[lints]
+workspace = true
diff --git a/foyer/src/lib.rs b/foyer/src/lib.rs
index d116d7e1..ecc1a3d4 100644
--- a/foyer/src/lib.rs
+++ b/foyer/src/lib.rs
@@ -13,8 +13,6 @@
 //  limitations under the License.
 
 #![cfg_attr(feature = "nightly", feature(allocator_api))]
-#![warn(missing_docs)]
-#![warn(clippy::allow_attributes)]
 
 //! A hybrid cache library that supports plug-and-play cache algorithms, in-memory cache and disk cache.
 //!

From 7b4474556062ad2b61771e9ffd0871a924ac56cd Mon Sep 17 00:00:00 2001
From: Croxx <mrcroxx@outlook.com>
Date: Mon, 28 Oct 2024 15:16:59 +0800
Subject: [PATCH 39/53] chore: update metrics and ot deps (#782)

Signed-off-by: MrCroxx <mrcroxx@outlook.com>
---
 Cargo.toml             | 2 +-
 examples/Cargo.toml    | 8 ++++----
 foyer-bench/Cargo.toml | 4 ++--
 3 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/Cargo.toml b/Cargo.toml
index 954908aa..e4de25c8 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -32,7 +32,7 @@ fastrace-jaeger = "0.7"
 fastrace-opentelemetry = "0.7"
 hashbrown = "0.14"
 itertools = "0.13"
-metrics = "0.23"
+metrics = "0.24"
 serde = { version = "1", features = ["derive", "rc"] }
 test-log = { version = "0.2", default-features = false, features = [
     "trace",
diff --git a/examples/Cargo.toml b/examples/Cargo.toml
index 3f732f81..42b574aa 100644
--- a/examples/Cargo.toml
+++ b/examples/Cargo.toml
@@ -22,10 +22,10 @@ fastrace = { workspace = true }
 fastrace-jaeger = { workspace = true, optional = true }
 fastrace-opentelemetry = { workspace = true, optional = true }
 foyer = { workspace = true }
-opentelemetry = { version = "0.25", optional = true }
-opentelemetry-otlp = { version = "0.25", optional = true }
-opentelemetry-semantic-conventions = { version = "0.25", optional = true }
-opentelemetry_sdk = { version = "0.25", features = [
+opentelemetry = { version = "0.26", optional = true }
+opentelemetry-otlp = { version = "0.26", optional = true }
+opentelemetry-semantic-conventions = { version = "0.26", optional = true }
+opentelemetry_sdk = { version = "0.26", features = [
     "rt-tokio",
     "trace",
 ], optional = true }
diff --git a/foyer-bench/Cargo.toml b/foyer-bench/Cargo.toml
index be44aba6..4a533437 100644
--- a/foyer-bench/Cargo.toml
+++ b/foyer-bench/Cargo.toml
@@ -25,14 +25,14 @@ hdrhistogram = "7"
 humantime = "2"
 itertools = { workspace = true }
 metrics = { workspace = true }
-metrics-exporter-prometheus = "0.15"
+metrics-exporter-prometheus = "0.16"
 parking_lot = "0.12"
 rand = "0.8.5"
 serde = { workspace = true }
 serde_bytes = "0.11.15"
 tokio = { workspace = true }
 tracing = "0.1"
-tracing-opentelemetry = { version = "0.26", optional = true }
+tracing-opentelemetry = { version = "0.27", optional = true }
 tracing-subscriber = { version = "0.3", features = ["env-filter"] }
 zipf = "7"
 

From 2e4844b99759511b6dc3141c1eb8b263c31e457b Mon Sep 17 00:00:00 2001
From: Croxx <mrcroxx@outlook.com>
Date: Thu, 31 Oct 2024 14:25:28 +0800
Subject: [PATCH 40/53] chore: add warning when region count is smaller than
 expected (#784)

Signed-off-by: MrCroxx <mrcroxx@outlook.com>
---
 foyer-storage/src/large/generic.rs | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/foyer-storage/src/large/generic.rs b/foyer-storage/src/large/generic.rs
index 047d96a7..35b4205a 100644
--- a/foyer-storage/src/large/generic.rs
+++ b/foyer-storage/src/large/generic.rs
@@ -183,6 +183,14 @@ where
     S: HashBuilder + Debug,
 {
     async fn open(mut config: GenericLargeStorageConfig<K, V, S>) -> Result<Self> {
+        if config.flushers + config.clean_region_threshold > config.device.regions() / 2 {
+            tracing::warn!("[lodc]: large object disk cache stable regions count is too small, flusher [{flushers}] + clean region threshold [{clean_region_threshold}] (default = reclaimers) is supposed to be much larger than the region count [{regions}]",
+                flushers = config.flushers,
+                clean_region_threshold = config.clean_region_threshold,
+                regions = config.device.regions()
+            );
+        }
+
         let stats = config.statistics.clone();
 
         let device = config.device.clone();

From 66a1d91259064b0fbe83e94065fe433a45ee4f32 Mon Sep 17 00:00:00 2001
From: Croxx <mrcroxx@outlook.com>
Date: Tue, 19 Nov 2024 13:53:26 +0800
Subject: [PATCH 41/53] feat: simplify foyer in-memory cache framework (#785)

* feat: optimize lock critical section

- Cloning and dropping cache entry no longer requires locking (ref > 0).
- Use rwlock for cache eviction algorighm without mutable acquire op.
- Use slab to replace object pool optimization.
- Refine abstractions.

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

* feat: adapt lru impl to new framework

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

* feat: adapt s3fifo to the new framework

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

* feat: print log when choosing between rwlock & mutex

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

* feat: adapt lfu to the new framework

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

* test: adapt eviction algorithm tests into new framework

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

* refactor: refine the generic types to support default type

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

* feat: adapt unified cache abstraction to the new framewrok, fix bugs

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

* refactor: refine some imports and lints

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

* refactor: make clippy happy

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

* refactor: bump msrv

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

* refactor: update nightly toolchain, temp reduce unused deps

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

* fix: fix release reinsertion

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

* fix: fix atomic reference count and reclamation with CAS

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

* fix: reset the atomic ref count when reclaimer gives up reclamation

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

* feat: replace the in-memory cache with the new framework

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

* refactor: port the benches, remove backups

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

* chore: udeps

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

* refactor: derice traits for cache hint

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

* refactor: and more

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

* refactor: remove slab

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

* chore: move crossbeam deps to workspace

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

* fix: typo

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

* feat: introduce dec ref slow path [MAY PANIC!]

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

* perf: use per shard object pool to optimize memory allocation

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

* feat: simplify foyer in-memory cache framework

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

* chore: make ffmt happy

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

* chore: make ffmt happy

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

* chore: remove unused comments and deps, format code

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

* chore: remove unused code

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

* chore: remove unused components

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

* docs: update rust docs

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

* chore: update changelog

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

---------

Signed-off-by: MrCroxx <mrcroxx@outlook.com>
---
 .github/workflows/ci.yml                      |    4 +-
 CHANGELOG.md                                  |   10 +
 Cargo.toml                                    |   17 +-
 Makefile                                      |   18 +-
 README.md                                     |    2 +-
 codecov.yml                                   |    1 -
 examples/event_listener.rs                    |   10 +-
 examples/hybrid_full.rs                       |    1 -
 foyer-bench/Cargo.toml                        |    2 +-
 foyer-common/Cargo.toml                       |    3 +-
 foyer-common/src/event.rs                     |   18 +-
 foyer-common/src/lib.rs                       |    6 +-
 foyer-common/src/object_pool.rs               |  106 --
 .../src/lib.rs => foyer-common/src/option.rs  |   31 +-
 foyer-common/src/scope.rs                     |   42 +
 foyer-intrusive/Cargo.toml                    |   23 -
 foyer-intrusive/src/adapter.rs                |  186 ---
 foyer-intrusive/src/dlist.rs                  |  586 -------
 foyer-intrusive/src/lib.rs                    |   46 -
 foyer-memory/Cargo.toml                       |   14 +-
 foyer-memory/benches/bench_hit_ratio.rs       |    7 +-
 foyer-memory/src/cache.rs                     |  261 ++-
 foyer-memory/src/context.rs                   |   37 -
 foyer-memory/src/eviction/fifo.rs             |  279 ++--
 foyer-memory/src/eviction/lfu.rs              |  735 +++++----
 foyer-memory/src/eviction/lru.rs              |  653 ++++----
 foyer-memory/src/eviction/mod.rs              |  145 +-
 foyer-memory/src/eviction/s3fifo.rs           |  572 +++----
 foyer-memory/src/eviction/sanity.rs           |  152 --
 foyer-memory/src/eviction/test_utils.rs       |   28 +-
 foyer-memory/src/generic.rs                   | 1403 -----------------
 foyer-memory/src/handle.rs                    |  263 ---
 foyer-memory/src/indexer/hash_table.rs        |   99 +-
 foyer-memory/src/indexer/mod.rs               |   25 +-
 foyer-memory/src/indexer/sanity.rs            |  108 --
 foyer-memory/src/indexer/sentry.rs            |   80 +
 foyer-memory/src/lib.rs                       |   46 +-
 foyer-memory/src/prelude.rs                   |    4 +-
 foyer-memory/src/raw.rs                       | 1020 ++++++++++++
 foyer-memory/src/record.rs                    |  247 +++
 foyer-storage/Cargo.toml                      |    2 +-
 foyer-storage/src/large/generic.rs            |    1 +
 foyer-storage/src/small/batch.rs              |    8 +-
 foyer-storage/src/small/bloom_filter.rs       |    2 +
 foyer-storage/src/small/generic.rs            |   20 +-
 foyer-storage/src/small/set_manager.rs        |    3 +
 foyer-storage/src/test_utils.rs               |   28 +-
 foyer-util/Cargo.toml                         |   34 -
 foyer-util/src/async_batch_pipeline.rs        |  193 ---
 foyer-util/src/batch.rs                       |   72 -
 foyer-util/src/compact_bloom_filter.rs        |  196 ---
 foyer-util/src/continuum.rs                   |  251 ---
 foyer-util/src/erwlock.rs                     |   63 -
 foyer-util/src/iostat.rs                      |  172 --
 foyer-util/src/judge.rs                       |   97 --
 foyer-util/src/slab/mod.rs                    |  172 --
 foyer-util/src/slab/slab_linked_list/mod.rs   |  432 -----
 foyer-util/src/slab/slab_linked_list/tests.rs |   58 -
 foyer-util/src/slab/tests.rs                  |   44 -
 foyer/src/hybrid/builder.rs                   |   14 -
 foyer/src/hybrid/cache.rs                     |   28 +-
 foyer/src/hybrid/writer.rs                    |   18 +-
 foyer/src/prelude.rs                          |    4 +-
 63 files changed, 3024 insertions(+), 6178 deletions(-)
 delete mode 100644 foyer-common/src/object_pool.rs
 rename foyer-util/src/lib.rs => foyer-common/src/option.rs (57%)
 create mode 100644 foyer-common/src/scope.rs
 delete mode 100644 foyer-intrusive/Cargo.toml
 delete mode 100644 foyer-intrusive/src/adapter.rs
 delete mode 100644 foyer-intrusive/src/dlist.rs
 delete mode 100644 foyer-intrusive/src/lib.rs
 delete mode 100644 foyer-memory/src/context.rs
 delete mode 100644 foyer-memory/src/eviction/sanity.rs
 delete mode 100644 foyer-memory/src/generic.rs
 delete mode 100644 foyer-memory/src/handle.rs
 delete mode 100644 foyer-memory/src/indexer/sanity.rs
 create mode 100644 foyer-memory/src/indexer/sentry.rs
 create mode 100644 foyer-memory/src/raw.rs
 create mode 100644 foyer-memory/src/record.rs
 delete mode 100644 foyer-util/Cargo.toml
 delete mode 100644 foyer-util/src/async_batch_pipeline.rs
 delete mode 100644 foyer-util/src/batch.rs
 delete mode 100644 foyer-util/src/compact_bloom_filter.rs
 delete mode 100644 foyer-util/src/continuum.rs
 delete mode 100644 foyer-util/src/erwlock.rs
 delete mode 100644 foyer-util/src/iostat.rs
 delete mode 100644 foyer-util/src/judge.rs
 delete mode 100644 foyer-util/src/slab/mod.rs
 delete mode 100644 foyer-util/src/slab/slab_linked_list/mod.rs
 delete mode 100644 foyer-util/src/slab/slab_linked_list/tests.rs
 delete mode 100644 foyer-util/src/slab/tests.rs

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 46dde79f..2457bd87 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -15,7 +15,7 @@ concurrency:
     cancel-in-progress: ${{ github.event_name == 'pull_request' }}
 
 env:
-  RUST_TOOLCHAIN_NIGHTLY: nightly-2024-07-19
+  RUST_TOOLCHAIN_NIGHTLY: nightly-2024-08-30
   CARGO_TERM_COLOR: always
   CACHE_KEY_SUFFIX: 20240821
 
@@ -122,7 +122,7 @@ jobs:
       fail-fast: false
       matrix:
         os: [ubuntu-latest, macos-latest, windows-latest]
-        rust_toolchain: [stable, 1.81.0]
+        rust_toolchain: [stable, 1.82.0]
     runs-on: ${{ matrix.os }}
     steps:
       - name: Checkout
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 31b8ee31..448e5470 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -9,6 +9,16 @@ date: 2023-05-12T11:02:09+08:00
 
 <!-- truncate -->
 
+## Unreleased
+
+### Changes
+
+- Refine in-memory cache framework:
+  - Allow "get"/"release"/"entry drop" to acquire read lock or lock-free if the algorithm allows.
+  - Make most `Eviction` APIs safe, only acquire unsafe Rust while accessing algorithm managed per-entry state with `UnsafeCell`.
+  - Replace the "reinsertion" design with `release` with real "release last ref" design.
+  - Rename some APIs.
+
 ## 2024-10-11
 
 ### Releases
diff --git a/Cargo.toml b/Cargo.toml
index e4de25c8..b2d2a350 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -6,16 +6,14 @@ members = [
     "foyer-bench",
     "foyer-cli",
     "foyer-common",
-    "foyer-intrusive",
     "foyer-memory",
     "foyer-storage",
-    "foyer-util",
 ]
 
 [workspace.package]
-version = "0.12.2"
+version = "0.13.0-dev"
 edition = "2021"
-rust-version = "1.81.0"
+rust-version = "1.82.0"
 repository = "https://github.com/foyer-rs/foyer"
 homepage = "https://foyer.rs"
 keywords = ["cache", "hybrid"]
@@ -26,6 +24,7 @@ readme = "README.md"
 [workspace.dependencies]
 bytesize = { package = "foyer-bytesize", version = "2" }
 clap = { version = "4", features = ["derive"] }
+crossbeam = "0.8"
 equivalent = "1"
 fastrace = "0.7"
 fastrace-jaeger = "0.7"
@@ -33,6 +32,7 @@ fastrace-opentelemetry = "0.7"
 hashbrown = "0.14"
 itertools = "0.13"
 metrics = "0.24"
+parking_lot = { version = "0.12" }
 serde = { version = "1", features = ["derive", "rc"] }
 test-log = { version = "0.2", default-features = false, features = [
     "trace",
@@ -49,11 +49,10 @@ tokio = { package = "madsim-tokio", version = "0.2", features = [
 ] }
 
 # foyer components
-foyer-common = { version = "0.12.2", path = "foyer-common" }
-foyer-intrusive = { version = "0.12.2", path = "foyer-intrusive" }
-foyer-memory = { version = "0.12.2", path = "foyer-memory" }
-foyer-storage = { version = "0.12.2", path = "foyer-storage" }
-foyer = { version = "0.12.2", path = "foyer" }
+foyer-common = { version = "0.13.0-dev", path = "foyer-common" }
+foyer-memory = { version = "0.13.0-dev", path = "foyer-memory" }
+foyer-storage = { version = "0.13.0-dev", path = "foyer-storage" }
+foyer = { version = "0.13.0-dev", path = "foyer" }
 
 [workspace.lints.rust]
 missing_docs = "warn"
diff --git a/Makefile b/Makefile
index c33e2b34..0b093b9a 100644
--- a/Makefile
+++ b/Makefile
@@ -52,17 +52,17 @@ fast: check test example
 msrv:
 	shellcheck ./scripts/*
 	./scripts/minimize-dashboards.sh
-	cargo +1.81.0 sort -w
-	cargo +1.81.0 fmt --all
-	cargo +1.81.0 clippy --all-targets --features deadlock
-	cargo +1.81.0 clippy --all-targets --features tokio-console
-	cargo +1.81.0 clippy --all-targets
-	RUST_BACKTRACE=1 cargo +1.81.0 nextest run --all
-	RUST_BACKTRACE=1 cargo +1.81.0 test --doc
-	RUST_BACKTRACE=1 cargo +1.81.0 nextest run --run-ignored ignored-only --no-capture --workspace
+	cargo +1.82.0 sort -w
+	cargo +1.82.0 fmt --all
+	cargo +1.82.0 clippy --all-targets --features deadlock
+	cargo +1.82.0 clippy --all-targets --features tokio-console
+	cargo +1.82.0 clippy --all-targets
+	RUST_BACKTRACE=1 cargo +1.82.0 nextest run --all
+	RUST_BACKTRACE=1 cargo +1.82.0 test --doc
+	RUST_BACKTRACE=1 cargo +1.82.0 nextest run --run-ignored ignored-only --no-capture --workspace
 
 udeps:
-	RUSTFLAGS="--cfg tokio_unstable -Awarnings" cargo +nightly-2024-07-19 udeps --all-targets
+	RUSTFLAGS="--cfg tokio_unstable -Awarnings" cargo +nightly-2024-08-30 udeps --all-targets
 
 monitor:
 	./scripts/monitor.sh
diff --git a/README.md b/README.md
index 5d97d0ed..ac424405 100644
--- a/README.md
+++ b/README.md
@@ -222,7 +222,7 @@ More examples and details can be found [here](https://github.com/foyer-rs/foyer/
 
 ## Supported Rust Versions
 
-*foyer* is built against the recent stable release. The minimum supported version is 1.81.0. The current *foyer* version is not guaranteed to build on Rust versions earlier than the minimum supported version.
+*foyer* is built against the recent stable release. The minimum supported version is 1.82.0. The current *foyer* version is not guaranteed to build on Rust versions earlier than the minimum supported version.
 
 ## Supported Platforms
 
diff --git a/codecov.yml b/codecov.yml
index 909bbf9d..f020ff7b 100644
--- a/codecov.yml
+++ b/codecov.yml
@@ -21,5 +21,4 @@ coverage:
         informational: true
         only_pulls: true
 ignore:
-  - "foyer-util"
   - "foyer-cli"
\ No newline at end of file
diff --git a/examples/event_listener.rs b/examples/event_listener.rs
index f7de1674..91feefbb 100644
--- a/examples/event_listener.rs
+++ b/examples/event_listener.rs
@@ -14,7 +14,7 @@
 
 use std::sync::Arc;
 
-use foyer::{Cache, CacheBuilder, EventListener, FifoConfig};
+use foyer::{Cache, CacheBuilder, Event, EventListener, FifoConfig};
 
 struct EchoEventListener;
 
@@ -22,11 +22,7 @@ impl EventListener for EchoEventListener {
     type Key = u64;
     type Value = String;
 
-    fn on_memory_release(&self, key: Self::Key, value: Self::Value)
-    where
-        Self::Key: foyer::Key,
-        Self::Value: foyer::Value,
-    {
+    fn on_leave(&self, _reason: Event, key: &Self::Key, value: &Self::Value) {
         println!("Entry [key = {key}] [value = {value}] is released.")
     }
 }
@@ -47,7 +43,7 @@ fn main() {
         .build();
 
     cache.insert(1, "Second".to_string());
-    cache.deposit(2, "First".to_string());
+    cache.insert_ephemeral(2, "First".to_string());
     cache.insert(3, "Third".to_string());
     cache.insert(3, "Forth".to_string());
 }
diff --git a/examples/hybrid_full.rs b/examples/hybrid_full.rs
index aba9510f..d02f2b8e 100644
--- a/examples/hybrid_full.rs
+++ b/examples/hybrid_full.rs
@@ -32,7 +32,6 @@ async fn main() -> Result<()> {
         .with_eviction_config(LruConfig {
             high_priority_pool_ratio: 0.1,
         })
-        .with_object_pool_capacity(1024)
         .with_hash_builder(ahash::RandomState::default())
         .with_weighter(|_key, value: &String| value.len())
         .storage(Engine::Mixed(0.1))
diff --git a/foyer-bench/Cargo.toml b/foyer-bench/Cargo.toml
index 4a533437..4ca43c11 100644
--- a/foyer-bench/Cargo.toml
+++ b/foyer-bench/Cargo.toml
@@ -26,7 +26,7 @@ humantime = "2"
 itertools = { workspace = true }
 metrics = { workspace = true }
 metrics-exporter-prometheus = "0.16"
-parking_lot = "0.12"
+parking_lot = { workspace = true }
 rand = "0.8.5"
 serde = { workspace = true }
 serde_bytes = "0.11.15"
diff --git a/foyer-common/Cargo.toml b/foyer-common/Cargo.toml
index 5eee4d06..8b1a99f1 100644
--- a/foyer-common/Cargo.toml
+++ b/foyer-common/Cargo.toml
@@ -15,13 +15,12 @@ readme = { workspace = true }
 [dependencies]
 bytes = "1"
 cfg-if = "1"
-crossbeam = "0.8"
 fastrace = { workspace = true }
 futures = "0.3"
 hashbrown = { workspace = true }
 itertools = { workspace = true }
 metrics = { workspace = true }
-parking_lot = { version = "0.12", features = ["arc_lock"] }
+parking_lot = { workspace = true }
 pin-project = "1"
 serde = { workspace = true }
 tokio = { workspace = true }
diff --git a/foyer-common/src/event.rs b/foyer-common/src/event.rs
index f6d17070..03d2c2ca 100644
--- a/foyer-common/src/event.rs
+++ b/foyer-common/src/event.rs
@@ -14,17 +14,29 @@
 
 use crate::code::{Key, Value};
 
-/// Trait for the customized event listener.
+/// Event identifier.
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum Event {
+    /// Cache eviction on insertion.
+    Evict,
+    /// Cache replacement on insertion.
+    Replace,
+    /// Cache remove.
+    Remove,
+    /// Cache clear.
+    Clear,
+}
 
+/// Trait for the customized event listener.
 pub trait EventListener: Send + Sync + 'static {
     /// Associated key type.
     type Key;
     /// Associated value type.
     type Value;
 
-    /// Called when a cache entry is released from the in-memory cache.
+    /// Called when a cache entry leaves the in-memory cache with the reason.
     #[expect(unused_variables)]
-    fn on_memory_release(&self, key: Self::Key, value: Self::Value)
+    fn on_leave(&self, reason: Event, key: &Self::Key, value: &Self::Value)
     where
         Self::Key: Key,
         Self::Value: Value,
diff --git a/foyer-common/src/lib.rs b/foyer-common/src/lib.rs
index 98356144..55383fb5 100644
--- a/foyer-common/src/lib.rs
+++ b/foyer-common/src/lib.rs
@@ -32,8 +32,8 @@ pub mod event;
 pub mod future;
 /// The shared metrics for foyer.
 pub mod metrics;
-/// A concurrent object pool.
-pub mod object_pool;
+/// Extensions for [`std::option::Option`].
+pub mod option;
 /// The range extensions.
 pub mod range;
 /// A rate limiter that returns the wait duration for limitation.
@@ -42,6 +42,8 @@ pub mod rate;
 pub mod rated_ticket;
 /// A runtime that automatically shutdown itself on drop.
 pub mod runtime;
+/// A kotlin like functional programming helper.
+pub mod scope;
 /// Tracing related components.
 pub mod tracing;
 /// An async wait group implementation.
diff --git a/foyer-common/src/object_pool.rs b/foyer-common/src/object_pool.rs
deleted file mode 100644
index 9e4d6048..00000000
--- a/foyer-common/src/object_pool.rs
+++ /dev/null
@@ -1,106 +0,0 @@
-//  Copyright 2024 foyer Project Authors
-//
-//  Licensed under the Apache License, Version 2.0 (the "License");
-//  you may not use this file except in compliance with the License.
-//  You may obtain a copy of the License at
-//
-//  http://www.apache.org/licenses/LICENSE-2.0
-//
-//  Unless required by applicable law or agreed to in writing, software
-//  distributed under the License is distributed on an "AS IS" BASIS,
-//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-//  See the License for the specific language governing permissions and
-//  limitations under the License.
-
-use std::sync::Arc;
-
-use crossbeam::queue::ArrayQueue;
-
-/// A concurrent object pool.
-pub struct ObjectPool<T> {
-    inner: Arc<ObjectPoolInner<T>>,
-}
-
-impl<T> Clone for ObjectPool<T> {
-    fn clone(&self) -> Self {
-        Self {
-            inner: self.inner.clone(),
-        }
-    }
-}
-
-struct ObjectPoolInner<T> {
-    queue: Option<ArrayQueue<T>>,
-    create: Box<dyn Fn() -> T + Send + Sync + 'static>,
-}
-
-impl<T> ObjectPool<T>
-where
-    T: Default + 'static,
-{
-    /// Create a new concurrent object pool.
-    pub fn new(capacity: usize) -> Self {
-        Self::new_with_create(capacity, T::default)
-    }
-}
-
-impl<T> ObjectPool<T> {
-    /// Create a new concurrent object pool with object creation method.
-    pub fn new_with_create(capacity: usize, create: impl Fn() -> T + Send + Sync + 'static) -> Self {
-        let inner = ObjectPoolInner {
-            queue: if capacity == 0 {
-                None
-            } else {
-                Some(ArrayQueue::new(capacity))
-            },
-            create: Box::new(create),
-        };
-        Self { inner: Arc::new(inner) }
-    }
-
-    /// Get or create an object from the object pool.
-    pub fn acquire(&self) -> T {
-        match self.inner.queue.as_ref() {
-            Some(queue) => queue.pop().unwrap_or((self.inner.create)()),
-            None => (self.inner.create)(),
-        }
-    }
-
-    /// Give back or release an object from the object pool.
-    pub fn release(&self, item: T) {
-        if let Some(queue) = self.inner.queue.as_ref() {
-            let _ = queue.push(item);
-        }
-    }
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-
-    fn case(capacity: usize) {
-        let pool: ObjectPool<usize> = ObjectPool::new(capacity);
-
-        for i in 0..capacity * 2 {
-            pool.release(i);
-        }
-
-        for i in 0..capacity {
-            assert_eq!(pool.acquire(), i);
-        }
-
-        for _ in 0..capacity {
-            assert_eq!(pool.acquire(), 0);
-        }
-    }
-
-    #[test]
-    fn test_object_pool() {
-        case(100);
-    }
-
-    #[test]
-    fn test_object_pool_zero() {
-        case(0);
-    }
-}
diff --git a/foyer-util/src/lib.rs b/foyer-common/src/option.rs
similarity index 57%
rename from foyer-util/src/lib.rs
rename to foyer-common/src/option.rs
index 7136494c..a1ea2bcd 100644
--- a/foyer-util/src/lib.rs
+++ b/foyer-common/src/option.rs
@@ -12,15 +12,26 @@
 //  See the License for the specific language governing permissions and
 //  limitations under the License.
 
-#![warn(clippy::allow_attributes)]
+/// Extension for [`std::option::Option`].
+pub trait OptionExt {
+    /// Wrapped type by [`Option`].
+    type Val;
 
-pub mod batch;
-pub mod compact_bloom_filter;
-pub mod continuum;
-pub mod erwlock;
-pub mod iostat;
-pub mod judge;
-pub mod slab;
+    /// Consume the wrapped value with the given function if there is.
+    fn then<F>(self, f: F)
+    where
+        F: FnOnce(Self::Val);
+}
 
-/// A structured async batch pipeline.
-pub mod async_batch_pipeline;
+impl<T> OptionExt for Option<T> {
+    type Val = T;
+
+    fn then<F>(self, f: F)
+    where
+        F: FnOnce(Self::Val),
+    {
+        if let Some(val) = self {
+            f(val)
+        }
+    }
+}
diff --git a/foyer-common/src/scope.rs b/foyer-common/src/scope.rs
new file mode 100644
index 00000000..da2cb94e
--- /dev/null
+++ b/foyer-common/src/scope.rs
@@ -0,0 +1,42 @@
+//  Copyright 2024 foyer Project Authors
+//
+//  Licensed under the Apache License, Version 2.0 (the "License");
+//  you may not use this file except in compliance with the License.
+//  You may obtain a copy of the License at
+//
+//  http://www.apache.org/licenses/LICENSE-2.0
+//
+//  Unless required by applicable law or agreed to in writing, software
+//  distributed under the License is distributed on an "AS IS" BASIS,
+//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//  See the License for the specific language governing permissions and
+//  limitations under the License.
+
+/// Scoped functional programming extensions.
+pub trait Scope {
+    /// Scoped with ownership.
+    fn with<F, R>(self, f: F) -> R
+    where
+        Self: Sized,
+        F: FnOnce(Self) -> R,
+    {
+        f(self)
+    }
+
+    /// Scoped with reference.
+    fn with_ref<F, R>(&self, f: F) -> R
+    where
+        F: FnOnce(&Self) -> R,
+    {
+        f(self)
+    }
+
+    /// Scoped with mutable reference.
+    fn with_mut<F, R>(&mut self, f: F) -> R
+    where
+        F: FnOnce(&mut Self) -> R,
+    {
+        f(self)
+    }
+}
+impl<T> Scope for T {}
diff --git a/foyer-intrusive/Cargo.toml b/foyer-intrusive/Cargo.toml
deleted file mode 100644
index 6c6d0eff..00000000
--- a/foyer-intrusive/Cargo.toml
+++ /dev/null
@@ -1,23 +0,0 @@
-[package]
-name = "foyer-intrusive"
-description = "intrusive data structures for foyer - Hybrid cache for Rust"
-version = { workspace = true }
-edition = { workspace = true }
-rust-version = { workspace = true }
-repository = { workspace = true }
-homepage = { workspace = true }
-keywords = { workspace = true }
-authors = { workspace = true }
-license = { workspace = true }
-readme = { workspace = true }
-# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
-
-[dependencies]
-foyer-common = { workspace = true }
-itertools = { workspace = true }
-
-[features]
-strict_assertions = ["foyer-common/strict_assertions"]
-
-[lints]
-workspace = true
diff --git a/foyer-intrusive/src/adapter.rs b/foyer-intrusive/src/adapter.rs
deleted file mode 100644
index 7a6a957a..00000000
--- a/foyer-intrusive/src/adapter.rs
+++ /dev/null
@@ -1,186 +0,0 @@
-//  Copyright 2024 foyer Project Authors
-//
-//  Licensed under the Apache License, Version 2.0 (the "License");
-//  you may not use this file except in compliance with the License.
-//  You may obtain a copy of the License at
-//
-//  http://www.apache.org/licenses/LICENSE-2.0
-//
-//  Unless required by applicable law or agreed to in writing, software
-//  distributed under the License is distributed on an "AS IS" BASIS,
-//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-//  See the License for the specific language governing permissions and
-//  limitations under the License.
-
-//  Copyright 2020 Amari Robinson
-//
-//  Licensed under the Apache License, Version 2.0 (the "License");
-//  you may not use this file except in compliance with the License.
-//  You may obtain a copy of the License at
-//
-//  http://www.apache.org/licenses/LICENSE-2.0
-//
-//  Unless required by applicable law or agreed to in writing, software
-//  distributed under the License is distributed on an "AS IS" BASIS,
-//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-//  See the License for the specific language governing permissions and
-//  limitations under the License.
-
-//! Intrusive data structure adapter that locates between pointer and item.
-
-use std::fmt::Debug;
-
-/// Intrusive data structure link.
-pub trait Link: Send + Sync + 'static + Default + Debug {
-    /// Check if the link is linked by the intrusive data structure.
-    fn is_linked(&self) -> bool;
-}
-
-/// Intrusive data structure adapter.
-///
-/// # Safety
-///
-/// Pointer operations MUST be valid.
-///
-/// [`Adapter`] is recommended to be generated by macro `intrusive_adapter!`.
-pub unsafe trait Adapter: Send + Sync + Debug + 'static {
-    /// Item type for the adapter.
-    type Item: ?Sized;
-    /// Link type for the adapter.
-    type Link: Link;
-
-    /// Create a new intrusive data structure link.
-    fn new() -> Self;
-
-    /// # Safety
-    ///
-    /// Pointer operations MUST be valid.
-    unsafe fn link2ptr(&self, link: std::ptr::NonNull<Self::Link>) -> std::ptr::NonNull<Self::Item>;
-
-    /// # Safety
-    ///
-    /// Pointer operations MUST be valid.
-    unsafe fn ptr2link(&self, item: std::ptr::NonNull<Self::Item>) -> std::ptr::NonNull<Self::Link>;
-}
-
-/// Macro to generate an implementation of [`Adapter`] for intrusive container and items.
-///
-/// The basic syntax to create an adapter is:
-///
-/// ```rust,ignore
-/// intrusive_adapter! { Adapter = Pointer: Item { link_field: LinkType } }
-/// ```
-///
-/// # Generics
-///
-/// This macro supports generic arguments:
-///
-/// Note that due to macro parsing limitations, `T: Trait` bounds are not
-/// supported in the generic argument list. You must list any trait bounds in
-/// a separate `where` clause at the end of the macro.
-///
-/// # Examples
-///
-/// ```
-/// use foyer_intrusive::intrusive_adapter;
-/// use foyer_intrusive::adapter::Link;
-///
-/// #[derive(Debug)]
-/// pub struct Item<L>
-/// where
-///     L: Link
-/// {
-///     link: L,
-///     key: u64,
-/// }
-///
-/// intrusive_adapter! { ItemAdapter<L> = Item<L> { link: L } where L: Link }
-/// ```
-#[macro_export]
-macro_rules! intrusive_adapter {
-    (@impl
-        $vis:vis $name:ident ($($args:tt),*) = $item:ty { $field:ident: $link:ty } $($where_:tt)*
-    ) => {
-        $vis struct $name<$($args),*> $($where_)* {
-            _marker: std::marker::PhantomData<($($args),*)>
-        }
-
-        unsafe impl<$($args),*> Send for $name<$($args),*> $($where_)* {}
-        unsafe impl<$($args),*> Sync for $name<$($args),*> $($where_)* {}
-
-        unsafe impl<$($args),*> $crate::adapter::Adapter for $name<$($args),*> $($where_)*{
-            type Item = $item;
-            type Link = $link;
-
-            fn new() -> Self {
-                Self {
-                    _marker: std::marker::PhantomData,
-                }
-            }
-
-            unsafe fn link2ptr(&self, link: std::ptr::NonNull<Self::Link>) -> std::ptr::NonNull<Self::Item> {
-                std::ptr::NonNull::new_unchecked($crate::container_of!(link.as_ptr(), $item, $field))
-            }
-
-            unsafe fn ptr2link(&self, item: std::ptr::NonNull<Self::Item>) -> std::ptr::NonNull<Self::Link> {
-                std::ptr::NonNull::new_unchecked((item.as_ptr() as *mut u8).add(std::mem::offset_of!($item, $field)) as *mut Self::Link)
-            }
-        }
-
-        impl<$($args),*> std::fmt::Debug for $name<$($args),*> $($where_)*{
-            fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> Result<(), std::fmt::Error> {
-                f.debug_struct(stringify!($name)).finish()
-            }
-        }
-    };
-    (
-        $vis:vis $name:ident = $($rest:tt)*
-    ) => {
-        intrusive_adapter! {@impl
-            $vis $name () = $($rest)*
-        }
-    };
-    (
-        $vis:vis $name:ident<$($args:tt),*> = $($rest:tt)*
-    ) => {
-        intrusive_adapter! {@impl
-            $vis $name ($($args),*) = $($rest)*
-        }
-    };
-}
-
-#[cfg(test)]
-mod tests {
-    use std::ptr::NonNull;
-
-    use itertools::Itertools;
-
-    use super::*;
-    use crate::{dlist::*, intrusive_adapter};
-
-    #[derive(Debug)]
-    struct DlistItem {
-        link: DlistLink,
-        val: u64,
-    }
-
-    impl DlistItem {
-        fn new(val: u64) -> Self {
-            Self {
-                link: DlistLink::default(),
-                val,
-            }
-        }
-    }
-
-    intrusive_adapter! { DlistItemAdapter = DlistItem { link: DlistLink }}
-
-    #[test]
-    fn test_adapter_macro() {
-        let mut l = Dlist::<DlistItemAdapter>::new();
-        l.push_front(unsafe { NonNull::new_unchecked(Box::into_raw(Box::new(DlistItem::new(1)))) });
-        let v = l.iter().map(|item| item.val).collect_vec();
-        assert_eq!(v, vec![1]);
-        let _ = unsafe { Box::from_raw(l.pop_front().unwrap().as_ptr()) };
-    }
-}
diff --git a/foyer-intrusive/src/dlist.rs b/foyer-intrusive/src/dlist.rs
deleted file mode 100644
index 04ecea48..00000000
--- a/foyer-intrusive/src/dlist.rs
+++ /dev/null
@@ -1,586 +0,0 @@
-//  Copyright 2024 foyer Project Authors
-//
-//  Licensed under the Apache License, Version 2.0 (the "License");
-//  you may not use this file except in compliance with the License.
-//  You may obtain a copy of the License at
-//
-//  http://www.apache.org/licenses/LICENSE-2.0
-//
-//  Unless required by applicable law or agreed to in writing, software
-//  distributed under the License is distributed on an "AS IS" BASIS,
-//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-//  See the License for the specific language governing permissions and
-//  limitations under the License.
-
-//! An intrusive double linked list implementation.
-
-use std::ptr::NonNull;
-
-use foyer_common::{assert::OptionExt, strict_assert};
-
-use crate::adapter::{Adapter, Link};
-
-/// The link for the intrusive double linked list.
-#[derive(Debug, Default)]
-pub struct DlistLink {
-    prev: Option<NonNull<DlistLink>>,
-    next: Option<NonNull<DlistLink>>,
-    is_linked: bool,
-}
-
-impl DlistLink {
-    /// Get the `NonNull` pointer of the link.
-    pub fn raw(&mut self) -> NonNull<DlistLink> {
-        unsafe { NonNull::new_unchecked(self as *mut _) }
-    }
-
-    /// Get the pointer of the prev link.
-    pub fn prev(&self) -> Option<NonNull<DlistLink>> {
-        self.prev
-    }
-
-    /// Get the pointer of the next link.
-    pub fn next(&self) -> Option<NonNull<DlistLink>> {
-        self.next
-    }
-}
-
-unsafe impl Send for DlistLink {}
-unsafe impl Sync for DlistLink {}
-
-impl Link for DlistLink {
-    fn is_linked(&self) -> bool {
-        self.is_linked
-    }
-}
-
-/// Intrusive double linked list.
-#[derive(Debug)]
-pub struct Dlist<A>
-where
-    A: Adapter<Link = DlistLink>,
-{
-    head: Option<NonNull<DlistLink>>,
-    tail: Option<NonNull<DlistLink>>,
-
-    len: usize,
-
-    adapter: A,
-}
-
-impl<A> Drop for Dlist<A>
-where
-    A: Adapter<Link = DlistLink>,
-{
-    fn drop(&mut self) {
-        let mut iter = self.iter_mut();
-        iter.front();
-        while iter.is_valid() {
-            iter.remove();
-        }
-        assert!(self.is_empty());
-    }
-}
-
-impl<A> Dlist<A>
-where
-    A: Adapter<Link = DlistLink>,
-{
-    /// Create a new intrusive double linked list.
-    pub fn new() -> Self {
-        Self {
-            head: None,
-            tail: None,
-            len: 0,
-
-            adapter: A::new(),
-        }
-    }
-
-    /// Get the reference of the first item of the intrusive double linked list.
-    pub fn front(&self) -> Option<&A::Item> {
-        unsafe { self.head.map(|link| self.adapter.link2ptr(link).as_ref()) }
-    }
-
-    /// Get the reference of the last item of the intrusive double linked list.
-    pub fn back(&self) -> Option<&A::Item> {
-        unsafe { self.tail.map(|link| self.adapter.link2ptr(link).as_ref()) }
-    }
-
-    /// Get the mutable reference of the first item of the intrusive double linked list.
-    pub fn front_mut(&mut self) -> Option<&mut A::Item> {
-        unsafe { self.head.map(|link| self.adapter.link2ptr(link).as_mut()) }
-    }
-
-    /// Get the mutable reference of the last item of the intrusive double linked list.
-    pub fn back_mut(&mut self) -> Option<&mut A::Item> {
-        unsafe { self.tail.map(|link| self.adapter.link2ptr(link).as_mut()) }
-    }
-
-    /// Push an item to the first position of the intrusive double linked list.
-    pub fn push_front(&mut self, ptr: NonNull<A::Item>) {
-        self.iter_mut().insert_after(ptr);
-    }
-
-    /// Push an item to the last position of the intrusive double linked list.
-    pub fn push_back(&mut self, ptr: NonNull<A::Item>) {
-        self.iter_mut().insert_before(ptr);
-    }
-
-    /// Pop an item from the first position of the intrusive double linked list.
-    pub fn pop_front(&mut self) -> Option<NonNull<A::Item>> {
-        let mut iter = self.iter_mut();
-        iter.next();
-        iter.remove()
-    }
-
-    /// Pop an item from the last position of the intrusive double linked list.
-    pub fn pop_back(&mut self) -> Option<NonNull<A::Item>> {
-        let mut iter = self.iter_mut();
-        iter.prev();
-        iter.remove()
-    }
-
-    /// Get the item reference iterator of the intrusive double linked list.
-    pub fn iter(&self) -> DlistIter<'_, A> {
-        DlistIter {
-            link: None,
-            dlist: self,
-        }
-    }
-
-    /// Get the item mutable reference iterator of the intrusive double linked list.
-    pub fn iter_mut(&mut self) -> DlistIterMut<'_, A> {
-        DlistIterMut {
-            link: None,
-            dlist: self,
-        }
-    }
-
-    /// Get the length of the intrusive double linked list.
-    pub fn len(&self) -> usize {
-        self.len
-    }
-
-    /// Check if the intrusive double linked list is empty.
-    pub fn is_empty(&self) -> bool {
-        self.len() == 0
-    }
-
-    /// Remove an node that holds the given raw link.
-    ///
-    /// # Safety
-    ///
-    /// `link` MUST be in this [`Dlist`].
-    pub unsafe fn remove_raw(&mut self, link: NonNull<DlistLink>) -> NonNull<A::Item> {
-        let mut iter = self.iter_mut_from_raw(link);
-        strict_assert!(iter.is_valid());
-        iter.remove().strict_unwrap_unchecked()
-    }
-
-    /// Create mutable iterator directly on raw link.
-    ///
-    /// # Safety
-    ///
-    /// `link` MUST be in this [`Dlist`].
-    pub unsafe fn iter_mut_from_raw(&mut self, link: NonNull<DlistLink>) -> DlistIterMut<'_, A> {
-        DlistIterMut {
-            link: Some(link),
-            dlist: self,
-        }
-    }
-
-    /// Create immutable iterator directly on raw link.
-    ///
-    /// # Safety
-    ///
-    /// `link` MUST be in this [`Dlist`].
-    pub unsafe fn iter_from_raw(&self, link: NonNull<DlistLink>) -> DlistIter<'_, A> {
-        DlistIter {
-            link: Some(link),
-            dlist: self,
-        }
-    }
-
-    /// Get the intrusive adapter of the double linked list.
-    pub fn adapter(&self) -> &A {
-        &self.adapter
-    }
-}
-
-/// Item reference iterator of the intrusive double linked list.
-pub struct DlistIter<'a, A>
-where
-    A: Adapter<Link = DlistLink>,
-{
-    link: Option<NonNull<A::Link>>,
-    dlist: &'a Dlist<A>,
-}
-
-impl<'a, A> DlistIter<'a, A>
-where
-    A: Adapter<Link = DlistLink>,
-{
-    /// Check if the iter is in a valid position.
-    pub fn is_valid(&self) -> bool {
-        self.link.is_some()
-    }
-
-    /// Get the item of the current position.
-    pub fn get(&self) -> Option<&A::Item> {
-        self.link
-            .map(|link| unsafe { self.dlist.adapter.link2ptr(link).as_ref() })
-    }
-
-    /// Move to next.
-    ///
-    /// If iter is on tail, move to null.
-    /// If iter is on null, move to head.
-    pub fn next(&mut self) {
-        unsafe {
-            match self.link {
-                Some(link) => self.link = link.as_ref().next,
-                None => self.link = self.dlist.head,
-            }
-        }
-    }
-
-    /// Move to prev.
-    ///
-    /// If iter is on head, move to null.
-    /// If iter is on null, move to tail.
-    pub fn prev(&mut self) {
-        unsafe {
-            match self.link {
-                Some(link) => self.link = link.as_ref().prev,
-                None => self.link = self.dlist.tail,
-            }
-        }
-    }
-
-    /// Move to head.
-    pub fn front(&mut self) {
-        self.link = self.dlist.head;
-    }
-
-    /// Move to head.
-    pub fn back(&mut self) {
-        self.link = self.dlist.tail;
-    }
-
-    /// Check if the iterator is in the first position of the intrusive double linked list.
-    pub fn is_front(&self) -> bool {
-        self.link == self.dlist.head
-    }
-
-    /// Check if the iterator is in the last position of the intrusive double linked list.
-    pub fn is_back(&self) -> bool {
-        self.link == self.dlist.tail
-    }
-}
-
-/// Item mutable reference iterator of the intrusive double linked list.
-pub struct DlistIterMut<'a, A>
-where
-    A: Adapter<Link = DlistLink>,
-{
-    link: Option<NonNull<A::Link>>,
-    dlist: &'a mut Dlist<A>,
-}
-
-impl<'a, A> DlistIterMut<'a, A>
-where
-    A: Adapter<Link = DlistLink>,
-{
-    /// Check if the iter is in a valid position.
-    pub fn is_valid(&self) -> bool {
-        self.link.is_some()
-    }
-
-    /// Get the item reference of the current position.
-    pub fn get(&self) -> Option<&A::Item> {
-        self.link
-            .map(|link| unsafe { self.dlist.adapter.link2ptr(link).as_ref() })
-    }
-
-    /// Get the item mutable reference of the current position.
-    pub fn get_mut(&mut self) -> Option<&mut A::Item> {
-        self.link
-            .map(|link| unsafe { self.dlist.adapter.link2ptr(link).as_mut() })
-    }
-
-    /// Move to next.
-    ///
-    /// If iter is on tail, move to null.
-    /// If iter is on null, move to head.
-    pub fn next(&mut self) {
-        unsafe {
-            match self.link {
-                Some(link) => self.link = link.as_ref().next,
-                None => self.link = self.dlist.head,
-            }
-        }
-    }
-
-    /// Move to prev.
-    ///
-    /// If iter is on head, move to null.
-    /// If iter is on null, move to tail.
-    pub fn prev(&mut self) {
-        unsafe {
-            match self.link {
-                Some(link) => self.link = link.as_ref().prev,
-                None => self.link = self.dlist.tail,
-            }
-        }
-    }
-
-    /// Move to front.
-    pub fn front(&mut self) {
-        self.link = self.dlist.head;
-    }
-
-    /// Move to back.
-    pub fn back(&mut self) {
-        self.link = self.dlist.tail;
-    }
-
-    /// Removes the current item from [`Dlist`] and move next.
-    pub fn remove(&mut self) -> Option<NonNull<A::Item>> {
-        unsafe {
-            if !self.is_valid() {
-                return None;
-            }
-
-            strict_assert!(self.is_valid());
-            let mut link = self.link.strict_unwrap_unchecked();
-            let ptr = self.dlist.adapter.link2ptr(link);
-
-            // fix head and tail if node is either of that
-            let mut prev = link.as_ref().prev;
-            let mut next = link.as_ref().next;
-            if Some(link) == self.dlist.head {
-                self.dlist.head = next;
-            }
-            if Some(link) == self.dlist.tail {
-                self.dlist.tail = prev;
-            }
-
-            // fix the next and prev ptrs of the node before and after this
-            if let Some(prev) = &mut prev {
-                prev.as_mut().next = next;
-            }
-            if let Some(next) = &mut next {
-                next.as_mut().prev = prev;
-            }
-
-            link.as_mut().next = None;
-            link.as_mut().prev = None;
-            link.as_mut().is_linked = false;
-
-            self.dlist.len -= 1;
-
-            self.link = next;
-
-            Some(ptr)
-        }
-    }
-
-    /// Link a new ptr before the current one.
-    ///
-    /// If iter is on null, link to tail.
-    pub fn insert_before(&mut self, ptr: NonNull<A::Item>) {
-        unsafe {
-            let mut link_new = self.dlist.adapter.ptr2link(ptr);
-            assert!(!link_new.as_ref().is_linked());
-
-            match self.link {
-                Some(link) => self.link_before(link_new, link),
-                None => {
-                    self.link_between(link_new, self.dlist.tail, None);
-                    self.dlist.tail = Some(link_new);
-                }
-            }
-
-            if self.dlist.head == self.link {
-                self.dlist.head = Some(link_new);
-            }
-
-            link_new.as_mut().is_linked = true;
-
-            self.dlist.len += 1;
-        }
-    }
-
-    /// Link a new ptr after the current one.
-    ///
-    /// If iter is on null, link to head.
-    pub fn insert_after(&mut self, ptr: NonNull<A::Item>) {
-        unsafe {
-            let mut link_new = self.dlist.adapter.ptr2link(ptr);
-            assert!(!link_new.as_ref().is_linked());
-
-            match self.link {
-                Some(link) => self.link_after(link_new, link),
-                None => {
-                    self.link_between(link_new, None, self.dlist.head);
-                    self.dlist.head = Some(link_new);
-                }
-            }
-
-            if self.dlist.tail == self.link {
-                self.dlist.tail = Some(link_new);
-            }
-
-            link_new.as_mut().is_linked = true;
-
-            self.dlist.len += 1;
-        }
-    }
-
-    unsafe fn link_before(&mut self, link: NonNull<A::Link>, next: NonNull<A::Link>) {
-        self.link_between(link, next.as_ref().prev, Some(next));
-    }
-
-    unsafe fn link_after(&mut self, link: NonNull<A::Link>, prev: NonNull<A::Link>) {
-        self.link_between(link, Some(prev), prev.as_ref().next);
-    }
-
-    unsafe fn link_between(
-        &mut self,
-        mut link: NonNull<A::Link>,
-        mut prev: Option<NonNull<A::Link>>,
-        mut next: Option<NonNull<A::Link>>,
-    ) {
-        if let Some(prev) = &mut prev {
-            prev.as_mut().next = Some(link);
-        }
-        if let Some(next) = &mut next {
-            next.as_mut().prev = Some(link);
-        }
-        link.as_mut().prev = prev;
-        link.as_mut().next = next;
-    }
-
-    /// Check if the iterator is in the first position of the intrusive double linked list.
-    pub fn is_front(&self) -> bool {
-        self.link == self.dlist.head
-    }
-
-    /// Check if the iterator is in the last position of the intrusive double linked list.
-    pub fn is_back(&self) -> bool {
-        self.link == self.dlist.tail
-    }
-}
-
-impl<'a, A> Iterator for DlistIter<'a, A>
-where
-    A: Adapter<Link = DlistLink>,
-{
-    type Item = &'a A::Item;
-
-    fn next(&mut self) -> Option<Self::Item> {
-        self.next();
-        match self.link {
-            Some(link) => Some(unsafe { self.dlist.adapter.link2ptr(link).as_ref() }),
-            None => None,
-        }
-    }
-}
-
-impl<'a, A> Iterator for DlistIterMut<'a, A>
-where
-    A: Adapter<Link = DlistLink>,
-{
-    type Item = &'a mut A::Item;
-
-    fn next(&mut self) -> Option<Self::Item> {
-        self.next();
-        match self.link {
-            Some(link) => Some(unsafe { self.dlist.adapter.link2ptr(link).as_mut() }),
-            None => None,
-        }
-    }
-}
-
-// TODO(MrCroxx): Need more tests.
-
-#[cfg(test)]
-mod tests {
-
-    use itertools::Itertools;
-
-    use super::*;
-    use crate::intrusive_adapter;
-
-    #[derive(Debug)]
-    struct DlistItem {
-        link: DlistLink,
-        val: u64,
-    }
-
-    impl DlistItem {
-        fn new(val: u64) -> Self {
-            Self {
-                link: DlistLink::default(),
-                val,
-            }
-        }
-    }
-
-    #[derive(Debug, Default)]
-    struct DlistAdapter;
-
-    unsafe impl Adapter for DlistAdapter {
-        type Item = DlistItem;
-        type Link = DlistLink;
-
-        fn new() -> Self {
-            Self
-        }
-
-        unsafe fn link2ptr(&self, link: NonNull<Self::Link>) -> NonNull<Self::Item> {
-            NonNull::new_unchecked(crate::container_of!(link.as_ptr(), DlistItem, link))
-        }
-
-        unsafe fn ptr2link(&self, item: NonNull<Self::Item>) -> NonNull<Self::Link> {
-            NonNull::new_unchecked((item.as_ptr() as *const u8).add(std::mem::offset_of!(DlistItem, link)) as *mut _)
-        }
-    }
-
-    intrusive_adapter! { DlistArcAdapter = DlistItem { link: DlistLink } }
-
-    #[test]
-    fn test_dlist_simple() {
-        let mut l = Dlist::<DlistAdapter>::new();
-
-        l.push_back(unsafe { NonNull::new_unchecked(Box::into_raw(Box::new(DlistItem::new(2)))) });
-        l.push_front(unsafe { NonNull::new_unchecked(Box::into_raw(Box::new(DlistItem::new(1)))) });
-        l.push_back(unsafe { NonNull::new_unchecked(Box::into_raw(Box::new(DlistItem::new(3)))) });
-
-        let v = l.iter_mut().map(|item| item.val).collect_vec();
-        assert_eq!(v, vec![1, 2, 3]);
-        assert_eq!(l.len(), 3);
-
-        let mut iter = l.iter_mut();
-        iter.next();
-        iter.next();
-        assert_eq!(DlistIterMut::get(&iter).unwrap().val, 2);
-        let p2 = iter.remove();
-        let i2 = unsafe { Box::from_raw(p2.unwrap().as_ptr()) };
-        assert_eq!(i2.val, 2);
-        assert_eq!(DlistIterMut::get(&iter).unwrap().val, 3);
-        let v = l.iter_mut().map(|item| item.val).collect_vec();
-        assert_eq!(v, vec![1, 3]);
-        assert_eq!(l.len(), 2);
-
-        let p3 = l.pop_back();
-        let i3 = unsafe { Box::from_raw(p3.unwrap().as_ptr()) };
-        assert_eq!(i3.val, 3);
-        let p1 = l.pop_front();
-        let i1 = unsafe { Box::from_raw(p1.unwrap().as_ptr()) };
-        assert_eq!(i1.val, 1);
-        assert!(l.pop_front().is_none());
-        assert_eq!(l.len(), 0);
-    }
-}
diff --git a/foyer-intrusive/src/lib.rs b/foyer-intrusive/src/lib.rs
deleted file mode 100644
index b176003e..00000000
--- a/foyer-intrusive/src/lib.rs
+++ /dev/null
@@ -1,46 +0,0 @@
-//  Copyright 2024 foyer Project Authors
-//
-//  Licensed under the Apache License, Version 2.0 (the "License");
-//  you may not use this file except in compliance with the License.
-//  You may obtain a copy of the License at
-//
-//  http://www.apache.org/licenses/LICENSE-2.0
-//
-//  Unless required by applicable law or agreed to in writing, software
-//  distributed under the License is distributed on an "AS IS" BASIS,
-//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-//  See the License for the specific language governing permissions and
-//  limitations under the License.
-
-#![expect(clippy::new_without_default)]
-
-//! Intrusive data structures and utils for foyer.
-
-/// Unsafe macro to get a raw pointer to an outer object from a pointer to one
-/// of its fields.
-///
-/// # Examples
-///
-/// ```
-/// use foyer_intrusive::container_of;
-///
-/// struct S { x: u32, y: u32 };
-/// let mut container = S { x: 1, y: 2 };
-/// let field = &mut container.x;
-/// let container2: *mut S = unsafe { container_of!(field, S, x) };
-/// assert_eq!(&mut container as *mut S, container2);
-/// ```
-///
-/// # Safety
-///
-/// This is unsafe because it assumes that the given expression is a valid
-/// pointer to the specified field of some container type.
-#[macro_export]
-macro_rules! container_of {
-    ($ptr:expr, $container:path, $field:ident) => {
-        ($ptr as *mut _ as *const u8).sub(std::mem::offset_of!($container, $field)) as *mut $container
-    };
-}
-
-pub mod adapter;
-pub mod dlist;
diff --git a/foyer-memory/Cargo.toml b/foyer-memory/Cargo.toml
index eab9cc62..cc1fbe61 100644
--- a/foyer-memory/Cargo.toml
+++ b/foyer-memory/Cargo.toml
@@ -19,18 +19,18 @@ cmsketch = "0.2.1"
 equivalent = { workspace = true }
 fastrace = { workspace = true }
 foyer-common = { workspace = true }
-foyer-intrusive = { workspace = true }
 futures = "0.3"
 hashbrown = { workspace = true }
+intrusive-collections = { git = "https://github.com/foyer-rs/intrusive-rs", rev = "94cfac4701dbc0033b7bc27e31c46bf3a12d96d7" }
 itertools = { workspace = true }
-parking_lot = "0.12"
+parking_lot = { workspace = true }
+paste = "1"
 pin-project = "1"
 serde = { workspace = true }
 tokio = { workspace = true }
 tracing = "0.1"
 
 [dev-dependencies]
-anyhow = "1"
 csv = "1.3.0"
 moka = { version = "0.12", features = ["sync"] }
 rand = { version = "0.8", features = ["small_rng"] }
@@ -39,11 +39,9 @@ zipf = "7.0.1"
 
 [features]
 deadlock = ["parking_lot/deadlock_detection"]
-strict_assertions = [
-    "foyer-common/strict_assertions",
-    "foyer-intrusive/strict_assertions",
-]
-sanity = ["strict_assertions"]
+# FIXME: remove sanity feature
+sanity = []
+strict_assertions = ["foyer-common/strict_assertions"]
 tracing = ["fastrace/enable", "foyer-common/tracing"]
 
 [[bench]]
diff --git a/foyer-memory/benches/bench_hit_ratio.rs b/foyer-memory/benches/bench_hit_ratio.rs
index 42dbfb38..5afafdad 100644
--- a/foyer-memory/benches/bench_hit_ratio.rs
+++ b/foyer-memory/benches/bench_hit_ratio.rs
@@ -27,7 +27,7 @@ const ITEMS: usize = 10_000;
 const ITERATIONS: usize = 5_000_000;
 
 const SHARDS: usize = 1;
-const OBJECT_POOL_CAPACITY: usize = 16;
+
 /*
 inspired by pingora/tinyufo/benches/bench_hit_ratio.rs
 cargo bench --bench bench_hit_ratio
@@ -89,7 +89,6 @@ fn new_fifo_cache(capacity: usize) -> Cache<CacheKey, CacheValue> {
     CacheBuilder::new(capacity)
         .with_shards(SHARDS)
         .with_eviction_config(FifoConfig {})
-        .with_object_pool_capacity(OBJECT_POOL_CAPACITY)
         .build()
 }
 
@@ -99,7 +98,6 @@ fn new_lru_cache(capacity: usize) -> Cache<CacheKey, CacheValue> {
         .with_eviction_config(LruConfig {
             high_priority_pool_ratio: 0.1,
         })
-        .with_object_pool_capacity(OBJECT_POOL_CAPACITY)
         .build()
 }
 
@@ -112,7 +110,6 @@ fn new_lfu_cache(capacity: usize) -> Cache<CacheKey, CacheValue> {
             cmsketch_eps: 0.001,
             cmsketch_confidence: 0.9,
         })
-        .with_object_pool_capacity(OBJECT_POOL_CAPACITY)
         .build()
 }
 
@@ -124,7 +121,6 @@ fn new_s3fifo_cache_wo_ghost(capacity: usize) -> Cache<CacheKey, CacheValue> {
             ghost_queue_capacity_ratio: 0.0,
             small_to_main_freq_threshold: 2,
         })
-        .with_object_pool_capacity(OBJECT_POOL_CAPACITY)
         .build()
 }
 
@@ -136,7 +132,6 @@ fn new_s3fifo_cache_w_ghost(capacity: usize) -> Cache<CacheKey, CacheValue> {
             ghost_queue_capacity_ratio: 1.0,
             small_to_main_freq_threshold: 2,
         })
-        .with_object_pool_capacity(OBJECT_POOL_CAPACITY)
         .build()
 }
 
diff --git a/foyer-memory/src/cache.rs b/foyer-memory/src/cache.rs
index e3554e48..4edf75cf 100644
--- a/foyer-memory/src/cache.rs
+++ b/foyer-memory/src/cache.rs
@@ -12,7 +12,7 @@
 //  See the License for the specific language governing permissions and
 //  limitations under the License.
 
-use std::{fmt::Debug, hash::Hash, ops::Deref, sync::Arc};
+use std::{fmt::Debug, future::Future, hash::Hash, ops::Deref, sync::Arc};
 
 use ahash::RandomState;
 use equivalent::Equivalent;
@@ -22,57 +22,36 @@ use foyer_common::{
     future::Diversion,
     runtime::SingletonHandle,
 };
-use futures::Future;
 use pin_project::pin_project;
 use serde::{Deserialize, Serialize};
 use tokio::sync::oneshot;
 
 use crate::{
-    context::CacheContext,
     eviction::{
-        fifo::{Fifo, FifoHandle},
-        lfu::{Lfu, LfuHandle},
-        lru::{Lru, LruHandle},
-        s3fifo::{S3Fifo, S3FifoHandle},
-        sanity::SanityEviction,
+        fifo::{Fifo, FifoConfig},
+        lfu::{Lfu, LfuConfig},
+        lru::{Lru, LruConfig},
+        s3fifo::{S3Fifo, S3FifoConfig},
     },
-    generic::{FetchMark, FetchState, GenericCache, GenericCacheConfig, GenericCacheEntry, GenericFetch, Weighter},
-    indexer::{hash_table::HashTableIndexer, sanity::SanityIndexer},
-    FifoConfig, LfuConfig, LruConfig, S3FifoConfig,
+    raw::{FetchMark, FetchState, RawCache, RawCacheConfig, RawCacheEntry, RawFetch, Weighter},
+    record::CacheHint,
 };
 
-pub type FifoCache<K, V, S = RandomState> =
-    GenericCache<K, V, SanityEviction<Fifo<(K, V)>>, SanityIndexer<HashTableIndexer<K, FifoHandle<(K, V)>>>, S>;
-pub type FifoCacheEntry<K, V, S = RandomState> =
-    GenericCacheEntry<K, V, SanityEviction<Fifo<(K, V)>>, SanityIndexer<HashTableIndexer<K, FifoHandle<(K, V)>>>, S>;
-pub type FifoFetch<K, V, ER, S = RandomState> =
-    GenericFetch<K, V, SanityEviction<Fifo<(K, V)>>, SanityIndexer<HashTableIndexer<K, FifoHandle<(K, V)>>>, S, ER>;
-
-pub type LruCache<K, V, S = RandomState> =
-    GenericCache<K, V, SanityEviction<Lru<(K, V)>>, SanityIndexer<HashTableIndexer<K, LruHandle<(K, V)>>>, S>;
-pub type LruCacheEntry<K, V, S = RandomState> =
-    GenericCacheEntry<K, V, SanityEviction<Lru<(K, V)>>, SanityIndexer<HashTableIndexer<K, LruHandle<(K, V)>>>, S>;
-pub type LruFetch<K, V, ER, S = RandomState> =
-    GenericFetch<K, V, SanityEviction<Lru<(K, V)>>, SanityIndexer<HashTableIndexer<K, LruHandle<(K, V)>>>, S, ER>;
-
-pub type LfuCache<K, V, S = RandomState> =
-    GenericCache<K, V, SanityEviction<Lfu<(K, V)>>, SanityIndexer<HashTableIndexer<K, LfuHandle<(K, V)>>>, S>;
-pub type LfuCacheEntry<K, V, S = RandomState> =
-    GenericCacheEntry<K, V, SanityEviction<Lfu<(K, V)>>, SanityIndexer<HashTableIndexer<K, LfuHandle<(K, V)>>>, S>;
-pub type LfuFetch<K, V, ER, S = RandomState> =
-    GenericFetch<K, V, SanityEviction<Lfu<(K, V)>>, SanityIndexer<HashTableIndexer<K, LfuHandle<(K, V)>>>, S, ER>;
-
-pub type S3FifoCache<K, V, S = RandomState> =
-    GenericCache<K, V, SanityEviction<S3Fifo<(K, V)>>, SanityIndexer<HashTableIndexer<K, S3FifoHandle<(K, V)>>>, S>;
-pub type S3FifoCacheEntry<K, V, S = RandomState> = GenericCacheEntry<
-    K,
-    V,
-    SanityEviction<S3Fifo<(K, V)>>,
-    SanityIndexer<HashTableIndexer<K, S3FifoHandle<(K, V)>>>,
-    S,
->;
-pub type S3FifoFetch<K, V, ER, S = RandomState> =
-    GenericFetch<K, V, SanityEviction<S3Fifo<(K, V)>>, SanityIndexer<HashTableIndexer<K, S3FifoHandle<(K, V)>>>, S, ER>;
+pub type FifoCache<K, V, S = RandomState> = RawCache<Fifo<K, V>, S>;
+pub type FifoCacheEntry<K, V, S = RandomState> = RawCacheEntry<Fifo<K, V>, S>;
+pub type FifoFetch<K, V, ER, S = RandomState> = RawFetch<Fifo<K, V>, ER, S>;
+
+pub type S3FifoCache<K, V, S = RandomState> = RawCache<S3Fifo<K, V>, S>;
+pub type S3FifoCacheEntry<K, V, S = RandomState> = RawCacheEntry<S3Fifo<K, V>, S>;
+pub type S3FifoFetch<K, V, ER, S = RandomState> = RawFetch<S3Fifo<K, V>, ER, S>;
+
+pub type LruCache<K, V, S = RandomState> = RawCache<Lru<K, V>, S>;
+pub type LruCacheEntry<K, V, S = RandomState> = RawCacheEntry<Lru<K, V>, S>;
+pub type LruFetch<K, V, ER, S = RandomState> = RawFetch<Lru<K, V>, ER, S>;
+
+pub type LfuCache<K, V, S = RandomState> = RawCache<Lfu<K, V>, S>;
+pub type LfuCacheEntry<K, V, S = RandomState> = RawCacheEntry<Lfu<K, V>, S>;
+pub type LfuFetch<K, V, ER, S = RandomState> = RawFetch<Lfu<K, V>, ER, S>;
 
 /// A cached entry holder of the in-memory cache.
 #[derive(Debug)]
@@ -84,12 +63,12 @@ where
 {
     /// A cached entry holder of the in-memory FIFO cache.
     Fifo(FifoCacheEntry<K, V, S>),
+    /// A cached entry holder of the in-memory S3FIFO cache.
+    S3Fifo(S3FifoCacheEntry<K, V, S>),
     /// A cached entry holder of the in-memory LRU cache.
     Lru(LruCacheEntry<K, V, S>),
     /// A cached entry holder of the in-memory LFU cache.
     Lfu(LfuCacheEntry<K, V, S>),
-    /// A cached entry holder of the in-memory S3FIFO cache.
-    S3Fifo(S3FifoCacheEntry<K, V, S>),
 }
 
 impl<K, V, S> Clone for CacheEntry<K, V, S>
@@ -206,13 +185,13 @@ where
         }
     }
 
-    /// Context of the cached entry.
-    pub fn context(&self) -> CacheContext {
+    /// Hint of the cached entry.
+    pub fn hint(&self) -> CacheHint {
         match self {
-            CacheEntry::Fifo(entry) => entry.context().clone().into(),
-            CacheEntry::Lru(entry) => entry.context().clone().into(),
-            CacheEntry::Lfu(entry) => entry.context().clone().into(),
-            CacheEntry::S3Fifo(entry) => entry.context().clone().into(),
+            CacheEntry::Fifo(entry) => entry.hint().clone().into(),
+            CacheEntry::Lru(entry) => entry.hint().clone().into(),
+            CacheEntry::Lfu(entry) => entry.hint().clone().into(),
+            CacheEntry::S3Fifo(entry) => entry.hint().clone().into(),
         }
     }
 
@@ -252,12 +231,12 @@ where
 pub enum EvictionConfig {
     /// FIFO eviction algorithm config.
     Fifo(FifoConfig),
+    /// S3FIFO eviction algorithm config.
+    S3Fifo(S3FifoConfig),
     /// LRU eviction algorithm config.
     Lru(LruConfig),
     /// LFU eviction algorithm config.
     Lfu(LfuConfig),
-    /// S3FIFO eviction algorithm config.
-    S3Fifo(S3FifoConfig),
 }
 
 impl From<FifoConfig> for EvictionConfig {
@@ -266,6 +245,12 @@ impl From<FifoConfig> for EvictionConfig {
     }
 }
 
+impl From<S3FifoConfig> for EvictionConfig {
+    fn from(value: S3FifoConfig) -> EvictionConfig {
+        EvictionConfig::S3Fifo(value)
+    }
+}
+
 impl From<LruConfig> for EvictionConfig {
     fn from(value: LruConfig) -> EvictionConfig {
         EvictionConfig::Lru(value)
@@ -278,12 +263,6 @@ impl From<LfuConfig> for EvictionConfig {
     }
 }
 
-impl From<S3FifoConfig> for EvictionConfig {
-    fn from(value: S3FifoConfig) -> EvictionConfig {
-        EvictionConfig::S3Fifo(value)
-    }
-}
-
 /// In-memory cache builder.
 pub struct CacheBuilder<K, V, S>
 where
@@ -296,7 +275,6 @@ where
     capacity: usize,
     shards: usize,
     eviction_config: EvictionConfig,
-    object_pool_capacity: usize,
 
     hash_builder: S,
     weighter: Arc<dyn Weighter<K, V>>,
@@ -316,14 +294,8 @@ where
 
             capacity,
             shards: 8,
-            eviction_config: LfuConfig {
-                window_capacity_ratio: 0.1,
-                protected_capacity_ratio: 0.8,
-                cmsketch_eps: 0.001,
-                cmsketch_confidence: 0.9,
-            }
-            .into(),
-            object_pool_capacity: 1024,
+            eviction_config: LruConfig::default().into(),
+
             hash_builder: RandomState::default(),
             weighter: Arc::new(|_, _| 1),
             event_listener: None,
@@ -362,16 +334,6 @@ where
         self
     }
 
-    /// Set object pool for handles. The object pool is used to reduce handle allocation.
-    ///
-    /// The optimized value is supposed to be equal to the max cache entry count.
-    ///
-    /// The default value is 1024.
-    pub fn with_object_pool_capacity(mut self, object_pool_capacity: usize) -> Self {
-        self.object_pool_capacity = object_pool_capacity;
-        self
-    }
-
     /// Set in-memory cache hash builder.
     pub fn with_hash_builder<OS>(self, hash_builder: OS) -> CacheBuilder<K, V, OS>
     where
@@ -382,7 +344,6 @@ where
             capacity: self.capacity,
             shards: self.shards,
             eviction_config: self.eviction_config,
-            object_pool_capacity: self.object_pool_capacity,
             hash_builder,
             weighter: self.weighter,
             event_listener: self.event_listener,
@@ -412,42 +373,38 @@ where
         }
 
         match self.eviction_config {
-            EvictionConfig::Fifo(eviction_config) => Cache::Fifo(Arc::new(GenericCache::new(GenericCacheConfig {
+            EvictionConfig::Fifo(eviction_config) => Cache::Fifo(Arc::new(RawCache::new(RawCacheConfig {
                 name: self.name,
                 capacity: self.capacity,
                 shards: self.shards,
                 eviction_config,
-                object_pool_capacity: self.object_pool_capacity,
                 hash_builder: self.hash_builder,
                 weighter: self.weighter,
                 event_listener: self.event_listener,
             }))),
-            EvictionConfig::Lru(eviction_config) => Cache::Lru(Arc::new(GenericCache::new(GenericCacheConfig {
+            EvictionConfig::S3Fifo(eviction_config) => Cache::S3Fifo(Arc::new(RawCache::new(RawCacheConfig {
                 name: self.name,
                 capacity: self.capacity,
                 shards: self.shards,
                 eviction_config,
-                object_pool_capacity: self.object_pool_capacity,
                 hash_builder: self.hash_builder,
                 weighter: self.weighter,
                 event_listener: self.event_listener,
             }))),
-            EvictionConfig::Lfu(eviction_config) => Cache::Lfu(Arc::new(GenericCache::new(GenericCacheConfig {
+            EvictionConfig::Lru(eviction_config) => Cache::Lru(Arc::new(RawCache::new(RawCacheConfig {
                 name: self.name,
                 capacity: self.capacity,
                 shards: self.shards,
                 eviction_config,
-                object_pool_capacity: self.object_pool_capacity,
                 hash_builder: self.hash_builder,
                 weighter: self.weighter,
                 event_listener: self.event_listener,
             }))),
-            EvictionConfig::S3Fifo(eviction_config) => Cache::S3Fifo(Arc::new(GenericCache::new(GenericCacheConfig {
+            EvictionConfig::Lfu(eviction_config) => Cache::Lfu(Arc::new(RawCache::new(RawCacheConfig {
                 name: self.name,
                 capacity: self.capacity,
                 shards: self.shards,
                 eviction_config,
-                object_pool_capacity: self.object_pool_capacity,
                 hash_builder: self.hash_builder,
                 weighter: self.weighter,
                 event_listener: self.event_listener,
@@ -482,9 +439,9 @@ where
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
         match self {
             Self::Fifo(_) => f.debug_tuple("Cache::FifoCache").finish(),
+            Self::S3Fifo(_) => f.debug_tuple("Cache::S3FifoCache").finish(),
             Self::Lru(_) => f.debug_tuple("Cache::LruCache").finish(),
             Self::Lfu(_) => f.debug_tuple("Cache::LfuCache").finish(),
-            Self::S3Fifo(_) => f.debug_tuple("Cache::S3FifoCache").finish(),
         }
     }
 }
@@ -498,9 +455,9 @@ where
     fn clone(&self) -> Self {
         match self {
             Self::Fifo(cache) => Self::Fifo(cache.clone()),
+            Self::S3Fifo(cache) => Self::S3Fifo(cache.clone()),
             Self::Lru(cache) => Self::Lru(cache.clone()),
             Self::Lfu(cache) => Self::Lfu(cache.clone()),
-            Self::S3Fifo(cache) => Self::S3Fifo(cache.clone()),
         }
     }
 }
@@ -516,20 +473,20 @@ where
     pub fn insert(&self, key: K, value: V) -> CacheEntry<K, V, S> {
         match self {
             Cache::Fifo(cache) => cache.insert(key, value).into(),
+            Cache::S3Fifo(cache) => cache.insert(key, value).into(),
             Cache::Lru(cache) => cache.insert(key, value).into(),
             Cache::Lfu(cache) => cache.insert(key, value).into(),
-            Cache::S3Fifo(cache) => cache.insert(key, value).into(),
         }
     }
 
-    /// Insert cache entry with cache context to the in-memory cache.
-    #[fastrace::trace(name = "foyer::memory::cache::insert_with_context")]
-    pub fn insert_with_context(&self, key: K, value: V, context: CacheContext) -> CacheEntry<K, V, S> {
+    /// Insert cache entry with cache hint to the in-memory cache.
+    #[fastrace::trace(name = "foyer::memory::cache::insert_with_hint")]
+    pub fn insert_with_hint(&self, key: K, value: V, hint: CacheHint) -> CacheEntry<K, V, S> {
         match self {
-            Cache::Fifo(cache) => cache.insert_with_context(key, value, context).into(),
-            Cache::Lru(cache) => cache.insert_with_context(key, value, context).into(),
-            Cache::Lfu(cache) => cache.insert_with_context(key, value, context).into(),
-            Cache::S3Fifo(cache) => cache.insert_with_context(key, value, context).into(),
+            Cache::Fifo(cache) => cache.insert_with_hint(key, value, hint.into()).into(),
+            Cache::S3Fifo(cache) => cache.insert_with_hint(key, value, hint.into()).into(),
+            Cache::Lru(cache) => cache.insert_with_hint(key, value, hint.into()).into(),
+            Cache::Lfu(cache) => cache.insert_with_hint(key, value, hint.into()).into(),
         }
     }
 
@@ -538,28 +495,28 @@ where
     /// The entry will be removed as soon as the returned entry is dropped.
     ///
     /// The entry will become a normal entry after it is accessed.
-    #[fastrace::trace(name = "foyer::memory::cache::deposit")]
-    pub fn deposit(&self, key: K, value: V) -> CacheEntry<K, V, S> {
+    #[fastrace::trace(name = "foyer::memory::cache::insert_ephemeral")]
+    pub fn insert_ephemeral(&self, key: K, value: V) -> CacheEntry<K, V, S> {
         match self {
-            Cache::Fifo(cache) => cache.deposit(key, value).into(),
-            Cache::Lru(cache) => cache.deposit(key, value).into(),
-            Cache::Lfu(cache) => cache.deposit(key, value).into(),
-            Cache::S3Fifo(cache) => cache.deposit(key, value).into(),
+            Cache::Fifo(cache) => cache.insert_ephemeral(key, value).into(),
+            Cache::S3Fifo(cache) => cache.insert_ephemeral(key, value).into(),
+            Cache::Lru(cache) => cache.insert_ephemeral(key, value).into(),
+            Cache::Lfu(cache) => cache.insert_ephemeral(key, value).into(),
         }
     }
 
-    /// Temporarily insert cache entry with cache context to the in-memory cache.
+    /// Temporarily insert cache entry with cache hint to the in-memory cache.
     ///
     /// The entry will be removed as soon as the returned entry is dropped.
     ///
     /// The entry will become a normal entry after it is accessed.
-    #[fastrace::trace(name = "foyer::memory::cache::deposit_with_context")]
-    pub fn deposit_with_context(&self, key: K, value: V, context: CacheContext) -> CacheEntry<K, V, S> {
+    #[fastrace::trace(name = "foyer::memory::cache::insert_ephemeral_with_hint")]
+    pub fn insert_ephemeral_with_hint(&self, key: K, value: V, hint: CacheHint) -> CacheEntry<K, V, S> {
         match self {
-            Cache::Fifo(cache) => cache.deposit_with_context(key, value, context).into(),
-            Cache::Lru(cache) => cache.deposit_with_context(key, value, context).into(),
-            Cache::Lfu(cache) => cache.deposit_with_context(key, value, context).into(),
-            Cache::S3Fifo(cache) => cache.deposit_with_context(key, value, context).into(),
+            Cache::Fifo(cache) => cache.insert_ephemeral_with_hint(key, value, hint.into()).into(),
+            Cache::Lru(cache) => cache.insert_ephemeral_with_hint(key, value, hint.into()).into(),
+            Cache::Lfu(cache) => cache.insert_ephemeral_with_hint(key, value, hint.into()).into(),
+            Cache::S3Fifo(cache) => cache.insert_ephemeral_with_hint(key, value, hint.into()).into(),
         }
     }
 
@@ -571,9 +528,9 @@ where
     {
         match self {
             Cache::Fifo(cache) => cache.remove(key).map(CacheEntry::from),
+            Cache::S3Fifo(cache) => cache.remove(key).map(CacheEntry::from),
             Cache::Lru(cache) => cache.remove(key).map(CacheEntry::from),
             Cache::Lfu(cache) => cache.remove(key).map(CacheEntry::from),
-            Cache::S3Fifo(cache) => cache.remove(key).map(CacheEntry::from),
         }
     }
 
@@ -585,9 +542,9 @@ where
     {
         match self {
             Cache::Fifo(cache) => cache.get(key).map(CacheEntry::from),
+            Cache::S3Fifo(cache) => cache.get(key).map(CacheEntry::from),
             Cache::Lru(cache) => cache.get(key).map(CacheEntry::from),
             Cache::Lfu(cache) => cache.get(key).map(CacheEntry::from),
-            Cache::S3Fifo(cache) => cache.get(key).map(CacheEntry::from),
         }
     }
 
@@ -599,9 +556,9 @@ where
     {
         match self {
             Cache::Fifo(cache) => cache.contains(key),
+            Cache::S3Fifo(cache) => cache.contains(key),
             Cache::Lru(cache) => cache.contains(key),
             Cache::Lfu(cache) => cache.contains(key),
-            Cache::S3Fifo(cache) => cache.contains(key),
         }
     }
 
@@ -615,9 +572,9 @@ where
     {
         match self {
             Cache::Fifo(cache) => cache.touch(key),
+            Cache::S3Fifo(cache) => cache.touch(key),
             Cache::Lru(cache) => cache.touch(key),
             Cache::Lfu(cache) => cache.touch(key),
-            Cache::S3Fifo(cache) => cache.touch(key),
         }
     }
 
@@ -626,9 +583,9 @@ where
     pub fn clear(&self) {
         match self {
             Cache::Fifo(cache) => cache.clear(),
+            Cache::S3Fifo(cache) => cache.clear(),
             Cache::Lru(cache) => cache.clear(),
             Cache::Lfu(cache) => cache.clear(),
-            Cache::S3Fifo(cache) => cache.clear(),
         }
     }
 
@@ -636,9 +593,9 @@ where
     pub fn capacity(&self) -> usize {
         match self {
             Cache::Fifo(cache) => cache.capacity(),
+            Cache::S3Fifo(cache) => cache.capacity(),
             Cache::Lru(cache) => cache.capacity(),
             Cache::Lfu(cache) => cache.capacity(),
-            Cache::S3Fifo(cache) => cache.capacity(),
         }
     }
 
@@ -646,9 +603,9 @@ where
     pub fn usage(&self) -> usize {
         match self {
             Cache::Fifo(cache) => cache.usage(),
+            Cache::S3Fifo(cache) => cache.usage(),
             Cache::Lru(cache) => cache.usage(),
             Cache::Lfu(cache) => cache.usage(),
-            Cache::S3Fifo(cache) => cache.usage(),
         }
     }
 
@@ -664,9 +621,9 @@ where
     pub fn hash_builder(&self) -> &S {
         match self {
             Cache::Fifo(cache) => cache.hash_builder(),
+            Cache::S3Fifo(cache) => cache.hash_builder(),
             Cache::Lru(cache) => cache.hash_builder(),
             Cache::Lfu(cache) => cache.hash_builder(),
-            Cache::S3Fifo(cache) => cache.hash_builder(),
         }
     }
 
@@ -674,9 +631,9 @@ where
     pub fn shards(&self) -> usize {
         match self {
             Cache::Fifo(cache) => cache.shards(),
+            Cache::S3Fifo(cache) => cache.shards(),
             Cache::Lru(cache) => cache.shards(),
             Cache::Lfu(cache) => cache.shards(),
-            Cache::S3Fifo(cache) => cache.shards(),
         }
     }
 }
@@ -691,12 +648,12 @@ where
 {
     /// A future that is used to get entry value from the remote storage for the in-memory FIFO cache.
     Fifo(#[pin] FifoFetch<K, V, ER, S>),
+    /// A future that is used to get entry value from the remote storage for the in-memory S3FIFO cache.
+    S3Fifo(#[pin] S3FifoFetch<K, V, ER, S>),
     /// A future that is used to get entry value from the remote storage for the in-memory LRU cache.
     Lru(#[pin] LruFetch<K, V, ER, S>),
     /// A future that is used to get entry value from the remote storage for the in-memory LFU cache.
     Lfu(#[pin] LfuFetch<K, V, ER, S>),
-    /// A future that is used to get entry value from the remote storage for the in-memory S3FIFO cache.
-    S3Fifo(#[pin] S3FifoFetch<K, V, ER, S>),
 }
 
 impl<K, V, ER, S> From<FifoFetch<K, V, ER, S>> for Fetch<K, V, ER, S>
@@ -710,36 +667,36 @@ where
     }
 }
 
-impl<K, V, ER, S> From<LruFetch<K, V, ER, S>> for Fetch<K, V, ER, S>
+impl<K, V, ER, S> From<S3FifoFetch<K, V, ER, S>> for Fetch<K, V, ER, S>
 where
     K: Key,
     V: Value,
     S: HashBuilder,
 {
-    fn from(entry: LruFetch<K, V, ER, S>) -> Self {
-        Self::Lru(entry)
+    fn from(entry: S3FifoFetch<K, V, ER, S>) -> Self {
+        Self::S3Fifo(entry)
     }
 }
 
-impl<K, V, ER, S> From<LfuFetch<K, V, ER, S>> for Fetch<K, V, ER, S>
+impl<K, V, ER, S> From<LruFetch<K, V, ER, S>> for Fetch<K, V, ER, S>
 where
     K: Key,
     V: Value,
     S: HashBuilder,
 {
-    fn from(entry: LfuFetch<K, V, ER, S>) -> Self {
-        Self::Lfu(entry)
+    fn from(entry: LruFetch<K, V, ER, S>) -> Self {
+        Self::Lru(entry)
     }
 }
 
-impl<K, V, ER, S> From<S3FifoFetch<K, V, ER, S>> for Fetch<K, V, ER, S>
+impl<K, V, ER, S> From<LfuFetch<K, V, ER, S>> for Fetch<K, V, ER, S>
 where
     K: Key,
     V: Value,
     S: HashBuilder,
 {
-    fn from(entry: S3FifoFetch<K, V, ER, S>) -> Self {
-        Self::S3Fifo(entry)
+    fn from(entry: LfuFetch<K, V, ER, S>) -> Self {
+        Self::Lfu(entry)
     }
 }
 
@@ -755,9 +712,9 @@ where
     fn poll(self: std::pin::Pin<&mut Self>, cx: &mut std::task::Context<'_>) -> std::task::Poll<Self::Output> {
         match self.project() {
             FetchProj::Fifo(entry) => entry.poll(cx).map(|res| res.map(CacheEntry::from)),
+            FetchProj::S3Fifo(entry) => entry.poll(cx).map(|res| res.map(CacheEntry::from)),
             FetchProj::Lru(entry) => entry.poll(cx).map(|res| res.map(CacheEntry::from)),
             FetchProj::Lfu(entry) => entry.poll(cx).map(|res| res.map(CacheEntry::from)),
-            FetchProj::S3Fifo(entry) => entry.poll(cx).map(|res| res.map(CacheEntry::from)),
         }
     }
 }
@@ -772,9 +729,9 @@ where
     pub fn state(&self) -> FetchState {
         match self {
             Fetch::Fifo(fetch) => fetch.state(),
+            Fetch::S3Fifo(fetch) => fetch.state(),
             Fetch::Lru(fetch) => fetch.state(),
             Fetch::Lfu(fetch) => fetch.state(),
-            Fetch::S3Fifo(fetch) => fetch.state(),
         }
     }
 
@@ -783,9 +740,9 @@ where
     pub fn store(&self) -> &Option<FetchMark> {
         match self {
             Fetch::Fifo(fetch) => fetch.store(),
+            Fetch::S3Fifo(fetch) => fetch.store(),
             Fetch::Lru(fetch) => fetch.store(),
             Fetch::Lfu(fetch) => fetch.store(),
-            Fetch::S3Fifo(fetch) => fetch.store(),
         }
     }
 }
@@ -810,29 +767,29 @@ where
     {
         match self {
             Cache::Fifo(cache) => Fetch::from(cache.fetch(key, fetch)),
+            Cache::S3Fifo(cache) => Fetch::from(cache.fetch(key, fetch)),
             Cache::Lru(cache) => Fetch::from(cache.fetch(key, fetch)),
             Cache::Lfu(cache) => Fetch::from(cache.fetch(key, fetch)),
-            Cache::S3Fifo(cache) => Fetch::from(cache.fetch(key, fetch)),
         }
     }
 
-    /// Get the cached entry with the given key and context from the in-memory cache.
+    /// Get the cached entry with the given key and hint from the in-memory cache.
     ///
     /// Use `fetch` to fetch the cache value from the remote storage on cache miss.
     ///
     /// The concurrent fetch requests will be deduplicated.
-    #[fastrace::trace(name = "foyer::memory::cache::fetch_with_context")]
-    pub fn fetch_with_context<F, FU, ER>(&self, key: K, context: CacheContext, fetch: F) -> Fetch<K, V, ER, S>
+    #[fastrace::trace(name = "foyer::memory::cache::fetch_with_hint")]
+    pub fn fetch_with_hint<F, FU, ER>(&self, key: K, hint: CacheHint, fetch: F) -> Fetch<K, V, ER, S>
     where
         F: FnOnce() -> FU,
         FU: Future<Output = std::result::Result<V, ER>> + Send + 'static,
         ER: Send + 'static + Debug,
     {
         match self {
-            Cache::Fifo(cache) => Fetch::from(cache.fetch_with_context(key, context, fetch)),
-            Cache::Lru(cache) => Fetch::from(cache.fetch_with_context(key, context, fetch)),
-            Cache::Lfu(cache) => Fetch::from(cache.fetch_with_context(key, context, fetch)),
-            Cache::S3Fifo(cache) => Fetch::from(cache.fetch_with_context(key, context, fetch)),
+            Cache::Fifo(cache) => Fetch::from(cache.fetch_with_hint(key, hint.into(), fetch)),
+            Cache::S3Fifo(cache) => Fetch::from(cache.fetch_with_hint(key, hint.into(), fetch)),
+            Cache::Lru(cache) => Fetch::from(cache.fetch_with_hint(key, hint.into(), fetch)),
+            Cache::Lfu(cache) => Fetch::from(cache.fetch_with_hint(key, hint.into(), fetch)),
         }
     }
 
@@ -847,7 +804,7 @@ where
     pub fn fetch_inner<F, FU, ER, ID>(
         &self,
         key: K,
-        context: CacheContext,
+        hint: CacheHint,
         fetch: F,
         runtime: &SingletonHandle,
     ) -> Fetch<K, V, ER, S>
@@ -858,10 +815,10 @@ where
         ID: Into<Diversion<std::result::Result<V, ER>, FetchMark>>,
     {
         match self {
-            Cache::Fifo(cache) => Fetch::from(cache.fetch_inner(key, context, fetch, runtime)),
-            Cache::Lru(cache) => Fetch::from(cache.fetch_inner(key, context, fetch, runtime)),
-            Cache::Lfu(cache) => Fetch::from(cache.fetch_inner(key, context, fetch, runtime)),
-            Cache::S3Fifo(cache) => Fetch::from(cache.fetch_inner(key, context, fetch, runtime)),
+            Cache::Fifo(cache) => Fetch::from(cache.fetch_inner(key, hint.into(), fetch, runtime)),
+            Cache::Lru(cache) => Fetch::from(cache.fetch_inner(key, hint.into(), fetch, runtime)),
+            Cache::Lfu(cache) => Fetch::from(cache.fetch_inner(key, hint.into(), fetch, runtime)),
+            Cache::S3Fifo(cache) => Fetch::from(cache.fetch_inner(key, hint.into(), fetch, runtime)),
         }
     }
 }
@@ -875,11 +832,10 @@ mod tests {
     use rand::{rngs::StdRng, seq::SliceRandom, Rng, SeedableRng};
 
     use super::*;
-    use crate::{eviction::s3fifo::S3FifoConfig, FifoConfig, LfuConfig, LruConfig};
+    use crate::eviction::{fifo::FifoConfig, lfu::LfuConfig, lru::LruConfig, s3fifo::S3FifoConfig};
 
     const CAPACITY: usize = 100;
     const SHARDS: usize = 4;
-    const OBJECT_POOL_CAPACITY: usize = 64;
     const RANGE: Range<u64> = 0..1000;
     const OPS: usize = 10000;
     const CONCURRENCY: usize = 8;
@@ -888,7 +844,6 @@ mod tests {
         CacheBuilder::new(CAPACITY)
             .with_shards(SHARDS)
             .with_eviction_config(FifoConfig {})
-            .with_object_pool_capacity(OBJECT_POOL_CAPACITY)
             .build()
     }
 
@@ -898,7 +853,6 @@ mod tests {
             .with_eviction_config(LruConfig {
                 high_priority_pool_ratio: 0.1,
             })
-            .with_object_pool_capacity(OBJECT_POOL_CAPACITY)
             .build()
     }
 
@@ -911,7 +865,6 @@ mod tests {
                 cmsketch_eps: 0.001,
                 cmsketch_confidence: 0.9,
             })
-            .with_object_pool_capacity(OBJECT_POOL_CAPACITY)
             .build()
     }
 
@@ -923,7 +876,6 @@ mod tests {
                 ghost_queue_capacity_ratio: 10.0,
                 small_to_main_freq_threshold: 2,
             })
-            .with_object_pool_capacity(OBJECT_POOL_CAPACITY)
             .build()
     }
 
@@ -1006,9 +958,4 @@ mod tests {
     async fn test_s3fifo_cache() {
         case(s3fifo()).await
     }
-
-    #[tokio::test]
-    async fn test_cache_with_zero_object_pool() {
-        case(CacheBuilder::new(8).with_object_pool_capacity(0).build()).await
-    }
 }
diff --git a/foyer-memory/src/context.rs b/foyer-memory/src/context.rs
deleted file mode 100644
index 3af8ab18..00000000
--- a/foyer-memory/src/context.rs
+++ /dev/null
@@ -1,37 +0,0 @@
-//  Copyright 2024 foyer Project Authors
-//
-//  Licensed under the Apache License, Version 2.0 (the "License");
-//  you may not use this file except in compliance with the License.
-//  You may obtain a copy of the License at
-//
-//  http://www.apache.org/licenses/LICENSE-2.0
-//
-//  Unless required by applicable law or agreed to in writing, software
-//  distributed under the License is distributed on an "AS IS" BASIS,
-//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-//  See the License for the specific language governing permissions and
-//  limitations under the License.
-
-/// Context of the cache entry.
-///
-/// It may be used by the eviction algorithm.
-#[derive(Debug, Clone, Copy, PartialEq, Eq)]
-pub enum CacheContext {
-    /// The default context shared by all eviction container implementations.
-    Default,
-    /// Mark the entry as low-priority.
-    ///
-    /// The behavior differs from different eviction algorithm.
-    LowPriority,
-}
-
-impl Default for CacheContext {
-    fn default() -> Self {
-        Self::Default
-    }
-}
-
-/// The overhead of `Context` itself and the conversion should be light.
-pub trait Context: From<CacheContext> + Into<CacheContext> + Send + Sync + 'static + Clone {}
-
-impl<T> Context for T where T: From<CacheContext> + Into<CacheContext> + Send + Sync + 'static + Clone {}
diff --git a/foyer-memory/src/eviction/fifo.rs b/foyer-memory/src/eviction/fifo.rs
index a1e0d974..680da0a9 100644
--- a/foyer-memory/src/eviction/fifo.rs
+++ b/foyer-memory/src/eviction/fifo.rs
@@ -12,149 +12,111 @@
 //  See the License for the specific language governing permissions and
 //  limitations under the License.
 
-use std::{fmt::Debug, ptr::NonNull};
+use std::{mem::offset_of, sync::Arc};
 
-use foyer_intrusive::{
-    dlist::{Dlist, DlistLink},
-    intrusive_adapter,
-};
+use foyer_common::code::{Key, Value};
+use intrusive_collections::{intrusive_adapter, LinkedList, LinkedListAtomicLink};
 use serde::{Deserialize, Serialize};
 
-use crate::{
-    eviction::Eviction,
-    handle::{BaseHandle, Handle},
-    CacheContext,
-};
+use super::{Eviction, Operator};
+use crate::record::{CacheHint, Record};
 
-#[derive(Debug, Clone)]
-pub struct FifoContext(CacheContext);
+/// Fifo eviction algorithm config.
+#[derive(Debug, Clone, Default, Serialize, Deserialize)]
+pub struct FifoConfig {}
 
-impl From<CacheContext> for FifoContext {
-    fn from(context: CacheContext) -> Self {
-        Self(context)
-    }
-}
+/// Fifo eviction algorithm hint.
+#[derive(Debug, Clone, Default)]
+pub struct FifoHint;
 
-impl From<FifoContext> for CacheContext {
-    fn from(context: FifoContext) -> Self {
-        context.0
+impl From<CacheHint> for FifoHint {
+    fn from(_: CacheHint) -> Self {
+        FifoHint
     }
 }
 
-pub struct FifoHandle<T>
-where
-    T: Send + Sync + 'static,
-{
-    link: DlistLink,
-    base: BaseHandle<T, FifoContext>,
-}
-
-impl<T> Debug for FifoHandle<T>
-where
-    T: Send + Sync + 'static,
-{
-    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-        f.debug_struct("FifoHandle").finish()
+impl From<FifoHint> for CacheHint {
+    fn from(_: FifoHint) -> Self {
+        CacheHint::Normal
     }
 }
 
-intrusive_adapter! { FifoHandleDlistAdapter<T> = FifoHandle<T> { link: DlistLink } where T: Send + Sync + 'static }
-
-impl<T> Default for FifoHandle<T>
-where
-    T: Send + Sync + 'static,
-{
-    fn default() -> Self {
-        Self {
-            link: DlistLink::default(),
-            base: BaseHandle::new(),
-        }
-    }
+/// Fifo eviction algorithm state.
+#[derive(Debug, Default)]
+pub struct FifoState {
+    link: LinkedListAtomicLink,
 }
 
-impl<T> Handle for FifoHandle<T>
-where
-    T: Send + Sync + 'static,
-{
-    type Data = T;
-    type Context = FifoContext;
+intrusive_adapter! { Adapter<K, V> = Arc<Record<Fifo<K, V>>>: Record<Fifo<K, V>> { ?offset = Record::<Fifo<K, V>>::STATE_OFFSET + offset_of!(FifoState, link) => LinkedListAtomicLink } where K: Key, V: Value }
 
-    fn base(&self) -> &BaseHandle<Self::Data, Self::Context> {
-        &self.base
-    }
-
-    fn base_mut(&mut self) -> &mut BaseHandle<Self::Data, Self::Context> {
-        &mut self.base
-    }
-}
-
-/// Fifo eviction algorithm config.
-#[derive(Debug, Clone, Default, Serialize, Deserialize)]
-pub struct FifoConfig {}
-
-pub struct Fifo<T>
+pub struct Fifo<K, V>
 where
-    T: Send + Sync + 'static,
+    K: Key,
+    V: Value,
 {
-    queue: Dlist<FifoHandleDlistAdapter<T>>,
+    queue: LinkedList<Adapter<K, V>>,
 }
 
-impl<T> Eviction for Fifo<T>
+impl<K, V> Eviction for Fifo<K, V>
 where
-    T: Send + Sync + 'static,
+    K: Key,
+    V: Value,
 {
-    type Handle = FifoHandle<T>;
     type Config = FifoConfig;
+    type Key = K;
+    type Value = V;
+    type Hint = FifoHint;
+    type State = FifoState;
 
-    unsafe fn new(_capacity: usize, _config: &Self::Config) -> Self
+    fn new(_capacity: usize, _config: &Self::Config) -> Self
     where
         Self: Sized,
     {
-        Self { queue: Dlist::new() }
+        Self {
+            queue: LinkedList::new(Adapter::new()),
+        }
     }
 
-    unsafe fn push(&mut self, mut ptr: NonNull<Self::Handle>) {
-        self.queue.push_back(ptr);
-        ptr.as_mut().base_mut().set_in_eviction(true);
+    fn update(&mut self, _: usize, _: &Self::Config) {}
+
+    fn push(&mut self, record: Arc<Record<Self>>) {
+        record.set_in_eviction(true);
+        self.queue.push_back(record);
     }
 
-    unsafe fn pop(&mut self) -> Option<NonNull<Self::Handle>> {
-        self.queue.pop_front().map(|mut ptr| {
-            ptr.as_mut().base_mut().set_in_eviction(false);
-            ptr
-        })
+    fn pop(&mut self) -> Option<Arc<Record<Self>>> {
+        self.queue.pop_front().inspect(|record| record.set_in_eviction(false))
     }
 
-    unsafe fn release(&mut self, _: NonNull<Self::Handle>) {}
+    fn remove(&mut self, record: &Arc<Record<Self>>) {
+        unsafe { self.queue.remove_from_ptr(Arc::as_ptr(record)) };
+        record.set_in_eviction(false);
+    }
 
-    unsafe fn acquire(&mut self, _: NonNull<Self::Handle>) {}
+    fn acquire_operator() -> super::Operator {
+        Operator::Noop
+    }
 
-    unsafe fn remove(&mut self, mut ptr: NonNull<Self::Handle>) {
-        let p = self.queue.iter_mut_from_raw(ptr.as_mut().link.raw()).remove().unwrap();
-        assert_eq!(p, ptr);
-        ptr.as_mut().base_mut().set_in_eviction(false);
+    fn acquire_immutable(&self, _record: &Arc<Record<Self>>) {
+        unreachable!()
     }
 
-    unsafe fn clear(&mut self) -> Vec<NonNull<Self::Handle>> {
-        let mut res = Vec::with_capacity(self.len());
-        while let Some(mut ptr) = self.queue.pop_front() {
-            ptr.as_mut().base_mut().set_in_eviction(false);
-            res.push(ptr);
-        }
-        res
+    fn acquire_mutable(&mut self, _record: &Arc<Record<Self>>) {
+        unreachable!()
     }
 
-    fn len(&self) -> usize {
-        self.queue.len()
+    fn release_operator() -> super::Operator {
+        Operator::Noop
     }
 
-    fn is_empty(&self) -> bool {
-        self.len() == 0
+    fn release_immutable(&self, _record: &Arc<Record<Self>>) {
+        unreachable!()
     }
-}
 
-unsafe impl<T> Send for Fifo<T> where T: Send + Sync + 'static {}
-unsafe impl<T> Sync for Fifo<T> where T: Send + Sync + 'static {}
+    fn release_mutable(&mut self, _record: &Arc<Record<Self>>) {
+        unreachable!()
+    }
+}
 
 #[cfg(test)]
 pub mod tests {
@@ -162,67 +124,76 @@ pub mod tests {
     use itertools::Itertools;
 
     use super::*;
-    use crate::{eviction::test_utils::TestEviction, handle::HandleExt};
+    use crate::{
+        eviction::test_utils::{assert_ptr_eq, assert_ptr_vec_eq, TestEviction},
+        record::Data,
+    };
 
-    impl<T> TestEviction for Fifo<T>
+    impl<K, V> TestEviction for Fifo<K, V>
     where
-        T: Send + Sync + 'static + Clone,
+        K: Key + Clone,
+        V: Value + Clone,
     {
-        fn dump(&self) -> Vec<T> {
-            self.queue
-                .iter()
-                .map(|handle| handle.base().data_unwrap_unchecked().clone())
-                .collect_vec()
+        type Dump = Vec<Arc<Record<Self>>>;
+        fn dump(&self) -> Self::Dump {
+            let mut res = vec![];
+            let mut cursor = self.queue.cursor();
+            loop {
+                cursor.move_next();
+                match cursor.clone_pointer() {
+                    Some(record) => res.push(record),
+                    None => break,
+                }
+            }
+            res
         }
     }
 
-    type TestFifoHandle = FifoHandle<u64>;
-    type TestFifo = Fifo<u64>;
-
-    unsafe fn new_test_fifo_handle_ptr(data: u64) -> NonNull<TestFifoHandle> {
-        let mut handle = Box::<TestFifoHandle>::default();
-        handle.init(0, data, 1, FifoContext(CacheContext::Default));
-        NonNull::new_unchecked(Box::into_raw(handle))
-    }
-
-    unsafe fn del_test_fifo_handle_ptr(ptr: NonNull<TestFifoHandle>) {
-        let _ = Box::from_raw(ptr.as_ptr());
-    }
+    type TestFifo = Fifo<u64, u64>;
 
     #[test]
     fn test_fifo() {
-        unsafe {
-            let ptrs = (0..8).map(|i| new_test_fifo_handle_ptr(i)).collect_vec();
-
-            let mut fifo = TestFifo::new(100, &FifoConfig {});
-
-            // 0, 1, 2, 3
-            fifo.push(ptrs[0]);
-            fifo.push(ptrs[1]);
-            fifo.push(ptrs[2]);
-            fifo.push(ptrs[3]);
-
-            // 2, 3
-            let p0 = fifo.pop().unwrap();
-            let p1 = fifo.pop().unwrap();
-            assert_eq!(ptrs[0], p0);
-            assert_eq!(ptrs[1], p1);
-
-            // 2, 3, 4, 5, 6
-            fifo.push(ptrs[4]);
-            fifo.push(ptrs[5]);
-            fifo.push(ptrs[6]);
-
-            // 2, 6
-            fifo.remove(ptrs[3]);
-            fifo.remove(ptrs[4]);
-            fifo.remove(ptrs[5]);
-
-            assert_eq!(fifo.clear(), vec![ptrs[2], ptrs[6]]);
-
-            for ptr in ptrs {
-                del_test_fifo_handle_ptr(ptr);
-            }
-        }
+        let rs = (0..8)
+            .map(|i| {
+                Arc::new(Record::new(Data {
+                    key: i,
+                    value: i,
+                    hint: FifoHint,
+                    hash: i,
+                    weight: 1,
+                }))
+            })
+            .collect_vec();
+        let r = |i: usize| rs[i].clone();
+
+        let mut fifo = TestFifo::new(100, &FifoConfig {});
+
+        // 0, 1, 2, 3
+        fifo.push(r(0));
+        fifo.push(r(1));
+        fifo.push(r(2));
+        fifo.push(r(3));
+
+        // 2, 3
+        let r0 = fifo.pop().unwrap();
+        let r1 = fifo.pop().unwrap();
+        assert_ptr_eq(&rs[0], &r0);
+        assert_ptr_eq(&rs[1], &r1);
+
+        // 2, 3, 4, 5, 6
+        fifo.push(r(4));
+        fifo.push(r(5));
+        fifo.push(r(6));
+
+        // 2, 6
+        fifo.remove(&rs[3]);
+        fifo.remove(&rs[4]);
+        fifo.remove(&rs[5]);
+
+        assert_ptr_vec_eq(fifo.dump(), vec![r(2), r(6)]);
+
+        fifo.clear();
+
+        assert_ptr_vec_eq(fifo.dump(), vec![]);
     }
 }
diff --git a/foyer-memory/src/eviction/lfu.rs b/foyer-memory/src/eviction/lfu.rs
index 5ed7f21c..88e1e215 100644
--- a/foyer-memory/src/eviction/lfu.rs
+++ b/foyer-memory/src/eviction/lfu.rs
@@ -12,22 +12,18 @@
 //  See the License for the specific language governing permissions and
 //  limitations under the License.
 
-use std::{fmt::Debug, ptr::NonNull};
+use std::{mem::offset_of, sync::Arc};
 
 use cmsketch::CMSketchU16;
-use foyer_common::{assert::OptionExt, strict_assert, strict_assert_eq, strict_assert_ne};
-use foyer_intrusive::{
-    adapter::Link,
-    dlist::{Dlist, DlistLink},
-    intrusive_adapter,
+use foyer_common::{
+    code::{Key, Value},
+    strict_assert, strict_assert_eq, strict_assert_ne,
 };
+use intrusive_collections::{intrusive_adapter, LinkedList, LinkedListAtomicLink};
 use serde::{Deserialize, Serialize};
 
-use crate::{
-    eviction::Eviction,
-    handle::{BaseHandle, Handle},
-    CacheContext,
-};
+use super::{Eviction, Operator};
+use crate::record::{CacheHint, Record};
 
 /// w-TinyLFU eviction algorithm config.
 #[derive(Debug, Clone, Serialize, Deserialize)]
@@ -67,22 +63,23 @@ impl Default for LfuConfig {
     }
 }
 
-#[derive(Debug, Clone)]
-pub struct LfuContext(CacheContext);
+/// w-TinyLFU eviction algorithm hint.
+#[derive(Debug, Clone, Default)]
+pub struct LfuHint;
 
-impl From<CacheContext> for LfuContext {
-    fn from(context: CacheContext) -> Self {
-        Self(context)
+impl From<CacheHint> for LfuHint {
+    fn from(_: CacheHint) -> Self {
+        LfuHint
     }
 }
 
-impl From<LfuContext> for CacheContext {
-    fn from(context: LfuContext) -> Self {
-        context.0
+impl From<LfuHint> for CacheHint {
+    fn from(_: LfuHint) -> Self {
+        CacheHint::Normal
     }
 }
 
-#[derive(Debug, PartialEq, Eq)]
+#[derive(Debug, PartialEq, Eq, Clone, Copy)]
 enum Queue {
     None,
     Window,
@@ -90,57 +87,20 @@ enum Queue {
     Protected,
 }
 
-pub struct LfuHandle<T>
-where
-    T: Send + Sync + 'static,
-{
-    link: DlistLink,
-    base: BaseHandle<T, LfuContext>,
-    queue: Queue,
-}
-
-impl<T> Debug for LfuHandle<T>
-where
-    T: Send + Sync + 'static,
-{
-    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-        f.debug_struct("LfuHandle").finish()
-    }
-}
-
-intrusive_adapter! { LfuHandleDlistAdapter<T> = LfuHandle<T> { link: DlistLink } where T: Send + Sync + 'static }
-
-impl<T> Default for LfuHandle<T>
-where
-    T: Send + Sync + 'static,
-{
+impl Default for Queue {
     fn default() -> Self {
-        Self {
-            link: DlistLink::default(),
-            base: BaseHandle::new(),
-            queue: Queue::None,
-        }
+        Self::None
     }
 }
 
-impl<T> Handle for LfuHandle<T>
-where
-    T: Send + Sync + 'static,
-{
-    type Data = T;
-    type Context = LfuContext;
-
-    fn base(&self) -> &BaseHandle<Self::Data, Self::Context> {
-        &self.base
-    }
-
-    fn base_mut(&mut self) -> &mut BaseHandle<Self::Data, Self::Context> {
-        &mut self.base
-    }
+/// w-TinyLFU eviction algorithm hint.
+#[derive(Debug, Default)]
+pub struct LfuState {
+    link: LinkedListAtomicLink,
+    queue: Queue,
 }
 
-unsafe impl<T> Send for LfuHandle<T> where T: Send + Sync + 'static {}
-unsafe impl<T> Sync for LfuHandle<T> where T: Send + Sync + 'static {}
+intrusive_adapter! { Adapter<K, V> = Arc<Record<Lfu<K, V>>>: Record<Lfu<K, V>> { ?offset = Record::<Lfu<K, V>>::STATE_OFFSET + offset_of!(LfuState, link) => LinkedListAtomicLink } where K: Key, V: Value }
 
 /// This implementation is inspired by [Caffeine](https://github.com/ben-manes/caffeine) under Apache License 2.0
 ///
@@ -154,13 +114,14 @@ unsafe impl<T> Sync for LfuHandle<T> where T: Send + Sync + 'static {}
 ///
 /// When evicting, the entry with a lower frequency from `window` or `probation` will be evicted first, then from
 /// `protected`.
-pub struct Lfu<T>
+pub struct Lfu<K, V>
 where
-    T: Send + Sync + 'static,
+    K: Key,
+    V: Value,
 {
-    window: Dlist<LfuHandleDlistAdapter<T>>,
-    probation: Dlist<LfuHandleDlistAdapter<T>>,
-    protected: Dlist<LfuHandleDlistAdapter<T>>,
+    window: LinkedList<Adapter<K, V>>,
+    probation: LinkedList<Adapter<K, V>>,
+    protected: LinkedList<Adapter<K, V>>,
 
     window_weight: usize,
     probation_weight: usize,
@@ -169,19 +130,20 @@ where
     window_weight_capacity: usize,
     protected_weight_capacity: usize,
 
+    // TODO(MrCroxx): use a count-min-sketch impl with atomic u16
     frequencies: CMSketchU16,
 
     step: usize,
     decay: usize,
 }
 
-impl<T> Lfu<T>
+impl<K, V> Lfu<K, V>
 where
-    T: Send + Sync + 'static,
+    K: Key,
+    V: Value,
 {
-    fn increase_queue_weight(&mut self, handle: &LfuHandle<T>) {
-        let weight = handle.base().weight();
-        match handle.queue {
+    fn increase_queue_weight(&mut self, queue: Queue, weight: usize) {
+        match queue {
             Queue::None => unreachable!(),
             Queue::Window => self.window_weight += weight,
             Queue::Probation => self.probation_weight += weight,
@@ -189,9 +151,8 @@ where
         }
     }
 
-    fn decrease_queue_weight(&mut self, handle: &LfuHandle<T>) {
-        let weight = handle.base().weight();
-        match handle.queue {
+    fn decrease_queue_weight(&mut self, queue: Queue, weight: usize) {
+        match queue {
             Queue::None => unreachable!(),
             Queue::Window => self.window_weight -= weight,
             Queue::Probation => self.probation_weight -= weight,
@@ -209,14 +170,18 @@ where
     }
 }
 
-impl<T> Eviction for Lfu<T>
+impl<K, V> Eviction for Lfu<K, V>
 where
-    T: Send + Sync + 'static,
+    K: Key,
+    V: Value,
 {
-    type Handle = LfuHandle<T>;
     type Config = LfuConfig;
+    type Key = K;
+    type Value = V;
+    type Hint = LfuHint;
+    type State = LfuState;
 
-    unsafe fn new(capacity: usize, config: &Self::Config) -> Self
+    fn new(capacity: usize, config: &Self::Config) -> Self
     where
         Self: Sized,
     {
@@ -244,9 +209,9 @@ where
         let decay = frequencies.width();
 
         Self {
-            window: Dlist::new(),
-            probation: Dlist::new(),
-            protected: Dlist::new(),
+            window: LinkedList::new(Adapter::new()),
+            probation: LinkedList::new(Adapter::new()),
+            protected: LinkedList::new(Adapter::new()),
             window_weight: 0,
             probation_weight: 0,
             protected_weight: 0,
@@ -258,307 +223,439 @@ where
         }
     }
 
-    unsafe fn push(&mut self, mut ptr: NonNull<Self::Handle>) {
-        let handle = ptr.as_mut();
+    fn update(&mut self, capacity: usize, config: &Self::Config) {
+        if config.window_capacity_ratio <= 0.0 || config.window_capacity_ratio >= 1.0 {
+            tracing::error!(
+                "window_capacity_ratio must be in (0, 1), given: {}, new config ignored",
+                config.window_capacity_ratio
+            );
+        }
+
+        if config.protected_capacity_ratio <= 0.0 || config.protected_capacity_ratio >= 1.0 {
+            tracing::error!(
+                "protected_capacity_ratio must be in (0, 1), given: {}, new config ignored",
+                config.protected_capacity_ratio
+            );
+        }
+
+        if config.window_capacity_ratio + config.protected_capacity_ratio >= 1.0 {
+            tracing::error!(
+                "must guarantee: window_capacity_ratio + protected_capacity_ratio < 1, given: {}, new config ignored",
+                config.window_capacity_ratio + config.protected_capacity_ratio
+            )
+        }
+
+        // TODO(MrCroxx): Raise a warn log the cmsketch args updates is not supported yet if it is modified.
+
+        let window_weight_capacity = (capacity as f64 * config.window_capacity_ratio) as usize;
+        let protected_weight_capacity = (capacity as f64 * config.protected_capacity_ratio) as usize;
+
+        self.window_weight_capacity = window_weight_capacity;
+        self.protected_weight_capacity = protected_weight_capacity;
+    }
 
-        strict_assert!(!handle.link.is_linked());
-        strict_assert!(!handle.base().is_in_eviction());
-        strict_assert_eq!(handle.queue, Queue::None);
+    /// Push a new record to `window`.
+    ///
+    /// Overflow record from `window` to `probation` if needed.
+    fn push(&mut self, record: Arc<Record<Self>>) {
+        let state = unsafe { &mut *record.state().get() };
 
-        self.window.push_back(ptr);
-        handle.base_mut().set_in_eviction(true);
-        handle.queue = Queue::Window;
+        strict_assert!(!state.link.is_linked());
+        strict_assert!(!record.is_in_eviction());
+        strict_assert_eq!(state.queue, Queue::None);
 
-        self.increase_queue_weight(handle);
-        self.update_frequencies(handle.base().hash());
+        record.set_in_eviction(true);
+        state.queue = Queue::Window;
+        self.increase_queue_weight(Queue::Window, record.weight());
+        self.update_frequencies(record.hash());
+        self.window.push_back(record);
 
         // If `window` weight exceeds the capacity, overflow entry from `window` to `probation`.
         while self.window_weight > self.window_weight_capacity {
             strict_assert!(!self.window.is_empty());
-            let mut ptr = self.window.pop_front().strict_unwrap_unchecked();
-            let handle = ptr.as_mut();
-            self.decrease_queue_weight(handle);
-            handle.queue = Queue::Probation;
-            self.increase_queue_weight(handle);
-            self.probation.push_back(ptr);
+            let r = self.window.pop_front().unwrap();
+            let s = unsafe { &mut *r.state().get() };
+            self.decrease_queue_weight(Queue::Window, r.weight());
+            s.queue = Queue::Probation;
+            self.increase_queue_weight(Queue::Probation, r.weight());
+            self.probation.push_back(r);
         }
     }
 
-    unsafe fn pop(&mut self) -> Option<NonNull<Self::Handle>> {
+    fn pop(&mut self) -> Option<Arc<Record<Self>>> {
         // Compare the frequency of the front element of `window` and `probation` queue, and evict the lower one.
         // If both `window` and `probation` are empty, try evict from `protected`.
-        let mut ptr = match (self.window.front(), self.probation.front()) {
+        let mut cw = self.window.front_mut();
+        let mut cp = self.probation.front_mut();
+        let record = match (cw.get(), cp.get()) {
             (None, None) => None,
-            (None, Some(_)) => self.probation.pop_front(),
-            (Some(_), None) => self.window.pop_front(),
-            (Some(window), Some(probation)) => {
-                if self.frequencies.estimate(window.base().hash()) < self.frequencies.estimate(probation.base().hash())
-                {
-                    self.window.pop_front()
+            (None, Some(_)) => cp.remove(),
+            (Some(_), None) => cw.remove(),
+            (Some(w), Some(p)) => {
+                if self.frequencies.estimate(w.hash()) < self.frequencies.estimate(p.hash()) {
+                    cw.remove()
 
                     // TODO(MrCroxx): Rotate probation to prevent a high frequency but cold head holds back promotion
                     // too long like CacheLib does?
                 } else {
-                    self.probation.pop_front()
+                    cp.remove()
                 }
             }
         }
         .or_else(|| self.protected.pop_front())?;
 
-        let handle = ptr.as_mut();
+        let state = unsafe { &mut *record.state().get() };
 
-        strict_assert!(!handle.link.is_linked());
-        strict_assert!(handle.base().is_in_eviction());
-        strict_assert_ne!(handle.queue, Queue::None);
+        strict_assert!(!state.link.is_linked());
+        strict_assert!(record.is_in_eviction());
+        strict_assert_ne!(state.queue, Queue::None);
 
-        self.decrease_queue_weight(handle);
-        handle.queue = Queue::None;
-        handle.base_mut().set_in_eviction(false);
+        self.decrease_queue_weight(state.queue, record.weight());
+        state.queue = Queue::None;
+        record.set_in_eviction(false);
 
-        Some(ptr)
+        Some(record)
     }
 
-    unsafe fn release(&mut self, mut ptr: NonNull<Self::Handle>) {
-        let handle = ptr.as_mut();
+    fn remove(&mut self, record: &Arc<Record<Self>>) {
+        let state = unsafe { &mut *record.state().get() };
 
-        match handle.queue {
-            Queue::None => {
-                strict_assert!(!handle.link.is_linked());
-                strict_assert!(!handle.base().is_in_eviction());
-                self.push(ptr);
-                strict_assert!(handle.link.is_linked());
-                strict_assert!(handle.base().is_in_eviction());
-            }
+        strict_assert!(state.link.is_linked());
+        strict_assert!(record.is_in_eviction());
+        strict_assert_ne!(state.queue, Queue::None);
+
+        match state.queue {
+            Queue::None => unreachable!(),
+            Queue::Window => unsafe { self.window.remove_from_ptr(Arc::as_ptr(record)) },
+            Queue::Probation => unsafe { self.probation.remove_from_ptr(Arc::as_ptr(record)) },
+            Queue::Protected => unsafe { self.protected.remove_from_ptr(Arc::as_ptr(record)) },
+        };
+
+        strict_assert!(!state.link.is_linked());
+
+        self.decrease_queue_weight(state.queue, record.weight());
+        state.queue = Queue::None;
+        record.set_in_eviction(false);
+    }
+
+    fn clear(&mut self) {
+        while let Some(record) = self.pop() {
+            let state = unsafe { &*record.state().get() };
+            strict_assert!(!record.is_in_eviction());
+            strict_assert!(!state.link.is_linked());
+            strict_assert_eq!(state.queue, Queue::None);
+        }
+    }
+
+    fn acquire_operator() -> super::Operator {
+        // TODO(MrCroxx): use a count-min-sketch with atomic u16 impl.
+        Operator::Mutable
+    }
+
+    fn acquire_immutable(&self, _record: &Arc<Record<Self>>) {
+        unreachable!()
+    }
+
+    fn acquire_mutable(&mut self, record: &Arc<Record<Self>>) {
+        // Update frequency by access.
+        self.update_frequencies(record.hash());
+
+        if !record.is_in_eviction() {
+            return;
+        }
+
+        let state = unsafe { &mut *record.state().get() };
+
+        strict_assert!(state.link.is_linked());
+
+        match state.queue {
+            Queue::None => unreachable!(),
             Queue::Window => {
                 // Move to MRU position of `window`.
-                strict_assert!(handle.link.is_linked());
-                strict_assert!(handle.base().is_in_eviction());
-                self.window.remove_raw(handle.link.raw());
-                self.window.push_back(ptr);
+                let r = unsafe { self.window.remove_from_ptr(Arc::as_ptr(record)) };
+                self.window.push_back(r);
             }
             Queue::Probation => {
                 // Promote to MRU position of `protected`.
-                strict_assert!(handle.link.is_linked());
-                strict_assert!(handle.base().is_in_eviction());
-                self.probation.remove_raw(handle.link.raw());
-                self.decrease_queue_weight(handle);
-                handle.queue = Queue::Protected;
-                self.increase_queue_weight(handle);
-                self.protected.push_back(ptr);
+                let r = unsafe { self.probation.remove_from_ptr(Arc::as_ptr(record)) };
+                self.decrease_queue_weight(Queue::Probation, record.weight());
+                state.queue = Queue::Protected;
+                self.increase_queue_weight(Queue::Protected, record.weight());
+                self.protected.push_back(r);
 
                 // If `protected` weight exceeds the capacity, overflow entry from `protected` to `probation`.
                 while self.protected_weight > self.protected_weight_capacity {
                     strict_assert!(!self.protected.is_empty());
-                    let mut ptr = self.protected.pop_front().strict_unwrap_unchecked();
-                    let handle = ptr.as_mut();
-                    self.decrease_queue_weight(handle);
-                    handle.queue = Queue::Probation;
-                    self.increase_queue_weight(handle);
-                    self.probation.push_back(ptr);
+                    let r = self.protected.pop_front().unwrap();
+                    let s = unsafe { &mut *r.state().get() };
+                    self.decrease_queue_weight(Queue::Protected, r.weight());
+                    s.queue = Queue::Probation;
+                    self.increase_queue_weight(Queue::Probation, r.weight());
+                    self.probation.push_back(r);
                 }
             }
             Queue::Protected => {
                 // Move to MRU position of `protected`.
-                strict_assert!(handle.link.is_linked());
-                strict_assert!(handle.base().is_in_eviction());
-                self.protected.remove_raw(handle.link.raw());
-                self.protected.push_back(ptr);
+                let r = unsafe { self.protected.remove_from_ptr(Arc::as_ptr(record)) };
+                self.protected.push_back(r);
             }
         }
     }
 
-    unsafe fn acquire(&mut self, ptr: NonNull<Self::Handle>) {
-        self.update_frequencies(ptr.as_ref().base().hash());
-    }
-
-    unsafe fn remove(&mut self, mut ptr: NonNull<Self::Handle>) {
-        let handle = ptr.as_mut();
-
-        strict_assert!(handle.link.is_linked());
-        strict_assert!(handle.base().is_in_eviction());
-        strict_assert_ne!(handle.queue, Queue::None);
-
-        match handle.queue {
-            Queue::None => unreachable!(),
-            Queue::Window => self.window.remove_raw(handle.link.raw()),
-            Queue::Probation => self.probation.remove_raw(handle.link.raw()),
-            Queue::Protected => self.protected.remove_raw(handle.link.raw()),
-        };
-
-        strict_assert!(!handle.link.is_linked());
-
-        self.decrease_queue_weight(handle);
-        handle.queue = Queue::None;
-        handle.base_mut().set_in_eviction(false);
-    }
-
-    unsafe fn clear(&mut self) -> Vec<NonNull<Self::Handle>> {
-        let mut res = Vec::with_capacity(self.len());
-
-        while !self.is_empty() {
-            let ptr = self.pop().strict_unwrap_unchecked();
-            strict_assert!(!ptr.as_ref().base().is_in_eviction());
-            strict_assert!(!ptr.as_ref().link.is_linked());
-            strict_assert_eq!(ptr.as_ref().queue, Queue::None);
-            res.push(ptr);
-        }
-
-        res
+    fn release_operator() -> Operator {
+        Operator::Noop
     }
 
-    fn len(&self) -> usize {
-        self.window.len() + self.probation.len() + self.protected.len()
+    fn release_immutable(&self, _record: &Arc<Record<Self>>) {
+        unreachable!()
     }
 
-    fn is_empty(&self) -> bool {
-        self.len() == 0
+    fn release_mutable(&mut self, _record: &Arc<Record<Self>>) {
+        unreachable!()
     }
 }
 
-unsafe impl<T> Send for Lfu<T> where T: Send + Sync + 'static {}
-unsafe impl<T> Sync for Lfu<T> where T: Send + Sync + 'static {}
-
 #[cfg(test)]
 mod tests {
 
     use itertools::Itertools;
 
     use super::*;
-    use crate::{eviction::test_utils::TestEviction, handle::HandleExt};
+    use crate::{
+        eviction::test_utils::{assert_ptr_eq, assert_ptr_vec_vec_eq, TestEviction},
+        record::Data,
+    };
 
-    impl<T> TestEviction for Lfu<T>
+    impl<K, V> TestEviction for Lfu<K, V>
     where
-        T: Send + Sync + 'static + Clone,
+        K: Key + Clone,
+        V: Value + Clone,
     {
-        fn dump(&self) -> Vec<T> {
-            self.window
-                .iter()
-                .chain(self.probation.iter())
-                .chain(self.protected.iter())
-                .map(|handle| handle.base().data_unwrap_unchecked().clone())
-                .collect_vec()
-        }
-    }
+        type Dump = Vec<Vec<Arc<Record<Self>>>>;
+        fn dump(&self) -> Self::Dump {
+            let mut window = vec![];
+            let mut probation = vec![];
+            let mut protected = vec![];
+
+            let mut cursor = self.window.cursor();
+            loop {
+                cursor.move_next();
+                match cursor.clone_pointer() {
+                    Some(record) => window.push(record),
+                    None => break,
+                }
+            }
 
-    type TestLfu = Lfu<u64>;
-    type TestLfuHandle = LfuHandle<u64>;
-
-    unsafe fn assert_test_lfu(
-        lfu: &TestLfu,
-        len: usize,
-        window: usize,
-        probation: usize,
-        protected: usize,
-        entries: Vec<u64>,
-    ) {
-        assert_eq!(lfu.len(), len);
-        assert_eq!(lfu.window.len(), window);
-        assert_eq!(lfu.probation.len(), probation);
-        assert_eq!(lfu.protected.len(), protected);
-        assert_eq!(lfu.window_weight, window);
-        assert_eq!(lfu.probation_weight, probation);
-        assert_eq!(lfu.protected_weight, protected);
-        let es = lfu.dump().into_iter().collect_vec();
-        assert_eq!(es, entries);
-    }
+            let mut cursor = self.probation.cursor();
+            loop {
+                cursor.move_next();
+                match cursor.clone_pointer() {
+                    Some(record) => probation.push(record),
+                    None => break,
+                }
+            }
 
-    fn assert_min_frequency(lfu: &TestLfu, hash: u64, count: usize) {
-        let freq = lfu.frequencies.estimate(hash);
-        assert!(freq >= count as u16, "assert {freq} >= {count} failed for {hash}");
+            let mut cursor = self.protected.cursor();
+            loop {
+                cursor.move_next();
+                match cursor.clone_pointer() {
+                    Some(record) => protected.push(record),
+                    None => break,
+                }
+            }
+
+            vec![window, probation, protected]
+        }
     }
 
+    type TestLfu = Lfu<u64, u64>;
+
     #[test]
     fn test_lfu() {
-        unsafe {
-            let ptrs = (0..100)
-                .map(|i| {
-                    let mut handle = Box::<TestLfuHandle>::default();
-                    handle.init(i, i, 1, LfuContext(CacheContext::Default));
-                    NonNull::new_unchecked(Box::into_raw(handle))
-                })
-                .collect_vec();
-
-            // window: 2, probation: 2, protected: 6
-            let config = LfuConfig {
-                window_capacity_ratio: 0.2,
-                protected_capacity_ratio: 0.6,
-                cmsketch_eps: 0.01,
-                cmsketch_confidence: 0.95,
-            };
-            let mut lfu = TestLfu::new(10, &config);
-
-            assert_eq!(lfu.window_weight_capacity, 2);
-            assert_eq!(lfu.protected_weight_capacity, 6);
-
-            lfu.push(ptrs[0]);
-            lfu.push(ptrs[1]);
-            assert_test_lfu(&lfu, 2, 2, 0, 0, vec![0, 1]);
-
-            lfu.push(ptrs[2]);
-            lfu.push(ptrs[3]);
-            assert_test_lfu(&lfu, 4, 2, 2, 0, vec![2, 3, 0, 1]);
-
-            (4..10).for_each(|i| lfu.push(ptrs[i]));
-            assert_test_lfu(&lfu, 10, 2, 8, 0, vec![8, 9, 0, 1, 2, 3, 4, 5, 6, 7]);
-
-            (0..10).for_each(|i| assert_min_frequency(&lfu, i, 1));
-
-            // [8, 9] [1, 2, 3, 4, 5, 6, 7]
-            let p0 = lfu.pop().unwrap();
-            assert_eq!(p0, ptrs[0]);
-
-            // [9, 0] [1, 2, 3, 4, 5, 6, 7, 8]
-            lfu.release(p0);
-            assert_test_lfu(&lfu, 10, 2, 8, 0, vec![9, 0, 1, 2, 3, 4, 5, 6, 7, 8]);
-
-            // [0, 9] [1, 2, 3, 4, 5, 6, 7, 8]
-            lfu.release(ptrs[9]);
-            assert_test_lfu(&lfu, 10, 2, 8, 0, vec![0, 9, 1, 2, 3, 4, 5, 6, 7, 8]);
-
-            // [0, 9] [1, 2, 7, 8] [3, 4, 5, 6]
-            (3..7).for_each(|i| lfu.release(ptrs[i]));
-            assert_test_lfu(&lfu, 10, 2, 4, 4, vec![0, 9, 1, 2, 7, 8, 3, 4, 5, 6]);
-
-            // [0, 9] [1, 2, 7, 8] [5, 6, 3, 4]
-            (3..5).for_each(|i| lfu.release(ptrs[i]));
-            assert_test_lfu(&lfu, 10, 2, 4, 4, vec![0, 9, 1, 2, 7, 8, 5, 6, 3, 4]);
-
-            // [0, 9] [5, 6] [3, 4, 1, 2, 7, 8]
-            [1, 2, 7, 8].into_iter().for_each(|i| lfu.release(ptrs[i]));
-            assert_test_lfu(&lfu, 10, 2, 2, 6, vec![0, 9, 5, 6, 3, 4, 1, 2, 7, 8]);
-
-            // [0, 9] [6] [3, 4, 1, 2, 7, 8]
-            let p5 = lfu.pop().unwrap();
-            assert_eq!(p5, ptrs[5]);
-            assert_test_lfu(&lfu, 9, 2, 1, 6, vec![0, 9, 6, 3, 4, 1, 2, 7, 8]);
-
-            (10..13).for_each(|i| lfu.push(ptrs[i]));
-
-            // [11, 12] [6, 0, 9, 10] [3, 4, 1, 2, 7, 8]
-            assert_test_lfu(&lfu, 12, 2, 4, 6, vec![11, 12, 6, 0, 9, 10, 3, 4, 1, 2, 7, 8]);
-            (1..13).for_each(|i| assert_min_frequency(&lfu, i, 0));
-            lfu.acquire(ptrs[0]);
-            assert_min_frequency(&lfu, 0, 2);
-
-            // evict 11 because freq(11) < freq(0)
-            // [12] [0, 9, 10] [3, 4, 1, 2, 7, 8]
-            let p6 = lfu.pop().unwrap();
-            let p11 = lfu.pop().unwrap();
-            assert_eq!(p6, ptrs[6]);
-            assert_eq!(p11, ptrs[11]);
-            assert_test_lfu(&lfu, 10, 1, 3, 6, vec![12, 0, 9, 10, 3, 4, 1, 2, 7, 8]);
-
-            assert_eq!(
-                lfu.clear(),
-                [12, 0, 9, 10, 3, 4, 1, 2, 7, 8]
-                    .into_iter()
-                    .map(|i| ptrs[i])
-                    .collect_vec()
-            );
+        let rs = (0..100)
+            .map(|i| {
+                Arc::new(Record::new(Data {
+                    key: i,
+                    value: i,
+                    hint: LfuHint,
+                    hash: i,
+                    weight: 1,
+                }))
+            })
+            .collect_vec();
+        let r = |i: usize| rs[i].clone();
+
+        // window: 2, probation: 2, protected: 6
+        let config = LfuConfig {
+            window_capacity_ratio: 0.2,
+            protected_capacity_ratio: 0.6,
+            cmsketch_eps: 0.01,
+            cmsketch_confidence: 0.95,
+        };
+        let mut lfu = TestLfu::new(10, &config);
+
+        assert_eq!(lfu.window_weight_capacity, 2);
+        assert_eq!(lfu.protected_weight_capacity, 6);
+
+        lfu.push(r(0));
+        lfu.push(r(1));
+        assert_ptr_vec_vec_eq(lfu.dump(), vec![vec![r(0), r(1)], vec![], vec![]]);
+
+        lfu.push(r(2));
+        lfu.push(r(3));
+        assert_ptr_vec_vec_eq(lfu.dump(), vec![vec![r(2), r(3)], vec![r(0), r(1)], vec![]]);
+
+        (4..10).for_each(|i| lfu.push(r(i)));
+        assert_ptr_vec_vec_eq(
+            lfu.dump(),
+            vec![
+                vec![r(8), r(9)],
+                vec![r(0), r(1), r(2), r(3), r(4), r(5), r(6), r(7)],
+                vec![],
+            ],
+        );
 
-            for ptr in ptrs {
-                let _ = Box::from_raw(ptr.as_ptr());
-            }
-        }
+        // [8, 9] [1, 2, 3, 4, 5, 6, 7]
+        let r0 = lfu.pop().unwrap();
+        assert_ptr_eq(&rs[0], &r0);
+
+        // [9, 0] [1, 2, 3, 4, 5, 6, 7, 8]
+        lfu.push(r(0));
+        assert_ptr_vec_vec_eq(
+            lfu.dump(),
+            vec![
+                vec![r(9), r(0)],
+                vec![r(1), r(2), r(3), r(4), r(5), r(6), r(7), r(8)],
+                vec![],
+            ],
+        );
+
+        // [0, 9] [1, 2, 3, 4, 5, 6, 7, 8]
+        lfu.acquire_mutable(&rs[9]);
+        assert_ptr_vec_vec_eq(
+            lfu.dump(),
+            vec![
+                vec![r(0), r(9)],
+                vec![r(1), r(2), r(3), r(4), r(5), r(6), r(7), r(8)],
+                vec![],
+            ],
+        );
+
+        // [0, 9] [1, 2, 7, 8] [3, 4, 5, 6]
+        (3..7).for_each(|i| lfu.acquire_mutable(&rs[i]));
+        assert_ptr_vec_vec_eq(
+            lfu.dump(),
+            vec![
+                vec![r(0), r(9)],
+                vec![r(1), r(2), r(7), r(8)],
+                vec![r(3), r(4), r(5), r(6)],
+            ],
+        );
+
+        // [0, 9] [1, 2, 7, 8] [5, 6, 3, 4]
+        (3..5).for_each(|i| lfu.acquire_mutable(&rs[i]));
+        assert_ptr_vec_vec_eq(
+            lfu.dump(),
+            vec![
+                vec![r(0), r(9)],
+                vec![r(1), r(2), r(7), r(8)],
+                vec![r(5), r(6), r(3), r(4)],
+            ],
+        );
+
+        // [0, 9] [5, 6] [3, 4, 1, 2, 7, 8]
+        [1, 2, 7, 8].into_iter().for_each(|i| lfu.acquire_mutable(&rs[i]));
+        assert_ptr_vec_vec_eq(
+            lfu.dump(),
+            vec![
+                vec![r(0), r(9)],
+                vec![r(5), r(6)],
+                vec![r(3), r(4), r(1), r(2), r(7), r(8)],
+            ],
+        );
+
+        // [0, 9] [6] [3, 4, 1, 2, 7, 8]
+        let r5 = lfu.pop().unwrap();
+        assert_ptr_eq(&rs[5], &r5);
+        assert_ptr_vec_vec_eq(
+            lfu.dump(),
+            vec![vec![r(0), r(9)], vec![r(6)], vec![r(3), r(4), r(1), r(2), r(7), r(8)]],
+        );
+
+        // [11, 12] [6, 0, 9, 10] [3, 4, 1, 2, 7, 8]
+        (10..13).for_each(|i| lfu.push(r(i)));
+        assert_ptr_vec_vec_eq(
+            lfu.dump(),
+            vec![
+                vec![r(11), r(12)],
+                vec![r(6), r(0), r(9), r(10)],
+                vec![r(3), r(4), r(1), r(2), r(7), r(8)],
+            ],
+        );
+
+        // 0: high freq
+        // [11, 12] [6, 9, 10, 3] [4, 1, 2, 7, 8, 0]
+        (0..10).for_each(|_| lfu.acquire_mutable(&rs[0]));
+        assert_ptr_vec_vec_eq(
+            lfu.dump(),
+            vec![
+                vec![r(11), r(12)],
+                vec![r(6), r(9), r(10), r(3)],
+                vec![r(4), r(1), r(2), r(7), r(8), r(0)],
+            ],
+        );
+
+        // 0: high freq
+        // [11, 12] [0, 6, 9, 10] [3, 4, 1, 2, 7, 8]
+        lfu.acquire_mutable(&rs[6]);
+        lfu.acquire_mutable(&rs[9]);
+        lfu.acquire_mutable(&rs[10]);
+        lfu.acquire_mutable(&rs[3]);
+        lfu.acquire_mutable(&rs[4]);
+        lfu.acquire_mutable(&rs[1]);
+        lfu.acquire_mutable(&rs[2]);
+        lfu.acquire_mutable(&rs[7]);
+        lfu.acquire_mutable(&rs[8]);
+        assert_ptr_vec_vec_eq(
+            lfu.dump(),
+            vec![
+                vec![r(11), r(12)],
+                vec![r(0), r(6), r(9), r(10)],
+                vec![r(3), r(4), r(1), r(2), r(7), r(8)],
+            ],
+        );
+
+        // evict 11, 12 because freq(11) < freq(0), freq(12) < freq(0)
+        // [12] [0, 9, 10] [3, 4, 1, 2, 7, 8]
+        assert!(lfu.frequencies.estimate(0) > lfu.frequencies.estimate(11));
+        assert!(lfu.frequencies.estimate(0) > lfu.frequencies.estimate(12));
+        let r11 = lfu.pop().unwrap();
+        let r12 = lfu.pop().unwrap();
+        assert_ptr_eq(&rs[11], &r11);
+        assert_ptr_eq(&rs[12], &r12);
+        assert_ptr_vec_vec_eq(
+            lfu.dump(),
+            vec![
+                vec![],
+                vec![r(0), r(6), r(9), r(10)],
+                vec![r(3), r(4), r(1), r(2), r(7), r(8)],
+            ],
+        );
+
+        // evict 0, high freq but cold
+        // [] [6, 9, 10] [3, 4, 1, 2, 7, 8]
+        let r0 = lfu.pop().unwrap();
+        assert_ptr_eq(&rs[0], &r0);
+        assert_ptr_vec_vec_eq(
+            lfu.dump(),
+            vec![
+                vec![],
+                vec![r(6), r(9), r(10)],
+                vec![r(3), r(4), r(1), r(2), r(7), r(8)],
+            ],
+        );
+
+        lfu.clear();
+        assert_ptr_vec_vec_eq(lfu.dump(), vec![vec![], vec![], vec![]]);
     }
 }
diff --git a/foyer-memory/src/eviction/lru.rs b/foyer-memory/src/eviction/lru.rs
index 132df3c6..74c77b11 100644
--- a/foyer-memory/src/eviction/lru.rs
+++ b/foyer-memory/src/eviction/lru.rs
@@ -12,23 +12,19 @@
 //  See the License for the specific language governing permissions and
 //  limitations under the License.
 
-use std::{fmt::Debug, ptr::NonNull};
+use std::{mem::offset_of, sync::Arc};
 
-use foyer_common::{assert::OptionExt, strict_assert, strict_assert_eq};
-use foyer_intrusive::{
-    adapter::Link,
-    dlist::{Dlist, DlistLink},
-    intrusive_adapter,
+use foyer_common::{
+    code::{Key, Value},
+    strict_assert,
 };
+use intrusive_collections::{intrusive_adapter, LinkedList, LinkedListAtomicLink};
 use serde::{Deserialize, Serialize};
 
-use crate::{
-    eviction::Eviction,
-    handle::{BaseHandle, Handle},
-    CacheContext,
-};
+use super::{Eviction, Operator};
+use crate::record::{CacheHint, Record};
 
-/// LRU eviction algorithm config.
+/// Lru eviction algorithm config.
 #[derive(Debug, Clone, Serialize, Deserialize)]
 pub struct LruConfig {
     /// The ratio of the high priority pool occupied.
@@ -45,248 +41,279 @@ pub struct LruConfig {
 impl Default for LruConfig {
     fn default() -> Self {
         Self {
-            high_priority_pool_ratio: 0.0,
+            high_priority_pool_ratio: 0.9,
         }
     }
 }
 
-#[derive(Debug, Clone, PartialEq, Eq)]
-pub enum LruContext {
+/// Lru eviction algorithm hint.
+#[derive(Debug, Clone)]
+pub enum LruHint {
     HighPriority,
     LowPriority,
 }
 
-impl From<CacheContext> for LruContext {
-    fn from(value: CacheContext) -> Self {
-        match value {
-            CacheContext::Default => Self::HighPriority,
-            CacheContext::LowPriority => Self::LowPriority,
-        }
+impl Default for LruHint {
+    fn default() -> Self {
+        Self::HighPriority
     }
 }
 
-impl From<LruContext> for CacheContext {
-    fn from(value: LruContext) -> Self {
-        match value {
-            LruContext::HighPriority => CacheContext::Default,
-            LruContext::LowPriority => CacheContext::LowPriority,
+impl From<CacheHint> for LruHint {
+    fn from(hint: CacheHint) -> Self {
+        match hint {
+            CacheHint::Normal => LruHint::HighPriority,
+            CacheHint::Low => LruHint::LowPriority,
         }
     }
 }
 
-pub struct LruHandle<T>
-where
-    T: Send + Sync + 'static,
-{
-    link: DlistLink,
-    base: BaseHandle<T, LruContext>,
-    in_high_priority_pool: bool,
-}
-
-impl<T> Debug for LruHandle<T>
-where
-    T: Send + Sync + 'static,
-{
-    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-        f.debug_struct("LruHandle").finish()
-    }
-}
-
-intrusive_adapter! { LruHandleDlistAdapter<T> = LruHandle<T> { link: DlistLink } where T: Send + Sync + 'static }
-
-impl<T> Default for LruHandle<T>
-where
-    T: Send + Sync + 'static,
-{
-    fn default() -> Self {
-        Self {
-            link: DlistLink::default(),
-            base: BaseHandle::new(),
-            in_high_priority_pool: false,
+impl From<LruHint> for CacheHint {
+    fn from(hint: LruHint) -> Self {
+        match hint {
+            LruHint::HighPriority => CacheHint::Normal,
+            LruHint::LowPriority => CacheHint::Low,
         }
     }
 }
 
-impl<T> Handle for LruHandle<T>
-where
-    T: Send + Sync + 'static,
-{
-    type Data = T;
-    type Context = LruContext;
-
-    fn base(&self) -> &BaseHandle<Self::Data, Self::Context> {
-        &self.base
-    }
-
-    fn base_mut(&mut self) -> &mut BaseHandle<Self::Data, Self::Context> {
-        &mut self.base
-    }
+/// Lru eviction algorithm state.
+#[derive(Debug, Default)]
+pub struct LruState {
+    link: LinkedListAtomicLink,
+    in_high_priority_pool: bool,
+    is_pinned: bool,
 }
 
-unsafe impl<T> Send for LruHandle<T> where T: Send + Sync + 'static {}
-unsafe impl<T> Sync for LruHandle<T> where T: Send + Sync + 'static {}
+intrusive_adapter! { Adapter<K, V> = Arc<Record<Lru<K, V>>>: Record<Lru<K, V>> { ?offset = Record::<Lru<K, V>>::STATE_OFFSET + offset_of!(LruState, link) => LinkedListAtomicLink } where K: Key, V: Value }
 
-pub struct Lru<T>
+pub struct Lru<K, V>
 where
-    T: Send + Sync + 'static,
+    K: Key,
+    V: Value,
 {
-    high_priority_list: Dlist<LruHandleDlistAdapter<T>>,
-    list: Dlist<LruHandleDlistAdapter<T>>,
+    high_priority_list: LinkedList<Adapter<K, V>>,
+    list: LinkedList<Adapter<K, V>>,
+    pin_list: LinkedList<Adapter<K, V>>,
 
     high_priority_weight: usize,
     high_priority_weight_capacity: usize,
 }
 
-impl<T> Lru<T>
+impl<K, V> Lru<K, V>
 where
-    T: Send + Sync + 'static,
+    K: Key,
+    V: Value,
 {
-    unsafe fn may_overflow_high_priority_pool(&mut self) {
+    fn may_overflow_high_priority_pool(&mut self) {
         while self.high_priority_weight > self.high_priority_weight_capacity {
             strict_assert!(!self.high_priority_list.is_empty());
 
             // overflow last entry in high priority pool to low priority pool
-            let mut ptr = self.high_priority_list.pop_front().strict_unwrap_unchecked();
-            strict_assert!(ptr.as_ref().in_high_priority_pool);
-            ptr.as_mut().in_high_priority_pool = false;
-            self.high_priority_weight -= ptr.as_ref().base().weight();
-            self.list.push_back(ptr);
+            let record = self.high_priority_list.pop_front().unwrap();
+            let state = unsafe { &mut *record.state().get() };
+            strict_assert!(state.in_high_priority_pool);
+            state.in_high_priority_pool = false;
+            self.high_priority_weight -= record.weight();
+            self.list.push_back(record);
         }
     }
 }
 
-impl<T> Eviction for Lru<T>
+impl<K, V> Eviction for Lru<K, V>
 where
-    T: Send + Sync + 'static,
+    K: Key,
+    V: Value,
 {
-    type Handle = LruHandle<T>;
     type Config = LruConfig;
+    type Key = K;
+    type Value = V;
+    type Hint = LruHint;
+    type State = LruState;
 
-    unsafe fn new(capacity: usize, config: &Self::Config) -> Self
+    fn new(capacity: usize, config: &Self::Config) -> Self
     where
         Self: Sized,
     {
         assert!(
-            config.high_priority_pool_ratio >= 0.0 && config.high_priority_pool_ratio <= 1.0,
-            "high_priority_pool_ratio_percentage must be in [0, 100], given: {}",
+            (0.0..=1.0).contains(&config.high_priority_pool_ratio),
+            "high_priority_pool_ratio_percentage must be in 0.0..=1.0, given: {}",
             config.high_priority_pool_ratio
         );
 
         let high_priority_weight_capacity = (capacity as f64 * config.high_priority_pool_ratio) as usize;
 
         Self {
-            high_priority_list: Dlist::new(),
-            list: Dlist::new(),
+            high_priority_list: LinkedList::new(Adapter::new()),
+            list: LinkedList::new(Adapter::new()),
+            pin_list: LinkedList::new(Adapter::new()),
             high_priority_weight: 0,
             high_priority_weight_capacity,
         }
     }
 
-    unsafe fn push(&mut self, mut ptr: NonNull<Self::Handle>) {
-        let handle = ptr.as_mut();
+    fn update(&mut self, capacity: usize, config: &Self::Config) {
+        if !(0.0..=1.0).contains(&config.high_priority_pool_ratio) {
+            tracing::error!(
+                "[lru]: high_priority_pool_ratio_percentage must be in 0.0..=1.0, given: {}, new configuration ignored",
+                config.high_priority_pool_ratio
+            );
+            return;
+        }
+
+        let high_priority_weight_capacity = (capacity as f64 * config.high_priority_pool_ratio) as usize;
+        self.high_priority_weight_capacity = high_priority_weight_capacity;
+
+        self.may_overflow_high_priority_pool();
+    }
 
-        strict_assert!(!handle.link.is_linked());
+    fn push(&mut self, record: Arc<Record<Self>>) {
+        let state = unsafe { &mut *record.state().get() };
 
-        match handle.base().context() {
-            LruContext::HighPriority => {
-                handle.in_high_priority_pool = true;
-                self.high_priority_weight += handle.base().weight();
-                self.high_priority_list.push_back(ptr);
+        strict_assert!(!state.link.is_linked());
+
+        record.set_in_eviction(true);
+
+        match record.hint() {
+            LruHint::HighPriority => {
+                state.in_high_priority_pool = true;
+                self.high_priority_weight += record.weight();
+                self.high_priority_list.push_back(record);
 
                 self.may_overflow_high_priority_pool();
             }
-            LruContext::LowPriority => {
-                handle.in_high_priority_pool = false;
-                self.list.push_back(ptr);
+            LruHint::LowPriority => {
+                state.in_high_priority_pool = false;
+                self.list.push_back(record);
             }
         }
-
-        handle.base_mut().set_in_eviction(true);
     }
 
-    unsafe fn pop(&mut self) -> Option<NonNull<Self::Handle>> {
-        let mut ptr = self.list.pop_front().or_else(|| self.high_priority_list.pop_front())?;
+    fn pop(&mut self) -> Option<Arc<Record<Self>>> {
+        let record = self.list.pop_front().or_else(|| self.high_priority_list.pop_front())?;
 
-        let handle = ptr.as_mut();
-        strict_assert!(!handle.link.is_linked());
+        let state = unsafe { &mut *record.state().get() };
 
-        if handle.in_high_priority_pool {
-            self.high_priority_weight -= handle.base().weight();
-            handle.in_high_priority_pool = false;
+        strict_assert!(!state.link.is_linked());
+
+        if state.in_high_priority_pool {
+            self.high_priority_weight -= record.weight();
+            state.in_high_priority_pool = false;
         }
 
-        handle.base_mut().set_in_eviction(false);
+        record.set_in_eviction(false);
 
-        Some(ptr)
+        Some(record)
     }
 
-    unsafe fn acquire(&mut self, _: NonNull<Self::Handle>) {}
+    fn remove(&mut self, record: &Arc<Record<Self>>) {
+        let state = unsafe { &mut *record.state().get() };
 
-    unsafe fn release(&mut self, mut ptr: NonNull<Self::Handle>) {
-        let handle = ptr.as_mut();
+        strict_assert!(state.link.is_linked());
 
-        if handle.base().is_in_eviction() {
-            strict_assert!(handle.link.is_linked());
-            self.remove(ptr);
-            self.push(ptr);
-        } else {
-            strict_assert!(!handle.link.is_linked());
-            self.push(ptr);
-        }
+        match (state.is_pinned, state.in_high_priority_pool) {
+            (true, _) => unsafe { self.pin_list.remove_from_ptr(Arc::as_ptr(record)) },
+            (false, true) => {
+                self.high_priority_weight -= record.weight();
+                state.in_high_priority_pool = false;
+                unsafe { self.high_priority_list.remove_from_ptr(Arc::as_ptr(record)) }
+            }
+            (false, false) => unsafe { self.list.remove_from_ptr(Arc::as_ptr(record)) },
+        };
 
-        strict_assert!(handle.base().is_in_eviction());
+        strict_assert!(!state.link.is_linked());
+
+        record.set_in_eviction(false);
     }
 
-    unsafe fn remove(&mut self, mut ptr: NonNull<Self::Handle>) {
-        let handle = ptr.as_mut();
-        strict_assert!(handle.link.is_linked());
+    fn clear(&mut self) {
+        while self.pop().is_some() {}
 
-        if handle.in_high_priority_pool {
-            self.high_priority_weight -= handle.base.weight();
-            self.high_priority_list.remove_raw(handle.link.raw());
-            handle.in_high_priority_pool = false;
-        } else {
-            self.list.remove_raw(handle.link.raw());
+        // Clear pin list to prevent from memory leak.
+        while let Some(record) = self.pin_list.pop_front() {
+            let state = unsafe { &mut *record.state().get() };
+            strict_assert!(!state.link.is_linked());
+
+            if state.in_high_priority_pool {
+                self.high_priority_weight -= record.weight();
+                state.in_high_priority_pool = false;
+            }
+
+            record.set_in_eviction(false);
         }
 
-        strict_assert!(!handle.link.is_linked());
+        assert!(self.list.is_empty());
+        assert!(self.high_priority_list.is_empty());
+        assert!(self.pin_list.is_empty());
+        assert_eq!(self.high_priority_weight, 0);
+    }
 
-        handle.base_mut().set_in_eviction(false);
+    fn acquire_operator() -> super::Operator {
+        Operator::Mutable
     }
 
-    unsafe fn clear(&mut self) -> Vec<NonNull<Self::Handle>> {
-        let mut res = Vec::with_capacity(self.len());
+    fn acquire_immutable(&self, _record: &Arc<Record<Self>>) {
+        unreachable!()
+    }
 
-        while !self.list.is_empty() {
-            let mut ptr = self.list.pop_front().strict_unwrap_unchecked();
-            ptr.as_mut().base_mut().set_in_eviction(false);
-            res.push(ptr);
+    fn acquire_mutable(&mut self, record: &Arc<Record<Self>>) {
+        if !record.is_in_eviction() {
+            return;
         }
 
-        while !self.high_priority_list.is_empty() {
-            let mut ptr = self.high_priority_list.pop_front().strict_unwrap_unchecked();
-            ptr.as_mut().base_mut().set_in_eviction(false);
-            ptr.as_mut().in_high_priority_pool = false;
-            self.high_priority_weight -= ptr.as_ref().base().weight();
-            res.push(ptr);
+        let state = unsafe { &mut *record.state().get() };
+        assert!(state.link.is_linked());
+
+        if state.is_pinned {
+            return;
         }
 
-        strict_assert_eq!(self.high_priority_weight, 0);
+        // Pin the record by moving it to the pin list.
+
+        let r = if state.in_high_priority_pool {
+            unsafe { self.high_priority_list.remove_from_ptr(Arc::as_ptr(record)) }
+        } else {
+            unsafe { self.list.remove_from_ptr(Arc::as_ptr(record)) }
+        };
 
-        res
+        self.pin_list.push_back(r);
+
+        state.is_pinned = true;
     }
 
-    fn len(&self) -> usize {
-        self.high_priority_list.len() + self.list.len()
+    fn release_operator() -> Operator {
+        Operator::Mutable
     }
 
-    fn is_empty(&self) -> bool {
-        self.len() == 0
+    fn release_immutable(&self, _record: &Arc<Record<Self>>) {
+        unreachable!()
     }
-}
 
-unsafe impl<T> Send for Lru<T> where T: Send + Sync + 'static {}
-unsafe impl<T> Sync for Lru<T> where T: Send + Sync + 'static {}
+    fn release_mutable(&mut self, record: &Arc<Record<Self>>) {
+        if !record.is_in_eviction() {
+            return;
+        }
+
+        let state = unsafe { &mut *record.state().get() };
+        assert!(state.link.is_linked());
+
+        if !state.is_pinned {
+            return;
+        }
+
+        // Unpin the record by moving it from the pin list.
+
+        unsafe { self.pin_list.remove_from_ptr(Arc::as_ptr(record)) };
+
+        if state.in_high_priority_pool {
+            self.high_priority_list.push_back(record.clone());
+        } else {
+            self.list.push_back(record.clone());
+        }
+
+        state.is_pinned = false;
+    }
+}
 
 #[cfg(test)]
 pub mod tests {
@@ -294,137 +321,209 @@ pub mod tests {
     use itertools::Itertools;
 
     use super::*;
-    use crate::{eviction::test_utils::TestEviction, handle::HandleExt};
+    use crate::{
+        eviction::test_utils::{assert_ptr_eq, assert_ptr_vec_vec_eq, TestEviction},
+        record::Data,
+    };
 
-    impl<T> TestEviction for Lru<T>
+    impl<K, V> TestEviction for Lru<K, V>
     where
-        T: Send + Sync + 'static + Clone,
+        K: Key + Clone,
+        V: Value + Clone,
     {
-        fn dump(&self) -> Vec<T> {
-            self.list
-                .iter()
-                .chain(self.high_priority_list.iter())
-                .map(|handle| handle.base().data_unwrap_unchecked().clone())
-                .collect_vec()
+        type Dump = Vec<Vec<Arc<Record<Self>>>>;
+        fn dump(&self) -> Self::Dump {
+            let mut low = vec![];
+            let mut high = vec![];
+            let mut pin = vec![];
+
+            let mut cursor = self.list.cursor();
+            loop {
+                cursor.move_next();
+                match cursor.clone_pointer() {
+                    Some(record) => low.push(record),
+                    None => break,
+                }
+            }
+
+            let mut cursor = self.high_priority_list.cursor();
+            loop {
+                cursor.move_next();
+                match cursor.clone_pointer() {
+                    Some(record) => high.push(record),
+                    None => break,
+                }
+            }
+
+            let mut cursor = self.pin_list.cursor();
+            loop {
+                cursor.move_next();
+                match cursor.clone_pointer() {
+                    Some(record) => pin.push(record),
+                    None => break,
+                }
+            }
+
+            vec![low, high, pin]
         }
     }
 
-    type TestLruHandle = LruHandle<u64>;
-    type TestLru = Lru<u64>;
+    type TestLru = Lru<u64, u64>;
 
-    unsafe fn new_test_lru_handle_ptr(data: u64, context: LruContext) -> NonNull<TestLruHandle> {
-        let mut handle = Box::<TestLruHandle>::default();
-        handle.init(0, data, 1, context);
-        NonNull::new_unchecked(Box::into_raw(handle))
-    }
+    #[test]
+    fn test_lru() {
+        let rs = (0..20)
+            .map(|i| {
+                Arc::new(Record::new(Data {
+                    key: i,
+                    value: i,
+                    hint: if i < 10 {
+                        LruHint::HighPriority
+                    } else {
+                        LruHint::LowPriority
+                    },
+                    hash: i,
+                    weight: 1,
+                }))
+            })
+            .collect_vec();
+        let r = |i: usize| rs[i].clone();
+
+        let config = LruConfig {
+            high_priority_pool_ratio: 0.5,
+        };
+        let mut lru = TestLru::new(8, &config);
+
+        assert_eq!(lru.high_priority_weight_capacity, 4);
+
+        // [0, 1, 2, 3]
+        lru.push(r(0));
+        lru.push(r(1));
+        lru.push(r(2));
+        lru.push(r(3));
+        assert_ptr_vec_vec_eq(lru.dump(), vec![vec![], vec![r(0), r(1), r(2), r(3)], vec![]]);
+
+        // 0, [1, 2, 3, 4]
+        lru.push(r(4));
+        assert_ptr_vec_vec_eq(lru.dump(), vec![vec![r(0)], vec![r(1), r(2), r(3), r(4)], vec![]]);
+
+        // 0, 10, [1, 2, 3, 4]
+        lru.push(r(10));
+        assert_ptr_vec_vec_eq(
+            lru.dump(),
+            vec![vec![r(0), r(10)], vec![r(1), r(2), r(3), r(4)], vec![]],
+        );
 
-    unsafe fn del_test_lru_handle_ptr(ptr: NonNull<TestLruHandle>) {
-        let _ = Box::from_raw(ptr.as_ptr());
-    }
+        // 10, [1, 2, 3, 4]
+        let r0 = lru.pop().unwrap();
+        assert_ptr_eq(&r(0), &r0);
+        assert_ptr_vec_vec_eq(lru.dump(), vec![vec![r(10)], vec![r(1), r(2), r(3), r(4)], vec![]]);
+
+        // 10, [1, 3, 4]
+        lru.remove(&rs[2]);
+        assert_ptr_vec_vec_eq(lru.dump(), vec![vec![r(10)], vec![r(1), r(3), r(4)], vec![]]);
+
+        // 10, 11, [1, 3, 4]
+        lru.push(r(11));
+        assert_ptr_vec_vec_eq(lru.dump(), vec![vec![r(10), r(11)], vec![r(1), r(3), r(4)], vec![]]);
+
+        // 10, 11, 1, [3, 4, 5, 6]
+        lru.push(r(5));
+        lru.push(r(6));
+        assert_ptr_vec_vec_eq(
+            lru.dump(),
+            vec![vec![r(10), r(11), r(1)], vec![r(3), r(4), r(5), r(6)], vec![]],
+        );
 
-    unsafe fn dump_test_lru(lru: &TestLru) -> (Vec<u64>, Vec<u64>) {
-        (
-            lru.list
-                .iter()
-                .map(|handle| *handle.base().data_unwrap_unchecked())
-                .collect_vec(),
-            lru.high_priority_list
-                .iter()
-                .map(|handle| *handle.base().data_unwrap_unchecked())
-                .collect_vec(),
-        )
+        // 10, 11, 1, 3, [4, 5, 6, 0]
+        lru.push(r(0));
+        assert_ptr_vec_vec_eq(
+            lru.dump(),
+            vec![vec![r(10), r(11), r(1), r(3)], vec![r(4), r(5), r(6), r(0)], vec![]],
+        );
+
+        lru.clear();
+        assert_ptr_vec_vec_eq(lru.dump(), vec![vec![], vec![], vec![]]);
     }
 
     #[test]
-    fn test_lru() {
-        unsafe {
-            let ptrs = (0..20)
-                .map(|i| {
-                    new_test_lru_handle_ptr(
-                        i,
-                        if i < 10 {
-                            LruContext::HighPriority
-                        } else {
-                            LruContext::LowPriority
-                        },
-                    )
-                })
-                .collect_vec();
-
-            let config = LruConfig {
-                high_priority_pool_ratio: 0.5,
-            };
-            let mut lru = TestLru::new(8, &config);
-
-            assert_eq!(lru.high_priority_weight_capacity, 4);
-
-            // [0, 1, 2, 3]
-            lru.push(ptrs[0]);
-            lru.push(ptrs[1]);
-            lru.push(ptrs[2]);
-            lru.push(ptrs[3]);
-            assert_eq!(lru.len(), 4);
-            assert_eq!(lru.high_priority_weight, 4);
-            assert_eq!(lru.high_priority_list.len(), 4);
-            assert_eq!(dump_test_lru(&lru), (vec![], vec![0, 1, 2, 3]));
-
-            // 0, [1, 2, 3, 4]
-            lru.push(ptrs[4]);
-            assert_eq!(lru.len(), 5);
-            assert_eq!(lru.high_priority_weight, 4);
-            assert_eq!(lru.high_priority_list.len(), 4);
-            assert_eq!(dump_test_lru(&lru), (vec![0], vec![1, 2, 3, 4]));
-
-            // 0, 10, [1, 2, 3, 4]
-            lru.push(ptrs[10]);
-            assert_eq!(lru.len(), 6);
-            assert_eq!(lru.high_priority_weight, 4);
-            assert_eq!(lru.high_priority_list.len(), 4);
-            assert_eq!(dump_test_lru(&lru), (vec![0, 10], vec![1, 2, 3, 4]));
-
-            // 10, [1, 2, 3, 4]
-            let p0 = lru.pop().unwrap();
-            assert_eq!(ptrs[0], p0);
-            assert_eq!(lru.len(), 5);
-            assert_eq!(lru.high_priority_weight, 4);
-            assert_eq!(lru.high_priority_list.len(), 4);
-            assert_eq!(dump_test_lru(&lru), (vec![10], vec![1, 2, 3, 4]));
-
-            // 10, [1, 3, 4]
-            lru.remove(ptrs[2]);
-            assert_eq!(lru.len(), 4);
-            assert_eq!(lru.high_priority_weight, 3);
-            assert_eq!(lru.high_priority_list.len(), 3);
-            assert_eq!(dump_test_lru(&lru), (vec![10], vec![1, 3, 4]));
-
-            // 10, 11, [1, 3, 4]
-            lru.push(ptrs[11]);
-            assert_eq!(lru.len(), 5);
-            assert_eq!(lru.high_priority_weight, 3);
-            assert_eq!(lru.high_priority_list.len(), 3);
-            assert_eq!(dump_test_lru(&lru), (vec![10, 11], vec![1, 3, 4]));
-
-            // 10, 11, 1, [3, 4, 5, 6]
-            lru.push(ptrs[5]);
-            lru.push(ptrs[6]);
-            assert_eq!(lru.len(), 7);
-            assert_eq!(lru.high_priority_weight, 4);
-            assert_eq!(lru.high_priority_list.len(), 4);
-            assert_eq!(dump_test_lru(&lru), (vec![10, 11, 1], vec![3, 4, 5, 6]));
-
-            // 10, 11, 1, 3, [4, 5, 6, 0]
-            lru.push(ptrs[0]);
-            assert_eq!(lru.len(), 8);
-            assert_eq!(lru.high_priority_weight, 4);
-            assert_eq!(lru.high_priority_list.len(), 4);
-            assert_eq!(dump_test_lru(&lru), (vec![10, 11, 1, 3], vec![4, 5, 6, 0]));
-
-            let ps = lru.clear();
-            assert_eq!(ps, [10, 11, 1, 3, 4, 5, 6, 0].map(|i| ptrs[i]));
-
-            for ptr in ptrs {
-                del_test_lru_handle_ptr(ptr);
-            }
-        }
+    fn test_lru_pin() {
+        let rs = (0..20)
+            .map(|i| {
+                Arc::new(Record::new(Data {
+                    key: i,
+                    value: i,
+                    hint: if i < 10 {
+                        LruHint::HighPriority
+                    } else {
+                        LruHint::LowPriority
+                    },
+                    hash: i,
+                    weight: 1,
+                }))
+            })
+            .collect_vec();
+        let r = |i: usize| rs[i].clone();
+
+        let config = LruConfig {
+            high_priority_pool_ratio: 0.5,
+        };
+        let mut lru = TestLru::new(8, &config);
+
+        assert_eq!(lru.high_priority_weight_capacity, 4);
+
+        // 10, 11, [0, 1]
+        lru.push(r(0));
+        lru.push(r(1));
+        lru.push(r(10));
+        lru.push(r(11));
+        assert_ptr_vec_vec_eq(lru.dump(), vec![vec![r(10), r(11)], vec![r(0), r(1)], vec![]]);
+
+        // pin: [0], 10
+        // 11, [1]
+        lru.acquire_mutable(&rs[0]);
+        lru.acquire_mutable(&rs[10]);
+        assert_ptr_vec_vec_eq(lru.dump(), vec![vec![r(11)], vec![r(1)], vec![r(0), r(10)]]);
+
+        // 11, 10, [1, 0]
+        lru.release_mutable(&rs[0]);
+        lru.release_mutable(&rs[10]);
+        assert_ptr_vec_vec_eq(lru.dump(), vec![vec![r(11), r(10)], vec![r(1), r(0)], vec![]]);
+
+        // acquire pinned
+        // pin: [0], 11
+        // 10, [1]
+        lru.acquire_mutable(&rs[0]);
+        lru.acquire_mutable(&rs[11]);
+        lru.acquire_mutable(&rs[0]);
+        lru.acquire_mutable(&rs[11]);
+        assert_ptr_vec_vec_eq(lru.dump(), vec![vec![r(10)], vec![r(1)], vec![r(0), r(11)]]);
+
+        // remove pinned
+        // pin: [0]
+        // 10, [1]
+        lru.remove(&rs[11]);
+        assert_ptr_vec_vec_eq(lru.dump(), vec![vec![r(10)], vec![r(1)], vec![r(0)]]);
+
+        // release removed
+        // pin: [0]
+        // 10, [1]
+        lru.release_mutable(&rs[11]);
+        assert_ptr_vec_vec_eq(lru.dump(), vec![vec![r(10)], vec![r(1)], vec![r(0)]]);
+
+        // release unpinned
+        // 10, [1, 0]
+        lru.release_mutable(&rs[0]);
+        lru.release_mutable(&rs[0]);
+        assert_ptr_vec_vec_eq(lru.dump(), vec![vec![r(10)], vec![r(1), r(0)], vec![]]);
+
+        // clear with pinned
+        // pin: [1]
+        // 10, [0]
+        lru.acquire_mutable(&rs[1]);
+        assert_ptr_vec_vec_eq(lru.dump(), vec![vec![r(10)], vec![r(0)], vec![r(1)]]);
+
+        lru.clear();
+        assert_ptr_vec_vec_eq(lru.dump(), vec![vec![], vec![], vec![]]);
     }
 }
diff --git a/foyer-memory/src/eviction/mod.rs b/foyer-memory/src/eviction/mod.rs
index fa7297be..612de1a6 100644
--- a/foyer-memory/src/eviction/mod.rs
+++ b/foyer-memory/src/eviction/mod.rs
@@ -12,89 +12,124 @@
 //  See the License for the specific language governing permissions and
 //  limitations under the License.
 
-use std::ptr::NonNull;
+use std::sync::Arc;
 
+use foyer_common::code::{Key, Value};
 use serde::{de::DeserializeOwned, Serialize};
 
-use crate::handle::Handle;
+use crate::record::{CacheHint, Record};
 
-pub trait EvictionConfig: Send + Sync + 'static + Clone + Serialize + DeserializeOwned + Default {}
-impl<T> EvictionConfig for T where T: Send + Sync + 'static + Clone + Serialize + DeserializeOwned + Default {}
+pub trait Hint: Send + Sync + 'static + Clone + Default + From<CacheHint> + Into<CacheHint> {}
+impl<T> Hint for T where T: Send + Sync + 'static + Clone + Default + From<CacheHint> + Into<CacheHint> {}
 
-/// The lifetime of `handle: Self::H` is managed by [`Indexer`].
+pub trait State: Send + Sync + 'static + Default {}
+impl<T> State for T where T: Send + Sync + 'static + Default {}
+
+pub trait Config: Send + Sync + 'static + Clone + Serialize + DeserializeOwned + Default {}
+impl<T> Config for T where T: Send + Sync + 'static + Clone + Serialize + DeserializeOwned + Default {}
+
+pub enum Operator {
+    Noop,
+    Immutable,
+    Mutable,
+}
+
+/// Cache eviction algorithm abstraction.
+///
+/// [`Eviction`] provides essential APIs for the plug-and-play algorithm abstraction.
 ///
-/// Each `handle`'s lifetime in [`Indexer`] must outlive the raw pointer in [`Eviction`].
-pub trait Eviction: Send + Sync + 'static {
-    type Handle: Handle;
-    type Config: EvictionConfig;
+/// [`Eviction`] is needs to be implemented to support a new cache eviction algorithm.
+///
+/// For performance considerations, most APIs pass parameters via [`NonNull`] pointers to implement intrusive data
+/// structures. It is not required to implement the cache eviction algorithm using the [`NonNull`] pointers. They can
+/// also be used as a token for the target entry.
+///
+/// # Safety
+///
+/// The pointer can be dereferenced as a mutable reference ***iff*** the `self` reference is also mutable.
+/// Dereferencing a pointer as a mutable reference when `self` is immutable will cause UB.
+pub trait Eviction: Send + Sync + 'static + Sized {
+    /// Cache eviction algorithm configurations.
+    type Config: Config;
+    /// Cache key. Generally, it is supposed to be a generic type of the implementation.
+    type Key: Key;
+    /// Cache value. Generally, it is supposed to be a generic type of the implementation.
+    type Value: Value;
+    /// Hint for a cache entry. Can be used to support priority at the entry granularity.
+    type Hint: Hint;
+    /// State for a cache entry. Mutable state for maintaining the cache eviction algorithm implementation.
+    type State: State;
 
-    /// Create a new empty eviction container.
-    ///
-    /// # Safety
-    unsafe fn new(capacity: usize, config: &Self::Config) -> Self
-    where
-        Self: Sized;
+    /// Create a new cache eviction algorithm instance with the given arguments.
+    fn new(capacity: usize, config: &Self::Config) -> Self;
 
-    /// Push a handle `ptr` into the eviction container.
-    ///
-    /// The caller guarantees that the `ptr` is NOT in the eviction container.
-    ///
-    /// # Safety
+    /// Update the arguments of the ache eviction algorithm instance.
+    fn update(&mut self, capacity: usize, config: &Self::Config);
+
+    /// Push a record into the cache eviction algorithm instance.
     ///
-    /// The `ptr` must be kept holding until `pop` or `remove`.
+    /// The caller guarantees that the record is NOT in the cache eviction algorithm instance.
     ///
-    /// The base handle associated to the `ptr` must be set in cache.
-    unsafe fn push(&mut self, ptr: NonNull<Self::Handle>);
+    /// The cache eviction algorithm instance MUST hold the record and set its `IN_EVICTION` flag to true.
+    fn push(&mut self, record: Arc<Record<Self>>);
 
-    /// Pop a handle `ptr` from the eviction container.
+    /// Push a record from the cache eviction algorithm instance.
     ///
-    /// # Safety
+    /// The cache eviction algorithm instance MUST remove the record and set its `IN_EVICTION` flag to false.
+    fn pop(&mut self) -> Option<Arc<Record<Self>>>;
+
+    /// Remove a record from the cache eviction algorithm instance.
     ///
-    /// The `ptr` must be taken from the eviction container.
-    /// Or it may become dangling and cause UB.
+    /// The caller guarantees that the record is in the cache eviction algorithm instance.
     ///
-    /// The base handle associated to the `ptr` must be set NOT in cache.
-    unsafe fn pop(&mut self) -> Option<NonNull<Self::Handle>>;
+    /// The cache eviction algorithm instance MUST remove the record and set its `IN_EVICTION` flag to false.
+    fn remove(&mut self, record: &Arc<Record<Self>>);
 
-    /// Notify the eviction container that the `ptr` is acquired by **AN** external user.
+    /// Remove all records from the cache eviction algorithm instance.
     ///
-    /// # Safety
+    /// The cache eviction algorithm instance MUST remove the records and set its `IN_EVICTION` flag to false.
+    fn clear(&mut self) {
+        while self.pop().is_some() {}
+    }
+
+    /// Determine if the immutable version or the mutable version to use for the `acquire` operation.
     ///
-    /// The given `ptr` can be EITHER in the eviction container OR not in the eviction container.
-    unsafe fn acquire(&mut self, ptr: NonNull<Self::Handle>);
+    /// Only the chosen version needs to be implemented. The other version is recommend to be left as `unreachable!()`.
+    fn acquire_operator() -> Operator;
 
-    /// Notify the eviction container that the `ptr` is released by **ALL** external users.
+    /// Immutable version of the `acquire` operation.
     ///
-    /// # Safety
+    /// `acquire` is called when an external caller acquire a cache entry from the cache.
     ///
-    /// The given `ptr` can be EITHER in the eviction container OR not in the eviction container.
-    unsafe fn release(&mut self, ptr: NonNull<Self::Handle>);
+    /// The entry can be EITHER in the cache eviction algorithm instance or not.
+    fn acquire_immutable(&self, record: &Arc<Record<Self>>);
 
-    /// Remove the given `ptr` from the eviction container.
-    ///
-    /// /// The caller guarantees that the `ptr` is NOT in the eviction container.
+    /// Mutable version of the `acquire` operation.
     ///
-    /// # Safety
+    /// `acquire` is called when an external caller acquire a cache entry from the cache.
     ///
-    /// The `ptr` must be taken from the eviction container, otherwise it may become dangling and cause UB.
+    /// The entry can be EITHER in the cache eviction algorithm instance or not.
+    fn acquire_mutable(&mut self, records: &Arc<Record<Self>>);
+
+    /// Determine if the immutable version or the mutable version to use for the `release` operation.
     ///
-    /// The base handle associated to the `ptr` must be set NOT in cache.
-    unsafe fn remove(&mut self, ptr: NonNull<Self::Handle>);
+    /// Only the chosen version needs to be implemented. The other version is recommend to be left as `unreachable!()`.
+    fn release_operator() -> Operator;
 
-    /// Remove all `ptr`s from the eviction container and reset.
+    /// Immutable version of the `release` operation.
     ///
-    /// # Safety
     ///
-    /// All `ptr` must be taken from the eviction container, otherwise it may become dangling and cause UB.
+    /// `release` is called when the last external caller drops the cache entry.
     ///
-    /// All base handles associated to the `ptr`s must be set NOT in cache.
-    unsafe fn clear(&mut self) -> Vec<NonNull<Self::Handle>>;
-
-    /// Return the count of the `ptr`s that in the eviction container.
-    fn len(&self) -> usize;
+    /// The entry can be EITHER in the cache eviction algorithm instance or not.
+    fn release_immutable(&self, record: &Arc<Record<Self>>);
 
-    /// Return `true` if the eviction container is empty.
-    fn is_empty(&self) -> bool;
+    /// Mutable version of the `release` operation.
+    ///
+    /// `release` is called when the last external caller drops the cache entry.
+    ///
+    /// The entry can be EITHER in the cache eviction algorithm instance or not.
+    fn release_mutable(&mut self, record: &Arc<Record<Self>>);
 }
 
 pub mod fifo;
@@ -102,7 +137,5 @@ pub mod lfu;
 pub mod lru;
 pub mod s3fifo;
 
-pub mod sanity;
-
 #[cfg(test)]
 pub mod test_utils;
diff --git a/foyer-memory/src/eviction/s3fifo.rs b/foyer-memory/src/eviction/s3fifo.rs
index fcee8588..0eb281aa 100644
--- a/foyer-memory/src/eviction/s3fifo.rs
+++ b/foyer-memory/src/eviction/s3fifo.rs
@@ -14,37 +14,57 @@
 
 use std::{
     collections::{HashSet, VecDeque},
-    fmt::Debug,
-    ptr::NonNull,
+    mem::offset_of,
+    sync::{
+        atomic::{AtomicU8, Ordering},
+        Arc,
+    },
 };
 
-use foyer_common::{assert::OptionExt, strict_assert, strict_assert_eq};
-use foyer_intrusive::{
-    dlist::{Dlist, DlistLink},
-    intrusive_adapter,
+use foyer_common::{
+    code::{Key, Value},
+    strict_assert, strict_assert_eq,
 };
+use intrusive_collections::{intrusive_adapter, LinkedList, LinkedListAtomicLink};
 use serde::{Deserialize, Serialize};
 
-use crate::{
-    eviction::Eviction,
-    handle::{BaseHandle, Handle},
-    CacheContext,
-};
+use super::{Eviction, Operator};
+use crate::record::{CacheHint, Record};
 
-const MAX_FREQ: u8 = 3;
+/// S3Fifo eviction algorithm config.
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct S3FifoConfig {
+    /// Capacity ratio of the small S3FIFO queue.
+    pub small_queue_capacity_ratio: f64,
+    /// Capacity ratio of the ghost S3FIFO queue.
+    pub ghost_queue_capacity_ratio: f64,
+    /// Minimum access times when population entry from small queue to main queue.
+    pub small_to_main_freq_threshold: u8,
+}
 
-#[derive(Debug, Clone)]
-pub struct S3FifoContext(CacheContext);
+impl Default for S3FifoConfig {
+    fn default() -> Self {
+        Self {
+            small_queue_capacity_ratio: 0.1,
+            ghost_queue_capacity_ratio: 1.0,
+            small_to_main_freq_threshold: 1,
+        }
+    }
+}
 
-impl From<CacheContext> for S3FifoContext {
-    fn from(context: CacheContext) -> Self {
-        Self(context)
+/// S3Fifo eviction algorithm hint.
+#[derive(Debug, Clone, Default)]
+pub struct S3FifoHint;
+
+impl From<CacheHint> for S3FifoHint {
+    fn from(_: CacheHint) -> Self {
+        S3FifoHint
     }
 }
 
-impl From<S3FifoContext> for CacheContext {
-    fn from(context: S3FifoContext) -> Self {
-        context.0
+impl From<S3FifoHint> for CacheHint {
+    fn from(_: S3FifoHint) -> Self {
+        CacheHint::Normal
     }
 }
 
@@ -55,100 +75,56 @@ enum Queue {
     Small,
 }
 
-pub struct S3FifoHandle<T>
-where
-    T: Send + Sync + 'static,
-{
-    link: DlistLink,
-    base: BaseHandle<T, S3FifoContext>,
-    freq: u8,
-    queue: Queue,
-}
-
-impl<T> Debug for S3FifoHandle<T>
-where
-    T: Send + Sync + 'static,
-{
-    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-        f.debug_struct("S3FifoHandle").finish()
+impl Default for Queue {
+    fn default() -> Self {
+        Self::None
     }
 }
 
-intrusive_adapter! { S3FifoHandleDlistAdapter<T> = S3FifoHandle<T> { link: DlistLink } where T: Send + Sync + 'static }
+/// S3Fifo eviction algorithm hint.
+#[derive(Debug, Default)]
+pub struct S3FifoState {
+    link: LinkedListAtomicLink,
+    frequency: AtomicU8,
+    queue: Queue,
+}
 
-impl<T> S3FifoHandle<T>
-where
-    T: Send + Sync + 'static,
-{
-    #[inline(always)]
-    pub fn freq_inc(&mut self) {
-        self.freq = std::cmp::min(self.freq + 1, MAX_FREQ);
-    }
+impl S3FifoState {
+    const MAX_FREQUENCY: u8 = 3;
 
-    #[inline(always)]
-    pub fn freq_dec(&mut self) {
-        self.freq = self.freq.saturating_sub(1);
+    fn frequency(&self) -> u8 {
+        self.frequency.load(Ordering::Acquire)
     }
-}
 
-impl<T> Default for S3FifoHandle<T>
-where
-    T: Send + Sync + 'static,
-{
-    fn default() -> Self {
-        Self {
-            link: DlistLink::default(),
-            freq: 0,
-            base: BaseHandle::new(),
-            queue: Queue::None,
-        }
+    fn set_frequency(&self, val: u8) {
+        self.frequency.store(val, Ordering::Release)
     }
-}
-
-impl<T> Handle for S3FifoHandle<T>
-where
-    T: Send + Sync + 'static,
-{
-    type Data = T;
-    type Context = S3FifoContext;
 
-    fn base(&self) -> &BaseHandle<Self::Data, Self::Context> {
-        &self.base
+    fn inc_frequency(&self) -> u8 {
+        self.frequency
+            .fetch_update(Ordering::Release, Ordering::Acquire, |v| {
+                Some(std::cmp::min(Self::MAX_FREQUENCY, v + 1))
+            })
+            .unwrap()
     }
 
-    fn base_mut(&mut self) -> &mut BaseHandle<Self::Data, Self::Context> {
-        &mut self.base
+    fn dec_frequency(&self) -> u8 {
+        self.frequency
+            .fetch_update(Ordering::Release, Ordering::Acquire, |v| Some(v.saturating_sub(1)))
+            .unwrap()
     }
 }
 
-/// S3FIFO eviction algorithm config.
-#[derive(Debug, Clone, Serialize, Deserialize)]
-pub struct S3FifoConfig {
-    /// Capacity ratio of the small S3FIFO queue.
-    pub small_queue_capacity_ratio: f64,
-    /// Capacity ratio of the ghost S3FIFO queue.
-    pub ghost_queue_capacity_ratio: f64,
-    /// Minimum access times when population entry from small queue to main queue.
-    pub small_to_main_freq_threshold: u8,
-}
-
-impl Default for S3FifoConfig {
-    fn default() -> Self {
-        Self {
-            small_queue_capacity_ratio: 0.1,
-            ghost_queue_capacity_ratio: 1.0,
-            small_to_main_freq_threshold: 1,
-        }
-    }
-}
+intrusive_adapter! { Adapter<K, V> = Arc<Record<S3Fifo<K, V>>>: Record<S3Fifo<K, V>> { ?offset = Record::<S3Fifo<K, V>>::STATE_OFFSET + offset_of!(S3FifoState, link) => LinkedListAtomicLink } where K: Key, V: Value }
 
-pub struct S3Fifo<T>
+pub struct S3Fifo<K, V>
 where
-    T: Send + Sync + 'static,
+    K: Key,
+    V: Value,
 {
     ghost_queue: GhostQueue,
-    small_queue: Dlist<S3FifoHandleDlistAdapter<T>>,
-    main_queue: Dlist<S3FifoHandleDlistAdapter<T>>,
+    small_queue: LinkedList<Adapter<K, V>>,
+    main_queue: LinkedList<Adapter<K, V>>,
 
     small_weight_capacity: usize,
 
@@ -158,120 +134,132 @@ where
     small_to_main_freq_threshold: u8,
 }
 
-impl<T> S3Fifo<T>
+impl<K, V> S3Fifo<K, V>
 where
-    T: Send + Sync + 'static,
+    K: Key,
+    V: Value,
 {
-    unsafe fn evict(&mut self) -> Option<NonNull<S3FifoHandle<T>>> {
+    fn evict(&mut self) -> Option<Arc<Record<S3Fifo<K, V>>>> {
         // TODO(MrCroxx): Use `let_chains` here after it is stable.
         if self.small_weight > self.small_weight_capacity {
-            if let Some(ptr) = self.evict_small() {
-                return Some(ptr);
+            if let Some(record) = self.evict_small() {
+                return Some(record);
             }
         }
-        if let Some(ptr) = self.evict_main() {
-            return Some(ptr);
+        if let Some(record) = self.evict_main() {
+            return Some(record);
         }
         self.evict_small_force()
     }
 
     #[expect(clippy::never_loop)]
-    unsafe fn evict_small_force(&mut self) -> Option<NonNull<S3FifoHandle<T>>> {
-        while let Some(mut ptr) = self.small_queue.pop_front() {
-            let handle = ptr.as_mut();
-            handle.queue = Queue::None;
-            handle.freq = 0;
-            self.small_weight -= handle.base().weight();
-            return Some(ptr);
+    fn evict_small_force(&mut self) -> Option<Arc<Record<S3Fifo<K, V>>>> {
+        while let Some(record) = self.small_queue.pop_front() {
+            let state = unsafe { &mut *record.state().get() };
+            state.queue = Queue::None;
+            state.set_frequency(0);
+            self.small_weight -= record.weight();
+            return Some(record);
         }
         None
     }
 
-    unsafe fn evict_small(&mut self) -> Option<NonNull<S3FifoHandle<T>>> {
-        while let Some(mut ptr) = self.small_queue.pop_front() {
-            let handle = ptr.as_mut();
-            if handle.freq >= self.small_to_main_freq_threshold {
-                handle.queue = Queue::Main;
-                self.main_queue.push_back(ptr);
-                self.small_weight -= handle.base().weight();
-                self.main_weight += handle.base().weight();
+    fn evict_small(&mut self) -> Option<Arc<Record<S3Fifo<K, V>>>> {
+        while let Some(record) = self.small_queue.pop_front() {
+            let state = unsafe { &mut *record.state().get() };
+            if state.frequency() >= self.small_to_main_freq_threshold {
+                state.queue = Queue::Main;
+                self.small_weight -= record.weight();
+                self.main_weight += record.weight();
+                self.main_queue.push_back(record);
             } else {
-                handle.queue = Queue::None;
-                handle.freq = 0;
-                self.small_weight -= handle.base().weight();
+                state.queue = Queue::None;
+                state.set_frequency(0);
+                self.small_weight -= record.weight();
 
-                self.ghost_queue.push(handle.base().hash(), handle.base().weight());
+                self.ghost_queue.push(record.hash(), record.weight());
 
-                return Some(ptr);
+                return Some(record);
             }
         }
         None
     }
 
-    unsafe fn evict_main(&mut self) -> Option<NonNull<S3FifoHandle<T>>> {
-        while let Some(mut ptr) = self.main_queue.pop_front() {
-            let handle = ptr.as_mut();
-            if handle.freq > 0 {
-                handle.freq_dec();
-                self.main_queue.push_back(ptr);
+    fn evict_main(&mut self) -> Option<Arc<Record<S3Fifo<K, V>>>> {
+        while let Some(record) = self.main_queue.pop_front() {
+            let state = unsafe { &mut *record.state().get() };
+            if state.dec_frequency() > 0 {
+                self.main_queue.push_back(record);
             } else {
-                handle.queue = Queue::None;
-                self.main_weight -= handle.base.weight();
-                return Some(ptr);
+                state.queue = Queue::None;
+                self.main_weight -= record.weight();
+                return Some(record);
             }
         }
         None
     }
 }
 
-impl<T> Eviction for S3Fifo<T>
+impl<K, V> Eviction for S3Fifo<K, V>
 where
-    T: Send + Sync + 'static,
+    K: Key,
+    V: Value,
 {
-    type Handle = S3FifoHandle<T>;
     type Config = S3FifoConfig;
+    type Key = K;
+    type Value = V;
+    type Hint = S3FifoHint;
+    type State = S3FifoState;
 
-    unsafe fn new(capacity: usize, config: &Self::Config) -> Self
+    fn new(capacity: usize, config: &Self::Config) -> Self
     where
         Self: Sized,
     {
-        let ghost_queue = GhostQueue::new((capacity as f64 * config.ghost_queue_capacity_ratio) as usize);
+        let ghost_queue_capacity = (capacity as f64 * config.ghost_queue_capacity_ratio) as usize;
+        let ghost_queue = GhostQueue::new(ghost_queue_capacity);
         let small_weight_capacity = (capacity as f64 * config.small_queue_capacity_ratio) as usize;
         Self {
             ghost_queue,
-            small_queue: Dlist::new(),
-            main_queue: Dlist::new(),
+            small_queue: LinkedList::new(Adapter::new()),
+            main_queue: LinkedList::new(Adapter::new()),
             small_weight_capacity,
             small_weight: 0,
             main_weight: 0,
-            small_to_main_freq_threshold: config.small_to_main_freq_threshold.min(MAX_FREQ),
+            small_to_main_freq_threshold: config.small_to_main_freq_threshold.min(S3FifoState::MAX_FREQUENCY),
         }
     }
 
-    unsafe fn push(&mut self, mut ptr: NonNull<Self::Handle>) {
-        let handle = ptr.as_mut();
-        strict_assert_eq!(handle.freq, 0);
-        strict_assert_eq!(handle.queue, Queue::None);
+    fn update(&mut self, capacity: usize, config: &Self::Config) {
+        let ghost_queue_capacity = (capacity as f64 * config.ghost_queue_capacity_ratio) as usize;
+        let small_weight_capacity = (capacity as f64 * config.small_queue_capacity_ratio) as usize;
+        self.ghost_queue.update(ghost_queue_capacity);
+        self.small_weight_capacity = small_weight_capacity;
+    }
+
+    fn push(&mut self, record: Arc<Record<Self>>) {
+        let state = unsafe { &mut *record.state().get() };
 
-        if self.ghost_queue.contains(handle.base().hash()) {
-            handle.queue = Queue::Main;
-            self.main_queue.push_back(ptr);
-            self.main_weight += handle.base().weight();
+        strict_assert_eq!(state.frequency(), 0);
+        strict_assert_eq!(state.queue, Queue::None);
+
+        record.set_in_eviction(true);
+
+        if self.ghost_queue.contains(record.hash()) {
+            state.queue = Queue::Main;
+            self.main_weight += record.weight();
+            self.main_queue.push_back(record);
         } else {
-            handle.queue = Queue::Small;
-            self.small_queue.push_back(ptr);
-            self.small_weight += handle.base().weight();
+            state.queue = Queue::Small;
+            self.small_weight += record.weight();
+            self.small_queue.push_back(record);
         }
-
-        handle.base_mut().set_in_eviction(true);
     }
 
-    unsafe fn pop(&mut self) -> Option<NonNull<Self::Handle>> {
-        if let Some(mut ptr) = self.evict() {
-            let handle = ptr.as_mut();
+    fn pop(&mut self) -> Option<Arc<Record<Self>>> {
+        if let Some(record) = self.evict() {
             // `handle.queue` has already been set with `evict()`
-            handle.base_mut().set_in_eviction(false);
-            Some(ptr)
+            record.set_in_eviction(false);
+            Some(record)
         } else {
             strict_assert!(self.small_queue.is_empty());
             strict_assert!(self.main_queue.is_empty());
@@ -279,69 +267,59 @@ where
         }
     }
 
-    unsafe fn release(&mut self, _: NonNull<Self::Handle>) {}
+    fn remove(&mut self, record: &Arc<Record<Self>>) {
+        let state = unsafe { &mut *record.state().get() };
 
-    unsafe fn acquire(&mut self, ptr: NonNull<Self::Handle>) {
-        let mut ptr = ptr;
-        ptr.as_mut().freq_inc();
-    }
-
-    unsafe fn remove(&mut self, mut ptr: NonNull<Self::Handle>) {
-        let handle = ptr.as_mut();
-
-        match handle.queue {
+        match state.queue {
             Queue::None => unreachable!(),
             Queue::Main => {
-                let p = self
-                    .main_queue
-                    .iter_mut_from_raw(ptr.as_mut().link.raw())
-                    .remove()
-                    .strict_unwrap_unchecked();
-                strict_assert_eq!(p, ptr);
-
-                handle.queue = Queue::None;
-                handle.freq = 0;
-                handle.base_mut().set_in_eviction(false);
-
-                self.main_weight -= handle.base().weight();
+                unsafe { self.main_queue.remove_from_ptr(Arc::as_ptr(record)) };
+
+                state.queue = Queue::None;
+                state.set_frequency(0);
+                record.set_in_eviction(false);
+
+                self.main_weight -= record.weight();
             }
             Queue::Small => {
-                let p = self
-                    .small_queue
-                    .iter_mut_from_raw(ptr.as_mut().link.raw())
-                    .remove()
-                    .strict_unwrap_unchecked();
-                strict_assert_eq!(p, ptr);
-
-                handle.queue = Queue::None;
-                handle.freq = 0;
-                handle.base_mut().set_in_eviction(false);
-
-                self.small_weight -= handle.base().weight();
+                unsafe { self.small_queue.remove_from_ptr(Arc::as_ptr(record)) };
+
+                state.queue = Queue::None;
+                state.set_frequency(0);
+                record.set_in_eviction(false);
+
+                self.small_weight -= record.weight();
             }
         }
     }
 
-    unsafe fn clear(&mut self) -> Vec<NonNull<Self::Handle>> {
-        let mut res = Vec::with_capacity(self.len());
-        while let Some(ptr) = self.pop() {
-            res.push(ptr);
-        }
-        res
+    fn acquire_operator() -> super::Operator {
+        Operator::Immutable
     }
 
-    fn len(&self) -> usize {
-        self.small_queue.len() + self.main_queue.len()
+    fn acquire_immutable(&self, record: &Arc<Record<Self>>) {
+        let state = unsafe { &mut *record.state().get() };
+        state.inc_frequency();
     }
 
-    fn is_empty(&self) -> bool {
-        self.small_queue.is_empty() && self.main_queue.is_empty()
+    fn acquire_mutable(&mut self, _record: &Arc<Record<Self>>) {
+        unreachable!()
+    }
+
+    fn release_operator() -> Operator {
+        Operator::Noop
+    }
+
+    fn release_immutable(&self, _record: &Arc<Record<Self>>) {
+        unreachable!()
     }
-}
 
-unsafe impl<T> Send for S3Fifo<T> where T: Send + Sync + 'static {}
-unsafe impl<T> Sync for S3Fifo<T> where T: Send + Sync + 'static {}
+    fn release_mutable(&mut self, _record: &Arc<Record<Self>>) {
+        unreachable!()
+    }
+}
 
+// TODO(MrCroxx): use ordered hash map?
 struct GhostQueue {
     counts: HashSet<u64>,
     queue: VecDeque<(u64, usize)>,
@@ -359,6 +337,16 @@ impl GhostQueue {
         }
     }
 
+    fn update(&mut self, capacity: usize) {
+        self.capacity = capacity;
+        if self.capacity == 0 {
+            return;
+        }
+        while self.weight > self.capacity && self.weight > 0 {
+            self.pop();
+        }
+    }
+
     fn push(&mut self, hash: u64, weight: usize) {
         if self.capacity == 0 {
             return;
@@ -390,97 +378,109 @@ mod tests {
     use itertools::Itertools;
 
     use super::*;
-    use crate::{eviction::test_utils::TestEviction, handle::HandleExt};
+    use crate::{
+        eviction::test_utils::{assert_ptr_eq, assert_ptr_vec_vec_eq, TestEviction},
+        record::Data,
+    };
 
-    impl<T> TestEviction for S3Fifo<T>
+    impl<K, V> TestEviction for S3Fifo<K, V>
     where
-        T: Send + Sync + 'static + Clone,
+        K: Key + Clone,
+        V: Value + Clone,
     {
-        fn dump(&self) -> Vec<T> {
-            self.small_queue
-                .iter()
-                .chain(self.main_queue.iter())
-                .map(|handle| handle.base().data_unwrap_unchecked().clone())
-                .collect_vec()
+        type Dump = Vec<Vec<Arc<Record<Self>>>>;
+
+        fn dump(&self) -> Self::Dump {
+            let mut small = vec![];
+            let mut main = vec![];
+
+            let mut cursor = self.small_queue.cursor();
+            loop {
+                cursor.move_next();
+                match cursor.clone_pointer() {
+                    Some(record) => small.push(record),
+                    None => break,
+                }
+            }
+
+            let mut cursor = self.main_queue.cursor();
+            loop {
+                cursor.move_next();
+                match cursor.clone_pointer() {
+                    Some(record) => main.push(record),
+                    None => break,
+                }
+            }
+
+            vec![small, main]
         }
     }
 
-    type TestS3Fifo = S3Fifo<u64>;
-    type TestS3FifoHandle = S3FifoHandle<u64>;
-
-    fn assert_test_s3fifo(s3fifo: &TestS3Fifo, small: Vec<u64>, main: Vec<u64>) {
-        let mut s = s3fifo.dump().into_iter().collect_vec();
-        assert_eq!(s.len(), s3fifo.small_queue.len() + s3fifo.main_queue.len());
-        let m = s.split_off(s3fifo.small_queue.len());
-        assert_eq!((&s, &m), (&small, &main));
-        assert_eq!(s3fifo.small_weight, s.len());
-        assert_eq!(s3fifo.main_weight, m.len());
-        assert_eq!(s3fifo.len(), s3fifo.small_queue.len() + s3fifo.main_queue.len());
-    }
+    type TestS3Fifo = S3Fifo<u64, u64>;
 
-    fn assert_count(ptrs: &[NonNull<TestS3FifoHandle>], range: Range<usize>, count: u8) {
-        unsafe {
-            ptrs[range].iter().for_each(|ptr| assert_eq!(ptr.as_ref().freq, count));
-        }
+    fn assert_frequencies(rs: &[Arc<Record<TestS3Fifo>>], range: Range<usize>, count: u8) {
+        rs[range]
+            .iter()
+            .for_each(|r| assert_eq!(unsafe { &*r.state().get() }.frequency(), count));
     }
 
     #[test]
     fn test_s3fifo() {
-        unsafe {
-            let ptrs = (0..100)
-                .map(|i| {
-                    let mut handle = Box::<TestS3FifoHandle>::default();
-                    handle.init(i, i, 1, S3FifoContext(CacheContext::Default));
-                    NonNull::new_unchecked(Box::into_raw(handle))
-                })
-                .collect_vec();
-
-            // capacity: 8, small: 2, ghost: 80
-            let config = S3FifoConfig {
-                small_queue_capacity_ratio: 0.25,
-                ghost_queue_capacity_ratio: 10.0,
-                small_to_main_freq_threshold: 2,
-            };
-            let mut s3fifo = TestS3Fifo::new(8, &config);
-
-            assert_eq!(s3fifo.small_weight_capacity, 2);
-
-            s3fifo.push(ptrs[0]);
-            s3fifo.push(ptrs[1]);
-            assert_test_s3fifo(&s3fifo, vec![0, 1], vec![]);
-
-            s3fifo.push(ptrs[2]);
-            s3fifo.push(ptrs[3]);
-            assert_test_s3fifo(&s3fifo, vec![0, 1, 2, 3], vec![]);
-
-            assert_count(&ptrs, 0..4, 0);
-
-            (0..4).for_each(|i| s3fifo.acquire(ptrs[i]));
-            s3fifo.acquire(ptrs[1]);
-            s3fifo.acquire(ptrs[2]);
-            assert_count(&ptrs, 0..1, 1);
-            assert_count(&ptrs, 1..3, 2);
-            assert_count(&ptrs, 3..4, 1);
-
-            let p0 = s3fifo.pop().unwrap();
-            let p3 = s3fifo.pop().unwrap();
-            assert_eq!(p0, ptrs[0]);
-            assert_eq!(p3, ptrs[3]);
-            assert_test_s3fifo(&s3fifo, vec![], vec![1, 2]);
-            assert_count(&ptrs, 0..1, 0);
-            assert_count(&ptrs, 1..3, 2);
-            assert_count(&ptrs, 3..4, 0);
-
-            let p1 = s3fifo.pop().unwrap();
-            assert_eq!(p1, ptrs[1]);
-            assert_test_s3fifo(&s3fifo, vec![], vec![2]);
-            assert_count(&ptrs, 0..4, 0);
-
-            assert_eq!(s3fifo.clear(), [2].into_iter().map(|i| ptrs[i]).collect_vec());
-
-            for ptr in ptrs {
-                let _ = Box::from_raw(ptr.as_ptr());
-            }
-        }
+        let rs = (0..100)
+            .map(|i| {
+                Arc::new(Record::new(Data {
+                    key: i,
+                    value: i,
+                    hint: S3FifoHint,
+                    hash: i,
+                    weight: 1,
+                }))
+            })
+            .collect_vec();
+        let r = |i: usize| rs[i].clone();
+
+        // capacity: 8, small: 2, ghost: 80
+        let config = S3FifoConfig {
+            small_queue_capacity_ratio: 0.25,
+            ghost_queue_capacity_ratio: 10.0,
+            small_to_main_freq_threshold: 2,
+        };
+        let mut s3fifo = TestS3Fifo::new(8, &config);
+
+        assert_eq!(s3fifo.small_weight_capacity, 2);
+
+        s3fifo.push(r(0));
+        s3fifo.push(r(1));
+        assert_ptr_vec_vec_eq(s3fifo.dump(), vec![vec![r(0), r(1)], vec![]]);
+
+        s3fifo.push(r(2));
+        s3fifo.push(r(3));
+        assert_ptr_vec_vec_eq(s3fifo.dump(), vec![vec![r(0), r(1), r(2), r(3)], vec![]]);
+
+        assert_frequencies(&rs, 0..4, 0);
+
+        (0..4).for_each(|i| s3fifo.acquire_immutable(&rs[i]));
+        s3fifo.acquire_immutable(&rs[1]);
+        s3fifo.acquire_immutable(&rs[2]);
+        assert_frequencies(&rs, 0..1, 1);
+        assert_frequencies(&rs, 1..3, 2);
+        assert_frequencies(&rs, 3..4, 1);
+
+        let r0 = s3fifo.pop().unwrap();
+        let r3 = s3fifo.pop().unwrap();
+        assert_ptr_eq(&rs[0], &r0);
+        assert_ptr_eq(&rs[3], &r3);
+        assert_ptr_vec_vec_eq(s3fifo.dump(), vec![vec![], vec![r(1), r(2)]]);
+        assert_frequencies(&rs, 0..1, 0);
+        assert_frequencies(&rs, 1..3, 2);
+        assert_frequencies(&rs, 3..4, 0);
+
+        let r1 = s3fifo.pop().unwrap();
+        assert_ptr_eq(&rs[1], &r1);
+        assert_ptr_vec_vec_eq(s3fifo.dump(), vec![vec![], vec![r(2)]]);
+        assert_frequencies(&rs, 0..4, 0);
+
+        s3fifo.clear();
+        assert_ptr_vec_vec_eq(s3fifo.dump(), vec![vec![], vec![]]);
     }
 }
diff --git a/foyer-memory/src/eviction/sanity.rs b/foyer-memory/src/eviction/sanity.rs
deleted file mode 100644
index 4d965a13..00000000
--- a/foyer-memory/src/eviction/sanity.rs
+++ /dev/null
@@ -1,152 +0,0 @@
-//  Copyright 2024 foyer Project Authors
-//
-//  Licensed under the Apache License, Version 2.0 (the "License");
-//  you may not use this file except in compliance with the License.
-//  You may obtain a copy of the License at
-//
-//  http://www.apache.org/licenses/LICENSE-2.0
-//
-//  Unless required by applicable law or agreed to in writing, software
-//  distributed under the License is distributed on an "AS IS" BASIS,
-//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-//  See the License for the specific language governing permissions and
-//  limitations under the License.
-
-use super::Eviction;
-#[cfg(feature = "sanity")]
-use crate::handle::Handle;
-
-pub struct SanityEviction<E>
-where
-    E: Eviction,
-{
-    eviction: E,
-}
-
-#[cfg(feature = "sanity")]
-impl<E> Eviction for SanityEviction<E>
-where
-    E: Eviction,
-{
-    type Handle = E::Handle;
-    type Config = E::Config;
-
-    unsafe fn new(capacity: usize, config: &Self::Config) -> Self
-    where
-        Self: Sized,
-    {
-        Self {
-            eviction: E::new(capacity, config),
-        }
-    }
-
-    unsafe fn push(&mut self, ptr: std::ptr::NonNull<Self::Handle>) {
-        assert!(!ptr.as_ref().base().is_in_eviction());
-        self.eviction.push(ptr);
-        assert!(ptr.as_ref().base().is_in_eviction());
-    }
-
-    unsafe fn pop(&mut self) -> Option<std::ptr::NonNull<Self::Handle>> {
-        let res = self.eviction.pop();
-        if let Some(ptr) = res {
-            assert!(!ptr.as_ref().base().is_in_eviction());
-        }
-        res
-    }
-
-    unsafe fn acquire(&mut self, ptr: std::ptr::NonNull<Self::Handle>) {
-        self.eviction.acquire(ptr)
-    }
-
-    unsafe fn release(&mut self, ptr: std::ptr::NonNull<Self::Handle>) {
-        self.eviction.release(ptr)
-    }
-
-    unsafe fn remove(&mut self, ptr: std::ptr::NonNull<Self::Handle>) {
-        assert!(ptr.as_ref().base().is_in_eviction());
-        self.eviction.remove(ptr);
-        assert!(!ptr.as_ref().base().is_in_eviction());
-    }
-
-    unsafe fn clear(&mut self) -> Vec<std::ptr::NonNull<Self::Handle>> {
-        let res = self.eviction.clear();
-        res.iter()
-            .for_each(|ptr| assert!(!ptr.as_ref().base().is_in_eviction()));
-        res
-    }
-
-    fn len(&self) -> usize {
-        self.eviction.len()
-    }
-
-    fn is_empty(&self) -> bool {
-        self.eviction.is_empty()
-    }
-}
-
-#[cfg(not(feature = "sanity"))]
-impl<E> Eviction for SanityEviction<E>
-where
-    E: Eviction,
-{
-    type Handle = E::Handle;
-    type Config = E::Config;
-
-    unsafe fn new(capacity: usize, config: &Self::Config) -> Self
-    where
-        Self: Sized,
-    {
-        Self {
-            eviction: E::new(capacity, config),
-        }
-    }
-
-    unsafe fn push(&mut self, ptr: std::ptr::NonNull<Self::Handle>) {
-        self.eviction.push(ptr)
-    }
-
-    unsafe fn pop(&mut self) -> Option<std::ptr::NonNull<Self::Handle>> {
-        self.eviction.pop()
-    }
-
-    unsafe fn acquire(&mut self, ptr: std::ptr::NonNull<Self::Handle>) {
-        self.eviction.acquire(ptr)
-    }
-
-    unsafe fn release(&mut self, ptr: std::ptr::NonNull<Self::Handle>) {
-        self.eviction.release(ptr)
-    }
-
-    unsafe fn remove(&mut self, ptr: std::ptr::NonNull<Self::Handle>) {
-        self.eviction.remove(ptr)
-    }
-
-    unsafe fn clear(&mut self) -> Vec<std::ptr::NonNull<Self::Handle>> {
-        self.eviction.clear()
-    }
-
-    fn len(&self) -> usize {
-        self.eviction.len()
-    }
-
-    fn is_empty(&self) -> bool {
-        self.eviction.is_empty()
-    }
-}
-
-#[cfg(test)]
-pub mod tests {
-    use super::*;
-    use crate::{eviction::test_utils::TestEviction, handle::Handle};
-
-    impl<T, E> TestEviction for SanityEviction<E>
-    where
-        T: Send + Sync + 'static + Clone,
-        E: TestEviction,
-        E::Handle: Handle<Data = T>,
-    {
-        fn dump(&self) -> Vec<T> {
-            self.eviction.dump()
-        }
-    }
-}
diff --git a/foyer-memory/src/eviction/test_utils.rs b/foyer-memory/src/eviction/test_utils.rs
index be965799..a1d6f387 100644
--- a/foyer-memory/src/eviction/test_utils.rs
+++ b/foyer-memory/src/eviction/test_utils.rs
@@ -12,12 +12,28 @@
 //  See the License for the specific language governing permissions and
 //  limitations under the License.
 
+use std::sync::Arc;
+
+use itertools::Itertools;
+
 use super::Eviction;
-use crate::handle::Handle;
 
-pub trait TestEviction: Eviction
-where
-    Self::Handle: Handle,
-{
-    fn dump(&self) -> Vec<<Self::Handle as Handle>::Data>;
+pub trait TestEviction: Eviction {
+    type Dump;
+    fn dump(&self) -> Self::Dump;
+}
+
+pub fn assert_ptr_eq<T>(a: &Arc<T>, b: &Arc<T>) {
+    assert_eq!(Arc::as_ptr(a), Arc::as_ptr(b));
+}
+
+pub fn assert_ptr_vec_eq<T>(va: Vec<Arc<T>>, vb: Vec<Arc<T>>) {
+    let trans = |v: Vec<Arc<T>>| v.iter().map(Arc::as_ptr).collect_vec();
+    assert_eq!(trans(va), trans(vb));
+}
+
+pub fn assert_ptr_vec_vec_eq<T>(vva: Vec<Vec<Arc<T>>>, vvb: Vec<Vec<Arc<T>>>) {
+    let trans = |vv: Vec<Vec<Arc<T>>>| vv.iter().map(|v| v.iter().map(Arc::as_ptr).collect_vec()).collect_vec();
+
+    assert_eq!(trans(vva), trans(vvb));
 }
diff --git a/foyer-memory/src/generic.rs b/foyer-memory/src/generic.rs
deleted file mode 100644
index 6d4504fe..00000000
--- a/foyer-memory/src/generic.rs
+++ /dev/null
@@ -1,1403 +0,0 @@
-//  Copyright 2024 foyer Project Authors
-//
-//  Licensed under the Apache License, Version 2.0 (the "License");
-//  you may not use this file except in compliance with the License.
-//  You may obtain a copy of the License at
-//
-//  http://www.apache.org/licenses/LICENSE-2.0
-//
-//  Unless required by applicable law or agreed to in writing, software
-//  distributed under the License is distributed on an "AS IS" BASIS,
-//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-//  See the License for the specific language governing permissions and
-//  limitations under the License.
-
-use std::{
-    fmt::Debug,
-    future::Future,
-    hash::Hash,
-    ops::Deref,
-    pin::Pin,
-    ptr::NonNull,
-    sync::{
-        atomic::{AtomicUsize, Ordering},
-        Arc,
-    },
-    task::{Context, Poll},
-};
-
-use ahash::RandomState;
-use equivalent::Equivalent;
-use fastrace::{future::InSpan, prelude::*};
-use foyer_common::{
-    code::{HashBuilder, Key, Value},
-    event::EventListener,
-    future::{Diversion, DiversionFuture},
-    metrics::Metrics,
-    object_pool::ObjectPool,
-    runtime::SingletonHandle,
-    strict_assert, strict_assert_eq,
-};
-use hashbrown::hash_map::{Entry as HashMapEntry, HashMap};
-use itertools::Itertools;
-use parking_lot::{lock_api::MutexGuard, Mutex, RawMutex};
-use pin_project::pin_project;
-use tokio::{sync::oneshot, task::JoinHandle};
-
-use crate::{
-    eviction::Eviction,
-    handle::{Handle, HandleExt, KeyedHandle},
-    indexer::Indexer,
-    CacheContext,
-};
-
-// TODO(MrCroxx): Use `trait_alias` after stable.
-/// The weighter for the in-memory cache.
-///
-/// The weighter is used to calculate the weight of the cache entry.
-pub trait Weighter<K, V>: Fn(&K, &V) -> usize + Send + Sync + 'static {}
-impl<K, V, T> Weighter<K, V> for T where T: Fn(&K, &V) -> usize + Send + Sync + 'static {}
-
-struct SharedState<K, V, T> {
-    metrics: Arc<Metrics>,
-    /// The object pool to avoid frequent handle allocating, shared by all shards.
-    object_pool: ObjectPool<Box<T>>,
-    event_listener: Option<Arc<dyn EventListener<Key = K, Value = V>>>,
-}
-
-#[expect(clippy::type_complexity)]
-struct GenericCacheShard<K, V, E, I, S>
-where
-    K: Key,
-    V: Value,
-    E: Eviction,
-    E::Handle: KeyedHandle<Key = K, Data = (K, V)>,
-    I: Indexer<Key = K, Handle = E::Handle>,
-    S: HashBuilder,
-{
-    indexer: I,
-    eviction: E,
-
-    capacity: usize,
-    usage: Arc<AtomicUsize>,
-
-    waiters: HashMap<K, Vec<oneshot::Sender<GenericCacheEntry<K, V, E, I, S>>>>,
-
-    state: Arc<SharedState<K, V, E::Handle>>,
-}
-
-impl<K, V, E, I, S> GenericCacheShard<K, V, E, I, S>
-where
-    K: Key,
-    V: Value,
-    E: Eviction,
-    E::Handle: KeyedHandle<Key = K, Data = (K, V)>,
-    I: Indexer<Key = K, Handle = E::Handle>,
-    S: HashBuilder,
-{
-    fn new(
-        capacity: usize,
-        eviction_config: &E::Config,
-        usage: Arc<AtomicUsize>,
-        context: Arc<SharedState<K, V, E::Handle>>,
-    ) -> Self {
-        let indexer = I::new();
-        let eviction = unsafe { E::new(capacity, eviction_config) };
-        let waiters = HashMap::default();
-        Self {
-            indexer,
-            eviction,
-            capacity,
-            usage,
-            waiters,
-            state: context,
-        }
-    }
-
-    /// Insert a new entry into the cache. The handle for the new entry is returned.
-
-    #[expect(clippy::too_many_arguments)]
-    #[fastrace::trace(name = "foyer::memory::generic::shard::emplace")]
-    unsafe fn emplace(
-        &mut self,
-        hash: u64,
-        key: K,
-        value: V,
-        weight: usize,
-        context: <E::Handle as Handle>::Context,
-        deposit: bool,
-        to_release: &mut Vec<(K, V, <E::Handle as Handle>::Context, usize)>,
-    ) -> NonNull<E::Handle> {
-        let mut handle = self.state.object_pool.acquire();
-        strict_assert!(!handle.base().has_refs());
-        strict_assert!(!handle.base().is_in_indexer());
-        strict_assert!(!handle.base().is_in_eviction());
-
-        handle.init(hash, (key, value), weight, context);
-        let mut ptr = unsafe { NonNull::new_unchecked(Box::into_raw(handle)) };
-
-        self.evict(weight, to_release);
-
-        strict_assert!(!ptr.as_ref().base().is_in_indexer());
-        if let Some(old) = self.indexer.insert(ptr) {
-            self.state.metrics.memory_replace.increment(1);
-
-            strict_assert!(!old.as_ref().base().is_in_indexer());
-            if old.as_ref().base().is_in_eviction() {
-                self.eviction.remove(old);
-            }
-            strict_assert!(!old.as_ref().base().is_in_eviction());
-            // Because the `old` handle is removed from the indexer, it will not be reinserted again.
-            if let Some(entry) = self.try_release_handle(old, false) {
-                to_release.push(entry);
-            }
-        } else {
-            self.state.metrics.memory_insert.increment(1);
-        }
-        strict_assert!(ptr.as_ref().base().is_in_indexer());
-
-        ptr.as_mut().base_mut().set_deposit(deposit);
-        if !deposit {
-            self.eviction.push(ptr);
-            strict_assert!(ptr.as_ref().base().is_in_eviction());
-        }
-
-        self.usage.fetch_add(weight, Ordering::Relaxed);
-        self.state.metrics.memory_usage.increment(weight as f64);
-        ptr.as_mut().base_mut().inc_refs();
-
-        ptr
-    }
-
-    unsafe fn get<Q>(&mut self, hash: u64, key: &Q) -> Option<NonNull<E::Handle>>
-    where
-        Q: Hash + Equivalent<K> + ?Sized,
-    {
-        let mut ptr = match self.indexer.get(hash, key) {
-            Some(ptr) => {
-                self.state.metrics.memory_hit.increment(1);
-                ptr
-            }
-            None => {
-                self.state.metrics.memory_miss.increment(1);
-                return None;
-            }
-        };
-        let base = ptr.as_mut().base_mut();
-        strict_assert!(base.is_in_indexer());
-
-        base.set_deposit(false);
-        base.inc_refs();
-        self.eviction.acquire(ptr);
-
-        Some(ptr)
-    }
-
-    unsafe fn contains<Q>(&mut self, hash: u64, key: &Q) -> bool
-    where
-        Q: Hash + Equivalent<K> + ?Sized,
-    {
-        self.indexer.get(hash, key).is_some()
-    }
-
-    unsafe fn touch<Q>(&mut self, hash: u64, key: &Q) -> bool
-    where
-        Q: Hash + Equivalent<K> + ?Sized,
-    {
-        let res = self.indexer.get(hash, key);
-        if let Some(ptr) = res {
-            self.eviction.acquire(ptr);
-        }
-        res.is_some()
-    }
-
-    /// Remove a key from the cache.
-    ///
-    /// Return `Some(..)` if the handle is released, or `None` if the handle is still in use.
-    unsafe fn remove<Q>(&mut self, hash: u64, key: &Q) -> Option<NonNull<E::Handle>>
-    where
-        Q: Hash + Equivalent<K> + ?Sized,
-    {
-        let mut ptr = self.indexer.remove(hash, key)?;
-        let handle = ptr.as_mut();
-
-        self.state.metrics.memory_remove.increment(1);
-
-        if handle.base().is_in_eviction() {
-            self.eviction.remove(ptr);
-        }
-
-        strict_assert!(!handle.base().is_in_indexer());
-        strict_assert!(!handle.base().is_in_eviction());
-
-        handle.base_mut().inc_refs();
-
-        Some(ptr)
-    }
-
-    /// Clear all cache entries.
-    unsafe fn clear(&mut self, to_release: &mut Vec<(K, V, <E::Handle as Handle>::Context, usize)>) {
-        // TODO(MrCroxx): Avoid collecting here?
-        let ptrs = self.indexer.drain().collect_vec();
-        let eptrs = self.eviction.clear();
-
-        // Assert that the handles in the indexer covers the handles in the eviction container.
-        if cfg!(debug_assertions) {
-            use std::{collections::HashSet as StdHashSet, hash::RandomState as StdRandomState};
-            let ptrs: StdHashSet<_, StdRandomState> = StdHashSet::from_iter(ptrs.iter().copied());
-            let eptrs: StdHashSet<_, StdRandomState> = StdHashSet::from_iter(eptrs.iter().copied());
-            assert!((&eptrs - &ptrs).is_empty());
-        }
-
-        self.state.metrics.memory_remove.increment(ptrs.len() as _);
-
-        // The handles in the indexer covers the handles in the eviction container.
-        // So only the handles drained from the indexer need to be released.
-        for ptr in ptrs {
-            strict_assert!(!ptr.as_ref().base().is_in_indexer());
-            strict_assert!(!ptr.as_ref().base().is_in_eviction());
-            if let Some(entry) = self.try_release_handle(ptr, false) {
-                to_release.push(entry);
-            }
-        }
-    }
-
-    #[fastrace::trace(name = "foyer::memory::generic::shard::evict")]
-    unsafe fn evict(&mut self, weight: usize, to_release: &mut Vec<(K, V, <E::Handle as Handle>::Context, usize)>) {
-        // TODO(MrCroxx): Use `let_chains` here after it is stable.
-        while self.usage.load(Ordering::Relaxed) + weight > self.capacity {
-            let evicted = match self.eviction.pop() {
-                Some(evicted) => evicted,
-                None => break,
-            };
-            self.state.metrics.memory_evict.increment(1);
-            let base = evicted.as_ref().base();
-            strict_assert!(base.is_in_indexer());
-            strict_assert!(!base.is_in_eviction());
-            if let Some(entry) = self.try_release_handle(evicted, false) {
-                to_release.push(entry);
-            }
-        }
-    }
-
-    /// Release a handle used by an external user.
-    ///
-    /// Return `Some(..)` if the handle is released, or `None` if the handle is still in use.
-    unsafe fn try_release_external_handle(
-        &mut self,
-        mut ptr: NonNull<E::Handle>,
-    ) -> Option<(K, V, <E::Handle as Handle>::Context, usize)> {
-        ptr.as_mut().base_mut().dec_refs();
-        self.try_release_handle(ptr, true)
-    }
-
-    /// Try release handle if there is no external reference and no reinsertion is needed.
-    ///
-    /// Return the entry if the handle is released.
-    ///
-    /// Recycle it if possible.
-    unsafe fn try_release_handle(
-        &mut self,
-        mut ptr: NonNull<E::Handle>,
-        reinsert: bool,
-    ) -> Option<(K, V, <E::Handle as Handle>::Context, usize)> {
-        let handle = ptr.as_mut();
-
-        if handle.base().has_refs() {
-            return None;
-        }
-
-        strict_assert!(handle.base().is_initialized());
-        strict_assert!(!handle.base().has_refs());
-
-        // If the entry is deposit (emplace by deposit & never read), remove it from indexer to skip reinsertion.
-        if handle.base().is_in_indexer() && handle.base().is_deposit() {
-            strict_assert!(!handle.base().is_in_eviction());
-            self.indexer.remove(handle.base().hash(), handle.key());
-            strict_assert!(!handle.base().is_in_indexer());
-        }
-
-        // If the entry is not updated or removed from the cache, try to reinsert it or remove it from the indexer and
-        // the eviction container.
-        if handle.base().is_in_indexer() {
-            // The usage is higher than the capacity means most handles are held externally,
-            // the cache shard cannot release enough weight for the new inserted entries.
-            // In this case, the reinsertion should be given up.
-            if reinsert && self.usage.load(Ordering::Relaxed) <= self.capacity {
-                let was_in_eviction = handle.base().is_in_eviction();
-                self.eviction.release(ptr);
-                if ptr.as_ref().base().is_in_eviction() {
-                    // The entry is not yep evicted, do NOT release it.
-                    if !was_in_eviction {
-                        self.state.metrics.memory_reinsert.increment(1);
-                    }
-                    strict_assert!(ptr.as_ref().base().is_in_indexer());
-                    strict_assert!(ptr.as_ref().base().is_in_eviction());
-                    strict_assert!(!handle.base().has_refs());
-                    return None;
-                }
-            }
-
-            // If the entry has not been reinserted, remove it from the indexer and the eviction container (if needed).
-            self.indexer.remove(handle.base().hash(), handle.key());
-            if ptr.as_ref().base().is_in_eviction() {
-                self.eviction.remove(ptr);
-            }
-        }
-
-        // Here the handle is neither in the indexer nor in the eviction container.
-        strict_assert!(!handle.base().is_in_indexer());
-        strict_assert!(!handle.base().is_in_eviction());
-        strict_assert!(!handle.base().has_refs());
-
-        self.state.metrics.memory_release.increment(1);
-
-        self.usage.fetch_sub(handle.base().weight(), Ordering::Relaxed);
-        self.state.metrics.memory_usage.decrement(handle.base().weight() as f64);
-        let ((key, value), context, weight) = handle.base_mut().take();
-
-        let handle = Box::from_raw(ptr.as_ptr());
-        self.state.object_pool.release(handle);
-
-        Some((key, value, context, weight))
-    }
-}
-
-pub struct GenericCacheConfig<K, V, E, S = RandomState>
-where
-    K: Key,
-    V: Value,
-    E: Eviction,
-    E::Handle: KeyedHandle<Key = K, Data = (K, V)>,
-    S: HashBuilder,
-{
-    pub name: String,
-    pub capacity: usize,
-    pub shards: usize,
-    pub eviction_config: E::Config,
-    pub object_pool_capacity: usize,
-    pub hash_builder: S,
-    pub weighter: Arc<dyn Weighter<K, V>>,
-    pub event_listener: Option<Arc<dyn EventListener<Key = K, Value = V>>>,
-}
-
-type GenericFetchHit<K, V, E, I, S> = Option<GenericCacheEntry<K, V, E, I, S>>;
-type GenericFetchWait<K, V, E, I, S> = InSpan<oneshot::Receiver<GenericCacheEntry<K, V, E, I, S>>>;
-type GenericFetchMiss<K, V, E, I, S, ER, DFS> =
-    JoinHandle<Diversion<std::result::Result<GenericCacheEntry<K, V, E, I, S>, ER>, DFS>>;
-
-/// The state of [`Fetch`].
-#[derive(Debug, Clone, Copy, PartialEq, Eq)]
-pub enum FetchState {
-    /// Cache hit.
-    Hit,
-    /// Cache miss, but wait in queue.
-    Wait,
-    /// Cache miss, and there is no other waiters at the moment.
-    Miss,
-}
-
-/// A mark for fetch calls.
-pub struct FetchMark;
-
-pub type GenericFetch<K, V, E, I, S, ER> = DiversionFuture<
-    GenericFetchInner<K, V, E, I, S, ER>,
-    std::result::Result<GenericCacheEntry<K, V, E, I, S>, ER>,
-    FetchMark,
->;
-
-#[pin_project(project = GenericFetchInnerProj)]
-pub enum GenericFetchInner<K, V, E, I, S, ER>
-where
-    K: Key,
-    V: Value,
-    E: Eviction,
-    E::Handle: KeyedHandle<Key = K, Data = (K, V)>,
-    I: Indexer<Key = K, Handle = E::Handle>,
-    S: HashBuilder,
-{
-    Hit(GenericFetchHit<K, V, E, I, S>),
-    Wait(#[pin] GenericFetchWait<K, V, E, I, S>),
-    Miss(#[pin] GenericFetchMiss<K, V, E, I, S, ER, FetchMark>),
-}
-
-impl<K, V, E, I, S, ER> GenericFetchInner<K, V, E, I, S, ER>
-where
-    K: Key,
-    V: Value,
-    E: Eviction,
-    E::Handle: KeyedHandle<Key = K, Data = (K, V)>,
-    I: Indexer<Key = K, Handle = E::Handle>,
-    S: HashBuilder,
-{
-    pub fn state(&self) -> FetchState {
-        match self {
-            GenericFetchInner::Hit(_) => FetchState::Hit,
-            GenericFetchInner::Wait(_) => FetchState::Wait,
-            GenericFetchInner::Miss(_) => FetchState::Miss,
-        }
-    }
-}
-
-impl<K, V, E, I, S, ER> Future for GenericFetchInner<K, V, E, I, S, ER>
-where
-    K: Key,
-    V: Value,
-    E: Eviction,
-    E::Handle: KeyedHandle<Key = K, Data = (K, V)>,
-    I: Indexer<Key = K, Handle = E::Handle>,
-    S: HashBuilder,
-    ER: From<oneshot::error::RecvError>,
-{
-    type Output = Diversion<std::result::Result<GenericCacheEntry<K, V, E, I, S>, ER>, FetchMark>;
-
-    fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Self::Output> {
-        match self.project() {
-            GenericFetchInnerProj::Hit(opt) => Poll::Ready(Ok(opt.take().unwrap()).into()),
-            GenericFetchInnerProj::Wait(waiter) => waiter.poll(cx).map_err(|err| err.into()).map(Diversion::from),
-            GenericFetchInnerProj::Miss(handle) => handle.poll(cx).map(|join| join.unwrap()),
-        }
-    }
-}
-
-#[expect(clippy::type_complexity)]
-pub struct GenericCache<K, V, E, I, S = RandomState>
-where
-    K: Key,
-    V: Value,
-    E: Eviction,
-    E::Handle: KeyedHandle<Key = K, Data = (K, V)>,
-    I: Indexer<Key = K, Handle = E::Handle>,
-    S: HashBuilder,
-{
-    shards: Vec<Mutex<GenericCacheShard<K, V, E, I, S>>>,
-
-    capacity: usize,
-    usages: Vec<Arc<AtomicUsize>>,
-
-    context: Arc<SharedState<K, V, E::Handle>>,
-
-    hash_builder: S,
-    weighter: Arc<dyn Weighter<K, V>>,
-
-    _metrics: Arc<Metrics>,
-}
-
-impl<K, V, E, I, S> GenericCache<K, V, E, I, S>
-where
-    K: Key,
-    V: Value,
-    E: Eviction,
-    E::Handle: KeyedHandle<Key = K, Data = (K, V)>,
-    I: Indexer<Key = K, Handle = E::Handle>,
-    S: HashBuilder,
-{
-    pub fn new(config: GenericCacheConfig<K, V, E, S>) -> Self {
-        let metrics = Arc::new(Metrics::new(&config.name));
-
-        let usages = (0..config.shards).map(|_| Arc::new(AtomicUsize::new(0))).collect_vec();
-        let context = Arc::new(SharedState {
-            metrics: metrics.clone(),
-            object_pool: ObjectPool::new_with_create(config.object_pool_capacity, Box::default),
-            event_listener: config.event_listener,
-        });
-
-        let shard_capacity = config.capacity / config.shards;
-
-        let shards = usages
-            .iter()
-            .map(|usage| {
-                GenericCacheShard::new(shard_capacity, &config.eviction_config, usage.clone(), context.clone())
-            })
-            .map(Mutex::new)
-            .collect_vec();
-
-        Self {
-            shards,
-            capacity: config.capacity,
-            usages,
-            context,
-            hash_builder: config.hash_builder,
-            weighter: config.weighter,
-            _metrics: metrics,
-        }
-    }
-
-    #[fastrace::trace(name = "foyer::memory::generic::insert")]
-    pub fn insert(self: &Arc<Self>, key: K, value: V) -> GenericCacheEntry<K, V, E, I, S> {
-        self.insert_with_context(key, value, CacheContext::default())
-    }
-
-    #[fastrace::trace(name = "foyer::memory::generic::insert_with_context")]
-    pub fn insert_with_context(
-        self: &Arc<Self>,
-        key: K,
-        value: V,
-        context: CacheContext,
-    ) -> GenericCacheEntry<K, V, E, I, S> {
-        self.emplace(key, value, context, false)
-    }
-
-    #[fastrace::trace(name = "foyer::memory::generic::deposit")]
-    pub fn deposit(self: &Arc<Self>, key: K, value: V) -> GenericCacheEntry<K, V, E, I, S> {
-        self.deposit_with_context(key, value, CacheContext::default())
-    }
-
-    #[fastrace::trace(name = "foyer::memory::generic::deposit_with_context")]
-    pub fn deposit_with_context(
-        self: &Arc<Self>,
-        key: K,
-        value: V,
-        context: CacheContext,
-    ) -> GenericCacheEntry<K, V, E, I, S> {
-        self.emplace(key, value, context, true)
-    }
-
-    #[fastrace::trace(name = "foyer::memory::generic::emplace")]
-    fn emplace(
-        self: &Arc<Self>,
-        key: K,
-        value: V,
-        context: CacheContext,
-        deposit: bool,
-    ) -> GenericCacheEntry<K, V, E, I, S> {
-        let hash = self.hash_builder.hash_one(&key);
-        let weight = (self.weighter)(&key, &value);
-
-        let mut to_release = vec![];
-
-        let (entry, waiters) = unsafe {
-            let mut shard = self.shard(hash as usize % self.shards.len());
-            let waiters = shard.waiters.remove(&key);
-            let mut ptr = shard.emplace(hash, key, value, weight, context.into(), deposit, &mut to_release);
-            if let Some(waiters) = waiters.as_ref() {
-                // Increase the reference count within the lock section.
-                ptr.as_mut().base_mut().inc_refs_by(waiters.len());
-                strict_assert_eq!(ptr.as_ref().base().refs(), waiters.len() + 1);
-            }
-            let entry = GenericCacheEntry {
-                cache: self.clone(),
-                ptr,
-            };
-            (entry, waiters)
-        };
-
-        if let Some(waiters) = waiters {
-            for waiter in waiters {
-                let _ = waiter.send(GenericCacheEntry {
-                    cache: self.clone(),
-                    ptr: entry.ptr,
-                });
-            }
-        }
-
-        // Do not deallocate data within the lock section.
-        if let Some(listener) = self.context.event_listener.as_ref() {
-            for (k, v, _c, _w) in to_release {
-                listener.on_memory_release(k, v);
-            }
-        }
-
-        entry
-    }
-
-    #[fastrace::trace(name = "foyer::memory::generic::remove")]
-    pub fn remove<Q>(self: &Arc<Self>, key: &Q) -> Option<GenericCacheEntry<K, V, E, I, S>>
-    where
-        Q: Hash + Equivalent<K> + ?Sized,
-    {
-        let hash = self.hash_builder.hash_one(key);
-
-        unsafe {
-            let mut shard = self.shard(hash as usize % self.shards.len());
-            shard.remove(hash, key).map(|ptr| GenericCacheEntry {
-                cache: self.clone(),
-                ptr,
-            })
-        }
-    }
-
-    #[fastrace::trace(name = "foyer::memory::generic::get")]
-    pub fn get<Q>(self: &Arc<Self>, key: &Q) -> Option<GenericCacheEntry<K, V, E, I, S>>
-    where
-        Q: Hash + Equivalent<K> + ?Sized,
-    {
-        let hash = self.hash_builder.hash_one(key);
-
-        unsafe {
-            let mut shard = self.shard(hash as usize % self.shards.len());
-            shard.get(hash, key).map(|ptr| GenericCacheEntry {
-                cache: self.clone(),
-                ptr,
-            })
-        }
-    }
-
-    pub fn contains<Q>(self: &Arc<Self>, key: &Q) -> bool
-    where
-        Q: Hash + Equivalent<K> + ?Sized,
-    {
-        let hash = self.hash_builder.hash_one(key);
-
-        unsafe {
-            let mut shard = self.shard(hash as usize % self.shards.len());
-            shard.contains(hash, key)
-        }
-    }
-
-    pub fn touch<Q>(&self, key: &Q) -> bool
-    where
-        Q: Hash + Equivalent<K> + ?Sized,
-    {
-        let hash = self.hash_builder.hash_one(key);
-
-        unsafe {
-            let mut shard = self.shard(hash as usize % self.shards.len());
-            shard.touch(hash, key)
-        }
-    }
-
-    #[fastrace::trace(name = "foyer::memory::generic::clear")]
-    pub fn clear(&self) {
-        let mut to_release = vec![];
-        for shard in self.shards.iter() {
-            let mut shard = shard.lock();
-            unsafe { shard.clear(&mut to_release) };
-        }
-
-        // Do not deallocate data within the lock section.
-        if let Some(listener) = self.context.event_listener.as_ref() {
-            for (k, v, _c, _w) in to_release {
-                listener.on_memory_release(k, v);
-            }
-        }
-    }
-
-    pub fn capacity(&self) -> usize {
-        self.capacity
-    }
-
-    pub fn usage(&self) -> usize {
-        self.usages.iter().map(|usage| usage.load(Ordering::Relaxed)).sum()
-    }
-
-    pub fn metrics(&self) -> &Metrics {
-        &self.context.metrics
-    }
-
-    pub fn hash_builder(&self) -> &S {
-        &self.hash_builder
-    }
-
-    pub fn shards(&self) -> usize {
-        self.shards.len()
-    }
-
-    unsafe fn try_release_external_handle(&self, ptr: NonNull<E::Handle>) {
-        let entry = {
-            let base = ptr.as_ref().base();
-            let mut shard = self.shard(base.hash() as usize % self.shards.len());
-            shard.try_release_external_handle(ptr)
-        };
-
-        // Do not deallocate data within the lock section.
-        if let Some(listener) = self.context.event_listener.as_ref() {
-            if let Some((k, v, _c, _w)) = entry {
-                listener.on_memory_release(k, v);
-            }
-        }
-    }
-
-    unsafe fn inc_refs(&self, mut ptr: NonNull<E::Handle>) {
-        let shard = self.shard(ptr.as_ref().base().hash() as usize % self.shards.len());
-        ptr.as_mut().base_mut().inc_refs();
-        drop(shard);
-    }
-
-    #[fastrace::trace(name = "foyer::memory::generic::shard")]
-    fn shard(&self, shard: usize) -> MutexGuard<'_, RawMutex, GenericCacheShard<K, V, E, I, S>> {
-        self.shards[shard].lock()
-    }
-}
-
-// TODO(MrCroxx): use `hashbrown::HashTable` with `Handle` may relax the `Clone` bound?
-impl<K, V, E, I, S> GenericCache<K, V, E, I, S>
-where
-    K: Key + Clone,
-    V: Value,
-    E: Eviction,
-    E::Handle: KeyedHandle<Key = K, Data = (K, V)>,
-    I: Indexer<Key = K, Handle = E::Handle>,
-    S: HashBuilder,
-{
-    pub fn fetch<F, FU, ER>(self: &Arc<Self>, key: K, fetch: F) -> GenericFetch<K, V, E, I, S, ER>
-    where
-        F: FnOnce() -> FU,
-        FU: Future<Output = std::result::Result<V, ER>> + Send + 'static,
-        ER: Send + 'static + Debug,
-    {
-        self.fetch_inner(
-            key,
-            CacheContext::default(),
-            fetch,
-            &tokio::runtime::Handle::current().into(),
-        )
-    }
-
-    pub fn fetch_with_context<F, FU, ER>(
-        self: &Arc<Self>,
-        key: K,
-        context: CacheContext,
-        fetch: F,
-    ) -> GenericFetch<K, V, E, I, S, ER>
-    where
-        F: FnOnce() -> FU,
-        FU: Future<Output = std::result::Result<V, ER>> + Send + 'static,
-        ER: Send + 'static + Debug,
-    {
-        self.fetch_inner(key, context, fetch, &tokio::runtime::Handle::current().into())
-    }
-
-    #[doc(hidden)]
-    pub fn fetch_inner<F, FU, ER, ID>(
-        self: &Arc<Self>,
-        key: K,
-        context: CacheContext,
-        fetch: F,
-        runtime: &SingletonHandle,
-    ) -> GenericFetch<K, V, E, I, S, ER>
-    where
-        F: FnOnce() -> FU,
-        FU: Future<Output = ID> + Send + 'static,
-        ER: Send + 'static + Debug,
-        ID: Into<Diversion<std::result::Result<V, ER>, FetchMark>>,
-    {
-        let hash = self.hash_builder.hash_one(&key);
-
-        {
-            let mut shard = self.shard(hash as usize % self.shards.len());
-
-            if let Some(ptr) = unsafe { shard.get(hash, &key) } {
-                return GenericFetch::new(GenericFetchInner::Hit(Some(GenericCacheEntry {
-                    cache: self.clone(),
-                    ptr,
-                })));
-            }
-            match shard.waiters.entry(key.clone()) {
-                HashMapEntry::Occupied(mut o) => {
-                    let (tx, rx) = oneshot::channel();
-                    o.get_mut().push(tx);
-                    shard.state.metrics.memory_queue.increment(1);
-                    return GenericFetch::new(GenericFetchInner::Wait(rx.in_span(Span::enter_with_local_parent(
-                        "foyer::memory::generic::fetch_with_runtime::wait",
-                    ))));
-                }
-                HashMapEntry::Vacant(v) => {
-                    v.insert(vec![]);
-                    shard.state.metrics.memory_fetch.increment(1);
-                }
-            }
-        }
-
-        let cache = self.clone();
-        let future = fetch();
-        let join = runtime.spawn(
-            async move {
-                let Diversion { target, store } = future
-                    .in_span(Span::enter_with_local_parent(
-                        "foyer::memory::generic::fetch_with_runtime::fn",
-                    ))
-                    .await
-                    .into();
-                let value = match target {
-                    Ok(value) => value,
-                    Err(e) => {
-                        let mut shard = cache.shard(hash as usize % cache.shards.len());
-                        tracing::debug!("[fetch]: error raise while fetching, all waiter are dropped, err: {e:?}");
-                        shard.waiters.remove(&key);
-                        return Diversion { target: Err(e), store };
-                    }
-                };
-                let entry = cache.insert_with_context(key, value, context);
-                Diversion {
-                    target: Ok(entry),
-                    store,
-                }
-            }
-            .in_span(Span::enter_with_local_parent(
-                "foyer::memory::generic::fetch_with_runtime::spawn",
-            )),
-        );
-        GenericFetch::new(GenericFetchInner::Miss(join))
-    }
-}
-
-impl<K, V, E, I, S> Drop for GenericCache<K, V, E, I, S>
-where
-    K: Key,
-    V: Value,
-    E: Eviction,
-    E::Handle: KeyedHandle<Key = K, Data = (K, V)>,
-    I: Indexer<Key = K, Handle = E::Handle>,
-    S: HashBuilder,
-{
-    fn drop(&mut self) {
-        self.clear();
-    }
-}
-
-pub struct GenericCacheEntry<K, V, E, I, S = RandomState>
-where
-    K: Key,
-    V: Value,
-    E: Eviction,
-    E::Handle: KeyedHandle<Key = K, Data = (K, V)>,
-    I: Indexer<Key = K, Handle = E::Handle>,
-    S: HashBuilder,
-{
-    cache: Arc<GenericCache<K, V, E, I, S>>,
-    ptr: NonNull<E::Handle>,
-}
-
-impl<K, V, E, I, S> Debug for GenericCacheEntry<K, V, E, I, S>
-where
-    K: Key,
-    V: Value,
-    E: Eviction,
-    E::Handle: KeyedHandle<Key = K, Data = (K, V)>,
-    I: Indexer<Key = K, Handle = E::Handle>,
-    S: HashBuilder,
-{
-    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-        f.debug_struct("GenericCacheEntry").finish()
-    }
-}
-
-impl<K, V, E, I, S> GenericCacheEntry<K, V, E, I, S>
-where
-    K: Key,
-    V: Value,
-    E: Eviction,
-    E::Handle: KeyedHandle<Key = K, Data = (K, V)>,
-    I: Indexer<Key = K, Handle = E::Handle>,
-    S: HashBuilder,
-{
-    pub fn hash(&self) -> u64 {
-        unsafe { self.ptr.as_ref().base().hash() }
-    }
-
-    pub fn key(&self) -> &K {
-        unsafe { &self.ptr.as_ref().base().data_unwrap_unchecked().0 }
-    }
-
-    pub fn value(&self) -> &V {
-        unsafe { &self.ptr.as_ref().base().data_unwrap_unchecked().1 }
-    }
-
-    pub fn context(&self) -> &<E::Handle as Handle>::Context {
-        unsafe { self.ptr.as_ref().base().context() }
-    }
-
-    pub fn weight(&self) -> usize {
-        unsafe { self.ptr.as_ref().base().weight() }
-    }
-
-    pub fn refs(&self) -> usize {
-        unsafe { self.ptr.as_ref().base().refs() }
-    }
-
-    pub fn is_outdated(&self) -> bool {
-        unsafe { !self.ptr.as_ref().base().is_in_indexer() }
-    }
-}
-
-impl<K, V, E, I, S> Clone for GenericCacheEntry<K, V, E, I, S>
-where
-    K: Key,
-    V: Value,
-    E: Eviction,
-    E::Handle: KeyedHandle<Key = K, Data = (K, V)>,
-    I: Indexer<Key = K, Handle = E::Handle>,
-    S: HashBuilder,
-{
-    fn clone(&self) -> Self {
-        unsafe { self.cache.inc_refs(self.ptr) };
-        Self {
-            cache: self.cache.clone(),
-            ptr: self.ptr,
-        }
-    }
-}
-
-impl<K, V, E, I, S> Drop for GenericCacheEntry<K, V, E, I, S>
-where
-    K: Key,
-    V: Value,
-    E: Eviction,
-    E::Handle: KeyedHandle<Key = K, Data = (K, V)>,
-    I: Indexer<Key = K, Handle = E::Handle>,
-    S: HashBuilder,
-{
-    fn drop(&mut self) {
-        unsafe { self.cache.try_release_external_handle(self.ptr) };
-    }
-}
-
-impl<K, V, E, I, S> Deref for GenericCacheEntry<K, V, E, I, S>
-where
-    K: Key,
-    V: Value,
-    E: Eviction,
-    E::Handle: KeyedHandle<Key = K, Data = (K, V)>,
-    I: Indexer<Key = K, Handle = E::Handle>,
-    S: HashBuilder,
-{
-    type Target = V;
-
-    fn deref(&self) -> &Self::Target {
-        self.value()
-    }
-}
-
-unsafe impl<K, V, E, I, S> Send for GenericCacheEntry<K, V, E, I, S>
-where
-    K: Key,
-    V: Value,
-    E: Eviction,
-    E::Handle: KeyedHandle<Key = K, Data = (K, V)>,
-    I: Indexer<Key = K, Handle = E::Handle>,
-    S: HashBuilder,
-{
-}
-unsafe impl<K, V, E, I, S> Sync for GenericCacheEntry<K, V, E, I, S>
-where
-    K: Key,
-    V: Value,
-    E: Eviction,
-    E::Handle: KeyedHandle<Key = K, Data = (K, V)>,
-    I: Indexer<Key = K, Handle = E::Handle>,
-    S: HashBuilder,
-{
-}
-
-#[cfg(test)]
-mod tests {
-    use std::time::Duration;
-
-    use futures::future::{join_all, try_join_all};
-    use rand::{rngs::SmallRng, Rng, RngCore, SeedableRng};
-
-    use super::*;
-    use crate::{
-        cache::{FifoCache, FifoCacheEntry, LfuCache, LruCache, LruCacheEntry, S3FifoCache},
-        eviction::{
-            fifo::{FifoConfig, FifoHandle},
-            lru::LruConfig,
-            test_utils::TestEviction,
-        },
-        indexer::{hash_table::HashTableIndexer, sanity::SanityIndexer},
-        LfuConfig, S3FifoConfig,
-    };
-
-    fn is_send_sync_static<T: Send + Sync + 'static>() {}
-
-    #[test]
-    fn test_send_sync_static() {
-        is_send_sync_static::<FifoCache<(), ()>>();
-        is_send_sync_static::<LruCache<(), ()>>();
-    }
-
-    #[expect(clippy::type_complexity)]
-    fn fuzzy<E>(cache: Arc<GenericCache<u64, u64, E, SanityIndexer<HashTableIndexer<u64, E::Handle>>>>)
-    where
-        E: Eviction,
-        E::Handle: KeyedHandle<Key = u64, Data = (u64, u64)>,
-    {
-        let handles = (0..8)
-            .map(|i| {
-                let c = cache.clone();
-                std::thread::spawn(move || {
-                    let mut rng = SmallRng::seed_from_u64(i);
-                    for _ in 0..100000 {
-                        let key = rng.next_u64();
-                        if let Some(entry) = c.get(&key) {
-                            assert_eq!(key, *entry);
-                            drop(entry);
-                            continue;
-                        }
-                        c.insert_with_context(
-                            key,
-                            key,
-                            if rng.gen_bool(0.5) {
-                                CacheContext::Default
-                            } else {
-                                CacheContext::LowPriority
-                            },
-                        );
-                    }
-                })
-            })
-            .collect_vec();
-
-        handles.into_iter().for_each(|handle| handle.join().unwrap());
-
-        assert_eq!(cache.usage(), cache.capacity());
-    }
-
-    #[test]
-    fn test_fifo_cache_fuzzy() {
-        fuzzy(Arc::new(FifoCache::<u64, u64>::new(GenericCacheConfig {
-            name: "test".to_string(),
-            capacity: 256,
-            shards: 4,
-            eviction_config: FifoConfig::default(),
-            object_pool_capacity: 16,
-            hash_builder: RandomState::default(),
-            weighter: Arc::new(|_, _| 1),
-            event_listener: None,
-        })))
-    }
-
-    #[test]
-    fn test_lru_cache_fuzzy() {
-        fuzzy(Arc::new(LruCache::<u64, u64>::new(GenericCacheConfig {
-            name: "test".to_string(),
-            capacity: 256,
-            shards: 4,
-            eviction_config: LruConfig::default(),
-            object_pool_capacity: 16,
-            hash_builder: RandomState::default(),
-            weighter: Arc::new(|_, _| 1),
-            event_listener: None,
-        })))
-    }
-
-    #[test]
-    fn test_lfu_cache_fuzzy() {
-        fuzzy(Arc::new(LfuCache::<u64, u64>::new(GenericCacheConfig {
-            name: "test".to_string(),
-            capacity: 256,
-            shards: 4,
-            eviction_config: LfuConfig::default(),
-            object_pool_capacity: 16,
-            hash_builder: RandomState::default(),
-            weighter: Arc::new(|_, _| 1),
-            event_listener: None,
-        })))
-    }
-
-    #[test]
-    fn test_s3fifo_cache_fuzzy() {
-        fuzzy(Arc::new(S3FifoCache::<u64, u64>::new(GenericCacheConfig {
-            name: "test".to_string(),
-            capacity: 256,
-            shards: 4,
-            eviction_config: S3FifoConfig::default(),
-            object_pool_capacity: 16,
-            hash_builder: RandomState::default(),
-            weighter: Arc::new(|_, _| 1),
-            event_listener: None,
-        })))
-    }
-
-    fn fifo(capacity: usize) -> Arc<FifoCache<u64, String>> {
-        let config = GenericCacheConfig {
-            name: "test".to_string(),
-            capacity,
-            shards: 1,
-            eviction_config: FifoConfig {},
-            object_pool_capacity: 1,
-            hash_builder: RandomState::default(),
-            weighter: Arc::new(|_, v: &String| v.len()),
-            event_listener: None,
-        };
-        Arc::new(FifoCache::<u64, String>::new(config))
-    }
-
-    fn lru(capacity: usize) -> Arc<LruCache<u64, String>> {
-        let config = GenericCacheConfig {
-            name: "test".to_string(),
-            capacity,
-            shards: 1,
-            eviction_config: LruConfig {
-                high_priority_pool_ratio: 0.0,
-            },
-            object_pool_capacity: 1,
-            hash_builder: RandomState::default(),
-            weighter: Arc::new(|_, v: &String| v.len()),
-            event_listener: None,
-        };
-        Arc::new(LruCache::<u64, String>::new(config))
-    }
-
-    fn insert_fifo(cache: &Arc<FifoCache<u64, String>>, key: u64, value: &str) -> FifoCacheEntry<u64, String> {
-        cache.insert(key, value.to_string())
-    }
-
-    fn insert_lru(cache: &Arc<LruCache<u64, String>>, key: u64, value: &str) -> LruCacheEntry<u64, String> {
-        cache.insert(key, value.to_string())
-    }
-
-    #[test]
-    fn test_reference_count() {
-        let cache = fifo(100);
-
-        let refs = |ptr: NonNull<FifoHandle<(u64, String)>>| unsafe { ptr.as_ref().base().refs() };
-
-        let e1 = insert_fifo(&cache, 42, "the answer to life, the universe, and everything");
-        let ptr = e1.ptr;
-        assert_eq!(refs(ptr), 1);
-
-        let e2 = cache.get(&42).unwrap();
-        assert_eq!(refs(ptr), 2);
-
-        let e3 = e2.clone();
-        assert_eq!(refs(ptr), 3);
-
-        drop(e2);
-        assert_eq!(refs(ptr), 2);
-
-        drop(e3);
-        assert_eq!(refs(ptr), 1);
-
-        drop(e1);
-        assert_eq!(refs(ptr), 0);
-    }
-
-    #[test]
-    fn test_deposit() {
-        let cache = lru(10);
-        let e = cache.deposit(42, "answer".to_string());
-        assert_eq!(cache.usage(), 6);
-        drop(e);
-        assert_eq!(cache.usage(), 0);
-
-        let e = cache.deposit(42, "answer".to_string());
-        assert_eq!(cache.usage(), 6);
-        assert_eq!(cache.get(&42).unwrap().value(), "answer");
-        drop(e);
-        assert_eq!(cache.usage(), 6);
-        assert_eq!(cache.get(&42).unwrap().value(), "answer");
-    }
-
-    #[test]
-    fn test_deposit_replace() {
-        let cache = lru(100);
-        let e1 = cache.deposit(42, "wrong answer".to_string());
-        let e2 = cache.insert(42, "answer".to_string());
-        drop(e1);
-        drop(e2);
-        assert_eq!(cache.get(&42).unwrap().value(), "answer");
-        assert_eq!(cache.usage(), 6);
-    }
-
-    #[test]
-    fn test_replace() {
-        let cache = fifo(10);
-
-        insert_fifo(&cache, 114, "xx");
-        assert_eq!(cache.usage(), 2);
-
-        insert_fifo(&cache, 514, "QwQ");
-        assert_eq!(cache.usage(), 5);
-
-        insert_fifo(&cache, 114, "(0.0)");
-        assert_eq!(cache.usage(), 8);
-
-        assert_eq!(
-            cache.shards[0].lock().eviction.dump(),
-            vec![(514, "QwQ".to_string()), (114, "(0.0)".to_string())],
-        );
-    }
-
-    #[test]
-    fn test_replace_with_external_refs() {
-        let cache = fifo(10);
-
-        insert_fifo(&cache, 514, "QwQ");
-        insert_fifo(&cache, 114, "(0.0)");
-
-        let e4 = cache.get(&514).unwrap();
-        let e5 = insert_fifo(&cache, 514, "bili");
-
-        assert_eq!(e4.refs(), 1);
-        assert_eq!(e5.refs(), 1);
-
-        // remains: 514 => QwQ (3), 514 => bili (4)
-        // evicted: 114 => (0.0) (5)
-        assert_eq!(cache.usage(), 7);
-
-        assert!(cache.get(&114).is_none());
-        assert_eq!(cache.get(&514).unwrap().value(), "bili");
-        assert_eq!(e4.value(), "QwQ");
-
-        let e6 = cache.remove(&514).unwrap();
-        assert_eq!(e6.value(), "bili");
-        drop(e6);
-
-        drop(e5);
-        assert!(cache.get(&514).is_none());
-        assert_eq!(e4.value(), "QwQ");
-
-        assert_eq!(cache.usage(), 3);
-        drop(e4);
-        assert_eq!(cache.usage(), 0);
-    }
-
-    #[test]
-    fn test_reinsert_while_all_referenced_lru() {
-        let cache = lru(10);
-
-        let e1 = insert_lru(&cache, 1, "111");
-        let e2 = insert_lru(&cache, 2, "222");
-        let e3 = insert_lru(&cache, 3, "333");
-        assert_eq!(cache.usage(), 9);
-
-        // No entry will be released because all of them are referenced externally.
-        let e4 = insert_lru(&cache, 4, "444");
-        assert_eq!(cache.usage(), 12);
-
-        // `111`, `222` and `333` are evicted from the eviction container to make space for `444`.
-        assert_eq!(cache.shards[0].lock().eviction.dump(), vec![(4, "444".to_string()),]);
-
-        // `e1` cannot be reinserted for the usage has already exceeds the capacity.
-        drop(e1);
-        assert_eq!(cache.usage(), 9);
-
-        // `222` and `333` will be reinserted
-        drop(e2);
-        drop(e3);
-        assert_eq!(
-            cache.shards[0].lock().eviction.dump(),
-            vec![(4, "444".to_string()), (2, "222".to_string()), (3, "333".to_string()),]
-        );
-        assert_eq!(cache.usage(), 9);
-
-        // `444` will be reinserted
-        drop(e4);
-        assert_eq!(
-            cache.shards[0].lock().eviction.dump(),
-            vec![(2, "222".to_string()), (3, "333".to_string()), (4, "444".to_string()),]
-        );
-        assert_eq!(cache.usage(), 9);
-    }
-
-    #[test]
-    fn test_reinsert_while_all_referenced_fifo() {
-        let cache = fifo(10);
-
-        let e1 = insert_fifo(&cache, 1, "111");
-        let e2 = insert_fifo(&cache, 2, "222");
-        let e3 = insert_fifo(&cache, 3, "333");
-        assert_eq!(cache.usage(), 9);
-
-        // No entry will be released because all of them are referenced externally.
-        let e4 = insert_fifo(&cache, 4, "444");
-        assert_eq!(cache.usage(), 12);
-
-        // `111`, `222` and `333` are evicted from the eviction container to make space for `444`.
-        assert_eq!(cache.shards[0].lock().eviction.dump(), vec![(4, "444".to_string()),]);
-
-        // `e1` cannot be reinserted for the usage has already exceeds the capacity.
-        drop(e1);
-        assert_eq!(cache.usage(), 9);
-
-        // `222` and `333` will be not reinserted because fifo will ignore reinsert operations.
-        drop([e2, e3, e4]);
-        assert_eq!(cache.shards[0].lock().eviction.dump(), vec![(4, "444".to_string()),]);
-        assert_eq!(cache.usage(), 3);
-
-        // Note:
-        //
-        // For cache policy like FIFO, the entries will not be reinserted while all handles are referenced.
-        // It's okay for this is not a common situation and is not supposed to happen in real workload.
-    }
-
-    #[test_log::test(tokio::test)]
-    async fn test_fetch() {
-        let cache = fifo(10);
-
-        let fetch = |s: &'static str| async move {
-            tokio::time::sleep(Duration::from_millis(100)).await;
-            Ok::<_, anyhow::Error>(s.to_string())
-        };
-
-        /* fetch with waiters */
-
-        let e1s = try_join_all([
-            cache.fetch(1, || fetch("111")),
-            cache.fetch(1, || fetch("111")),
-            cache.fetch(1, || fetch("111")),
-        ])
-        .await
-        .unwrap();
-
-        assert_eq!(e1s[0].value(), "111");
-        assert_eq!(e1s[1].value(), "111");
-        assert_eq!(e1s[2].value(), "111");
-
-        let e1 = cache.fetch(1, || fetch("111")).await.unwrap();
-
-        assert_eq!(e1.value(), "111");
-        assert_eq!(e1.refs(), 4);
-
-        /* insert before fetch finish */
-
-        let c = cache.clone();
-        let h2 = tokio::spawn(async move {
-            tokio::time::sleep(Duration::from_millis(10)).await;
-            c.insert(2, "222222".to_string())
-        });
-        let e2s = try_join_all([
-            cache.fetch(2, || fetch("222")),
-            cache.fetch(2, || fetch("222")),
-            cache.fetch(2, || fetch("222")),
-        ])
-        .await
-        .unwrap();
-        let e2 = h2.await.unwrap();
-
-        assert_eq!(e2s[0].value(), "222");
-        assert_eq!(e2s[1].value(), "222222");
-        assert_eq!(e2s[2].value(), "222222");
-        assert_eq!(e2.value(), "222222");
-
-        assert_eq!(e2s[0].refs(), 1);
-        assert_eq!(e2s[1].refs(), 3);
-        assert_eq!(e2s[2].refs(), 3);
-        assert_eq!(e2.refs(), 3);
-
-        /* fetch cancel */
-
-        let c = cache.clone();
-        let h3a = tokio::spawn(async move { c.fetch(3, || fetch("333")).await.unwrap() });
-        let c = cache.clone();
-        let h3b = tokio::spawn(async move {
-            tokio::time::sleep(Duration::from_millis(10)).await;
-            tokio::time::timeout(Duration::from_millis(10), c.fetch(3, || fetch("333"))).await
-        });
-
-        let _ = h3b.await.unwrap();
-        let e3 = h3a.await.unwrap();
-        assert_eq!(e3.value(), "333");
-        assert_eq!(e3.refs(), 1);
-
-        /* fetch error */
-
-        let r4s = join_all([
-            cache.fetch(4, || async move {
-                tokio::time::sleep(Duration::from_millis(100)).await;
-                Err(anyhow::anyhow!("fetch error"))
-            }),
-            cache.fetch(4, || fetch("444")),
-        ])
-        .await;
-
-        assert!(r4s[0].is_err());
-        assert!(r4s[1].is_err());
-
-        let e4 = cache.fetch(4, || fetch("444")).await.unwrap();
-        assert_eq!(e4.value(), "444");
-        assert_eq!(e4.refs(), 1);
-    }
-}
diff --git a/foyer-memory/src/handle.rs b/foyer-memory/src/handle.rs
deleted file mode 100644
index 9476cda2..00000000
--- a/foyer-memory/src/handle.rs
+++ /dev/null
@@ -1,263 +0,0 @@
-//  Copyright 2024 foyer Project Authors
-//
-//  Licensed under the Apache License, Version 2.0 (the "License");
-//  you may not use this file except in compliance with the License.
-//  You may obtain a copy of the License at
-//
-//  http://www.apache.org/licenses/LICENSE-2.0
-//
-//  Unless required by applicable law or agreed to in writing, software
-//  distributed under the License is distributed on an "AS IS" BASIS,
-//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-//  See the License for the specific language governing permissions and
-//  limitations under the License.
-
-use bitflags::bitflags;
-use foyer_common::{
-    assert::OptionExt,
-    code::{Key, Value},
-    strict_assert,
-};
-
-use crate::context::Context;
-
-bitflags! {
-    #[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)]
-    struct BaseHandleFlags: u8 {
-        const IN_INDEXER = 0b00000001;
-        const IN_EVICTION = 0b00000010;
-        const IS_DEPOSIT= 0b00000100;
-    }
-}
-
-pub trait Handle: Send + Sync + 'static + Default {
-    type Data;
-    type Context: Context;
-
-    fn base(&self) -> &BaseHandle<Self::Data, Self::Context>;
-    fn base_mut(&mut self) -> &mut BaseHandle<Self::Data, Self::Context>;
-}
-
-pub trait HandleExt: Handle {
-    fn init(&mut self, hash: u64, data: Self::Data, weight: usize, context: Self::Context) {
-        self.base_mut().init(hash, data, weight, context);
-    }
-}
-impl<H: Handle> HandleExt for H {}
-
-pub trait KeyedHandle: Handle {
-    type Key;
-
-    fn key(&self) -> &Self::Key;
-}
-
-impl<K, V, T> KeyedHandle for T
-where
-    K: Key,
-    V: Value,
-    T: Handle<Data = (K, V)>,
-{
-    type Key = K;
-
-    fn key(&self) -> &Self::Key {
-        &self.base().data_unwrap_unchecked().0
-    }
-}
-
-#[derive(Debug)]
-pub struct BaseHandle<T, C> {
-    /// key, value, context
-    entry: Option<(T, C)>,
-    /// key hash
-    hash: u64,
-    /// entry weight
-    weight: usize,
-    /// external reference count
-    refs: usize,
-    /// flags that used by the general cache abstraction
-    flags: BaseHandleFlags,
-}
-
-impl<T, C> Default for BaseHandle<T, C> {
-    fn default() -> Self {
-        Self::new()
-    }
-}
-
-impl<T, C> BaseHandle<T, C> {
-    /// Create a uninitialized handle.
-    #[inline(always)]
-    pub fn new() -> Self {
-        Self {
-            entry: None,
-            hash: 0,
-            weight: 0,
-            refs: 0,
-            flags: BaseHandleFlags::empty(),
-        }
-    }
-
-    /// Init handle with args.
-    #[inline(always)]
-    pub fn init(&mut self, hash: u64, data: T, weight: usize, context: C) {
-        strict_assert!(self.entry.is_none());
-        assert_ne!(weight, 0);
-        self.hash = hash;
-        self.entry = Some((data, context));
-        self.weight = weight;
-        self.refs = 0;
-        self.flags = BaseHandleFlags::empty();
-    }
-
-    /// Take key and value from the handle and reset it to the uninitialized state.
-    #[inline(always)]
-    pub fn take(&mut self) -> (T, C, usize) {
-        strict_assert!(self.entry.is_some());
-        unsafe {
-            self.entry
-                .take()
-                .map(|(data, context)| (data, context, self.weight))
-                .strict_unwrap_unchecked()
-        }
-    }
-
-    /// Return `true` if the handle is initialized.
-    #[inline(always)]
-    pub fn is_initialized(&self) -> bool {
-        self.entry.is_some()
-    }
-
-    /// Get key hash.
-    ///
-    /// # Panics
-    ///
-    /// Panics if the handle is uninitialized.
-    #[inline(always)]
-    pub fn hash(&self) -> u64 {
-        self.hash
-    }
-
-    /// Get data reference.
-    ///
-    /// # Panics
-    ///
-    /// Panics if the handle is uninitialized.
-    #[inline(always)]
-    pub fn data_unwrap_unchecked(&self) -> &T {
-        strict_assert!(self.entry.is_some());
-        unsafe { self.entry.as_ref().map(|entry| &entry.0).strict_unwrap_unchecked() }
-    }
-
-    /// Get context reference.
-    ///  
-    /// # Panics
-    ///
-    /// Panics if the handle is uninitialized.
-    #[inline(always)]
-    pub fn context(&self) -> &C {
-        strict_assert!(self.entry.is_some());
-        unsafe { self.entry.as_ref().map(|entry| &entry.1).strict_unwrap_unchecked() }
-    }
-
-    /// Get the weight of the handle.
-    #[inline(always)]
-    pub fn weight(&self) -> usize {
-        self.weight
-    }
-
-    /// Increase the external reference count of the handle, returns the new reference count.
-    #[inline(always)]
-    pub fn inc_refs(&mut self) -> usize {
-        self.inc_refs_by(1)
-    }
-
-    /// Increase the external reference count of the handle, returns the new reference count.
-    #[inline(always)]
-    pub fn inc_refs_by(&mut self, val: usize) -> usize {
-        self.refs += val;
-        self.refs
-    }
-
-    /// Decrease the external reference count of the handle, returns the new reference count.
-    #[inline(always)]
-    pub fn dec_refs(&mut self) -> usize {
-        self.refs -= 1;
-        self.refs
-    }
-
-    /// Get the external reference count of the handle.
-    #[inline(always)]
-    pub fn refs(&self) -> usize {
-        self.refs
-    }
-
-    /// Return `true` if there are external references.
-    #[inline(always)]
-    pub fn has_refs(&self) -> bool {
-        self.refs() > 0
-    }
-
-    #[inline(always)]
-    pub fn set_in_indexer(&mut self, in_cache: bool) {
-        if in_cache {
-            self.flags |= BaseHandleFlags::IN_INDEXER;
-        } else {
-            self.flags -= BaseHandleFlags::IN_INDEXER;
-        }
-    }
-
-    #[inline(always)]
-    pub fn is_in_indexer(&self) -> bool {
-        self.flags.contains(BaseHandleFlags::IN_INDEXER)
-    }
-
-    #[inline(always)]
-    pub fn set_in_eviction(&mut self, in_eviction: bool) {
-        if in_eviction {
-            self.flags |= BaseHandleFlags::IN_EVICTION;
-        } else {
-            self.flags -= BaseHandleFlags::IN_EVICTION;
-        }
-    }
-
-    #[inline(always)]
-    pub fn is_in_eviction(&self) -> bool {
-        self.flags.contains(BaseHandleFlags::IN_EVICTION)
-    }
-
-    #[inline(always)]
-    pub fn set_deposit(&mut self, deposit: bool) {
-        if deposit {
-            self.flags |= BaseHandleFlags::IS_DEPOSIT;
-        } else {
-            self.flags -= BaseHandleFlags::IS_DEPOSIT;
-        }
-    }
-
-    #[inline(always)]
-    pub fn is_deposit(&self) -> bool {
-        self.flags.contains(BaseHandleFlags::IS_DEPOSIT)
-    }
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-
-    #[test]
-    fn test_base_handle_basic() {
-        let mut h = BaseHandle::<(), ()>::new();
-        assert!(!h.is_in_indexer());
-        assert!(!h.is_in_eviction());
-
-        h.set_in_indexer(true);
-        h.set_in_eviction(true);
-        assert!(h.is_in_indexer());
-        assert!(h.is_in_eviction());
-
-        h.set_in_indexer(false);
-        h.set_in_eviction(false);
-        assert!(!h.is_in_indexer());
-        assert!(!h.is_in_eviction());
-    }
-}
diff --git a/foyer-memory/src/indexer/hash_table.rs b/foyer-memory/src/indexer/hash_table.rs
index 47ed6dd6..f9611750 100644
--- a/foyer-memory/src/indexer/hash_table.rs
+++ b/foyer-memory/src/indexer/hash_table.rs
@@ -12,106 +12,77 @@
 //  See the License for the specific language governing permissions and
 //  limitations under the License.
 
-use std::{hash::Hash, ptr::NonNull};
+use std::sync::Arc;
 
-use equivalent::Equivalent;
-use foyer_common::{code::Key, strict_assert};
 use hashbrown::hash_table::{Entry as HashTableEntry, HashTable};
 
 use super::Indexer;
-use crate::handle::KeyedHandle;
+use crate::{eviction::Eviction, record::Record};
 
-pub struct HashTableIndexer<K, H>
+pub struct HashTableIndexer<E>
 where
-    K: Key,
-    H: KeyedHandle<Key = K>,
+    E: Eviction,
 {
-    table: HashTable<NonNull<H>>,
+    table: HashTable<Arc<Record<E>>>,
 }
 
-unsafe impl<K, H> Send for HashTableIndexer<K, H>
-where
-    K: Key,
-    H: KeyedHandle<Key = K>,
-{
-}
+unsafe impl<E> Send for HashTableIndexer<E> where E: Eviction {}
+unsafe impl<E> Sync for HashTableIndexer<E> where E: Eviction {}
 
-unsafe impl<K, H> Sync for HashTableIndexer<K, H>
+impl<E> Default for HashTableIndexer<E>
 where
-    K: Key,
-    H: KeyedHandle<Key = K>,
+    E: Eviction,
 {
-}
-
-impl<K, H> Indexer for HashTableIndexer<K, H>
-where
-    K: Key,
-    H: KeyedHandle<Key = K>,
-{
-    type Key = K;
-    type Handle = H;
-
-    fn new() -> Self {
+    fn default() -> Self {
         Self {
-            table: HashTable::new(),
+            table: Default::default(),
         }
     }
+}
 
-    unsafe fn insert(&mut self, mut ptr: NonNull<Self::Handle>) -> Option<NonNull<Self::Handle>> {
-        let handle = ptr.as_mut();
-
-        strict_assert!(!handle.base().is_in_indexer());
-        handle.base_mut().set_in_indexer(true);
+impl<E> Indexer for HashTableIndexer<E>
+where
+    E: Eviction,
+{
+    type Eviction = E;
 
-        match self.table.entry(
-            handle.base().hash(),
-            |p| p.as_ref().key() == handle.key(),
-            |p| p.as_ref().base().hash(),
-        ) {
+    fn insert(&mut self, mut record: Arc<Record<Self::Eviction>>) -> Option<Arc<Record<Self::Eviction>>> {
+        match self
+            .table
+            .entry(record.hash(), |r| r.key() == record.key(), |r| r.hash())
+        {
             HashTableEntry::Occupied(mut o) => {
-                std::mem::swap(o.get_mut(), &mut ptr);
-                let b = ptr.as_mut().base_mut();
-                strict_assert!(b.is_in_indexer());
-                b.set_in_indexer(false);
-                Some(ptr)
+                std::mem::swap(o.get_mut(), &mut record);
+                Some(record)
             }
             HashTableEntry::Vacant(v) => {
-                v.insert(ptr);
+                v.insert(record);
                 None
             }
         }
     }
 
-    unsafe fn get<Q>(&self, hash: u64, key: &Q) -> Option<NonNull<Self::Handle>>
+    fn get<Q>(&self, hash: u64, key: &Q) -> Option<&Arc<Record<Self::Eviction>>>
     where
-        Q: Hash + Equivalent<Self::Key> + ?Sized,
+        Q: std::hash::Hash + equivalent::Equivalent<<Self::Eviction as Eviction>::Key> + ?Sized,
     {
-        self.table.find(hash, |p| key.equivalent(p.as_ref().key())).copied()
+        self.table.find(hash, |r| key.equivalent(r.key()))
     }
 
-    unsafe fn remove<Q>(&mut self, hash: u64, key: &Q) -> Option<NonNull<Self::Handle>>
+    fn remove<Q>(&mut self, hash: u64, key: &Q) -> Option<Arc<Record<Self::Eviction>>>
     where
-        Q: Hash + Equivalent<Self::Key> + ?Sized,
+        Q: std::hash::Hash + equivalent::Equivalent<<Self::Eviction as Eviction>::Key> + ?Sized,
     {
-        match self
-            .table
-            .entry(hash, |p| key.equivalent(p.as_ref().key()), |p| p.as_ref().base().hash())
-        {
+        match self.table.entry(hash, |r| key.equivalent(r.key()), |r| r.hash()) {
             HashTableEntry::Occupied(o) => {
-                let (mut p, _) = o.remove();
-                let b = p.as_mut().base_mut();
-                strict_assert!(b.is_in_indexer());
-                b.set_in_indexer(false);
-                Some(p)
+                let (r, _) = o.remove();
+                Some(r)
             }
             HashTableEntry::Vacant(_) => None,
         }
     }
 
-    unsafe fn drain(&mut self) -> impl Iterator<Item = NonNull<Self::Handle>> {
-        self.table.drain().map(|mut ptr| {
-            ptr.as_mut().base_mut().set_in_indexer(false);
-            ptr
-        })
+    fn drain(&mut self) -> impl Iterator<Item = Arc<Record<Self::Eviction>>> {
+        self.table.drain()
     }
 }
diff --git a/foyer-memory/src/indexer/mod.rs b/foyer-memory/src/indexer/mod.rs
index bacfd2c2..7ff78e4a 100644
--- a/foyer-memory/src/indexer/mod.rs
+++ b/foyer-memory/src/indexer/mod.rs
@@ -12,27 +12,24 @@
 //  See the License for the specific language governing permissions and
 //  limitations under the License.
 
-use std::{hash::Hash, ptr::NonNull};
+use std::{hash::Hash, sync::Arc};
 
 use equivalent::Equivalent;
-use foyer_common::code::Key;
 
-use crate::handle::KeyedHandle;
+use crate::{eviction::Eviction, record::Record};
 
-pub trait Indexer: Send + Sync + 'static {
-    type Key: Key;
-    type Handle: KeyedHandle<Key = Self::Key>;
+pub trait Indexer: Send + Sync + 'static + Default {
+    type Eviction: Eviction;
 
-    fn new() -> Self;
-    unsafe fn insert(&mut self, ptr: NonNull<Self::Handle>) -> Option<NonNull<Self::Handle>>;
-    unsafe fn get<Q>(&self, hash: u64, key: &Q) -> Option<NonNull<Self::Handle>>
+    fn insert(&mut self, record: Arc<Record<Self::Eviction>>) -> Option<Arc<Record<Self::Eviction>>>;
+    fn get<Q>(&self, hash: u64, key: &Q) -> Option<&Arc<Record<Self::Eviction>>>
     where
-        Q: Hash + Equivalent<Self::Key> + ?Sized;
-    unsafe fn remove<Q>(&mut self, hash: u64, key: &Q) -> Option<NonNull<Self::Handle>>
+        Q: Hash + Equivalent<<Self::Eviction as Eviction>::Key> + ?Sized;
+    fn remove<Q>(&mut self, hash: u64, key: &Q) -> Option<Arc<Record<Self::Eviction>>>
     where
-        Q: Hash + Equivalent<Self::Key> + ?Sized;
-    unsafe fn drain(&mut self) -> impl Iterator<Item = NonNull<Self::Handle>>;
+        Q: Hash + Equivalent<<Self::Eviction as Eviction>::Key> + ?Sized;
+    fn drain(&mut self) -> impl Iterator<Item = Arc<Record<Self::Eviction>>>;
 }
 
 pub mod hash_table;
-pub mod sanity;
+pub mod sentry;
diff --git a/foyer-memory/src/indexer/sanity.rs b/foyer-memory/src/indexer/sanity.rs
deleted file mode 100644
index a0962758..00000000
--- a/foyer-memory/src/indexer/sanity.rs
+++ /dev/null
@@ -1,108 +0,0 @@
-//  Copyright 2024 foyer Project Authors
-//
-//  Licensed under the Apache License, Version 2.0 (the "License");
-//  you may not use this file except in compliance with the License.
-//  You may obtain a copy of the License at
-//
-//  http://www.apache.org/licenses/LICENSE-2.0
-//
-//  Unless required by applicable law or agreed to in writing, software
-//  distributed under the License is distributed on an "AS IS" BASIS,
-//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-//  See the License for the specific language governing permissions and
-//  limitations under the License.
-
-use std::ptr::NonNull;
-
-use equivalent::Equivalent;
-
-use super::Indexer;
-#[cfg(feature = "sanity")]
-use crate::handle::Handle;
-
-pub struct SanityIndexer<I>
-where
-    I: Indexer,
-{
-    indexer: I,
-}
-
-#[cfg(feature = "sanity")]
-impl<I> Indexer for SanityIndexer<I>
-where
-    I: Indexer,
-{
-    type Key = I::Key;
-    type Handle = I::Handle;
-
-    fn new() -> Self {
-        Self { indexer: I::new() }
-    }
-
-    unsafe fn insert(&mut self, ptr: NonNull<Self::Handle>) -> Option<NonNull<Self::Handle>> {
-        assert!(!ptr.as_ref().base().is_in_indexer());
-        let res = self
-            .indexer
-            .insert(ptr)
-            .inspect(|old| assert!(!old.as_ref().base().is_in_indexer()));
-        assert!(ptr.as_ref().base().is_in_indexer());
-        res
-    }
-
-    unsafe fn get<Q>(&self, hash: u64, key: &Q) -> Option<NonNull<Self::Handle>>
-    where
-        Q: std::hash::Hash + Equivalent<Self::Key> + ?Sized,
-    {
-        self.indexer
-            .get(hash, key)
-            .inspect(|ptr| assert!(ptr.as_ref().base().is_in_indexer()))
-    }
-
-    unsafe fn remove<Q>(&mut self, hash: u64, key: &Q) -> Option<NonNull<Self::Handle>>
-    where
-        Q: std::hash::Hash + Equivalent<Self::Key> + ?Sized,
-    {
-        self.indexer
-            .remove(hash, key)
-            .inspect(|ptr| assert!(!ptr.as_ref().base().is_in_indexer()))
-    }
-
-    unsafe fn drain(&mut self) -> impl Iterator<Item = NonNull<Self::Handle>> {
-        self.indexer.drain()
-    }
-}
-
-#[cfg(not(feature = "sanity"))]
-impl<I> Indexer for SanityIndexer<I>
-where
-    I: Indexer,
-{
-    type Key = I::Key;
-    type Handle = I::Handle;
-
-    fn new() -> Self {
-        Self { indexer: I::new() }
-    }
-
-    unsafe fn insert(&mut self, handle: NonNull<Self::Handle>) -> Option<NonNull<Self::Handle>> {
-        self.indexer.insert(handle)
-    }
-
-    unsafe fn get<Q>(&self, hash: u64, key: &Q) -> Option<NonNull<Self::Handle>>
-    where
-        Q: std::hash::Hash + Equivalent<Self::Key> + ?Sized,
-    {
-        self.indexer.get(hash, key)
-    }
-
-    unsafe fn remove<Q>(&mut self, hash: u64, key: &Q) -> Option<NonNull<Self::Handle>>
-    where
-        Q: std::hash::Hash + Equivalent<Self::Key> + ?Sized,
-    {
-        self.indexer.remove(hash, key)
-    }
-
-    unsafe fn drain(&mut self) -> impl Iterator<Item = NonNull<Self::Handle>> {
-        self.indexer.drain()
-    }
-}
diff --git a/foyer-memory/src/indexer/sentry.rs b/foyer-memory/src/indexer/sentry.rs
new file mode 100644
index 00000000..5c91afd2
--- /dev/null
+++ b/foyer-memory/src/indexer/sentry.rs
@@ -0,0 +1,80 @@
+//  Copyright 2024 foyer Project Authors
+//
+//  Licensed under the Apache License, Version 2.0 (the "License");
+//  you may not use this file except in compliance with the License.
+//  You may obtain a copy of the License at
+//
+//  http://www.apache.org/licenses/LICENSE-2.0
+//
+//  Unless required by applicable law or agreed to in writing, software
+//  distributed under the License is distributed on an "AS IS" BASIS,
+//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//  See the License for the specific language governing permissions and
+//  limitations under the License.
+
+use std::{hash::Hash, sync::Arc};
+
+use equivalent::Equivalent;
+use foyer_common::strict_assert;
+
+use super::Indexer;
+use crate::{eviction::Eviction, record::Record};
+
+/// [`Sentry`] is a guard for all [`Indexer`] implementations to set `IN_INDEXER` flag properly.
+pub struct Sentry<I>
+where
+    I: Indexer,
+{
+    indexer: I,
+}
+
+impl<I> Default for Sentry<I>
+where
+    I: Indexer,
+{
+    fn default() -> Self {
+        Self { indexer: I::default() }
+    }
+}
+
+impl<I> Indexer for Sentry<I>
+where
+    I: Indexer,
+{
+    type Eviction = I::Eviction;
+
+    fn insert(&mut self, record: Arc<Record<Self::Eviction>>) -> Option<Arc<Record<Self::Eviction>>> {
+        strict_assert!(!record.is_in_indexer());
+        record.set_in_indexer(true);
+        self.indexer.insert(record).inspect(|old| {
+            strict_assert!(old.is_in_indexer());
+            old.set_in_indexer(false);
+        })
+    }
+
+    fn get<Q>(&self, hash: u64, key: &Q) -> Option<&Arc<Record<Self::Eviction>>>
+    where
+        Q: Hash + Equivalent<<Self::Eviction as Eviction>::Key> + ?Sized,
+    {
+        self.indexer.get(hash, key).inspect(|r| {
+            strict_assert!(r.is_in_indexer());
+        })
+    }
+
+    fn remove<Q>(&mut self, hash: u64, key: &Q) -> Option<Arc<Record<Self::Eviction>>>
+    where
+        Q: Hash + Equivalent<<Self::Eviction as Eviction>::Key> + ?Sized,
+    {
+        self.indexer.remove(hash, key).inspect(|r| {
+            strict_assert!(r.is_in_indexer());
+            r.set_in_indexer(false)
+        })
+    }
+
+    fn drain(&mut self) -> impl Iterator<Item = Arc<Record<Self::Eviction>>> {
+        self.indexer.drain().inspect(|r| {
+            strict_assert!(r.is_in_indexer());
+            r.set_in_indexer(false)
+        })
+    }
+}
diff --git a/foyer-memory/src/lib.rs b/foyer-memory/src/lib.rs
index b92cf831..e38e82fd 100644
--- a/foyer-memory/src/lib.rs
+++ b/foyer-memory/src/lib.rs
@@ -24,50 +24,22 @@
 //!
 //! To achieve them, the crate needs to combine the advantages of the implementations of RocksDB and CacheLib.
 //!
-//! # Design
+//! # Components
 //!
 //! The cache is mainly composed of the following components:
-//! 1. handle             : Carries the cached entry, reference count, pointer links in the eviction container, etc.
-//! 2. indexer            : Indexes cached keys to the handles.
+//! 1. record             : Carries the cached entry, reference count, pointer links in the eviction container, etc.
+//! 2. indexer            : Indexes cached keys to the records.
 //! 3. eviction container : Defines the order of eviction. Usually implemented with intrusive data structures.
 //!
-//! Because a handle needs to be referenced and mutated by both the indexer and the eviction container in the same
-//! thread, it is hard to implement in 100% safe Rust without overhead. So, the APIs of the indexer and the eviction
-//! container are defined with `NonNull` pointers of the handles.
-//!
-//! When some entry is inserted into the cache, the associated handle should be transmuted into pointer without
-//! dropping. When some entry is removed from the cache, the pointer of the associated handle should be transmuted into
-//! an owned data structure.
-//!
-//! # Handle Lifetime
-//!
-//! The handle is created during a new entry is being inserted, and then inserted into both the indexer and the eviction
-//! container.
-//!
-//! The handle is return if the entry is retrieved from the cache. The handle will track the count of the external
-//! owners to decide the time to reclaim.
-//!
-//! When a key is removed or updated, the original handle will be removed from the indexer and the eviction container,
-//! and waits to be released by all the external owners before reclamation.
-//!
-//! When the cache is full and being inserted, a handle will be evicted from the eviction container based on the
-//! eviction algorithm. The evicted handle will NOT be removed from the indexer immediately because it still occupies
-//! memory and can be used by queries followed up.
-//!
-//! After the handle is released by all the external owners, the eviction container will update its order or evict it
-//! based on the eviction algorithm. If it doesn't appear in the eviction container, it may be reinserted if it still in
-//! the indexer and there is enough space. Otherwise, it will be removed from both the indexer and the eviction
-//! container.
-//!
-//! The handle that does not appear in either the indexer or the eviction container, and has no external owner, will be
-//! destroyed.
+//! Because a record needs to be referenced and mutated by both the indexer and the eviction container in the same
+//! thread, it is hard to implement in 100% safe Rust without overhead. So, accessing the algorithm managed per-entry
+//! state requires operation on the `UnsafeCell`.
 
 mod cache;
-mod context;
 mod eviction;
-mod generic;
-mod handle;
 mod indexer;
-mod prelude;
+mod raw;
+mod record;
 
+mod prelude;
 pub use prelude::*;
diff --git a/foyer-memory/src/prelude.rs b/foyer-memory/src/prelude.rs
index ee2925da..3ce516b4 100644
--- a/foyer-memory/src/prelude.rs
+++ b/foyer-memory/src/prelude.rs
@@ -16,7 +16,7 @@ pub use ahash::RandomState;
 
 pub use crate::{
     cache::{Cache, CacheBuilder, CacheEntry, EvictionConfig, Fetch},
-    context::CacheContext,
     eviction::{fifo::FifoConfig, lfu::LfuConfig, lru::LruConfig, s3fifo::S3FifoConfig},
-    generic::{FetchMark, FetchState, Weighter},
+    raw::{FetchMark, FetchState, Weighter},
+    record::{CacheHint, Record},
 };
diff --git a/foyer-memory/src/raw.rs b/foyer-memory/src/raw.rs
new file mode 100644
index 00000000..3912d67b
--- /dev/null
+++ b/foyer-memory/src/raw.rs
@@ -0,0 +1,1020 @@
+//  Copyright 2024 foyer Project Authors
+//
+//  Licensed under the Apache License, Version 2.0 (the "License");
+//  you may not use this file except in compliance with the License.
+//  You may obtain a copy of the License at
+//
+//  http://www.apache.org/licenses/LICENSE-2.0
+//
+//  Unless required by applicable law or agreed to in writing, software
+//  distributed under the License is distributed on an "AS IS" BASIS,
+//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//  See the License for the specific language governing permissions and
+//  limitations under the License.
+
+use std::{
+    collections::hash_map::{Entry as HashMapEntry, HashMap},
+    fmt::Debug,
+    future::Future,
+    hash::Hash,
+    ops::Deref,
+    pin::Pin,
+    sync::Arc,
+    task::{Context, Poll},
+};
+
+use equivalent::Equivalent;
+use fastrace::{
+    future::{FutureExt, InSpan},
+    Span,
+};
+use foyer_common::{
+    code::HashBuilder,
+    event::{Event, EventListener},
+    future::{Diversion, DiversionFuture},
+    metrics::Metrics,
+    runtime::SingletonHandle,
+    scope::Scope,
+    strict_assert,
+};
+use itertools::Itertools;
+use parking_lot::{Mutex, RwLock};
+use pin_project::pin_project;
+use tokio::{sync::oneshot, task::JoinHandle};
+
+use crate::{
+    eviction::{Eviction, Operator},
+    indexer::{hash_table::HashTableIndexer, sentry::Sentry, Indexer},
+    record::{Data, Record},
+};
+
+/// The weighter for the in-memory cache.
+///
+/// The weighter is used to calculate the weight of the cache entry.
+pub trait Weighter<K, V>: Fn(&K, &V) -> usize + Send + Sync + 'static {}
+impl<K, V, T> Weighter<K, V> for T where T: Fn(&K, &V) -> usize + Send + Sync + 'static {}
+
+pub struct RawCacheConfig<E, S>
+where
+    E: Eviction,
+    S: HashBuilder,
+{
+    pub name: String,
+    pub capacity: usize,
+    pub shards: usize,
+    pub eviction_config: E::Config,
+    pub hash_builder: S,
+    pub weighter: Arc<dyn Weighter<E::Key, E::Value>>,
+    pub event_listener: Option<Arc<dyn EventListener<Key = E::Key, Value = E::Value>>>,
+}
+
+struct RawCacheShard<E, S, I>
+where
+    E: Eviction,
+    S: HashBuilder,
+    I: Indexer<Eviction = E>,
+{
+    eviction: E,
+    indexer: Sentry<I>,
+
+    usage: usize,
+    capacity: usize,
+
+    #[expect(clippy::type_complexity)]
+    waiters: Mutex<HashMap<E::Key, Vec<oneshot::Sender<RawCacheEntry<E, S, I>>>>>,
+
+    metrics: Arc<Metrics>,
+    _event_listener: Option<Arc<dyn EventListener<Key = E::Key, Value = E::Value>>>,
+}
+
+impl<E, S, I> RawCacheShard<E, S, I>
+where
+    E: Eviction,
+    S: HashBuilder,
+    I: Indexer<Eviction = E>,
+{
+    fn emplace(
+        &mut self,
+        data: Data<E>,
+        ephemeral: bool,
+        garbages: &mut Vec<(Event, Arc<Record<E>>)>,
+        waiters: &mut Vec<oneshot::Sender<RawCacheEntry<E, S, I>>>,
+    ) -> Arc<Record<E>> {
+        std::mem::swap(waiters, &mut self.waiters.lock().remove(&data.key).unwrap_or_default());
+
+        let weight = data.weight;
+
+        let record = Arc::new(Record::new(data));
+
+        // Evict overflow records.
+        while self.usage + weight > self.capacity {
+            let evicted = match self.eviction.pop() {
+                Some(evicted) => evicted,
+                None => break,
+            };
+            self.metrics.memory_evict.increment(1);
+
+            let e = self.indexer.remove(evicted.hash(), evicted.key()).unwrap();
+            assert_eq!(Arc::as_ptr(&evicted), Arc::as_ptr(&e));
+
+            strict_assert!(!evicted.as_ref().is_in_indexer());
+            strict_assert!(!evicted.as_ref().is_in_eviction());
+
+            self.usage -= evicted.weight();
+
+            garbages.push((Event::Evict, evicted));
+        }
+
+        // Insert new record
+        if let Some(old) = self.indexer.insert(record.clone()) {
+            self.metrics.memory_replace.increment(1);
+
+            strict_assert!(!old.is_in_indexer());
+
+            if old.is_in_eviction() {
+                self.eviction.remove(&old);
+            }
+            strict_assert!(!old.is_in_eviction());
+
+            self.usage -= old.weight();
+
+            garbages.push((Event::Replace, old));
+        } else {
+            self.metrics.memory_insert.increment(1);
+        }
+        strict_assert!(record.is_in_indexer());
+
+        record.set_ephemeral(ephemeral);
+        if !ephemeral {
+            self.eviction.push(record.clone());
+            strict_assert!(record.is_in_eviction());
+        }
+
+        self.usage += weight;
+        self.metrics.memory_usage.increment(weight as f64);
+        // Increase the reference count within the lock section.
+        // The reference count of the new record must be at the moment.
+        let refs = waiters.len() + 1;
+        let inc = record.inc_refs(refs);
+        assert_eq!(refs, inc);
+
+        record
+    }
+
+    #[fastrace::trace(name = "foyer::memory::raw::shard::remove")]
+    fn remove<Q>(&mut self, hash: u64, key: &Q) -> Option<Arc<Record<E>>>
+    where
+        Q: Hash + Equivalent<E::Key> + ?Sized,
+    {
+        let record = self.indexer.remove(hash, key)?;
+
+        if record.is_in_eviction() {
+            self.eviction.remove(&record);
+        }
+        strict_assert!(!record.is_in_indexer());
+        strict_assert!(!record.is_in_eviction());
+
+        self.usage -= record.weight();
+
+        self.metrics.memory_remove.increment(1);
+
+        record.inc_refs(1);
+
+        Some(record)
+    }
+
+    #[fastrace::trace(name = "foyer::memory::raw::shard::get_noop")]
+    fn get_noop<Q>(&self, hash: u64, key: &Q) -> Option<Arc<Record<E>>>
+    where
+        Q: Hash + Equivalent<E::Key> + ?Sized,
+    {
+        self.get_inner(hash, key)
+    }
+
+    #[fastrace::trace(name = "foyer::memory::raw::shard::get_immutable")]
+    fn get_immutable<Q>(&self, hash: u64, key: &Q) -> Option<Arc<Record<E>>>
+    where
+        Q: Hash + Equivalent<E::Key> + ?Sized,
+    {
+        self.get_inner(hash, key)
+            .inspect(|record| self.acquire_immutable(record))
+    }
+
+    #[fastrace::trace(name = "foyer::memory::raw::shard::get_mutable")]
+    fn get_mutable<Q>(&mut self, hash: u64, key: &Q) -> Option<Arc<Record<E>>>
+    where
+        Q: Hash + Equivalent<E::Key> + ?Sized,
+    {
+        self.get_inner(hash, key).inspect(|record| self.acquire_mutable(record))
+    }
+
+    #[fastrace::trace(name = "foyer::memory::raw::shard::get_inner")]
+    fn get_inner<Q>(&self, hash: u64, key: &Q) -> Option<Arc<Record<E>>>
+    where
+        Q: Hash + Equivalent<E::Key> + ?Sized,
+    {
+        let record = match self.indexer.get(hash, key).cloned() {
+            Some(record) => {
+                self.metrics.memory_hit.increment(1);
+                record
+            }
+            None => {
+                self.metrics.memory_miss.increment(1);
+                return None;
+            }
+        };
+
+        strict_assert!(record.is_in_indexer());
+
+        record.set_ephemeral(false);
+
+        record.inc_refs(1);
+
+        Some(record)
+    }
+
+    #[fastrace::trace(name = "foyer::memory::raw::shard::clear")]
+    fn clear(&mut self, garbages: &mut Vec<Arc<Record<E>>>) {
+        let records = self.indexer.drain().collect_vec();
+        self.eviction.clear();
+
+        let mut count = 0;
+
+        for record in records {
+            count += 1;
+            strict_assert!(!record.is_in_indexer());
+            strict_assert!(!record.is_in_eviction());
+
+            garbages.push(record);
+        }
+
+        self.metrics.memory_remove.increment(count);
+    }
+
+    #[fastrace::trace(name = "foyer::memory::raw::shard::acquire_immutable")]
+    fn acquire_immutable(&self, record: &Arc<Record<E>>) {
+        self.eviction.acquire_immutable(record);
+    }
+
+    #[fastrace::trace(name = "foyer::memory::raw::shard::acquire_mutable")]
+    fn acquire_mutable(&mut self, record: &Arc<Record<E>>) {
+        self.eviction.acquire_mutable(record);
+    }
+
+    #[fastrace::trace(name = "foyer::memory::raw::shard::release_immutable")]
+    fn release_immutable(&self, record: &Arc<Record<E>>) {
+        self.eviction.release_immutable(record);
+    }
+
+    #[fastrace::trace(name = "foyer::memory::raw::shard::release_mutable")]
+    fn release_mutable(&mut self, record: &Arc<Record<E>>) {
+        self.eviction.release_mutable(record);
+    }
+
+    #[fastrace::trace(name = "foyer::memory::raw::shard::fetch_noop")]
+    fn fetch_noop(&self, hash: u64, key: &E::Key) -> RawShardFetch<E, S, I>
+    where
+        E::Key: Clone,
+    {
+        if let Some(record) = self.get_noop(hash, key) {
+            return RawShardFetch::Hit(record);
+        }
+
+        self.fetch_queue(key.clone())
+    }
+
+    #[fastrace::trace(name = "foyer::memory::raw::shard::fetch_immutable")]
+    fn fetch_immutable(&self, hash: u64, key: &E::Key) -> RawShardFetch<E, S, I>
+    where
+        E::Key: Clone,
+    {
+        if let Some(record) = self.get_immutable(hash, key) {
+            return RawShardFetch::Hit(record);
+        }
+
+        self.fetch_queue(key.clone())
+    }
+
+    #[fastrace::trace(name = "foyer::memory::raw::shard::fetch_mutable")]
+    fn fetch_mutable(&mut self, hash: u64, key: &E::Key) -> RawShardFetch<E, S, I>
+    where
+        E::Key: Clone,
+    {
+        if let Some(record) = self.get_mutable(hash, key) {
+            return RawShardFetch::Hit(record);
+        }
+
+        self.fetch_queue(key.clone())
+    }
+
+    #[fastrace::trace(name = "foyer::memory::raw::shard::fetch_queue")]
+    fn fetch_queue(&self, key: E::Key) -> RawShardFetch<E, S, I> {
+        match self.waiters.lock().entry(key) {
+            HashMapEntry::Occupied(mut o) => {
+                let (tx, rx) = oneshot::channel();
+                o.get_mut().push(tx);
+                self.metrics.memory_queue.increment(1);
+                RawShardFetch::Wait(rx.in_span(Span::enter_with_local_parent(
+                    "foyer::memory::raw::fetch_with_runtime::wait",
+                )))
+            }
+            HashMapEntry::Vacant(v) => {
+                v.insert(vec![]);
+                self.metrics.memory_fetch.increment(1);
+                RawShardFetch::Miss
+            }
+        }
+    }
+}
+
+struct RawCacheInner<E, S, I>
+where
+    E: Eviction,
+    S: HashBuilder,
+    I: Indexer<Eviction = E>,
+{
+    shards: Vec<RwLock<RawCacheShard<E, S, I>>>,
+
+    capacity: usize,
+
+    hash_builder: S,
+    weighter: Arc<dyn Weighter<E::Key, E::Value>>,
+
+    metrics: Arc<Metrics>,
+    event_listener: Option<Arc<dyn EventListener<Key = E::Key, Value = E::Value>>>,
+}
+
+impl<E, S, I> RawCacheInner<E, S, I>
+where
+    E: Eviction,
+    S: HashBuilder,
+    I: Indexer<Eviction = E>,
+{
+    #[fastrace::trace(name = "foyer::memory::raw::inner::clear")]
+    fn clear(&self) {
+        let mut garbages = vec![];
+
+        self.shards
+            .iter()
+            .map(|shard| shard.write())
+            .for_each(|mut shard| shard.clear(&mut garbages));
+
+        // Do not deallocate data within the lock section.
+        if let Some(listener) = self.event_listener.as_ref() {
+            for record in garbages {
+                listener.on_leave(Event::Clear, record.key(), record.value());
+            }
+        }
+    }
+}
+
+pub struct RawCache<E, S = ahash::RandomState, I = HashTableIndexer<E>>
+where
+    E: Eviction,
+    S: HashBuilder,
+    I: Indexer<Eviction = E>,
+{
+    inner: Arc<RawCacheInner<E, S, I>>,
+}
+
+impl<E, S, I> Drop for RawCacheInner<E, S, I>
+where
+    E: Eviction,
+    S: HashBuilder,
+    I: Indexer<Eviction = E>,
+{
+    fn drop(&mut self) {
+        self.clear();
+    }
+}
+
+impl<E, S, I> Clone for RawCache<E, S, I>
+where
+    E: Eviction,
+    S: HashBuilder,
+    I: Indexer<Eviction = E>,
+{
+    fn clone(&self) -> Self {
+        Self {
+            inner: self.inner.clone(),
+        }
+    }
+}
+
+impl<E, S, I> RawCache<E, S, I>
+where
+    E: Eviction,
+    S: HashBuilder,
+    I: Indexer<Eviction = E>,
+{
+    pub fn new(config: RawCacheConfig<E, S>) -> Self {
+        let metrics = Arc::new(Metrics::new(&config.name));
+
+        let shard_capacity = config.capacity / config.shards;
+
+        let shards = (0..config.shards)
+            .map(|_| RawCacheShard {
+                eviction: E::new(shard_capacity, &config.eviction_config),
+                indexer: Sentry::default(),
+                usage: 0,
+                capacity: shard_capacity,
+                waiters: Mutex::default(),
+                metrics: metrics.clone(),
+                _event_listener: config.event_listener.clone(),
+            })
+            .map(RwLock::new)
+            .collect_vec();
+
+        let inner = RawCacheInner {
+            shards,
+            capacity: config.capacity,
+            hash_builder: config.hash_builder,
+            weighter: config.weighter,
+            metrics,
+            event_listener: config.event_listener,
+        };
+
+        Self { inner: Arc::new(inner) }
+    }
+
+    #[fastrace::trace(name = "foyer::memory::raw::insert")]
+    pub fn insert(&self, key: E::Key, value: E::Value) -> RawCacheEntry<E, S, I> {
+        self.insert_with_hint(key, value, Default::default())
+    }
+
+    #[fastrace::trace(name = "foyer::memory::raw::insert_with_hint")]
+    pub fn insert_with_hint(&self, key: E::Key, value: E::Value, hint: E::Hint) -> RawCacheEntry<E, S, I> {
+        self.emplace(key, value, hint, false)
+    }
+
+    #[fastrace::trace(name = "foyer::memory::raw::insert_ephemeral")]
+    pub fn insert_ephemeral(&self, key: E::Key, value: E::Value) -> RawCacheEntry<E, S, I> {
+        self.insert_ephemeral_with_hint(key, value, Default::default())
+    }
+
+    #[fastrace::trace(name = "foyer::memory::raw::insert_ephemeral_with_hint")]
+    pub fn insert_ephemeral_with_hint(&self, key: E::Key, value: E::Value, hint: E::Hint) -> RawCacheEntry<E, S, I> {
+        self.emplace(key, value, hint, true)
+    }
+
+    #[fastrace::trace(name = "foyer::memory::raw::emplace")]
+    fn emplace(&self, key: E::Key, value: E::Value, hint: E::Hint, ephemeral: bool) -> RawCacheEntry<E, S, I> {
+        let hash = self.inner.hash_builder.hash_one(&key);
+        let weight = (self.inner.weighter)(&key, &value);
+
+        let mut garbages = vec![];
+        let mut waiters = vec![];
+
+        let record = self.inner.shards[self.shard(hash)].write().with(|mut shard| {
+            shard.emplace(
+                Data {
+                    key,
+                    value,
+                    hint,
+                    hash,
+                    weight,
+                },
+                ephemeral,
+                &mut garbages,
+                &mut waiters,
+            )
+        });
+
+        // Notify waiters out of the lock critical section.
+        for waiter in waiters {
+            let _ = waiter.send(RawCacheEntry {
+                record: record.clone(),
+                inner: self.inner.clone(),
+            });
+        }
+
+        // Deallocate data out of the lock critical section.
+        if let Some(listener) = self.inner.event_listener.as_ref() {
+            for (event, record) in garbages {
+                listener.on_leave(event, record.key(), record.value());
+            }
+        }
+
+        RawCacheEntry {
+            record,
+            inner: self.inner.clone(),
+        }
+    }
+
+    #[fastrace::trace(name = "foyer::memory::raw::remove")]
+    pub fn remove<Q>(&self, key: &Q) -> Option<RawCacheEntry<E, S, I>>
+    where
+        Q: Hash + Equivalent<E::Key> + ?Sized,
+    {
+        let hash = self.inner.hash_builder.hash_one(key);
+
+        self.inner.shards[self.shard(hash)]
+            .write()
+            .with(|mut shard| {
+                shard.remove(hash, key).map(|record| RawCacheEntry {
+                    inner: self.inner.clone(),
+                    record,
+                })
+            })
+            .inspect(|record| {
+                // Deallocate data out of the lock critical section.
+                if let Some(listener) = self.inner.event_listener.as_ref() {
+                    listener.on_leave(Event::Remove, record.key(), record.value());
+                }
+            })
+    }
+
+    #[fastrace::trace(name = "foyer::memory::raw::get")]
+    pub fn get<Q>(&self, key: &Q) -> Option<RawCacheEntry<E, S, I>>
+    where
+        Q: Hash + Equivalent<E::Key> + ?Sized,
+    {
+        let hash = self.inner.hash_builder.hash_one(key);
+
+        let record = match E::acquire_operator() {
+            Operator::Noop => self.inner.shards[self.shard(hash)].read().get_noop(hash, key),
+            Operator::Immutable => self.inner.shards[self.shard(hash)]
+                .read()
+                .with(|shard| shard.get_immutable(hash, key)),
+            Operator::Mutable => self.inner.shards[self.shard(hash)]
+                .write()
+                .with(|mut shard| shard.get_mutable(hash, key)),
+        }?;
+
+        Some(RawCacheEntry {
+            inner: self.inner.clone(),
+            record,
+        })
+    }
+
+    #[fastrace::trace(name = "foyer::memory::raw::contains")]
+    pub fn contains<Q>(&self, key: &Q) -> bool
+    where
+        Q: Hash + Equivalent<E::Key> + ?Sized,
+    {
+        let hash = self.inner.hash_builder.hash_one(key);
+
+        self.inner.shards[self.shard(hash)]
+            .read()
+            .with(|shard| shard.indexer.get(hash, key).is_some())
+    }
+
+    #[fastrace::trace(name = "foyer::memory::raw::touch")]
+    pub fn touch<Q>(&self, key: &Q) -> bool
+    where
+        Q: Hash + Equivalent<E::Key> + ?Sized,
+    {
+        let hash = self.inner.hash_builder.hash_one(key);
+
+        match E::acquire_operator() {
+            Operator::Noop => self.inner.shards[self.shard(hash)]
+                .read()
+                .with(|shard| shard.get_noop(hash, key)),
+            Operator::Immutable => self.inner.shards[self.shard(hash)]
+                .read()
+                .with(|shard| shard.get_immutable(hash, key)),
+            Operator::Mutable => self.inner.shards[self.shard(hash)]
+                .write()
+                .with(|mut shard| shard.get_mutable(hash, key)),
+        }
+        .is_some()
+    }
+
+    #[fastrace::trace(name = "foyer::memory::raw::clear")]
+    pub fn clear(&self) {
+        self.inner.clear();
+    }
+
+    pub fn capacity(&self) -> usize {
+        self.inner.capacity
+    }
+
+    pub fn usage(&self) -> usize {
+        self.inner.shards.iter().map(|shard| shard.read().usage).sum()
+    }
+
+    pub fn metrics(&self) -> &Metrics {
+        &self.inner.metrics
+    }
+
+    pub fn hash_builder(&self) -> &S {
+        &self.inner.hash_builder
+    }
+
+    pub fn shards(&self) -> usize {
+        self.inner.shards.len()
+    }
+
+    fn shard(&self, hash: u64) -> usize {
+        hash as usize % self.inner.shards.len()
+    }
+}
+
+pub struct RawCacheEntry<E, S = ahash::RandomState, I = HashTableIndexer<E>>
+where
+    E: Eviction,
+    S: HashBuilder,
+    I: Indexer<Eviction = E>,
+{
+    inner: Arc<RawCacheInner<E, S, I>>,
+    record: Arc<Record<E>>,
+}
+
+impl<E, S, I> Debug for RawCacheEntry<E, S, I>
+where
+    E: Eviction,
+    S: HashBuilder,
+    I: Indexer<Eviction = E>,
+{
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        f.debug_struct("RawCacheEntry").field("record", &self.record).finish()
+    }
+}
+
+impl<E, S, I> Drop for RawCacheEntry<E, S, I>
+where
+    E: Eviction,
+    S: HashBuilder,
+    I: Indexer<Eviction = E>,
+{
+    fn drop(&mut self) {
+        let hash = self.record.hash();
+        let shard = &self.inner.shards[hash as usize % self.inner.shards.len()];
+
+        if self.record.dec_refs(1) == 0 {
+            match E::release_operator() {
+                Operator::Noop => {}
+                Operator::Immutable => shard.read().with(|shard| shard.release_immutable(&self.record)),
+                Operator::Mutable => shard.write().with(|mut shard| shard.release_mutable(&self.record)),
+            }
+        }
+    }
+}
+
+impl<E, S, I> Clone for RawCacheEntry<E, S, I>
+where
+    E: Eviction,
+    S: HashBuilder,
+    I: Indexer<Eviction = E>,
+{
+    fn clone(&self) -> Self {
+        self.record.inc_refs(1);
+        Self {
+            inner: self.inner.clone(),
+            record: self.record.clone(),
+        }
+    }
+}
+
+impl<E, S, I> Deref for RawCacheEntry<E, S, I>
+where
+    E: Eviction,
+    S: HashBuilder,
+    I: Indexer<Eviction = E>,
+{
+    type Target = E::Value;
+
+    fn deref(&self) -> &Self::Target {
+        self.value()
+    }
+}
+
+unsafe impl<E, S, I> Send for RawCacheEntry<E, S, I>
+where
+    E: Eviction,
+    S: HashBuilder,
+    I: Indexer<Eviction = E>,
+{
+}
+
+unsafe impl<E, S, I> Sync for RawCacheEntry<E, S, I>
+where
+    E: Eviction,
+    S: HashBuilder,
+    I: Indexer<Eviction = E>,
+{
+}
+
+impl<E, S, I> RawCacheEntry<E, S, I>
+where
+    E: Eviction,
+    S: HashBuilder,
+    I: Indexer<Eviction = E>,
+{
+    pub fn hash(&self) -> u64 {
+        self.record.hash()
+    }
+
+    pub fn key(&self) -> &E::Key {
+        self.record.key()
+    }
+
+    pub fn value(&self) -> &E::Value {
+        self.record.value()
+    }
+
+    pub fn hint(&self) -> &E::Hint {
+        self.record.hint()
+    }
+
+    pub fn weight(&self) -> usize {
+        self.record.weight()
+    }
+
+    pub fn refs(&self) -> usize {
+        self.record.refs()
+    }
+
+    pub fn is_outdated(&self) -> bool {
+        !self.record.is_in_indexer()
+    }
+}
+
+/// The state of [`Fetch`].
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum FetchState {
+    /// Cache hit.
+    Hit,
+    /// Cache miss, but wait in queue.
+    Wait,
+    /// Cache miss, and there is no other waiters at the moment.
+    Miss,
+}
+
+/// A mark for fetch calls.
+pub struct FetchMark;
+
+enum RawShardFetch<E, S, I>
+where
+    E: Eviction,
+    S: HashBuilder,
+    I: Indexer<Eviction = E>,
+{
+    Hit(Arc<Record<E>>),
+    Wait(InSpan<oneshot::Receiver<RawCacheEntry<E, S, I>>>),
+    Miss,
+}
+
+pub type RawFetch<E, ER, S = ahash::RandomState, I = HashTableIndexer<E>> =
+    DiversionFuture<RawFetchInner<E, ER, S, I>, std::result::Result<RawCacheEntry<E, S, I>, ER>, FetchMark>;
+
+type RawFetchHit<E, S, I> = Option<RawCacheEntry<E, S, I>>;
+type RawFetchWait<E, S, I> = InSpan<oneshot::Receiver<RawCacheEntry<E, S, I>>>;
+type RawFetchMiss<E, I, S, ER, DFS> = JoinHandle<Diversion<std::result::Result<RawCacheEntry<E, S, I>, ER>, DFS>>;
+
+#[pin_project(project = RawFetchInnerProj)]
+pub enum RawFetchInner<E, ER, S, I>
+where
+    E: Eviction,
+    S: HashBuilder,
+    I: Indexer<Eviction = E>,
+{
+    Hit(RawFetchHit<E, S, I>),
+    Wait(#[pin] RawFetchWait<E, S, I>),
+    Miss(#[pin] RawFetchMiss<E, I, S, ER, FetchMark>),
+}
+
+impl<E, ER, S, I> RawFetchInner<E, ER, S, I>
+where
+    E: Eviction,
+    S: HashBuilder,
+    I: Indexer<Eviction = E>,
+{
+    pub fn state(&self) -> FetchState {
+        match self {
+            RawFetchInner::Hit(_) => FetchState::Hit,
+            RawFetchInner::Wait(_) => FetchState::Wait,
+            RawFetchInner::Miss(_) => FetchState::Miss,
+        }
+    }
+}
+
+impl<E, ER, S, I> Future for RawFetchInner<E, ER, S, I>
+where
+    E: Eviction,
+    ER: From<oneshot::error::RecvError>,
+    S: HashBuilder,
+    I: Indexer<Eviction = E>,
+{
+    type Output = Diversion<std::result::Result<RawCacheEntry<E, S, I>, ER>, FetchMark>;
+
+    fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Self::Output> {
+        match self.project() {
+            RawFetchInnerProj::Hit(opt) => Poll::Ready(Ok(opt.take().unwrap()).into()),
+            RawFetchInnerProj::Wait(waiter) => waiter.poll(cx).map_err(|err| err.into()).map(Diversion::from),
+            RawFetchInnerProj::Miss(handle) => handle.poll(cx).map(|join| join.unwrap()),
+        }
+    }
+}
+
+// TODO(MrCroxx): use `hashbrown::HashTable` with `Handle` may relax the `Clone` bound?
+impl<E, S, I> RawCache<E, S, I>
+where
+    E: Eviction,
+    S: HashBuilder,
+    I: Indexer<Eviction = E>,
+    E::Key: Clone,
+{
+    #[fastrace::trace(name = "foyer::memory::raw::fetch")]
+    pub fn fetch<F, FU, ER>(&self, key: E::Key, fetch: F) -> RawFetch<E, ER, S, I>
+    where
+        F: FnOnce() -> FU,
+        FU: Future<Output = std::result::Result<E::Value, ER>> + Send + 'static,
+        ER: Send + 'static + Debug,
+    {
+        self.fetch_inner(
+            key,
+            Default::default(),
+            fetch,
+            &tokio::runtime::Handle::current().into(),
+        )
+    }
+
+    #[fastrace::trace(name = "foyer::memory::raw::fetch_with_hint")]
+    pub fn fetch_with_hint<F, FU, ER>(&self, key: E::Key, hint: E::Hint, fetch: F) -> RawFetch<E, ER, S, I>
+    where
+        F: FnOnce() -> FU,
+        FU: Future<Output = std::result::Result<E::Value, ER>> + Send + 'static,
+        ER: Send + 'static + Debug,
+    {
+        self.fetch_inner(key, hint, fetch, &tokio::runtime::Handle::current().into())
+    }
+
+    /// Internal fetch function, only for other foyer crates usages only, so the doc is hidden.
+    #[doc(hidden)]
+    #[fastrace::trace(name = "foyer::memory::raw::fetch_inner")]
+    pub fn fetch_inner<F, FU, ER, ID>(
+        &self,
+        key: E::Key,
+        hint: E::Hint,
+        fetch: F,
+        runtime: &SingletonHandle,
+    ) -> RawFetch<E, ER, S, I>
+    where
+        F: FnOnce() -> FU,
+        FU: Future<Output = ID> + Send + 'static,
+        ER: Send + 'static + Debug,
+        ID: Into<Diversion<std::result::Result<E::Value, ER>, FetchMark>>,
+    {
+        let hash = self.inner.hash_builder.hash_one(&key);
+
+        let raw = match E::acquire_operator() {
+            Operator::Noop => self.inner.shards[self.shard(hash)].read().fetch_noop(hash, &key),
+            Operator::Immutable => self.inner.shards[self.shard(hash)].read().fetch_immutable(hash, &key),
+            Operator::Mutable => self.inner.shards[self.shard(hash)].write().fetch_mutable(hash, &key),
+        };
+
+        match raw {
+            RawShardFetch::Hit(record) => {
+                return RawFetch::new(RawFetchInner::Hit(Some(RawCacheEntry {
+                    record,
+                    inner: self.inner.clone(),
+                })))
+            }
+            RawShardFetch::Wait(future) => return RawFetch::new(RawFetchInner::Wait(future)),
+            RawShardFetch::Miss => {}
+        }
+
+        let cache = self.clone();
+        let future = fetch();
+        let join = runtime.spawn(
+            async move {
+                let Diversion { target, store } = future
+                    .in_span(Span::enter_with_local_parent("foyer::memory::raw::fetch_inner::fn"))
+                    .await
+                    .into();
+                let value = match target {
+                    Ok(value) => value,
+                    Err(e) => {
+                        cache.inner.shards[cache.shard(hash)].read().waiters.lock().remove(&key);
+                        tracing::debug!("[fetch]: error raise while fetching, all waiter are dropped, err: {e:?}");
+                        return Diversion { target: Err(e), store };
+                    }
+                };
+                let entry = cache.insert_with_hint(key, value, hint);
+                Diversion {
+                    target: Ok(entry),
+                    store,
+                }
+            }
+            .in_span(Span::enter_with_local_parent(
+                "foyer::memory::generic::fetch_with_runtime::spawn",
+            )),
+        );
+
+        RawFetch::new(RawFetchInner::Miss(join))
+    }
+}
+
+#[cfg(test)]
+mod tests {
+
+    use rand::{rngs::SmallRng, seq::SliceRandom, RngCore, SeedableRng};
+
+    use super::*;
+    use crate::eviction::{
+        fifo::{Fifo, FifoConfig, FifoHint},
+        lfu::{Lfu, LfuConfig, LfuHint},
+        lru::{Lru, LruConfig, LruHint},
+        s3fifo::{S3Fifo, S3FifoConfig, S3FifoHint},
+    };
+
+    fn is_send_sync_static<T: Send + Sync + 'static>() {}
+
+    #[test]
+    fn test_send_sync_static() {
+        is_send_sync_static::<RawCache<Fifo<(), ()>>>();
+        is_send_sync_static::<RawCache<S3Fifo<(), ()>>>();
+        is_send_sync_static::<RawCache<Lfu<(), ()>>>();
+        is_send_sync_static::<RawCache<Lru<(), ()>>>();
+    }
+
+    fn fuzzy<E>(cache: RawCache<E>, hints: Vec<E::Hint>)
+    where
+        E: Eviction<Key = u64, Value = u64>,
+    {
+        let handles = (0..8)
+            .map(|i| {
+                let c = cache.clone();
+                let hints = hints.clone();
+                std::thread::spawn(move || {
+                    let mut rng = SmallRng::seed_from_u64(i);
+                    for _ in 0..100000 {
+                        let key = rng.next_u64();
+                        if let Some(entry) = c.get(&key) {
+                            assert_eq!(key, *entry);
+                            drop(entry);
+                            continue;
+                        }
+                        let hint = hints.choose(&mut rng).cloned().unwrap();
+                        c.insert_with_hint(key, key, hint);
+                    }
+                })
+            })
+            .collect_vec();
+
+        handles.into_iter().for_each(|handle| handle.join().unwrap());
+
+        assert_eq!(cache.usage(), cache.capacity());
+    }
+
+    #[test_log::test]
+    fn test_fifo_cache_fuzzy() {
+        let cache: RawCache<Fifo<u64, u64>> = RawCache::new(RawCacheConfig {
+            name: "test".to_string(),
+            capacity: 256,
+            shards: 4,
+            eviction_config: FifoConfig::default(),
+            hash_builder: Default::default(),
+            weighter: Arc::new(|_, _| 1),
+            event_listener: None,
+        });
+        let hints = vec![FifoHint];
+        fuzzy(cache, hints);
+    }
+
+    #[test_log::test]
+    fn test_s3fifo_cache_fuzzy() {
+        let cache: RawCache<S3Fifo<u64, u64>> = RawCache::new(RawCacheConfig {
+            name: "test".to_string(),
+            capacity: 256,
+            shards: 4,
+            eviction_config: S3FifoConfig::default(),
+            hash_builder: Default::default(),
+            weighter: Arc::new(|_, _| 1),
+            event_listener: None,
+        });
+        let hints = vec![S3FifoHint];
+        fuzzy(cache, hints);
+    }
+
+    #[test_log::test]
+    fn test_lru_cache_fuzzy() {
+        let cache: RawCache<Lru<u64, u64>> = RawCache::new(RawCacheConfig {
+            name: "test".to_string(),
+            capacity: 256,
+            shards: 4,
+            eviction_config: LruConfig::default(),
+            hash_builder: Default::default(),
+            weighter: Arc::new(|_, _| 1),
+            event_listener: None,
+        });
+        let hints = vec![LruHint::HighPriority, LruHint::LowPriority];
+        fuzzy(cache, hints);
+    }
+
+    #[test_log::test]
+    fn test_lfu_cache_fuzzy() {
+        let cache: RawCache<Lfu<u64, u64>> = RawCache::new(RawCacheConfig {
+            name: "test".to_string(),
+            capacity: 256,
+            shards: 4,
+            eviction_config: LfuConfig::default(),
+            hash_builder: Default::default(),
+            weighter: Arc::new(|_, _| 1),
+            event_listener: None,
+        });
+        let hints = vec![LfuHint];
+        fuzzy(cache, hints);
+    }
+}
diff --git a/foyer-memory/src/record.rs b/foyer-memory/src/record.rs
new file mode 100644
index 00000000..cecfc831
--- /dev/null
+++ b/foyer-memory/src/record.rs
@@ -0,0 +1,247 @@
+//  Copyright 2024 foyer Project Authors
+//
+//  Licensed under the Apache License, Version 2.0 (the "License");
+//  you may not use this file except in compliance with the License.
+//  You may obtain a copy of the License at
+//
+//  http://www.apache.org/licenses/LICENSE-2.0
+//
+//  Unless required by applicable law or agreed to in writing, software
+//  distributed under the License is distributed on an "AS IS" BASIS,
+//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//  See the License for the specific language governing permissions and
+//  limitations under the License.
+
+use std::{
+    cell::UnsafeCell,
+    fmt::Debug,
+    sync::atomic::{AtomicU64, AtomicUsize, Ordering},
+};
+
+use bitflags::bitflags;
+use serde::{Deserialize, Serialize};
+
+use crate::eviction::Eviction;
+
+/// Hint for the cache eviction algorithm to decide the priority of the specific entry if needed.
+///
+/// The meaning of the hint differs in each cache eviction algorithm, and some of them can be ignore by specific
+/// algorithm.
+///
+/// If the given cache hint does not suitable for the cache eviction algorithm that is active, the algorithm may modify
+/// it to a proper one.
+///
+/// For more details, please refer to the document of each enum options.
+#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)]
+pub enum CacheHint {
+    /// The default hint shared by all cache eviction algorithms.
+    Normal,
+    /// Suggest the priority of the entry is low.
+    ///
+    /// Used by [`crate::eviction::lru::Lru`].
+    Low,
+}
+
+bitflags! {
+    #[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)]
+    pub struct Flags: u64 {
+        const IN_INDEXER = 0b00000001;
+        const IN_EVICTION = 0b00000010;
+        const EPHEMERAL= 0b00000100;
+    }
+}
+
+pub struct Data<E>
+where
+    E: Eviction,
+{
+    pub key: E::Key,
+    pub value: E::Value,
+    pub hint: E::Hint,
+    pub hash: u64,
+    pub weight: usize,
+}
+
+/// [`Record`] holds the information of the cached entry.
+pub struct Record<E>
+where
+    E: Eviction,
+{
+    data: Option<Data<E>>,
+    state: UnsafeCell<E::State>,
+    refs: AtomicUsize,
+    flags: AtomicU64,
+}
+
+unsafe impl<E> Send for Record<E> where E: Eviction {}
+unsafe impl<E> Sync for Record<E> where E: Eviction {}
+
+impl<E> Debug for Record<E>
+where
+    E: Eviction,
+{
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        let mut s = f.debug_struct("Record");
+        if let Some(data) = self.data.as_ref() {
+            s.field("hash", &data.hash);
+        }
+        s.finish()
+    }
+}
+
+impl<E> Record<E>
+where
+    E: Eviction,
+{
+    /// `state` field memory layout offset of the [`Record`].
+    pub const STATE_OFFSET: usize = std::mem::offset_of!(Self, state);
+
+    /// Create a record with data.
+    pub fn new(data: Data<E>) -> Self {
+        Record {
+            data: Some(data),
+            state: Default::default(),
+            refs: AtomicUsize::new(0),
+            flags: AtomicU64::new(0),
+        }
+    }
+
+    /// Create a record without data.
+    pub fn empty() -> Self {
+        Record {
+            data: None,
+            state: Default::default(),
+            refs: AtomicUsize::new(0),
+            flags: AtomicU64::new(0),
+        }
+    }
+
+    /// Wrap the data in the record.
+    ///
+    /// # Safety
+    ///
+    /// Panics if the record is already wrapping data.
+    pub fn insert(&mut self, data: Data<E>) {
+        assert!(self.data.replace(data).is_none());
+    }
+
+    /// Unwrap the inner data.
+    ///
+    /// # Safety
+    ///
+    /// Panics if the record is not wrapping data.
+    pub fn take(&mut self) -> Data<E> {
+        self.state = Default::default();
+        self.refs.store(0, Ordering::SeqCst);
+        self.flags.store(0, Ordering::SeqCst);
+        self.data.take().unwrap()
+    }
+
+    /// Get the immutable reference of the record key.
+    pub fn key(&self) -> &E::Key {
+        &self.data.as_ref().unwrap().key
+    }
+
+    /// Get the immutable reference of the record value.
+    pub fn value(&self) -> &E::Value {
+        &self.data.as_ref().unwrap().value
+    }
+
+    /// Get the immutable reference of the record hint.
+    pub fn hint(&self) -> &E::Hint {
+        &self.data.as_ref().unwrap().hint
+    }
+
+    /// Get the record hash.
+    pub fn hash(&self) -> u64 {
+        self.data.as_ref().unwrap().hash
+    }
+
+    /// Get the record weight.
+    pub fn weight(&self) -> usize {
+        self.data.as_ref().unwrap().weight
+    }
+
+    /// Get the record state wrapped with [`UnsafeCell`].
+    ///
+    /// # Safety
+    pub fn state(&self) -> &UnsafeCell<E::State> {
+        &self.state
+    }
+
+    /// Set in eviction flag with relaxed memory order.
+    pub fn set_in_eviction(&self, val: bool) {
+        self.set_flags(Flags::IN_EVICTION, val, Ordering::Release);
+    }
+
+    /// Get in eviction flag with relaxed memory order.
+    pub fn is_in_eviction(&self) -> bool {
+        self.get_flags(Flags::IN_EVICTION, Ordering::Acquire)
+    }
+
+    /// Set in indexer flag with relaxed memory order.
+    pub fn set_in_indexer(&self, val: bool) {
+        self.set_flags(Flags::IN_INDEXER, val, Ordering::Release);
+    }
+
+    /// Get in indexer flag with relaxed memory order.
+    pub fn is_in_indexer(&self) -> bool {
+        self.get_flags(Flags::IN_INDEXER, Ordering::Acquire)
+    }
+
+    /// Set ephemeral flag with relaxed memory order.
+    pub fn set_ephemeral(&self, val: bool) {
+        self.set_flags(Flags::EPHEMERAL, val, Ordering::Release);
+    }
+
+    /// Get ephemeral flag with relaxed memory order.
+    pub fn is_ephemeral(&self) -> bool {
+        self.get_flags(Flags::EPHEMERAL, Ordering::Acquire)
+    }
+
+    /// Set the record atomic flags.
+    pub fn set_flags(&self, flags: Flags, val: bool, order: Ordering) {
+        match val {
+            true => self.flags.fetch_or(flags.bits(), order),
+            false => self.flags.fetch_and(!flags.bits(), order),
+        };
+    }
+
+    /// Get the record atomic flags.
+    pub fn get_flags(&self, flags: Flags, order: Ordering) -> bool {
+        self.flags.load(order) & flags.bits() == flags.bits()
+    }
+
+    /// Get the atomic reference count.
+    pub fn refs(&self) -> usize {
+        self.refs.load(Ordering::Acquire)
+    }
+
+    /// Increase the atomic reference count.
+    ///
+    /// This function returns the new reference count after the op.
+    pub fn inc_refs(&self, val: usize) -> usize {
+        let old = self.refs.fetch_add(val, Ordering::SeqCst);
+        tracing::trace!(
+            "[record]: inc record (hash: {}) refs: {} => {}",
+            self.hash(),
+            old,
+            old + val
+        );
+        old + val
+    }
+
+    /// Decrease the atomic reference count.
+    ///
+    /// This function returns the new reference count after the op.
+    pub fn dec_refs(&self, val: usize) -> usize {
+        let old = self.refs.fetch_sub(val, Ordering::SeqCst);
+        tracing::trace!(
+            "[record]: dec record (hash: {}) refs: {} => {}",
+            self.hash(),
+            old,
+            old - val
+        );
+        old - val
+    }
+}
diff --git a/foyer-storage/Cargo.toml b/foyer-storage/Cargo.toml
index 5563db83..db5d8be8 100644
--- a/foyer-storage/Cargo.toml
+++ b/foyer-storage/Cargo.toml
@@ -38,7 +38,7 @@ itertools = { workspace = true }
 libc = "0.2"
 lz4 = "1.24"
 ordered_hash_map = "0.4"
-parking_lot = { version = "0.12", features = ["arc_lock"] }
+parking_lot = { workspace = true }
 paste = "1"
 pin-project = "1"
 rand = "0.8"
diff --git a/foyer-storage/src/large/generic.rs b/foyer-storage/src/large/generic.rs
index 35b4205a..f4b4b788 100644
--- a/foyer-storage/src/large/generic.rs
+++ b/foyer-storage/src/large/generic.rs
@@ -533,6 +533,7 @@ mod tests {
 
     fn cache_for_test() -> Cache<u64, Vec<u8>> {
         CacheBuilder::new(10)
+            .with_shards(1)
             .with_eviction_config(FifoConfig::default())
             .build()
     }
diff --git a/foyer-storage/src/small/batch.rs b/foyer-storage/src/small/batch.rs
index 30fb3385..9c3baeaf 100644
--- a/foyer-storage/src/small/batch.rs
+++ b/foyer-storage/src/small/batch.rs
@@ -137,7 +137,13 @@ where
 
         set.deletes.insert(entry.hash(), self.sequence);
 
-        if entry.is_outdated() || self.len + len > self.buffer.len() {
+        if entry.is_outdated() {
+            tracing::trace!("[sodc batch]: insert {} ignored, reason: outdated", entry.hash());
+            return false;
+        }
+
+        if self.len + len > self.buffer.len() {
+            tracing::trace!("[sodc batch]: insert {} ignored, reason: buffer overflow", entry.hash());
             return false;
         }
 
diff --git a/foyer-storage/src/small/bloom_filter.rs b/foyer-storage/src/small/bloom_filter.rs
index a2707ebe..f400e451 100644
--- a/foyer-storage/src/small/bloom_filter.rs
+++ b/foyer-storage/src/small/bloom_filter.rs
@@ -52,6 +52,7 @@ macro_rules! bloom_filter {
                     }
 
                     pub fn insert(&mut self, hash: u64) {
+                        tracing::trace!("[bloom filter]: insert hash {hash}");
                         for i in 0..N {
                             let seed = twang_mix64(i as _);
                             let hash = combine_hashes(hash, seed);
@@ -73,6 +74,7 @@ macro_rules! bloom_filter {
                     }
 
                     pub fn clear(&mut self) {
+                        tracing::trace!("[bloom filter]: clear");
                         self.data = [0; N];
                     }
                 }
diff --git a/foyer-storage/src/small/generic.rs b/foyer-storage/src/small/generic.rs
index 2f94e6b8..c27e263e 100644
--- a/foyer-storage/src/small/generic.rs
+++ b/foyer-storage/src/small/generic.rs
@@ -280,7 +280,6 @@ where
 mod tests {
     use std::path::Path;
 
-    use ahash::RandomState;
     use bytesize::ByteSize;
     use foyer_common::metrics::Metrics;
     use foyer_memory::{Cache, CacheBuilder, FifoConfig};
@@ -293,11 +292,14 @@ mod tests {
             Dev,
         },
         serde::EntrySerializer,
+        test_utils::ModHasher,
         DevExt, DirectFsDeviceOptions,
     };
 
-    fn cache_for_test() -> Cache<u64, Vec<u8>> {
+    fn cache_for_test() -> Cache<u64, Vec<u8>, ModHasher> {
         CacheBuilder::new(10)
+            .with_shards(1)
+            .with_hash_builder(ModHasher::default())
             .with_eviction_config(FifoConfig::default())
             .build()
     }
@@ -318,7 +320,7 @@ mod tests {
         .unwrap()
     }
 
-    async fn store_for_test(dir: impl AsRef<Path>) -> GenericSmallStorage<u64, Vec<u8>, RandomState> {
+    async fn store_for_test(dir: impl AsRef<Path>) -> GenericSmallStorage<u64, Vec<u8>, ModHasher> {
         let device = device_for_test(dir).await;
         let regions = 0..device.regions() as RegionId;
         let config = GenericSmallStorageConfig {
@@ -337,19 +339,25 @@ mod tests {
         GenericSmallStorage::open(config).await.unwrap()
     }
 
-    fn enqueue(store: &GenericSmallStorage<u64, Vec<u8>, RandomState>, entry: &CacheEntry<u64, Vec<u8>>) {
+    fn enqueue(store: &GenericSmallStorage<u64, Vec<u8>, ModHasher>, entry: &CacheEntry<u64, Vec<u8>, ModHasher>) {
         let estimated_size = EntrySerializer::estimated_size(entry.key(), entry.value());
         store.enqueue(entry.clone(), estimated_size);
     }
 
-    async fn assert_some(store: &GenericSmallStorage<u64, Vec<u8>, RandomState>, entry: &CacheEntry<u64, Vec<u8>>) {
+    async fn assert_some(
+        store: &GenericSmallStorage<u64, Vec<u8>, ModHasher>,
+        entry: &CacheEntry<u64, Vec<u8>, ModHasher>,
+    ) {
         assert_eq!(
             store.load(entry.hash()).await.unwrap().unwrap(),
             (*entry.key(), entry.value().clone())
         );
     }
 
-    async fn assert_none(store: &GenericSmallStorage<u64, Vec<u8>, RandomState>, entry: &CacheEntry<u64, Vec<u8>>) {
+    async fn assert_none(
+        store: &GenericSmallStorage<u64, Vec<u8>, ModHasher>,
+        entry: &CacheEntry<u64, Vec<u8>, ModHasher>,
+    ) {
         assert!(store.load(entry.hash()).await.unwrap().is_none());
     }
 
diff --git a/foyer-storage/src/small/set_manager.rs b/foyer-storage/src/small/set_manager.rs
index 46d366b3..ef42a471 100644
--- a/foyer-storage/src/small/set_manager.rs
+++ b/foyer-storage/src/small/set_manager.rs
@@ -142,8 +142,11 @@ impl SetManager {
     {
         let sid = self.inner.set_picker.sid(hash);
 
+        tracing::trace!("[sodc set manager]: load {hash} from set {sid}");
+
         // Query bloom filter.
         if !self.inner.loose_bloom_filters[sid as usize].read().lookup(hash) {
+            tracing::trace!("[sodc set manager]: set {sid} bloom filter miss for {hash}");
             return Ok(None);
         }
 
diff --git a/foyer-storage/src/test_utils.rs b/foyer-storage/src/test_utils.rs
index 115c7f53..b043758f 100644
--- a/foyer-storage/src/test_utils.rs
+++ b/foyer-storage/src/test_utils.rs
@@ -18,7 +18,7 @@ use std::{
     borrow::Borrow,
     collections::HashSet,
     fmt::Debug,
-    hash::Hash,
+    hash::{BuildHasher, Hash, Hasher},
     sync::{Arc, OnceLock},
 };
 
@@ -169,3 +169,29 @@ where
         false
     }
 }
+
+/// A hasher return u64 mod result.
+#[derive(Debug, Default)]
+pub struct ModHasher {
+    state: u64,
+}
+
+impl Hasher for ModHasher {
+    fn finish(&self) -> u64 {
+        self.state
+    }
+
+    fn write(&mut self, bytes: &[u8]) {
+        for byte in bytes {
+            self.state = (self.state << 8) + *byte as u64;
+        }
+    }
+}
+
+impl BuildHasher for ModHasher {
+    type Hasher = Self;
+
+    fn build_hasher(&self) -> Self::Hasher {
+        Self::default()
+    }
+}
diff --git a/foyer-util/Cargo.toml b/foyer-util/Cargo.toml
deleted file mode 100644
index 1b485ee4..00000000
--- a/foyer-util/Cargo.toml
+++ /dev/null
@@ -1,34 +0,0 @@
-[package]
-name = "foyer-utils"
-description = "utils for foyer - Hybrid cache for Rust"
-version = { workspace = true }
-edition = { workspace = true }
-rust-version = { workspace = true }
-repository = { workspace = true }
-homepage = { workspace = true }
-keywords = { workspace = true }
-authors = { workspace = true }
-license = { workspace = true }
-readme = { workspace = true }
-publish = false
-# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
-
-[dependencies]
-bitmaps = "3"
-bitvec = "1"
-bytes = "1"
-cfg-if = "1"
-foyer-common = { workspace = true }
-futures = "0.3"
-hashbrown = { workspace = true }
-itertools = { workspace = true }
-parking_lot = { version = "0.12", features = ["arc_lock"] }
-serde = { workspace = true }
-tokio = { workspace = true }
-
-[target.'cfg(unix)'.dependencies]
-libc = "0.2"
-nix = { version = "0.29", features = ["fs"] }
-
-[dev-dependencies]
-rand = "0.8.5"
diff --git a/foyer-util/src/async_batch_pipeline.rs b/foyer-util/src/async_batch_pipeline.rs
deleted file mode 100644
index 6278f919..00000000
--- a/foyer-util/src/async_batch_pipeline.rs
+++ /dev/null
@@ -1,193 +0,0 @@
-//  Copyright 2024 foyer Project Authors
-//
-//  Licensed under the Apache License, Version 2.0 (the "License");
-//  you may not use this file except in compliance with the License.
-//  You may obtain a copy of the License at
-//
-//  http://www.apache.org/licenses/LICENSE-2.0
-//
-//  Unless required by applicable law or agreed to in writing, software
-//  distributed under the License is distributed on an "AS IS" BASIS,
-//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-//  See the License for the specific language governing permissions and
-//  limitations under the License.
-
-use std::{future::Future, sync::Arc};
-
-use parking_lot::Mutex;
-use tokio::{runtime::Handle, task::JoinHandle};
-
-/// A structured async batch pipeline.
-#[derive(Debug)]
-pub struct AsyncBatchPipeline<T, R> {
-    inner: Arc<Mutex<AsyncBatchPipelineInner<T, R>>>,
-    runtime: Handle,
-}
-
-impl<T, R> Clone for AsyncBatchPipeline<T, R> {
-    fn clone(&self) -> Self {
-        Self {
-            inner: self.inner.clone(),
-            runtime: self.runtime.clone(),
-        }
-    }
-}
-
-#[derive(Debug)]
-struct AsyncBatchPipelineInner<T, R> {
-    state: T,
-    has_leader: bool,
-    handle: Option<JoinHandle<R>>,
-}
-
-/// The token returns by [`AsyncBatchPipeline::accumulate`] if the caller is the leader of the batch.
-pub struct LeaderToken<T, R> {
-    batch: AsyncBatchPipeline<T, R>,
-    handle: Option<JoinHandle<R>>,
-}
-
-impl<T, R> AsyncBatchPipeline<T, R> {
-    /// Create a new structured async batch pipeline with the given state as its initial state.
-    pub fn new(state: T) -> Self {
-        Self::with_runtime(state, Handle::current())
-    }
-
-    /// Create a new structured async batch pipeline with the given state as its initial state.
-    ///
-    /// The pipeline will use the given runtime for spawning tasks.
-    pub fn with_runtime(state: T, runtime: Handle) -> Self {
-        Self {
-            inner: Arc::new(Mutex::new(AsyncBatchPipelineInner {
-                state,
-                has_leader: false,
-                handle: None,
-            })),
-            runtime,
-        }
-    }
-
-    /// Accumulate the batch state with the given method.
-    ///
-    /// `accumulate` returns a leader token if the caller is the leader of the batch.
-    ///
-    /// The leader must call [`LeaderToken::pipeline`] to handle the batch and progress the pipeline.
-    pub fn accumulate<F>(&self, f: F) -> Option<LeaderToken<T, R>>
-    where
-        F: FnOnce(&mut T),
-    {
-        let mut inner = self.inner.lock();
-
-        let token = if !inner.has_leader {
-            inner.has_leader = true;
-            Some(LeaderToken {
-                batch: self.clone(),
-                handle: inner.handle.take(),
-            })
-        } else {
-            None
-        };
-
-        f(&mut inner.state);
-
-        token
-    }
-
-    /// Wait for the last batch pipeline to finish.
-    pub fn wait(&self) -> Option<JoinHandle<R>> {
-        self.inner.lock().handle.take()
-    }
-}
-
-impl<T, R> LeaderToken<T, R> {
-    /// Pipeline execute futures.
-    ///
-    /// `new_state`
-    /// - Receives the reference of the old state and returns the new state.
-    ///
-    /// `f`
-    /// - Receives the owned old state and returns a future.
-    /// - The future will be polled after handling the previous result.
-    /// - The future is guaranteed to be execute one by one in order.
-    ///
-    /// `fr`
-    /// - Handle the previous result.
-    pub fn pipeline<FR, F, FU, NS>(mut self, new_state: NS, fr: FR, f: F) -> JoinHandle<()>
-    where
-        T: Send + 'static,
-        R: Send + 'static,
-        FR: FnOnce(R) + Send + 'static,
-        F: FnOnce(T) -> FU + Send + 'static,
-        FU: Future<Output = R> + Send + 'static,
-        NS: FnOnce(&T) -> T + Send + 'static,
-    {
-        let handle = self.handle.take();
-        let inner = self.batch.inner.clone();
-        let runtime = self.batch.runtime.clone();
-
-        self.batch.runtime.spawn(async move {
-            if let Some(handle) = handle {
-                fr(handle.await.unwrap());
-            }
-
-            let mut guard = inner.lock();
-            let mut state = new_state(&guard.state);
-            std::mem::swap(&mut guard.state, &mut state);
-            let future = f(state);
-            let handle = runtime.spawn(future);
-            guard.handle = Some(handle);
-            guard.has_leader = false;
-        })
-    }
-}
-
-#[cfg(test)]
-mod tests {
-
-    use futures::future::join_all;
-    use itertools::Itertools;
-
-    use super::*;
-
-    #[tokio::test]
-    async fn test_async_batch_pipeline() {
-        let batch: AsyncBatchPipeline<Vec<u64>, Vec<u64>> = AsyncBatchPipeline::new(vec![]);
-        let res = join_all((0..100).map(|i| {
-            let batch = batch.clone();
-            async move { batch.accumulate(|state| state.push(i)) }
-        }))
-        .await;
-
-        let mut res = res.into_iter().flatten().collect_vec();
-        assert_eq!(res.len(), 1);
-        let token = res.remove(0);
-        token
-            .pipeline(|_| vec![], |_| unreachable!(), |state| async move { state })
-            .await
-            .unwrap();
-
-        let res = join_all((100..200).map(|i| {
-            let batch = batch.clone();
-            async move { batch.accumulate(|state| state.push(i)) }
-        }))
-        .await;
-
-        let mut res = res.into_iter().flatten().collect_vec();
-        assert_eq!(res.len(), 1);
-        let token = res.remove(0);
-        token
-            .pipeline(
-                |_| vec![],
-                |mut res| {
-                    res.sort();
-                    assert_eq!(res, (0..100).collect_vec());
-                },
-                |state| async move { state },
-            )
-            .await
-            .unwrap();
-
-        let mut res = batch.wait().unwrap().await.unwrap();
-        res.sort();
-        assert_eq!(res, (100..200).collect_vec());
-    }
-}
diff --git a/foyer-util/src/batch.rs b/foyer-util/src/batch.rs
deleted file mode 100644
index 016e3591..00000000
--- a/foyer-util/src/batch.rs
+++ /dev/null
@@ -1,72 +0,0 @@
-//  Copyright 2024 foyer Project Authors
-//
-//  Licensed under the Apache License, Version 2.0 (the "License");
-//  you may not use this file except in compliance with the License.
-//  You may obtain a copy of the License at
-//
-//  http://www.apache.org/licenses/LICENSE-2.0
-//
-//  Unless required by applicable law or agreed to in writing, software
-//  distributed under the License is distributed on an "AS IS" BASIS,
-//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-//  See the License for the specific language governing permissions and
-//  limitations under the License.
-
-use parking_lot::Mutex;
-use tokio::sync::oneshot;
-
-const DEFAULT_CAPACITY: usize = 16;
-
-pub enum Identity<T> {
-    Leader(oneshot::Receiver<T>),
-    Follower(oneshot::Receiver<T>),
-}
-
-#[derive(Debug)]
-pub struct Item<A, T> {
-    pub arg: A,
-    pub tx: oneshot::Sender<T>,
-}
-
-#[derive(Debug)]
-pub struct Batch<A, T> {
-    queue: Mutex<Vec<Item<A, T>>>,
-}
-
-impl<A, T> Default for Batch<A, T> {
-    fn default() -> Self {
-        Self::new()
-    }
-}
-
-impl<A, T> Batch<A, T> {
-    pub fn new() -> Self {
-        Self::with_capacity(DEFAULT_CAPACITY)
-    }
-
-    pub fn with_capacity(capacity: usize) -> Self {
-        Self {
-            queue: Mutex::new(Vec::with_capacity(capacity)),
-        }
-    }
-
-    pub fn push(&self, arg: A) -> Identity<T> {
-        let (tx, rx) = oneshot::channel();
-        let item = Item { arg, tx };
-        let mut queue = self.queue.lock();
-        let is_leader = queue.is_empty();
-        queue.push(item);
-        if is_leader {
-            Identity::Leader(rx)
-        } else {
-            Identity::Follower(rx)
-        }
-    }
-
-    pub fn rotate(&self) -> Vec<Item<A, T>> {
-        let mut queue = self.queue.lock();
-        let mut q = Vec::with_capacity(queue.capacity());
-        std::mem::swap(&mut *queue, &mut q);
-        q
-    }
-}
diff --git a/foyer-util/src/compact_bloom_filter.rs b/foyer-util/src/compact_bloom_filter.rs
deleted file mode 100644
index 9c7505d2..00000000
--- a/foyer-util/src/compact_bloom_filter.rs
+++ /dev/null
@@ -1,196 +0,0 @@
-//  Copyright 2024 foyer Project Authors
-//
-//  Licensed under the Apache License, Version 2.0 (the "License");
-//  you may not use this file except in compliance with the License.
-//  You may obtain a copy of the License at
-//
-//  http://www.apache.org/licenses/LICENSE-2.0
-//
-//  Unless required by applicable law or agreed to in writing, software
-//  distributed under the License is distributed on an "AS IS" BASIS,
-//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-//  See the License for the specific language governing permissions and
-//  limitations under the License.
-
-use std::{cell::UnsafeCell, sync::Arc};
-
-use bitvec::prelude::*;
-use foyer_common::strict_assert;
-use itertools::Itertools;
-
-/// Reduce two 64-bit hashes into one.
-///
-/// Ported from CacheLib, which uses the `Hash128to64` function from Google's city hash.
-#[inline(always)]
-fn combine_hashes(upper: u64, lower: u64) -> u64 {
-    const MUL: u64 = 0x9ddfea08eb382d69;
-
-    let mut a = (lower ^ upper).wrapping_mul(MUL);
-    a ^= a >> 47;
-    let mut b = (upper ^ a).wrapping_mul(MUL);
-    b ^= b >> 47;
-    b = b.wrapping_mul(MUL);
-    b
-}
-
-#[inline(always)]
-fn twang_mix64(val: u64) -> u64 {
-    let mut val = (!val).wrapping_add(val << 21); // val *= (1 << 21); val -= 1
-    val = val ^ (val >> 24);
-    val = val.wrapping_add(val << 3).wrapping_add(val << 8); // val *= 1 + (1 << 3) + (1 << 8)
-    val = val ^ (val >> 14);
-    val = val.wrapping_add(val << 2).wrapping_add(val << 4); // va; *= 1 + (1 << 2) + (1 << 4)
-    val = val ^ (val >> 28);
-    val = val.wrapping_add(val << 31); // val *= 1 + (1 << 31)
-    val
-}
-
-/// [`CompactBloomFilter`] is composed of a series of contiguous bloom filters with each size is smaller than cache
-/// line.
-pub struct CompactBloomFilter {
-    /// Bloom filter data.
-    data: BitVec,
-    /// Seeds for each hash function.
-    seeds: Box<[u64]>,
-    /// Count of hash functions apply to each input hash.
-    hashes: usize,
-    /// Bit count of each hash function result.
-    bits: usize,
-    /// Count of bloom filters
-    filters: usize,
-}
-
-impl CompactBloomFilter {
-    /// Create a new compact bloom filter.
-    pub fn new(filters: usize, hashes: usize, bits: usize) -> Self {
-        let data = bitvec![0; filters * hashes * bits];
-        let seeds = (0..hashes)
-            .map(|i| twang_mix64(i as _))
-            .collect_vec()
-            .into_boxed_slice();
-        Self {
-            data,
-            seeds,
-            hashes,
-            bits,
-            filters,
-        }
-    }
-
-    /// Insert the given hash `hash` into the `idx` filter.
-    pub fn insert(&mut self, idx: usize, hash: u64) {
-        strict_assert!(idx < self.filters);
-        for (i, seed) in self.seeds.iter().enumerate() {
-            let bit =
-                (idx * self.hashes * self.bits) + i * self.bits + (combine_hashes(hash, *seed) as usize % self.bits);
-            self.data.set(bit, true);
-        }
-    }
-
-    /// Lookup for if the `idx` filter may contains the given key `hash`.
-    pub fn lookup(&self, idx: usize, hash: u64) -> bool {
-        strict_assert!(idx < self.filters);
-        for (i, seed) in self.seeds.iter().enumerate() {
-            let bit =
-                (idx * self.hashes * self.bits) + i * self.bits + (combine_hashes(hash, *seed) as usize % self.bits);
-            if unsafe { !*self.data.get_unchecked(bit) } {
-                return false;
-            }
-        }
-        true
-    }
-
-    /// Clear the `idx` filter.
-    pub fn clear(&mut self, idx: usize) {
-        strict_assert!(idx < self.filters);
-        let start = idx * self.hashes * self.bits;
-        let end = (idx + 1) * self.hashes * self.bits;
-        self.data.as_mut_bitslice()[start..end].fill(false);
-    }
-
-    /// Reset the all filters.
-    pub fn reset(&mut self) {
-        self.data.fill(false);
-    }
-
-    /// Create a new compact bloom filter and return the shards.
-    ///
-    /// See [`CompactBloomFilterShard`].
-    pub fn shards(filters: usize, hashes: usize, bits: usize) -> Vec<CompactBloomFilterShard> {
-        #[expect(clippy::arc_with_non_send_sync)]
-        let filter = Arc::new(UnsafeCell::new(Self::new(filters, hashes, bits)));
-        (0..filters)
-            .map(|idx| CompactBloomFilterShard {
-                inner: filter.clone(),
-                idx,
-            })
-            .collect_vec()
-    }
-}
-
-/// A shard of the compact bloom filter.
-///
-/// [`CompactBloomFilterShard`] takes the partial ownership of the compact bloom filter.
-///
-/// Operations from different shards don't affect each other.
-#[derive(Debug)]
-pub struct CompactBloomFilterShard {
-    inner: Arc<UnsafeCell<CompactBloomFilter>>,
-    idx: usize,
-}
-
-impl CompactBloomFilterShard {
-    /// Insert the given hash `hash` the filter.
-    pub fn insert(&mut self, hash: u64) {
-        let inner = unsafe { &mut *self.inner.get() };
-        inner.insert(self.idx, hash);
-    }
-
-    /// Lookup for if the filter may contains the given key `hash`.
-    pub fn lookup(&self, hash: u64) -> bool {
-        let inner = unsafe { &mut *self.inner.get() };
-        inner.lookup(self.idx, hash)
-    }
-
-    /// Clear the filter.
-    pub fn clear(&mut self) {
-        let inner = unsafe { &mut *self.inner.get() };
-        inner.clear(self.idx)
-    }
-}
-
-unsafe impl Send for CompactBloomFilterShard {}
-unsafe impl Sync for CompactBloomFilterShard {}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-
-    fn is_send_sync_static<T: Send + Sync + 'static>() {}
-
-    #[test]
-    fn ensure_send_sync_static() {
-        is_send_sync_static::<CompactBloomFilter>();
-        is_send_sync_static::<CompactBloomFilterShard>();
-    }
-
-    #[test]
-    fn test_compact_bloom_filter() {
-        let mut shards = CompactBloomFilter::shards(10, 4, 8);
-        shards[0].insert(42);
-        shards[9].insert(42);
-        for (i, shard) in shards.iter().enumerate() {
-            let res = shard.lookup(42);
-            if i == 0 || i == 9 {
-                assert!(res);
-            } else {
-                assert!(!res);
-            }
-        }
-        shards[0].clear();
-        shards[9].clear();
-        for shard in shards.iter() {
-            assert!(!shard.lookup(42));
-        }
-    }
-}
diff --git a/foyer-util/src/continuum.rs b/foyer-util/src/continuum.rs
deleted file mode 100644
index 1ee8a089..00000000
--- a/foyer-util/src/continuum.rs
+++ /dev/null
@@ -1,251 +0,0 @@
-//  Copyright 2024 foyer Project Authors
-//
-//  Licensed under the Apache License, Version 2.0 (the "License");
-//  you may not use this file except in compliance with the License.
-//  You may obtain a copy of the License at
-//
-//  http://www.apache.org/licenses/LICENSE-2.0
-//
-//  Unless required by applicable law or agreed to in writing, software
-//  distributed under the License is distributed on an "AS IS" BASIS,
-//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-//  See the License for the specific language governing permissions and
-//  limitations under the License.
-
-use std::{
-    ops::Range,
-    sync::atomic::{AtomicU16, AtomicU32, AtomicU64, AtomicU8, AtomicUsize, Ordering},
-};
-
-use itertools::Itertools;
-
-macro_rules! def_continuum {
-    ($( { $name:ident, $uint:ty, $atomic:ty }, )*) => {
-        $(
-            #[derive(Debug)]
-            pub struct $name {
-                slots: Vec<$atomic>,
-                capacity: $uint,
-                continuum: $atomic,
-            }
-
-            impl $name {
-                pub fn new(capacity: $uint) -> Self {
-                    let slots = (0..capacity).map(|_| <$atomic>::default()).collect_vec();
-                    let continuum = <$atomic>::default();
-                    Self {
-                        slots,
-                        capacity,
-                        continuum,
-                    }
-                }
-
-                pub fn is_occupied(&self, start: $uint) -> bool {
-                    !self.is_vacant(start)
-                }
-
-                pub fn is_vacant(&self, start: $uint) -> bool {
-                    let continuum = self.continuum.load(Ordering::Acquire);
-                    if continuum + self.capacity > start {
-                        return true;
-                    }
-
-                    self.advance_until(|_, _| false, 0);
-
-                    let continuum = self.continuum.load(Ordering::Acquire);
-                    continuum + self.capacity > start
-                }
-
-                /// Submit a range.
-                pub fn submit(&self, range: Range<$uint>) {
-                    debug_assert!(range.start < range.end);
-
-                    self.slots[self.slot(range.start)].store(range.end, Ordering::SeqCst);
-                }
-
-                /// Submit a range, may advance continuum till the given range.
-                ///
-                /// Return `true` if advanced, else `false`.
-                pub fn submit_advance(&self, range: Range<$uint>) -> bool {
-                    debug_assert!(range.start < range.end);
-
-                    let continuum = self.continuum.load(Ordering::Acquire);
-
-                    debug_assert!(continuum <= range.start, "assert continuum <= range.start failed: {} <= {}", continuum, range.start);
-
-                    if continuum == range.start {
-                        // continuum can be advanced directly and exclusively
-                        self.continuum.store(range.end, Ordering::Release);
-                        true
-                    } else {
-                        let slot = &self.slots[self.slot(range.start)];
-                        slot.store(range.end, Ordering::Release);
-                        let stop = move |current: $uint, _next: $uint| {
-                            current > range.start
-                        };
-                        self.advance_until(stop, 1)
-                    }
-                }
-
-                pub fn advance(&self) -> bool {
-                    self.advance_until(|_, _| false, 0)
-                }
-
-                pub fn continuum(&self) -> $uint {
-                    self.continuum.load(Ordering::Acquire)
-                }
-
-                fn slot(&self, position: $uint) -> usize {
-                    (position % self.capacity) as usize
-                }
-
-                /// `stop: Fn(continuum, next) -> bool`.
-                fn advance_until<P>(&self, stop: P, retry: usize) -> bool
-                where
-                    P: Fn($uint, $uint) -> bool + Send + Sync + 'static,
-                {
-                    let mut continuum = self.continuum.load(Ordering::Acquire);
-                    let mut start = continuum;
-
-                    let mut times = 0;
-                    loop {
-                        let slot = &self.slots[self.slot(continuum)];
-
-                        let next = slot.load(Ordering::Acquire);
-
-                        if next >= continuum + self.capacity {
-                            // case 1: `range` >= `capacity`
-                            // case 2: continuum has rotated before `next` is loaded
-                            continuum = self.continuum.load(Ordering::Acquire);
-                            if continuum != start {
-                                start = continuum;
-                                continue;
-                            }
-                        }
-
-                        if next <= continuum || stop(continuum, next) {
-                            // nothing to advance
-                            return false;
-                        }
-
-                        // make sure `continuum` can be modified exclusively and lock
-                        if let Ok(_) = slot.compare_exchange(next, continuum, Ordering::AcqRel, Ordering::Relaxed) {
-                            // If this thread is scheduled for a long time after `continuum` is loaded,
-                            // the `slot` may refer to a rotated slot with actual index `continuum + N * capacity`,
-                            // and the loaded `continuum` lags. `continuum` needs to be checked if it is still behind
-                            // the slot.
-                            continuum = self.continuum.load(Ordering::Acquire);
-                            if continuum == start {
-                                // exclusive
-                                continuum = next;
-                                break;
-                            }
-                        }
-
-                        // prepare for the next retry
-                        times += 1;
-                        if times > retry {
-                            return false;
-                        }
-
-                        continuum = self.continuum.load(Ordering::Acquire);
-                        if continuum == start {
-                            return false;
-                        }
-                        start = continuum;
-                    }
-
-                    loop {
-                        let next = self.slots[self.slot(continuum)].load(Ordering::Relaxed);
-
-                        if next <= continuum || stop(continuum, next) {
-                            break;
-                        }
-
-                        continuum = next;
-                    }
-
-                    debug_assert_eq!(start, self.continuum.load(Ordering::Acquire));
-
-                    // modify continuum exclusively and unlock
-                    self.continuum.store(continuum, Ordering::Release);
-
-                    if continuum == start {
-                        return false;
-                    }
-                    return true;
-                }
-            }
-        )*
-    }
-}
-
-macro_rules! for_all_primitives {
-    ($macro:ident) => {
-        $macro! {
-            { ContinuumU8, u8, AtomicU8 },
-            { ContinuumU16, u16, AtomicU16 },
-            { ContinuumU32, u32, AtomicU32 },
-            { ContinuumU64, u64, AtomicU64 },
-            { ContinuumUsize, usize, AtomicUsize },
-        }
-    };
-}
-
-for_all_primitives! { def_continuum }
-
-#[cfg(test)]
-mod tests {
-    use std::{sync::Arc, time::Duration};
-
-    use rand::{rngs::OsRng, Rng};
-    use tokio::sync::Semaphore;
-
-    use super::*;
-
-    #[ignore]
-    #[tokio::test(flavor = "multi_thread")]
-    async fn test_continuum_fuzzy() {
-        const CAPACITY: u64 = 4096;
-        const CURRENCY: usize = 16;
-        const UNIT: u64 = 16;
-        const LOOP: usize = 1000;
-
-        let s = Arc::new(Semaphore::new(CAPACITY as usize));
-        let c = Arc::new(ContinuumU64::new(CAPACITY));
-        let v = Arc::new(AtomicU64::new(0));
-
-        let tasks = (0..CURRENCY)
-            .map(|_| {
-                let s = s.clone();
-                let c = c.clone();
-                let v = v.clone();
-                async move {
-                    for _ in 0..LOOP {
-                        let unit = OsRng.gen_range(1..UNIT);
-                        let start = v.fetch_add(unit, Ordering::Relaxed);
-                        let end = start + unit;
-
-                        let permit = s.acquire_many(unit as u32).await.unwrap();
-
-                        let sleep = OsRng.gen_range(0..10);
-                        tokio::time::sleep(Duration::from_millis(sleep)).await;
-                        c.submit(start..end);
-                        c.advance();
-
-                        drop(permit);
-                    }
-                }
-            })
-            .collect_vec();
-
-        let handles = tasks.into_iter().map(tokio::spawn).collect_vec();
-        for handle in handles {
-            handle.await.unwrap();
-        }
-
-        c.advance();
-
-        assert_eq!(v.load(Ordering::Relaxed), c.continuum());
-    }
-}
diff --git a/foyer-util/src/erwlock.rs b/foyer-util/src/erwlock.rs
deleted file mode 100644
index 1c95264e..00000000
--- a/foyer-util/src/erwlock.rs
+++ /dev/null
@@ -1,63 +0,0 @@
-//  Copyright 2024 foyer Project Authors
-//
-//  Licensed under the Apache License, Version 2.0 (the "License");
-//  you may not use this file except in compliance with the License.
-//  You may obtain a copy of the License at
-//
-//  http://www.apache.org/licenses/LICENSE-2.0
-//
-//  Unless required by applicable law or agreed to in writing, software
-//  distributed under the License is distributed on an "AS IS" BASIS,
-//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-//  See the License for the specific language governing permissions and
-//  limitations under the License.
-
-use std::sync::Arc;
-
-use parking_lot::{lock_api::ArcRwLockWriteGuard, RawRwLock, RwLock, RwLockReadGuard, RwLockWriteGuard};
-
-pub trait ErwLockInner {
-    type R;
-    fn is_exclusive(&self, require: &Self::R) -> bool;
-}
-
-#[derive(Debug)]
-pub struct ErwLock<T: ErwLockInner> {
-    inner: Arc<RwLock<T>>,
-}
-
-impl<T: ErwLockInner> Clone for ErwLock<T> {
-    fn clone(&self) -> Self {
-        Self {
-            inner: self.inner.clone(),
-        }
-    }
-}
-
-impl<T: ErwLockInner> ErwLock<T> {
-    pub fn new(inner: T) -> Self {
-        Self {
-            inner: Arc::new(RwLock::new(inner)),
-        }
-    }
-
-    pub fn read(&self) -> RwLockReadGuard<'_, T> {
-        self.inner.read()
-    }
-
-    pub fn write(&self) -> RwLockWriteGuard<'_, T> {
-        self.inner.write()
-    }
-
-    pub async fn exclusive(&self, require: &T::R) -> ArcRwLockWriteGuard<RawRwLock, T> {
-        loop {
-            {
-                let guard = self.inner.clone().write_arc();
-                if guard.is_exclusive(require) {
-                    return guard;
-                }
-            }
-            tokio::time::sleep(std::time::Duration::from_millis(1)).await;
-        }
-    }
-}
diff --git a/foyer-util/src/iostat.rs b/foyer-util/src/iostat.rs
deleted file mode 100644
index 7a02e12d..00000000
--- a/foyer-util/src/iostat.rs
+++ /dev/null
@@ -1,172 +0,0 @@
-//  Copyright 2024 foyer Project Authors
-//
-//  Licensed under the Apache License, Version 2.0 (the "License");
-//  you may not use this file except in compliance with the License.
-//  You may obtain a copy of the License at
-//
-//  http://www.apache.org/licenses/LICENSE-2.0
-//
-//  Unless required by applicable law or agreed to in writing, software
-//  distributed under the License is distributed on an "AS IS" BASIS,
-//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-//  See the License for the specific language governing permissions and
-//  limitations under the License.
-
-// Copyright 2023 RisingWave Labs
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-use std::path::Path;
-#[cfg(unix)]
-use std::path::PathBuf;
-
-use itertools::Itertools;
-#[cfg(unix)]
-use nix::{fcntl::readlink, sys::stat::stat};
-
-#[derive(PartialEq, Clone, Copy, Debug)]
-pub enum FsType {
-    Xfs,
-    Ext4,
-    Btrfs,
-    Tmpfs,
-    Others,
-}
-
-#[cfg_attr(not(target_os = "linux"), expect(unused_variables))]
-pub fn detect_fs_type(path: impl AsRef<Path>) -> FsType {
-    #[cfg(target_os = "linux")]
-    {
-        use nix::sys::statfs::{statfs, BTRFS_SUPER_MAGIC, EXT4_SUPER_MAGIC, TMPFS_MAGIC, XFS_SUPER_MAGIC};
-        let fs_stat = statfs(path.as_ref()).unwrap();
-        match fs_stat.filesystem_type() {
-            XFS_SUPER_MAGIC => FsType::Xfs,
-            EXT4_SUPER_MAGIC => FsType::Ext4,
-            BTRFS_SUPER_MAGIC => FsType::Btrfs,
-            TMPFS_MAGIC => FsType::Tmpfs,
-            _ => FsType::Others,
-        }
-    }
-
-    #[cfg(not(target_os = "linux"))]
-    FsType::Others
-}
-
-/// Given a normal file path, returns the containing block device static file path (of the
-/// partition).
-#[cfg(unix)]
-pub fn file_stat_path(path: impl AsRef<Path>) -> PathBuf {
-    let st_dev = stat(path.as_ref()).unwrap().st_dev;
-
-    let major = unsafe { libc::major(st_dev) };
-    let minor = unsafe { libc::minor(st_dev) };
-
-    let dev = PathBuf::from("/dev/block").join(format!("{}:{}", major, minor));
-
-    let linkname = readlink(&dev).unwrap();
-    let devname = Path::new(linkname.as_os_str()).file_name().unwrap();
-    dev_stat_path(devname.to_str().unwrap())
-}
-
-#[cfg(unix)]
-pub fn dev_stat_path(devname: &str) -> PathBuf {
-    let classpath = Path::new("/sys/class/block").join(devname);
-    let devclass = readlink(&classpath).unwrap();
-
-    let devpath = Path::new(&devclass);
-    Path::new("/sys")
-        .join(devpath.strip_prefix("../..").unwrap())
-        .join("stat")
-}
-
-#[derive(Debug, Clone, Copy)]
-pub struct IoStat {
-    /// read I/Os       requests      number of read I/Os processed
-    pub read_ios: usize,
-    /// read merges     requests      number of read I/Os merged with in-queue I/O
-    pub read_merges: usize,
-    /// read sectors    sectors       number of sectors read
-    pub read_sectors: usize,
-    /// read ticks      milliseconds  total wait time for read requests
-    pub read_ticks: usize,
-    /// write I/Os      requests      number of write I/Os processed
-    pub write_ios: usize,
-    /// write merges    requests      number of write I/Os merged with in-queue I/O
-    pub write_merges: usize,
-    /// write sectors   sectors       number of sectors written
-    pub write_sectors: usize,
-    /// write ticks     milliseconds  total wait time for write requests
-    pub write_ticks: usize,
-    /// in_flight       requests      number of I/Os currently in flight
-    pub in_flight: usize,
-    /// io_ticks        milliseconds  total time this block device has been active
-    pub io_ticks: usize,
-    /// time_in_queue   milliseconds  total wait time for all requests
-    pub time_in_queue: usize,
-    /// discard I/Os    requests      number of discard I/Os processed
-    pub discard_ios: usize,
-    /// discard merges  requests      number of discard I/Os merged with in-queue I/O
-    pub discard_merges: usize,
-    /// discard sectors sectors       number of sectors discarded
-    pub discard_sectors: usize,
-    /// discard ticks   milliseconds  total wait time for discard requests
-    pub discard_ticks: usize,
-    /// flush I/Os      requests      number of flush I/Os processed
-    pub flush_ios: usize,
-    /// flush ticks     milliseconds  total wait time for flush requests
-    pub flush_ticks: usize,
-}
-
-/// Given the device static file path and get the iostat.
-pub fn iostat(path: impl AsRef<Path>) -> IoStat {
-    let content = std::fs::read_to_string(path.as_ref()).unwrap();
-    let nums = content.split_ascii_whitespace().collect_vec();
-
-    let read_ios = nums[0].parse().unwrap();
-    let read_merges = nums[1].parse().unwrap();
-    let read_sectors = nums[2].parse().unwrap();
-    let read_ticks = nums[3].parse().unwrap();
-    let write_ios = nums[4].parse().unwrap();
-    let write_merges = nums[5].parse().unwrap();
-    let write_sectors = nums[6].parse().unwrap();
-    let write_ticks = nums[7].parse().unwrap();
-    let in_flight = nums[8].parse().unwrap();
-    let io_ticks = nums[9].parse().unwrap();
-    let time_in_queue = nums[10].parse().unwrap();
-    let discard_ios = nums[11].parse().unwrap();
-    let discard_merges = nums[12].parse().unwrap();
-    let discard_sectors = nums[13].parse().unwrap();
-    let discard_ticks = nums[14].parse().unwrap();
-    let flush_ios = nums[15].parse().unwrap();
-    let flush_ticks = nums[16].parse().unwrap();
-
-    IoStat {
-        read_ios,
-        read_merges,
-        read_sectors,
-        read_ticks,
-        write_ios,
-        write_merges,
-        write_sectors,
-        write_ticks,
-        in_flight,
-        io_ticks,
-        time_in_queue,
-        discard_ios,
-        discard_merges,
-        discard_sectors,
-        discard_ticks,
-        flush_ios,
-        flush_ticks,
-    }
-}
diff --git a/foyer-util/src/judge.rs b/foyer-util/src/judge.rs
deleted file mode 100644
index d654429c..00000000
--- a/foyer-util/src/judge.rs
+++ /dev/null
@@ -1,97 +0,0 @@
-//  Copyright 2024 foyer Project Authors
-//
-//  Licensed under the Apache License, Version 2.0 (the "License");
-//  you may not use this file except in compliance with the License.
-//  You may obtain a copy of the License at
-//
-//  http://www.apache.org/licenses/LICENSE-2.0
-//
-//  Unless required by applicable law or agreed to in writing, software
-//  distributed under the License is distributed on an "AS IS" BASIS,
-//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-//  See the License for the specific language governing permissions and
-//  limitations under the License.
-
-use std::ops::{BitAnd, BitOr};
-
-use bitmaps::Bitmap;
-
-#[derive(Debug)]
-pub struct Judges {
-    /// 1: admit
-    /// 0: reject
-    judge: Bitmap<64>,
-    /// 1: use
-    /// 0: ignore
-    umask: Bitmap<64>,
-}
-
-impl Judges {
-    pub fn new(size: usize) -> Self {
-        let mut mask = Bitmap::default();
-        mask.invert();
-        Self::with_mask(size, mask)
-    }
-
-    pub fn with_mask(size: usize, mask: Bitmap<64>) -> Self {
-        let mut umask = mask.bitand(Bitmap::from_value(1u64.wrapping_shl(size as u32).wrapping_sub(1)));
-        umask.invert();
-
-        Self {
-            judge: Bitmap::default(),
-            umask,
-        }
-    }
-
-    pub fn get(&mut self, index: usize) -> bool {
-        self.judge.get(index)
-    }
-
-    pub fn set(&mut self, index: usize, judge: bool) {
-        self.judge.set(index, judge);
-    }
-
-    pub fn set_mask(&mut self, mut mask: Bitmap<64>) {
-        mask.invert();
-        self.umask = mask;
-    }
-
-    /// judge | ( ~mask )
-    ///
-    /// | judge | mask | ~mask | result |
-    /// |   0   |  0   |   1   |    1   |
-    /// |   0   |  1   |   0   |    0   |
-    /// |   1   |  0   |   1   |    1   |
-    /// |   1   |  1   |   0   |    1   |
-    pub fn judge(&self) -> bool {
-        self.judge.bitor(self.umask).is_full()
-    }
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-
-    impl Judges {
-        pub fn apply(&mut self, judge: Bitmap<64>) {
-            self.judge = judge;
-        }
-    }
-
-    #[test]
-    fn test_judge() {
-        let mask = Bitmap::from_value(0b_0011);
-
-        let dataset = vec![
-            (mask, Bitmap::from_value(0b_0011), true),
-            (mask, Bitmap::from_value(0b_1011), true),
-            (mask, Bitmap::from_value(0b_1010), false),
-        ];
-
-        for (i, (mask, j, e)) in dataset.into_iter().enumerate() {
-            let mut judge = Judges::with_mask(4, mask);
-            judge.apply(j);
-            assert_eq!(judge.judge(), e, "case {}, {} != {}", i, judge.judge(), e);
-        }
-    }
-}
diff --git a/foyer-util/src/slab/mod.rs b/foyer-util/src/slab/mod.rs
deleted file mode 100644
index c1b889ab..00000000
--- a/foyer-util/src/slab/mod.rs
+++ /dev/null
@@ -1,172 +0,0 @@
-//  Copyright 2024 foyer Project Authors
-//
-//  Licensed under the Apache License, Version 2.0 (the "License");
-//  you may not use this file except in compliance with the License.
-//  You may obtain a copy of the License at
-//
-//  http://www.apache.org/licenses/LICENSE-2.0
-//
-//  Unless required by applicable law or agreed to in writing, software
-//  distributed under the License is distributed on an "AS IS" BASIS,
-//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-//  See the License for the specific language governing permissions and
-//  limitations under the License.
-
-use std::num::NonZeroUsize;
-
-use serde::{Deserialize, Serialize};
-
-#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)]
-pub struct Token(NonZeroUsize);
-
-impl Token {
-    const MASK: usize = 1 << (usize::BITS - 1);
-
-    fn new(index: usize) -> Self {
-        unsafe { Self(NonZeroUsize::new_unchecked(index | Self::MASK)) }
-    }
-
-    pub fn index(&self) -> usize {
-        self.0.get() & !Self::MASK
-    }
-}
-
-pub struct Slab<T> {
-    entries: Vec<Entry<T>>,
-    len: usize,
-    next: usize,
-}
-
-impl<T> Default for Slab<T> {
-    fn default() -> Self {
-        Self::new()
-    }
-}
-
-impl<T> Slab<T> {
-    pub const fn new() -> Self {
-        Self {
-            entries: Vec::new(),
-            next: 0,
-            len: 0,
-        }
-    }
-
-    pub fn with_capacity(capacity: usize) -> Self {
-        Self {
-            entries: Vec::with_capacity(capacity),
-            next: 0,
-            len: 0,
-        }
-    }
-
-    pub fn insert(&mut self, val: T) -> Token {
-        let index = self.next;
-        self.insert_at(index, val);
-        Token::new(index)
-    }
-
-    pub fn remove(&mut self, token: Token) -> Option<T> {
-        self.remove_at(token.index())
-    }
-
-    /// Remove a value by token.
-    ///
-    /// # Safety
-    ///
-    /// The token must be valid.
-    pub unsafe fn remove_unchecked(&mut self, token: Token) -> T {
-        self.remove_at(token.index()).unwrap_unchecked()
-    }
-
-    pub fn get(&self, token: Token) -> Option<&T> {
-        match self.entries.get(token.index()) {
-            Some(Entry::Occupied(val)) => Some(val),
-            _ => None,
-        }
-    }
-
-    pub fn get_mut(&mut self, token: Token) -> Option<&mut T> {
-        match self.entries.get_mut(token.index()) {
-            Some(Entry::Occupied(val)) => Some(val),
-            _ => None,
-        }
-    }
-
-    /// Remove the immutable reference of a value by token.
-    ///
-    /// # Safety
-    ///
-    /// The token must be valid.
-    pub unsafe fn get_unchecked(&self, token: Token) -> &T {
-        match self.entries.get_unchecked(token.index()) {
-            Entry::Occupied(val) => val,
-            _ => unreachable!(),
-        }
-    }
-
-    /// Remove the mutable reference of a value by token.
-    ///
-    /// # Safety
-    ///
-    /// The token must be valid.
-    pub unsafe fn get_unchecked_mut(&mut self, token: Token) -> &mut T {
-        match self.entries.get_unchecked_mut(token.index()) {
-            Entry::Occupied(val) => val,
-            _ => unreachable!(),
-        }
-    }
-
-    pub fn len(&self) -> usize {
-        self.len
-    }
-
-    pub fn is_empty(&self) -> bool {
-        self.len() == 0
-    }
-
-    fn insert_at(&mut self, index: usize, val: T) {
-        self.len += 1;
-
-        if index == self.entries.len() {
-            self.entries.push(Entry::Occupied(val));
-            self.next = index + 1;
-        } else {
-            self.next = match self.entries.get(index) {
-                Some(&Entry::Vacant(next)) => next,
-                _ => unreachable!(),
-            };
-            self.entries[index] = Entry::Occupied(val);
-        }
-    }
-
-    fn remove_at(&mut self, index: usize) -> Option<T> {
-        let entry = self.entries.get_mut(index)?;
-
-        if matches!(entry, Entry::Vacant(_)) {
-            return None;
-        }
-
-        let entry = std::mem::replace(entry, Entry::Vacant(self.next));
-
-        match entry {
-            Entry::Vacant(_) => unreachable!(),
-            Entry::Occupied(val) => {
-                self.len -= 1;
-                self.next = index;
-                Some(val)
-            }
-        }
-    }
-}
-
-#[derive(Debug, Clone)]
-enum Entry<T> {
-    Vacant(usize),
-    Occupied(T),
-}
-
-#[cfg(test)]
-mod tests;
-
-pub mod slab_linked_list;
diff --git a/foyer-util/src/slab/slab_linked_list/mod.rs b/foyer-util/src/slab/slab_linked_list/mod.rs
deleted file mode 100644
index 5f7dc928..00000000
--- a/foyer-util/src/slab/slab_linked_list/mod.rs
+++ /dev/null
@@ -1,432 +0,0 @@
-//  Copyright 2024 foyer Project Authors
-//
-//  Licensed under the Apache License, Version 2.0 (the "License");
-//  you may not use this file except in compliance with the License.
-//  You may obtain a copy of the License at
-//
-//  http://www.apache.org/licenses/LICENSE-2.0
-//
-//  Unless required by applicable law or agreed to in writing, software
-//  distributed under the License is distributed on an "AS IS" BASIS,
-//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-//  See the License for the specific language governing permissions and
-//  limitations under the License.
-
-use super::{Slab, Token};
-
-pub struct SlabLinkedList<T> {
-    slab: Slab<SlabLinkedListNode<T>>,
-    head: Option<Token>,
-    tail: Option<Token>,
-}
-
-impl<T> Default for SlabLinkedList<T> {
-    fn default() -> Self {
-        Self::new()
-    }
-}
-
-impl<T> SlabLinkedList<T> {
-    pub const fn new() -> Self {
-        Self {
-            slab: Slab::new(),
-            head: None,
-            tail: None,
-        }
-    }
-
-    pub fn with_capacity(capacity: usize) -> Self {
-        Self {
-            slab: Slab::with_capacity(capacity),
-            head: None,
-            tail: None,
-        }
-    }
-
-    pub fn front(&self) -> Option<&T> {
-        self.head
-            .map(|token| unsafe { self.slab.get_unchecked(token).as_ref() })
-    }
-
-    pub fn front_mut(&mut self) -> Option<&mut T> {
-        self.head
-            .map(|token| unsafe { self.slab.get_unchecked_mut(token).as_mut() })
-    }
-
-    pub fn back(&self) -> Option<&T> {
-        self.tail
-            .map(|token| unsafe { self.slab.get_unchecked(token).as_ref() })
-    }
-
-    pub fn back_mut(&mut self) -> Option<&mut T> {
-        self.tail
-            .map(|token| unsafe { self.slab.get_unchecked_mut(token).as_mut() })
-    }
-
-    pub fn push_front(&mut self, val: T) -> Token {
-        self.iter_mut().insert_after(val)
-    }
-
-    pub fn push_back(&mut self, val: T) -> Token {
-        self.iter_mut().insert_before(val)
-    }
-
-    pub fn pop_front(&mut self) -> Option<T> {
-        let mut iter = self.iter_mut();
-        iter.move_forward();
-        iter.remove()
-    }
-
-    pub fn pop_back(&mut self) -> Option<T> {
-        let mut iter = self.iter_mut();
-        iter.move_backward();
-        iter.remove()
-    }
-
-    pub fn clear(&mut self) {
-        let mut iter = self.iter_mut();
-        iter.move_to_head();
-        while iter.is_valid() {
-            iter.remove();
-        }
-        assert!(self.is_empty());
-    }
-
-    pub fn iter(&self) -> Iter<'_, T> {
-        Iter {
-            token: None,
-            list: self,
-        }
-    }
-
-    pub fn iter_mut(&mut self) -> IterMut<'_, T> {
-        IterMut {
-            token: None,
-            list: self,
-        }
-    }
-
-    pub fn len(&self) -> usize {
-        self.slab.len()
-    }
-
-    pub fn is_empty(&self) -> bool {
-        self.len() == 0
-    }
-
-    /// Remove a node by slab token.
-    ///
-    /// # Safety
-    ///
-    /// The slab token must be valid.
-    pub unsafe fn remove_with_token(&mut self, token: Token) -> T {
-        self.iter_mut_with_token(token).remove().unwrap_unchecked()
-    }
-
-    unsafe fn iter_mut_with_token(&mut self, token: Token) -> IterMut<'_, T> {
-        IterMut {
-            token: Some(token),
-            list: self,
-        }
-    }
-}
-
-impl<T> Drop for SlabLinkedList<T> {
-    fn drop(&mut self) {
-        self.clear();
-    }
-}
-
-impl<T> IntoIterator for SlabLinkedList<T> {
-    type Item = T;
-    type IntoIter = IntoIter<T>;
-
-    fn into_iter(self) -> Self::IntoIter {
-        IntoIter { list: self }
-    }
-}
-
-impl<T> Extend<T> for SlabLinkedList<T> {
-    fn extend<I: IntoIterator<Item = T>>(&mut self, iter: I) {
-        iter.into_iter().for_each(|elt| {
-            self.push_back(elt);
-        })
-    }
-}
-
-impl<T> FromIterator<T> for SlabLinkedList<T> {
-    fn from_iter<I: IntoIterator<Item = T>>(iter: I) -> Self {
-        let mut list = Self::new();
-        list.extend(iter);
-        list
-    }
-}
-
-struct SlabLinkedListNode<T> {
-    val: T,
-
-    prev: Option<Token>,
-    next: Option<Token>,
-}
-
-impl<T> AsRef<T> for SlabLinkedListNode<T> {
-    fn as_ref(&self) -> &T {
-        &self.val
-    }
-}
-
-impl<T> AsMut<T> for SlabLinkedListNode<T> {
-    fn as_mut(&mut self) -> &mut T {
-        &mut self.val
-    }
-}
-
-pub struct IterMut<'a, T: 'a> {
-    token: Option<Token>,
-    list: &'a mut SlabLinkedList<T>,
-}
-
-impl<'a, T> IterMut<'a, T> {
-    pub fn is_valid(&self) -> bool {
-        self.token.is_some()
-    }
-
-    pub fn get(&self) -> Option<&T> {
-        self.token
-            .map(|token| unsafe { self.list.slab.get_unchecked(token).as_ref() })
-    }
-
-    pub fn get_mut(&mut self) -> Option<&mut T> {
-        self.token
-            .map(|token| unsafe { self.list.slab.get_unchecked_mut(token).as_mut() })
-    }
-
-    /// Move forward.
-    ///
-    /// If iter is on tail, move to null.
-    /// If iter is on null, move to head.
-    pub fn move_forward(&mut self) {
-        match self.token {
-            Some(token) => unsafe { self.token = self.list.slab.get_unchecked(token).next },
-            None => self.token = self.list.head,
-        }
-    }
-
-    /// Move Backward.
-    ///
-    /// If iter is on head, move to null.
-    /// If iter is on null, move to tail.
-    pub fn move_backward(&mut self) {
-        match self.token {
-            Some(token) => unsafe { self.token = self.list.slab.get_unchecked(token).prev },
-            None => self.token = self.list.head,
-        }
-    }
-
-    pub fn move_to_head(&mut self) {
-        self.token = self.list.head
-    }
-
-    pub fn move_to_tail(&mut self) {
-        self.token = self.list.tail
-    }
-
-    pub fn remove(&mut self) -> Option<T> {
-        if !self.is_valid() {
-            return None;
-        }
-
-        let token = unsafe { self.token.unwrap_unchecked() };
-        let mut node = unsafe { self.list.slab.remove_unchecked(token) };
-
-        if Some(token) == self.list.head {
-            self.list.head = node.next;
-        }
-        if Some(token) == self.list.tail {
-            self.list.tail = node.prev;
-        }
-
-        if let Some(token) = node.prev {
-            unsafe { self.list.slab.get_unchecked_mut(token).next = node.next };
-        }
-        if let Some(token) = node.next {
-            unsafe { self.list.slab.get_unchecked_mut(token).prev = node.prev };
-        }
-
-        self.token = node.next;
-
-        node.next = None;
-        node.prev = None;
-
-        Some(node.val)
-    }
-
-    /// Link a new ptr before the current one.
-    ///
-    /// If iter is on null, link to tail.
-    pub fn insert_before(&mut self, val: T) -> Token {
-        let token_new = self.list.slab.insert(SlabLinkedListNode {
-            val,
-            prev: None,
-            next: None,
-        });
-
-        match self.token {
-            Some(token) => self.link_before(token_new, token),
-            None => {
-                self.link_between(token_new, self.list.tail, None);
-                self.list.tail = Some(token_new)
-            }
-        }
-
-        if self.list.head == self.token {
-            self.list.head = Some(token_new);
-        }
-
-        token_new
-    }
-
-    /// Link a new ptr after the current one.
-    ///
-    /// If iter is on null, link to head.
-    pub fn insert_after(&mut self, val: T) -> Token {
-        let token_new = self.list.slab.insert(SlabLinkedListNode {
-            val,
-            prev: None,
-            next: None,
-        });
-
-        match self.token {
-            Some(token) => self.link_after(token_new, token),
-            None => {
-                self.link_between(token_new, None, self.list.head);
-                self.list.head = Some(token_new)
-            }
-        }
-
-        if self.list.tail == self.token {
-            self.list.tail = Some(token_new);
-        }
-
-        token_new
-    }
-
-    pub fn is_head(&self) -> bool {
-        self.token == self.list.head
-    }
-
-    pub fn is_tail(&self) -> bool {
-        self.token == self.list.tail
-    }
-
-    fn link_before(&mut self, token: Token, next: Token) {
-        self.link_between(token, unsafe { self.list.slab.get_unchecked(next).prev }, Some(next));
-    }
-
-    fn link_after(&mut self, token: Token, prev: Token) {
-        self.link_between(token, Some(prev), unsafe { self.list.slab.get_unchecked(prev).next });
-    }
-
-    fn link_between(&mut self, token: Token, prev: Option<Token>, next: Option<Token>) {
-        if let Some(prev) = prev {
-            unsafe { self.list.slab.get_unchecked_mut(prev).next = Some(token) };
-        }
-        if let Some(next) = next {
-            unsafe { self.list.slab.get_unchecked_mut(next).prev = Some(token) };
-        }
-        let node = unsafe { self.list.slab.get_unchecked_mut(token) };
-        node.prev = prev;
-        node.next = next;
-    }
-}
-
-impl<'a, T> Iterator for IterMut<'a, T> {
-    type Item = &'a mut T;
-
-    fn next(&mut self) -> Option<Self::Item> {
-        self.move_forward();
-        self.get_mut().map(|val| unsafe { &mut *(val as *mut _) })
-    }
-}
-
-pub struct Iter<'a, T: 'a> {
-    token: Option<Token>,
-    list: &'a SlabLinkedList<T>,
-}
-
-impl<'a, T> Iter<'a, T> {
-    pub fn is_valid(&self) -> bool {
-        self.token.is_some()
-    }
-
-    pub fn get(&self) -> Option<&T> {
-        self.token
-            .map(|token| unsafe { self.list.slab.get_unchecked(token).as_ref() })
-    }
-
-    /// Move forward.
-    ///
-    /// If iter is on tail, move to null.
-    /// If iter is on null, move to head.
-    pub fn move_forward(&mut self) {
-        match self.token {
-            Some(token) => unsafe { self.token = self.list.slab.get_unchecked(token).next },
-            None => self.token = self.list.head,
-        }
-    }
-
-    /// Move Backward.
-    ///
-    /// If iter is on head, move to null.
-    /// If iter is on null, move to tail.
-    pub fn move_backward(&mut self) {
-        match self.token {
-            Some(token) => unsafe { self.token = self.list.slab.get_unchecked(token).prev },
-            None => self.token = self.list.head,
-        }
-    }
-
-    pub fn move_to_head(&mut self) {
-        self.token = self.list.head
-    }
-
-    pub fn move_to_tail(&mut self) {
-        self.token = self.list.tail
-    }
-
-    pub fn is_head(&self) -> bool {
-        self.token == self.list.head
-    }
-
-    pub fn is_tail(&self) -> bool {
-        self.token == self.list.tail
-    }
-}
-
-impl<'a, T> Iterator for Iter<'a, T> {
-    type Item = &'a T;
-
-    fn next(&mut self) -> Option<Self::Item> {
-        self.move_forward();
-        self.get().map(|val| unsafe { &*(val as *const _) })
-    }
-}
-
-pub struct IntoIter<T> {
-    list: SlabLinkedList<T>,
-}
-
-impl<T> Iterator for IntoIter<T> {
-    type Item = T;
-
-    fn next(&mut self) -> Option<Self::Item> {
-        self.list.pop_front()
-    }
-
-    fn size_hint(&self) -> (usize, Option<usize>) {
-        (self.list.len(), Some(self.list.len()))
-    }
-}
-
-#[cfg(test)]
-mod tests;
diff --git a/foyer-util/src/slab/slab_linked_list/tests.rs b/foyer-util/src/slab/slab_linked_list/tests.rs
deleted file mode 100644
index 0189d777..00000000
--- a/foyer-util/src/slab/slab_linked_list/tests.rs
+++ /dev/null
@@ -1,58 +0,0 @@
-//  Copyright 2024 foyer Project Authors
-//
-//  Licensed under the Apache License, Version 2.0 (the "License");
-//  you may not use this file except in compliance with the License.
-//  You may obtain a copy of the License at
-//
-//  http://www.apache.org/licenses/LICENSE-2.0
-//
-//  Unless required by applicable law or agreed to in writing, software
-//  distributed under the License is distributed on an "AS IS" BASIS,
-//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-//  See the License for the specific language governing permissions and
-//  limitations under the License.
-
-// TODO(MrCroxx): We need more tests!
-
-use super::*;
-
-#[test]
-fn test_basic() {
-    let mut l1 = SlabLinkedList::<Box<_>>::new();
-
-    assert_eq!(l1.pop_front(), None);
-    assert_eq!(l1.pop_back(), None);
-    assert_eq!(l1.pop_front(), None);
-    l1.push_front(Box::new(1));
-    assert_eq!(l1.pop_front(), Some(Box::new(1)));
-    l1.push_back(Box::new(2));
-    l1.push_back(Box::new(3));
-    assert_eq!(l1.len(), 2);
-    assert_eq!(l1.pop_front(), Some(Box::new(2)));
-    assert_eq!(l1.pop_front(), Some(Box::new(3)));
-    assert_eq!(l1.len(), 0);
-    assert_eq!(l1.pop_front(), None);
-    l1.push_back(Box::new(1));
-    l1.push_back(Box::new(3));
-    l1.push_back(Box::new(5));
-    l1.push_back(Box::new(7));
-    assert_eq!(l1.pop_front(), Some(Box::new(1)));
-
-    let mut l2 = SlabLinkedList::new();
-    l2.push_front(2);
-    l2.push_front(3);
-    {
-        assert_eq!(l2.front().unwrap(), &3);
-        let x = l2.front_mut().unwrap();
-        assert_eq!(*x, 3);
-        *x = 0;
-    }
-    {
-        assert_eq!(l2.back().unwrap(), &2);
-        let y = l2.back_mut().unwrap();
-        assert_eq!(*y, 2);
-        *y = 1;
-    }
-    assert_eq!(l2.pop_front(), Some(0));
-    assert_eq!(l2.pop_front(), Some(1));
-}
diff --git a/foyer-util/src/slab/tests.rs b/foyer-util/src/slab/tests.rs
deleted file mode 100644
index 75a0abbe..00000000
--- a/foyer-util/src/slab/tests.rs
+++ /dev/null
@@ -1,44 +0,0 @@
-//  Copyright 2024 foyer Project Authors
-//
-//  Licensed under the Apache License, Version 2.0 (the "License");
-//  you may not use this file except in compliance with the License.
-//  You may obtain a copy of the License at
-//
-//  http://www.apache.org/licenses/LICENSE-2.0
-//
-//  Unless required by applicable law or agreed to in writing, software
-//  distributed under the License is distributed on an "AS IS" BASIS,
-//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-//  See the License for the specific language governing permissions and
-//  limitations under the License.
-
-// TODO(MrCroxx): We need more tests!
-
-use super::*;
-
-#[test]
-fn test_token_null_pointer_optimization() {
-    assert_eq!(std::mem::size_of::<Token>(), std::mem::size_of::<Option<Token>>());
-}
-
-#[test]
-fn test_slab() {
-    let mut slab = Slab::new();
-
-    let t1 = slab.insert(1);
-    let t2 = slab.insert(2);
-    let t3 = slab.insert(3);
-
-    assert_eq!(slab.get(t1).unwrap(), &1);
-    assert_eq!(slab.get(t2).unwrap(), &2);
-    assert_eq!(slab.get(t3).unwrap(), &3);
-    assert_eq!(slab.len(), 3);
-
-    *slab.get_mut(t2).unwrap() = 4;
-    assert_eq!(slab.get(t2).unwrap(), &4);
-    assert_eq!(slab.len(), 3);
-
-    let v2 = slab.remove(t2).unwrap();
-    assert_eq!(v2, 4);
-    assert_eq!(slab.len(), 2);
-}
diff --git a/foyer/src/hybrid/builder.rs b/foyer/src/hybrid/builder.rs
index 826c091a..08bc2c7d 100644
--- a/foyer/src/hybrid/builder.rs
+++ b/foyer/src/hybrid/builder.rs
@@ -136,20 +136,6 @@ where
         }
     }
 
-    /// Set object pool for handles. The object pool is used to reduce handle allocation.
-    ///
-    /// The optimized value is supposed to be equal to the max cache entry count.
-    ///
-    /// The default value is 1024.
-    pub fn with_object_pool_capacity(self, object_pool_capacity: usize) -> Self {
-        let builder = self.builder.with_object_pool_capacity(object_pool_capacity);
-        HybridCacheBuilderPhaseMemory {
-            name: self.name,
-            tracing_options: self.tracing_options,
-            builder,
-        }
-    }
-
     /// Set in-memory cache hash builder.
     pub fn with_hash_builder<OS>(self, hash_builder: OS) -> HybridCacheBuilderPhaseMemory<K, V, OS>
     where
diff --git a/foyer/src/hybrid/cache.rs b/foyer/src/hybrid/cache.rs
index e784d85f..fb15a588 100644
--- a/foyer/src/hybrid/cache.rs
+++ b/foyer/src/hybrid/cache.rs
@@ -35,7 +35,7 @@ use foyer_common::{
     metrics::Metrics,
     tracing::{InRootSpan, TracingConfig, TracingOptions},
 };
-use foyer_memory::{Cache, CacheContext, CacheEntry, Fetch, FetchMark, FetchState};
+use foyer_memory::{Cache, CacheEntry, CacheHint, Fetch, FetchMark, FetchState};
 use foyer_storage::{DeviceStats, Store};
 use futures::FutureExt;
 use pin_project::pin_project;
@@ -189,15 +189,15 @@ where
         entry
     }
 
-    /// Insert cache entry with cache context to the hybrid cache.
-    pub fn insert_with_context(&self, key: K, value: V, context: CacheContext) -> HybridCacheEntry<K, V, S> {
+    /// Insert cache entry with cache hint to the hybrid cache.
+    pub fn insert_with_hint(&self, key: K, value: V, hint: CacheHint) -> HybridCacheEntry<K, V, S> {
         root_span!(self, mut span, "foyer::hybrid::cache::insert_with_context");
 
         let _guard = span.set_local_parent();
 
         let now = Instant::now();
 
-        let entry = self.memory.insert_with_context(key, value, context);
+        let entry = self.memory.insert_with_hint(key, value, hint);
         self.storage.enqueue(entry.clone(), false);
 
         self.metrics.hybrid_insert.increment(1);
@@ -457,14 +457,14 @@ where
         F: FnOnce() -> FU,
         FU: Future<Output = anyhow::Result<V>> + Send + 'static,
     {
-        self.fetch_with_context(key, CacheContext::default(), fetch)
+        self.fetch_with_hint(key, CacheHint::Normal, fetch)
     }
 
-    /// Fetch and insert a cache entry with the given key, context, and method if there is a cache miss.
+    /// Fetch and insert a cache entry with the given key, hint, and method if there is a cache miss.
     ///
     /// If the dedicated runtime of the foyer storage engine is enabled, `fetch` will spawn task with the dedicated
     /// runtime. Otherwise, the user's runtime will be used.
-    pub fn fetch_with_context<F, FU>(&self, key: K, context: CacheContext, fetch: F) -> HybridFetch<K, V, S>
+    pub fn fetch_with_hint<F, FU>(&self, key: K, hint: CacheHint, fetch: F) -> HybridFetch<K, V, S>
     where
         F: FnOnce() -> FU,
         FU: Future<Output = anyhow::Result<V>> + Send + 'static,
@@ -480,7 +480,7 @@ where
         let future = fetch();
         let inner = self.memory.fetch_inner(
             key.clone(),
-            context,
+            hint,
             || {
                 let metrics = self.metrics.clone();
                 let runtime = self.storage().runtime().clone();
@@ -586,14 +586,14 @@ mod tests {
         let hybrid = open(dir.path()).await;
 
         let e1 = hybrid.insert(1, vec![1; 7 * KB]);
-        let e2 = hybrid.insert_with_context(2, vec![2; 7 * KB], CacheContext::default());
+        let e2 = hybrid.insert_with_hint(2, vec![2; 7 * KB], CacheHint::Normal);
         assert_eq!(e1.value(), &vec![1; 7 * KB]);
         assert_eq!(e2.value(), &vec![2; 7 * KB]);
 
         let e3 = hybrid.storage_writer(3).insert(vec![3; 7 * KB]).unwrap();
         let e4 = hybrid
             .storage_writer(4)
-            .insert_with_context(vec![4; 7 * KB], CacheContext::default())
+            .insert_with_context(vec![4; 7 * KB], CacheHint::Normal)
             .unwrap();
         assert_eq!(e3.value(), &vec![3; 7 * KB]);
         assert_eq!(e4.value(), &vec![4; 7 * KB]);
@@ -622,17 +622,13 @@ mod tests {
         let hybrid = open_with_biased_admission_picker(dir.path(), [1, 2, 3, 4]).await;
 
         let e1 = hybrid.writer(1).insert(vec![1; 7 * KB]);
-        let e2 = hybrid
-            .writer(2)
-            .insert_with_context(vec![2; 7 * KB], CacheContext::default());
+        let e2 = hybrid.writer(2).insert_with_hint(vec![2; 7 * KB], CacheHint::Normal);
 
         assert_eq!(e1.value(), &vec![1; 7 * KB]);
         assert_eq!(e2.value(), &vec![2; 7 * KB]);
 
         let e3 = hybrid.writer(3).storage().insert(vec![3; 7 * KB]).unwrap();
-        let e4 = hybrid
-            .writer(4)
-            .insert_with_context(vec![4; 7 * KB], CacheContext::default());
+        let e4 = hybrid.writer(4).insert_with_hint(vec![4; 7 * KB], CacheHint::Normal);
 
         assert_eq!(e3.value(), &vec![3; 7 * KB]);
         assert_eq!(e4.value(), &vec![4; 7 * KB]);
diff --git a/foyer/src/hybrid/writer.rs b/foyer/src/hybrid/writer.rs
index 5d4e72ae..14f63fdb 100644
--- a/foyer/src/hybrid/writer.rs
+++ b/foyer/src/hybrid/writer.rs
@@ -19,7 +19,7 @@ use std::{
 
 use ahash::RandomState;
 use foyer_common::code::{HashBuilder, StorageKey, StorageValue};
-use foyer_memory::CacheContext;
+use foyer_memory::CacheHint;
 
 use crate::{HybridCache, HybridCacheEntry};
 
@@ -49,9 +49,9 @@ where
         self.hybrid.insert(self.key, value)
     }
 
-    /// Insert the entry with context to the hybrid cache.
-    pub fn insert_with_context(self, value: V, context: CacheContext) -> HybridCacheEntry<K, V, S> {
-        self.hybrid.insert_with_context(self.key, value, context)
+    /// Insert the entry with hint to the hybrid cache.
+    pub fn insert_with_hint(self, value: V, hint: CacheHint) -> HybridCacheEntry<K, V, S> {
+        self.hybrid.insert_with_hint(self.key, value, hint)
     }
 
     /// Convert [`HybridCacheWriter`] to [`HybridCacheStorageWriter`].
@@ -113,16 +113,16 @@ where
         self
     }
 
-    fn insert_inner(mut self, value: V, context: Option<CacheContext>) -> Option<HybridCacheEntry<K, V, S>> {
+    fn insert_inner(mut self, value: V, hint: Option<CacheHint>) -> Option<HybridCacheEntry<K, V, S>> {
         let now = Instant::now();
 
         if !self.pick() {
             return None;
         }
 
-        let entry = match context {
-            Some(context) => self.hybrid.memory().deposit_with_context(self.key, value, context),
-            None => self.hybrid.memory().deposit(self.key, value),
+        let entry = match hint {
+            Some(hint) => self.hybrid.memory().insert_ephemeral_with_hint(self.key, value, hint),
+            None => self.hybrid.memory().insert_ephemeral(self.key, value),
         };
         self.hybrid.storage().enqueue(entry.clone(), true);
 
@@ -141,7 +141,7 @@ where
     }
 
     /// Insert the entry with context to the disk cache only.
-    pub fn insert_with_context(self, value: V, context: CacheContext) -> Option<HybridCacheEntry<K, V, S>> {
+    pub fn insert_with_context(self, value: V, context: CacheHint) -> Option<HybridCacheEntry<K, V, S>> {
         self.insert_inner(value, Some(context))
     }
 }
diff --git a/foyer/src/prelude.rs b/foyer/src/prelude.rs
index 9ca7aec4..779d253f 100644
--- a/foyer/src/prelude.rs
+++ b/foyer/src/prelude.rs
@@ -16,7 +16,7 @@ pub use crate::{
     common::{
         buf::{BufExt, BufMutExt},
         code::{Key, StorageKey, StorageValue, Value},
-        event::EventListener,
+        event::{Event, EventListener},
         range::RangeBoundsExt,
         tracing::TracingOptions,
     },
@@ -26,7 +26,7 @@ pub use crate::{
         writer::{HybridCacheStorageWriter, HybridCacheWriter},
     },
     memory::{
-        Cache, CacheBuilder, CacheContext, CacheEntry, EvictionConfig, FetchState, FifoConfig, LfuConfig, LruConfig,
+        Cache, CacheBuilder, CacheEntry, CacheHint, EvictionConfig, FetchState, FifoConfig, LfuConfig, LruConfig,
         S3FifoConfig, Weighter,
     },
     storage::{

From d633ae2b57b625eb61868b3c12cfd847fb2ae973 Mon Sep 17 00:00:00 2001
From: Croxx <mrcroxx@outlook.com>
Date: Tue, 19 Nov 2024 14:28:15 +0800
Subject: [PATCH 42/53] fix: fix insert ephemeral (#786)

Signed-off-by: MrCroxx <mrcroxx@outlook.com>
---
 foyer-memory/src/raw.rs | 204 ++++++++++++++++++++++++----------------
 1 file changed, 125 insertions(+), 79 deletions(-)

diff --git a/foyer-memory/src/raw.rs b/foyer-memory/src/raw.rs
index 3912d67b..fe9d2e5f 100644
--- a/foyer-memory/src/raw.rs
+++ b/foyer-memory/src/raw.rs
@@ -647,6 +647,18 @@ where
                 Operator::Immutable => shard.read().with(|shard| shard.release_immutable(&self.record)),
                 Operator::Mutable => shard.write().with(|mut shard| shard.release_mutable(&self.record)),
             }
+
+            if self.record.is_ephemeral() {
+                shard
+                    .write()
+                    .with(|mut shard| shard.remove(hash, self.key()))
+                    .inspect(|record| {
+                        // Deallocate data out of the lock critical section.
+                        if let Some(listener) = self.inner.event_listener.as_ref() {
+                            listener.on_leave(Event::Remove, record.key(), record.value());
+                        }
+                    });
+            }
         }
     }
 }
@@ -921,6 +933,18 @@ mod tests {
 
     fn is_send_sync_static<T: Send + Sync + 'static>() {}
 
+    fn fifo_cache_for_test() -> RawCache<Fifo<u64, u64>> {
+        RawCache::new(RawCacheConfig {
+            name: "test".to_string(),
+            capacity: 256,
+            shards: 4,
+            eviction_config: FifoConfig::default(),
+            hash_builder: Default::default(),
+            weighter: Arc::new(|_, _| 1),
+            event_listener: None,
+        })
+    }
+
     #[test]
     fn test_send_sync_static() {
         is_send_sync_static::<RawCache<Fifo<(), ()>>>();
@@ -929,92 +953,114 @@ mod tests {
         is_send_sync_static::<RawCache<Lru<(), ()>>>();
     }
 
-    fn fuzzy<E>(cache: RawCache<E>, hints: Vec<E::Hint>)
-    where
-        E: Eviction<Key = u64, Value = u64>,
-    {
-        let handles = (0..8)
-            .map(|i| {
-                let c = cache.clone();
-                let hints = hints.clone();
-                std::thread::spawn(move || {
-                    let mut rng = SmallRng::seed_from_u64(i);
-                    for _ in 0..100000 {
-                        let key = rng.next_u64();
-                        if let Some(entry) = c.get(&key) {
-                            assert_eq!(key, *entry);
-                            drop(entry);
-                            continue;
+    #[test]
+    fn test_insert_ephemeral() {
+        let fifo = fifo_cache_for_test();
+
+        let e1 = fifo.insert_ephemeral(1, 1);
+        assert_eq!(fifo.usage(), 1);
+        drop(e1);
+        assert_eq!(fifo.usage(), 0);
+
+        let e2a = fifo.insert_ephemeral(2, 2);
+        assert_eq!(fifo.usage(), 1);
+        let e2b = fifo.get(&2).expect("entry 2 should exist");
+        drop(e2a);
+        assert_eq!(fifo.usage(), 1);
+        drop(e2b);
+        assert_eq!(fifo.usage(), 1);
+    }
+
+    mod fuzzy {
+        use super::*;
+
+        fn fuzzy<E>(cache: RawCache<E>, hints: Vec<E::Hint>)
+        where
+            E: Eviction<Key = u64, Value = u64>,
+        {
+            let handles = (0..8)
+                .map(|i| {
+                    let c = cache.clone();
+                    let hints = hints.clone();
+                    std::thread::spawn(move || {
+                        let mut rng = SmallRng::seed_from_u64(i);
+                        for _ in 0..100000 {
+                            let key = rng.next_u64();
+                            if let Some(entry) = c.get(&key) {
+                                assert_eq!(key, *entry);
+                                drop(entry);
+                                continue;
+                            }
+                            let hint = hints.choose(&mut rng).cloned().unwrap();
+                            c.insert_with_hint(key, key, hint);
                         }
-                        let hint = hints.choose(&mut rng).cloned().unwrap();
-                        c.insert_with_hint(key, key, hint);
-                    }
+                    })
                 })
-            })
-            .collect_vec();
+                .collect_vec();
 
-        handles.into_iter().for_each(|handle| handle.join().unwrap());
+            handles.into_iter().for_each(|handle| handle.join().unwrap());
 
-        assert_eq!(cache.usage(), cache.capacity());
-    }
+            assert_eq!(cache.usage(), cache.capacity());
+        }
 
-    #[test_log::test]
-    fn test_fifo_cache_fuzzy() {
-        let cache: RawCache<Fifo<u64, u64>> = RawCache::new(RawCacheConfig {
-            name: "test".to_string(),
-            capacity: 256,
-            shards: 4,
-            eviction_config: FifoConfig::default(),
-            hash_builder: Default::default(),
-            weighter: Arc::new(|_, _| 1),
-            event_listener: None,
-        });
-        let hints = vec![FifoHint];
-        fuzzy(cache, hints);
-    }
+        #[test_log::test]
+        fn test_fifo_cache_fuzzy() {
+            let cache: RawCache<Fifo<u64, u64>> = RawCache::new(RawCacheConfig {
+                name: "test".to_string(),
+                capacity: 256,
+                shards: 4,
+                eviction_config: FifoConfig::default(),
+                hash_builder: Default::default(),
+                weighter: Arc::new(|_, _| 1),
+                event_listener: None,
+            });
+            let hints = vec![FifoHint];
+            fuzzy(cache, hints);
+        }
 
-    #[test_log::test]
-    fn test_s3fifo_cache_fuzzy() {
-        let cache: RawCache<S3Fifo<u64, u64>> = RawCache::new(RawCacheConfig {
-            name: "test".to_string(),
-            capacity: 256,
-            shards: 4,
-            eviction_config: S3FifoConfig::default(),
-            hash_builder: Default::default(),
-            weighter: Arc::new(|_, _| 1),
-            event_listener: None,
-        });
-        let hints = vec![S3FifoHint];
-        fuzzy(cache, hints);
-    }
+        #[test_log::test]
+        fn test_s3fifo_cache_fuzzy() {
+            let cache: RawCache<S3Fifo<u64, u64>> = RawCache::new(RawCacheConfig {
+                name: "test".to_string(),
+                capacity: 256,
+                shards: 4,
+                eviction_config: S3FifoConfig::default(),
+                hash_builder: Default::default(),
+                weighter: Arc::new(|_, _| 1),
+                event_listener: None,
+            });
+            let hints = vec![S3FifoHint];
+            fuzzy(cache, hints);
+        }
 
-    #[test_log::test]
-    fn test_lru_cache_fuzzy() {
-        let cache: RawCache<Lru<u64, u64>> = RawCache::new(RawCacheConfig {
-            name: "test".to_string(),
-            capacity: 256,
-            shards: 4,
-            eviction_config: LruConfig::default(),
-            hash_builder: Default::default(),
-            weighter: Arc::new(|_, _| 1),
-            event_listener: None,
-        });
-        let hints = vec![LruHint::HighPriority, LruHint::LowPriority];
-        fuzzy(cache, hints);
-    }
+        #[test_log::test]
+        fn test_lru_cache_fuzzy() {
+            let cache: RawCache<Lru<u64, u64>> = RawCache::new(RawCacheConfig {
+                name: "test".to_string(),
+                capacity: 256,
+                shards: 4,
+                eviction_config: LruConfig::default(),
+                hash_builder: Default::default(),
+                weighter: Arc::new(|_, _| 1),
+                event_listener: None,
+            });
+            let hints = vec![LruHint::HighPriority, LruHint::LowPriority];
+            fuzzy(cache, hints);
+        }
 
-    #[test_log::test]
-    fn test_lfu_cache_fuzzy() {
-        let cache: RawCache<Lfu<u64, u64>> = RawCache::new(RawCacheConfig {
-            name: "test".to_string(),
-            capacity: 256,
-            shards: 4,
-            eviction_config: LfuConfig::default(),
-            hash_builder: Default::default(),
-            weighter: Arc::new(|_, _| 1),
-            event_listener: None,
-        });
-        let hints = vec![LfuHint];
-        fuzzy(cache, hints);
+        #[test_log::test]
+        fn test_lfu_cache_fuzzy() {
+            let cache: RawCache<Lfu<u64, u64>> = RawCache::new(RawCacheConfig {
+                name: "test".to_string(),
+                capacity: 256,
+                shards: 4,
+                eviction_config: LfuConfig::default(),
+                hash_builder: Default::default(),
+                weighter: Arc::new(|_, _| 1),
+                event_listener: None,
+            });
+            let hints = vec![LfuHint];
+            fuzzy(cache, hints);
+        }
     }
 }

From f9ac2f95ecd30b957665b732b2da9e8622d8310f Mon Sep 17 00:00:00 2001
From: Croxx <mrcroxx@outlook.com>
Date: Tue, 19 Nov 2024 17:23:03 +0800
Subject: [PATCH 43/53] refactor: wrapper no/immutable/mutable op into Op
 (#788)

Signed-off-by: MrCroxx <mrcroxx@outlook.com>
---
 foyer-memory/src/eviction/fifo.rs       |  34 ++------
 foyer-memory/src/eviction/lfu.rs        | 109 ++++++++++--------------
 foyer-memory/src/eviction/lru.rs        |  98 ++++++++++-----------
 foyer-memory/src/eviction/mod.rs        |  77 +++++++++--------
 foyer-memory/src/eviction/s3fifo.rs     |  38 +++------
 foyer-memory/src/eviction/test_utils.rs |  42 ++++++++-
 foyer-memory/src/prelude.rs             |   2 +-
 foyer-memory/src/raw.rs                 |  56 +++++++-----
 8 files changed, 226 insertions(+), 230 deletions(-)

diff --git a/foyer-memory/src/eviction/fifo.rs b/foyer-memory/src/eviction/fifo.rs
index 680da0a9..bcc14188 100644
--- a/foyer-memory/src/eviction/fifo.rs
+++ b/foyer-memory/src/eviction/fifo.rs
@@ -18,7 +18,7 @@ use foyer_common::code::{Key, Value};
 use intrusive_collections::{intrusive_adapter, LinkedList, LinkedListAtomicLink};
 use serde::{Deserialize, Serialize};
 
-use super::{Eviction, Operator};
+use super::{Eviction, Op};
 use crate::record::{CacheHint, Record};
 
 /// Fifo eviction algorithm config.
@@ -93,28 +93,12 @@ where
         record.set_in_eviction(false);
     }
 
-    fn acquire_operator() -> super::Operator {
-        Operator::Noop
+    fn acquire() -> Op<Self> {
+        Op::noop()
     }
 
-    fn acquire_immutable(&self, _record: &Arc<Record<Self>>) {
-        unreachable!()
-    }
-
-    fn acquire_mutable(&mut self, _record: &Arc<Record<Self>>) {
-        unreachable!()
-    }
-
-    fn release_operator() -> super::Operator {
-        Operator::Noop
-    }
-
-    fn release_immutable(&self, _record: &Arc<Record<Self>>) {
-        unreachable!()
-    }
-
-    fn release_mutable(&mut self, _record: &Arc<Record<Self>>) {
-        unreachable!()
+    fn release() -> Op<Self> {
+        Op::noop()
     }
 }
 
@@ -125,17 +109,17 @@ pub mod tests {
 
     use super::*;
     use crate::{
-        eviction::test_utils::{assert_ptr_eq, assert_ptr_vec_eq, TestEviction},
+        eviction::test_utils::{assert_ptr_eq, assert_ptr_vec_eq, Dump},
         record::Data,
     };
 
-    impl<K, V> TestEviction for Fifo<K, V>
+    impl<K, V> Dump for Fifo<K, V>
     where
         K: Key + Clone,
         V: Value + Clone,
     {
-        type Dump = Vec<Arc<Record<Self>>>;
-        fn dump(&self) -> Self::Dump {
+        type Output = Vec<Arc<Record<Self>>>;
+        fn dump(&self) -> Self::Output {
             let mut res = vec![];
             let mut cursor = self.queue.cursor();
             loop {
diff --git a/foyer-memory/src/eviction/lfu.rs b/foyer-memory/src/eviction/lfu.rs
index 88e1e215..182f2f9f 100644
--- a/foyer-memory/src/eviction/lfu.rs
+++ b/foyer-memory/src/eviction/lfu.rs
@@ -22,7 +22,7 @@ use foyer_common::{
 use intrusive_collections::{intrusive_adapter, LinkedList, LinkedListAtomicLink};
 use serde::{Deserialize, Serialize};
 
-use super::{Eviction, Operator};
+use super::{Eviction, Op};
 use crate::record::{CacheHint, Record};
 
 /// w-TinyLFU eviction algorithm config.
@@ -347,71 +347,56 @@ where
         }
     }
 
-    fn acquire_operator() -> super::Operator {
-        // TODO(MrCroxx): use a count-min-sketch with atomic u16 impl.
-        Operator::Mutable
-    }
-
-    fn acquire_immutable(&self, _record: &Arc<Record<Self>>) {
-        unreachable!()
-    }
-
-    fn acquire_mutable(&mut self, record: &Arc<Record<Self>>) {
-        // Update frequency by access.
-        self.update_frequencies(record.hash());
+    fn acquire() -> Op<Self> {
+        Op::mutable(|this: &mut Self, record| {
+            // Update frequency by access.
+            this.update_frequencies(record.hash());
 
-        if !record.is_in_eviction() {
-            return;
-        }
+            if !record.is_in_eviction() {
+                return;
+            }
 
-        let state = unsafe { &mut *record.state().get() };
+            let state = unsafe { &mut *record.state().get() };
 
-        strict_assert!(state.link.is_linked());
+            strict_assert!(state.link.is_linked());
 
-        match state.queue {
-            Queue::None => unreachable!(),
-            Queue::Window => {
-                // Move to MRU position of `window`.
-                let r = unsafe { self.window.remove_from_ptr(Arc::as_ptr(record)) };
-                self.window.push_back(r);
-            }
-            Queue::Probation => {
-                // Promote to MRU position of `protected`.
-                let r = unsafe { self.probation.remove_from_ptr(Arc::as_ptr(record)) };
-                self.decrease_queue_weight(Queue::Probation, record.weight());
-                state.queue = Queue::Protected;
-                self.increase_queue_weight(Queue::Protected, record.weight());
-                self.protected.push_back(r);
-
-                // If `protected` weight exceeds the capacity, overflow entry from `protected` to `probation`.
-                while self.protected_weight > self.protected_weight_capacity {
-                    strict_assert!(!self.protected.is_empty());
-                    let r = self.protected.pop_front().unwrap();
-                    let s = unsafe { &mut *r.state().get() };
-                    self.decrease_queue_weight(Queue::Protected, r.weight());
-                    s.queue = Queue::Probation;
-                    self.increase_queue_weight(Queue::Probation, r.weight());
-                    self.probation.push_back(r);
+            match state.queue {
+                Queue::None => unreachable!(),
+                Queue::Window => {
+                    // Move to MRU position of `window`.
+                    let r = unsafe { this.window.remove_from_ptr(Arc::as_ptr(record)) };
+                    this.window.push_back(r);
+                }
+                Queue::Probation => {
+                    // Promote to MRU position of `protected`.
+                    let r = unsafe { this.probation.remove_from_ptr(Arc::as_ptr(record)) };
+                    this.decrease_queue_weight(Queue::Probation, record.weight());
+                    state.queue = Queue::Protected;
+                    this.increase_queue_weight(Queue::Protected, record.weight());
+                    this.protected.push_back(r);
+
+                    // If `protected` weight exceeds the capacity, overflow entry from `protected` to `probation`.
+                    while this.protected_weight > this.protected_weight_capacity {
+                        strict_assert!(!this.protected.is_empty());
+                        let r = this.protected.pop_front().unwrap();
+                        let s = unsafe { &mut *r.state().get() };
+                        this.decrease_queue_weight(Queue::Protected, r.weight());
+                        s.queue = Queue::Probation;
+                        this.increase_queue_weight(Queue::Probation, r.weight());
+                        this.probation.push_back(r);
+                    }
+                }
+                Queue::Protected => {
+                    // Move to MRU position of `protected`.
+                    let r = unsafe { this.protected.remove_from_ptr(Arc::as_ptr(record)) };
+                    this.protected.push_back(r);
                 }
             }
-            Queue::Protected => {
-                // Move to MRU position of `protected`.
-                let r = unsafe { self.protected.remove_from_ptr(Arc::as_ptr(record)) };
-                self.protected.push_back(r);
-            }
-        }
-    }
-
-    fn release_operator() -> Operator {
-        Operator::Noop
-    }
-
-    fn release_immutable(&self, _record: &Arc<Record<Self>>) {
-        unreachable!()
+        })
     }
 
-    fn release_mutable(&mut self, _record: &Arc<Record<Self>>) {
-        unreachable!()
+    fn release() -> Op<Self> {
+        Op::noop()
     }
 }
 
@@ -422,17 +407,17 @@ mod tests {
 
     use super::*;
     use crate::{
-        eviction::test_utils::{assert_ptr_eq, assert_ptr_vec_vec_eq, TestEviction},
+        eviction::test_utils::{assert_ptr_eq, assert_ptr_vec_vec_eq, Dump, OpExt},
         record::Data,
     };
 
-    impl<K, V> TestEviction for Lfu<K, V>
+    impl<K, V> Dump for Lfu<K, V>
     where
         K: Key + Clone,
         V: Value + Clone,
     {
-        type Dump = Vec<Vec<Arc<Record<Self>>>>;
-        fn dump(&self) -> Self::Dump {
+        type Output = Vec<Vec<Arc<Record<Self>>>>;
+        fn dump(&self) -> Self::Output {
             let mut window = vec![];
             let mut probation = vec![];
             let mut protected = vec![];
diff --git a/foyer-memory/src/eviction/lru.rs b/foyer-memory/src/eviction/lru.rs
index 74c77b11..d0a2fa80 100644
--- a/foyer-memory/src/eviction/lru.rs
+++ b/foyer-memory/src/eviction/lru.rs
@@ -21,7 +21,7 @@ use foyer_common::{
 use intrusive_collections::{intrusive_adapter, LinkedList, LinkedListAtomicLink};
 use serde::{Deserialize, Serialize};
 
-use super::{Eviction, Operator};
+use super::{Eviction, Op};
 use crate::record::{CacheHint, Record};
 
 /// Lru eviction algorithm config.
@@ -248,70 +248,58 @@ where
         assert_eq!(self.high_priority_weight, 0);
     }
 
-    fn acquire_operator() -> super::Operator {
-        Operator::Mutable
-    }
-
-    fn acquire_immutable(&self, _record: &Arc<Record<Self>>) {
-        unreachable!()
-    }
-
-    fn acquire_mutable(&mut self, record: &Arc<Record<Self>>) {
-        if !record.is_in_eviction() {
-            return;
-        }
-
-        let state = unsafe { &mut *record.state().get() };
-        assert!(state.link.is_linked());
+    fn acquire() -> Op<Self> {
+        Op::mutable(|this: &mut Self, record| {
+            if !record.is_in_eviction() {
+                return;
+            }
 
-        if state.is_pinned {
-            return;
-        }
+            let state = unsafe { &mut *record.state().get() };
+            assert!(state.link.is_linked());
 
-        // Pin the record by moving it to the pin list.
+            if state.is_pinned {
+                return;
+            }
 
-        let r = if state.in_high_priority_pool {
-            unsafe { self.high_priority_list.remove_from_ptr(Arc::as_ptr(record)) }
-        } else {
-            unsafe { self.list.remove_from_ptr(Arc::as_ptr(record)) }
-        };
+            // Pin the record by moving it to the pin list.
 
-        self.pin_list.push_back(r);
+            let r = if state.in_high_priority_pool {
+                unsafe { this.high_priority_list.remove_from_ptr(Arc::as_ptr(record)) }
+            } else {
+                unsafe { this.list.remove_from_ptr(Arc::as_ptr(record)) }
+            };
 
-        state.is_pinned = true;
-    }
+            this.pin_list.push_back(r);
 
-    fn release_operator() -> Operator {
-        Operator::Mutable
+            state.is_pinned = true;
+        })
     }
 
-    fn release_immutable(&self, _record: &Arc<Record<Self>>) {
-        unreachable!()
-    }
-
-    fn release_mutable(&mut self, record: &Arc<Record<Self>>) {
-        if !record.is_in_eviction() {
-            return;
-        }
+    fn release() -> Op<Self> {
+        Op::mutable(|this: &mut Self, record| {
+            if !record.is_in_eviction() {
+                return;
+            }
 
-        let state = unsafe { &mut *record.state().get() };
-        assert!(state.link.is_linked());
+            let state = unsafe { &mut *record.state().get() };
+            assert!(state.link.is_linked());
 
-        if !state.is_pinned {
-            return;
-        }
+            if !state.is_pinned {
+                return;
+            }
 
-        // Unpin the record by moving it from the pin list.
+            // Unpin the record by moving it from the pin list.
 
-        unsafe { self.pin_list.remove_from_ptr(Arc::as_ptr(record)) };
+            unsafe { this.pin_list.remove_from_ptr(Arc::as_ptr(record)) };
 
-        if state.in_high_priority_pool {
-            self.high_priority_list.push_back(record.clone());
-        } else {
-            self.list.push_back(record.clone());
-        }
+            if state.in_high_priority_pool {
+                this.high_priority_list.push_back(record.clone());
+            } else {
+                this.list.push_back(record.clone());
+            }
 
-        state.is_pinned = false;
+            state.is_pinned = false;
+        })
     }
 }
 
@@ -322,17 +310,17 @@ pub mod tests {
 
     use super::*;
     use crate::{
-        eviction::test_utils::{assert_ptr_eq, assert_ptr_vec_vec_eq, TestEviction},
+        eviction::test_utils::{assert_ptr_eq, assert_ptr_vec_vec_eq, Dump, OpExt},
         record::Data,
     };
 
-    impl<K, V> TestEviction for Lru<K, V>
+    impl<K, V> Dump for Lru<K, V>
     where
         K: Key + Clone,
         V: Value + Clone,
     {
-        type Dump = Vec<Vec<Arc<Record<Self>>>>;
-        fn dump(&self) -> Self::Dump {
+        type Output = Vec<Vec<Arc<Record<Self>>>>;
+        fn dump(&self) -> Self::Output {
             let mut low = vec![];
             let mut high = vec![];
             let mut pin = vec![];
diff --git a/foyer-memory/src/eviction/mod.rs b/foyer-memory/src/eviction/mod.rs
index 612de1a6..564b5008 100644
--- a/foyer-memory/src/eviction/mod.rs
+++ b/foyer-memory/src/eviction/mod.rs
@@ -28,10 +28,48 @@ impl<T> State for T where T: Send + Sync + 'static + Default {}
 pub trait Config: Send + Sync + 'static + Clone + Serialize + DeserializeOwned + Default {}
 impl<T> Config for T where T: Send + Sync + 'static + Clone + Serialize + DeserializeOwned + Default {}
 
-pub enum Operator {
+/// Wrapper for one of the three kind of operations for the eviction container:
+///
+/// 1. no operation
+/// 2. immutable operation
+/// 3. mutable operation
+#[expect(clippy::type_complexity)]
+pub enum Op<E>
+where
+    E: Eviction,
+{
+    /// no operation
     Noop,
-    Immutable,
-    Mutable,
+    /// immutable operation
+    Immutable(Box<dyn Fn(&E, &Arc<Record<E>>) + Send + Sync + 'static>),
+    /// mutable operation
+    Mutable(Box<dyn FnMut(&mut E, &Arc<Record<E>>) + Send + Sync + 'static>),
+}
+
+impl<E> Op<E>
+where
+    E: Eviction,
+{
+    /// no operation
+    pub fn noop() -> Self {
+        Self::Noop
+    }
+
+    /// immutable operation
+    pub fn immutable<F>(f: F) -> Self
+    where
+        F: Fn(&E, &Arc<Record<E>>) + Send + Sync + 'static,
+    {
+        Self::Immutable(Box::new(f))
+    }
+
+    /// mutable operation
+    pub fn mutable<F>(f: F) -> Self
+    where
+        F: FnMut(&mut E, &Arc<Record<E>>) + Send + Sync + 'static,
+    {
+        Self::Mutable(Box::new(f))
+    }
 }
 
 /// Cache eviction algorithm abstraction.
@@ -92,44 +130,15 @@ pub trait Eviction: Send + Sync + 'static + Sized {
         while self.pop().is_some() {}
     }
 
-    /// Determine if the immutable version or the mutable version to use for the `acquire` operation.
-    ///
-    /// Only the chosen version needs to be implemented. The other version is recommend to be left as `unreachable!()`.
-    fn acquire_operator() -> Operator;
-
-    /// Immutable version of the `acquire` operation.
-    ///
     /// `acquire` is called when an external caller acquire a cache entry from the cache.
     ///
     /// The entry can be EITHER in the cache eviction algorithm instance or not.
-    fn acquire_immutable(&self, record: &Arc<Record<Self>>);
+    fn acquire() -> Op<Self>;
 
-    /// Mutable version of the `acquire` operation.
-    ///
-    /// `acquire` is called when an external caller acquire a cache entry from the cache.
-    ///
-    /// The entry can be EITHER in the cache eviction algorithm instance or not.
-    fn acquire_mutable(&mut self, records: &Arc<Record<Self>>);
-
-    /// Determine if the immutable version or the mutable version to use for the `release` operation.
-    ///
-    /// Only the chosen version needs to be implemented. The other version is recommend to be left as `unreachable!()`.
-    fn release_operator() -> Operator;
-
-    /// Immutable version of the `release` operation.
-    ///
-    ///
-    /// `release` is called when the last external caller drops the cache entry.
-    ///
-    /// The entry can be EITHER in the cache eviction algorithm instance or not.
-    fn release_immutable(&self, record: &Arc<Record<Self>>);
-
-    /// Mutable version of the `release` operation.
-    ///
     /// `release` is called when the last external caller drops the cache entry.
     ///
     /// The entry can be EITHER in the cache eviction algorithm instance or not.
-    fn release_mutable(&mut self, record: &Arc<Record<Self>>);
+    fn release() -> Op<Self>;
 }
 
 pub mod fifo;
diff --git a/foyer-memory/src/eviction/s3fifo.rs b/foyer-memory/src/eviction/s3fifo.rs
index 0eb281aa..c32a1dc4 100644
--- a/foyer-memory/src/eviction/s3fifo.rs
+++ b/foyer-memory/src/eviction/s3fifo.rs
@@ -28,7 +28,7 @@ use foyer_common::{
 use intrusive_collections::{intrusive_adapter, LinkedList, LinkedListAtomicLink};
 use serde::{Deserialize, Serialize};
 
-use super::{Eviction, Operator};
+use super::{Eviction, Op};
 use crate::record::{CacheHint, Record};
 
 /// S3Fifo eviction algorithm config.
@@ -293,29 +293,15 @@ where
         }
     }
 
-    fn acquire_operator() -> super::Operator {
-        Operator::Immutable
-    }
-
-    fn acquire_immutable(&self, record: &Arc<Record<Self>>) {
-        let state = unsafe { &mut *record.state().get() };
-        state.inc_frequency();
-    }
-
-    fn acquire_mutable(&mut self, _record: &Arc<Record<Self>>) {
-        unreachable!()
-    }
-
-    fn release_operator() -> Operator {
-        Operator::Noop
-    }
-
-    fn release_immutable(&self, _record: &Arc<Record<Self>>) {
-        unreachable!()
+    fn acquire() -> Op<Self> {
+        Op::immutable(|_: &Self, record| {
+            let state = unsafe { &mut *record.state().get() };
+            state.inc_frequency();
+        })
     }
 
-    fn release_mutable(&mut self, _record: &Arc<Record<Self>>) {
-        unreachable!()
+    fn release() -> Op<Self> {
+        Op::noop()
     }
 }
 
@@ -379,18 +365,18 @@ mod tests {
 
     use super::*;
     use crate::{
-        eviction::test_utils::{assert_ptr_eq, assert_ptr_vec_vec_eq, TestEviction},
+        eviction::test_utils::{assert_ptr_eq, assert_ptr_vec_vec_eq, Dump, OpExt},
         record::Data,
     };
 
-    impl<K, V> TestEviction for S3Fifo<K, V>
+    impl<K, V> Dump for S3Fifo<K, V>
     where
         K: Key + Clone,
         V: Value + Clone,
     {
-        type Dump = Vec<Vec<Arc<Record<Self>>>>;
+        type Output = Vec<Vec<Arc<Record<Self>>>>;
 
-        fn dump(&self) -> Self::Dump {
+        fn dump(&self) -> Self::Output {
             let mut small = vec![];
             let mut main = vec![];
 
diff --git a/foyer-memory/src/eviction/test_utils.rs b/foyer-memory/src/eviction/test_utils.rs
index a1d6f387..dfc18c4e 100644
--- a/foyer-memory/src/eviction/test_utils.rs
+++ b/foyer-memory/src/eviction/test_utils.rs
@@ -16,11 +16,45 @@ use std::sync::Arc;
 
 use itertools::Itertools;
 
-use super::Eviction;
+use super::{Eviction, Op};
+use crate::Record;
 
-pub trait TestEviction: Eviction {
-    type Dump;
-    fn dump(&self) -> Self::Dump;
+#[expect(dead_code)]
+pub trait OpExt: Eviction {
+    fn acquire_immutable(&self, record: &Arc<Record<Self>>) {
+        match Self::acquire() {
+            Op::Immutable(f) => f(self, record),
+            _ => unreachable!(),
+        }
+    }
+
+    fn acquire_mutable(&mut self, record: &Arc<Record<Self>>) {
+        match Self::acquire() {
+            Op::Mutable(mut f) => f(self, record),
+            _ => unreachable!(),
+        }
+    }
+
+    fn release_immutable(&self, record: &Arc<Record<Self>>) {
+        match Self::release() {
+            Op::Immutable(f) => f(self, record),
+            _ => unreachable!(),
+        }
+    }
+
+    fn release_mutable(&mut self, record: &Arc<Record<Self>>) {
+        match Self::release() {
+            Op::Mutable(mut f) => f(self, record),
+            _ => unreachable!(),
+        }
+    }
+}
+
+impl<E> OpExt for E where E: Eviction {}
+
+pub trait Dump: Eviction {
+    type Output;
+    fn dump(&self) -> Self::Output;
 }
 
 pub fn assert_ptr_eq<T>(a: &Arc<T>, b: &Arc<T>) {
diff --git a/foyer-memory/src/prelude.rs b/foyer-memory/src/prelude.rs
index 3ce516b4..4e36ee44 100644
--- a/foyer-memory/src/prelude.rs
+++ b/foyer-memory/src/prelude.rs
@@ -16,7 +16,7 @@ pub use ahash::RandomState;
 
 pub use crate::{
     cache::{Cache, CacheBuilder, CacheEntry, EvictionConfig, Fetch},
-    eviction::{fifo::FifoConfig, lfu::LfuConfig, lru::LruConfig, s3fifo::S3FifoConfig},
+    eviction::{fifo::FifoConfig, lfu::LfuConfig, lru::LruConfig, s3fifo::S3FifoConfig, Eviction, Op},
     raw::{FetchMark, FetchState, Weighter},
     record::{CacheHint, Record},
 };
diff --git a/foyer-memory/src/raw.rs b/foyer-memory/src/raw.rs
index fe9d2e5f..45ba0a40 100644
--- a/foyer-memory/src/raw.rs
+++ b/foyer-memory/src/raw.rs
@@ -43,7 +43,7 @@ use pin_project::pin_project;
 use tokio::{sync::oneshot, task::JoinHandle};
 
 use crate::{
-    eviction::{Eviction, Operator},
+    eviction::{Eviction, Op},
     indexer::{hash_table::HashTableIndexer, sentry::Sentry, Indexer},
     record::{Data, Record},
 };
@@ -253,22 +253,34 @@ where
 
     #[fastrace::trace(name = "foyer::memory::raw::shard::acquire_immutable")]
     fn acquire_immutable(&self, record: &Arc<Record<E>>) {
-        self.eviction.acquire_immutable(record);
+        match E::acquire() {
+            Op::Immutable(f) => f(&self.eviction, record),
+            _ => unreachable!(),
+        }
     }
 
     #[fastrace::trace(name = "foyer::memory::raw::shard::acquire_mutable")]
     fn acquire_mutable(&mut self, record: &Arc<Record<E>>) {
-        self.eviction.acquire_mutable(record);
+        match E::acquire() {
+            Op::Mutable(mut f) => f(&mut self.eviction, record),
+            _ => unreachable!(),
+        }
     }
 
     #[fastrace::trace(name = "foyer::memory::raw::shard::release_immutable")]
     fn release_immutable(&self, record: &Arc<Record<E>>) {
-        self.eviction.release_immutable(record);
+        match E::release() {
+            Op::Immutable(f) => f(&self.eviction, record),
+            _ => unreachable!(),
+        }
     }
 
     #[fastrace::trace(name = "foyer::memory::raw::shard::release_mutable")]
     fn release_mutable(&mut self, record: &Arc<Record<E>>) {
-        self.eviction.release_mutable(record);
+        match E::release() {
+            Op::Mutable(mut f) => f(&mut self.eviction, record),
+            _ => unreachable!(),
+        }
     }
 
     #[fastrace::trace(name = "foyer::memory::raw::shard::fetch_noop")]
@@ -531,12 +543,12 @@ where
     {
         let hash = self.inner.hash_builder.hash_one(key);
 
-        let record = match E::acquire_operator() {
-            Operator::Noop => self.inner.shards[self.shard(hash)].read().get_noop(hash, key),
-            Operator::Immutable => self.inner.shards[self.shard(hash)]
+        let record = match E::acquire() {
+            Op::Noop => self.inner.shards[self.shard(hash)].read().get_noop(hash, key),
+            Op::Immutable(_) => self.inner.shards[self.shard(hash)]
                 .read()
                 .with(|shard| shard.get_immutable(hash, key)),
-            Operator::Mutable => self.inner.shards[self.shard(hash)]
+            Op::Mutable(_) => self.inner.shards[self.shard(hash)]
                 .write()
                 .with(|mut shard| shard.get_mutable(hash, key)),
         }?;
@@ -566,14 +578,12 @@ where
     {
         let hash = self.inner.hash_builder.hash_one(key);
 
-        match E::acquire_operator() {
-            Operator::Noop => self.inner.shards[self.shard(hash)]
-                .read()
-                .with(|shard| shard.get_noop(hash, key)),
-            Operator::Immutable => self.inner.shards[self.shard(hash)]
+        match E::acquire() {
+            Op::Noop => self.inner.shards[self.shard(hash)].read().get_noop(hash, key),
+            Op::Immutable(_) => self.inner.shards[self.shard(hash)]
                 .read()
                 .with(|shard| shard.get_immutable(hash, key)),
-            Operator::Mutable => self.inner.shards[self.shard(hash)]
+            Op::Mutable(_) => self.inner.shards[self.shard(hash)]
                 .write()
                 .with(|mut shard| shard.get_mutable(hash, key)),
         }
@@ -642,10 +652,10 @@ where
         let shard = &self.inner.shards[hash as usize % self.inner.shards.len()];
 
         if self.record.dec_refs(1) == 0 {
-            match E::release_operator() {
-                Operator::Noop => {}
-                Operator::Immutable => shard.read().with(|shard| shard.release_immutable(&self.record)),
-                Operator::Mutable => shard.write().with(|mut shard| shard.release_mutable(&self.record)),
+            match E::release() {
+                Op::Noop => {}
+                Op::Immutable(_) => shard.read().with(|shard| shard.release_immutable(&self.record)),
+                Op::Mutable(_) => shard.write().with(|mut shard| shard.release_mutable(&self.record)),
             }
 
             if self.record.is_ephemeral() {
@@ -870,10 +880,10 @@ where
     {
         let hash = self.inner.hash_builder.hash_one(&key);
 
-        let raw = match E::acquire_operator() {
-            Operator::Noop => self.inner.shards[self.shard(hash)].read().fetch_noop(hash, &key),
-            Operator::Immutable => self.inner.shards[self.shard(hash)].read().fetch_immutable(hash, &key),
-            Operator::Mutable => self.inner.shards[self.shard(hash)].write().fetch_mutable(hash, &key),
+        let raw = match E::acquire() {
+            Op::Noop => self.inner.shards[self.shard(hash)].read().fetch_noop(hash, &key),
+            Op::Immutable(_) => self.inner.shards[self.shard(hash)].read().fetch_immutable(hash, &key),
+            Op::Mutable(_) => self.inner.shards[self.shard(hash)].write().fetch_mutable(hash, &key),
         };
 
         match raw {

From 430045c580854d449f178389eae22ab9e0d0480f Mon Sep 17 00:00:00 2001
From: Croxx <mrcroxx@outlook.com>
Date: Sat, 23 Nov 2024 01:04:49 +0800
Subject: [PATCH 44/53] chore: update default grafana view to a mroe compacted
 layout (#781)

Signed-off-by: MrCroxx <mrcroxx@outlook.com>
---
 etc/grafana/dashboards/foyer.json | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/etc/grafana/dashboards/foyer.json b/etc/grafana/dashboards/foyer.json
index de81fb91..190288d7 100644
--- a/etc/grafana/dashboards/foyer.json
+++ b/etc/grafana/dashboards/foyer.json
@@ -1 +1 @@
-{"annotations":{"list":[{"builtIn":1,"datasource":{"type":"grafana","uid":"-- Grafana --"},"enable":true,"hide":true,"iconColor":"rgba(0, 211, 255, 1)","name":"Annotations & Alerts","type":"dashboard"}]},"editable":true,"fiscalYearStartMonth":0,"graphTooltip":1,"links":[],"liveNow":false,"panels":[{"gridPos":{"h":1,"w":24,"x":0,"y":0},"id":22,"title":"Hybrid","type":"row"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":1},"id":23,"options":{"legend":{"calcs":["lastNotNull"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"maxHeight":600,"mode":"multi","sort":"none"}},"targets":[{"datasource":{"type":"prometheus","uid":"a2641a73-8591-446b-9d69-7869ebf43899"},"editorMode":"code","expr":"sum(rate(foyer_hybrid_op_total[$__rate_interval])) by (name, op)","instant":false,"legendFormat":"{{name}} - hybrid - {{op}}","range":true,"refId":"A"}],"title":"Op","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":1},"id":24,"options":{"legend":{"calcs":["lastNotNull"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"maxHeight":600,"mode":"multi","sort":"none"}},"targets":[{"datasource":{"type":"prometheus","uid":"a2641a73-8591-446b-9d69-7869ebf43899"},"editorMode":"code","expr":"histogram_quantile(0.5, sum(rate(foyer_hybrid_op_duration_bucket[$__rate_interval])) by (le, name, op)) ","instant":false,"legendFormat":"p50 - {{name}} - hybrid - {{op}}","range":true,"refId":"A"},{"datasource":{"type":"prometheus","uid":"a2641a73-8591-446b-9d69-7869ebf43899"},"editorMode":"code","expr":"histogram_quantile(0.9, sum(rate(foyer_hybrid_op_duration_bucket[$__rate_interval])) by (le, name, op)) ","hide":false,"instant":false,"legendFormat":"p90 - {{name}} - hybrid - {{op}}","range":true,"refId":"B"},{"datasource":{"type":"prometheus","uid":"a2641a73-8591-446b-9d69-7869ebf43899"},"editorMode":"code","expr":"histogram_quantile(0.99, sum(rate(foyer_hybrid_op_duration_bucket[$__rate_interval])) by (le, name, op)) ","hide":false,"instant":false,"legendFormat":"p99 - {{name}} - hybrid - {{op}}","range":true,"refId":"C"},{"datasource":{"type":"prometheus","uid":"a2641a73-8591-446b-9d69-7869ebf43899"},"editorMode":"code","expr":"histogram_quantile(1.0, sum(rate(foyer_hybrid_op_duration_bucket[$__rate_interval])) by (le, name, op)) ","hide":false,"instant":false,"legendFormat":"pmax - {{name}} - hybrid - {{op}}","range":true,"refId":"D"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"editorMode":"code","expr":"sum(rate(foyer_hybrid_op_duration_sum[$__rate_interval])) by (le, name, op) / sum(rate(foyer_hybrid_op_duration_count[$__rate_interval])) by (le, name, op)","hide":false,"instant":false,"legendFormat":"pavg - {{name}} - hybrid - {{op}}","range":true,"refId":"E"}],"title":"Op Duration","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":9},"id":25,"options":{"legend":{"calcs":["lastNotNull"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"maxHeight":600,"mode":"multi","sort":"none"}},"targets":[{"datasource":{"type":"prometheus","uid":"a2641a73-8591-446b-9d69-7869ebf43899"},"editorMode":"code","expr":"sum(rate(foyer_hybrid_op_total{op=\"hit\"}[$__rate_interval])) by (name) / (sum(rate(foyer_hybrid_op_total{op=\"hit\"}[$__rate_interval])) by (name) + sum(rate(foyer_hybrid_op_total{op=\"miss\"}[$__rate_interval])) by (name)) ","instant":false,"legendFormat":"{{name}} - hybrid - hit ratio","range":true,"refId":"A"}],"title":"Hit Ratio","type":"timeseries"},{"gridPos":{"h":1,"w":24,"x":0,"y":17},"id":14,"title":"Memory","type":"row"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":18},"id":13,"options":{"legend":{"calcs":["lastNotNull"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"maxHeight":600,"mode":"multi","sort":"none"}},"targets":[{"datasource":{"type":"prometheus","uid":"a2641a73-8591-446b-9d69-7869ebf43899"},"editorMode":"code","expr":"sum(rate(foyer_memory_op_total[$__rate_interval])) by (name, op)","instant":false,"legendFormat":"{{name}} - memory - {{op}}","range":true,"refId":"A"}],"title":"Op","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"decbytes"},"overrides":[{"matcher":{"id":"byFrameRefID","options":"B"},"properties":[{"id":"unit"}]}]},"gridPos":{"h":8,"w":12,"x":12,"y":18},"id":15,"options":{"legend":{"calcs":["lastNotNull"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"maxHeight":600,"mode":"multi","sort":"none"}},"targets":[{"datasource":{"type":"prometheus","uid":"a2641a73-8591-446b-9d69-7869ebf43899"},"editorMode":"code","expr":"sum(foyer_memory_usage) by (name)","instant":false,"legendFormat":"{{name}} - memory - usage (bytes)","range":true,"refId":"A"},{"datasource":{"type":"prometheus","uid":"a2641a73-8591-446b-9d69-7869ebf43899"},"editorMode":"code","expr":"sum(foyer_memory_usage) by (name)","hide":true,"instant":false,"legendFormat":"{{name}} - memory - usage (count)","range":true,"refId":"B"}],"title":"Usage","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":26},"id":7,"options":{"legend":{"calcs":["lastNotNull"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"maxHeight":600,"mode":"multi","sort":"none"}},"targets":[{"datasource":{"type":"prometheus","uid":"a2641a73-8591-446b-9d69-7869ebf43899"},"editorMode":"code","expr":"sum(rate(foyer_memory_op_total{op=\"hit\"}[$__rate_interval])) by (name) / (sum(rate(foyer_memory_op_total{op=\"hit\"}[$__rate_interval])) by (name) + sum(rate(foyer_memory_op_total{op=\"miss\"}[$__rate_interval])) by (name)) ","instant":false,"legendFormat":"{{name}} - memory - hit ratio","range":true,"refId":"A"}],"title":"Hit Ratio","type":"timeseries"},{"collapsed":false,"gridPos":{"h":1,"w":24,"x":0,"y":34},"id":8,"panels":[],"title":"Storage","type":"row"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":35},"id":1,"options":{"legend":{"calcs":["lastNotNull"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"maxHeight":600,"mode":"multi","sort":"none"}},"targets":[{"datasource":{"type":"prometheus","uid":"a2641a73-8591-446b-9d69-7869ebf43899"},"editorMode":"code","expr":"sum(rate(foyer_storage_op_total[$__rate_interval])) by (name, op)","instant":false,"legendFormat":"{{name}} - storage - {{op}}","range":true,"refId":"A"}],"title":"Op","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":35},"id":2,"options":{"legend":{"calcs":["lastNotNull"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"maxHeight":600,"mode":"multi","sort":"none"}},"targets":[{"datasource":{"type":"prometheus","uid":"a2641a73-8591-446b-9d69-7869ebf43899"},"editorMode":"code","expr":"sum(rate(foyer_storage_inner_op_total[$__rate_interval])) by (name, op)","instant":false,"legendFormat":"{{name}} - storage - {{op}}","range":true,"refId":"A"}],"title":"Inner Op","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":43},"id":16,"options":{"legend":{"calcs":["lastNotNull"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"maxHeight":600,"mode":"multi","sort":"none"}},"targets":[{"datasource":{"type":"prometheus","uid":"a2641a73-8591-446b-9d69-7869ebf43899"},"editorMode":"code","expr":"histogram_quantile(0.5, sum(rate(foyer_storage_op_duration_bucket[$__rate_interval])) by (le, name, op)) ","instant":false,"legendFormat":"p50 - {{name}} - storage - {{op}}","range":true,"refId":"A"},{"datasource":{"type":"prometheus","uid":"a2641a73-8591-446b-9d69-7869ebf43899"},"editorMode":"code","expr":"histogram_quantile(0.9, sum(rate(foyer_storage_op_duration_bucket[$__rate_interval])) by (le, name, op)) ","hide":false,"instant":false,"legendFormat":"p90 - {{name}} - storage - {{op}}","range":true,"refId":"B"},{"datasource":{"type":"prometheus","uid":"a2641a73-8591-446b-9d69-7869ebf43899"},"editorMode":"code","expr":"histogram_quantile(0.99, sum(rate(foyer_storage_op_duration_bucket[$__rate_interval])) by (le, name, op)) ","hide":false,"instant":false,"legendFormat":"p99 - {{name}} - storage - {{op}}","range":true,"refId":"C"},{"datasource":{"type":"prometheus","uid":"a2641a73-8591-446b-9d69-7869ebf43899"},"editorMode":"code","expr":"histogram_quantile(1.0, sum(rate(foyer_storage_op_duration_bucket[$__rate_interval])) by (le, name, op)) ","hide":false,"instant":false,"legendFormat":"pmax - {{name}} - storage - {{op}}","range":true,"refId":"D"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"editorMode":"code","expr":"sum(rate(foyer_storage_op_duration_sum[$__rate_interval])) by (le, name, op) / sum(rate(foyer_storage_op_duration_count[$__rate_interval])) by (le, name, op)","hide":false,"instant":false,"legendFormat":"pavg - {{name}} - storage - {{op}}","range":true,"refId":"E"}],"title":"Op Duration","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":43},"id":17,"options":{"legend":{"calcs":["lastNotNull"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"maxHeight":600,"mode":"multi","sort":"none"}},"targets":[{"datasource":{"type":"prometheus","uid":"a2641a73-8591-446b-9d69-7869ebf43899"},"editorMode":"code","expr":"histogram_quantile(0.5, sum(rate(foyer_storage_inner_op_duration_bucket[$__rate_interval])) by (le, name, op)) ","instant":false,"legendFormat":"p50 - {{name}} - storage - {{op}}","range":true,"refId":"A"},{"datasource":{"type":"prometheus","uid":"a2641a73-8591-446b-9d69-7869ebf43899"},"editorMode":"code","expr":"histogram_quantile(0.9, sum(rate(foyer_storage_inner_op_duration_bucket[$__rate_interval])) by (le, name, op)) ","hide":false,"instant":false,"legendFormat":"p90 - {{name}} - storage - {{op}}","range":true,"refId":"B"},{"datasource":{"type":"prometheus","uid":"a2641a73-8591-446b-9d69-7869ebf43899"},"editorMode":"code","expr":"histogram_quantile(0.99, sum(rate(foyer_storage_inner_op_duration_bucket[$__rate_interval])) by (le, name, op)) ","hide":false,"instant":false,"legendFormat":"p99 - {{name}} - storage - {{op}}","range":true,"refId":"C"},{"datasource":{"type":"prometheus","uid":"a2641a73-8591-446b-9d69-7869ebf43899"},"editorMode":"code","expr":"histogram_quantile(1.0, sum(rate(foyer_storage_inner_op_duration_bucket[$__rate_interval])) by (le, name, op)) ","hide":false,"instant":false,"legendFormat":"pmax - {{name}} - storage - {{op}}","range":true,"refId":"D"},{"datasource":{"type":"prometheus","uid":"a2641a73-8591-446b-9d69-7869ebf43899"},"editorMode":"code","expr":"sum(rate(foyer_storage_inner_op_duration_sum[$__rate_interval])) by (le, name, op) / sum(rate(foyer_storage_inner_op_duration_count[$__rate_interval])) by (le, name, op)","hide":false,"instant":false,"legendFormat":"pavg - {{name}} - storage - {{op}}","range":true,"refId":"E"}],"title":"Inner Op Duration","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]}},"overrides":[{"matcher":{"id":"byFrameRefID","options":"B"},"properties":[{"id":"unit"}]}]},"gridPos":{"h":8,"w":12,"x":0,"y":51},"id":27,"options":{"legend":{"calcs":["lastNotNull"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"maxHeight":600,"mode":"multi","sort":"none"}},"targets":[{"datasource":{"type":"prometheus","uid":"a2641a73-8591-446b-9d69-7869ebf43899"},"editorMode":"code","expr":"sum(foyer_storage_region) by (name, type)","instant":false,"legendFormat":"{{name}} - region - {{type}}","range":true,"refId":"A"}],"title":"Region Count","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"decbytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":51},"id":28,"options":{"legend":{"calcs":["lastNotNull"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"maxHeight":600,"mode":"multi","sort":"none"}},"targets":[{"datasource":{"type":"prometheus","uid":"a2641a73-8591-446b-9d69-7869ebf43899"},"editorMode":"code","expr":"sum(foyer_storage_region) by (name, type) * on(name) group_left() foyer_storage_region_size_bytes","instant":false,"legendFormat":"{{name}} - region - {{type}}","range":true,"refId":"A"}],"title":"Region Size","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":59},"id":18,"options":{"legend":{"calcs":["lastNotNull"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"maxHeight":600,"mode":"multi","sort":"none"}},"targets":[{"datasource":{"type":"prometheus","uid":"a2641a73-8591-446b-9d69-7869ebf43899"},"editorMode":"code","expr":"sum(rate(foyer_storage_op_total{op=\"hit\"}[$__rate_interval])) by (name) / (sum(rate(foyer_storage_op_total{op=\"hit\"}[$__rate_interval])) by (name) + sum(rate(foyer_storage_op_total{op=\"miss\"}[$__rate_interval])) by (name)) ","instant":false,"legendFormat":"{{name}} - storage - hit ratio","range":true,"refId":"A"}],"title":"Hit Ratio","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":59},"id":29,"options":{"legend":{"calcs":["lastNotNull"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"maxHeight":600,"mode":"multi","sort":"none"}},"targets":[{"datasource":{"type":"prometheus","uid":"a2641a73-8591-446b-9d69-7869ebf43899"},"editorMode":"code","expr":"histogram_quantile(0.5, sum(rate(foyer_storage_entry_serde_duration_bucket[$__rate_interval])) by (le, name, op))","instant":false,"legendFormat":"p50 - {{name}} - storage - {{op}}","range":true,"refId":"A"},{"datasource":{"type":"prometheus","uid":"a2641a73-8591-446b-9d69-7869ebf43899"},"editorMode":"code","expr":"histogram_quantile(0.9, sum(rate(foyer_storage_entry_serde_duration_bucket[$__rate_interval])) by (le, name, op))","hide":false,"instant":false,"legendFormat":"p90 - {{name}} - storage - {{op}}","range":true,"refId":"B"},{"datasource":{"type":"prometheus","uid":"a2641a73-8591-446b-9d69-7869ebf43899"},"editorMode":"code","expr":"histogram_quantile(0.99, sum(rate(foyer_storage_entry_serde_duration_bucket[$__rate_interval])) by (le, name, op))","hide":false,"instant":false,"legendFormat":"p99 - {{name}} - storage - {{op}}","range":true,"refId":"C"},{"datasource":{"type":"prometheus","uid":"a2641a73-8591-446b-9d69-7869ebf43899"},"editorMode":"code","expr":"histogram_quantile(1.0, sum(rate(foyer_storage_entry_serde_duration_bucket[$__rate_interval])) by (le, name, op))","hide":false,"instant":false,"legendFormat":"pmax - {{name}} - storage - {{op}}","range":true,"refId":"D"},{"datasource":{"type":"prometheus","uid":"a2641a73-8591-446b-9d69-7869ebf43899"},"editorMode":"code","expr":"sum(rate(foyer_storage_entry_serde_duration_sum[$__rate_interval])) by (le, name, op) / sum(rate(foyer_storage_entry_serde_duration_count[$__rate_interval])) by (le, name, op)","hide":false,"instant":false,"legendFormat":"pavg - {{name}} - storage - {{op}}","range":true,"refId":"E"}],"title":"Serde Duration","type":"timeseries"},{"collapsed":false,"gridPos":{"h":1,"w":24,"x":0,"y":67},"id":19,"panels":[],"title":"Storage (Disk)","type":"row"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":68},"id":20,"options":{"legend":{"calcs":["lastNotNull"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"maxHeight":600,"mode":"multi","sort":"none"}},"targets":[{"datasource":{"type":"prometheus","uid":"a2641a73-8591-446b-9d69-7869ebf43899"},"editorMode":"code","expr":"sum(rate(foyer_storage_disk_io_total[$__rate_interval])) by (name, op)","instant":false,"legendFormat":"{{name}} - disk io - {{op}}","range":true,"refId":"A"}],"title":"Disk IO","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":68},"id":21,"options":{"legend":{"calcs":["lastNotNull"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"maxHeight":600,"mode":"multi","sort":"none"}},"targets":[{"datasource":{"type":"prometheus","uid":"a2641a73-8591-446b-9d69-7869ebf43899"},"editorMode":"code","expr":"histogram_quantile(0.5, sum(rate(foyer_storage_disk_io_duration_bucket[$__rate_interval])) by (le, name, op)) ","instant":false,"legendFormat":"p50 - {{name}} - disk io - {{op}}","range":true,"refId":"A"},{"datasource":{"type":"prometheus","uid":"a2641a73-8591-446b-9d69-7869ebf43899"},"editorMode":"code","expr":"histogram_quantile(0.9, sum(rate(foyer_storage_disk_io_duration_bucket[$__rate_interval])) by (le, name, op)) ","hide":false,"instant":false,"legendFormat":"p90 - {{name}} - disk io - {{op}}","range":true,"refId":"B"},{"datasource":{"type":"prometheus","uid":"a2641a73-8591-446b-9d69-7869ebf43899"},"editorMode":"code","expr":"histogram_quantile(0.99, sum(rate(foyer_storage_disk_io_duration_bucket[$__rate_interval])) by (le, name, op)) ","hide":false,"instant":false,"legendFormat":"p99 - {{name}} - disk io - {{op}}","range":true,"refId":"C"},{"datasource":{"type":"prometheus","uid":"a2641a73-8591-446b-9d69-7869ebf43899"},"editorMode":"code","expr":"histogram_quantile(1.0, sum(rate(foyer_storage_disk_io_duration_bucket[$__rate_interval])) by (le, name, op)) ","hide":false,"instant":false,"legendFormat":"pmax - {{name}} - disk io - {{op}}","range":true,"refId":"D"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"editorMode":"code","expr":"sum(rate(foyer_storage_disk_io_duration_sum[$__rate_interval])) by (le, name, op) / sum(rate(foyer_storage_disk_io_duration_count[$__rate_interval])) by (le, name, op)","hide":false,"instant":false,"legendFormat":"pavg - {{name}} - disk io - {{op}}","range":true,"refId":"E"}],"title":"Disk IO Duration","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":76},"id":5,"options":{"legend":{"calcs":["lastNotNull"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"maxHeight":600,"mode":"multi","sort":"none"}},"targets":[{"datasource":{"type":"prometheus","uid":"a2641a73-8591-446b-9d69-7869ebf43899"},"editorMode":"code","expr":"sum(rate(foyer_storage_disk_io_bytes[$__rate_interval])) by (foyer, op, extra) ","instant":false,"legendFormat":"{{foyer}} foyer storage - {{op}} {{extra}}","range":true,"refId":"A"}],"title":"Op Thoughput","type":"timeseries"}],"refresh":"5s","schemaVersion":39,"tags":[],"templating":{"list":[]},"time":{"from":"now-30m","to":"now"},"timepicker":{},"timezone":"","title":"foyer","uid":"f0e2058b-b292-457c-8ddf-9dbdf7c60035","version":1,"weekStart":""}
+{"annotations":{"list":[{"builtIn":1,"datasource":{"type":"grafana","uid":"-- Grafana --"},"enable":true,"hide":true,"iconColor":"rgba(0, 211, 255, 1)","name":"Annotations & Alerts","type":"dashboard"}]},"editable":true,"fiscalYearStartMonth":0,"graphTooltip":1,"links":[],"liveNow":false,"panels":[{"gridPos":{"h":1,"w":24,"x":0,"y":0},"id":22,"title":"Hybrid","type":"row"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"ops"},"overrides":[]},"gridPos":{"h":7,"w":8,"x":0,"y":1},"id":23,"options":{"legend":{"calcs":["lastNotNull"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"maxHeight":600,"mode":"multi","sort":"none"}},"targets":[{"datasource":{"type":"prometheus","uid":"a2641a73-8591-446b-9d69-7869ebf43899"},"editorMode":"code","expr":"sum(rate(foyer_hybrid_op_total[$__rate_interval])) by (name, op)","instant":false,"legendFormat":"{{name}} - hybrid - {{op}}","range":true,"refId":"A"}],"title":"Op","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"s"},"overrides":[]},"gridPos":{"h":7,"w":8,"x":8,"y":1},"id":24,"options":{"legend":{"calcs":["lastNotNull"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"maxHeight":600,"mode":"multi","sort":"none"}},"targets":[{"datasource":{"type":"prometheus","uid":"a2641a73-8591-446b-9d69-7869ebf43899"},"editorMode":"code","expr":"histogram_quantile(0.5, sum(rate(foyer_hybrid_op_duration_bucket[$__rate_interval])) by (le, name, op)) ","instant":false,"legendFormat":"p50 - {{name}} - hybrid - {{op}}","range":true,"refId":"A"},{"datasource":{"type":"prometheus","uid":"a2641a73-8591-446b-9d69-7869ebf43899"},"editorMode":"code","expr":"histogram_quantile(0.9, sum(rate(foyer_hybrid_op_duration_bucket[$__rate_interval])) by (le, name, op)) ","hide":false,"instant":false,"legendFormat":"p90 - {{name}} - hybrid - {{op}}","range":true,"refId":"B"},{"datasource":{"type":"prometheus","uid":"a2641a73-8591-446b-9d69-7869ebf43899"},"editorMode":"code","expr":"histogram_quantile(0.99, sum(rate(foyer_hybrid_op_duration_bucket[$__rate_interval])) by (le, name, op)) ","hide":false,"instant":false,"legendFormat":"p99 - {{name}} - hybrid - {{op}}","range":true,"refId":"C"},{"datasource":{"type":"prometheus","uid":"a2641a73-8591-446b-9d69-7869ebf43899"},"editorMode":"code","expr":"histogram_quantile(1.0, sum(rate(foyer_hybrid_op_duration_bucket[$__rate_interval])) by (le, name, op)) ","hide":false,"instant":false,"legendFormat":"pmax - {{name}} - hybrid - {{op}}","range":true,"refId":"D"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"editorMode":"code","expr":"sum(rate(foyer_hybrid_op_duration_sum[$__rate_interval])) by (le, name, op) / sum(rate(foyer_hybrid_op_duration_count[$__rate_interval])) by (le, name, op)","hide":false,"instant":false,"legendFormat":"pavg - {{name}} - hybrid - {{op}}","range":true,"refId":"E"}],"title":"Op Duration","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":7,"w":8,"x":16,"y":1},"id":25,"options":{"legend":{"calcs":["lastNotNull"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"maxHeight":600,"mode":"multi","sort":"none"}},"targets":[{"datasource":{"type":"prometheus","uid":"a2641a73-8591-446b-9d69-7869ebf43899"},"editorMode":"code","expr":"sum(rate(foyer_hybrid_op_total{op=\"hit\"}[$__rate_interval])) by (name) / (sum(rate(foyer_hybrid_op_total{op=\"hit\"}[$__rate_interval])) by (name) + sum(rate(foyer_hybrid_op_total{op=\"miss\"}[$__rate_interval])) by (name)) ","instant":false,"legendFormat":"{{name}} - hybrid - hit ratio","range":true,"refId":"A"}],"title":"Hit Ratio","type":"timeseries"},{"gridPos":{"h":1,"w":24,"x":0,"y":8},"id":14,"title":"Memory","type":"row"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"ops"},"overrides":[]},"gridPos":{"h":7,"w":8,"x":0,"y":9},"id":13,"options":{"legend":{"calcs":["lastNotNull"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"maxHeight":600,"mode":"multi","sort":"none"}},"targets":[{"datasource":{"type":"prometheus","uid":"a2641a73-8591-446b-9d69-7869ebf43899"},"editorMode":"code","expr":"sum(rate(foyer_memory_op_total[$__rate_interval])) by (name, op)","instant":false,"legendFormat":"{{name}} - memory - {{op}}","range":true,"refId":"A"}],"title":"Op","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"decbytes"},"overrides":[{"matcher":{"id":"byFrameRefID","options":"B"},"properties":[{"id":"unit"}]}]},"gridPos":{"h":7,"w":8,"x":8,"y":9},"id":15,"options":{"legend":{"calcs":["lastNotNull"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"maxHeight":600,"mode":"multi","sort":"none"}},"targets":[{"datasource":{"type":"prometheus","uid":"a2641a73-8591-446b-9d69-7869ebf43899"},"editorMode":"code","expr":"sum(foyer_memory_usage) by (name)","instant":false,"legendFormat":"{{name}} - memory - usage (bytes)","range":true,"refId":"A"},{"datasource":{"type":"prometheus","uid":"a2641a73-8591-446b-9d69-7869ebf43899"},"editorMode":"code","expr":"sum(foyer_memory_usage) by (name)","hide":true,"instant":false,"legendFormat":"{{name}} - memory - usage (count)","range":true,"refId":"B"}],"title":"Usage","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":7,"w":8,"x":16,"y":9},"id":7,"options":{"legend":{"calcs":["lastNotNull"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"maxHeight":600,"mode":"multi","sort":"none"}},"targets":[{"datasource":{"type":"prometheus","uid":"a2641a73-8591-446b-9d69-7869ebf43899"},"editorMode":"code","expr":"sum(rate(foyer_memory_op_total{op=\"hit\"}[$__rate_interval])) by (name) / (sum(rate(foyer_memory_op_total{op=\"hit\"}[$__rate_interval])) by (name) + sum(rate(foyer_memory_op_total{op=\"miss\"}[$__rate_interval])) by (name)) ","instant":false,"legendFormat":"{{name}} - memory - hit ratio","range":true,"refId":"A"}],"title":"Hit Ratio","type":"timeseries"},{"collapsed":false,"gridPos":{"h":1,"w":24,"x":0,"y":16},"id":8,"panels":[],"title":"Storage","type":"row"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"ops"},"overrides":[]},"gridPos":{"h":7,"w":8,"x":0,"y":17},"id":1,"options":{"legend":{"calcs":["lastNotNull"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"maxHeight":600,"mode":"multi","sort":"none"}},"targets":[{"datasource":{"type":"prometheus","uid":"a2641a73-8591-446b-9d69-7869ebf43899"},"editorMode":"code","expr":"sum(rate(foyer_storage_op_total[$__rate_interval])) by (name, op)","instant":false,"legendFormat":"{{name}} - storage - {{op}}","range":true,"refId":"A"}],"title":"Op","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"s"},"overrides":[]},"gridPos":{"h":7,"w":8,"x":8,"y":17},"id":16,"options":{"legend":{"calcs":["lastNotNull"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"maxHeight":600,"mode":"multi","sort":"none"}},"targets":[{"datasource":{"type":"prometheus","uid":"a2641a73-8591-446b-9d69-7869ebf43899"},"editorMode":"code","expr":"histogram_quantile(0.5, sum(rate(foyer_storage_op_duration_bucket[$__rate_interval])) by (le, name, op)) ","instant":false,"legendFormat":"p50 - {{name}} - storage - {{op}}","range":true,"refId":"A"},{"datasource":{"type":"prometheus","uid":"a2641a73-8591-446b-9d69-7869ebf43899"},"editorMode":"code","expr":"histogram_quantile(0.9, sum(rate(foyer_storage_op_duration_bucket[$__rate_interval])) by (le, name, op)) ","hide":false,"instant":false,"legendFormat":"p90 - {{name}} - storage - {{op}}","range":true,"refId":"B"},{"datasource":{"type":"prometheus","uid":"a2641a73-8591-446b-9d69-7869ebf43899"},"editorMode":"code","expr":"histogram_quantile(0.99, sum(rate(foyer_storage_op_duration_bucket[$__rate_interval])) by (le, name, op)) ","hide":false,"instant":false,"legendFormat":"p99 - {{name}} - storage - {{op}}","range":true,"refId":"C"},{"datasource":{"type":"prometheus","uid":"a2641a73-8591-446b-9d69-7869ebf43899"},"editorMode":"code","expr":"histogram_quantile(1.0, sum(rate(foyer_storage_op_duration_bucket[$__rate_interval])) by (le, name, op)) ","hide":false,"instant":false,"legendFormat":"pmax - {{name}} - storage - {{op}}","range":true,"refId":"D"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"editorMode":"code","expr":"sum(rate(foyer_storage_op_duration_sum[$__rate_interval])) by (le, name, op) / sum(rate(foyer_storage_op_duration_count[$__rate_interval])) by (le, name, op)","hide":false,"instant":false,"legendFormat":"pavg - {{name}} - storage - {{op}}","range":true,"refId":"E"}],"title":"Op Duration","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]}},"overrides":[{"matcher":{"id":"byFrameRefID","options":"B"},"properties":[{"id":"unit"}]}]},"gridPos":{"h":7,"w":8,"x":16,"y":17},"id":27,"options":{"legend":{"calcs":["lastNotNull"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"maxHeight":600,"mode":"multi","sort":"none"}},"targets":[{"datasource":{"type":"prometheus","uid":"a2641a73-8591-446b-9d69-7869ebf43899"},"editorMode":"code","expr":"sum(foyer_storage_region) by (name, type)","instant":false,"legendFormat":"{{name}} - region - {{type}}","range":true,"refId":"A"}],"title":"Region Count","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"ops"},"overrides":[]},"gridPos":{"h":7,"w":8,"x":0,"y":24},"id":2,"options":{"legend":{"calcs":["lastNotNull"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"maxHeight":600,"mode":"multi","sort":"none"}},"targets":[{"datasource":{"type":"prometheus","uid":"a2641a73-8591-446b-9d69-7869ebf43899"},"editorMode":"code","expr":"sum(rate(foyer_storage_inner_op_total[$__rate_interval])) by (name, op)","instant":false,"legendFormat":"{{name}} - storage - {{op}}","range":true,"refId":"A"}],"title":"Inner Op","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"s"},"overrides":[]},"gridPos":{"h":7,"w":8,"x":8,"y":24},"id":17,"options":{"legend":{"calcs":["lastNotNull"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"maxHeight":600,"mode":"multi","sort":"none"}},"targets":[{"datasource":{"type":"prometheus","uid":"a2641a73-8591-446b-9d69-7869ebf43899"},"editorMode":"code","expr":"histogram_quantile(0.5, sum(rate(foyer_storage_inner_op_duration_bucket[$__rate_interval])) by (le, name, op)) ","instant":false,"legendFormat":"p50 - {{name}} - storage - {{op}}","range":true,"refId":"A"},{"datasource":{"type":"prometheus","uid":"a2641a73-8591-446b-9d69-7869ebf43899"},"editorMode":"code","expr":"histogram_quantile(0.9, sum(rate(foyer_storage_inner_op_duration_bucket[$__rate_interval])) by (le, name, op)) ","hide":false,"instant":false,"legendFormat":"p90 - {{name}} - storage - {{op}}","range":true,"refId":"B"},{"datasource":{"type":"prometheus","uid":"a2641a73-8591-446b-9d69-7869ebf43899"},"editorMode":"code","expr":"histogram_quantile(0.99, sum(rate(foyer_storage_inner_op_duration_bucket[$__rate_interval])) by (le, name, op)) ","hide":false,"instant":false,"legendFormat":"p99 - {{name}} - storage - {{op}}","range":true,"refId":"C"},{"datasource":{"type":"prometheus","uid":"a2641a73-8591-446b-9d69-7869ebf43899"},"editorMode":"code","expr":"histogram_quantile(1.0, sum(rate(foyer_storage_inner_op_duration_bucket[$__rate_interval])) by (le, name, op)) ","hide":false,"instant":false,"legendFormat":"pmax - {{name}} - storage - {{op}}","range":true,"refId":"D"},{"datasource":{"type":"prometheus","uid":"a2641a73-8591-446b-9d69-7869ebf43899"},"editorMode":"code","expr":"sum(rate(foyer_storage_inner_op_duration_sum[$__rate_interval])) by (le, name, op) / sum(rate(foyer_storage_inner_op_duration_count[$__rate_interval])) by (le, name, op)","hide":false,"instant":false,"legendFormat":"pavg - {{name}} - storage - {{op}}","range":true,"refId":"E"}],"title":"Inner Op Duration","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"decbytes"},"overrides":[]},"gridPos":{"h":7,"w":8,"x":16,"y":24},"id":28,"options":{"legend":{"calcs":["lastNotNull"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"maxHeight":600,"mode":"multi","sort":"none"}},"targets":[{"datasource":{"type":"prometheus","uid":"a2641a73-8591-446b-9d69-7869ebf43899"},"editorMode":"code","expr":"sum(foyer_storage_region) by (name, type) * on(name) group_left() foyer_storage_region_size_bytes","instant":false,"legendFormat":"{{name}} - region - {{type}}","range":true,"refId":"A"}],"title":"Region Size","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":7,"w":8,"x":0,"y":31},"id":18,"options":{"legend":{"calcs":["lastNotNull"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"maxHeight":600,"mode":"multi","sort":"none"}},"targets":[{"datasource":{"type":"prometheus","uid":"a2641a73-8591-446b-9d69-7869ebf43899"},"editorMode":"code","expr":"sum(rate(foyer_storage_op_total{op=\"hit\"}[$__rate_interval])) by (name) / (sum(rate(foyer_storage_op_total{op=\"hit\"}[$__rate_interval])) by (name) + sum(rate(foyer_storage_op_total{op=\"miss\"}[$__rate_interval])) by (name)) ","instant":false,"legendFormat":"{{name}} - storage - hit ratio","range":true,"refId":"A"}],"title":"Hit Ratio","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"s"},"overrides":[]},"gridPos":{"h":7,"w":8,"x":8,"y":31},"id":29,"options":{"legend":{"calcs":["lastNotNull"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"maxHeight":600,"mode":"multi","sort":"none"}},"targets":[{"datasource":{"type":"prometheus","uid":"a2641a73-8591-446b-9d69-7869ebf43899"},"editorMode":"code","expr":"histogram_quantile(0.5, sum(rate(foyer_storage_entry_serde_duration_bucket[$__rate_interval])) by (le, name, op))","instant":false,"legendFormat":"p50 - {{name}} - storage - {{op}}","range":true,"refId":"A"},{"datasource":{"type":"prometheus","uid":"a2641a73-8591-446b-9d69-7869ebf43899"},"editorMode":"code","expr":"histogram_quantile(0.9, sum(rate(foyer_storage_entry_serde_duration_bucket[$__rate_interval])) by (le, name, op))","hide":false,"instant":false,"legendFormat":"p90 - {{name}} - storage - {{op}}","range":true,"refId":"B"},{"datasource":{"type":"prometheus","uid":"a2641a73-8591-446b-9d69-7869ebf43899"},"editorMode":"code","expr":"histogram_quantile(0.99, sum(rate(foyer_storage_entry_serde_duration_bucket[$__rate_interval])) by (le, name, op))","hide":false,"instant":false,"legendFormat":"p99 - {{name}} - storage - {{op}}","range":true,"refId":"C"},{"datasource":{"type":"prometheus","uid":"a2641a73-8591-446b-9d69-7869ebf43899"},"editorMode":"code","expr":"histogram_quantile(1.0, sum(rate(foyer_storage_entry_serde_duration_bucket[$__rate_interval])) by (le, name, op))","hide":false,"instant":false,"legendFormat":"pmax - {{name}} - storage - {{op}}","range":true,"refId":"D"},{"datasource":{"type":"prometheus","uid":"a2641a73-8591-446b-9d69-7869ebf43899"},"editorMode":"code","expr":"sum(rate(foyer_storage_entry_serde_duration_sum[$__rate_interval])) by (le, name, op) / sum(rate(foyer_storage_entry_serde_duration_count[$__rate_interval])) by (le, name, op)","hide":false,"instant":false,"legendFormat":"pavg - {{name}} - storage - {{op}}","range":true,"refId":"E"}],"title":"Serde Duration","type":"timeseries"},{"collapsed":false,"gridPos":{"h":1,"w":24,"x":0,"y":38},"id":19,"panels":[],"title":"Storage (Disk)","type":"row"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"ops"},"overrides":[]},"gridPos":{"h":7,"w":8,"x":0,"y":39},"id":20,"options":{"legend":{"calcs":["lastNotNull"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"maxHeight":600,"mode":"multi","sort":"none"}},"targets":[{"datasource":{"type":"prometheus","uid":"a2641a73-8591-446b-9d69-7869ebf43899"},"editorMode":"code","expr":"sum(rate(foyer_storage_disk_io_total[$__rate_interval])) by (name, op)","instant":false,"legendFormat":"{{name}} - disk io - {{op}}","range":true,"refId":"A"}],"title":"Disk IO","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"s"},"overrides":[]},"gridPos":{"h":7,"w":8,"x":8,"y":39},"id":21,"options":{"legend":{"calcs":["lastNotNull"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"maxHeight":600,"mode":"multi","sort":"none"}},"targets":[{"datasource":{"type":"prometheus","uid":"a2641a73-8591-446b-9d69-7869ebf43899"},"editorMode":"code","expr":"histogram_quantile(0.5, sum(rate(foyer_storage_disk_io_duration_bucket[$__rate_interval])) by (le, name, op)) ","instant":false,"legendFormat":"p50 - {{name}} - disk io - {{op}}","range":true,"refId":"A"},{"datasource":{"type":"prometheus","uid":"a2641a73-8591-446b-9d69-7869ebf43899"},"editorMode":"code","expr":"histogram_quantile(0.9, sum(rate(foyer_storage_disk_io_duration_bucket[$__rate_interval])) by (le, name, op)) ","hide":false,"instant":false,"legendFormat":"p90 - {{name}} - disk io - {{op}}","range":true,"refId":"B"},{"datasource":{"type":"prometheus","uid":"a2641a73-8591-446b-9d69-7869ebf43899"},"editorMode":"code","expr":"histogram_quantile(0.99, sum(rate(foyer_storage_disk_io_duration_bucket[$__rate_interval])) by (le, name, op)) ","hide":false,"instant":false,"legendFormat":"p99 - {{name}} - disk io - {{op}}","range":true,"refId":"C"},{"datasource":{"type":"prometheus","uid":"a2641a73-8591-446b-9d69-7869ebf43899"},"editorMode":"code","expr":"histogram_quantile(1.0, sum(rate(foyer_storage_disk_io_duration_bucket[$__rate_interval])) by (le, name, op)) ","hide":false,"instant":false,"legendFormat":"pmax - {{name}} - disk io - {{op}}","range":true,"refId":"D"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"editorMode":"code","expr":"sum(rate(foyer_storage_disk_io_duration_sum[$__rate_interval])) by (le, name, op) / sum(rate(foyer_storage_disk_io_duration_count[$__rate_interval])) by (le, name, op)","hide":false,"instant":false,"legendFormat":"pavg - {{name}} - disk io - {{op}}","range":true,"refId":"E"}],"title":"Disk IO Duration","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P92AEBB27A9B79E22"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":7,"w":8,"x":16,"y":39},"id":5,"options":{"legend":{"calcs":["lastNotNull"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"maxHeight":600,"mode":"multi","sort":"none"}},"targets":[{"datasource":{"type":"prometheus","uid":"a2641a73-8591-446b-9d69-7869ebf43899"},"editorMode":"code","expr":"sum(rate(foyer_storage_disk_io_bytes[$__rate_interval])) by (foyer, op, extra) ","instant":false,"legendFormat":"{{foyer}} foyer storage - {{op}} {{extra}}","range":true,"refId":"A"}],"title":"Op Thoughput","type":"timeseries"}],"refresh":"5s","schemaVersion":39,"tags":[],"templating":{"list":[]},"time":{"from":"now-5m","to":"now"},"timepicker":{},"timezone":"","title":"foyer","uid":"f0e2058b-b292-457c-8ddf-9dbdf7c60035","version":1,"weekStart":""}

From 85d0ab8dbb58c16b0f50f4667c5a5325a3037c0c Mon Sep 17 00:00:00 2001
From: Croxx <mrcroxx@outlook.com>
Date: Sun, 24 Nov 2024 17:09:43 +0800
Subject: [PATCH 45/53] feat: introduce metrics framework for different metrics
 backend (#790)

* feat: introduce metrics framework for different metrics backend

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

* feat: introduce metrics model

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

* refactor: apply new metrics framework

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

* fix: fix memory usage

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

* refactor: make clippy happy

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

---------

Signed-off-by: MrCroxx <mrcroxx@outlook.com>
---
 .github/workflows/ci.yml                      |   3 +-
 Cargo.toml                                    |   3 +-
 Makefile                                      |   5 +-
 foyer-bench/Cargo.toml                        |  13 +-
 foyer-bench/src/exporter.rs                   |  91 +++++
 foyer-bench/src/main.rs                       |  36 +-
 foyer-common/Cargo.toml                       |   5 +-
 foyer-common/src/metrics.rs                   | 280 -------------
 foyer-common/src/metrics/mod.rs               |  94 +++++
 foyer-common/src/metrics/model.rs             | 376 ++++++++++++++++++
 foyer-common/src/metrics/registry/mod.rs      |  22 +
 foyer-common/src/metrics/registry/noop.rs     |  96 +++++
 .../src/metrics/registry/opentelemetry.rs     | 208 ++++++++++
 .../src/metrics/registry/prometheus.rs        | 140 +++++++
 foyer-memory/src/cache.rs                     |  71 +++-
 foyer-memory/src/raw.rs                       |  47 ++-
 foyer-storage/src/device/monitor.rs           |  40 +-
 foyer-storage/src/large/batch.rs              |   2 +-
 foyer-storage/src/large/flusher.rs            |  10 +-
 foyer-storage/src/large/generic.rs            |  31 +-
 foyer-storage/src/large/reclaimer.rs          |   2 +-
 foyer-storage/src/large/recover.rs            |   2 +-
 foyer-storage/src/large/scanner.rs            |   4 +-
 foyer-storage/src/large/tombstone.rs          |   6 +-
 foyer-storage/src/region.rs                   |  14 +-
 foyer-storage/src/serde.rs                    |  10 +-
 foyer-storage/src/small/batch.rs              |   2 +-
 foyer-storage/src/small/flusher.rs            |   4 +-
 foyer-storage/src/small/generic.rs            |   4 +-
 foyer-storage/src/small/set.rs                |   5 +-
 foyer-storage/src/store.rs                    |  31 +-
 foyer-storage/src/test_utils.rs               |  12 +-
 foyer-storage/tests/storage_test.rs           |   3 +-
 foyer/Cargo.toml                              |   2 +
 foyer/src/hybrid/builder.rs                   |  72 +++-
 foyer/src/hybrid/cache.rs                     |  45 +--
 foyer/src/hybrid/writer.rs                    |   4 +-
 foyer/src/prelude.rs                          |   8 +
 38 files changed, 1335 insertions(+), 468 deletions(-)
 create mode 100644 foyer-bench/src/exporter.rs
 delete mode 100644 foyer-common/src/metrics.rs
 create mode 100644 foyer-common/src/metrics/mod.rs
 create mode 100644 foyer-common/src/metrics/model.rs
 create mode 100644 foyer-common/src/metrics/registry/mod.rs
 create mode 100644 foyer-common/src/metrics/registry/noop.rs
 create mode 100644 foyer-common/src/metrics/registry/opentelemetry.rs
 create mode 100644 foyer-common/src/metrics/registry/prometheus.rs

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 2457bd87..cb8ecec4 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -165,6 +165,7 @@ jobs:
           cargo clippy --all-targets --features tokio-console -- -D warnings
           cargo clippy --all-targets --features deadlock -- -D warnings
           cargo clippy --all-targets --features tracing -- -D warnings
+          cargo clippy --all-targets --features prometheus,opentelemetry -- -D warnings
           cargo clippy --all-targets -- -D warnings
       - if: steps.cache.outputs.cache-hit != 'true'
         uses: taiki-e/install-action@cargo-llvm-cov
@@ -181,7 +182,7 @@ jobs:
           RUST_BACKTRACE: 1
           CI: true
         run: |
-          cargo llvm-cov --no-report nextest --features "strict_assertions,sanity"
+          cargo llvm-cov --no-report nextest --features "strict_assertions,sanity,prometheus,opentelemetry"
       - name: Run examples with coverage
         env:
           RUST_BACKTRACE: 1
diff --git a/Cargo.toml b/Cargo.toml
index b2d2a350..e42e2a47 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -31,7 +31,6 @@ fastrace-jaeger = "0.7"
 fastrace-opentelemetry = "0.7"
 hashbrown = "0.14"
 itertools = "0.13"
-metrics = "0.24"
 parking_lot = { version = "0.12" }
 serde = { version = "1", features = ["derive", "rc"] }
 test-log = { version = "0.2", default-features = false, features = [
@@ -47,6 +46,8 @@ tokio = { package = "madsim-tokio", version = "0.2", features = [
     "signal",
     "fs",
 ] }
+prometheus = "0.13"
+opentelemetry = "0.27"
 
 # foyer components
 foyer-common = { version = "0.13.0-dev", path = "foyer-common" }
diff --git a/Makefile b/Makefile
index 0b093b9a..40c3a225 100644
--- a/Makefile
+++ b/Makefile
@@ -10,7 +10,7 @@ check:
 	./scripts/minimize-dashboards.sh
 	cargo sort -w
 	cargo fmt --all
-	cargo clippy --all-targets
+	cargo clippy --all-targets --features prometheus,opentelemetry
 
 check-all:
 	shellcheck ./scripts/*
@@ -21,10 +21,11 @@ check-all:
 	cargo clippy --all-targets --features tokio-console
 	cargo clippy --all-targets --features sanity
 	cargo clippy --all-targets --features tracing
+	cargo clippy --all-targets --features prometheus,opentelemetry
 	cargo clippy --all-targets
 
 test:
-	RUST_BACKTRACE=1 cargo nextest run --all --features "strict_assertions,sanity"
+	RUST_BACKTRACE=1 cargo nextest run --all --features "strict_assertions,sanity,prometheus,opentelemetry"
 	RUST_BACKTRACE=1 cargo test --doc
 
 test-ignored:
diff --git a/foyer-bench/Cargo.toml b/foyer-bench/Cargo.toml
index 4ca43c11..77c78a46 100644
--- a/foyer-bench/Cargo.toml
+++ b/foyer-bench/Cargo.toml
@@ -19,18 +19,23 @@ clap = { workspace = true }
 console-subscriber = { version = "0.4", optional = true }
 fastrace = { workspace = true, optional = true }
 fastrace-jaeger = { workspace = true, optional = true }
-foyer = { workspace = true }
+foyer = { workspace = true, features = ["prometheus"] }
 futures = "0.3"
 hdrhistogram = "7"
+http-body-util = "0.1"
 humantime = "2"
+hyper = { version = "1", default-features = false, features = [
+    "server",
+    "http1",
+] }
+hyper-util = { version = "0.1", default-features = false, features = ["tokio"] }
 itertools = { workspace = true }
-metrics = { workspace = true }
-metrics-exporter-prometheus = "0.16"
 parking_lot = { workspace = true }
+prometheus = { workspace = true }
 rand = "0.8.5"
 serde = { workspace = true }
 serde_bytes = "0.11.15"
-tokio = { workspace = true }
+tokio = { workspace = true, features = ["net"] }
 tracing = "0.1"
 tracing-opentelemetry = { version = "0.27", optional = true }
 tracing-subscriber = { version = "0.3", features = ["env-filter"] }
diff --git a/foyer-bench/src/exporter.rs b/foyer-bench/src/exporter.rs
new file mode 100644
index 00000000..3008a3bc
--- /dev/null
+++ b/foyer-bench/src/exporter.rs
@@ -0,0 +1,91 @@
+//  Copyright 2024 foyer Project Authors
+//
+//  Licensed under the Apache License, Version 2.0 (the "License");
+//  you may not use this file except in compliance with the License.
+//  You may obtain a copy of the License at
+//
+//  http://www.apache.org/licenses/LICENSE-2.0
+//
+//  Unless required by applicable law or agreed to in writing, software
+//  distributed under the License is distributed on an "AS IS" BASIS,
+//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//  See the License for the specific language governing permissions and
+//  limitations under the License.
+
+use std::{future::Future, net::SocketAddr, pin::Pin};
+
+use anyhow::Ok;
+use http_body_util::Full;
+use hyper::{
+    body::{Bytes, Incoming},
+    header::CONTENT_TYPE,
+    server::conn::http1,
+    service::Service,
+    Request, Response,
+};
+use hyper_util::rt::TokioIo;
+use prometheus::{Encoder, Registry, TextEncoder};
+use tokio::net::TcpListener;
+
+pub struct PrometheusExporter {
+    registry: Registry,
+    addr: SocketAddr,
+}
+
+impl PrometheusExporter {
+    pub fn new(registry: Registry, addr: SocketAddr) -> Self {
+        Self { registry, addr }
+    }
+
+    pub fn run(self) {
+        tokio::spawn(async move {
+            let listener = TcpListener::bind(&self.addr).await.unwrap();
+            loop {
+                let (stream, _) = match listener.accept().await {
+                    Result::Ok(res) => res,
+                    Err(e) => {
+                        tracing::error!("[prometheus exporter]: accept connection error: {e}");
+                        continue;
+                    }
+                };
+
+                let io = TokioIo::new(stream);
+                let handle = Handle {
+                    registry: self.registry.clone(),
+                };
+
+                tokio::spawn(async move {
+                    if let Err(e) = http1::Builder::new().serve_connection(io, handle).await {
+                        tracing::error!("[prometheus exporter]: serve request error: {e}");
+                    }
+                });
+            }
+        });
+    }
+}
+
+struct Handle {
+    registry: Registry,
+}
+
+impl Service<Request<Incoming>> for Handle {
+    type Response = Response<Full<Bytes>>;
+    type Error = anyhow::Error;
+    type Future = Pin<Box<dyn Future<Output = Result<Self::Response, Self::Error>> + Send>>;
+
+    fn call(&self, _: Request<Incoming>) -> Self::Future {
+        let mfs = self.registry.gather();
+
+        Box::pin(async move {
+            let encoder = TextEncoder::new();
+            let mut buffer = vec![];
+            encoder.encode(&mfs, &mut buffer)?;
+
+            Ok(Response::builder()
+                .status(200)
+                .header(CONTENT_TYPE, encoder.format_type())
+                .body(Full::new(Bytes::from(buffer)))
+                .unwrap())
+        })
+    }
+}
diff --git a/foyer-bench/src/main.rs b/foyer-bench/src/main.rs
index 251084ae..9a6d5ba4 100644
--- a/foyer-bench/src/main.rs
+++ b/foyer-bench/src/main.rs
@@ -17,6 +17,7 @@
 #![warn(clippy::allow_attributes)]
 
 mod analyze;
+mod exporter;
 mod rate;
 mod text;
 
@@ -35,14 +36,16 @@ use std::{
 use analyze::{analyze, monitor, Metrics};
 use bytesize::ByteSize;
 use clap::{builder::PossibleValuesParser, ArgGroup, Parser};
+use exporter::PrometheusExporter;
 use foyer::{
     Compression, DirectFileDeviceOptions, DirectFsDeviceOptions, Engine, FifoConfig, FifoPicker, HybridCache,
-    HybridCacheBuilder, InvalidRatioPicker, LargeEngineOptions, LfuConfig, LruConfig, RateLimitPicker, RecoverMode,
-    RuntimeOptions, S3FifoConfig, SmallEngineOptions, TokioRuntimeOptions, TracingOptions,
+    HybridCacheBuilder, InvalidRatioPicker, LargeEngineOptions, LfuConfig, LruConfig, PrometheusMetricsRegistry,
+    RateLimitPicker, RecoverMode, RuntimeOptions, S3FifoConfig, SmallEngineOptions, TokioRuntimeOptions,
+    TracingOptions,
 };
 use futures::future::join_all;
 use itertools::Itertools;
-use metrics_exporter_prometheus::PrometheusBuilder;
+use prometheus::Registry;
 use rand::{
     distributions::Distribution,
     rngs::{OsRng, StdRng},
@@ -431,16 +434,6 @@ async fn benchmark(args: Args) {
 
     assert!(args.get_range > 0, "\"--get-range\" value must be greater than 0");
 
-    if args.metrics {
-        let addr: SocketAddr = "0.0.0.0:19970".parse().unwrap();
-        PrometheusBuilder::new()
-            .with_http_listener(addr)
-            .set_buckets(&[0.000_001, 0.000_01, 0.000_1, 0.001, 0.01, 0.1, 1.0])
-            .unwrap()
-            .install()
-            .unwrap();
-    }
-
     let tracing_options = TracingOptions::new()
         .with_record_hybrid_insert_threshold(args.trace_insert.into())
         .with_record_hybrid_get_threshold(args.trace_get.into())
@@ -448,10 +441,19 @@ async fn benchmark(args: Args) {
         .with_record_hybrid_remove_threshold(args.trace_remove.into())
         .with_record_hybrid_fetch_threshold(args.trace_fetch.into());
 
-    let builder = HybridCacheBuilder::new()
-        .with_tracing_options(tracing_options)
-        .memory(args.mem.as_u64() as _)
-        .with_shards(args.shards);
+    let builder = HybridCacheBuilder::new().with_tracing_options(tracing_options);
+
+    let builder = if args.metrics {
+        let registry = Registry::new();
+        let addr: SocketAddr = "0.0.0.0:19970".parse().unwrap();
+        PrometheusExporter::new(registry.clone(), addr).run();
+        builder
+            .with_metrics_registry(PrometheusMetricsRegistry::new(registry))
+            .memory(args.mem.as_u64() as _)
+            .with_shards(args.shards)
+    } else {
+        builder.memory(args.mem.as_u64() as _).with_shards(args.shards)
+    };
 
     let builder = match args.eviction.as_str() {
         "lru" => builder.with_eviction_config(LruConfig::default()),
diff --git a/foyer-common/Cargo.toml b/foyer-common/Cargo.toml
index 8b1a99f1..4c6a7e38 100644
--- a/foyer-common/Cargo.toml
+++ b/foyer-common/Cargo.toml
@@ -19,9 +19,10 @@ fastrace = { workspace = true }
 futures = "0.3"
 hashbrown = { workspace = true }
 itertools = { workspace = true }
-metrics = { workspace = true }
+opentelemetry = { workspace = true, optional = true }
 parking_lot = { workspace = true }
 pin-project = "1"
+prometheus = { workspace = true, optional = true }
 serde = { workspace = true }
 tokio = { workspace = true }
 
@@ -32,6 +33,8 @@ rand = "0.8.5"
 [features]
 strict_assertions = []
 tracing = ["fastrace/enable"]
+prometheus = ["dep:prometheus"]
+opentelemetry = ["dep:opentelemetry"]
 
 [lints]
 workspace = true
diff --git a/foyer-common/src/metrics.rs b/foyer-common/src/metrics.rs
deleted file mode 100644
index d8ea3adc..00000000
--- a/foyer-common/src/metrics.rs
+++ /dev/null
@@ -1,280 +0,0 @@
-//  Copyright 2024 foyer Project Authors
-//
-//  Licensed under the Apache License, Version 2.0 (the "License");
-//  you may not use this file except in compliance with the License.
-//  You may obtain a copy of the License at
-//
-//  http://www.apache.org/licenses/LICENSE-2.0
-//
-//  Unless required by applicable law or agreed to in writing, software
-//  distributed under the License is distributed on an "AS IS" BASIS,
-//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-//  See the License for the specific language governing permissions and
-//  limitations under the License.
-
-use std::fmt::Debug;
-
-use metrics::{counter, gauge, histogram, Counter, Gauge, Histogram};
-
-// FIXME: https://github.com/rust-lang/rust-analyzer/issues/17685
-// #[expect(missing_docs)]
-/// ... ...
-#[derive(Clone)]
-pub struct Metrics {
-    /* in-memory cache metrics */
-    /// ... ...
-    pub memory_insert: Counter,
-    /// ... ...
-    pub memory_replace: Counter,
-    /// ... ...
-    pub memory_hit: Counter,
-    /// ... ...
-    pub memory_miss: Counter,
-    /// ... ...
-    pub memory_remove: Counter,
-    /// ... ...
-    pub memory_evict: Counter,
-    /// ... ...
-    pub memory_reinsert: Counter,
-    /// ... ...
-    pub memory_release: Counter,
-    /// ... ...
-    pub memory_queue: Counter,
-    /// ... ...
-    pub memory_fetch: Counter,
-
-    /// ... ...
-    pub memory_usage: Gauge,
-
-    /* disk cache metrics */
-    /// ... ...
-    pub storage_enqueue: Counter,
-    /// ... ...
-    pub storage_hit: Counter,
-    /// ... ...
-    pub storage_miss: Counter,
-    /// ... ...
-    pub storage_delete: Counter,
-
-    /// ... ...
-    pub storage_enqueue_duration: Histogram,
-    /// ... ...
-    pub storage_hit_duration: Histogram,
-    /// ... ...
-    pub storage_miss_duration: Histogram,
-    /// ... ...
-    pub storage_delete_duration: Histogram,
-
-    /// ... ...
-    pub storage_queue_rotate: Counter,
-    /// ... ...
-    pub storage_queue_rotate_duration: Histogram,
-    /// ... ...
-    pub storage_queue_drop: Counter,
-
-    /// ... ...
-    pub storage_disk_write: Counter,
-    /// ... ...
-    pub storage_disk_read: Counter,
-    /// ... ...
-    pub storage_disk_flush: Counter,
-
-    /// ... ...
-    pub storage_disk_write_bytes: Counter,
-    /// ... ...
-    pub storage_disk_read_bytes: Counter,
-
-    /// ... ...
-    pub storage_disk_write_duration: Histogram,
-    /// ... ...
-    pub storage_disk_read_duration: Histogram,
-    /// ... ...
-    pub storage_disk_flush_duration: Histogram,
-
-    /// ... ...
-    pub storage_region_total: Gauge,
-    /// ... ...
-    pub storage_region_clean: Gauge,
-    /// ... ...
-    pub storage_region_evictable: Gauge,
-
-    /// ... ...
-    pub storage_region_size_bytes: Gauge,
-
-    /// ... ...
-    pub storage_entry_serialize_duration: Histogram,
-    /// ... ...
-    pub storage_entry_deserialize_duration: Histogram,
-
-    /* hybrid cache metrics */
-    /// ... ...
-    pub hybrid_insert: Counter,
-    /// ... ...
-    pub hybrid_hit: Counter,
-    /// ... ...
-    pub hybrid_miss: Counter,
-    /// ... ...
-    pub hybrid_remove: Counter,
-
-    /// ... ...
-    pub hybrid_insert_duration: Histogram,
-    /// ... ...
-    pub hybrid_hit_duration: Histogram,
-    /// ... ...
-    pub hybrid_miss_duration: Histogram,
-    /// ... ...
-    pub hybrid_remove_duration: Histogram,
-    /// ... ...
-    pub hybrid_fetch_duration: Histogram,
-}
-
-impl Debug for Metrics {
-    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-        f.debug_struct("Metrics").finish()
-    }
-}
-
-impl Metrics {
-    /// Create a new metrics with the given name.
-    pub fn new(name: &str) -> Self {
-        /* in-memory cache metrics */
-
-        let memory_insert = counter!(format!("foyer_memory_op_total"), "name" => name.to_string(), "op" => "insert");
-        let memory_replace = counter!(format!("foyer_memory_op_total"), "name" => name.to_string(), "op" => "replace");
-        let memory_hit = counter!(format!("foyer_memory_op_total"), "name" => name.to_string(), "op" => "hit");
-        let memory_miss = counter!(format!("foyer_memory_op_total"), "name" => name.to_string(), "op" => "miss");
-        let memory_remove = counter!(format!("foyer_memory_op_total"), "name" => name.to_string(), "op" => "remove");
-        let memory_evict = counter!(format!("foyer_memory_op_total"), "name" => name.to_string(), "op" => "evict");
-        let memory_reinsert =
-            counter!(format!("foyer_memory_op_total"), "name" => name.to_string(), "op" => "reinsert");
-        let memory_release = counter!(format!("foyer_memory_op_total"), "name" => name.to_string(), "op" => "release");
-        let memory_queue = counter!(format!("foyer_memory_op_total"), "name" => name.to_string(), "op" => "queue");
-        let memory_fetch = counter!(format!("foyer_memory_op_total"), "name" => name.to_string(), "op" => "fetch");
-
-        let memory_usage = gauge!(format!("foyer_memory_usage"), "name" => name.to_string(), "op" => "usage");
-
-        /* disk cache metrics */
-
-        let storage_enqueue =
-            counter!(format!("foyer_storage_op_total"), "name" => name.to_string(), "op" => "enqueue");
-        let storage_hit = counter!(format!("foyer_storage_op_total"), "name" => name.to_string(), "op" => "hit");
-        let storage_miss = counter!(format!("foyer_storage_op_total"), "name" => name.to_string(), "op" => "miss");
-        let storage_delete = counter!(format!("foyer_storage_op_total"), "name" => name.to_string(), "op" => "delete");
-
-        let storage_enqueue_duration =
-            histogram!(format!("foyer_storage_op_duration"), "name" => name.to_string(), "op" => "enqueue");
-        let storage_hit_duration =
-            histogram!(format!("foyer_storage_op_duration"), "name" => name.to_string(), "op" => "hit");
-        let storage_miss_duration =
-            histogram!(format!("foyer_storage_op_duration"), "name" => name.to_string(), "op" => "miss");
-        let storage_delete_duration =
-            histogram!(format!("foyer_storage_op_duration"), "name" => name.to_string(), "op" => "delete");
-
-        let storage_queue_rotate =
-            counter!(format!("foyer_storage_inner_op_total"), "name" => name.to_string(), "op" => "queue_rotate");
-        let storage_queue_rotate_duration =
-            histogram!(format!("foyer_storage_inner_op_duration"), "name" => name.to_string(), "op" => "queue_rotate");
-        let storage_queue_drop =
-            counter!(format!("foyer_storage_inner_op_total"), "name" => name.to_string(), "op" => "queue_drop");
-
-        let storage_disk_write =
-            counter!(format!("foyer_storage_disk_io_total"), "name" => name.to_string(), "op" => "write");
-        let storage_disk_read =
-            counter!(format!("foyer_storage_disk_io_total"), "name" => name.to_string(), "op" => "read");
-        let storage_disk_flush =
-            counter!(format!("foyer_storage_disk_io_total"), "name" => name.to_string(), "op" => "flush");
-
-        let storage_disk_write_bytes =
-            counter!(format!("foyer_storage_disk_io_bytes"), "name" => name.to_string(), "op" => "write");
-        let storage_disk_read_bytes =
-            counter!(format!("foyer_storage_disk_io_bytes"), "name" => name.to_string(), "op" => "read");
-
-        let storage_disk_write_duration =
-            histogram!(format!("foyer_storage_disk_io_duration"), "name" => name.to_string(), "op" => "write");
-        let storage_disk_read_duration =
-            histogram!(format!("foyer_storage_disk_io_duration"), "name" => name.to_string(), "op" => "read");
-        let storage_disk_flush_duration =
-            histogram!(format!("foyer_storage_disk_io_duration"), "name" => name.to_string(), "op" => "flush");
-
-        let storage_region_total =
-            gauge!(format!("foyer_storage_region"), "name" => name.to_string(), "type" => "total");
-        let storage_region_clean =
-            gauge!(format!("foyer_storage_region"), "name" => name.to_string(), "type" => "clean");
-        let storage_region_evictable =
-            gauge!(format!("foyer_storage_region"), "name" => name.to_string(), "type" => "evictable");
-
-        let storage_region_size_bytes = gauge!(format!("foyer_storage_region_size_bytes"), "name" => name.to_string());
-
-        let storage_entry_serialize_duration =
-            histogram!(format!("foyer_storage_entry_serde_duration"), "name" => name.to_string(), "op" => "serialize");
-        let storage_entry_deserialize_duration = histogram!(format!("foyer_storage_entry_serde_duration"), "name" => name.to_string(), "op" => "deserialize");
-
-        /* hybrid cache metrics */
-
-        let hybrid_insert = counter!(format!("foyer_hybrid_op_total"), "name" => name.to_string(), "op" => "insert");
-        let hybrid_hit = counter!(format!("foyer_hybrid_op_total"), "name" => name.to_string(), "op" => "hit");
-        let hybrid_miss = counter!(format!("foyer_hybrid_op_total"), "name" => name.to_string(), "op" => "miss");
-        let hybrid_remove = counter!(format!("foyer_hybrid_op_total"), "name" => name.to_string(), "op" => "remove");
-
-        let hybrid_insert_duration =
-            histogram!(format!("foyer_hybrid_op_duration"), "name" => name.to_string(), "op" => "insert");
-        let hybrid_hit_duration =
-            histogram!(format!("foyer_hybrid_op_duration"), "name" => name.to_string(), "op" => "hit");
-        let hybrid_miss_duration =
-            histogram!(format!("foyer_hybrid_op_duration"), "name" => name.to_string(), "op" => "miss");
-        let hybrid_remove_duration =
-            histogram!(format!("foyer_hybrid_op_duration"), "name" => name.to_string(), "op" => "remove");
-        let hybrid_fetch_duration =
-            histogram!(format!("foyer_hybrid_op_duration"), "name" => name.to_string(), "op" => "fetch");
-
-        Self {
-            memory_insert,
-            memory_replace,
-            memory_hit,
-            memory_miss,
-            memory_remove,
-            memory_evict,
-            memory_reinsert,
-            memory_release,
-            memory_queue,
-            memory_fetch,
-            memory_usage,
-
-            storage_enqueue,
-            storage_hit,
-            storage_miss,
-            storage_delete,
-            storage_enqueue_duration,
-            storage_hit_duration,
-            storage_miss_duration,
-            storage_delete_duration,
-            storage_queue_rotate,
-            storage_queue_rotate_duration,
-            storage_queue_drop,
-            storage_disk_write,
-            storage_disk_read,
-            storage_disk_flush,
-            storage_disk_write_bytes,
-            storage_disk_read_bytes,
-            storage_disk_write_duration,
-            storage_disk_read_duration,
-            storage_disk_flush_duration,
-            storage_region_total,
-            storage_region_clean,
-            storage_region_evictable,
-            storage_region_size_bytes,
-            storage_entry_serialize_duration,
-            storage_entry_deserialize_duration,
-
-            hybrid_insert,
-            hybrid_hit,
-            hybrid_miss,
-            hybrid_remove,
-            hybrid_insert_duration,
-            hybrid_hit_duration,
-            hybrid_miss_duration,
-            hybrid_remove_duration,
-            hybrid_fetch_duration,
-        }
-    }
-}
diff --git a/foyer-common/src/metrics/mod.rs b/foyer-common/src/metrics/mod.rs
new file mode 100644
index 00000000..576182f7
--- /dev/null
+++ b/foyer-common/src/metrics/mod.rs
@@ -0,0 +1,94 @@
+//  Copyright 2024 foyer Project Authors
+//
+//  Licensed under the Apache License, Version 2.0 (the "License");
+//  you may not use this file except in compliance with the License.
+//  You may obtain a copy of the License at
+//
+//  http://www.apache.org/licenses/LICENSE-2.0
+//
+//  Unless required by applicable law or agreed to in writing, software
+//  distributed under the License is distributed on an "AS IS" BASIS,
+//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//  See the License for the specific language governing permissions and
+//  limitations under the License.
+
+use std::fmt::Debug;
+
+/// Counter metric operations.
+pub trait CounterOps: Send + Sync + 'static + Debug {
+    /// Increase record by `val`.
+    fn increase(&self, val: u64);
+}
+
+/// Gauge metric operations.
+pub trait GaugeOps: Send + Sync + 'static + Debug {
+    /// Increase record by `val`.
+    fn increase(&self, val: u64);
+    /// Decrease record by `val`.
+    fn decrease(&self, val: u64);
+    /// Set the record as a absolute value `val`.
+    fn absolute(&self, val: u64);
+}
+
+/// Histogram metric operations.
+pub trait HistogramOps: Send + Sync + 'static + Debug {
+    /// Record a value.
+    fn record(&self, val: f64);
+}
+
+/// A vector of counters.
+pub trait CounterVecOps: Send + Sync + 'static + Debug {
+    /// Get a counter within the vector of counters.
+    fn counter(&self, labels: &[&str]) -> impl CounterOps;
+}
+
+/// A vector of gauges.
+pub trait GaugeVecOps: Send + Sync + 'static + Debug {
+    /// Get a gauge within the vector of gauges.
+    fn gauge(&self, labels: &[&str]) -> impl GaugeOps;
+}
+
+/// A vector of histograms.
+pub trait HistogramVecOps: Send + Sync + 'static + Debug {
+    /// Get a histogram within the vector of histograms.
+    fn histogram(&self, labels: &[&str]) -> impl HistogramOps;
+}
+
+/// Metrics registry.
+pub trait RegistryOps: Send + Sync + 'static + Debug {
+    /// Register a vector of counters to the registry.
+    fn register_counter_vec(
+        &self,
+        name: &'static str,
+        desc: &'static str,
+        label_names: &'static [&'static str],
+    ) -> impl CounterVecOps;
+
+    /// Register a vector of gauges to the registry.
+    fn register_gauge_vec(
+        &self,
+        name: &'static str,
+        desc: &'static str,
+        label_names: &'static [&'static str],
+    ) -> impl GaugeVecOps;
+
+    /// Register a vector of histograms to the registry.
+    fn register_histogram_vec(
+        &self,
+        name: &'static str,
+        desc: &'static str,
+        label_names: &'static [&'static str],
+    ) -> impl HistogramVecOps;
+}
+
+/// Boxed generic counter.
+pub type BoxedCounter = Box<dyn CounterOps>;
+/// Boxed generic gauge.
+pub type BoxedGauge = Box<dyn GaugeOps>;
+/// Boxed generic histogram.
+pub type BoxedHistogram = Box<dyn HistogramOps>;
+
+/// Shared metrics model.
+pub mod model;
+/// Provisioned metrics registries.
+pub mod registry;
diff --git a/foyer-common/src/metrics/model.rs b/foyer-common/src/metrics/model.rs
new file mode 100644
index 00000000..3db3f536
--- /dev/null
+++ b/foyer-common/src/metrics/model.rs
@@ -0,0 +1,376 @@
+//  Copyright 2024 foyer Project Authors
+//
+//  Licensed under the Apache License, Version 2.0 (the "License");
+//  you may not use this file except in compliance with the License.
+//  You may obtain a copy of the License at
+//
+//  http://www.apache.org/licenses/LICENSE-2.0
+//
+//  Unless required by applicable law or agreed to in writing, software
+//  distributed under the License is distributed on an "AS IS" BASIS,
+//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//  See the License for the specific language governing permissions and
+//  limitations under the License.
+
+use super::{BoxedCounter, BoxedGauge, BoxedHistogram, GaugeVecOps, HistogramVecOps, RegistryOps};
+use crate::metrics::CounterVecOps;
+
+trait Boxer {
+    fn boxed(self) -> Box<Self>
+    where
+        Self: Sized,
+    {
+        Box::new(self)
+    }
+}
+impl<T> Boxer for T {}
+
+// FIXME: https://github.com/rust-lang/rust-analyzer/issues/17685
+// #[expect(missing_docs)]
+/// ... ...
+#[derive(Debug)]
+pub struct Metrics {
+    /* in-memory cache metrics */
+    /// ... ...
+    pub memory_insert: BoxedCounter,
+    /// ... ...
+    pub memory_replace: BoxedCounter,
+    /// ... ...
+    pub memory_hit: BoxedCounter,
+    /// ... ...
+    pub memory_miss: BoxedCounter,
+    /// ... ...
+    pub memory_remove: BoxedCounter,
+    /// ... ...
+    pub memory_evict: BoxedCounter,
+    /// ... ...
+    pub memory_reinsert: BoxedCounter,
+    /// ... ...
+    pub memory_release: BoxedCounter,
+    /// ... ...
+    pub memory_queue: BoxedCounter,
+    /// ... ...
+    pub memory_fetch: BoxedCounter,
+
+    /// ... ...
+    pub memory_usage: BoxedGauge,
+
+    /* disk cache metrics */
+    /// ... ...
+    pub storage_enqueue: BoxedCounter,
+    /// ... ...
+    pub storage_hit: BoxedCounter,
+    /// ... ...
+    pub storage_miss: BoxedCounter,
+    /// ... ...
+    pub storage_delete: BoxedCounter,
+
+    /// ... ...
+    pub storage_enqueue_duration: BoxedHistogram,
+    /// ... ...
+    pub storage_hit_duration: BoxedHistogram,
+    /// ... ...
+    pub storage_miss_duration: BoxedHistogram,
+    /// ... ...
+    pub storage_delete_duration: BoxedHistogram,
+
+    /// ... ...
+    pub storage_queue_rotate: BoxedCounter,
+    /// ... ...
+    pub storage_queue_rotate_duration: BoxedHistogram,
+    /// ... ...
+    pub storage_queue_drop: BoxedCounter,
+
+    /// ... ...
+    pub storage_disk_write: BoxedCounter,
+    /// ... ...
+    pub storage_disk_read: BoxedCounter,
+    /// ... ...
+    pub storage_disk_flush: BoxedCounter,
+
+    /// ... ...
+    pub storage_disk_write_bytes: BoxedCounter,
+    /// ... ...
+    pub storage_disk_read_bytes: BoxedCounter,
+
+    /// ... ...
+    pub storage_disk_write_duration: BoxedHistogram,
+    /// ... ...
+    pub storage_disk_read_duration: BoxedHistogram,
+    /// ... ...
+    pub storage_disk_flush_duration: BoxedHistogram,
+
+    /// ... ...
+    pub storage_region_total: BoxedGauge,
+    /// ... ...
+    pub storage_region_clean: BoxedGauge,
+    /// ... ...
+    pub storage_region_evictable: BoxedGauge,
+
+    /// ... ...
+    pub storage_region_size_bytes: BoxedGauge,
+
+    /// ... ...
+    pub storage_entry_serialize_duration: BoxedHistogram,
+    /// ... ...
+    pub storage_entry_deserialize_duration: BoxedHistogram,
+
+    /* hybrid cache metrics */
+    /// ... ...
+    pub hybrid_insert: BoxedCounter,
+    /// ... ...
+    pub hybrid_hit: BoxedCounter,
+    /// ... ...
+    pub hybrid_miss: BoxedCounter,
+    /// ... ...
+    pub hybrid_remove: BoxedCounter,
+
+    /// ... ...
+    pub hybrid_insert_duration: BoxedHistogram,
+    /// ... ...
+    pub hybrid_hit_duration: BoxedHistogram,
+    /// ... ...
+    pub hybrid_miss_duration: BoxedHistogram,
+    /// ... ...
+    pub hybrid_remove_duration: BoxedHistogram,
+    /// ... ...
+    pub hybrid_fetch_duration: BoxedHistogram,
+}
+
+impl Metrics {
+    /// Create a new metric with the given name.
+    pub fn new<R>(name: &'static str, registry: &R) -> Self
+    where
+        R: RegistryOps,
+    {
+        /* in-memory cache metrics */
+
+        let foyer_memory_op_total = registry.register_counter_vec(
+            "foyer_memory_op_total",
+            "foyer in-memory cache operations",
+            &["name", "op"],
+        );
+        let foyer_memory_usage =
+            registry.register_gauge_vec("foyer_memory_usage", "foyer in-memory cache usage", &["name"]);
+
+        let memory_insert = foyer_memory_op_total.counter(&[name, "insert"]).boxed();
+        let memory_replace = foyer_memory_op_total.counter(&[name, "replace"]).boxed();
+        let memory_hit = foyer_memory_op_total.counter(&[name, "hit"]).boxed();
+        let memory_miss = foyer_memory_op_total.counter(&[name, "miss"]).boxed();
+        let memory_remove = foyer_memory_op_total.counter(&[name, "remove"]).boxed();
+        let memory_evict = foyer_memory_op_total.counter(&[name, "evict"]).boxed();
+        let memory_reinsert = foyer_memory_op_total.counter(&[name, "reinsert"]).boxed();
+        let memory_release = foyer_memory_op_total.counter(&[name, "release"]).boxed();
+        let memory_queue = foyer_memory_op_total.counter(&[name, "queue"]).boxed();
+        let memory_fetch = foyer_memory_op_total.counter(&[name, "fetch"]).boxed();
+
+        let memory_usage = foyer_memory_usage.gauge(&[name]).boxed();
+
+        /* disk cache metrics */
+
+        let foyer_storage_op_total =
+            registry.register_counter_vec("foyer_storage_op_total", "foyer disk cache operations", &["name", "op"]);
+        let foyer_storage_op_duration = registry.register_histogram_vec(
+            "foyer_storage_op_duration",
+            "foyer disk cache op durations",
+            &["name", "op"],
+        );
+
+        let foyer_storage_inner_op_total = registry.register_counter_vec(
+            "foyer_storage_inner_op_total",
+            "foyer disk cache inner operations",
+            &["name", "op"],
+        );
+        let foyer_storage_inner_op_duration = registry.register_histogram_vec(
+            "foyer_storage_inner_op_duration",
+            "foyer disk cache inner op durations",
+            &["name", "op"],
+        );
+
+        let foyer_storage_disk_io_total = registry.register_counter_vec(
+            "foyer_storage_disk_io_total",
+            "foyer disk cache disk operations",
+            &["name", "op"],
+        );
+        let foyer_storage_disk_io_bytes = registry.register_counter_vec(
+            "foyer_storage_disk_io_bytes",
+            "foyer disk cache disk io bytes",
+            &["name", "op"],
+        );
+        let foyer_storage_disk_io_duration = registry.register_histogram_vec(
+            "foyer_storage_disk_io_duration",
+            "foyer disk cache disk io duration",
+            &["name", "op"],
+        );
+
+        let foyer_storage_region =
+            registry.register_gauge_vec("foyer_storage_region", "foyer disk cache regions", &["name", "type"]);
+        let foyer_storage_region_size_bytes = registry.register_gauge_vec(
+            "foyer_storage_region_size_bytes",
+            "foyer disk cache region sizes",
+            &["name"],
+        );
+
+        let foyer_storage_entry_serde_duration = registry.register_histogram_vec(
+            "foyer_storage_entry_serde_duration",
+            "foyer disk cache entry serde durations",
+            &["name", "op"],
+        );
+
+        let storage_enqueue = foyer_storage_op_total.counter(&[name, "enqueue"]).boxed();
+        let storage_hit = foyer_storage_op_total.counter(&[name, "hit"]).boxed();
+        let storage_miss = foyer_storage_op_total.counter(&[name, "miss"]).boxed();
+        let storage_delete = foyer_storage_op_total.counter(&[name, "delete"]).boxed();
+
+        let storage_enqueue_duration = foyer_storage_op_duration.histogram(&[name, "enqueue"]).boxed();
+        let storage_hit_duration = foyer_storage_op_duration.histogram(&[name, "hit"]).boxed();
+        let storage_miss_duration = foyer_storage_op_duration.histogram(&[name, "miss"]).boxed();
+        let storage_delete_duration = foyer_storage_op_duration.histogram(&[name, "delete"]).boxed();
+
+        let storage_queue_rotate = foyer_storage_inner_op_total.counter(&[name, "queue_rotate"]).boxed();
+        let storage_queue_drop = foyer_storage_inner_op_total.counter(&[name, "queue_drop"]).boxed();
+
+        let storage_queue_rotate_duration = foyer_storage_inner_op_duration
+            .histogram(&[name, "queue_rotate"])
+            .boxed();
+
+        let storage_disk_write = foyer_storage_disk_io_total.counter(&[name, "write"]).boxed();
+        let storage_disk_read = foyer_storage_disk_io_total.counter(&[name, "read"]).boxed();
+        let storage_disk_flush = foyer_storage_disk_io_total.counter(&[name, "flush"]).boxed();
+
+        let storage_disk_write_bytes = foyer_storage_disk_io_bytes.counter(&[name, "write"]).boxed();
+        let storage_disk_read_bytes = foyer_storage_disk_io_bytes.counter(&[name, "read"]).boxed();
+
+        let storage_disk_write_duration = foyer_storage_disk_io_duration.histogram(&[name, "write"]).boxed();
+        let storage_disk_read_duration = foyer_storage_disk_io_duration.histogram(&[name, "read"]).boxed();
+        let storage_disk_flush_duration = foyer_storage_disk_io_duration.histogram(&[name, "flush"]).boxed();
+
+        let storage_region_total = foyer_storage_region.gauge(&[name, "total"]).boxed();
+        let storage_region_clean = foyer_storage_region.gauge(&[name, "clean"]).boxed();
+        let storage_region_evictable = foyer_storage_region.gauge(&[name, "evictable"]).boxed();
+
+        let storage_region_size_bytes = foyer_storage_region_size_bytes.gauge(&[name]).boxed();
+
+        let storage_entry_serialize_duration = foyer_storage_entry_serde_duration
+            .histogram(&[name, "serialize"])
+            .boxed();
+        let storage_entry_deserialize_duration = foyer_storage_entry_serde_duration
+            .histogram(&[name, "deserialize"])
+            .boxed();
+
+        /* hybrid cache metrics */
+
+        let foyer_hybrid_op_total = registry.register_counter_vec(
+            "foyer_hybrid_op_total",
+            "foyer hybrid cache operations",
+            &["name", "op"],
+        );
+        let foyer_hybrid_op_duration = registry.register_histogram_vec(
+            "foyer_hybrid_op_duration",
+            "foyer hybrid cache operation durations",
+            &["name", "op"],
+        );
+
+        let hybrid_insert = foyer_hybrid_op_total.counter(&[name, "insert"]).boxed();
+        let hybrid_hit = foyer_hybrid_op_total.counter(&[name, "hit"]).boxed();
+        let hybrid_miss = foyer_hybrid_op_total.counter(&[name, "miss"]).boxed();
+        let hybrid_remove = foyer_hybrid_op_total.counter(&[name, "remove"]).boxed();
+
+        let hybrid_insert_duration = foyer_hybrid_op_duration.histogram(&[name, "insert"]).boxed();
+        let hybrid_hit_duration = foyer_hybrid_op_duration.histogram(&[name, "hit"]).boxed();
+        let hybrid_miss_duration = foyer_hybrid_op_duration.histogram(&[name, "miss"]).boxed();
+        let hybrid_remove_duration = foyer_hybrid_op_duration.histogram(&[name, "remove"]).boxed();
+        let hybrid_fetch_duration = foyer_hybrid_op_duration.histogram(&[name, "fetch"]).boxed();
+
+        Self {
+            memory_insert,
+            memory_replace,
+            memory_hit,
+            memory_miss,
+            memory_remove,
+            memory_evict,
+            memory_reinsert,
+            memory_release,
+            memory_queue,
+            memory_fetch,
+            memory_usage,
+
+            storage_enqueue,
+            storage_hit,
+            storage_miss,
+            storage_delete,
+            storage_enqueue_duration,
+            storage_hit_duration,
+            storage_miss_duration,
+            storage_delete_duration,
+            storage_queue_rotate,
+            storage_queue_rotate_duration,
+            storage_queue_drop,
+            storage_disk_write,
+            storage_disk_read,
+            storage_disk_flush,
+            storage_disk_write_bytes,
+            storage_disk_read_bytes,
+            storage_disk_write_duration,
+            storage_disk_read_duration,
+            storage_disk_flush_duration,
+            storage_region_total,
+            storage_region_clean,
+            storage_region_evictable,
+            storage_region_size_bytes,
+            storage_entry_serialize_duration,
+            storage_entry_deserialize_duration,
+
+            hybrid_insert,
+            hybrid_hit,
+            hybrid_miss,
+            hybrid_remove,
+            hybrid_insert_duration,
+            hybrid_hit_duration,
+            hybrid_miss_duration,
+            hybrid_remove_duration,
+            hybrid_fetch_duration,
+        }
+    }
+
+    /// Build noop metrics.
+    ///
+    /// Note: `noop` is only supposed to be called by other foyer components.
+    #[doc(hidden)]
+    pub fn noop() -> Self {
+        use super::registry::noop::NoopMetricsRegistry;
+
+        Self::new("test", &NoopMetricsRegistry)
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::metrics::registry::noop::NoopMetricsRegistry;
+
+    fn case(registry: &impl RegistryOps) {
+        let _ = Metrics::new("test", registry);
+    }
+
+    #[test]
+    fn test_metrics_noop() {
+        case(&NoopMetricsRegistry);
+    }
+
+    #[cfg(feature = "prometheus")]
+    #[test]
+    fn test_metrics_prometheus() {
+        use crate::metrics::registry::prometheus::PrometheusMetricsRegistry;
+
+        case(&PrometheusMetricsRegistry::new(prometheus::Registry::new()));
+    }
+
+    #[cfg(feature = "opentelemetry")]
+    #[test]
+    fn test_metrics_opentelemetry() {
+        use crate::metrics::registry::opentelemetry::OpenTelemetryMetricsRegistry;
+
+        case(&OpenTelemetryMetricsRegistry::new(opentelemetry::global::meter("test")));
+    }
+}
diff --git a/foyer-common/src/metrics/registry/mod.rs b/foyer-common/src/metrics/registry/mod.rs
new file mode 100644
index 00000000..840f7fde
--- /dev/null
+++ b/foyer-common/src/metrics/registry/mod.rs
@@ -0,0 +1,22 @@
+//  Copyright 2024 foyer Project Authors
+//
+//  Licensed under the Apache License, Version 2.0 (the "License");
+//  you may not use this file except in compliance with the License.
+//  You may obtain a copy of the License at
+//
+//  http://www.apache.org/licenses/LICENSE-2.0
+//
+//  Unless required by applicable law or agreed to in writing, software
+//  distributed under the License is distributed on an "AS IS" BASIS,
+//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//  See the License for the specific language governing permissions and
+//  limitations under the License.
+
+/// Some phantom metrics components that do nothing.
+pub mod noop;
+/// OpenTelemetry metrics components.
+#[cfg(feature = "opentelemetry")]
+pub mod opentelemetry;
+/// Prometheus metrics components.
+#[cfg(feature = "prometheus")]
+pub mod prometheus;
diff --git a/foyer-common/src/metrics/registry/noop.rs b/foyer-common/src/metrics/registry/noop.rs
new file mode 100644
index 00000000..16cc4215
--- /dev/null
+++ b/foyer-common/src/metrics/registry/noop.rs
@@ -0,0 +1,96 @@
+//  Copyright 2024 foyer Project Authors
+//
+//  Licensed under the Apache License, Version 2.0 (the "License");
+//  you may not use this file except in compliance with the License.
+//  You may obtain a copy of the License at
+//
+//  http://www.apache.org/licenses/LICENSE-2.0
+//
+//  Unless required by applicable law or agreed to in writing, software
+//  distributed under the License is distributed on an "AS IS" BASIS,
+//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//  See the License for the specific language governing permissions and
+//  limitations under the License.
+
+use crate::metrics::{CounterOps, CounterVecOps, GaugeOps, GaugeVecOps, HistogramOps, HistogramVecOps, RegistryOps};
+
+/// Noop metrics placeholder.
+#[derive(Debug)]
+pub struct NoopMetricsRegistry;
+
+impl CounterOps for NoopMetricsRegistry {
+    fn increase(&self, _: u64) {}
+}
+
+impl CounterVecOps for NoopMetricsRegistry {
+    fn counter(&self, _: &[&str]) -> impl CounterOps {
+        NoopMetricsRegistry
+    }
+}
+
+impl GaugeOps for NoopMetricsRegistry {
+    fn increase(&self, _: u64) {}
+
+    fn decrease(&self, _: u64) {}
+
+    fn absolute(&self, _: u64) {}
+}
+
+impl GaugeVecOps for NoopMetricsRegistry {
+    fn gauge(&self, _: &[&str]) -> impl GaugeOps {
+        NoopMetricsRegistry
+    }
+}
+
+impl HistogramOps for NoopMetricsRegistry {
+    fn record(&self, _: f64) {}
+}
+
+impl HistogramVecOps for NoopMetricsRegistry {
+    fn histogram(&self, _: &[&str]) -> impl HistogramOps {
+        NoopMetricsRegistry
+    }
+}
+
+impl RegistryOps for NoopMetricsRegistry {
+    fn register_counter_vec(&self, _: &'static str, _: &'static str, _: &'static [&'static str]) -> impl CounterVecOps {
+        NoopMetricsRegistry
+    }
+
+    fn register_gauge_vec(&self, _: &'static str, _: &'static str, _: &'static [&'static str]) -> impl GaugeVecOps {
+        NoopMetricsRegistry
+    }
+
+    fn register_histogram_vec(
+        &self,
+        _: &'static str,
+        _: &'static str,
+        _: &'static [&'static str],
+    ) -> impl HistogramVecOps {
+        NoopMetricsRegistry
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test() {
+        let noop = NoopMetricsRegistry;
+
+        let cv = noop.register_counter_vec("test_counter_1", "test counter 1", &["label1", "label2"]);
+        let c = cv.counter(&["l1", "l2"]);
+        c.increase(42);
+
+        let gv = noop.register_gauge_vec("test_gauge_1", "test gauge 1", &["label1", "label2"]);
+        let g = gv.gauge(&["l1", "l2"]);
+        g.increase(514);
+        g.decrease(114);
+        g.absolute(114514);
+
+        let hv = noop.register_histogram_vec("test_histogram_1", "test histogram 1", &["label1", "label2"]);
+        let h = hv.histogram(&["l1", "l2"]);
+        h.record(114.514);
+    }
+}
diff --git a/foyer-common/src/metrics/registry/opentelemetry.rs b/foyer-common/src/metrics/registry/opentelemetry.rs
new file mode 100644
index 00000000..349d0802
--- /dev/null
+++ b/foyer-common/src/metrics/registry/opentelemetry.rs
@@ -0,0 +1,208 @@
+//  Copyright 2024 foyer Project Authors
+//
+//  Licensed under the Apache License, Version 2.0 (the "License");
+//  you may not use this file except in compliance with the License.
+//  You may obtain a copy of the License at
+//
+//  http://www.apache.org/licenses/LICENSE-2.0
+//
+//  Unless required by applicable law or agreed to in writing, software
+//  distributed under the License is distributed on an "AS IS" BASIS,
+//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//  See the License for the specific language governing permissions and
+//  limitations under the License.
+
+use std::sync::atomic::{AtomicU64, Ordering};
+
+use itertools::Itertools;
+use opentelemetry::{
+    metrics::{Counter as OtCounter, Gauge as OtGauge, Histogram as OtHistogram, Meter},
+    KeyValue,
+};
+
+use crate::metrics::{CounterOps, CounterVecOps, GaugeOps, GaugeVecOps, HistogramOps, HistogramVecOps, RegistryOps};
+
+/// OpenTelemetry counter metric.
+#[derive(Debug)]
+pub struct Counter {
+    counter: OtCounter<u64>,
+    labels: Vec<KeyValue>,
+}
+
+impl CounterOps for Counter {
+    fn increase(&self, val: u64) {
+        self.counter.add(val, &self.labels);
+    }
+}
+
+/// OpenTelemetry gauge metric.
+#[derive(Debug)]
+pub struct Gauge {
+    val: AtomicU64,
+    gauge: OtGauge<u64>,
+    labels: Vec<KeyValue>,
+}
+
+impl GaugeOps for Gauge {
+    fn increase(&self, val: u64) {
+        let v = self.val.fetch_add(val, Ordering::Relaxed) + val;
+        self.gauge.record(v, &self.labels);
+    }
+
+    fn decrease(&self, val: u64) {
+        let v = self.val.fetch_sub(val, Ordering::Relaxed) - val;
+        self.gauge.record(v, &self.labels);
+    }
+
+    fn absolute(&self, val: u64) {
+        self.val.store(val, Ordering::Relaxed);
+        self.gauge.record(val, &self.labels);
+    }
+}
+
+/// OpenTelemetry histogram metric.
+#[derive(Debug)]
+pub struct Histogram {
+    histogram: OtHistogram<f64>,
+    labels: Vec<KeyValue>,
+}
+
+impl HistogramOps for Histogram {
+    fn record(&self, val: f64) {
+        self.histogram.record(val, &self.labels);
+    }
+}
+
+/// OpenTelemetry metric vector.
+#[derive(Debug)]
+pub struct MetricVec {
+    meter: Meter,
+    name: &'static str,
+    desc: &'static str,
+    label_names: &'static [&'static str],
+}
+
+impl CounterVecOps for MetricVec {
+    fn counter(&self, labels: &[&str]) -> impl CounterOps {
+        let counter = self.meter.u64_counter(self.name).with_description(self.desc).build();
+        let labels = self
+            .label_names
+            .iter()
+            .zip_eq(labels.iter())
+            .map(|(name, label)| KeyValue::new(name.to_string(), label.to_string()))
+            .collect();
+        Counter { counter, labels }
+    }
+}
+
+impl GaugeVecOps for MetricVec {
+    fn gauge(&self, labels: &[&str]) -> impl GaugeOps {
+        let gauge = self.meter.u64_gauge(self.name).with_description(self.desc).build();
+        let labels = self
+            .label_names
+            .iter()
+            .zip_eq(labels.iter())
+            .map(|(name, label)| KeyValue::new(name.to_string(), label.to_string()))
+            .collect();
+        let val = AtomicU64::new(0);
+        Gauge { val, gauge, labels }
+    }
+}
+
+impl HistogramVecOps for MetricVec {
+    fn histogram(&self, labels: &[&str]) -> impl HistogramOps {
+        let histogram = self.meter.f64_histogram(self.name).with_description(self.desc).build();
+        let labels = self
+            .label_names
+            .iter()
+            .zip_eq(labels.iter())
+            .map(|(name, label)| KeyValue::new(name.to_string(), label.to_string()))
+            .collect();
+        Histogram { histogram, labels }
+    }
+}
+
+/// OpenTelemetry metrics registry.
+#[derive(Debug)]
+pub struct OpenTelemetryMetricsRegistry {
+    meter: Meter,
+}
+
+impl OpenTelemetryMetricsRegistry {
+    /// Create an OpenTelemetry metrics registry.
+    pub fn new(meter: Meter) -> Self {
+        Self { meter }
+    }
+}
+
+impl RegistryOps for OpenTelemetryMetricsRegistry {
+    fn register_counter_vec(
+        &self,
+        name: &'static str,
+        desc: &'static str,
+        label_names: &'static [&'static str],
+    ) -> impl CounterVecOps {
+        let meter = self.meter.clone();
+        MetricVec {
+            meter,
+            name,
+            desc,
+            label_names,
+        }
+    }
+
+    fn register_gauge_vec(
+        &self,
+        name: &'static str,
+        desc: &'static str,
+        label_names: &'static [&'static str],
+    ) -> impl GaugeVecOps {
+        let meter = self.meter.clone();
+        MetricVec {
+            meter,
+            name,
+            desc,
+            label_names,
+        }
+    }
+
+    fn register_histogram_vec(
+        &self,
+        name: &'static str,
+        desc: &'static str,
+        label_names: &'static [&'static str],
+    ) -> impl HistogramVecOps {
+        let meter = self.meter.clone();
+        MetricVec {
+            meter,
+            name,
+            desc,
+            label_names,
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test() {
+        let meter = opentelemetry::global::meter("test");
+        let ot = OpenTelemetryMetricsRegistry::new(meter);
+
+        let cv = ot.register_counter_vec("test_counter_1", "test counter 1", &["label1", "label2"]);
+        let c = cv.counter(&["l1", "l2"]);
+        c.increase(42);
+
+        let gv = ot.register_gauge_vec("test_gauge_1", "test gauge 1", &["label1", "label2"]);
+        let g = gv.gauge(&["l1", "l2"]);
+        g.increase(514);
+        g.decrease(114);
+        g.absolute(114514);
+
+        let hv = ot.register_histogram_vec("test_histogram_1", "test histogram 1", &["label1", "label2"]);
+        let h = hv.histogram(&["l1", "l2"]);
+        h.record(114.514);
+    }
+}
diff --git a/foyer-common/src/metrics/registry/prometheus.rs b/foyer-common/src/metrics/registry/prometheus.rs
new file mode 100644
index 00000000..18a9bc35
--- /dev/null
+++ b/foyer-common/src/metrics/registry/prometheus.rs
@@ -0,0 +1,140 @@
+//  Copyright 2024 foyer Project Authors
+//
+//  Licensed under the Apache License, Version 2.0 (the "License");
+//  you may not use this file except in compliance with the License.
+//  You may obtain a copy of the License at
+//
+//  http://www.apache.org/licenses/LICENSE-2.0
+//
+//  Unless required by applicable law or agreed to in writing, software
+//  distributed under the License is distributed on an "AS IS" BASIS,
+//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//  See the License for the specific language governing permissions and
+//  limitations under the License.
+
+use prometheus::{
+    register_histogram_vec_with_registry, register_int_counter_vec_with_registry, register_int_gauge_vec_with_registry,
+    Histogram, HistogramVec, IntCounter, IntCounterVec, IntGauge, IntGaugeVec, Registry,
+};
+
+use crate::metrics::{CounterOps, CounterVecOps, GaugeOps, GaugeVecOps, HistogramOps, HistogramVecOps, RegistryOps};
+
+impl CounterOps for IntCounter {
+    fn increase(&self, val: u64) {
+        self.inc_by(val);
+    }
+}
+
+impl CounterVecOps for IntCounterVec {
+    fn counter(&self, labels: &[&str]) -> impl CounterOps {
+        self.with_label_values(labels)
+    }
+}
+
+impl GaugeOps for IntGauge {
+    fn increase(&self, val: u64) {
+        self.add(val as _);
+    }
+
+    fn decrease(&self, val: u64) {
+        self.sub(val as _);
+    }
+
+    fn absolute(&self, val: u64) {
+        self.set(val as _);
+    }
+}
+
+impl GaugeVecOps for IntGaugeVec {
+    fn gauge(&self, labels: &[&str]) -> impl GaugeOps {
+        self.with_label_values(labels)
+    }
+}
+
+impl HistogramOps for Histogram {
+    fn record(&self, val: f64) {
+        self.observe(val);
+    }
+}
+
+impl HistogramVecOps for HistogramVec {
+    fn histogram(&self, labels: &[&str]) -> impl HistogramOps {
+        self.with_label_values(labels)
+    }
+}
+
+/// Prometheus metrics registry.
+#[derive(Debug)]
+pub struct PrometheusMetricsRegistry {
+    registry: Registry,
+}
+
+impl PrometheusMetricsRegistry {
+    /// Create an Prometheus metrics registry.
+    pub fn new(registry: Registry) -> Self {
+        Self { registry }
+    }
+}
+
+impl RegistryOps for PrometheusMetricsRegistry {
+    fn register_counter_vec(
+        &self,
+        name: &'static str,
+        desc: &'static str,
+        label_names: &'static [&'static str],
+    ) -> impl CounterVecOps {
+        register_int_counter_vec_with_registry! {
+            name, desc, label_names, self.registry
+        }
+        .unwrap()
+    }
+
+    fn register_gauge_vec(
+        &self,
+        name: &'static str,
+        desc: &'static str,
+        label_names: &'static [&'static str],
+    ) -> impl GaugeVecOps {
+        register_int_gauge_vec_with_registry! {
+            name, desc, label_names, self.registry
+        }
+        .unwrap()
+    }
+
+    fn register_histogram_vec(
+        &self,
+        name: &'static str,
+        desc: &'static str,
+        label_names: &'static [&'static str],
+    ) -> impl HistogramVecOps {
+        register_histogram_vec_with_registry! {
+            name, desc, label_names, self.registry
+        }
+        .unwrap()
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test() {
+        let registry = Registry::new();
+        let p8s = PrometheusMetricsRegistry::new(registry);
+
+        let cv = p8s.register_counter_vec("test_counter_1", "test counter 1", &["label1", "label2"]);
+        let c = cv.counter(&["l1", "l2"]);
+        c.increase(42);
+
+        let gv = p8s.register_gauge_vec("test_gauge_1", "test gauge 1", &["label1", "label2"]);
+        let g = gv.gauge(&["l1", "l2"]);
+        g.increase(514);
+        g.decrease(114);
+        g.absolute(114514);
+
+        let hv = p8s.register_histogram_vec("test_histogram_1", "test histogram 1", &["label1", "label2"]);
+        let h = hv.histogram(&["l1", "l2"]);
+        h.record(114.514);
+    }
+}
diff --git a/foyer-memory/src/cache.rs b/foyer-memory/src/cache.rs
index 4edf75cf..af22cd45 100644
--- a/foyer-memory/src/cache.rs
+++ b/foyer-memory/src/cache.rs
@@ -20,6 +20,7 @@ use foyer_common::{
     code::{HashBuilder, Key, Value},
     event::EventListener,
     future::Diversion,
+    metrics::{model::Metrics, registry::noop::NoopMetricsRegistry, RegistryOps},
     runtime::SingletonHandle,
 };
 use pin_project::pin_project;
@@ -264,13 +265,13 @@ impl From<LfuConfig> for EvictionConfig {
 }
 
 /// In-memory cache builder.
-pub struct CacheBuilder<K, V, S>
+pub struct CacheBuilder<K, V, S, M>
 where
     K: Key,
     V: Value,
     S: HashBuilder,
 {
-    name: String,
+    name: &'static str,
 
     capacity: usize,
     shards: usize,
@@ -280,9 +281,12 @@ where
     weighter: Arc<dyn Weighter<K, V>>,
 
     event_listener: Option<Arc<dyn EventListener<Key = K, Value = V>>>,
+
+    registry: M,
+    metrics: Option<Arc<Metrics>>,
 }
 
-impl<K, V> CacheBuilder<K, V, RandomState>
+impl<K, V> CacheBuilder<K, V, RandomState, NoopMetricsRegistry>
 where
     K: Key,
     V: Value,
@@ -290,7 +294,7 @@ where
     /// Create a new in-memory cache builder.
     pub fn new(capacity: usize) -> Self {
         Self {
-            name: "foyer".to_string(),
+            name: "foyer",
 
             capacity,
             shards: 8,
@@ -299,11 +303,14 @@ where
             hash_builder: RandomState::default(),
             weighter: Arc::new(|_, _| 1),
             event_listener: None,
+
+            registry: NoopMetricsRegistry,
+            metrics: None,
         }
     }
 }
 
-impl<K, V, S> CacheBuilder<K, V, S>
+impl<K, V, S, M> CacheBuilder<K, V, S, M>
 where
     K: Key,
     V: Value,
@@ -314,8 +321,8 @@ where
     /// foyer will use the name as the prefix of the metric names.
     ///
     /// Default: `foyer`.
-    pub fn with_name(mut self, name: &str) -> Self {
-        self.name = name.to_string();
+    pub fn with_name(mut self, name: &'static str) -> Self {
+        self.name = name;
         self
     }
 
@@ -335,7 +342,7 @@ where
     }
 
     /// Set in-memory cache hash builder.
-    pub fn with_hash_builder<OS>(self, hash_builder: OS) -> CacheBuilder<K, V, OS>
+    pub fn with_hash_builder<OS>(self, hash_builder: OS) -> CacheBuilder<K, V, OS, M>
     where
         OS: HashBuilder,
     {
@@ -347,6 +354,8 @@ where
             hash_builder,
             weighter: self.weighter,
             event_listener: self.event_listener,
+            registry: self.registry,
+            metrics: self.metrics,
         }
     }
 
@@ -362,8 +371,40 @@ where
         self
     }
 
+    /// Set metrics registry.
+    ///
+    /// Default: [`NoopMetricsRegistry`].
+    pub fn with_metrics_registry<OM>(self, registry: OM) -> CacheBuilder<K, V, S, OM>
+    where
+        OM: RegistryOps,
+    {
+        CacheBuilder {
+            name: self.name,
+            capacity: self.capacity,
+            shards: self.shards,
+            eviction_config: self.eviction_config,
+            hash_builder: self.hash_builder,
+            weighter: self.weighter,
+            event_listener: self.event_listener,
+            registry,
+            metrics: self.metrics,
+        }
+    }
+
+    /// Set metrics.
+    ///
+    /// Note: `with_metrics` is only supposed to be called by other foyer components.
+    #[doc(hidden)]
+    pub fn with_metrics(mut self, metrics: Arc<Metrics>) -> Self {
+        self.metrics = Some(metrics);
+        self
+    }
+
     /// Build in-memory cache with the given configuration.
-    pub fn build(self) -> Cache<K, V, S> {
+    pub fn build(self) -> Cache<K, V, S>
+    where
+        M: RegistryOps,
+    {
         if self.capacity < self.shards {
             tracing::warn!(
                 "The in-memory cache capacity({}) < shards({}).",
@@ -372,42 +413,46 @@ where
             );
         }
 
+        let metrics = self
+            .metrics
+            .unwrap_or_else(|| Arc::new(Metrics::new(self.name, &self.registry)));
+
         match self.eviction_config {
             EvictionConfig::Fifo(eviction_config) => Cache::Fifo(Arc::new(RawCache::new(RawCacheConfig {
-                name: self.name,
                 capacity: self.capacity,
                 shards: self.shards,
                 eviction_config,
                 hash_builder: self.hash_builder,
                 weighter: self.weighter,
                 event_listener: self.event_listener,
+                metrics,
             }))),
             EvictionConfig::S3Fifo(eviction_config) => Cache::S3Fifo(Arc::new(RawCache::new(RawCacheConfig {
-                name: self.name,
                 capacity: self.capacity,
                 shards: self.shards,
                 eviction_config,
                 hash_builder: self.hash_builder,
                 weighter: self.weighter,
                 event_listener: self.event_listener,
+                metrics,
             }))),
             EvictionConfig::Lru(eviction_config) => Cache::Lru(Arc::new(RawCache::new(RawCacheConfig {
-                name: self.name,
                 capacity: self.capacity,
                 shards: self.shards,
                 eviction_config,
                 hash_builder: self.hash_builder,
                 weighter: self.weighter,
                 event_listener: self.event_listener,
+                metrics,
             }))),
             EvictionConfig::Lfu(eviction_config) => Cache::Lfu(Arc::new(RawCache::new(RawCacheConfig {
-                name: self.name,
                 capacity: self.capacity,
                 shards: self.shards,
                 eviction_config,
                 hash_builder: self.hash_builder,
                 weighter: self.weighter,
                 event_listener: self.event_listener,
+                metrics,
             }))),
         }
     }
diff --git a/foyer-memory/src/raw.rs b/foyer-memory/src/raw.rs
index 45ba0a40..93d85980 100644
--- a/foyer-memory/src/raw.rs
+++ b/foyer-memory/src/raw.rs
@@ -32,7 +32,7 @@ use foyer_common::{
     code::HashBuilder,
     event::{Event, EventListener},
     future::{Diversion, DiversionFuture},
-    metrics::Metrics,
+    metrics::model::Metrics,
     runtime::SingletonHandle,
     scope::Scope,
     strict_assert,
@@ -59,13 +59,13 @@ where
     E: Eviction,
     S: HashBuilder,
 {
-    pub name: String,
     pub capacity: usize,
     pub shards: usize,
     pub eviction_config: E::Config,
     pub hash_builder: S,
     pub weighter: Arc<dyn Weighter<E::Key, E::Value>>,
     pub event_listener: Option<Arc<dyn EventListener<Key = E::Key, Value = E::Value>>>,
+    pub metrics: Arc<Metrics>,
 }
 
 struct RawCacheShard<E, S, I>
@@ -103,6 +103,7 @@ where
         std::mem::swap(waiters, &mut self.waiters.lock().remove(&data.key).unwrap_or_default());
 
         let weight = data.weight;
+        let old_usage = self.usage;
 
         let record = Arc::new(Record::new(data));
 
@@ -112,7 +113,7 @@ where
                 Some(evicted) => evicted,
                 None => break,
             };
-            self.metrics.memory_evict.increment(1);
+            self.metrics.memory_evict.increase(1);
 
             let e = self.indexer.remove(evicted.hash(), evicted.key()).unwrap();
             assert_eq!(Arc::as_ptr(&evicted), Arc::as_ptr(&e));
@@ -127,7 +128,7 @@ where
 
         // Insert new record
         if let Some(old) = self.indexer.insert(record.clone()) {
-            self.metrics.memory_replace.increment(1);
+            self.metrics.memory_replace.increase(1);
 
             strict_assert!(!old.is_in_indexer());
 
@@ -140,7 +141,7 @@ where
 
             garbages.push((Event::Replace, old));
         } else {
-            self.metrics.memory_insert.increment(1);
+            self.metrics.memory_insert.increase(1);
         }
         strict_assert!(record.is_in_indexer());
 
@@ -151,13 +152,18 @@ where
         }
 
         self.usage += weight;
-        self.metrics.memory_usage.increment(weight as f64);
         // Increase the reference count within the lock section.
         // The reference count of the new record must be at the moment.
         let refs = waiters.len() + 1;
         let inc = record.inc_refs(refs);
         assert_eq!(refs, inc);
 
+        match self.usage.cmp(&old_usage) {
+            std::cmp::Ordering::Greater => self.metrics.memory_usage.increase((self.usage - old_usage) as _),
+            std::cmp::Ordering::Less => self.metrics.memory_usage.decrease((old_usage - self.usage) as _),
+            std::cmp::Ordering::Equal => {}
+        }
+
         record
     }
 
@@ -176,7 +182,8 @@ where
 
         self.usage -= record.weight();
 
-        self.metrics.memory_remove.increment(1);
+        self.metrics.memory_remove.increase(1);
+        self.metrics.memory_usage.decrease(record.weight() as _);
 
         record.inc_refs(1);
 
@@ -215,11 +222,11 @@ where
     {
         let record = match self.indexer.get(hash, key).cloned() {
             Some(record) => {
-                self.metrics.memory_hit.increment(1);
+                self.metrics.memory_hit.increase(1);
                 record
             }
             None => {
-                self.metrics.memory_miss.increment(1);
+                self.metrics.memory_miss.increase(1);
                 return None;
             }
         };
@@ -248,7 +255,7 @@ where
             garbages.push(record);
         }
 
-        self.metrics.memory_remove.increment(count);
+        self.metrics.memory_remove.increase(count);
     }
 
     #[fastrace::trace(name = "foyer::memory::raw::shard::acquire_immutable")]
@@ -325,14 +332,14 @@ where
             HashMapEntry::Occupied(mut o) => {
                 let (tx, rx) = oneshot::channel();
                 o.get_mut().push(tx);
-                self.metrics.memory_queue.increment(1);
+                self.metrics.memory_queue.increase(1);
                 RawShardFetch::Wait(rx.in_span(Span::enter_with_local_parent(
                     "foyer::memory::raw::fetch_with_runtime::wait",
                 )))
             }
             HashMapEntry::Vacant(v) => {
                 v.insert(vec![]);
-                self.metrics.memory_fetch.increment(1);
+                self.metrics.memory_fetch.increase(1);
                 RawShardFetch::Miss
             }
         }
@@ -420,8 +427,6 @@ where
     I: Indexer<Eviction = E>,
 {
     pub fn new(config: RawCacheConfig<E, S>) -> Self {
-        let metrics = Arc::new(Metrics::new(&config.name));
-
         let shard_capacity = config.capacity / config.shards;
 
         let shards = (0..config.shards)
@@ -431,7 +436,7 @@ where
                 usage: 0,
                 capacity: shard_capacity,
                 waiters: Mutex::default(),
-                metrics: metrics.clone(),
+                metrics: config.metrics.clone(),
                 _event_listener: config.event_listener.clone(),
             })
             .map(RwLock::new)
@@ -442,7 +447,7 @@ where
             capacity: config.capacity,
             hash_builder: config.hash_builder,
             weighter: config.weighter,
-            metrics,
+            metrics: config.metrics,
             event_listener: config.event_listener,
         };
 
@@ -945,13 +950,13 @@ mod tests {
 
     fn fifo_cache_for_test() -> RawCache<Fifo<u64, u64>> {
         RawCache::new(RawCacheConfig {
-            name: "test".to_string(),
             capacity: 256,
             shards: 4,
             eviction_config: FifoConfig::default(),
             hash_builder: Default::default(),
             weighter: Arc::new(|_, _| 1),
             event_listener: None,
+            metrics: Arc::new(Metrics::noop()),
         })
     }
 
@@ -1016,13 +1021,13 @@ mod tests {
         #[test_log::test]
         fn test_fifo_cache_fuzzy() {
             let cache: RawCache<Fifo<u64, u64>> = RawCache::new(RawCacheConfig {
-                name: "test".to_string(),
                 capacity: 256,
                 shards: 4,
                 eviction_config: FifoConfig::default(),
                 hash_builder: Default::default(),
                 weighter: Arc::new(|_, _| 1),
                 event_listener: None,
+                metrics: Arc::new(Metrics::noop()),
             });
             let hints = vec![FifoHint];
             fuzzy(cache, hints);
@@ -1031,13 +1036,13 @@ mod tests {
         #[test_log::test]
         fn test_s3fifo_cache_fuzzy() {
             let cache: RawCache<S3Fifo<u64, u64>> = RawCache::new(RawCacheConfig {
-                name: "test".to_string(),
                 capacity: 256,
                 shards: 4,
                 eviction_config: S3FifoConfig::default(),
                 hash_builder: Default::default(),
                 weighter: Arc::new(|_, _| 1),
                 event_listener: None,
+                metrics: Arc::new(Metrics::noop()),
             });
             let hints = vec![S3FifoHint];
             fuzzy(cache, hints);
@@ -1046,13 +1051,13 @@ mod tests {
         #[test_log::test]
         fn test_lru_cache_fuzzy() {
             let cache: RawCache<Lru<u64, u64>> = RawCache::new(RawCacheConfig {
-                name: "test".to_string(),
                 capacity: 256,
                 shards: 4,
                 eviction_config: LruConfig::default(),
                 hash_builder: Default::default(),
                 weighter: Arc::new(|_, _| 1),
                 event_listener: None,
+                metrics: Arc::new(Metrics::noop()),
             });
             let hints = vec![LruHint::HighPriority, LruHint::LowPriority];
             fuzzy(cache, hints);
@@ -1061,13 +1066,13 @@ mod tests {
         #[test_log::test]
         fn test_lfu_cache_fuzzy() {
             let cache: RawCache<Lfu<u64, u64>> = RawCache::new(RawCacheConfig {
-                name: "test".to_string(),
                 capacity: 256,
                 shards: 4,
                 eviction_config: LfuConfig::default(),
                 hash_builder: Default::default(),
                 weighter: Arc::new(|_, _| 1),
                 event_listener: None,
+                metrics: Arc::new(Metrics::noop()),
             });
             let hints = vec![LfuHint];
             fuzzy(cache, hints);
diff --git a/foyer-storage/src/device/monitor.rs b/foyer-storage/src/device/monitor.rs
index 1e897b51..bfef7774 100644
--- a/foyer-storage/src/device/monitor.rs
+++ b/foyer-storage/src/device/monitor.rs
@@ -21,7 +21,7 @@ use std::{
     time::Instant,
 };
 
-use foyer_common::{bits, metrics::Metrics};
+use foyer_common::{bits, metrics::model::Metrics};
 
 use super::RegionId;
 use crate::{error::Result, Dev, DevExt, DirectFileDevice, IoBytes, IoBytesMut, Runtime};
@@ -97,9 +97,11 @@ where
 
         let res = self.device.write(buf, region, offset).await;
 
-        self.metrics.storage_disk_write.increment(1);
-        self.metrics.storage_disk_write_bytes.increment(bytes as u64);
-        self.metrics.storage_disk_write_duration.record(now.elapsed());
+        self.metrics.storage_disk_write.increase(1);
+        self.metrics.storage_disk_write_bytes.increase(bytes as u64);
+        self.metrics
+            .storage_disk_write_duration
+            .record(now.elapsed().as_secs_f64());
 
         res
     }
@@ -114,9 +116,11 @@ where
 
         let res = self.device.read(region, offset, len).await;
 
-        self.metrics.storage_disk_read.increment(1);
-        self.metrics.storage_disk_read_bytes.increment(bytes as u64);
-        self.metrics.storage_disk_read_duration.record(now.elapsed());
+        self.metrics.storage_disk_read.increase(1);
+        self.metrics.storage_disk_read_bytes.increase(bytes as u64);
+        self.metrics
+            .storage_disk_read_duration
+            .record(now.elapsed().as_secs_f64());
 
         res
     }
@@ -129,8 +133,10 @@ where
 
         let res = self.device.flush(region).await;
 
-        self.metrics.storage_disk_flush.increment(1);
-        self.metrics.storage_disk_flush_duration.record(now.elapsed());
+        self.metrics.storage_disk_flush.increase(1);
+        self.metrics
+            .storage_disk_flush_duration
+            .record(now.elapsed().as_secs_f64());
 
         res
     }
@@ -178,9 +184,11 @@ impl Monitored<DirectFileDevice> {
 
         let res = self.device.pwrite(buf, offset).await;
 
-        self.metrics.storage_disk_write.increment(1);
-        self.metrics.storage_disk_write_bytes.increment(bytes as u64);
-        self.metrics.storage_disk_write_duration.record(now.elapsed());
+        self.metrics.storage_disk_write.increase(1);
+        self.metrics.storage_disk_write_bytes.increase(bytes as u64);
+        self.metrics
+            .storage_disk_write_duration
+            .record(now.elapsed().as_secs_f64());
 
         res
     }
@@ -195,9 +203,11 @@ impl Monitored<DirectFileDevice> {
 
         let res = self.device.pread(offset, len).await;
 
-        self.metrics.storage_disk_read.increment(1);
-        self.metrics.storage_disk_read_bytes.increment(bytes as u64);
-        self.metrics.storage_disk_read_duration.record(now.elapsed());
+        self.metrics.storage_disk_read.increase(1);
+        self.metrics.storage_disk_read_bytes.increase(bytes as u64);
+        self.metrics
+            .storage_disk_read_duration
+            .record(now.elapsed().as_secs_f64());
 
         res
     }
diff --git a/foyer-storage/src/large/batch.rs b/foyer-storage/src/large/batch.rs
index 5adcea96..bc8d22b8 100644
--- a/foyer-storage/src/large/batch.rs
+++ b/foyer-storage/src/large/batch.rs
@@ -17,7 +17,7 @@ use std::{fmt::Debug, ops::Range, sync::Arc, time::Instant};
 use foyer_common::{
     bits,
     code::{HashBuilder, StorageKey, StorageValue},
-    metrics::Metrics,
+    metrics::model::Metrics,
     range::RangeBoundsExt,
     strict_assert_eq,
 };
diff --git a/foyer-storage/src/large/flusher.rs b/foyer-storage/src/large/flusher.rs
index c3e1053d..e562b85f 100644
--- a/foyer-storage/src/large/flusher.rs
+++ b/foyer-storage/src/large/flusher.rs
@@ -23,7 +23,7 @@ use std::{
 
 use foyer_common::{
     code::{HashBuilder, StorageKey, StorageValue},
-    metrics::Metrics,
+    metrics::model::Metrics,
 };
 use foyer_memory::CacheEntry;
 use futures::future::{try_join, try_join_all};
@@ -253,7 +253,7 @@ where
     fn submit(&mut self, submission: Submission<K, V, S>) {
         let report = |enqueued: bool| {
             if !enqueued {
-                self.metrics.storage_queue_drop.increment(1);
+                self.metrics.storage_queue_drop.increase(1);
             }
         };
 
@@ -345,8 +345,10 @@ where
         }
 
         if let Some(init) = batch.init.as_ref() {
-            self.metrics.storage_queue_rotate.increment(1);
-            self.metrics.storage_queue_rotate_duration.record(init.elapsed());
+            self.metrics.storage_queue_rotate.increase(1);
+            self.metrics
+                .storage_queue_rotate_duration
+                .record(init.elapsed().as_secs_f64());
         }
 
         drop(permit);
diff --git a/foyer-storage/src/large/generic.rs b/foyer-storage/src/large/generic.rs
index f4b4b788..75ae6269 100644
--- a/foyer-storage/src/large/generic.rs
+++ b/foyer-storage/src/large/generic.rs
@@ -28,7 +28,7 @@ use fastrace::prelude::*;
 use foyer_common::{
     bits,
     code::{HashBuilder, StorageKey, StorageValue},
-    metrics::Metrics,
+    metrics::model::Metrics,
 };
 use foyer_memory::CacheEntry;
 use futures::future::{join_all, try_join_all};
@@ -64,7 +64,6 @@ where
     V: StorageValue,
     S: HashBuilder + Debug,
 {
-    pub name: String,
     pub device: MonitoredDevice,
     pub regions: Range<RegionId>,
     pub compression: Compression,
@@ -93,7 +92,6 @@ where
 {
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
         f.debug_struct("GenericStoreConfig")
-            .field("name", &self.name)
             .field("device", &self.device)
             .field("compression", &self.compression)
             .field("flush", &self.flush)
@@ -341,8 +339,8 @@ where
             let addr = match indexer.get(hash) {
                 Some(addr) => addr,
                 None => {
-                    metrics.storage_miss.increment(1);
-                    metrics.storage_miss_duration.record(now.elapsed());
+                    metrics.storage_miss.increase(1);
+                    metrics.storage_miss_duration.record(now.elapsed().as_secs_f64());
                     return Ok(None);
                 }
             };
@@ -362,8 +360,8 @@ where
                 | Err(e @ Error::CompressionAlgorithmNotSupported(_)) => {
                     tracing::trace!("deserialize entry header error: {e}, remove this entry and skip");
                     indexer.remove(hash);
-                    metrics.storage_miss.increment(1);
-                    metrics.storage_miss_duration.record(now.elapsed());
+                    metrics.storage_miss.increase(1);
+                    metrics.storage_miss_duration.record(now.elapsed().as_secs_f64());
                     return Ok(None);
                 }
                 Err(e) => return Err(e),
@@ -381,15 +379,15 @@ where
                 Err(e @ Error::MagicMismatch { .. }) | Err(e @ Error::ChecksumMismatch { .. }) => {
                     tracing::trace!("deserialize read buffer raise error: {e}, remove this entry and skip");
                     indexer.remove(hash);
-                    metrics.storage_miss.increment(1);
-                    metrics.storage_miss_duration.record(now.elapsed());
+                    metrics.storage_miss.increase(1);
+                    metrics.storage_miss_duration.record(now.elapsed().as_secs_f64());
                     return Ok(None);
                 }
                 Err(e) => return Err(e),
             };
 
-            metrics.storage_hit.increment(1);
-            metrics.storage_hit_duration.record(now.elapsed());
+            metrics.storage_hit.increase(1);
+            metrics.storage_hit_duration.record(now.elapsed().as_secs_f64());
 
             Ok(Some((k, v)))
         }
@@ -418,8 +416,11 @@ where
             });
         });
 
-        self.inner.metrics.storage_delete.increment(1);
-        self.inner.metrics.storage_miss_duration.record(now.elapsed());
+        self.inner.metrics.storage_delete.increase(1);
+        self.inner
+            .metrics
+            .storage_miss_duration
+            .record(now.elapsed().as_secs_f64());
     }
 
     fn may_contains(&self, hash: u64) -> bool {
@@ -546,7 +547,7 @@ mod tests {
                     .with_capacity(ByteSize::kib(64).as_u64() as _)
                     .with_file_size(ByteSize::kib(16).as_u64() as _)
                     .into(),
-                metrics: Arc::new(Metrics::new("test")),
+                metrics: Arc::new(Metrics::noop()),
             },
             runtime,
         )
@@ -566,7 +567,6 @@ mod tests {
         let device = device_for_test(dir).await;
         let regions = 0..device.regions() as RegionId;
         let config = GenericLargeStorageConfig {
-            name: "test".to_string(),
             device,
             regions,
             compression: Compression::None,
@@ -596,7 +596,6 @@ mod tests {
         let device = device_for_test(dir).await;
         let regions = 0..device.regions() as RegionId;
         let config = GenericLargeStorageConfig {
-            name: "test".to_string(),
             device,
             regions,
             compression: Compression::None,
diff --git a/foyer-storage/src/large/reclaimer.rs b/foyer-storage/src/large/reclaimer.rs
index ac2ffb64..62be4b19 100644
--- a/foyer-storage/src/large/reclaimer.rs
+++ b/foyer-storage/src/large/reclaimer.rs
@@ -16,7 +16,7 @@ use std::{fmt::Debug, future::Future, sync::Arc, time::Duration};
 
 use foyer_common::{
     code::{HashBuilder, StorageKey, StorageValue},
-    metrics::Metrics,
+    metrics::model::Metrics,
 };
 use futures::future::join_all;
 use itertools::Itertools;
diff --git a/foyer-storage/src/large/recover.rs b/foyer-storage/src/large/recover.rs
index cc4bf827..08df950e 100644
--- a/foyer-storage/src/large/recover.rs
+++ b/foyer-storage/src/large/recover.rs
@@ -22,7 +22,7 @@ use std::{
 use clap::ValueEnum;
 use foyer_common::{
     code::{HashBuilder, StorageKey, StorageValue},
-    metrics::Metrics,
+    metrics::model::Metrics,
 };
 use futures::future::try_join_all;
 use itertools::Itertools;
diff --git a/foyer-storage/src/large/scanner.rs b/foyer-storage/src/large/scanner.rs
index 9704f6f0..de1a7146 100644
--- a/foyer-storage/src/large/scanner.rs
+++ b/foyer-storage/src/large/scanner.rs
@@ -17,7 +17,7 @@ use std::sync::Arc;
 use foyer_common::{
     bits,
     code::{StorageKey, StorageValue},
-    metrics::Metrics,
+    metrics::model::Metrics,
     strict_assert,
 };
 
@@ -261,7 +261,7 @@ mod tests {
                     .with_capacity(ByteSize::kib(64).as_u64() as _)
                     .with_file_size(ByteSize::kib(16).as_u64() as _)
                     .into(),
-                metrics: Arc::new(Metrics::new("test")),
+                metrics: Arc::new(Metrics::noop()),
             },
             runtime,
         )
diff --git a/foyer-storage/src/large/tombstone.rs b/foyer-storage/src/large/tombstone.rs
index c5da6a5a..88a1f196 100644
--- a/foyer-storage/src/large/tombstone.rs
+++ b/foyer-storage/src/large/tombstone.rs
@@ -19,7 +19,7 @@ use std::{
 
 use array_util::SliceExt;
 use bytes::{Buf, BufMut};
-use foyer_common::{bits, metrics::Metrics, strict_assert_eq};
+use foyer_common::{bits, metrics::model::Metrics, strict_assert_eq};
 use futures::future::try_join_all;
 use tokio::sync::Mutex;
 
@@ -335,7 +335,7 @@ mod tests {
             device.clone(),
             true,
             &mut vec![],
-            Arc::new(Metrics::new("test")),
+            Arc::new(Metrics::noop()),
             runtime.clone(),
         )
         .await
@@ -365,7 +365,7 @@ mod tests {
             device,
             true,
             &mut vec![],
-            Arc::new(Metrics::new("test")),
+            Arc::new(Metrics::noop()),
             runtime,
         )
         .await
diff --git a/foyer-storage/src/region.rs b/foyer-storage/src/region.rs
index fad51781..969f409f 100644
--- a/foyer-storage/src/region.rs
+++ b/foyer-storage/src/region.rs
@@ -25,7 +25,7 @@ use std::{
 };
 
 use async_channel::{Receiver, Sender};
-use foyer_common::{countdown::Countdown, metrics::Metrics};
+use foyer_common::{countdown::Countdown, metrics::model::Metrics};
 use futures::{
     future::{BoxFuture, Shared},
     FutureExt,
@@ -158,8 +158,8 @@ impl RegionManager {
             .collect_vec();
         let (clean_region_tx, clean_region_rx) = async_channel::unbounded();
 
-        metrics.storage_region_total.set(device.regions() as f64);
-        metrics.storage_region_size_bytes.set(device.region_size() as f64);
+        metrics.storage_region_total.absolute(device.regions() as _);
+        metrics.storage_region_size_bytes.absolute(device.region_size() as _);
 
         Self {
             inner: Arc::new(RegionManagerInner {
@@ -199,7 +199,7 @@ impl RegionManager {
         std::mem::swap(&mut eviction.eviction_pickers, &mut pickers);
         assert!(pickers.is_empty());
 
-        self.inner.metrics.storage_region_evictable.increment(1);
+        self.inner.metrics.storage_region_evictable.increase(1);
 
         tracing::debug!("[region manager]: Region {region} is marked evictable.");
     }
@@ -237,7 +237,7 @@ impl RegionManager {
 
         // Update evictable map.
         eviction.evictable.remove(&picked).unwrap();
-        self.inner.metrics.storage_region_evictable.decrement(1);
+        self.inner.metrics.storage_region_evictable.decrease(1);
 
         // Notify pickers.
         for picker in pickers.iter_mut() {
@@ -260,7 +260,7 @@ impl RegionManager {
             .send(self.region(region).clone())
             .await
             .unwrap();
-        self.inner.metrics.storage_region_clean.increment(1);
+        self.inner.metrics.storage_region_clean.increase(1);
     }
 
     pub fn get_clean_region(&self) -> GetCleanRegionHandle {
@@ -277,7 +277,7 @@ impl RegionManager {
                 if reclaim_semaphore_countdown.countdown() {
                     reclaim_semaphore.add_permits(1);
                 }
-                metrics.storage_region_clean.decrement(1);
+                metrics.storage_region_clean.decrease(1);
                 region
             }
             .boxed(),
diff --git a/foyer-storage/src/serde.rs b/foyer-storage/src/serde.rs
index a466cbba..2fbf6028 100644
--- a/foyer-storage/src/serde.rs
+++ b/foyer-storage/src/serde.rs
@@ -16,7 +16,7 @@ use std::{fmt::Debug, hash::Hasher, io::Write, time::Instant};
 
 use foyer_common::{
     code::{StorageKey, StorageValue},
-    metrics::Metrics,
+    metrics::model::Metrics,
 };
 use twox_hash::{XxHash32, XxHash64};
 
@@ -145,7 +145,9 @@ impl EntrySerializer {
         bincode::serialize_into(&mut writer, &key).map_err(Error::from)?;
         let key_len = writer.written();
 
-        metrics.storage_entry_serialize_duration.record(now.elapsed());
+        metrics
+            .storage_entry_serialize_duration
+            .record(now.elapsed().as_secs_f64());
 
         Ok(KvInfo { key_len, value_len })
     }
@@ -195,7 +197,9 @@ impl EntryDeserializer {
             }
         }
 
-        metrics.storage_entry_deserialize_duration.record(now.elapsed());
+        metrics
+            .storage_entry_deserialize_duration
+            .record(now.elapsed().as_secs_f64());
 
         Ok((key, value))
     }
diff --git a/foyer-storage/src/small/batch.rs b/foyer-storage/src/small/batch.rs
index 9c3baeaf..ce2a99ab 100644
--- a/foyer-storage/src/small/batch.rs
+++ b/foyer-storage/src/small/batch.rs
@@ -23,7 +23,7 @@ use std::{
 use foyer_common::{
     bits,
     code::{HashBuilder, StorageKey, StorageValue},
-    metrics::Metrics,
+    metrics::model::Metrics,
 };
 use foyer_memory::CacheEntry;
 use itertools::Itertools;
diff --git a/foyer-storage/src/small/flusher.rs b/foyer-storage/src/small/flusher.rs
index 902c1ee4..1aa0aa3c 100644
--- a/foyer-storage/src/small/flusher.rs
+++ b/foyer-storage/src/small/flusher.rs
@@ -20,7 +20,7 @@ use std::{
 
 use foyer_common::{
     code::{HashBuilder, StorageKey, StorageValue},
-    metrics::Metrics,
+    metrics::model::Metrics,
 };
 use foyer_memory::CacheEntry;
 use futures::future::try_join_all;
@@ -182,7 +182,7 @@ where
     fn submit(&mut self, submission: Submission<K, V, S>) {
         let report = |enqueued: bool| {
             if !enqueued {
-                self.metrics.storage_queue_drop.increment(1);
+                self.metrics.storage_queue_drop.increase(1);
             }
         };
 
diff --git a/foyer-storage/src/small/generic.rs b/foyer-storage/src/small/generic.rs
index c27e263e..fedf7487 100644
--- a/foyer-storage/src/small/generic.rs
+++ b/foyer-storage/src/small/generic.rs
@@ -281,7 +281,7 @@ mod tests {
     use std::path::Path;
 
     use bytesize::ByteSize;
-    use foyer_common::metrics::Metrics;
+    use foyer_common::metrics::model::Metrics;
     use foyer_memory::{Cache, CacheBuilder, FifoConfig};
     use tokio::runtime::Handle;
 
@@ -312,7 +312,7 @@ mod tests {
                     .with_capacity(ByteSize::kib(64).as_u64() as _)
                     .with_file_size(ByteSize::kib(16).as_u64() as _)
                     .into(),
-                metrics: Arc::new(Metrics::new("test")),
+                metrics: Arc::new(Metrics::noop()),
             },
             runtime,
         )
diff --git a/foyer-storage/src/small/set.rs b/foyer-storage/src/small/set.rs
index aa24fc0a..9ec0bb5c 100644
--- a/foyer-storage/src/small/set.rs
+++ b/foyer-storage/src/small/set.rs
@@ -353,10 +353,11 @@ impl SetTimestamp {
 #[cfg(test)]
 mod tests {
 
+    use foyer_common::metrics::model::Metrics;
     use foyer_memory::{Cache, CacheBuilder, CacheEntry};
 
     use super::*;
-    use crate::{serde::EntrySerializer, test_utils::metrics_for_test, Compression};
+    use crate::{serde::EntrySerializer, Compression};
 
     const PAGE: usize = 4096;
 
@@ -372,7 +373,7 @@ mod tests {
             entry.value(),
             &Compression::None,
             &mut buf,
-            metrics_for_test(),
+            &Metrics::noop(),
         )
         .unwrap();
 
diff --git a/foyer-storage/src/store.rs b/foyer-storage/src/store.rs
index f2379c02..7a7bada8 100644
--- a/foyer-storage/src/store.rs
+++ b/foyer-storage/src/store.rs
@@ -26,7 +26,7 @@ use equivalent::Equivalent;
 use foyer_common::{
     bits,
     code::{HashBuilder, StorageKey, StorageValue},
-    metrics::Metrics,
+    metrics::model::Metrics,
     runtime::BackgroundShutdownRuntime,
 };
 use foyer_memory::{Cache, CacheEntry};
@@ -144,8 +144,11 @@ where
             self.inner.engine.enqueue(entry, estimated_size);
         }
 
-        self.inner.metrics.storage_enqueue.increment(1);
-        self.inner.metrics.storage_enqueue_duration.record(now.elapsed());
+        self.inner.metrics.storage_enqueue.increase(1);
+        self.inner
+            .metrics
+            .storage_enqueue_duration
+            .record(now.elapsed().as_secs_f64());
     }
 
     /// Load a cache entry from the disk cache.
@@ -341,8 +344,9 @@ where
     V: StorageValue,
     S: HashBuilder + Debug,
 {
-    name: String,
+    name: &'static str,
     memory: Cache<K, V, S>,
+    metrics: Arc<Metrics>,
 
     device_options: DeviceOptions,
     engine: Engine,
@@ -364,14 +368,15 @@ where
     S: HashBuilder + Debug,
 {
     /// Setup disk cache store for the given in-memory cache.
-    pub fn new(memory: Cache<K, V, S>, engine: Engine) -> Self {
+    pub fn new(name: &'static str, memory: Cache<K, V, S>, metrics: Arc<Metrics>, engine: Engine) -> Self {
         if matches!(engine, Engine::Mixed(ratio) if !(0.0..=1.0).contains(&ratio)) {
             panic!("mixed engine small object disk cache ratio must be a f64 in range [0.0, 1.0]");
         }
 
         Self {
-            name: "foyer".to_string(),
+            name,
             memory,
+            metrics,
 
             device_options: DeviceOptions::None,
             engine,
@@ -387,16 +392,6 @@ where
         }
     }
 
-    /// Set the name of the foyer disk cache instance.
-    ///
-    /// foyer will use the name as the prefix of the metric names.
-    ///
-    /// Default: `foyer`.
-    pub fn with_name(mut self, name: &str) -> Self {
-        self.name = name.to_string();
-        self
-    }
-
     /// Set device options for the disk cache store.
     pub fn with_device_options(mut self, device_options: impl Into<DeviceOptions>) -> Self {
         self.device_options = device_options.into();
@@ -470,9 +465,9 @@ where
     /// Build the disk cache store with the given configuration.
     pub async fn build(self) -> Result<Store<K, V, S>> {
         let memory = self.memory.clone();
+        let metrics = self.metrics.clone();
         let admission_picker = self.admission_picker.clone();
 
-        let metrics = Arc::new(Metrics::new(&self.name));
         let statistics = Arc::<Statistics>::default();
 
         let compression = self.compression;
@@ -540,7 +535,6 @@ where
                         Engine::Large => {
                             let regions = 0..device.regions() as RegionId;
                             EngineEnum::open(EngineConfig::Large(GenericLargeStorageConfig {
-                                name: self.name,
                                 device,
                                 regions,
                                 compression: self.compression,
@@ -599,7 +593,6 @@ where
                                     marker: PhantomData,
                                 },
                                 right: GenericLargeStorageConfig {
-                                    name: self.name,
                                     device,
                                     regions: large_regions,
                                     compression: self.compression,
diff --git a/foyer-storage/src/test_utils.rs b/foyer-storage/src/test_utils.rs
index b043758f..d80b707b 100644
--- a/foyer-storage/src/test_utils.rs
+++ b/foyer-storage/src/test_utils.rs
@@ -19,10 +19,10 @@ use std::{
     collections::HashSet,
     fmt::Debug,
     hash::{BuildHasher, Hash, Hasher},
-    sync::{Arc, OnceLock},
+    sync::Arc,
 };
 
-use foyer_common::{code::StorageKey, metrics::Metrics};
+use foyer_common::code::StorageKey;
 use parking_lot::Mutex;
 
 use crate::{
@@ -30,14 +30,6 @@ use crate::{
     statistics::Statistics,
 };
 
-/// A phantom metrics for test.
-static METRICS_FOR_TEST: OnceLock<Metrics> = OnceLock::new();
-
-/// Get a phantom metrics for test.
-pub fn metrics_for_test() -> &'static Metrics {
-    METRICS_FOR_TEST.get_or_init(|| Metrics::new("test"))
-}
-
 /// A picker that only admits key from the given list.
 pub struct BiasedPicker<K> {
     admits: HashSet<K>,
diff --git a/foyer-storage/tests/storage_test.rs b/foyer-storage/tests/storage_test.rs
index a2304352..2c9f4b4c 100644
--- a/foyer-storage/tests/storage_test.rs
+++ b/foyer-storage/tests/storage_test.rs
@@ -17,6 +17,7 @@
 use std::{path::Path, sync::Arc, time::Duration};
 
 use ahash::RandomState;
+use foyer_common::metrics::model::Metrics;
 use foyer_memory::{Cache, CacheBuilder, CacheEntry, FifoConfig};
 use foyer_storage::{
     test_utils::Recorder, Compression, DirectFsDeviceOptions, Engine, LargeEngineOptions, StoreBuilder,
@@ -109,7 +110,7 @@ fn basic(
     recorder: &Arc<Recorder<u64>>,
 ) -> StoreBuilder<u64, Vec<u8>> {
     // TODO(MrCroxx): Test mixed engine here.
-    StoreBuilder::new(memory.clone(), Engine::Large)
+    StoreBuilder::new("test", memory.clone(), Arc::new(Metrics::noop()), Engine::Large)
         .with_device_options(
             DirectFsDeviceOptions::new(path)
                 .with_capacity(4 * MB)
diff --git a/foyer/Cargo.toml b/foyer/Cargo.toml
index 2d932cae..f08108a3 100644
--- a/foyer/Cargo.toml
+++ b/foyer/Cargo.toml
@@ -45,6 +45,8 @@ tracing = [
     "foyer-memory/tracing",
     "foyer-storage/tracing",
 ]
+prometheus = ["foyer-common/prometheus"]
+opentelemetry = ["foyer-common/opentelemetry"]
 
 [lints]
 workspace = true
diff --git a/foyer/src/hybrid/builder.rs b/foyer/src/hybrid/builder.rs
index 08bc2c7d..da04045e 100644
--- a/foyer/src/hybrid/builder.rs
+++ b/foyer/src/hybrid/builder.rs
@@ -18,6 +18,7 @@ use ahash::RandomState;
 use foyer_common::{
     code::{HashBuilder, StorageKey, StorageValue},
     event::EventListener,
+    metrics::{model::Metrics, registry::noop::NoopMetricsRegistry, RegistryOps},
     tracing::TracingOptions,
 };
 use foyer_memory::{Cache, CacheBuilder, EvictionConfig, Weighter};
@@ -29,35 +30,39 @@ use foyer_storage::{
 use crate::HybridCache;
 
 /// Hybrid cache builder.
-pub struct HybridCacheBuilder<K, V> {
-    name: String,
+pub struct HybridCacheBuilder<K, V, M> {
+    name: &'static str,
     event_listener: Option<Arc<dyn EventListener<Key = K, Value = V>>>,
     tracing_options: TracingOptions,
+    registry: M,
 }
 
-impl<K, V> Default for HybridCacheBuilder<K, V> {
+impl<K, V> Default for HybridCacheBuilder<K, V, NoopMetricsRegistry> {
     fn default() -> Self {
         Self::new()
     }
 }
 
-impl<K, V> HybridCacheBuilder<K, V> {
+impl<K, V> HybridCacheBuilder<K, V, NoopMetricsRegistry> {
     /// Create a new hybrid cache builder.
     pub fn new() -> Self {
         Self {
-            name: "foyer".to_string(),
+            name: "foyer",
             event_listener: None,
             tracing_options: TracingOptions::default(),
+            registry: NoopMetricsRegistry,
         }
     }
+}
 
+impl<K, V, M> HybridCacheBuilder<K, V, M> {
     /// Set the name of the foyer hybrid cache instance.
     ///
     /// foyer will use the name as the prefix of the metric names.
     ///
     /// Default: `foyer`.
-    pub fn with_name(mut self, name: &str) -> Self {
-        self.name = name.to_string();
+    pub fn with_name(mut self, name: &'static str) -> Self {
+        self.name = name;
         self
     }
 
@@ -77,19 +82,39 @@ impl<K, V> HybridCacheBuilder<K, V> {
         self
     }
 
+    /// Set metrics registry.
+    ///
+    /// Default: [`NoopMetricsRegistry`].
+    pub fn with_metrics_registry<OM>(self, registry: OM) -> HybridCacheBuilder<K, V, OM>
+    where
+        OM: RegistryOps,
+    {
+        HybridCacheBuilder {
+            name: self.name,
+            event_listener: self.event_listener,
+            tracing_options: self.tracing_options,
+            registry,
+        }
+    }
+
     /// Continue to modify the in-memory cache configurations.
     pub fn memory(self, capacity: usize) -> HybridCacheBuilderPhaseMemory<K, V, RandomState>
     where
         K: StorageKey,
         V: StorageValue,
+        M: RegistryOps,
     {
-        let mut builder = CacheBuilder::new(capacity).with_name(&self.name);
+        let metrics = Arc::new(Metrics::new(self.name, &self.registry));
+        let mut builder = CacheBuilder::new(capacity)
+            .with_name(self.name)
+            .with_metrics(metrics.clone());
         if let Some(event_listener) = self.event_listener {
             builder = builder.with_event_listener(event_listener);
         }
         HybridCacheBuilderPhaseMemory {
             builder,
             name: self.name,
+            metrics,
             tracing_options: self.tracing_options,
         }
     }
@@ -102,9 +127,11 @@ where
     V: StorageValue,
     S: HashBuilder + Debug,
 {
-    name: String,
+    name: &'static str,
     tracing_options: TracingOptions,
-    builder: CacheBuilder<K, V, S>,
+    metrics: Arc<Metrics>,
+    // `NoopMetricsRegistry` here will be ignored, for its metrics is already set.
+    builder: CacheBuilder<K, V, S, NoopMetricsRegistry>,
 }
 
 impl<K, V, S> HybridCacheBuilderPhaseMemory<K, V, S>
@@ -120,6 +147,7 @@ where
         HybridCacheBuilderPhaseMemory {
             name: self.name,
             tracing_options: self.tracing_options,
+            metrics: self.metrics,
             builder,
         }
     }
@@ -132,6 +160,7 @@ where
         HybridCacheBuilderPhaseMemory {
             name: self.name,
             tracing_options: self.tracing_options,
+            metrics: self.metrics,
             builder,
         }
     }
@@ -145,6 +174,7 @@ where
         HybridCacheBuilderPhaseMemory {
             name: self.name,
             tracing_options: self.tracing_options,
+            metrics: self.metrics,
             builder,
         }
     }
@@ -155,6 +185,7 @@ where
         HybridCacheBuilderPhaseMemory {
             name: self.name,
             tracing_options: self.tracing_options,
+            metrics: self.metrics,
             builder,
         }
     }
@@ -163,9 +194,10 @@ where
     pub fn storage(self, engine: Engine) -> HybridCacheBuilderPhaseStorage<K, V, S> {
         let memory = self.builder.build();
         HybridCacheBuilderPhaseStorage {
-            builder: StoreBuilder::new(memory.clone(), engine).with_name(&self.name),
+            builder: StoreBuilder::new(self.name, memory.clone(), self.metrics.clone(), engine),
             name: self.name,
             tracing_options: self.tracing_options,
+            metrics: self.metrics,
             memory,
         }
     }
@@ -178,8 +210,9 @@ where
     V: StorageValue,
     S: HashBuilder + Debug,
 {
-    name: String,
+    name: &'static str,
     tracing_options: TracingOptions,
+    metrics: Arc<Metrics>,
     memory: Cache<K, V, S>,
     builder: StoreBuilder<K, V, S>,
 }
@@ -196,6 +229,7 @@ where
         Self {
             name: self.name,
             tracing_options: self.tracing_options,
+            metrics: self.metrics,
             memory: self.memory,
             builder,
         }
@@ -209,6 +243,7 @@ where
         Self {
             name: self.name,
             tracing_options: self.tracing_options,
+            metrics: self.metrics,
             memory: self.memory,
             builder,
         }
@@ -224,6 +259,7 @@ where
         Self {
             name: self.name,
             tracing_options: self.tracing_options,
+            metrics: self.metrics,
             memory: self.memory,
             builder,
         }
@@ -239,6 +275,7 @@ where
         Self {
             name: self.name,
             tracing_options: self.tracing_options,
+            metrics: self.metrics,
             memory: self.memory,
             builder,
         }
@@ -252,6 +289,7 @@ where
         Self {
             name: self.name,
             tracing_options: self.tracing_options,
+            metrics: self.metrics,
             memory: self.memory,
             builder,
         }
@@ -263,6 +301,7 @@ where
         Self {
             name: self.name,
             tracing_options: self.tracing_options,
+            metrics: self.metrics,
             memory: self.memory,
             builder,
         }
@@ -276,6 +315,7 @@ where
         Self {
             name: self.name,
             tracing_options: self.tracing_options,
+            metrics: self.metrics,
             memory: self.memory,
             builder,
         }
@@ -289,6 +329,7 @@ where
         Self {
             name: self.name,
             tracing_options: self.tracing_options,
+            metrics: self.metrics,
             memory: self.memory,
             builder,
         }
@@ -297,6 +338,11 @@ where
     /// Build and open the hybrid cache with the given configurations.
     pub async fn build(self) -> anyhow::Result<HybridCache<K, V, S>> {
         let storage = self.builder.build().await?;
-        Ok(HybridCache::new(self.name, self.memory, storage, self.tracing_options))
+        Ok(HybridCache::new(
+            self.memory,
+            storage,
+            self.tracing_options,
+            self.metrics,
+        ))
     }
 }
diff --git a/foyer/src/hybrid/cache.rs b/foyer/src/hybrid/cache.rs
index fb15a588..adf72ff2 100644
--- a/foyer/src/hybrid/cache.rs
+++ b/foyer/src/hybrid/cache.rs
@@ -32,7 +32,7 @@ use fastrace::prelude::*;
 use foyer_common::{
     code::{HashBuilder, StorageKey, StorageValue},
     future::Diversion,
-    metrics::Metrics,
+    metrics::model::Metrics,
     tracing::{InRootSpan, TracingConfig, TracingOptions},
 };
 use foyer_memory::{Cache, CacheEntry, CacheHint, Fetch, FetchMark, FetchState};
@@ -127,12 +127,11 @@ where
     S: HashBuilder + Debug,
 {
     pub(crate) fn new(
-        name: String,
         memory: Cache<K, V, S>,
         storage: Store<K, V, S>,
         tracing_options: TracingOptions,
+        metrics: Arc<Metrics>,
     ) -> Self {
-        let metrics = Arc::new(Metrics::new(&name));
         let tracing_config = Arc::<TracingConfig>::default();
         tracing_config.update(tracing_options);
         let tracing = Arc::new(AtomicBool::new(false));
@@ -181,8 +180,8 @@ where
         let entry = self.memory.insert(key, value);
         self.storage.enqueue(entry.clone(), false);
 
-        self.metrics.hybrid_insert.increment(1);
-        self.metrics.hybrid_insert_duration.record(now.elapsed());
+        self.metrics.hybrid_insert.increase(1);
+        self.metrics.hybrid_insert_duration.record(now.elapsed().as_secs_f64());
 
         try_cancel!(self, span, record_hybrid_insert_threshold);
 
@@ -200,8 +199,8 @@ where
         let entry = self.memory.insert_with_hint(key, value, hint);
         self.storage.enqueue(entry.clone(), false);
 
-        self.metrics.hybrid_insert.increment(1);
-        self.metrics.hybrid_insert_duration.record(now.elapsed());
+        self.metrics.hybrid_insert.increase(1);
+        self.metrics.hybrid_insert_duration.record(now.elapsed().as_secs_f64());
 
         try_cancel!(self, span, record_hybrid_insert_threshold);
 
@@ -218,12 +217,12 @@ where
         let now = Instant::now();
 
         let record_hit = || {
-            self.metrics.hybrid_hit.increment(1);
-            self.metrics.hybrid_hit_duration.record(now.elapsed());
+            self.metrics.hybrid_hit.increase(1);
+            self.metrics.hybrid_hit_duration.record(now.elapsed().as_secs_f64());
         };
         let record_miss = || {
-            self.metrics.hybrid_miss.increment(1);
-            self.metrics.hybrid_miss_duration.record(now.elapsed());
+            self.metrics.hybrid_miss.increase(1);
+            self.metrics.hybrid_miss_duration.record(now.elapsed().as_secs_f64());
         };
 
         let guard = span.set_local_parent();
@@ -286,14 +285,14 @@ where
 
         match res {
             Ok(entry) => {
-                self.metrics.hybrid_hit.increment(1);
-                self.metrics.hybrid_hit_duration.record(now.elapsed());
+                self.metrics.hybrid_hit.increase(1);
+                self.metrics.hybrid_hit_duration.record(now.elapsed().as_secs_f64());
                 try_cancel!(self, span, record_hybrid_obtain_threshold);
                 Ok(Some(entry))
             }
             Err(ObtainFetchError::NotExist) => {
-                self.metrics.hybrid_miss.increment(1);
-                self.metrics.hybrid_miss_duration.record(now.elapsed());
+                self.metrics.hybrid_miss.increase(1);
+                self.metrics.hybrid_miss_duration.record(now.elapsed().as_secs_f64());
                 try_cancel!(self, span, record_hybrid_obtain_threshold);
                 Ok(None)
             }
@@ -322,8 +321,8 @@ where
         self.memory.remove(key);
         self.storage.delete(key);
 
-        self.metrics.hybrid_remove.increment(1);
-        self.metrics.hybrid_remove_duration.record(now.elapsed());
+        self.metrics.hybrid_remove.increase(1);
+        self.metrics.hybrid_remove_duration.record(now.elapsed().as_secs_f64());
 
         try_cancel!(self, span, record_hybrid_remove_threshold);
     }
@@ -488,8 +487,8 @@ where
                 async move {
                     match store.load(&key).await.map_err(anyhow::Error::from) {
                         Ok(Some((_k, v))) => {
-                            metrics.hybrid_hit.increment(1);
-                            metrics.hybrid_hit_duration.record(now.elapsed());
+                            metrics.hybrid_hit.increase(1);
+                            metrics.hybrid_hit_duration.record(now.elapsed().as_secs_f64());
 
                             return Ok(v).into();
                         }
@@ -497,8 +496,8 @@ where
                         Err(e) => return Err(e).into(),
                     };
 
-                    metrics.hybrid_miss.increment(1);
-                    metrics.hybrid_miss_duration.record(now.elapsed());
+                    metrics.hybrid_miss.increase(1);
+                    metrics.hybrid_miss_duration.record(now.elapsed().as_secs_f64());
 
                     runtime
                         .user()
@@ -518,8 +517,8 @@ where
         );
 
         if inner.state() == FetchState::Hit {
-            self.metrics.hybrid_hit.increment(1);
-            self.metrics.hybrid_hit_duration.record(now.elapsed());
+            self.metrics.hybrid_hit.increase(1);
+            self.metrics.hybrid_hit_duration.record(now.elapsed().as_secs_f64());
         }
 
         let inner = HybridFetchInner {
diff --git a/foyer/src/hybrid/writer.rs b/foyer/src/hybrid/writer.rs
index 14f63fdb..c0087068 100644
--- a/foyer/src/hybrid/writer.rs
+++ b/foyer/src/hybrid/writer.rs
@@ -126,11 +126,11 @@ where
         };
         self.hybrid.storage().enqueue(entry.clone(), true);
 
-        self.hybrid.metrics().hybrid_insert.increment(1);
+        self.hybrid.metrics().hybrid_insert.increase(1);
         self.hybrid
             .metrics()
             .hybrid_insert_duration
-            .record(now.elapsed() + self.pick_duration);
+            .record((now.elapsed() + self.pick_duration).as_secs_f64());
 
         Some(entry)
     }
diff --git a/foyer/src/prelude.rs b/foyer/src/prelude.rs
index 779d253f..e6a878fa 100644
--- a/foyer/src/prelude.rs
+++ b/foyer/src/prelude.rs
@@ -12,11 +12,19 @@
 //  See the License for the specific language governing permissions and
 //  limitations under the License.
 
+#[cfg(feature = "opentelemetry")]
+pub use crate::common::metrics::registry::opentelemetry::OpenTelemetryMetricsRegistry;
+#[cfg(feature = "prometheus")]
+pub use crate::common::metrics::registry::prometheus::PrometheusMetricsRegistry;
 pub use crate::{
     common::{
         buf::{BufExt, BufMutExt},
         code::{Key, StorageKey, StorageValue, Value},
         event::{Event, EventListener},
+        metrics::{
+            registry::noop::NoopMetricsRegistry, CounterOps, CounterVecOps, GaugeOps, GaugeVecOps, HistogramOps,
+            HistogramVecOps, RegistryOps,
+        },
         range::RangeBoundsExt,
         tracing::TracingOptions,
     },

From fd2bed421ebcedd02bbb7c8445f5ee16ad603a7a Mon Sep 17 00:00:00 2001
From: Croxx <mrcroxx@outlook.com>
Date: Mon, 25 Nov 2024 14:12:26 +0800
Subject: [PATCH 46/53] feat: introduce both opentelemetry 0.26 and 0.27
 exporter (#793)

Signed-off-by: MrCroxx <mrcroxx@outlook.com>
---
 .github/workflows/ci.yml                      |   4 +-
 Cargo.toml                                    |   3 +-
 Makefile                                      |   6 +-
 foyer-common/Cargo.toml                       |   7 +-
 foyer-common/src/metrics/model.rs             |  20 +-
 foyer-common/src/metrics/registry/mod.rs      |  15 +-
 .../metrics/registry/opentelemetry_0_26.rs    | 209 ++++++++++++++++++
 ...opentelemetry.rs => opentelemetry_0_27.rs} |   1 +
 foyer/Cargo.toml                              |   2 +
 9 files changed, 252 insertions(+), 15 deletions(-)
 create mode 100644 foyer-common/src/metrics/registry/opentelemetry_0_26.rs
 rename foyer-common/src/metrics/registry/{opentelemetry.rs => opentelemetry_0_27.rs} (99%)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index cb8ecec4..3d51fc3b 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -165,7 +165,7 @@ jobs:
           cargo clippy --all-targets --features tokio-console -- -D warnings
           cargo clippy --all-targets --features deadlock -- -D warnings
           cargo clippy --all-targets --features tracing -- -D warnings
-          cargo clippy --all-targets --features prometheus,opentelemetry -- -D warnings
+          cargo clippy --all-targets --features prometheus,opentelemetry_0_26,opentelemetry_0_27 -- -D warnings
           cargo clippy --all-targets -- -D warnings
       - if: steps.cache.outputs.cache-hit != 'true'
         uses: taiki-e/install-action@cargo-llvm-cov
@@ -182,7 +182,7 @@ jobs:
           RUST_BACKTRACE: 1
           CI: true
         run: |
-          cargo llvm-cov --no-report nextest --features "strict_assertions,sanity,prometheus,opentelemetry"
+          cargo llvm-cov --no-report nextest --features "strict_assertions,sanity,prometheus,opentelemetry_0_26,opentelemetry_0_27"
       - name: Run examples with coverage
         env:
           RUST_BACKTRACE: 1
diff --git a/Cargo.toml b/Cargo.toml
index e42e2a47..62838d1d 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -47,7 +47,8 @@ tokio = { package = "madsim-tokio", version = "0.2", features = [
     "fs",
 ] }
 prometheus = "0.13"
-opentelemetry = "0.27"
+opentelemetry_0_27 = { package = "opentelemetry", version = "0.27" }
+opentelemetry_0_26 = { package = "opentelemetry", version = "0.26" }
 
 # foyer components
 foyer-common = { version = "0.13.0-dev", path = "foyer-common" }
diff --git a/Makefile b/Makefile
index 40c3a225..67714e85 100644
--- a/Makefile
+++ b/Makefile
@@ -10,7 +10,7 @@ check:
 	./scripts/minimize-dashboards.sh
 	cargo sort -w
 	cargo fmt --all
-	cargo clippy --all-targets --features prometheus,opentelemetry
+	cargo clippy --all-targets --features prometheus,opentelemetry_0_26,opentelemetry_0_27
 
 check-all:
 	shellcheck ./scripts/*
@@ -21,11 +21,11 @@ check-all:
 	cargo clippy --all-targets --features tokio-console
 	cargo clippy --all-targets --features sanity
 	cargo clippy --all-targets --features tracing
-	cargo clippy --all-targets --features prometheus,opentelemetry
+	cargo clippy --all-targets --features prometheus,opentelemetry_0_26,opentelemetry_0_27
 	cargo clippy --all-targets
 
 test:
-	RUST_BACKTRACE=1 cargo nextest run --all --features "strict_assertions,sanity,prometheus,opentelemetry"
+	RUST_BACKTRACE=1 cargo nextest run --all --features "strict_assertions,sanity,prometheus,opentelemetry_0_26,opentelemetry_0_27"
 	RUST_BACKTRACE=1 cargo test --doc
 
 test-ignored:
diff --git a/foyer-common/Cargo.toml b/foyer-common/Cargo.toml
index 4c6a7e38..6c6b3dfa 100644
--- a/foyer-common/Cargo.toml
+++ b/foyer-common/Cargo.toml
@@ -19,7 +19,8 @@ fastrace = { workspace = true }
 futures = "0.3"
 hashbrown = { workspace = true }
 itertools = { workspace = true }
-opentelemetry = { workspace = true, optional = true }
+opentelemetry_0_26 = { workspace = true, optional = true }
+opentelemetry_0_27 = { workspace = true, optional = true }
 parking_lot = { workspace = true }
 pin-project = "1"
 prometheus = { workspace = true, optional = true }
@@ -34,7 +35,9 @@ rand = "0.8.5"
 strict_assertions = []
 tracing = ["fastrace/enable"]
 prometheus = ["dep:prometheus"]
-opentelemetry = ["dep:opentelemetry"]
+opentelemetry = ["opentelemetry_0_27"]
+opentelemetry_0_27 = ["dep:opentelemetry_0_27"]
+opentelemetry_0_26 = ["dep:opentelemetry_0_26"]
 
 [lints]
 workspace = true
diff --git a/foyer-common/src/metrics/model.rs b/foyer-common/src/metrics/model.rs
index 3db3f536..233f03a6 100644
--- a/foyer-common/src/metrics/model.rs
+++ b/foyer-common/src/metrics/model.rs
@@ -366,11 +366,23 @@ mod tests {
         case(&PrometheusMetricsRegistry::new(prometheus::Registry::new()));
     }
 
-    #[cfg(feature = "opentelemetry")]
+    #[cfg(feature = "opentelemetry_0_27")]
     #[test]
-    fn test_metrics_opentelemetry() {
-        use crate::metrics::registry::opentelemetry::OpenTelemetryMetricsRegistry;
+    fn test_metrics_opentelemetry_0_27() {
+        use crate::metrics::registry::opentelemetry_0_27::OpenTelemetryMetricsRegistry;
 
-        case(&OpenTelemetryMetricsRegistry::new(opentelemetry::global::meter("test")));
+        case(&OpenTelemetryMetricsRegistry::new(opentelemetry_0_27::global::meter(
+            "test",
+        )));
+    }
+
+    #[cfg(feature = "opentelemetry_0_26")]
+    #[test]
+    fn test_metrics_opentelemetry_0_26() {
+        use crate::metrics::registry::opentelemetry_0_26::OpenTelemetryMetricsRegistry;
+
+        case(&OpenTelemetryMetricsRegistry::new(opentelemetry_0_26::global::meter(
+            "test",
+        )));
     }
 }
diff --git a/foyer-common/src/metrics/registry/mod.rs b/foyer-common/src/metrics/registry/mod.rs
index 840f7fde..a88fecc5 100644
--- a/foyer-common/src/metrics/registry/mod.rs
+++ b/foyer-common/src/metrics/registry/mod.rs
@@ -14,9 +14,18 @@
 
 /// Some phantom metrics components that do nothing.
 pub mod noop;
-/// OpenTelemetry metrics components.
-#[cfg(feature = "opentelemetry")]
-pub mod opentelemetry;
+
 /// Prometheus metrics components.
 #[cfg(feature = "prometheus")]
 pub mod prometheus;
+
+#[cfg(feature = "opentelemetry")]
+pub use opentelemetry_0_27 as opentelemetry;
+
+/// OpenTelemetry metrics components.
+#[cfg(feature = "opentelemetry_0_27")]
+pub mod opentelemetry_0_27;
+
+/// OpenTelemetry metrics components.
+#[cfg(feature = "opentelemetry_0_26")]
+pub mod opentelemetry_0_26;
diff --git a/foyer-common/src/metrics/registry/opentelemetry_0_26.rs b/foyer-common/src/metrics/registry/opentelemetry_0_26.rs
new file mode 100644
index 00000000..e4374c16
--- /dev/null
+++ b/foyer-common/src/metrics/registry/opentelemetry_0_26.rs
@@ -0,0 +1,209 @@
+//  Copyright 2024 foyer Project Authors
+//
+//  Licensed under the Apache License, Version 2.0 (the "License");
+//  you may not use this file except in compliance with the License.
+//  You may obtain a copy of the License at
+//
+//  http://www.apache.org/licenses/LICENSE-2.0
+//
+//  Unless required by applicable law or agreed to in writing, software
+//  distributed under the License is distributed on an "AS IS" BASIS,
+//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//  See the License for the specific language governing permissions and
+//  limitations under the License.
+
+use std::sync::atomic::{AtomicU64, Ordering};
+
+use itertools::Itertools;
+use opentelemetry::{
+    metrics::{Counter as OtCounter, Gauge as OtGauge, Histogram as OtHistogram, Meter},
+    KeyValue,
+};
+use opentelemetry_0_26 as opentelemetry;
+
+use crate::metrics::{CounterOps, CounterVecOps, GaugeOps, GaugeVecOps, HistogramOps, HistogramVecOps, RegistryOps};
+
+/// OpenTelemetry counter metric.
+#[derive(Debug)]
+pub struct Counter {
+    counter: OtCounter<u64>,
+    labels: Vec<KeyValue>,
+}
+
+impl CounterOps for Counter {
+    fn increase(&self, val: u64) {
+        self.counter.add(val, &self.labels);
+    }
+}
+
+/// OpenTelemetry gauge metric.
+#[derive(Debug)]
+pub struct Gauge {
+    val: AtomicU64,
+    gauge: OtGauge<u64>,
+    labels: Vec<KeyValue>,
+}
+
+impl GaugeOps for Gauge {
+    fn increase(&self, val: u64) {
+        let v = self.val.fetch_add(val, Ordering::Relaxed) + val;
+        self.gauge.record(v, &self.labels);
+    }
+
+    fn decrease(&self, val: u64) {
+        let v = self.val.fetch_sub(val, Ordering::Relaxed) - val;
+        self.gauge.record(v, &self.labels);
+    }
+
+    fn absolute(&self, val: u64) {
+        self.val.store(val, Ordering::Relaxed);
+        self.gauge.record(val, &self.labels);
+    }
+}
+
+/// OpenTelemetry histogram metric.
+#[derive(Debug)]
+pub struct Histogram {
+    histogram: OtHistogram<f64>,
+    labels: Vec<KeyValue>,
+}
+
+impl HistogramOps for Histogram {
+    fn record(&self, val: f64) {
+        self.histogram.record(val, &self.labels);
+    }
+}
+
+/// OpenTelemetry metric vector.
+#[derive(Debug)]
+pub struct MetricVec {
+    meter: Meter,
+    name: &'static str,
+    desc: &'static str,
+    label_names: &'static [&'static str],
+}
+
+impl CounterVecOps for MetricVec {
+    fn counter(&self, labels: &[&str]) -> impl CounterOps {
+        let counter = self.meter.u64_counter(self.name).with_description(self.desc).init();
+        let labels = self
+            .label_names
+            .iter()
+            .zip_eq(labels.iter())
+            .map(|(name, label)| KeyValue::new(name.to_string(), label.to_string()))
+            .collect();
+        Counter { counter, labels }
+    }
+}
+
+impl GaugeVecOps for MetricVec {
+    fn gauge(&self, labels: &[&str]) -> impl GaugeOps {
+        let gauge = self.meter.u64_gauge(self.name).with_description(self.desc).init();
+        let labels = self
+            .label_names
+            .iter()
+            .zip_eq(labels.iter())
+            .map(|(name, label)| KeyValue::new(name.to_string(), label.to_string()))
+            .collect();
+        let val = AtomicU64::new(0);
+        Gauge { val, gauge, labels }
+    }
+}
+
+impl HistogramVecOps for MetricVec {
+    fn histogram(&self, labels: &[&str]) -> impl HistogramOps {
+        let histogram = self.meter.f64_histogram(self.name).with_description(self.desc).init();
+        let labels = self
+            .label_names
+            .iter()
+            .zip_eq(labels.iter())
+            .map(|(name, label)| KeyValue::new(name.to_string(), label.to_string()))
+            .collect();
+        Histogram { histogram, labels }
+    }
+}
+
+/// OpenTelemetry metrics registry.
+#[derive(Debug)]
+pub struct OpenTelemetryMetricsRegistry {
+    meter: Meter,
+}
+
+impl OpenTelemetryMetricsRegistry {
+    /// Create an OpenTelemetry metrics registry.
+    pub fn new(meter: Meter) -> Self {
+        Self { meter }
+    }
+}
+
+impl RegistryOps for OpenTelemetryMetricsRegistry {
+    fn register_counter_vec(
+        &self,
+        name: &'static str,
+        desc: &'static str,
+        label_names: &'static [&'static str],
+    ) -> impl CounterVecOps {
+        let meter = self.meter.clone();
+        MetricVec {
+            meter,
+            name,
+            desc,
+            label_names,
+        }
+    }
+
+    fn register_gauge_vec(
+        &self,
+        name: &'static str,
+        desc: &'static str,
+        label_names: &'static [&'static str],
+    ) -> impl GaugeVecOps {
+        let meter = self.meter.clone();
+        MetricVec {
+            meter,
+            name,
+            desc,
+            label_names,
+        }
+    }
+
+    fn register_histogram_vec(
+        &self,
+        name: &'static str,
+        desc: &'static str,
+        label_names: &'static [&'static str],
+    ) -> impl HistogramVecOps {
+        let meter = self.meter.clone();
+        MetricVec {
+            meter,
+            name,
+            desc,
+            label_names,
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test() {
+        let meter = opentelemetry::global::meter("test");
+        let ot = OpenTelemetryMetricsRegistry::new(meter);
+
+        let cv = ot.register_counter_vec("test_counter_1", "test counter 1", &["label1", "label2"]);
+        let c = cv.counter(&["l1", "l2"]);
+        c.increase(42);
+
+        let gv = ot.register_gauge_vec("test_gauge_1", "test gauge 1", &["label1", "label2"]);
+        let g = gv.gauge(&["l1", "l2"]);
+        g.increase(514);
+        g.decrease(114);
+        g.absolute(114514);
+
+        let hv = ot.register_histogram_vec("test_histogram_1", "test histogram 1", &["label1", "label2"]);
+        let h = hv.histogram(&["l1", "l2"]);
+        h.record(114.514);
+    }
+}
diff --git a/foyer-common/src/metrics/registry/opentelemetry.rs b/foyer-common/src/metrics/registry/opentelemetry_0_27.rs
similarity index 99%
rename from foyer-common/src/metrics/registry/opentelemetry.rs
rename to foyer-common/src/metrics/registry/opentelemetry_0_27.rs
index 349d0802..99ae98c1 100644
--- a/foyer-common/src/metrics/registry/opentelemetry.rs
+++ b/foyer-common/src/metrics/registry/opentelemetry_0_27.rs
@@ -19,6 +19,7 @@ use opentelemetry::{
     metrics::{Counter as OtCounter, Gauge as OtGauge, Histogram as OtHistogram, Meter},
     KeyValue,
 };
+use opentelemetry_0_27 as opentelemetry;
 
 use crate::metrics::{CounterOps, CounterVecOps, GaugeOps, GaugeVecOps, HistogramOps, HistogramVecOps, RegistryOps};
 
diff --git a/foyer/Cargo.toml b/foyer/Cargo.toml
index f08108a3..cb831682 100644
--- a/foyer/Cargo.toml
+++ b/foyer/Cargo.toml
@@ -47,6 +47,8 @@ tracing = [
 ]
 prometheus = ["foyer-common/prometheus"]
 opentelemetry = ["foyer-common/opentelemetry"]
+opentelemetry_0_27 = ["foyer-common/opentelemetry_0_27"]
+opentelemetry_0_26 = ["foyer-common/opentelemetry_0_26"]
 
 [lints]
 workspace = true

From e0dc273d529ee760242f9163454f4b9e9383f289 Mon Sep 17 00:00:00 2001
From: Croxx <mrcroxx@outlook.com>
Date: Mon, 25 Nov 2024 16:46:31 +0800
Subject: [PATCH 47/53] feat: support in-memory cache online resize (#794)

* feat: support in-memory cache online resize

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

* chore: update comments

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

---------

Signed-off-by: MrCroxx <mrcroxx@outlook.com>
---
 Cargo.toml                          |   2 +
 examples/Cargo.toml                 |   2 +-
 foyer-cli/Cargo.toml                |   2 +-
 foyer-common/Cargo.toml             |   1 +
 foyer-common/src/hasher.rs          | 103 +++++++++++++++++
 foyer-common/src/lib.rs             |   2 +
 foyer-memory/Cargo.toml             |   3 +-
 foyer-memory/src/cache.rs           |  12 ++
 foyer-memory/src/error.rs           |  53 +++++++++
 foyer-memory/src/eviction/fifo.rs   |   9 +-
 foyer-memory/src/eviction/lfu.rs    |  59 ++++++----
 foyer-memory/src/eviction/lru.rs    |  32 ++++--
 foyer-memory/src/eviction/mod.rs    |   7 +-
 foyer-memory/src/eviction/s3fifo.rs |  35 +++++-
 foyer-memory/src/lib.rs             |   1 +
 foyer-memory/src/prelude.rs         |   1 +
 foyer-memory/src/raw.rs             | 169 ++++++++++++++++++++++++----
 foyer-storage/Cargo.toml            |   4 +-
 foyer-storage/src/small/generic.rs  |  22 ++--
 foyer-storage/src/test_utils.rs     |  34 +-----
 foyer/Cargo.toml                    |   2 +-
 21 files changed, 446 insertions(+), 109 deletions(-)
 create mode 100644 foyer-common/src/hasher.rs
 create mode 100644 foyer-memory/src/error.rs

diff --git a/Cargo.toml b/Cargo.toml
index 62838d1d..de78d52c 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -22,6 +22,7 @@ license = "Apache-2.0"
 readme = "README.md"
 
 [workspace.dependencies]
+ahash = "0.8"
 bytesize = { package = "foyer-bytesize", version = "2" }
 clap = { version = "4", features = ["derive"] }
 crossbeam = "0.8"
@@ -37,6 +38,7 @@ test-log = { version = "0.2", default-features = false, features = [
     "trace",
     "color",
 ] }
+thiserror = "2"
 tokio = { package = "madsim-tokio", version = "0.2", features = [
     "rt",
     "rt-multi-thread",
diff --git a/examples/Cargo.toml b/examples/Cargo.toml
index 42b574aa..92f2b274 100644
--- a/examples/Cargo.toml
+++ b/examples/Cargo.toml
@@ -14,7 +14,7 @@ publish = false
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
-ahash = "0.8"
+ahash = { workspace = true }
 anyhow = "1"
 chrono = "0.4"
 equivalent = { workspace = true }
diff --git a/foyer-cli/Cargo.toml b/foyer-cli/Cargo.toml
index ca230e84..c07556b9 100644
--- a/foyer-cli/Cargo.toml
+++ b/foyer-cli/Cargo.toml
@@ -17,7 +17,7 @@ publish = false
 anyhow = "1"
 bytesize = { workspace = true }
 clap = { workspace = true }
-thiserror = "1"
+thiserror = { workspace = true }
 
 [dev-dependencies]
 
diff --git a/foyer-common/Cargo.toml b/foyer-common/Cargo.toml
index 6c6b3dfa..10f9fcde 100644
--- a/foyer-common/Cargo.toml
+++ b/foyer-common/Cargo.toml
@@ -13,6 +13,7 @@ readme = { workspace = true }
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
+ahash = { workspace = true }
 bytes = "1"
 cfg-if = "1"
 fastrace = { workspace = true }
diff --git a/foyer-common/src/hasher.rs b/foyer-common/src/hasher.rs
new file mode 100644
index 00000000..d11ae1c3
--- /dev/null
+++ b/foyer-common/src/hasher.rs
@@ -0,0 +1,103 @@
+//  Copyright 2024 foyer Project Authors
+//
+//  Licensed under the Apache License, Version 2.0 (the "License");
+//  you may not use this file except in compliance with the License.
+//  You may obtain a copy of the License at
+//
+//  http://www.apache.org/licenses/LICENSE-2.0
+//
+//  Unless required by applicable law or agreed to in writing, software
+//  distributed under the License is distributed on an "AS IS" BASIS,
+//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//  See the License for the specific language governing permissions and
+//  limitations under the License.
+
+use std::hash::{BuildHasher, Hasher};
+
+pub use ahash::RandomState as AhashRandomState;
+
+/// A hasher return u64 mod result.
+#[derive(Debug, Default)]
+pub struct ModRandomState {
+    state: u64,
+}
+
+impl Hasher for ModRandomState {
+    fn finish(&self) -> u64 {
+        self.state
+    }
+
+    fn write(&mut self, bytes: &[u8]) {
+        for byte in bytes {
+            self.state = (self.state << 8) + *byte as u64;
+        }
+    }
+
+    fn write_u8(&mut self, i: u8) {
+        self.write(&[i])
+    }
+
+    fn write_u16(&mut self, i: u16) {
+        self.write(&i.to_be_bytes())
+    }
+
+    fn write_u32(&mut self, i: u32) {
+        self.write(&i.to_be_bytes())
+    }
+
+    fn write_u64(&mut self, i: u64) {
+        self.write(&i.to_be_bytes())
+    }
+
+    fn write_u128(&mut self, i: u128) {
+        self.write(&i.to_be_bytes())
+    }
+
+    fn write_usize(&mut self, i: usize) {
+        self.write(&i.to_be_bytes())
+    }
+
+    fn write_i8(&mut self, i: i8) {
+        self.write_u8(i as u8)
+    }
+
+    fn write_i16(&mut self, i: i16) {
+        self.write_u16(i as u16)
+    }
+
+    fn write_i32(&mut self, i: i32) {
+        self.write_u32(i as u32)
+    }
+
+    fn write_i64(&mut self, i: i64) {
+        self.write_u64(i as u64)
+    }
+
+    fn write_i128(&mut self, i: i128) {
+        self.write_u128(i as u128)
+    }
+
+    fn write_isize(&mut self, i: isize) {
+        self.write_usize(i as usize)
+    }
+}
+
+impl BuildHasher for ModRandomState {
+    type Hasher = Self;
+
+    fn build_hasher(&self) -> Self::Hasher {
+        Self::default()
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_mod_hasher() {
+        for i in 0..255u8 {
+            assert_eq!(i, ModRandomState::default().hash_one(i) as u8,)
+        }
+    }
+}
diff --git a/foyer-common/src/lib.rs b/foyer-common/src/lib.rs
index 55383fb5..49565a6f 100644
--- a/foyer-common/src/lib.rs
+++ b/foyer-common/src/lib.rs
@@ -30,6 +30,8 @@ pub mod countdown;
 pub mod event;
 /// Future extensions.
 pub mod future;
+/// Provisioned hashers.
+pub mod hasher;
 /// The shared metrics for foyer.
 pub mod metrics;
 /// Extensions for [`std::option::Option`].
diff --git a/foyer-memory/Cargo.toml b/foyer-memory/Cargo.toml
index cc1fbe61..42469281 100644
--- a/foyer-memory/Cargo.toml
+++ b/foyer-memory/Cargo.toml
@@ -13,7 +13,7 @@ readme = { workspace = true }
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
-ahash = "0.8"
+ahash = { workspace = true }
 bitflags = "2"
 cmsketch = "0.2.1"
 equivalent = { workspace = true }
@@ -27,6 +27,7 @@ parking_lot = { workspace = true }
 paste = "1"
 pin-project = "1"
 serde = { workspace = true }
+thiserror = { workspace = true }
 tokio = { workspace = true }
 tracing = "0.1"
 
diff --git a/foyer-memory/src/cache.rs b/foyer-memory/src/cache.rs
index af22cd45..4a4ec71e 100644
--- a/foyer-memory/src/cache.rs
+++ b/foyer-memory/src/cache.rs
@@ -36,6 +36,7 @@ use crate::{
     },
     raw::{FetchMark, FetchState, RawCache, RawCacheConfig, RawCacheEntry, RawFetch, Weighter},
     record::CacheHint,
+    Result,
 };
 
 pub type FifoCache<K, V, S = RandomState> = RawCache<Fifo<K, V>, S>;
@@ -513,6 +514,17 @@ where
     V: Value,
     S: HashBuilder,
 {
+    /// Update capacity and evict overflowed entries.
+    #[fastrace::trace(name = "foyer::memory::cache::resize")]
+    pub fn resize(&self, capacity: usize) -> Result<()> {
+        match self {
+            Cache::Fifo(cache) => cache.resize(capacity),
+            Cache::S3Fifo(cache) => cache.resize(capacity),
+            Cache::Lru(cache) => cache.resize(capacity),
+            Cache::Lfu(cache) => cache.resize(capacity),
+        }
+    }
+
     /// Insert cache entry to the in-memory cache.
     #[fastrace::trace(name = "foyer::memory::cache::insert")]
     pub fn insert(&self, key: K, value: V) -> CacheEntry<K, V, S> {
diff --git a/foyer-memory/src/error.rs b/foyer-memory/src/error.rs
new file mode 100644
index 00000000..59b74ac3
--- /dev/null
+++ b/foyer-memory/src/error.rs
@@ -0,0 +1,53 @@
+//  Copyright 2024 foyer Project Authors
+//
+//  Licensed under the Apache License, Version 2.0 (the "License");
+//  you may not use this file except in compliance with the License.
+//  You may obtain a copy of the License at
+//
+//  http://www.apache.org/licenses/LICENSE-2.0
+//
+//  Unless required by applicable law or agreed to in writing, software
+//  distributed under the License is distributed on an "AS IS" BASIS,
+//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//  See the License for the specific language governing permissions and
+//  limitations under the License.
+
+use std::fmt::Display;
+
+/// In-memory cache error.
+#[derive(thiserror::Error, Debug)]
+pub enum Error {
+    /// Multiple error list.
+    #[error(transparent)]
+    Multiple(MultipleError),
+    /// Config error.
+    #[error("config error: {0}")]
+    ConfigError(String),
+}
+
+impl Error {
+    /// Combine multiple errors into one error.
+    pub fn multiple(errs: Vec<Error>) -> Self {
+        Self::Multiple(MultipleError(errs))
+    }
+}
+
+#[derive(thiserror::Error, Debug)]
+pub struct MultipleError(Vec<Error>);
+
+impl Display for MultipleError {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        write!(f, "multiple errors: [")?;
+        if let Some((last, errs)) = self.0.as_slice().split_last() {
+            for err in errs {
+                write!(f, "{}, ", err)?;
+            }
+            write!(f, "{}", last)?;
+        }
+        write!(f, "]")?;
+        Ok(())
+    }
+}
+
+/// In-memory cache result.
+pub type Result<T> = std::result::Result<T, Error>;
diff --git a/foyer-memory/src/eviction/fifo.rs b/foyer-memory/src/eviction/fifo.rs
index bcc14188..07baeceb 100644
--- a/foyer-memory/src/eviction/fifo.rs
+++ b/foyer-memory/src/eviction/fifo.rs
@@ -19,7 +19,10 @@ use intrusive_collections::{intrusive_adapter, LinkedList, LinkedListAtomicLink}
 use serde::{Deserialize, Serialize};
 
 use super::{Eviction, Op};
-use crate::record::{CacheHint, Record};
+use crate::{
+    error::Result,
+    record::{CacheHint, Record},
+};
 
 /// Fifo eviction algorithm config.
 #[derive(Debug, Clone, Default, Serialize, Deserialize)]
@@ -77,7 +80,9 @@ where
         }
     }
 
-    fn update(&mut self, _: usize, _: &Self::Config) {}
+    fn update(&mut self, _: usize, _: Option<&Self::Config>) -> Result<()> {
+        Ok(())
+    }
 
     fn push(&mut self, record: Arc<Record<Self>>) {
         record.set_in_eviction(true);
diff --git a/foyer-memory/src/eviction/lfu.rs b/foyer-memory/src/eviction/lfu.rs
index 182f2f9f..222db9da 100644
--- a/foyer-memory/src/eviction/lfu.rs
+++ b/foyer-memory/src/eviction/lfu.rs
@@ -23,7 +23,10 @@ use intrusive_collections::{intrusive_adapter, LinkedList, LinkedListAtomicLink}
 use serde::{Deserialize, Serialize};
 
 use super::{Eviction, Op};
-use crate::record::{CacheHint, Record};
+use crate::{
+    error::{Error, Result},
+    record::{CacheHint, Record},
+};
 
 /// w-TinyLFU eviction algorithm config.
 #[derive(Debug, Clone, Serialize, Deserialize)]
@@ -135,6 +138,8 @@ where
 
     step: usize,
     decay: usize,
+
+    config: LfuConfig,
 }
 
 impl<K, V> Lfu<K, V>
@@ -203,6 +208,8 @@ where
             config.window_capacity_ratio + config.protected_capacity_ratio
         );
 
+        let config = config.clone();
+
         let window_weight_capacity = (capacity as f64 * config.window_capacity_ratio) as usize;
         let protected_weight_capacity = (capacity as f64 * config.protected_capacity_ratio) as usize;
         let frequencies = CMSketchU16::new(config.cmsketch_eps, config.cmsketch_confidence);
@@ -220,38 +227,48 @@ where
             frequencies,
             step: 0,
             decay,
+            config,
         }
     }
 
-    fn update(&mut self, capacity: usize, config: &Self::Config) {
-        if config.window_capacity_ratio <= 0.0 || config.window_capacity_ratio >= 1.0 {
-            tracing::error!(
-                "window_capacity_ratio must be in (0, 1), given: {}, new config ignored",
-                config.window_capacity_ratio
-            );
-        }
+    fn update(&mut self, capacity: usize, config: Option<&Self::Config>) -> Result<()> {
+        if let Some(config) = config {
+            let mut msgs = vec![];
+            if config.window_capacity_ratio <= 0.0 || config.window_capacity_ratio >= 1.0 {
+                msgs.push(format!(
+                    "window_capacity_ratio must be in (0, 1), given: {}, new config ignored",
+                    config.window_capacity_ratio
+                ));
+            }
+            if config.protected_capacity_ratio <= 0.0 || config.protected_capacity_ratio >= 1.0 {
+                msgs.push(format!(
+                    "protected_capacity_ratio must be in (0, 1), given: {}, new config ignored",
+                    config.protected_capacity_ratio
+                ));
+            }
+            if config.window_capacity_ratio + config.protected_capacity_ratio >= 1.0 {
+                msgs.push(format!(
+                    "must guarantee: window_capacity_ratio + protected_capacity_ratio < 1, given: {}, new config ignored",
+                    config.window_capacity_ratio + config.protected_capacity_ratio
+                ));
+            }
 
-        if config.protected_capacity_ratio <= 0.0 || config.protected_capacity_ratio >= 1.0 {
-            tracing::error!(
-                "protected_capacity_ratio must be in (0, 1), given: {}, new config ignored",
-                config.protected_capacity_ratio
-            );
-        }
+            if !msgs.is_empty() {
+                return Err(Error::ConfigError(msgs.join(" | ")));
+            }
 
-        if config.window_capacity_ratio + config.protected_capacity_ratio >= 1.0 {
-            tracing::error!(
-                "must guarantee: window_capacity_ratio + protected_capacity_ratio < 1, given: {}, new config ignored",
-                config.window_capacity_ratio + config.protected_capacity_ratio
-            )
+            self.config = config.clone();
         }
 
         // TODO(MrCroxx): Raise a warn log the cmsketch args updates is not supported yet if it is modified.
 
-        let window_weight_capacity = (capacity as f64 * config.window_capacity_ratio) as usize;
-        let protected_weight_capacity = (capacity as f64 * config.protected_capacity_ratio) as usize;
+        let window_weight_capacity = (capacity as f64 * self.config.window_capacity_ratio) as usize;
+        let protected_weight_capacity = (capacity as f64 * self.config.protected_capacity_ratio) as usize;
 
         self.window_weight_capacity = window_weight_capacity;
         self.protected_weight_capacity = protected_weight_capacity;
+
+        Ok(())
     }
 
     /// Push a new record to `window`.
diff --git a/foyer-memory/src/eviction/lru.rs b/foyer-memory/src/eviction/lru.rs
index d0a2fa80..d5ccea49 100644
--- a/foyer-memory/src/eviction/lru.rs
+++ b/foyer-memory/src/eviction/lru.rs
@@ -22,7 +22,10 @@ use intrusive_collections::{intrusive_adapter, LinkedList, LinkedListAtomicLink}
 use serde::{Deserialize, Serialize};
 
 use super::{Eviction, Op};
-use crate::record::{CacheHint, Record};
+use crate::{
+    error::{Error, Result},
+    record::{CacheHint, Record},
+};
 
 /// Lru eviction algorithm config.
 #[derive(Debug, Clone, Serialize, Deserialize)]
@@ -98,6 +101,8 @@ where
 
     high_priority_weight: usize,
     high_priority_weight_capacity: usize,
+
+    config: LruConfig,
 }
 
 impl<K, V> Lru<K, V>
@@ -141,6 +146,8 @@ where
             config.high_priority_pool_ratio
         );
 
+        let config = config.clone();
+
         let high_priority_weight_capacity = (capacity as f64 * config.high_priority_pool_ratio) as usize;
 
         Self {
@@ -149,22 +156,29 @@ where
             pin_list: LinkedList::new(Adapter::new()),
             high_priority_weight: 0,
             high_priority_weight_capacity,
+            config,
         }
     }
 
-    fn update(&mut self, capacity: usize, config: &Self::Config) {
-        if !(0.0..=1.0).contains(&config.high_priority_pool_ratio) {
-            tracing::error!(
-                "[lru]: high_priority_pool_ratio_percentage must be in 0.0..=1.0, given: {}, new configuration ignored",
-                config.high_priority_pool_ratio
-            );
-            return;
+    fn update(&mut self, capacity: usize, config: Option<&Self::Config>) -> Result<()> {
+        if let Some(config) = config {
+            if !(0.0..=1.0).contains(&config.high_priority_pool_ratio) {
+                return Err(Error::ConfigError(
+                    format!(
+                        "[lru]: high_priority_pool_ratio_percentage must be in 0.0..=1.0, given: {}, new configuration ignored",
+                        config.high_priority_pool_ratio
+                    )
+                ));
+            }
+            self.config = config.clone();
         }
 
-        let high_priority_weight_capacity = (capacity as f64 * config.high_priority_pool_ratio) as usize;
+        let high_priority_weight_capacity = (capacity as f64 * self.config.high_priority_pool_ratio) as usize;
         self.high_priority_weight_capacity = high_priority_weight_capacity;
 
         self.may_overflow_high_priority_pool();
+
+        Ok(())
     }
 
     fn push(&mut self, record: Arc<Record<Self>>) {
diff --git a/foyer-memory/src/eviction/mod.rs b/foyer-memory/src/eviction/mod.rs
index 564b5008..f3401fb5 100644
--- a/foyer-memory/src/eviction/mod.rs
+++ b/foyer-memory/src/eviction/mod.rs
@@ -17,7 +17,10 @@ use std::sync::Arc;
 use foyer_common::code::{Key, Value};
 use serde::{de::DeserializeOwned, Serialize};
 
-use crate::record::{CacheHint, Record};
+use crate::{
+    error::Result,
+    record::{CacheHint, Record},
+};
 
 pub trait Hint: Send + Sync + 'static + Clone + Default + From<CacheHint> + Into<CacheHint> {}
 impl<T> Hint for T where T: Send + Sync + 'static + Clone + Default + From<CacheHint> + Into<CacheHint> {}
@@ -102,7 +105,7 @@ pub trait Eviction: Send + Sync + 'static + Sized {
     fn new(capacity: usize, config: &Self::Config) -> Self;
 
     /// Update the arguments of the ache eviction algorithm instance.
-    fn update(&mut self, capacity: usize, config: &Self::Config);
+    fn update(&mut self, capacity: usize, config: Option<&Self::Config>) -> Result<()>;
 
     /// Push a record into the cache eviction algorithm instance.
     ///
diff --git a/foyer-memory/src/eviction/s3fifo.rs b/foyer-memory/src/eviction/s3fifo.rs
index c32a1dc4..cf210aac 100644
--- a/foyer-memory/src/eviction/s3fifo.rs
+++ b/foyer-memory/src/eviction/s3fifo.rs
@@ -29,7 +29,10 @@ use intrusive_collections::{intrusive_adapter, LinkedList, LinkedListAtomicLink}
 use serde::{Deserialize, Serialize};
 
 use super::{Eviction, Op};
-use crate::record::{CacheHint, Record};
+use crate::{
+    error::{Error, Result},
+    record::{CacheHint, Record},
+};
 
 /// S3Fifo eviction algorithm config.
 #[derive(Debug, Clone, Serialize, Deserialize)]
@@ -132,6 +135,8 @@ where
     main_weight: usize,
 
     small_to_main_freq_threshold: u8,
+
+    config: S3FifoConfig,
 }
 
 impl<K, V> S3Fifo<K, V>
@@ -215,9 +220,18 @@ where
     where
         Self: Sized,
     {
+        assert!(
+            config.small_queue_capacity_ratio > 0.0 && config.small_queue_capacity_ratio < 1.0,
+            "small_queue_capacity_ratio must be in (0, 1), given: {}",
+            config.small_queue_capacity_ratio
+        );
+
+        let config = config.clone();
+
         let ghost_queue_capacity = (capacity as f64 * config.ghost_queue_capacity_ratio) as usize;
         let ghost_queue = GhostQueue::new(ghost_queue_capacity);
         let small_weight_capacity = (capacity as f64 * config.small_queue_capacity_ratio) as usize;
+
         Self {
             ghost_queue,
             small_queue: LinkedList::new(Adapter::new()),
@@ -226,14 +240,27 @@ where
             small_weight: 0,
             main_weight: 0,
             small_to_main_freq_threshold: config.small_to_main_freq_threshold.min(S3FifoState::MAX_FREQUENCY),
+            config,
         }
     }
 
-    fn update(&mut self, capacity: usize, config: &Self::Config) {
-        let ghost_queue_capacity = (capacity as f64 * config.ghost_queue_capacity_ratio) as usize;
-        let small_weight_capacity = (capacity as f64 * config.small_queue_capacity_ratio) as usize;
+    fn update(&mut self, capacity: usize, config: Option<&Self::Config>) -> Result<()> {
+        if let Some(config) = config {
+            if config.small_queue_capacity_ratio > 0.0 && config.small_queue_capacity_ratio < 1.0 {
+                return Err(Error::ConfigError(format!(
+                    "small_queue_capacity_ratio must be in (0, 1), given: {}",
+                    config.small_queue_capacity_ratio
+                )));
+            }
+            self.config = config.clone();
+        }
+
+        let ghost_queue_capacity = (capacity as f64 * self.config.ghost_queue_capacity_ratio) as usize;
+        let small_weight_capacity = (capacity as f64 * self.config.small_queue_capacity_ratio) as usize;
         self.ghost_queue.update(ghost_queue_capacity);
         self.small_weight_capacity = small_weight_capacity;
+
+        Ok(())
     }
 
     fn push(&mut self, record: Arc<Record<Self>>) {
diff --git a/foyer-memory/src/lib.rs b/foyer-memory/src/lib.rs
index e38e82fd..6ca1fdf5 100644
--- a/foyer-memory/src/lib.rs
+++ b/foyer-memory/src/lib.rs
@@ -36,6 +36,7 @@
 //! state requires operation on the `UnsafeCell`.
 
 mod cache;
+mod error;
 mod eviction;
 mod indexer;
 mod raw;
diff --git a/foyer-memory/src/prelude.rs b/foyer-memory/src/prelude.rs
index 4e36ee44..d33ce8c5 100644
--- a/foyer-memory/src/prelude.rs
+++ b/foyer-memory/src/prelude.rs
@@ -16,6 +16,7 @@ pub use ahash::RandomState;
 
 pub use crate::{
     cache::{Cache, CacheBuilder, CacheEntry, EvictionConfig, Fetch},
+    error::{Error, Result},
     eviction::{fifo::FifoConfig, lfu::LfuConfig, lru::LruConfig, s3fifo::S3FifoConfig, Eviction, Op},
     raw::{FetchMark, FetchState, Weighter},
     record::{CacheHint, Record},
diff --git a/foyer-memory/src/raw.rs b/foyer-memory/src/raw.rs
index 93d85980..ed2a0ff1 100644
--- a/foyer-memory/src/raw.rs
+++ b/foyer-memory/src/raw.rs
@@ -43,6 +43,7 @@ use pin_project::pin_project;
 use tokio::{sync::oneshot, task::JoinHandle};
 
 use crate::{
+    error::{Error, Result},
     eviction::{Eviction, Op},
     indexer::{hash_table::HashTableIndexer, sentry::Sentry, Indexer},
     record::{Data, Record},
@@ -93,22 +94,10 @@ where
     S: HashBuilder,
     I: Indexer<Eviction = E>,
 {
-    fn emplace(
-        &mut self,
-        data: Data<E>,
-        ephemeral: bool,
-        garbages: &mut Vec<(Event, Arc<Record<E>>)>,
-        waiters: &mut Vec<oneshot::Sender<RawCacheEntry<E, S, I>>>,
-    ) -> Arc<Record<E>> {
-        std::mem::swap(waiters, &mut self.waiters.lock().remove(&data.key).unwrap_or_default());
-
-        let weight = data.weight;
-        let old_usage = self.usage;
-
-        let record = Arc::new(Record::new(data));
-
+    /// Evict entries to fit the target usage.
+    fn evict(&mut self, target: usize, garbages: &mut Vec<(Event, Arc<Record<E>>)>) {
         // Evict overflow records.
-        while self.usage + weight > self.capacity {
+        while self.usage > target {
             let evicted = match self.eviction.pop() {
                 Some(evicted) => evicted,
                 None => break,
@@ -125,6 +114,24 @@ where
 
             garbages.push((Event::Evict, evicted));
         }
+    }
+
+    fn emplace(
+        &mut self,
+        data: Data<E>,
+        ephemeral: bool,
+        garbages: &mut Vec<(Event, Arc<Record<E>>)>,
+        waiters: &mut Vec<oneshot::Sender<RawCacheEntry<E, S, I>>>,
+    ) -> Arc<Record<E>> {
+        std::mem::swap(waiters, &mut self.waiters.lock().remove(&data.key).unwrap_or_default());
+
+        let weight = data.weight;
+        let old_usage = self.usage;
+
+        let record = Arc::new(Record::new(data));
+
+        // Evict overflow records.
+        self.evict(self.capacity.saturating_sub(weight), garbages);
 
         // Insert new record
         if let Some(old) = self.indexer.insert(record.clone()) {
@@ -454,6 +461,46 @@ where
         Self { inner: Arc::new(inner) }
     }
 
+    #[fastrace::trace(name = "foyer::memory::raw::resize")]
+    pub fn resize(&self, capacity: usize) -> Result<()> {
+        let shards = self.inner.shards.len();
+        let shard_capacity = capacity / shards;
+
+        let handles = (0..shards)
+            .map(|i| {
+                let inner = self.inner.clone();
+                std::thread::spawn(move || {
+                    let mut garbages = vec![];
+                    let res = inner.shards[i].write().with(|mut shard| {
+                        shard.eviction.update(shard_capacity, None).inspect(|_| {
+                            shard.capacity = shard_capacity;
+                            shard.evict(shard_capacity, &mut garbages)
+                        })
+                    });
+                    // Deallocate data out of the lock critical section.
+                    if let Some(listener) = inner.event_listener.as_ref() {
+                        for (event, record) in garbages {
+                            listener.on_leave(event, record.key(), record.value());
+                        }
+                    }
+                    res
+                })
+            })
+            .collect_vec();
+
+        let errs = handles
+            .into_iter()
+            .map(|handle| handle.join().unwrap())
+            .filter(|res| res.is_err())
+            .map(|res| res.unwrap_err())
+            .collect_vec();
+        if !errs.is_empty() {
+            return Err(Error::multiple(errs));
+        }
+
+        Ok(())
+    }
+
     #[fastrace::trace(name = "foyer::memory::raw::insert")]
     pub fn insert(&self, key: E::Key, value: E::Value) -> RawCacheEntry<E, S, I> {
         self.insert_with_hint(key, value, Default::default())
@@ -936,6 +983,7 @@ where
 #[cfg(test)]
 mod tests {
 
+    use foyer_common::hasher::ModRandomState;
     use rand::{rngs::SmallRng, seq::SliceRandom, RngCore, SeedableRng};
 
     use super::*;
@@ -948,7 +996,15 @@ mod tests {
 
     fn is_send_sync_static<T: Send + Sync + 'static>() {}
 
-    fn fifo_cache_for_test() -> RawCache<Fifo<u64, u64>> {
+    #[test]
+    fn test_send_sync_static() {
+        is_send_sync_static::<RawCache<Fifo<(), ()>>>();
+        is_send_sync_static::<RawCache<S3Fifo<(), ()>>>();
+        is_send_sync_static::<RawCache<Lfu<(), ()>>>();
+        is_send_sync_static::<RawCache<Lru<(), ()>>>();
+    }
+
+    fn fifo_cache_for_test() -> RawCache<Fifo<u64, u64>, ModRandomState, HashTableIndexer<Fifo<u64, u64>>> {
         RawCache::new(RawCacheConfig {
             capacity: 256,
             shards: 4,
@@ -960,12 +1016,40 @@ mod tests {
         })
     }
 
-    #[test]
-    fn test_send_sync_static() {
-        is_send_sync_static::<RawCache<Fifo<(), ()>>>();
-        is_send_sync_static::<RawCache<S3Fifo<(), ()>>>();
-        is_send_sync_static::<RawCache<Lfu<(), ()>>>();
-        is_send_sync_static::<RawCache<Lru<(), ()>>>();
+    fn s3fifo_cache_for_test() -> RawCache<S3Fifo<u64, u64>, ModRandomState, HashTableIndexer<S3Fifo<u64, u64>>> {
+        RawCache::new(RawCacheConfig {
+            capacity: 256,
+            shards: 4,
+            eviction_config: S3FifoConfig::default(),
+            hash_builder: Default::default(),
+            weighter: Arc::new(|_, _| 1),
+            event_listener: None,
+            metrics: Arc::new(Metrics::noop()),
+        })
+    }
+
+    fn lru_cache_for_test() -> RawCache<Lru<u64, u64>, ModRandomState, HashTableIndexer<Lru<u64, u64>>> {
+        RawCache::new(RawCacheConfig {
+            capacity: 256,
+            shards: 4,
+            eviction_config: LruConfig::default(),
+            hash_builder: Default::default(),
+            weighter: Arc::new(|_, _| 1),
+            event_listener: None,
+            metrics: Arc::new(Metrics::noop()),
+        })
+    }
+
+    fn lfu_cache_for_test() -> RawCache<Lfu<u64, u64>, ModRandomState, HashTableIndexer<Lfu<u64, u64>>> {
+        RawCache::new(RawCacheConfig {
+            capacity: 256,
+            shards: 4,
+            eviction_config: LfuConfig::default(),
+            hash_builder: Default::default(),
+            weighter: Arc::new(|_, _| 1),
+            event_listener: None,
+            metrics: Arc::new(Metrics::noop()),
+        })
     }
 
     #[test]
@@ -986,6 +1070,47 @@ mod tests {
         assert_eq!(fifo.usage(), 1);
     }
 
+    fn test_resize<E>(cache: &RawCache<E, ModRandomState, HashTableIndexer<E>>)
+    where
+        E: Eviction<Key = u64, Value = u64>,
+    {
+        let capacity = cache.capacity();
+        for i in 0..capacity as u64 * 2 {
+            cache.insert(i, i);
+        }
+        assert_eq!(cache.usage(), capacity);
+        cache.resize(capacity / 2).unwrap();
+        assert_eq!(cache.usage(), capacity / 2);
+        for i in 0..capacity as u64 * 2 {
+            cache.insert(i, i);
+        }
+        assert_eq!(cache.usage(), capacity / 2);
+    }
+
+    #[test]
+    fn test_fifo_cache_resize() {
+        let cache = fifo_cache_for_test();
+        test_resize(&cache);
+    }
+
+    #[test]
+    fn test_s3fifo_cache_resize() {
+        let cache = s3fifo_cache_for_test();
+        test_resize(&cache);
+    }
+
+    #[test]
+    fn test_lru_cache_resize() {
+        let cache = lru_cache_for_test();
+        test_resize(&cache);
+    }
+
+    #[test]
+    fn test_lfu_cache_resize() {
+        let cache = lfu_cache_for_test();
+        test_resize(&cache);
+    }
+
     mod fuzzy {
         use super::*;
 
diff --git a/foyer-storage/Cargo.toml b/foyer-storage/Cargo.toml
index db5d8be8..e4be9b44 100644
--- a/foyer-storage/Cargo.toml
+++ b/foyer-storage/Cargo.toml
@@ -13,7 +13,7 @@ readme = { workspace = true }
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
-ahash = "0.8"
+ahash = { workspace = true }
 # TODO(MrCroxx): Remove this after `allocator_api` is stable.
 allocator-api2 = "0.2"
 anyhow = "1.0"
@@ -43,7 +43,7 @@ paste = "1"
 pin-project = "1"
 rand = "0.8"
 serde = { workspace = true }
-thiserror = "1"
+thiserror = { workspace = true }
 tokio = { workspace = true }
 tracing = "0.1"
 twox-hash = "1"
diff --git a/foyer-storage/src/small/generic.rs b/foyer-storage/src/small/generic.rs
index fedf7487..ed8ed87d 100644
--- a/foyer-storage/src/small/generic.rs
+++ b/foyer-storage/src/small/generic.rs
@@ -281,7 +281,7 @@ mod tests {
     use std::path::Path;
 
     use bytesize::ByteSize;
-    use foyer_common::metrics::model::Metrics;
+    use foyer_common::{hasher::ModRandomState, metrics::model::Metrics};
     use foyer_memory::{Cache, CacheBuilder, FifoConfig};
     use tokio::runtime::Handle;
 
@@ -292,14 +292,13 @@ mod tests {
             Dev,
         },
         serde::EntrySerializer,
-        test_utils::ModHasher,
         DevExt, DirectFsDeviceOptions,
     };
 
-    fn cache_for_test() -> Cache<u64, Vec<u8>, ModHasher> {
+    fn cache_for_test() -> Cache<u64, Vec<u8>, ModRandomState> {
         CacheBuilder::new(10)
             .with_shards(1)
-            .with_hash_builder(ModHasher::default())
+            .with_hash_builder(ModRandomState::default())
             .with_eviction_config(FifoConfig::default())
             .build()
     }
@@ -320,7 +319,7 @@ mod tests {
         .unwrap()
     }
 
-    async fn store_for_test(dir: impl AsRef<Path>) -> GenericSmallStorage<u64, Vec<u8>, ModHasher> {
+    async fn store_for_test(dir: impl AsRef<Path>) -> GenericSmallStorage<u64, Vec<u8>, ModRandomState> {
         let device = device_for_test(dir).await;
         let regions = 0..device.regions() as RegionId;
         let config = GenericSmallStorageConfig {
@@ -339,14 +338,17 @@ mod tests {
         GenericSmallStorage::open(config).await.unwrap()
     }
 
-    fn enqueue(store: &GenericSmallStorage<u64, Vec<u8>, ModHasher>, entry: &CacheEntry<u64, Vec<u8>, ModHasher>) {
+    fn enqueue(
+        store: &GenericSmallStorage<u64, Vec<u8>, ModRandomState>,
+        entry: &CacheEntry<u64, Vec<u8>, ModRandomState>,
+    ) {
         let estimated_size = EntrySerializer::estimated_size(entry.key(), entry.value());
         store.enqueue(entry.clone(), estimated_size);
     }
 
     async fn assert_some(
-        store: &GenericSmallStorage<u64, Vec<u8>, ModHasher>,
-        entry: &CacheEntry<u64, Vec<u8>, ModHasher>,
+        store: &GenericSmallStorage<u64, Vec<u8>, ModRandomState>,
+        entry: &CacheEntry<u64, Vec<u8>, ModRandomState>,
     ) {
         assert_eq!(
             store.load(entry.hash()).await.unwrap().unwrap(),
@@ -355,8 +357,8 @@ mod tests {
     }
 
     async fn assert_none(
-        store: &GenericSmallStorage<u64, Vec<u8>, ModHasher>,
-        entry: &CacheEntry<u64, Vec<u8>, ModHasher>,
+        store: &GenericSmallStorage<u64, Vec<u8>, ModRandomState>,
+        entry: &CacheEntry<u64, Vec<u8>, ModRandomState>,
     ) {
         assert!(store.load(entry.hash()).await.unwrap().is_none());
     }
diff --git a/foyer-storage/src/test_utils.rs b/foyer-storage/src/test_utils.rs
index d80b707b..11f124d7 100644
--- a/foyer-storage/src/test_utils.rs
+++ b/foyer-storage/src/test_utils.rs
@@ -14,13 +14,7 @@
 
 //! Test utils for the `foyer-storage` crate.
 
-use std::{
-    borrow::Borrow,
-    collections::HashSet,
-    fmt::Debug,
-    hash::{BuildHasher, Hash, Hasher},
-    sync::Arc,
-};
+use std::{borrow::Borrow, collections::HashSet, fmt::Debug, hash::Hash, sync::Arc};
 
 use foyer_common::code::StorageKey;
 use parking_lot::Mutex;
@@ -161,29 +155,3 @@ where
         false
     }
 }
-
-/// A hasher return u64 mod result.
-#[derive(Debug, Default)]
-pub struct ModHasher {
-    state: u64,
-}
-
-impl Hasher for ModHasher {
-    fn finish(&self) -> u64 {
-        self.state
-    }
-
-    fn write(&mut self, bytes: &[u8]) {
-        for byte in bytes {
-            self.state = (self.state << 8) + *byte as u64;
-        }
-    }
-}
-
-impl BuildHasher for ModHasher {
-    type Hasher = Self;
-
-    fn build_hasher(&self) -> Self::Hasher {
-        Self::default()
-    }
-}
diff --git a/foyer/Cargo.toml b/foyer/Cargo.toml
index cb831682..a33e6885 100644
--- a/foyer/Cargo.toml
+++ b/foyer/Cargo.toml
@@ -13,7 +13,7 @@ readme = { workspace = true }
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
-ahash = "0.8"
+ahash = { workspace = true }
 anyhow = "1"
 equivalent = { workspace = true }
 fastrace = { workspace = true }

From eabeb1489cacdc5b0e546ad74f170ac14dd50269 Mon Sep 17 00:00:00 2001
From: Croxx <mrcroxx@outlook.com>
Date: Tue, 26 Nov 2024 14:58:36 +0800
Subject: [PATCH 48/53] fix: fix lru high priority weight calculation (#797)

Signed-off-by: MrCroxx <mrcroxx@outlook.com>
---
 foyer-memory/src/eviction/lru.rs | 22 ++++++++++++++++++++--
 1 file changed, 20 insertions(+), 2 deletions(-)

diff --git a/foyer-memory/src/eviction/lru.rs b/foyer-memory/src/eviction/lru.rs
index d5ccea49..63038e5e 100644
--- a/foyer-memory/src/eviction/lru.rs
+++ b/foyer-memory/src/eviction/lru.rs
@@ -226,7 +226,12 @@ where
         strict_assert!(state.link.is_linked());
 
         match (state.is_pinned, state.in_high_priority_pool) {
-            (true, _) => unsafe { self.pin_list.remove_from_ptr(Arc::as_ptr(record)) },
+            (true, false) => unsafe { self.pin_list.remove_from_ptr(Arc::as_ptr(record)) },
+            (true, true) => unsafe {
+                self.high_priority_weight -= record.weight();
+                state.in_high_priority_pool = false;
+                self.pin_list.remove_from_ptr(Arc::as_ptr(record))
+            },
             (false, true) => {
                 self.high_priority_weight -= record.weight();
                 state.in_high_priority_pool = false;
@@ -501,12 +506,25 @@ pub mod tests {
         lru.acquire_mutable(&rs[11]);
         assert_ptr_vec_vec_eq(lru.dump(), vec![vec![r(10)], vec![r(1)], vec![r(0), r(11)]]);
 
-        // remove pinned
+        // remove pinned (low priority)
         // pin: [0]
         // 10, [1]
         lru.remove(&rs[11]);
         assert_ptr_vec_vec_eq(lru.dump(), vec![vec![r(10)], vec![r(1)], vec![r(0)]]);
 
+        // remove pinned (high priority)
+        // step 1:
+        //   pin: [0], [2]
+        //   10, [1]
+        lru.push(r(2));
+        lru.acquire_mutable(&rs[2]);
+        assert_ptr_vec_vec_eq(lru.dump(), vec![vec![r(10)], vec![r(1)], vec![r(0), r(2)]]);
+        // step 2:
+        //   pin: [0]
+        //   10, [1]
+        lru.remove(&rs[2]);
+        assert_ptr_vec_vec_eq(lru.dump(), vec![vec![r(10)], vec![r(1)], vec![r(0)]]);
+
         // release removed
         // pin: [0]
         // 10, [1]

From 153b618c52a418e79d57116c22337f7ebad39b1b Mon Sep 17 00:00:00 2001
From: Croxx <mrcroxx@outlook.com>
Date: Tue, 26 Nov 2024 16:30:45 +0800
Subject: [PATCH 49/53] doc: add example for export metrics with prometheus and
 hyper (#799)

Signed-off-by: MrCroxx <mrcroxx@outlook.com>
---
 .github/workflows/ci.yml                    |   1 +
 Cargo.toml                                  |   3 +-
 Makefile                                    |   1 +
 examples/Cargo.toml                         |  18 ++-
 examples/export_metrics_prometheus_hyper.rs | 125 ++++++++++++++++++++
 foyer-bench/Cargo.toml                      |   4 +-
 foyer-memory/Cargo.toml                     |   2 +-
 foyer-storage/Cargo.toml                    |   2 +-
 foyer/Cargo.toml                            |   2 +-
 9 files changed, 148 insertions(+), 10 deletions(-)
 create mode 100644 examples/export_metrics_prometheus_hyper.rs

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 3d51fc3b..3bb0c12a 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -195,6 +195,7 @@ jobs:
           cargo llvm-cov --no-report run --features "tracing,jaeger" --example tail_based_tracing
           cargo llvm-cov --no-report run --features "tracing,ot" --example tail_based_tracing
           cargo llvm-cov --no-report run --example equivalent
+          cargo llvm-cov --no-report run --example export_metrics_prometheus_hyper
       - name: Run foyer-bench with coverage
         if: runner.os == 'Linux'
         env:
diff --git a/Cargo.toml b/Cargo.toml
index de78d52c..e01b3d65 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -28,8 +28,6 @@ clap = { version = "4", features = ["derive"] }
 crossbeam = "0.8"
 equivalent = "1"
 fastrace = "0.7"
-fastrace-jaeger = "0.7"
-fastrace-opentelemetry = "0.7"
 hashbrown = "0.14"
 itertools = "0.13"
 parking_lot = { version = "0.12" }
@@ -48,6 +46,7 @@ tokio = { package = "madsim-tokio", version = "0.2", features = [
     "signal",
     "fs",
 ] }
+tracing = "0.1"
 prometheus = "0.13"
 opentelemetry_0_27 = { package = "opentelemetry", version = "0.27" }
 opentelemetry_0_26 = { package = "opentelemetry", version = "0.26" }
diff --git a/Makefile b/Makefile
index 67714e85..e2c9831f 100644
--- a/Makefile
+++ b/Makefile
@@ -45,6 +45,7 @@ example:
 	cargo run --features "tracing,jaeger" --example tail_based_tracing
 	cargo run --features "tracing,ot" --example tail_based_tracing
 	cargo run --example equivalent
+	cargo run --example export_metrics_prometheus_hyper
 
 full: check-all test-all example udeps
 
diff --git a/examples/Cargo.toml b/examples/Cargo.toml
index 92f2b274..04c26b62 100644
--- a/examples/Cargo.toml
+++ b/examples/Cargo.toml
@@ -19,9 +19,15 @@ anyhow = "1"
 chrono = "0.4"
 equivalent = { workspace = true }
 fastrace = { workspace = true }
-fastrace-jaeger = { workspace = true, optional = true }
-fastrace-opentelemetry = { workspace = true, optional = true }
-foyer = { workspace = true }
+fastrace-jaeger = { version = "0.7", optional = true }
+fastrace-opentelemetry = { version = "0.7", optional = true }
+foyer = { workspace = true, features = ["prometheus"] }
+http-body-util = "0.1"
+hyper = { version = "1", default-features = false, features = [
+    "server",
+    "http1",
+] }
+hyper-util = { version = "0.1", default-features = false, features = ["tokio"] }
 opentelemetry = { version = "0.26", optional = true }
 opentelemetry-otlp = { version = "0.26", optional = true }
 opentelemetry-semantic-conventions = { version = "0.26", optional = true }
@@ -29,8 +35,10 @@ opentelemetry_sdk = { version = "0.26", features = [
     "rt-tokio",
     "trace",
 ], optional = true }
+prometheus = { workspace = true }
 tempfile = "3"
 tokio = { version = "1", features = ["rt"] }
+tracing = { workspace = true }
 
 [features]
 jaeger = ["fastrace-jaeger"]
@@ -65,3 +73,7 @@ path = "tail_based_tracing.rs"
 [[example]]
 name = "equivalent"
 path = "equivalent.rs"
+
+[[example]]
+name = "export_metrics_prometheus_hyper"
+path = "export_metrics_prometheus_hyper.rs"
diff --git a/examples/export_metrics_prometheus_hyper.rs b/examples/export_metrics_prometheus_hyper.rs
new file mode 100644
index 00000000..56af8aa9
--- /dev/null
+++ b/examples/export_metrics_prometheus_hyper.rs
@@ -0,0 +1,125 @@
+//  Copyright 2024 foyer Project Authors
+//
+//  Licensed under the Apache License, Version 2.0 (the "License");
+//  you may not use this file except in compliance with the License.
+//  You may obtain a copy of the License at
+//
+//  http://www.apache.org/licenses/LICENSE-2.0
+//
+//  Unless required by applicable law or agreed to in writing, software
+//  distributed under the License is distributed on an "AS IS" BASIS,
+//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//  See the License for the specific language governing permissions and
+//  limitations under the License.
+
+use std::{future::Future, net::SocketAddr, pin::Pin};
+
+use anyhow::Ok;
+use foyer::{Cache, CacheBuilder, PrometheusMetricsRegistry};
+use http_body_util::Full;
+use hyper::{
+    body::{Bytes, Incoming},
+    header::CONTENT_TYPE,
+    server::conn::http1,
+    service::Service,
+    Request, Response,
+};
+use hyper_util::rt::TokioIo;
+use prometheus::{Encoder, Registry, TextEncoder};
+use tokio::net::TcpListener;
+
+pub struct PrometheusExporter {
+    registry: Registry,
+    addr: SocketAddr,
+}
+
+impl PrometheusExporter {
+    pub fn new(registry: Registry, addr: SocketAddr) -> Self {
+        Self { registry, addr }
+    }
+
+    pub fn run(self) {
+        tokio::spawn(async move {
+            let listener = TcpListener::bind(&self.addr).await.unwrap();
+            loop {
+                let (stream, _) = match listener.accept().await {
+                    Result::Ok(res) => res,
+                    Err(e) => {
+                        tracing::error!("[prometheus exporter]: accept connection error: {e}");
+                        continue;
+                    }
+                };
+
+                let io = TokioIo::new(stream);
+                let handle = Handle {
+                    registry: self.registry.clone(),
+                };
+
+                tokio::spawn(async move {
+                    if let Err(e) = http1::Builder::new().serve_connection(io, handle).await {
+                        tracing::error!("[prometheus exporter]: serve request error: {e}");
+                    }
+                });
+            }
+        });
+    }
+}
+
+struct Handle {
+    registry: Registry,
+}
+
+impl Service<Request<Incoming>> for Handle {
+    type Response = Response<Full<Bytes>>;
+    type Error = anyhow::Error;
+    type Future = Pin<Box<dyn Future<Output = Result<Self::Response, Self::Error>> + Send>>;
+
+    fn call(&self, _: Request<Incoming>) -> Self::Future {
+        let mfs = self.registry.gather();
+
+        Box::pin(async move {
+            let encoder = TextEncoder::new();
+            let mut buffer = vec![];
+            encoder.encode(&mfs, &mut buffer)?;
+
+            Ok(Response::builder()
+                .status(200)
+                .header(CONTENT_TYPE, encoder.format_type())
+                .body(Full::new(Bytes::from(buffer)))
+                .unwrap())
+        })
+    }
+}
+
+#[tokio::main]
+async fn main() {
+    // Create a new registry or use the global registry of `prometheus` lib.
+    let registry = Registry::new();
+
+    // Create a `PrometheusExporter` powered by hyper and run it.
+    let addr = "127.0.0.1:19970".parse().unwrap();
+    PrometheusExporter::new(registry.clone(), addr).run();
+
+    // Build a cache with `PrometheusMetricsRegistry`.
+    let _: Cache<String, String> = CacheBuilder::new(100)
+        .with_metrics_registry(PrometheusMetricsRegistry::new(registry))
+        .build();
+
+    // > curl http://127.0.0.1:7890
+    //
+    // # HELP foyer_hybrid_op_duration foyer hybrid cache operation durations
+    // # TYPE foyer_hybrid_op_duration histogram
+    // foyer_hybrid_op_duration_bucket{name="foyer",op="fetch",le="0.005"} 0
+    // foyer_hybrid_op_duration_bucket{name="foyer",op="fetch",le="0.01"} 0
+    // foyer_hybrid_op_duration_bucket{name="foyer",op="fetch",le="0.025"} 0
+    // foyer_hybrid_op_duration_bucket{name="foyer",op="fetch",le="0.05"} 0
+    // foyer_hybrid_op_duration_bucket{name="foyer",op="fetch",le="0.1"} 0
+    // foyer_hybrid_op_duration_bucket{name="foyer",op="fetch",le="0.25"} 0
+    // foyer_hybrid_op_duration_bucket{name="foyer",op="fetch",le="0.5"} 0
+    // foyer_hybrid_op_duration_bucket{name="foyer",op="fetch",le="1"} 0
+    // foyer_hybrid_op_duration_bucket{name="foyer",op="fetch",le="2.5"} 0
+    // foyer_hybrid_op_duration_bucket{name="foyer",op="fetch",le="5"} 0
+    // foyer_hybrid_op_duration_bucket{name="foyer",op="fetch",le="10"} 0
+    // foyer_hybrid_op_duration_bucket{name="foyer",op="fetch",le="+Inf"} 0
+    // ... ...
+}
diff --git a/foyer-bench/Cargo.toml b/foyer-bench/Cargo.toml
index 77c78a46..b629d8b9 100644
--- a/foyer-bench/Cargo.toml
+++ b/foyer-bench/Cargo.toml
@@ -18,7 +18,7 @@ bytesize = { workspace = true }
 clap = { workspace = true }
 console-subscriber = { version = "0.4", optional = true }
 fastrace = { workspace = true, optional = true }
-fastrace-jaeger = { workspace = true, optional = true }
+fastrace-jaeger = { version = "0.7", optional = true }
 foyer = { workspace = true, features = ["prometheus"] }
 futures = "0.3"
 hdrhistogram = "7"
@@ -36,7 +36,7 @@ rand = "0.8.5"
 serde = { workspace = true }
 serde_bytes = "0.11.15"
 tokio = { workspace = true, features = ["net"] }
-tracing = "0.1"
+tracing = { workspace = true }
 tracing-opentelemetry = { version = "0.27", optional = true }
 tracing-subscriber = { version = "0.3", features = ["env-filter"] }
 zipf = "7"
diff --git a/foyer-memory/Cargo.toml b/foyer-memory/Cargo.toml
index 42469281..d0419a8d 100644
--- a/foyer-memory/Cargo.toml
+++ b/foyer-memory/Cargo.toml
@@ -29,7 +29,7 @@ pin-project = "1"
 serde = { workspace = true }
 thiserror = { workspace = true }
 tokio = { workspace = true }
-tracing = "0.1"
+tracing = { workspace = true }
 
 [dev-dependencies]
 csv = "1.3.0"
diff --git a/foyer-storage/Cargo.toml b/foyer-storage/Cargo.toml
index e4be9b44..a79e7f61 100644
--- a/foyer-storage/Cargo.toml
+++ b/foyer-storage/Cargo.toml
@@ -45,7 +45,7 @@ rand = "0.8"
 serde = { workspace = true }
 thiserror = { workspace = true }
 tokio = { workspace = true }
-tracing = "0.1"
+tracing = { workspace = true }
 twox-hash = "1"
 zstd = "0.13"
 
diff --git a/foyer/Cargo.toml b/foyer/Cargo.toml
index a33e6885..b61e65c2 100644
--- a/foyer/Cargo.toml
+++ b/foyer/Cargo.toml
@@ -23,7 +23,7 @@ foyer-storage = { workspace = true }
 futures = "0.3"
 pin-project = "1"
 tokio = { workspace = true }
-tracing = "0.1"
+tracing = { workspace = true }
 
 [dev-dependencies]
 tempfile = "3"

From 19bc2def14dfa8bb7d6581fb625584694bc7f4a0 Mon Sep 17 00:00:00 2001
From: Croxx <mrcroxx@outlook.com>
Date: Wed, 27 Nov 2024 00:25:52 +0800
Subject: [PATCH 50/53] feat: support multiple foyer instance share the same
 prometheus (#801)

Signed-off-by: MrCroxx <mrcroxx@outlook.com>
---
 Makefile                                      |   2 +-
 .../src/metrics/registry/prometheus.rs        | 195 +++++++++++++++---
 2 files changed, 172 insertions(+), 25 deletions(-)

diff --git a/Makefile b/Makefile
index e2c9831f..b836120b 100644
--- a/Makefile
+++ b/Makefile
@@ -49,7 +49,7 @@ example:
 
 full: check-all test-all example udeps
 
-fast: check test example
+fast: check test example1
 
 msrv:
 	shellcheck ./scripts/*
diff --git a/foyer-common/src/metrics/registry/prometheus.rs b/foyer-common/src/metrics/registry/prometheus.rs
index 18a9bc35..fbb3fbe6 100644
--- a/foyer-common/src/metrics/registry/prometheus.rs
+++ b/foyer-common/src/metrics/registry/prometheus.rs
@@ -12,12 +12,105 @@
 //  See the License for the specific language governing permissions and
 //  limitations under the License.
 
+use std::{
+    collections::HashMap,
+    hash::{Hash, Hasher},
+    sync::{Arc, LazyLock},
+};
+
+use parking_lot::Mutex;
 use prometheus::{
     register_histogram_vec_with_registry, register_int_counter_vec_with_registry, register_int_gauge_vec_with_registry,
     Histogram, HistogramVec, IntCounter, IntCounterVec, IntGauge, IntGaugeVec, Registry,
 };
 
-use crate::metrics::{CounterOps, CounterVecOps, GaugeOps, GaugeVecOps, HistogramOps, HistogramVecOps, RegistryOps};
+use crate::{
+    metrics::{CounterOps, CounterVecOps, GaugeOps, GaugeVecOps, HistogramOps, HistogramVecOps, RegistryOps},
+    scope::Scope,
+};
+
+static METRICS: LazyLock<Mutex<HashMap<PrometheusMetricsRegistry, HashMap<Metadata, MetricVec>>>> =
+    LazyLock::new(|| Mutex::new(HashMap::new()));
+
+fn get_or_register_counter_vec(registry: &PrometheusMetricsRegistry, metadata: Metadata) -> IntCounterVec {
+    let vec = METRICS.lock().with(|mut metrics| {
+        metrics
+            .get_mut(registry)
+            .expect("registry must be registered when creating")
+            .entry(metadata.clone())
+            .or_insert_with(|| {
+                MetricVec::Counter(
+                    register_int_counter_vec_with_registry! {
+                        metadata.name, metadata.desc, metadata.label_names, registry.registry
+                    }
+                    .unwrap(),
+                )
+            })
+            .clone()
+    });
+    match vec {
+        MetricVec::Counter(v) => v,
+        _ => unreachable!(),
+    }
+}
+
+fn get_or_register_gauge_vec(registry: &PrometheusMetricsRegistry, metadata: Metadata) -> IntGaugeVec {
+    let vec = METRICS.lock().with(|mut metrics| {
+        metrics
+            .get_mut(registry)
+            .expect("registry must be registered when creating")
+            .entry(metadata.clone())
+            .or_insert_with(|| {
+                MetricVec::Gauge(
+                    register_int_gauge_vec_with_registry! {
+                        metadata.name, metadata.desc, metadata.label_names, registry.registry
+                    }
+                    .unwrap(),
+                )
+            })
+            .clone()
+    });
+    match vec {
+        MetricVec::Gauge(v) => v,
+        _ => unreachable!(),
+    }
+}
+
+fn get_or_register_histogram_vec(registry: &PrometheusMetricsRegistry, metadata: Metadata) -> HistogramVec {
+    let vec = METRICS.lock().with(|mut metrics| {
+        metrics
+            .get_mut(registry)
+            .expect("registry must be registered when creating")
+            .entry(metadata.clone())
+            .or_insert_with(|| {
+                MetricVec::Histogram(
+                    register_histogram_vec_with_registry! {
+                        metadata.name, metadata.desc, metadata.label_names, registry.registry
+                    }
+                    .unwrap(),
+                )
+            })
+            .clone()
+    });
+    match vec {
+        MetricVec::Histogram(v) => v,
+        _ => unreachable!(),
+    }
+}
+
+#[derive(Debug, Clone)]
+enum MetricVec {
+    Counter(IntCounterVec),
+    Gauge(IntGaugeVec),
+    Histogram(HistogramVec),
+}
+
+#[derive(Debug, Clone, Hash, PartialEq, Eq)]
+struct Metadata {
+    name: &'static str,
+    desc: &'static str,
+    label_names: &'static [&'static str],
+}
 
 impl CounterOps for IntCounter {
     fn increase(&self, val: u64) {
@@ -64,15 +157,35 @@ impl HistogramVecOps for HistogramVec {
 }
 
 /// Prometheus metrics registry.
-#[derive(Debug)]
+///
+/// The [`PrometheusMetricsRegistry`] can be cloned and used by multiple foyer instances, without worrying about
+/// duplicately registering.
+#[derive(Debug, Clone)]
 pub struct PrometheusMetricsRegistry {
-    registry: Registry,
+    registry: Arc<Registry>,
+}
+
+impl PartialEq for PrometheusMetricsRegistry {
+    fn eq(&self, other: &Self) -> bool {
+        Arc::ptr_eq(&self.registry, &other.registry)
+    }
+}
+
+impl Eq for PrometheusMetricsRegistry {}
+
+impl Hash for PrometheusMetricsRegistry {
+    fn hash<H: Hasher>(&self, state: &mut H) {
+        Arc::as_ptr(&self.registry).hash(state);
+    }
 }
 
 impl PrometheusMetricsRegistry {
     /// Create an Prometheus metrics registry.
     pub fn new(registry: Registry) -> Self {
-        Self { registry }
+        let registry = Arc::new(registry);
+        let this = Self { registry };
+        METRICS.lock().insert(this.clone(), HashMap::new());
+        this
     }
 }
 
@@ -83,10 +196,14 @@ impl RegistryOps for PrometheusMetricsRegistry {
         desc: &'static str,
         label_names: &'static [&'static str],
     ) -> impl CounterVecOps {
-        register_int_counter_vec_with_registry! {
-            name, desc, label_names, self.registry
-        }
-        .unwrap()
+        get_or_register_counter_vec(
+            self,
+            Metadata {
+                name,
+                desc,
+                label_names,
+            },
+        )
     }
 
     fn register_gauge_vec(
@@ -95,10 +212,14 @@ impl RegistryOps for PrometheusMetricsRegistry {
         desc: &'static str,
         label_names: &'static [&'static str],
     ) -> impl GaugeVecOps {
-        register_int_gauge_vec_with_registry! {
-            name, desc, label_names, self.registry
-        }
-        .unwrap()
+        get_or_register_gauge_vec(
+            self,
+            Metadata {
+                name,
+                desc,
+                label_names,
+            },
+        )
     }
 
     fn register_histogram_vec(
@@ -107,10 +228,14 @@ impl RegistryOps for PrometheusMetricsRegistry {
         desc: &'static str,
         label_names: &'static [&'static str],
     ) -> impl HistogramVecOps {
-        register_histogram_vec_with_registry! {
-            name, desc, label_names, self.registry
-        }
-        .unwrap()
+        get_or_register_histogram_vec(
+            self,
+            Metadata {
+                name,
+                desc,
+                label_names,
+            },
+        )
     }
 }
 
@@ -118,23 +243,45 @@ impl RegistryOps for PrometheusMetricsRegistry {
 mod tests {
     use super::*;
 
-    #[test]
-    fn test() {
-        let registry = Registry::new();
-        let p8s = PrometheusMetricsRegistry::new(registry);
-
-        let cv = p8s.register_counter_vec("test_counter_1", "test counter 1", &["label1", "label2"]);
+    fn case(registry: &PrometheusMetricsRegistry) {
+        let cv = registry.register_counter_vec("test_counter_1", "test counter 1", &["label1", "label2"]);
         let c = cv.counter(&["l1", "l2"]);
         c.increase(42);
 
-        let gv = p8s.register_gauge_vec("test_gauge_1", "test gauge 1", &["label1", "label2"]);
+        let gv = registry.register_gauge_vec("test_gauge_1", "test gauge 1", &["label1", "label2"]);
         let g = gv.gauge(&["l1", "l2"]);
         g.increase(514);
         g.decrease(114);
         g.absolute(114514);
 
-        let hv = p8s.register_histogram_vec("test_histogram_1", "test histogram 1", &["label1", "label2"]);
+        let hv = registry.register_histogram_vec("test_histogram_1", "test histogram 1", &["label1", "label2"]);
         let h = hv.histogram(&["l1", "l2"]);
         h.record(114.514);
     }
+
+    #[test]
+    fn test_prometheus_metrics_registry() {
+        let registry = Registry::new();
+        let p8s = PrometheusMetricsRegistry::new(registry);
+        case(&p8s);
+    }
+
+    #[should_panic]
+    #[test]
+    fn test_duplicated_prometheus_metrics_registry_wrongly() {
+        let registry = Registry::new();
+        let p8s1 = PrometheusMetricsRegistry::new(registry.clone());
+        let p8s2 = PrometheusMetricsRegistry::new(registry);
+        case(&p8s1);
+        case(&p8s2);
+    }
+
+    #[test]
+    fn test_duplicated_prometheus_metrics_registry() {
+        let registry = Registry::new();
+        let p8s1 = PrometheusMetricsRegistry::new(registry);
+        let p8s2 = p8s1.clone();
+        case(&p8s1);
+        case(&p8s2);
+    }
 }

From 2cac366bf83494acc7b200a3742ee6bc30bd9503 Mon Sep 17 00:00:00 2001
From: Croxx <mrcroxx@outlook.com>
Date: Wed, 27 Nov 2024 15:37:29 +0800
Subject: [PATCH 51/53] refacor: give hybrid cache builder a default registry
 type (#803)

Signed-off-by: MrCroxx <mrcroxx@outlook.com>
---
 foyer/src/hybrid/builder.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/foyer/src/hybrid/builder.rs b/foyer/src/hybrid/builder.rs
index da04045e..45252d87 100644
--- a/foyer/src/hybrid/builder.rs
+++ b/foyer/src/hybrid/builder.rs
@@ -30,7 +30,7 @@ use foyer_storage::{
 use crate::HybridCache;
 
 /// Hybrid cache builder.
-pub struct HybridCacheBuilder<K, V, M> {
+pub struct HybridCacheBuilder<K, V, M = NoopMetricsRegistry> {
     name: &'static str,
     event_listener: Option<Arc<dyn EventListener<Key = K, Value = V>>>,
     tracing_options: TracingOptions,

From 7e0b714a065a54b15b649bc9f58b5e36de20964f Mon Sep 17 00:00:00 2001
From: Croxx <mrcroxx@outlook.com>
Date: Wed, 27 Nov 2024 16:18:00 +0800
Subject: [PATCH 52/53] refactor: export metrics exporters in mod level (#804)

To avoid conflict on multiple features enabled.

Signed-off-by: MrCroxx <mrcroxx@outlook.com>
---
 examples/export_metrics_prometheus_hyper.rs | 2 +-
 foyer-bench/src/main.rs                     | 6 +++---
 foyer/src/prelude.rs                        | 8 ++++++--
 3 files changed, 10 insertions(+), 6 deletions(-)

diff --git a/examples/export_metrics_prometheus_hyper.rs b/examples/export_metrics_prometheus_hyper.rs
index 56af8aa9..b5770301 100644
--- a/examples/export_metrics_prometheus_hyper.rs
+++ b/examples/export_metrics_prometheus_hyper.rs
@@ -15,7 +15,7 @@
 use std::{future::Future, net::SocketAddr, pin::Pin};
 
 use anyhow::Ok;
-use foyer::{Cache, CacheBuilder, PrometheusMetricsRegistry};
+use foyer::{prometheus::PrometheusMetricsRegistry, Cache, CacheBuilder};
 use http_body_util::Full;
 use hyper::{
     body::{Bytes, Incoming},
diff --git a/foyer-bench/src/main.rs b/foyer-bench/src/main.rs
index 9a6d5ba4..0ed18eba 100644
--- a/foyer-bench/src/main.rs
+++ b/foyer-bench/src/main.rs
@@ -38,9 +38,9 @@ use bytesize::ByteSize;
 use clap::{builder::PossibleValuesParser, ArgGroup, Parser};
 use exporter::PrometheusExporter;
 use foyer::{
-    Compression, DirectFileDeviceOptions, DirectFsDeviceOptions, Engine, FifoConfig, FifoPicker, HybridCache,
-    HybridCacheBuilder, InvalidRatioPicker, LargeEngineOptions, LfuConfig, LruConfig, PrometheusMetricsRegistry,
-    RateLimitPicker, RecoverMode, RuntimeOptions, S3FifoConfig, SmallEngineOptions, TokioRuntimeOptions,
+    prometheus::PrometheusMetricsRegistry, Compression, DirectFileDeviceOptions, DirectFsDeviceOptions, Engine,
+    FifoConfig, FifoPicker, HybridCache, HybridCacheBuilder, InvalidRatioPicker, LargeEngineOptions, LfuConfig,
+    LruConfig, RateLimitPicker, RecoverMode, RuntimeOptions, S3FifoConfig, SmallEngineOptions, TokioRuntimeOptions,
     TracingOptions,
 };
 use futures::future::join_all;
diff --git a/foyer/src/prelude.rs b/foyer/src/prelude.rs
index e6a878fa..c85ab6d2 100644
--- a/foyer/src/prelude.rs
+++ b/foyer/src/prelude.rs
@@ -13,9 +13,13 @@
 //  limitations under the License.
 
 #[cfg(feature = "opentelemetry")]
-pub use crate::common::metrics::registry::opentelemetry::OpenTelemetryMetricsRegistry;
+pub use crate::common::metrics::registry::opentelemetry;
+#[cfg(feature = "opentelemetry_0_26")]
+pub use crate::common::metrics::registry::opentelemetry_0_26;
+#[cfg(feature = "opentelemetry_0_27")]
+pub use crate::common::metrics::registry::opentelemetry_0_27;
 #[cfg(feature = "prometheus")]
-pub use crate::common::metrics::registry::prometheus::PrometheusMetricsRegistry;
+pub use crate::common::metrics::registry::prometheus;
 pub use crate::{
     common::{
         buf::{BufExt, BufMutExt},

From 9321f60a2de69ce433086c907d24203ef165edec Mon Sep 17 00:00:00 2001
From: Croxx <mrcroxx@outlook.com>
Date: Thu, 28 Nov 2024 15:16:56 +0800
Subject: [PATCH 53/53] feat: support prometheus-client metrics exporter (#805)

* feat: support prometheus-client metrics exporter

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

* test: text encode in ut

Signed-off-by: MrCroxx <mrcroxx@outlook.com>

---------

Signed-off-by: MrCroxx <mrcroxx@outlook.com>
---
 .github/workflows/ci.yml                      |   4 +-
 Cargo.toml                                    |   1 +
 Makefile                                      |   6 +-
 foyer-common/Cargo.toml                       |   3 +
 foyer-common/src/metrics/mod.rs               |   6 +-
 foyer-common/src/metrics/model.rs             |  14 ++
 foyer-common/src/metrics/registry/mod.rs      |   8 +
 foyer-common/src/metrics/registry/noop.rs     |   6 +-
 .../metrics/registry/opentelemetry_0_26.rs    |   6 +-
 .../metrics/registry/opentelemetry_0_27.rs    |   6 +-
 .../src/metrics/registry/prometheus.rs        |   8 +-
 .../registry/prometheus_client_0_22.rs        | 232 ++++++++++++++++++
 foyer/Cargo.toml                              |   2 +
 foyer/src/prelude.rs                          |   4 +
 14 files changed, 285 insertions(+), 21 deletions(-)
 create mode 100644 foyer-common/src/metrics/registry/prometheus_client_0_22.rs

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 3bb0c12a..af66656a 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -165,7 +165,7 @@ jobs:
           cargo clippy --all-targets --features tokio-console -- -D warnings
           cargo clippy --all-targets --features deadlock -- -D warnings
           cargo clippy --all-targets --features tracing -- -D warnings
-          cargo clippy --all-targets --features prometheus,opentelemetry_0_26,opentelemetry_0_27 -- -D warnings
+          cargo clippy --all-targets --features prometheus,prometheus-client_0_22,opentelemetry_0_26,opentelemetry_0_27 -- -D warnings
           cargo clippy --all-targets -- -D warnings
       - if: steps.cache.outputs.cache-hit != 'true'
         uses: taiki-e/install-action@cargo-llvm-cov
@@ -182,7 +182,7 @@ jobs:
           RUST_BACKTRACE: 1
           CI: true
         run: |
-          cargo llvm-cov --no-report nextest --features "strict_assertions,sanity,prometheus,opentelemetry_0_26,opentelemetry_0_27"
+          cargo llvm-cov --no-report nextest --features "strict_assertions,sanity,prometheus,prometheus-client_0_22,opentelemetry_0_26,opentelemetry_0_27"
       - name: Run examples with coverage
         env:
           RUST_BACKTRACE: 1
diff --git a/Cargo.toml b/Cargo.toml
index e01b3d65..fe308b1a 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -50,6 +50,7 @@ tracing = "0.1"
 prometheus = "0.13"
 opentelemetry_0_27 = { package = "opentelemetry", version = "0.27" }
 opentelemetry_0_26 = { package = "opentelemetry", version = "0.26" }
+prometheus-client_0_22 = { package = "prometheus-client", version = "0.22" }
 
 # foyer components
 foyer-common = { version = "0.13.0-dev", path = "foyer-common" }
diff --git a/Makefile b/Makefile
index b836120b..7ea82934 100644
--- a/Makefile
+++ b/Makefile
@@ -10,7 +10,7 @@ check:
 	./scripts/minimize-dashboards.sh
 	cargo sort -w
 	cargo fmt --all
-	cargo clippy --all-targets --features prometheus,opentelemetry_0_26,opentelemetry_0_27
+	cargo clippy --all-targets --features prometheus,prometheus-client_0_22,opentelemetry_0_26,opentelemetry_0_27
 
 check-all:
 	shellcheck ./scripts/*
@@ -21,11 +21,11 @@ check-all:
 	cargo clippy --all-targets --features tokio-console
 	cargo clippy --all-targets --features sanity
 	cargo clippy --all-targets --features tracing
-	cargo clippy --all-targets --features prometheus,opentelemetry_0_26,opentelemetry_0_27
+	cargo clippy --all-targets --features prometheus,prometheus-client_0_22,opentelemetry_0_26,opentelemetry_0_27
 	cargo clippy --all-targets
 
 test:
-	RUST_BACKTRACE=1 cargo nextest run --all --features "strict_assertions,sanity,prometheus,opentelemetry_0_26,opentelemetry_0_27"
+	RUST_BACKTRACE=1 cargo nextest run --all --features "strict_assertions,sanity,prometheus,prometheus-client_0_22,opentelemetry_0_26,opentelemetry_0_27"
 	RUST_BACKTRACE=1 cargo test --doc
 
 test-ignored:
diff --git a/foyer-common/Cargo.toml b/foyer-common/Cargo.toml
index 10f9fcde..ab862bb8 100644
--- a/foyer-common/Cargo.toml
+++ b/foyer-common/Cargo.toml
@@ -25,6 +25,7 @@ opentelemetry_0_27 = { workspace = true, optional = true }
 parking_lot = { workspace = true }
 pin-project = "1"
 prometheus = { workspace = true, optional = true }
+prometheus-client_0_22 = { workspace = true, optional = true }
 serde = { workspace = true }
 tokio = { workspace = true }
 
@@ -36,6 +37,8 @@ rand = "0.8.5"
 strict_assertions = []
 tracing = ["fastrace/enable"]
 prometheus = ["dep:prometheus"]
+prometheus-client = ["prometheus-client_0_22"]
+prometheus-client_0_22 = ["dep:prometheus-client_0_22"]
 opentelemetry = ["opentelemetry_0_27"]
 opentelemetry_0_27 = ["dep:opentelemetry_0_27"]
 opentelemetry_0_26 = ["dep:opentelemetry_0_26"]
diff --git a/foyer-common/src/metrics/mod.rs b/foyer-common/src/metrics/mod.rs
index 576182f7..8e63bb8f 100644
--- a/foyer-common/src/metrics/mod.rs
+++ b/foyer-common/src/metrics/mod.rs
@@ -39,19 +39,19 @@ pub trait HistogramOps: Send + Sync + 'static + Debug {
 /// A vector of counters.
 pub trait CounterVecOps: Send + Sync + 'static + Debug {
     /// Get a counter within the vector of counters.
-    fn counter(&self, labels: &[&str]) -> impl CounterOps;
+    fn counter(&self, labels: &[&'static str]) -> impl CounterOps;
 }
 
 /// A vector of gauges.
 pub trait GaugeVecOps: Send + Sync + 'static + Debug {
     /// Get a gauge within the vector of gauges.
-    fn gauge(&self, labels: &[&str]) -> impl GaugeOps;
+    fn gauge(&self, labels: &[&'static str]) -> impl GaugeOps;
 }
 
 /// A vector of histograms.
 pub trait HistogramVecOps: Send + Sync + 'static + Debug {
     /// Get a histogram within the vector of histograms.
-    fn histogram(&self, labels: &[&str]) -> impl HistogramOps;
+    fn histogram(&self, labels: &[&'static str]) -> impl HistogramOps;
 }
 
 /// Metrics registry.
diff --git a/foyer-common/src/metrics/model.rs b/foyer-common/src/metrics/model.rs
index 233f03a6..f8093cef 100644
--- a/foyer-common/src/metrics/model.rs
+++ b/foyer-common/src/metrics/model.rs
@@ -366,6 +366,20 @@ mod tests {
         case(&PrometheusMetricsRegistry::new(prometheus::Registry::new()));
     }
 
+    #[cfg(feature = "prometheus-client_0_22")]
+    #[test]
+    fn test_metrics_prometheus_client_0_22() {
+        use std::sync::Arc;
+
+        use parking_lot::Mutex;
+
+        use crate::metrics::registry::prometheus_client_0_22::PrometheusClientMetricsRegistry;
+
+        case(&PrometheusClientMetricsRegistry::new(Arc::new(Mutex::new(
+            prometheus_client_0_22::registry::Registry::default(),
+        ))));
+    }
+
     #[cfg(feature = "opentelemetry_0_27")]
     #[test]
     fn test_metrics_opentelemetry_0_27() {
diff --git a/foyer-common/src/metrics/registry/mod.rs b/foyer-common/src/metrics/registry/mod.rs
index a88fecc5..4a1194a2 100644
--- a/foyer-common/src/metrics/registry/mod.rs
+++ b/foyer-common/src/metrics/registry/mod.rs
@@ -19,6 +19,14 @@ pub mod noop;
 #[cfg(feature = "prometheus")]
 pub mod prometheus;
 
+/// Prometheus metrics components.
+#[cfg(feature = "prometheus-client")]
+pub use prometheus_client_0_22 as prometheus_client;
+
+/// Prometheus metrics components.
+#[cfg(feature = "prometheus-client_0_22")]
+pub mod prometheus_client_0_22;
+
 #[cfg(feature = "opentelemetry")]
 pub use opentelemetry_0_27 as opentelemetry;
 
diff --git a/foyer-common/src/metrics/registry/noop.rs b/foyer-common/src/metrics/registry/noop.rs
index 16cc4215..73199d9a 100644
--- a/foyer-common/src/metrics/registry/noop.rs
+++ b/foyer-common/src/metrics/registry/noop.rs
@@ -23,7 +23,7 @@ impl CounterOps for NoopMetricsRegistry {
 }
 
 impl CounterVecOps for NoopMetricsRegistry {
-    fn counter(&self, _: &[&str]) -> impl CounterOps {
+    fn counter(&self, _: &[&'static str]) -> impl CounterOps {
         NoopMetricsRegistry
     }
 }
@@ -37,7 +37,7 @@ impl GaugeOps for NoopMetricsRegistry {
 }
 
 impl GaugeVecOps for NoopMetricsRegistry {
-    fn gauge(&self, _: &[&str]) -> impl GaugeOps {
+    fn gauge(&self, _: &[&'static str]) -> impl GaugeOps {
         NoopMetricsRegistry
     }
 }
@@ -47,7 +47,7 @@ impl HistogramOps for NoopMetricsRegistry {
 }
 
 impl HistogramVecOps for NoopMetricsRegistry {
-    fn histogram(&self, _: &[&str]) -> impl HistogramOps {
+    fn histogram(&self, _: &[&'static str]) -> impl HistogramOps {
         NoopMetricsRegistry
     }
 }
diff --git a/foyer-common/src/metrics/registry/opentelemetry_0_26.rs b/foyer-common/src/metrics/registry/opentelemetry_0_26.rs
index e4374c16..26e7ef06 100644
--- a/foyer-common/src/metrics/registry/opentelemetry_0_26.rs
+++ b/foyer-common/src/metrics/registry/opentelemetry_0_26.rs
@@ -84,7 +84,7 @@ pub struct MetricVec {
 }
 
 impl CounterVecOps for MetricVec {
-    fn counter(&self, labels: &[&str]) -> impl CounterOps {
+    fn counter(&self, labels: &[&'static str]) -> impl CounterOps {
         let counter = self.meter.u64_counter(self.name).with_description(self.desc).init();
         let labels = self
             .label_names
@@ -97,7 +97,7 @@ impl CounterVecOps for MetricVec {
 }
 
 impl GaugeVecOps for MetricVec {
-    fn gauge(&self, labels: &[&str]) -> impl GaugeOps {
+    fn gauge(&self, labels: &[&'static str]) -> impl GaugeOps {
         let gauge = self.meter.u64_gauge(self.name).with_description(self.desc).init();
         let labels = self
             .label_names
@@ -111,7 +111,7 @@ impl GaugeVecOps for MetricVec {
 }
 
 impl HistogramVecOps for MetricVec {
-    fn histogram(&self, labels: &[&str]) -> impl HistogramOps {
+    fn histogram(&self, labels: &[&'static str]) -> impl HistogramOps {
         let histogram = self.meter.f64_histogram(self.name).with_description(self.desc).init();
         let labels = self
             .label_names
diff --git a/foyer-common/src/metrics/registry/opentelemetry_0_27.rs b/foyer-common/src/metrics/registry/opentelemetry_0_27.rs
index 99ae98c1..8df31ef4 100644
--- a/foyer-common/src/metrics/registry/opentelemetry_0_27.rs
+++ b/foyer-common/src/metrics/registry/opentelemetry_0_27.rs
@@ -84,7 +84,7 @@ pub struct MetricVec {
 }
 
 impl CounterVecOps for MetricVec {
-    fn counter(&self, labels: &[&str]) -> impl CounterOps {
+    fn counter(&self, labels: &[&'static str]) -> impl CounterOps {
         let counter = self.meter.u64_counter(self.name).with_description(self.desc).build();
         let labels = self
             .label_names
@@ -97,7 +97,7 @@ impl CounterVecOps for MetricVec {
 }
 
 impl GaugeVecOps for MetricVec {
-    fn gauge(&self, labels: &[&str]) -> impl GaugeOps {
+    fn gauge(&self, labels: &[&'static str]) -> impl GaugeOps {
         let gauge = self.meter.u64_gauge(self.name).with_description(self.desc).build();
         let labels = self
             .label_names
@@ -111,7 +111,7 @@ impl GaugeVecOps for MetricVec {
 }
 
 impl HistogramVecOps for MetricVec {
-    fn histogram(&self, labels: &[&str]) -> impl HistogramOps {
+    fn histogram(&self, labels: &[&'static str]) -> impl HistogramOps {
         let histogram = self.meter.f64_histogram(self.name).with_description(self.desc).build();
         let labels = self
             .label_names
diff --git a/foyer-common/src/metrics/registry/prometheus.rs b/foyer-common/src/metrics/registry/prometheus.rs
index fbb3fbe6..3d492264 100644
--- a/foyer-common/src/metrics/registry/prometheus.rs
+++ b/foyer-common/src/metrics/registry/prometheus.rs
@@ -119,7 +119,7 @@ impl CounterOps for IntCounter {
 }
 
 impl CounterVecOps for IntCounterVec {
-    fn counter(&self, labels: &[&str]) -> impl CounterOps {
+    fn counter(&self, labels: &[&'static str]) -> impl CounterOps {
         self.with_label_values(labels)
     }
 }
@@ -139,7 +139,7 @@ impl GaugeOps for IntGauge {
 }
 
 impl GaugeVecOps for IntGaugeVec {
-    fn gauge(&self, labels: &[&str]) -> impl GaugeOps {
+    fn gauge(&self, labels: &[&'static str]) -> impl GaugeOps {
         self.with_label_values(labels)
     }
 }
@@ -151,12 +151,12 @@ impl HistogramOps for Histogram {
 }
 
 impl HistogramVecOps for HistogramVec {
-    fn histogram(&self, labels: &[&str]) -> impl HistogramOps {
+    fn histogram(&self, labels: &[&'static str]) -> impl HistogramOps {
         self.with_label_values(labels)
     }
 }
 
-/// Prometheus metrics registry.
+/// Prometheus metric registry with lib `prometheus`.
 ///
 /// The [`PrometheusMetricsRegistry`] can be cloned and used by multiple foyer instances, without worrying about
 /// duplicately registering.
diff --git a/foyer-common/src/metrics/registry/prometheus_client_0_22.rs b/foyer-common/src/metrics/registry/prometheus_client_0_22.rs
new file mode 100644
index 00000000..bef64f12
--- /dev/null
+++ b/foyer-common/src/metrics/registry/prometheus_client_0_22.rs
@@ -0,0 +1,232 @@
+//  Copyright 2024 foyer Project Authors
+//
+//  Licensed under the Apache License, Version 2.0 (the "License");
+//  you may not use this file except in compliance with the License.
+//  You may obtain a copy of the License at
+//
+//  http://www.apache.org/licenses/LICENSE-2.0
+//
+//  Unless required by applicable law or agreed to in writing, software
+//  distributed under the License is distributed on an "AS IS" BASIS,
+//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//  See the License for the specific language governing permissions and
+//  limitations under the License.
+
+use std::sync::Arc;
+
+use itertools::Itertools;
+use parking_lot::Mutex;
+use prometheus_client_0_22::{
+    encoding::{EncodeLabel, EncodeLabelSet, LabelSetEncoder},
+    metrics::{
+        counter::Counter as PcCounter, family::Family, gauge::Gauge as PcGauge, histogram::Histogram as PcHistogram,
+    },
+    registry::Registry,
+};
+
+use crate::metrics::{CounterOps, CounterVecOps, GaugeOps, GaugeVecOps, HistogramOps, HistogramVecOps, RegistryOps};
+
+#[derive(Debug, Clone, PartialEq, Eq, Hash)]
+struct Labels {
+    pairs: Vec<(&'static str, &'static str)>,
+}
+
+impl EncodeLabelSet for Labels {
+    fn encode(&self, mut encoder: LabelSetEncoder) -> Result<(), std::fmt::Error> {
+        for pair in self.pairs.iter() {
+            pair.encode(encoder.encode_label())?;
+        }
+        Ok(())
+    }
+}
+
+#[derive(Debug)]
+struct Counter {
+    counter: Family<Labels, PcCounter>,
+    labels: Labels,
+}
+
+impl CounterOps for Counter {
+    fn increase(&self, val: u64) {
+        self.counter.get_or_create(&self.labels).inc_by(val);
+    }
+}
+
+#[derive(Debug)]
+struct CounterVec {
+    counter: Family<Labels, PcCounter>,
+    label_names: &'static [&'static str],
+}
+
+impl CounterVecOps for CounterVec {
+    fn counter(&self, labels: &[&'static str]) -> impl CounterOps {
+        Counter {
+            counter: self.counter.clone(),
+            labels: Labels {
+                pairs: self
+                    .label_names
+                    .iter()
+                    .zip_eq(labels.iter())
+                    .map(|(name, label)| (*name, *label))
+                    .collect(),
+            },
+        }
+    }
+}
+
+#[derive(Debug)]
+struct Gauge {
+    gauge: Family<Labels, PcGauge>,
+    labels: Labels,
+}
+
+impl GaugeOps for Gauge {
+    fn increase(&self, val: u64) {
+        self.gauge.get_or_create(&self.labels).inc_by(val as _);
+    }
+
+    fn decrease(&self, val: u64) {
+        self.gauge.get_or_create(&self.labels).dec_by(val as _);
+    }
+
+    fn absolute(&self, val: u64) {
+        self.gauge.get_or_create(&self.labels).set(val as _);
+    }
+}
+
+#[derive(Debug)]
+struct GaugeVec {
+    gauge: Family<Labels, PcGauge>,
+    label_names: &'static [&'static str],
+}
+
+impl GaugeVecOps for GaugeVec {
+    fn gauge(&self, labels: &[&'static str]) -> impl GaugeOps {
+        Gauge {
+            gauge: self.gauge.clone(),
+            labels: Labels {
+                pairs: self
+                    .label_names
+                    .iter()
+                    .zip_eq(labels.iter())
+                    .map(|(name, label)| (*name, *label))
+                    .collect(),
+            },
+        }
+    }
+}
+
+#[derive(Debug)]
+struct Histogram {
+    histogram: Family<Labels, PcHistogram>,
+    labels: Labels,
+}
+
+impl HistogramOps for Histogram {
+    fn record(&self, val: f64) {
+        self.histogram.get_or_create(&self.labels).observe(val);
+    }
+}
+
+#[derive(Debug)]
+struct HistogramVec {
+    histogram: Family<Labels, PcHistogram>,
+    label_names: &'static [&'static str],
+}
+
+impl HistogramVecOps for HistogramVec {
+    fn histogram(&self, labels: &[&'static str]) -> impl HistogramOps {
+        Histogram {
+            histogram: self.histogram.clone(),
+            labels: Labels {
+                pairs: self
+                    .label_names
+                    .iter()
+                    .zip_eq(labels.iter())
+                    .map(|(name, label)| (*name, *label))
+                    .collect(),
+            },
+        }
+    }
+}
+
+#[derive(Debug, Clone)]
+/// Prometheus metric registry with lib `prometheus-client`.
+pub struct PrometheusClientMetricsRegistry {
+    registry: Arc<Mutex<Registry>>,
+}
+
+impl PrometheusClientMetricsRegistry {
+    /// Create an Prometheus metrics registry.
+    pub fn new(registry: Arc<Mutex<Registry>>) -> Self {
+        Self { registry }
+    }
+}
+
+impl RegistryOps for PrometheusClientMetricsRegistry {
+    fn register_counter_vec(
+        &self,
+        name: &'static str,
+        desc: &'static str,
+        label_names: &'static [&'static str],
+    ) -> impl CounterVecOps {
+        let counter = Family::<Labels, PcCounter>::default();
+        self.registry.lock().register(name, desc, counter.clone());
+        CounterVec { counter, label_names }
+    }
+
+    fn register_gauge_vec(
+        &self,
+        name: &'static str,
+        desc: &'static str,
+        label_names: &'static [&'static str],
+    ) -> impl GaugeVecOps {
+        let gauge = Family::<Labels, PcGauge>::default();
+        self.registry.lock().register(name, desc, gauge.clone());
+        GaugeVec { gauge, label_names }
+    }
+
+    fn register_histogram_vec(
+        &self,
+        name: &'static str,
+        desc: &'static str,
+        label_names: &'static [&'static str],
+    ) -> impl HistogramVecOps {
+        let histogram = Family::<Labels, PcHistogram>::new_with_constructor(|| {
+            PcHistogram::new([0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1.0, 2.5, 5.0, 10.0].into_iter())
+        });
+        self.registry.lock().register(name, desc, histogram.clone());
+        HistogramVec { histogram, label_names }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use prometheus_client_0_22::encoding::text::encode;
+
+    use super::*;
+
+    #[test]
+    fn test() {
+        let registry = Arc::new(Mutex::new(Registry::default()));
+        let pc = PrometheusClientMetricsRegistry::new(registry.clone());
+
+        let cv = pc.register_counter_vec("test_counter_1", "test counter 1", &["label1", "label2"]);
+        let c = cv.counter(&["l1", "l2"]);
+        c.increase(42);
+
+        let gv = pc.register_gauge_vec("test_gauge_1", "test gauge 1", &["label1", "label2"]);
+        let g = gv.gauge(&["l1", "l2"]);
+        g.increase(514);
+        g.decrease(114);
+        g.absolute(114514);
+
+        let hv = pc.register_histogram_vec("test_histogram_1", "test histogram 1", &["label1", "label2"]);
+        let h = hv.histogram(&["l1", "l2"]);
+        h.record(114.514);
+
+        let mut text = String::new();
+        encode(&mut text, &registry.lock()).unwrap();
+        println!("{text}");
+    }
+}
diff --git a/foyer/Cargo.toml b/foyer/Cargo.toml
index b61e65c2..f695652c 100644
--- a/foyer/Cargo.toml
+++ b/foyer/Cargo.toml
@@ -46,6 +46,8 @@ tracing = [
     "foyer-storage/tracing",
 ]
 prometheus = ["foyer-common/prometheus"]
+prometheus-client = ["foyer-common/prometheus-client"]
+prometheus-client_0_22 = ["foyer-common/prometheus-client_0_22"]
 opentelemetry = ["foyer-common/opentelemetry"]
 opentelemetry_0_27 = ["foyer-common/opentelemetry_0_27"]
 opentelemetry_0_26 = ["foyer-common/opentelemetry_0_26"]
diff --git a/foyer/src/prelude.rs b/foyer/src/prelude.rs
index c85ab6d2..2873547b 100644
--- a/foyer/src/prelude.rs
+++ b/foyer/src/prelude.rs
@@ -20,6 +20,10 @@ pub use crate::common::metrics::registry::opentelemetry_0_26;
 pub use crate::common::metrics::registry::opentelemetry_0_27;
 #[cfg(feature = "prometheus")]
 pub use crate::common::metrics::registry::prometheus;
+#[cfg(feature = "prometheus-client")]
+pub use crate::common::metrics::registry::prometheus_client;
+#[cfg(feature = "prometheus-client_0_22")]
+pub use crate::common::metrics::registry::prometheus_client_0_22;
 pub use crate::{
     common::{
         buf::{BufExt, BufMutExt},