From 0df1ccc7ccf3fd0aad6d6574a085e050df346bfe Mon Sep 17 00:00:00 2001 From: Ahmed Yasin Koculu Date: Sun, 1 Sep 2024 00:58:21 +0200 Subject: [PATCH 1/2] Improve naming to clarify block cache life time and update the default life time. --- src/ZoneTree/Core/ZoneTreeMaintainer.cs | 6 +-- src/ZoneTree/Directory.Build.props | 4 +- src/ZoneTree/IMaintainer.cs | 63 +++++++++++-------------- 3 files changed, 33 insertions(+), 40 deletions(-) diff --git a/src/ZoneTree/Core/ZoneTreeMaintainer.cs b/src/ZoneTree/Core/ZoneTreeMaintainer.cs index 4f69f27..8984857 100644 --- a/src/ZoneTree/Core/ZoneTreeMaintainer.cs +++ b/src/ZoneTree/Core/ZoneTreeMaintainer.cs @@ -57,10 +57,10 @@ public bool EnableJobForCleaningInactiveCaches } /// - public long DiskSegmentBufferLifeTime { get; set; } = 10_000; + public TimeSpan BlockCacheLifeTime { get; set; } = TimeSpan.FromMinutes(1); /// - public TimeSpan InactiveBlockCacheCleanupInterval { get; set; } = TimeSpan.FromSeconds(5); + public TimeSpan InactiveBlockCacheCleanupInterval { get; set; } = TimeSpan.FromSeconds(30); /// /// Creates a ZoneTreeMaintainer. @@ -261,7 +261,7 @@ async Task StartPeriodicTimer() var zoneTreeMaintenance = ZoneTree.Maintenance; var diskSegment = ZoneTree.Maintenance.DiskSegment; var now = Environment.TickCount64; - var ticks = now - DiskSegmentBufferLifeTime; + var ticks = now - (long)BlockCacheLifeTime.TotalMilliseconds; var releasedCount = zoneTreeMaintenance.ReleaseReadBuffers(ticks); var releasedCacheKeyRecordCount = zoneTreeMaintenance.ReleaseCircularKeyCacheRecords(); var releasedCacheValueRecordCount = zoneTreeMaintenance.ReleaseCircularValueCacheRecords(); diff --git a/src/ZoneTree/Directory.Build.props b/src/ZoneTree/Directory.Build.props index c96b403..68a73ed 100644 --- a/src/ZoneTree/Directory.Build.props +++ b/src/ZoneTree/Directory.Build.props @@ -5,8 +5,8 @@ Ahmed Yasin Koculu ZoneTree ZoneTree - 1.7.9.0 - 1.7.9.0 + 1.8.0.0 + 1.8.0.0 Ahmed Yasin Koculu ZoneTree ZoneTree is a persistent, high-performance, transactional, ACID-compliant ordered key-value database for NET. It can operate in memory or on local/cloud storage. diff --git a/src/ZoneTree/IMaintainer.cs b/src/ZoneTree/IMaintainer.cs index c7c4ba0..25746c7 100644 --- a/src/ZoneTree/IMaintainer.cs +++ b/src/ZoneTree/IMaintainer.cs @@ -1,87 +1,80 @@ namespace Tenray.ZoneTree; /// -/// The maintainer for ZoneTree to control -/// merge operations and memory compaction. +/// Provides functionality to manage merge operations and memory compaction within the ZoneTree. /// /// -/// You must complete or cancel all pending threads of this maintainer -/// before disposing. +/// Ensure that all pending operations are either completed or cancelled before disposing of this maintainer. /// public interface IMaintainer : IDisposable { /// - /// Starts merge operation when records count - /// in read-only segments exceeds this value. - /// Default value is 0. + /// The threshold of record count in read-only segments that triggers the start of a merge operation. + /// The default value is 0. /// int ThresholdForMergeOperationStart { get; set; } /// - /// Starts merge operation when read-only segments - /// count exceeds this value. - /// Default value is 64. + /// The maximum number of read-only segments allowed before triggering a merge operation. + /// The default value is 64. /// int MaximumReadOnlySegmentCount { get; set; } /// - /// Gets or sets a value indicating whether a periodic timer is enabled to release - /// unused block and key/value record caches in the disk segment. - /// Changing this property will start or stop the periodic timer accordingly. + /// Gets or sets a value indicating whether a periodic cleanup job is enabled to release + /// unused block and key/value caches in disk segments. + /// Toggling this property starts or stops the associated periodic timer. /// The default value is false. /// bool EnableJobForCleaningInactiveCaches { get; set; } /// - /// Sets or gets Disk Segment block cache life time in milliseconds. - /// Default value is 10_000 milliseconds. + /// Gets or sets the lifespan of the disk segment block cache. + /// The default value is 1 minute. /// - long DiskSegmentBufferLifeTime { get; set; } + TimeSpan BlockCacheLifeTime { get; set; } /// - /// Gets or sets the interval for the periodic timer that triggers the cleanup job. - /// The default value is 5 seconds. + /// Gets or sets the interval at which the cleanup job runs to remove inactive block caches. + /// The default value is 30 seconds. /// TimeSpan InactiveBlockCacheCleanupInterval { get; set; } /// - /// Tries cancel background threads. + /// Attempts to cancel all background threads associated with the maintainer. /// void TryCancelBackgroundThreads(); /// - /// Blocks the calling thread until all background threads have completed their execution. + /// Blocks the calling thread until all background threads have finished execution. /// void WaitForBackgroundThreads(); /// - /// Asynchronously waits for all background threads to complete. + /// Asynchronously waits for the completion of all background threads. /// - /// A task that represents the asynchronous wait operation. + /// A task representing the asynchronous operation. Task WaitForBackgroundThreadsAsync(); /// - /// Evicts all in-memory data to disk by moving the mutable segment forward and initiating a merge process. + /// Evicts in-memory data to disk by advancing the mutable segment and initiating a merge process. /// /// - /// This method is responsible for freeing up memory in the LSM tree by moving data from the mutable in-memory segment to disk storage. - /// It first advances the current mutable segment to a new state, ensuring that any data currently in memory is prepared for disk storage. - /// Afterward, it starts the merging process, which combines the in-memory data with existing on-disk data to maintain the integrity - /// and efficiency of the LSM tree structure. + /// This operation is crucial for freeing up memory by transferring data from the mutable in-memory segment + /// to disk storage. It advances the current mutable segment, ensuring all in-memory data is properly prepared + /// for disk storage, and then initiates the merging process to maintain the efficiency and integrity of the LSM tree. /// void EvictToDisk(); /// - /// Initiates the merge process in a new thread. + /// Starts the merge process in a new background thread. /// void StartMerge(); /// - /// Initiates a merge of selected bottom segments into a single bottom disk segment. + /// Initiates a merge operation for selected bottom segments, combining them into a single bottom disk segment. /// - /// The lower bound - /// The upper bound - /// - void StartBottomSegmentsMerge( - int fromIndex = 0, int toIndex = int.MaxValue); -} + /// The starting index of the range of segments to merge. + /// The ending index of the range of segments to merge. + void StartBottomSegmentsMerge(int fromIndex = 0, int toIndex = int.MaxValue); +} \ No newline at end of file From 1c4739be922f92118b3069763240d107dd601d23 Mon Sep 17 00:00:00 2001 From: Ahmed Yasin Koculu Date: Sun, 1 Sep 2024 02:09:41 +0200 Subject: [PATCH 2/2] Improve docs. --- .github/CONTRIBUTING.md | 45 +- README.md | 672 ++++++++++++------- src/ZoneTree/docs/ZoneTree/README-NUGET.md | 715 +++++++++++++-------- 3 files changed, 911 insertions(+), 521 deletions(-) diff --git a/.github/CONTRIBUTING.md b/.github/CONTRIBUTING.md index 96a56b8..fa99fc4 100644 --- a/.github/CONTRIBUTING.md +++ b/.github/CONTRIBUTING.md @@ -1,16 +1,29 @@ -# Contributing - -When contributing to this repository, please first discuss the change you wish to make via issue, -email, or any other method with the owners of this repository before making a change. - -Please note we have a code of conduct. Please follow it in all your interactions with the project. - -## Rules -1. [KISS](https://en.wikipedia.org/wiki/KISS_principle) is the primary principle at ZoneTree. Please make your changes without violating the KISS principle. -2. ZoneTree is a low-level database engine. Thus it is good to make micro-optimization for better speed and less memory usage. For example, use *for* loop instead of a *foreach* iteration. -3. Add unit tests that cover the functionality of your changes including edge cases. -4. By contributing ZoneTree, you agree that your changes are included in ZoneTree with the [MIT license](https://github.com/koculu/ZoneTree/blob/main/LICENSE). This license agreement cannot be undone. -5. Make sure you are not violating some other 3rd party licenses with your contribution. For example, don't copy source code from other open source or commercial software without adding the necessary LICENSE copyright notice! -6. ZoneTree does not reference any 3rd party library except compression libraries. If you are going to add a dependency to a 3rd party library, there should be a good reason for it. Discuss with the maintainers if you need to add a 3rd party reference. -7. When you do a performance optimization make sure that you measure the difference with real benchmark code included in your pull request. -8. Rules are subject to change. Please add your complaints and suggestions to the discussion page. +# Contributing to ZoneTree + +Thank you for considering contributing to ZoneTree! To ensure a smooth collaboration, please take a moment to review the guidelines below. + +## Before You Start + +Before making any changes, please discuss the proposed modifications via an issue, email, or any other communication method with the repository owners. This helps ensure that your efforts align with the project's goals and avoids unnecessary work. + +Please also familiarize yourself with our [Code of Conduct](CODE_OF_CONDUCT.md) and adhere to it in all interactions related to the project. + +## Contribution Guidelines + +1. **KISS Principle**: The [KISS](https://en.wikipedia.org/wiki/KISS_principle) (Keep It Simple, Stupid) principle is fundamental at ZoneTree. Ensure that your contributions are straightforward and do not add unnecessary complexity. + +2. **Micro-Optimizations**: ZoneTree is a low-level database engine, and performance is critical. Micro-optimizations for speed and memory usage are encouraged. For example, prefer using a `for` loop over a `foreach` loop where performance gains can be realized. + +3. **Unit Testing**: Every contribution should be accompanied by unit tests that thoroughly cover the new or modified functionality, including edge cases. This ensures that ZoneTree remains reliable and robust. + +4. **License Agreement**: By contributing to ZoneTree, you agree that your changes will be included under the [MIT license](LICENSE). This agreement is permanent and cannot be revoked. + +5. **Respect Third-Party Licenses**: Ensure that your contributions do not violate any third-party licenses. Do not copy code from other open-source or commercial software without including the necessary LICENSE and copyright notices. + +6. **Minimizing Dependencies**: ZoneTree avoids dependencies on third-party libraries except for compression libraries. If your contribution requires adding a new dependency, it must be justified. Please discuss with the maintainers before proceeding. + +7. **Performance Benchmarking**: When submitting performance optimizations, include benchmark code in your pull request to demonstrate the impact of your changes. This allows us to verify and appreciate the improvements. + +8. **Open to Feedback**: The contribution rules are subject to change based on community feedback and project needs. If you have suggestions or concerns, please share them in the discussion page or via an issue. + +Thank you for helping to make ZoneTree better! diff --git a/README.md b/README.md index 1f2c521..7c735ed 100644 --- a/README.md +++ b/README.md @@ -1,367 +1,541 @@ -![img](https://raw.githubusercontent.com/koculu/ZoneTree/main/src/ZoneTree/docs/ZoneTree/images/logo2.png) +![ZoneTree Logo](https://raw.githubusercontent.com/koculu/ZoneTree/main/src/ZoneTree/docs/ZoneTree/images/logo2.png) # ZoneTree -ZoneTree is a persistent, high-performance, transactional, and ACID-compliant [ordered key-value database](https://en.wikipedia.org/wiki/Ordered_Key-Value_Store) for .NET. -It can operate in memory or on local/cloud storage. +ZoneTree is a **persistent**, **high-performance**, **transactional**, and **ACID-compliant** [ordered key-value database](https://en.wikipedia.org/wiki/Ordered_Key-Value_Store) for .NET. It operates seamlessly both **in-memory** and on **local/cloud storage**, making it an ideal choice for a wide range of applications requiring efficient data management. ![License](https://img.shields.io/badge/license-MIT-blue.svg) [![Downloads](https://img.shields.io/nuget/dt/ZoneTree)](https://www.nuget.org/packages/ZoneTree/) ![Platform](https://img.shields.io/badge/platform-.NET-blue.svg) ![Build](https://img.shields.io/badge/build-passing-brightgreen.svg) -[![](https://dcbadge.vercel.app/api/server/d9aDtzVNNv?logoColor=f1c400&theme=discord&style=flat)](https://discord.gg/d9aDtzVNNv) +[![Join Discord](https://dcbadge.vercel.app/api/server/d9aDtzVNNv?logoColor=f1c400&theme=discord&style=flat)](https://discord.gg/d9aDtzVNNv) -ZoneTree is a lightweight, transactional and high-performance LSM Tree for .NET. +## Table of Contents -It is several times faster than Facebook's RocksDB and hundreds of times faster than SQLite. It is faster than any other alternative that I have tested so far. -100 Million integer key-value pair inserts in 20 seconds. You may get longer durations based on the durability level. -For example, with async-compressed WAL mode, you can insert 100M integer key-value pairs in 28 seconds. Background merge operation that might take a bit longer is excluded from the insert duration because your inserted data is immediately queryable. -Loading 100M integer key-value pair database is in 812 ms. The iteration on 100M key-value pairs takes 24 seconds. -There are so many tuning options wait you to discover. +- [Why Choose ZoneTree?](#why-choose-zonetree) +- [Performance](#performance) + - [Benchmark Results](#benchmark-results) + - [Write-Ahead Log (WAL) Modes](#write-ahead-log-wal-modes) + - [Benchmark Environment](#benchmark-environment) +- [Getting Started](#getting-started) + - [Basic Usage](#basic-usage) + - [Creating a Database](#creating-a-database) +- [Maintaining the LSM Tree](#maintaining-the-lsm-tree) +- [Handling Deletions](#handling-deletions) + - [Using an Integer Deletion Flag](#using-an-integer-deletion-flag) + - [Using a Custom Struct for Deletion](#using-a-custom-struct-for-deletion) +- [Data Iteration](#data-iteration) + - [Forward and Backward Iteration](#forward-and-backward-iteration) + - [Seekable Iterator](#seekable-iterator) +- [Transaction Support](#transaction-support) + - [Fluent Transactions](#fluent-transactions) + - [Classical Transactions](#classical-transactions) + - [Exceptionless Transactions](#exceptionless-transactions) +- [Feature Highlights](#feature-highlights) +- [ZoneTree.FullTextSearch](#zonetreefulltextsearch) +- [Documentation](#documentation) +- [Contributing](#contributing) +- [License](#license) -## [INTRODUCTION](https://tenray.io/docs/ZoneTree/guide/introduction.html) +--- -## [QUICK START GUIDE](https://tenray.io/docs/ZoneTree/guide/quick-start.html) +## Why Choose ZoneTree? -## [API DOCS](https://tenray.io/docs/ZoneTree/api/Tenray.ZoneTree.html) +1. **Pure C# Implementation**: ZoneTree is developed entirely in C#, ensuring seamless integration and deployment within .NET ecosystems without external dependencies. -## [TUNING ZONETREE](https://tenray.io/docs/ZoneTree/guide/tuning-disk-segment.html) +2. **Exceptional Performance**: Demonstrates performance several times faster than Facebook's RocksDB and hundreds of times faster than SQLite. Optimized for both speed and efficiency. -## [FEATURES](https://tenray.io/docs/ZoneTree/guide/features.html) +3. **Data Durability and Crash Resilience**: Provides optional durability features that protect data against crashes and power outages, ensuring data integrity under all circumstances. -## [TERMINOLOGY](https://tenray.io/docs/ZoneTree/guide/terminology.html) +4. **Transactional and ACID-Compliant**: Supports both transactional and non-transactional access with full ACID guarantees, offering flexibility for various application requirements. -## [PERFORMANCE](https://tenray.io/docs/ZoneTree/guide/performance.html) +5. **Embeddable Database Engine**: Easily embed ZoneTree into applications, eliminating the overhead of maintaining or shipping separate database products. -## Why ZoneTree? +6. **Scalability**: Capable of handling small to massive datasets, allowing the creation of both scalable and non-scalable databases tailored to specific needs. -1. It is pure C#. -2. It is fast. See benchmark below. -3. Your data is protected against crashes / power cuts (optional). -4. Supports transactional and non-transactional access with blazing speeds and ACID guarantees. -5. You can embed your database into your assembly. Therefore, you don't have to pay the cost of maintaining/shipping another database product along with yours. -6. You can create scalable and non-scalable databases using ZoneTree as core database engine. +--- -## How fast is it? +## Performance -It is possible with ZoneTree to insert 100 Million integer key-value pairs in 20 seconds using WAL mode = NONE. +ZoneTree sets new standards in database performance, showcasing remarkable speeds in data insertion, loading, and iteration operations. -Benchmark for all modes: [benchmark](https://raw.githubusercontent.com/koculu/ZoneTree/main/src/Playground/BenchmarkForAllModes.txt) +### Benchmark Results -| Insert Benchmarks | 1M | 2M | 3M | 10M | -| ------------------------------------- | ------------- | -------- | -------- | -------- | -| int-int ZoneTree async-compressed WAL | 267 ms | 464 ms | 716 ms | 2693 ms | -| int-int ZoneTree sync-compressed WAL | 834 ms | 1617 ms | 2546 ms | 8642 ms | -| int-int ZoneTree sync WAL | 2742 ms | 5533 ms | 8242 ms | 27497 ms | -| | -| str-str ZoneTree async-compressed WAL | 892 ms | 1833 ms | 2711 ms | 9443 ms | -| str-str ZoneTree sync-compressed WAL | 1752 ms | 3397 ms | 5070 ms | 19153 ms | -| str-str ZoneTree sync WAL | 3488 ms | 7002 ms | 10483 ms | 38727 ms | -| | -| RocksDb sync WAL (10K => 11 sec) | ~1.100.000 ms | N/A | N/A | N/A | -| int-int RocksDb sync-compressed WAL | 8059 ms | 16188 ms | 23599 ms | 61947 ms | -| str-str RocksDb sync-compressed WAL | 8215 ms | 16146 ms | 23760 ms | 72491 ms | -| | +- **100 Million integer key-value pairs** inserted in **20 seconds** using **WAL mode = NONE**. +- **Loading** database with 100 million integer key-value pairs takes **812 milliseconds**. +- **Iterating** over 100 million key-value pairs completes in **24 seconds**. -Benchmark Configuration: +**Detailed Benchmark for Various Modes:** -```c# -DiskCompressionBlockSize = 1024 * 1024 * 10; -WALCompressionBlockSize = 1024 * 32 * 8; +| Insert Benchmarks | 1M | 2M | 3M | 10M | +| ----------------------------------------- | ------------- | -------- | -------- | -------- | +| **int-int ZoneTree async-compressed WAL** | 267 ms | 464 ms | 716 ms | 2693 ms | +| **int-int ZoneTree sync-compressed WAL** | 834 ms | 1617 ms | 2546 ms | 8642 ms | +| **int-int ZoneTree sync WAL** | 2742 ms | 5533 ms | 8242 ms | 27497 ms | +| | | | | | +| **str-str ZoneTree async-compressed WAL** | 892 ms | 1833 ms | 2711 ms | 9443 ms | +| **str-str ZoneTree sync-compressed WAL** | 1752 ms | 3397 ms | 5070 ms | 19153 ms | +| **str-str ZoneTree sync WAL** | 3488 ms | 7002 ms | 10483 ms | 38727 ms | +| | | | | | +| **RocksDb sync WAL (10K => 11 sec)** | ~1,100,000 ms | N/A | N/A | N/A | +| **int-int RocksDb sync-compressed WAL** | 8059 ms | 16188 ms | 23599 ms | 61947 ms | +| **str-str RocksDb sync-compressed WAL** | 8215 ms | 16146 ms | 23760 ms | 72491 ms | + +**[Full Benchmark Results](https://raw.githubusercontent.com/koculu/ZoneTree/main/src/Playground/BenchmarkForAllModes.txt)** + +**Benchmark Configuration:** + +```csharp +DiskCompressionBlockSize = 10 * 1024 * 1024; // 10 MB +WALCompressionBlockSize = 32 * 1024 * 8; // 256 KB DiskSegmentMode = DiskSegmentMode.SingleDiskSegment; MutableSegmentMaxItemCount = 1_000_000; ThresholdForMergeOperationStart = 2_000_000; ``` -Additional Notes: -According to our tests, ZoneTree is stable and fast even with big data. -Tested up to 1 billion records in desktop computers till now. +**Additional Notes:** + +- ZoneTree has been tested successfully with up to **1 billion records** on standard desktop computers, demonstrating stability and efficiency even with very large datasets. + +### Write-Ahead Log (WAL) Modes + +ZoneTree offers **four WAL modes** to provide flexibility between performance and durability: -### ZoneTree offers 4 WAL modes to let you make a flexible tradeoff. +1. **Sync Mode**: -- The sync mode provides maximum durability but slower write speed. - In case of a crash/power cut, the sync mode ensures that the inserted data is not lost. + - **Durability**: Maximum. + - **Performance**: Slower write speed. + - **Use Case**: Ensures data is not lost in case of crashes or power cuts. -- The sync-compressed mode provides faster write speed but less durability. - Compression requires chunks to be filled before appending them into the WAL file. - It is possible to enable a periodic job to persist decompressed tail records into a separate location in a specified interval. - See IWriteAheadLogProvider options for more details. +2. **Sync-Compressed Mode**: -- The async-compressed mode provides faster write speed but less durability. - Log entries are queued to be written in a separate thread. - async-compressed mode uses compression in WAL files and provides immediate tail record persistence. + - **Durability**: High, but slightly less than sync mode. + - **Performance**: Faster write speed due to compression. + - **Use Case**: Balances durability and performance; periodic jobs can persist decompressed tail records for added safety. -- None WAL mode disables WAL completely to get maximum performance. Data still can be saved to disk by tree maintainer automatically or manually. +3. **Async-Compressed Mode**: -### Environment: + - **Durability**: Moderate. + - **Performance**: Very fast write speed; logs are written in a separate thread. + - **Use Case**: Suitable where immediate durability is less critical but performance is paramount. + +4. **None Mode**: + - **Durability**: No immediate durability; relies on manual or automatic disk saves. + - **Performance**: Maximum possible. + - **Use Case**: Ideal for scenarios where performance is critical and data can be reconstructed or is not mission-critical. + +### Benchmark Environment ``` BenchmarkDotNet=v0.13.1, OS=Windows 10.0.22000 -Intel Core i7-6850K CPU 3.60GHz (Skylake), 1 CPU, 12 logical and 6 physical cores -64 GB DDR4 Memory -SSD: Samsung SSD 850 EVO 1TB -Config: 1M mutable segment size, 2M readonly segments merge-threshold +Processor: Intel Core i7-6850K CPU @ 3.60GHz (Skylake), 12 logical cores, 6 physical cores +Memory: 64 GB DDR4 +Storage: Samsung SSD 850 EVO 1TB +Configuration: 1M mutable segment size, 2M readonly segments merge-threshold ``` -## How to use ZoneTree? +--- + +## Getting Started + +ZoneTree is designed for ease of use, allowing developers to integrate and utilize its capabilities with minimal setup. + +### Basic Usage -The following sample shows the most basic setup of a ZoneTree database. +```csharp +using Tenray.ZoneTree; -```C# using var zoneTree = new ZoneTreeFactory() - .OpenOrCreate(); -zoneTree.Upsert(39, "Hello Zone Tree"); + .OpenOrCreate(); + +zoneTree.Upsert(39, "Hello ZoneTree"); ``` -The following sample demonstrates creating a database. +### Creating a Database + +```csharp +using Tenray.ZoneTree; +using Tenray.ZoneTree.Comparers; +using Tenray.ZoneTree.Serializers; -```c# - var dataPath = "data/mydatabase"; - using var zoneTree = new ZoneTreeFactory() +var dataPath = "data/mydatabase"; + +using var zoneTree = new ZoneTreeFactory() .SetComparer(new Int32ComparerAscending()) .SetDataDirectory(dataPath) .SetKeySerializer(new Int32Serializer()) .SetValueSerializer(new Utf8StringSerializer()) .OpenOrCreate(); - // atomic (thread-safe) on single mutable-segment. - zoneTree.Upsert(39, "Hello Zone Tree!"); +// Atomic (thread-safe) operation on single mutable-segment. +zoneTree.Upsert(39, "Hello ZoneTree!"); - // atomic across all segments - zoneTree.TryAtomicAddOrUpdate(39, "a", - bool (ref string x) => - { - x += "b"; - return true; - }); +// Atomic operation across all segments. +zoneTree.TryAtomicAddOrUpdate(39, "a", (ref string x) => +{ + x += "b"; + return true; +}); ``` -## How to maintain LSM Tree? +--- + +## Maintaining the LSM Tree + +Large-scale LSM Trees require periodic maintenance to ensure optimal performance and resource utilization. ZoneTree provides the `IZoneTreeMaintenance` interface to facilitate comprehensive maintenance tasks. + +**Example Usage:** -Big LSM Trees require maintenance tasks. ZoneTree provides the IZoneTreeMaintenance interface to give you full power on maintenance tasks. -It also comes with a default maintainer to let you focus on your business logic without wasting time with LSM details. -You can start using the default maintainer like in the following sample code. -Note: For small data you don't need a maintainer. +```csharp +using Tenray.ZoneTree; +using Tenray.ZoneTree.Maintenance; -```c# - var dataPath = "data/mydatabase"; +var dataPath = "data/mydatabase"; - // 1. Create your ZoneTree - using var zoneTree = new ZoneTreeFactory() +using var zoneTree = new ZoneTreeFactory() .SetComparer(new Int32ComparerAscending()) .SetDataDirectory(dataPath) .SetKeySerializer(new Int32Serializer()) .SetValueSerializer(new Utf8StringSerializer()) .OpenOrCreate(); - using var maintainer = zoneTree.CreateMaintainer(); - maintainer.EnableJobForCleaningInactiveCaches = true; +using var maintainer = zoneTree.CreateMaintainer(); +maintainer.EnableJobForCleaningInactiveCaches = true; - // 2. Read/Write data - zoneTree.Upsert(39, "Hello ZoneTree!"); +// Perform read/write operations. +zoneTree.Upsert(39, "Hello ZoneTree!"); - // 3. Wait for background threads. - maintainer.WaitForBackgroundThreads(); +// Wait for background maintenance tasks to complete. +maintainer.WaitForBackgroundThreads(); ``` -## How to delete keys? - -In Log-Structured Merge (LSM) trees, deletions are managed by upserting a key/value pair with a deletion marker. The actual removal of the data occurs during the compaction stage. In ZoneTree, by default, the system assumes that the default values indicate deletion. However, you can customize this behavior by defining a specific deletion flag, such as using -1 for integer values or completely disable deletion by calling DisableDeletion method. - -### Custom Deletion Flag +**Note:** For smaller datasets, maintenance tasks may not be necessary. The default maintainer allows developers to focus on core business logic without delving into LSM tree intricacies. -If you need more control over how deletions are handled, you can define a custom structure to represent your values and their deletion status. For example: +--- -```c# -[StructLayout(LayoutKind.Sequential)] -struct MyDeletableValueType { - int Number; - bool IsDeleted; -} -``` +## Handling Deletions -This struct allows you to include a boolean flag indicating whether a value is deleted. You can then use this custom type as the value type in your ZoneTree. +In **Log-Structured Merge (LSM) trees**, deletions are managed by upserting a key/value pair with a **deletion marker**. Actual data removal occurs during the **compaction** stage. -### Configuring Deletion Markers +By default, ZoneTree assumes that **default values** indicate deletion. This behavior can be customized by defining specific deletion flags or disabling deletions entirely using the `DisableDeletion` method. -ZoneTree provides flexibility in managing the tree size by allowing you to configure how deletion markers are set and identified. Below are examples of how you can configure these markers for your database: +### Using an Integer Deletion Flag -#### Example 1: Using an Integer Deletion Flag +In this example, `-1` is used as the deletion marker for integer values: -In this example, -1 is used as the deletion marker for integer values: +```csharp +using Tenray.ZoneTree; -```c# using var zoneTree = new ZoneTreeFactory() - // Additional stuff goes here - .SetIsValueDeletedDelegate((in int x) => x == -1) - .SetMarkValueDeletedDelegate((ref int x) => x = -1) - .OpenOrCreate(); + .SetIsValueDeletedDelegate((in int x) => x == -1) + .SetMarkValueDeletedDelegate((ref int x) => x = -1) + .OpenOrCreate(); + +// Deleting a key by setting its value to -1 +zoneTree.Upsert(42, -1); ``` -#### Example 2: Using a Custom Struct for Deletion +### Using a Custom Struct for Deletion + +For more control, define a custom structure to represent values and their deletion status: + +```csharp +using System.Runtime.InteropServices; +using Tenray.ZoneTree; -Alternatively, if you're using a custom struct to manage deletions, you can configure ZoneTree to recognize and mark deletions as follows: +[StructLayout(LayoutKind.Sequential)] +struct MyDeletableValueType +{ + public int Number; + public bool IsDeleted; +} -```c# using var zoneTree = new ZoneTreeFactory() - // Additional stuff goes here - .SetIsValueDeletedDelegate((in MyDeletableValueType x) => x.IsDeleted) - .SetMarkValueDeletedDelegate((ref MyDeletableValueType x) => x.IsDeleted = true) - .OpenOrCreate(); + .SetIsValueDeletedDelegate((in MyDeletableValueType x) => x.IsDeleted) + .SetMarkValueDeletedDelegate((ref MyDeletableValueType x) => x.IsDeleted = true) + .OpenOrCreate(); + +// Deleting a key by setting the IsDeleted flag +zoneTree.Upsert(42, new MyDeletableValueType { Number = 0, IsDeleted = true }); ``` -## How to iterate over data? +--- + +## Data Iteration -Iteration is possible in both directions, forward and backward. -Unlike other LSM tree implementations, iteration performance is equal in both directions. -The following sample shows how to do the iteration. +ZoneTree provides efficient mechanisms to iterate over data both **forward** and **backward**, with equal performance in both directions. Iterators also support **seek** operations for quick access to specific keys. -```c# - using var zoneTree = new ZoneTreeFactory() - // Additional stuff goes here +### Forward and Backward Iteration + +```csharp +using Tenray.ZoneTree; +using Tenray.ZoneTree.Collections; + +using var zoneTree = new ZoneTreeFactory() .OpenOrCreate(); - using var iterator = zoneTree.CreateIterator(); - while(iterator.Next()) { + +// Forward iteration +using var iterator = zoneTree.CreateIterator(); +while (iterator.Next()) +{ var key = iterator.CurrentKey; var value = iterator.CurrentValue; - } + // Process key and value +} - using var reverseIterator = zoneTree.CreateReverseIterator(); - while(reverseIterator.Next()) { +// Backward iteration +using var reverseIterator = zoneTree.CreateReverseIterator(); +while (reverseIterator.Next()) +{ var key = reverseIterator.CurrentKey; var value = reverseIterator.CurrentValue; - } + // Process key and value +} ``` -## How to iterate starting with a key (Seekable Iterator)? +### Seekable Iterator + +The `ZoneTreeIterator` supports the `Seek()` method to jump to any record with **O(log(n))** complexity, useful for prefix searches and range queries. -ZoneTreeIterator provides Seek() method to jump into any record with in O(log(n)) complexity. -That is useful for doing prefix search with forward-iterator or with backward-iterator. +```csharp +using Tenray.ZoneTree; +using Tenray.ZoneTree.Collections; -```c# - using var zoneTree = new ZoneTreeFactory() - // Additional stuff goes here +using var zoneTree = new ZoneTreeFactory() .OpenOrCreate(); - using var iterator = zoneTree.CreateIterator(); - // iterator jumps into the first record starting with "SomePrefix" in O(log(n)) complexity. - iterator.Seek("SomePrefix"); - //iterator.Next() complexity is O(1) - while(iterator.Next()) { - var key = iterator.CurrentKey; - var value = iterator.CurrentValue; - } +using var iterator = zoneTree.CreateIterator(); + +// Jump to the first record starting with "SomePrefix" +if (iterator.Seek("SomePrefix")) +{ + do + { + var key = iterator.CurrentKey; + var value = iterator.CurrentValue; + // Process key and value + } + while (iterator.Next()); +} ``` +--- + ## Transaction Support -ZoneTree supports Optimistic Transactions. It is proud to announce that the ZoneTree is ACID-compliant. Of course, you can use non-transactional API for the scenarios where eventual consistency is sufficient. +ZoneTree supports **Optimistic Transactions**, ensuring **ACID compliance** while offering flexibility through various transaction models: -ZoneTree supports 3 way of doing transactions. +1. **Fluent Transactions**: Provides an intuitive, chainable API with built-in retry capabilities. +2. **Classical Transactions**: Traditional approach with explicit control over transaction lifecycle. +3. **Exceptionless Transactions**: Allows transaction management without relying on exceptions for control flow. -1. Fluent Transactions with ready to use retry capability. -2. Classical Transaction API. -3. Exceptionless Transaction API. +### Fluent Transactions -The following sample shows how to do the transactions with ZoneTree Fluent Transaction API. +```csharp +using System.Threading.Tasks; +using Tenray.ZoneTree; +using Tenray.ZoneTree.Transaction; -```c# using var zoneTree = new ZoneTreeFactory() - // Additional stuff goes here .OpenOrCreateTransactional(); -using var transaction = - zoneTree - .BeginFluentTransaction() - .Do((tx) => zoneTree.UpsertNoThrow(tx, 3, 9)) - .Do((tx) => - { - if (zoneTree.TryGetNoThrow(tx, 3, out var value).IsAborted) - return TransactionResult.Aborted(); - if (zoneTree.UpsertNoThrow(tx, 3, 21).IsAborted) - return TransactionResult.Aborted(); - return TransactionResult.Success(); - }) - .SetRetryCountForPendingTransactions(100) - .SetRetryCountForAbortedTransactions(10); - await transaction.CommitAsync(); + +using var transaction = zoneTree + .BeginFluentTransaction() + .Do(tx => zoneTree.UpsertNoThrow(tx, 3, 9)) + .Do(tx => + { + if (zoneTree.TryGetNoThrow(tx, 3, out var value).IsAborted) + return TransactionResult.Aborted(); + + if (zoneTree.UpsertNoThrow(tx, 3, 21).IsAborted) + return TransactionResult.Aborted(); + + return TransactionResult.Success(); + }) + .SetRetryCountForPendingTransactions(100) + .SetRetryCountForAbortedTransactions(10); + +await transaction.CommitAsync(); ``` -The following sample shows traditional way of doing transactions with ZoneTree. +### Classical Transactions -```c# - using var zoneTree = new ZoneTreeFactory() - // Additional stuff goes here +```csharp +using System; +using System.Threading; +using Tenray.ZoneTree; +using Tenray.ZoneTree.Transaction; + +using var zoneTree = new ZoneTreeFactory() .OpenOrCreateTransactional(); - try - { - var txId = zoneTree.BeginTransaction(); - zoneTree.TryGet(txId, 3, out var value); - zoneTree.Upsert(txId, 3, 9); - var result = zoneTree.Prepare(txId); - while (result.IsPendingTransactions) { - Thread.Sleep(100); - result = zoneTree.Prepare(txId); - } - zoneTree.Commit(txId); - } - catch(TransactionAbortedException e) - { - //retry or cancel - } + +try +{ + var txId = zoneTree.BeginTransaction(); + + zoneTree.TryGet(txId, 3, out var value); + zoneTree.Upsert(txId, 3, 9); + + var result = zoneTree.Prepare(txId); + while (result.IsPendingTransactions) + { + Thread.Sleep(100); + result = zoneTree.Prepare(txId); + } + + zoneTree.Commit(txId); +} +catch (TransactionAbortedException) +{ + // Handle aborted transaction (retry or cancel) +} ``` -## Features - -| ZoneTree Features | -| --------------------------------------------------------------------------------------------- | -| Works with .NET primitives, structs and classes. | -| High Speed and Low Memory consumption. | -| Crash Resilience | -| Optimum disk space utilization. | -| WAL and DiskSegment data compression. | -| Very fast load/unload. | -| Standard read/upsert/delete functions. | -| Optimistic Transaction Support | -| Atomic Read Modify Update | -| Can work in memory. | -| Can work with any disk device including cloud devices. | -| Supports optimistic transactions. | -| Supports Atomicity, Consistency, Isolation, Durability. | -| Supports Read Committed Isolation. | -| 4 different modes for write ahead log. | -| Audit support with incremental transaction log backup. | -| Live backup. | -| Configurable amount of data that can stay in memory. | -| Partially (with sparse arrays) or completely load/unload data on disk to/from memory. | -| Forward/Backward iteration. | -| Allow optional dirty reads. | -| Embeddable. | -| Optimized for SSDs. | -| Exceptionless Transaction API. | -| Fluent Transaction API with ready to use retry capabilities. | -| Easy Maintenance. | -| Configurable LSM merger. | -| Transparent and simple implementation that reveals your database's internals. | -| Fully open-source with unrestrictive MIT license. | -| Transaction Log compaction. | -| Analyze / control transactions. | -| Concurrency Control with minimum overhead by novel separation of Concurrency Stamps and Data. | -| TTL support. | -| Use your custom serializer for keys and values. | -| Use your custom comparer. | -| MultipleDiskSegments Mode to enable dividing data files into configurable sized chunks. | -| Snapshot iterators. | - -## I want to contribute. What can I do? - -I appreciate any contribution to the project. -These are the things I do think we need at the moment: - -1. Write tests / benchmarks. -2. Write documentation. -3. Feature requests & bug fixes. -4. Performance improvements. +### Exceptionless Transactions + +```csharp +using System; +using System.Threading.Tasks; +using Tenray.ZoneTree.Transactional; + +public async Task ExecuteTransactionWithRetryAsync(ZoneTreeFactory zoneTreeFactory, int maxRetries = 3) +{ + using var zoneTree = zoneTreeFactory.OpenOrCreateTransactional(); + var transactionId = zoneTree.BeginTransaction(); + int retryCount = 0; + + while (retryCount <= maxRetries) + { + var result = zoneTree.UpsertNoThrow(transactionId, 1, 100); + if (result.IsAborted) + { + // Abort the transaction and return false on failure. + return false; + } + + result = zoneTree.UpsertNoThrow(transactionId, 2, 200); + if (result.IsAborted) + { + // Abort the transaction and return false on failure. + return false; + } + + var prepareResult = zoneTree.PrepareNoThrow(transactionId); + if (prepareResult.IsAborted) + { + // Abort the transaction and return false on failure. + return false; + } + + if (prepareResult.IsPendingTransactions) + { + // Optionally wait or handle pending transactions + await Task.Delay(100); // Simple delay before retrying + retryCount++; + continue; // Retry the transaction + } + + if (prepareResult.IsReadyToCommit) + { + var commitResult = zoneTree.CommitNoThrow(transactionId); + if (commitResult.IsAborted) + { + // Abort the transaction and return false on failure. + return false; + } + // Transaction committed successfully + return true; + } + } + + // Return false if retries are exhausted without a successful commit. + return false; +} +``` + +--- + +## Feature Highlights + +| Feature | Description | +| ----------------------------------------------- | ------------------------------------------------------------------------------------ | +| **.NET Compatibility** | Works seamlessly with .NET primitives, structs, and classes. | +| **High Performance and Low Memory Consumption** | Optimized algorithms ensure fast operations with minimal resource usage. | +| **Crash Resilience** | Robust mechanisms protect data integrity against unexpected failures. | +| **Efficient Disk Space Utilization** | Optimized storage strategies minimize disk space requirements. | +| **Data Compression** | Supports WAL and DiskSegment data compression for efficient storage. | +| **Fast Load/Unload** | Quickly load and unload large datasets as needed. | +| **Standard CRUD Operations** | Provides intuitive and straightforward Create, Read, Update, Delete functionalities. | +| **Optimistic Transactions** | Supports concurrent operations with minimal locking overhead. | +| **Atomic Read-Modify-Update** | Ensures data consistency during complex update operations. | +| **In-Memory and Disk Storage** | Flexibly operate entirely in memory or persist data to various storage backends. | +| **Cloud Storage Support** | Compatible with cloud-based storage solutions for scalable deployments. | +| **ACID Compliance** | Guarantees Atomicity, Consistency, Isolation, and Durability across transactions. | +| **Multiple WAL Modes** | Choose from four different WAL modes to balance performance and durability. | +| **Configurable Memory Usage** | Adjust the amount of data retained in memory based on application needs. | +| **Partial and Complete Data Loading** | Load data partially (with sparse arrays) or completely to and from disk. | +| **Bidirectional Iteration** | Efficiently iterate over data both forward and backward. | +| **Optional Dirty Reads** | Allow for faster reads when strict consistency is not required. | +| **Embeddable Design** | Integrate ZoneTree directly into applications without external dependencies. | +| **SSD Optimization** | Tailored for optimal performance on solid-state drives. | +| **Exceptionless Transaction API** | Manage transactions smoothly without relying on exceptions for control flow. | +| **Fluent Transaction API** | Utilize an intuitive, chainable transaction interface with retry capabilities. | +| **Easy Maintenance** | Simplified maintenance processes ensure consistent performance. | +| **Configurable LSM Merger** | Customize merge operations to suit specific workload patterns. | +| **Transparent Implementation** | Clear and straightforward codebase reveals internal workings for easy understanding. | +| **Open-Source with MIT License** | Freely use, modify, and distribute under a permissive license. | +| **Transaction Log Compaction** | Efficiently manage and reduce the size of transaction logs. | +| **Transaction Analysis and Control** | Analyze and manage transactions for improved performance and reliability. | +| **Efficient Concurrency Control** | Minimal overhead through innovative separation of concurrency stamps and data. | +| **Time-To-Live (TTL) Support** | Automatically expire data after a specified duration. | +| **Custom Serializer and Comparer Support** | Implement custom logic for data serialization and comparison. | +| **Multiple Disk Segments Mode** | Divide data files into configurable chunks for better manageability and performance. | +| **Snapshot Iterators** | Create consistent snapshots for data analysis and backup purposes. | + +--- + +## ZoneTree.FullTextSearch + +[ZoneTree.FullTextSearch](https://www.nuget.org/packages/ZoneTree.FullTextSearch/) is an extension library built upon ZoneTree, providing a **high-performance** and **flexible full-text search engine** for .NET applications. + +**Key Features Include:** + +- **Fast and Efficient Indexing**: Handles large volumes of text data with impressive speed. +- **Advanced Query Support**: Enables complex search queries with support for Boolean operators and faceted search. +- **Customizable Components**: Allows integration of custom tokenizers, stemmers, and normalizers to suit specific application needs. +- **Scalable Architecture**: Designed to scale seamlessly with growing data and usage demands. + +For more information and detailed documentation, visit the [ZoneTree.FullTextSearch GitHub Repository](https://github.com/koculu/ZoneTree.FullTextSearch). + +--- + +## Documentation + +Explore comprehensive guides and API references to get the most out of ZoneTree: + +- **[Introduction](https://tenray.io/docs/ZoneTree/guide/introduction.html)** +- **[Quick Start Guide](https://tenray.io/docs/ZoneTree/guide/quick-start.html)** +- **[API Documentation](https://tenray.io/docs/ZoneTree/api/Tenray.ZoneTree.html)** +- **[Tuning ZoneTree](https://tenray.io/docs/ZoneTree/guide/tuning-disk-segment.html)** +- **[Features Overview](https://tenray.io/docs/ZoneTree/guide/features.html)** +- **[Terminology](https://tenray.io/docs/ZoneTree/guide/terminology.html)** +- **[Performance Details](https://tenray.io/docs/ZoneTree/guide/performance.html)** + +--- ## Contributing -This project welcomes contributions and suggestions. Please follow [CONTRIBUTING.md](.github/CONTRIBUTING.md) instructions. +Contributions are highly appreciated and welcomed! Here’s how you can help: + +1. **Write Tests and Benchmarks**: Improve code reliability and performance analysis. +2. **Enhance Documentation**: Help others understand and utilize ZoneTree effectively. +3. **Submit Feature Requests and Bug Reports**: Share ideas and report issues to refine ZoneTree further. +4. **Optimize Performance**: Contribute optimizations and improvements to existing functionalities. + +Please follow the guidelines outlined in **[CONTRIBUTING.md](https://github.com/koculu/ZoneTree/blob/main/.github/CONTRIBUTING.md)** to get started. + +--- + +## License + +ZoneTree is licensed under the **[MIT License](https://github.com/koculu/ZoneTree?tab=MIT-1-ov-file#readme)**, allowing for flexible use and distribution. diff --git a/src/ZoneTree/docs/ZoneTree/README-NUGET.md b/src/ZoneTree/docs/ZoneTree/README-NUGET.md index 499ce26..c0321d9 100644 --- a/src/ZoneTree/docs/ZoneTree/README-NUGET.md +++ b/src/ZoneTree/docs/ZoneTree/README-NUGET.md @@ -1,337 +1,540 @@ -![img](https://raw.githubusercontent.com/koculu/ZoneTree/main/src/ZoneTree/docs/ZoneTree/images/logo2.png) +![ZoneTree Logo](https://raw.githubusercontent.com/koculu/ZoneTree/main/src/ZoneTree/docs/ZoneTree/images/logo2.png) # ZoneTree -ZoneTree is a persistent, high-performance, transactional, and ACID-compliant [ordered key-value database](https://en.wikipedia.org/wiki/Ordered_Key-Value_Store) for .NET. -It can operate in memory or on local/cloud storage. + +ZoneTree is a **persistent**, **high-performance**, **transactional**, and **ACID-compliant** [ordered key-value database](https://en.wikipedia.org/wiki/Ordered_Key-Value_Store) for .NET. It operates seamlessly both **in-memory** and on **local/cloud storage**, making it an ideal choice for a wide range of applications requiring efficient data management. ![License](https://img.shields.io/badge/license-MIT-blue.svg) [![Downloads](https://img.shields.io/nuget/dt/ZoneTree)](https://www.nuget.org/packages/ZoneTree/) ![Platform](https://img.shields.io/badge/platform-.NET-blue.svg) ![Build](https://img.shields.io/badge/build-passing-brightgreen.svg) -ZoneTree is a lightweight, transactional and high-performance LSM Tree for .NET. - -It is several times faster than Facebook's RocksDB and hundreds of times faster than SQLite. It is faster than any other alternative that I have tested so far. -100 Million integer key-value pair inserts in 20 seconds. You may get longer durations based on the durability level. -For example, with async-compressed WAL mode, you can insert 100M integer key-value pairs in 28 seconds. Background merge operation that might take a bit longer is excluded from the insert duration because your inserted data is immediately queryable. -Loading 100M integer key-value pair database is in 812 ms. The iteration on 100M key-value pairs takes 24 seconds. -There are so many tuning options wait you to discover. - -## [INTRODUCTION](https://tenray.io/docs/ZoneTree/guide/introduction.html) -## [QUICK START GUIDE](https://tenray.io/docs/ZoneTree/guide/quick-start.html) -## [API DOCS](https://tenray.io/docs/ZoneTree/api/Tenray.ZoneTree.html) -## [TUNING ZONETREE](https://tenray.io/docs/ZoneTree/guide/tuning-disk-segment.html) -## [FEATURES](https://tenray.io/docs/ZoneTree/guide/features.html) -## [TERMINOLOGY](https://tenray.io/docs/ZoneTree/guide/terminology.html) -## [PERFORMANCE](https://tenray.io/docs/ZoneTree/guide/performance.html) - -## Why ZoneTree? -1. It is pure C#. -2. It is fast. See benchmark below. -3. Your data is protected against crashes / power cuts (optional). -4. Supports transactional and non-transactional access with blazing speeds and ACID guarantees. -5. You can embed your database into your assembly. Therefore, you don't have to pay the cost of maintaining/shipping another database product along with yours. -6. You can create scalable and non-scalable databases using ZoneTree as core database engine. - -## How fast is it? -It is possible with ZoneTree to insert 100 Million integer key-value pairs in 20 seconds using WAL mode = NONE. - -Benchmark for all modes: [benchmark](https://raw.githubusercontent.com/koculu/ZoneTree/main/src/Playground/BenchmarkForAllModes.txt) - -| Insert Benchmarks | 1M | 2M | 3M | 10M | -| ------------------------------------- | ------------- | -------- | -------- | -------- | -| int-int ZoneTree async-compressed WAL | 267 ms | 464 ms | 716 ms | 2693 ms | -| int-int ZoneTree sync-compressed WAL | 834 ms | 1617 ms | 2546 ms | 8642 ms | -| int-int ZoneTree sync WAL | 2742 ms | 5533 ms | 8242 ms | 27497 ms | -| | -| str-str ZoneTree async-compressed WAL | 892 ms | 1833 ms | 2711 ms | 9443 ms | -| str-str ZoneTree sync-compressed WAL | 1752 ms | 3397 ms | 5070 ms | 19153 ms | -| str-str ZoneTree sync WAL | 3488 ms | 7002 ms | 10483 ms | 38727 ms | -| | -| RocksDb sync WAL (10K => 11 sec) | ~1.100.000 ms | N/A | N/A | N/A | -| int-int RocksDb sync-compressed WAL | 8059 ms | 16188 ms | 23599 ms | 61947 ms | -| str-str RocksDb sync-compressed WAL | 8215 ms | 16146 ms | 23760 ms | 72491 ms | -| | - -Benchmark Configuration: -```c# -DiskCompressionBlockSize = 1024 * 1024 * 10; -WALCompressionBlockSize = 1024 * 32 * 8; +## Table of Contents + +- [Why Choose ZoneTree?](#why-choose-zonetree) +- [Performance](#performance) + - [Benchmark Results](#benchmark-results) + - [Write-Ahead Log (WAL) Modes](#write-ahead-log-wal-modes) + - [Benchmark Environment](#benchmark-environment) +- [Getting Started](#getting-started) + - [Basic Usage](#basic-usage) + - [Creating a Database](#creating-a-database) +- [Maintaining the LSM Tree](#maintaining-the-lsm-tree) +- [Handling Deletions](#handling-deletions) + - [Using an Integer Deletion Flag](#using-an-integer-deletion-flag) + - [Using a Custom Struct for Deletion](#using-a-custom-struct-for-deletion) +- [Data Iteration](#data-iteration) + - [Forward and Backward Iteration](#forward-and-backward-iteration) + - [Seekable Iterator](#seekable-iterator) +- [Transaction Support](#transaction-support) + - [Fluent Transactions](#fluent-transactions) + - [Classical Transactions](#classical-transactions) + - [Exceptionless Transactions](#exceptionless-transactions) +- [Feature Highlights](#feature-highlights) +- [ZoneTree.FullTextSearch](#zonetreefulltextsearch) +- [Documentation](#documentation) +- [Contributing](#contributing) +- [License](#license) + +--- + +## Why Choose ZoneTree? + +1. **Pure C# Implementation**: ZoneTree is developed entirely in C#, ensuring seamless integration and deployment within .NET ecosystems without external dependencies. + +2. **Exceptional Performance**: Demonstrates performance several times faster than Facebook's RocksDB and hundreds of times faster than SQLite. Optimized for both speed and efficiency. + +3. **Data Durability and Crash Resilience**: Provides optional durability features that protect data against crashes and power outages, ensuring data integrity under all circumstances. + +4. **Transactional and ACID-Compliant**: Supports both transactional and non-transactional access with full ACID guarantees, offering flexibility for various application requirements. + +5. **Embeddable Database Engine**: Easily embed ZoneTree into applications, eliminating the overhead of maintaining or shipping separate database products. + +6. **Scalability**: Capable of handling small to massive datasets, allowing the creation of both scalable and non-scalable databases tailored to specific needs. + +--- + +## Performance + +ZoneTree sets new standards in database performance, showcasing remarkable speeds in data insertion, loading, and iteration operations. + +### Benchmark Results + +- **100 Million integer key-value pairs** inserted in **20 seconds** using **WAL mode = NONE**. +- **Loading** database with 100 million integer key-value pairs takes **812 milliseconds**. +- **Iterating** over 100 million key-value pairs completes in **24 seconds**. + +**Detailed Benchmark for Various Modes:** + +| Insert Benchmarks | 1M | 2M | 3M | 10M | +| ----------------------------------------- | ------------- | -------- | -------- | -------- | +| **int-int ZoneTree async-compressed WAL** | 267 ms | 464 ms | 716 ms | 2693 ms | +| **int-int ZoneTree sync-compressed WAL** | 834 ms | 1617 ms | 2546 ms | 8642 ms | +| **int-int ZoneTree sync WAL** | 2742 ms | 5533 ms | 8242 ms | 27497 ms | +| | | | | | +| **str-str ZoneTree async-compressed WAL** | 892 ms | 1833 ms | 2711 ms | 9443 ms | +| **str-str ZoneTree sync-compressed WAL** | 1752 ms | 3397 ms | 5070 ms | 19153 ms | +| **str-str ZoneTree sync WAL** | 3488 ms | 7002 ms | 10483 ms | 38727 ms | +| | | | | | +| **RocksDb sync WAL (10K => 11 sec)** | ~1,100,000 ms | N/A | N/A | N/A | +| **int-int RocksDb sync-compressed WAL** | 8059 ms | 16188 ms | 23599 ms | 61947 ms | +| **str-str RocksDb sync-compressed WAL** | 8215 ms | 16146 ms | 23760 ms | 72491 ms | + +**[Full Benchmark Results](https://raw.githubusercontent.com/koculu/ZoneTree/main/src/Playground/BenchmarkForAllModes.txt)** + +**Benchmark Configuration:** + +```csharp +DiskCompressionBlockSize = 10 * 1024 * 1024; // 10 MB +WALCompressionBlockSize = 32 * 1024 * 8; // 256 KB DiskSegmentMode = DiskSegmentMode.SingleDiskSegment; MutableSegmentMaxItemCount = 1_000_000; ThresholdForMergeOperationStart = 2_000_000; ``` -Additional Notes: -According to our tests, ZoneTree is stable and fast even with big data. -Tested up to 1 billion records in desktop computers till now. +**Additional Notes:** + +- ZoneTree has been tested successfully with up to **1 billion records** on standard desktop computers, demonstrating stability and efficiency even with very large datasets. + +### Write-Ahead Log (WAL) Modes + +ZoneTree offers **four WAL modes** to provide flexibility between performance and durability: + +1. **Sync Mode**: + + - **Durability**: Maximum. + - **Performance**: Slower write speed. + - **Use Case**: Ensures data is not lost in case of crashes or power cuts. -### ZoneTree offers 4 WAL modes to let you make a flexible tradeoff. +2. **Sync-Compressed Mode**: -* The sync mode provides maximum durability but slower write speed. - In case of a crash/power cut, the sync mode ensures that the inserted data is not lost. + - **Durability**: High, but slightly less than sync mode. + - **Performance**: Faster write speed due to compression. + - **Use Case**: Balances durability and performance; periodic jobs can persist decompressed tail records for added safety. -* The sync-compressed mode provides faster write speed but less durability. - Compression requires chunks to be filled before appending them into the WAL file. - It is possible to enable a periodic job to persist decompressed tail records into a separate location in a specified interval. - See IWriteAheadLogProvider options for more details. +3. **Async-Compressed Mode**: -* The async-compressed mode provides faster write speed but less durability. - Log entries are queued to be written in a separate thread. - async-compressed mode uses compression in WAL files and provides immediate tail record persistence. + - **Durability**: Moderate. + - **Performance**: Very fast write speed; logs are written in a separate thread. + - **Use Case**: Suitable where immediate durability is less critical but performance is paramount. -* None WAL mode disables WAL completely to get maximum performance. Data still can be saved to disk by tree maintainer automatically or manually. +4. **None Mode**: + - **Durability**: No immediate durability; relies on manual or automatic disk saves. + - **Performance**: Maximum possible. + - **Use Case**: Ideal for scenarios where performance is critical and data can be reconstructed or is not mission-critical. + +### Benchmark Environment -### Environment: ``` BenchmarkDotNet=v0.13.1, OS=Windows 10.0.22000 -Intel Core i7-6850K CPU 3.60GHz (Skylake), 1 CPU, 12 logical and 6 physical cores -64 GB DDR4 Memory -SSD: Samsung SSD 850 EVO 1TB -Config: 1M mutable segment size, 2M readonly segments merge-threshold +Processor: Intel Core i7-6850K CPU @ 3.60GHz (Skylake), 12 logical cores, 6 physical cores +Memory: 64 GB DDR4 +Storage: Samsung SSD 850 EVO 1TB +Configuration: 1M mutable segment size, 2M readonly segments merge-threshold ``` -## How to use ZoneTree? +--- + +## Getting Started + +ZoneTree is designed for ease of use, allowing developers to integrate and utilize its capabilities with minimal setup. + +### Basic Usage -The following sample shows the most basic setup of a ZoneTree database. +```csharp +using Tenray.ZoneTree; -```C# using var zoneTree = new ZoneTreeFactory() - .OpenOrCreate(); -zoneTree.Upsert(39, "Hello Zone Tree"); + .OpenOrCreate(); + +zoneTree.Upsert(39, "Hello ZoneTree"); ``` -The following sample demonstrates creating a database. -```c# - var dataPath = "data/mydatabase"; - using var zoneTree = new ZoneTreeFactory() +### Creating a Database + +```csharp +using Tenray.ZoneTree; +using Tenray.ZoneTree.Comparers; +using Tenray.ZoneTree.Serializers; + +var dataPath = "data/mydatabase"; + +using var zoneTree = new ZoneTreeFactory() .SetComparer(new Int32ComparerAscending()) .SetDataDirectory(dataPath) .SetKeySerializer(new Int32Serializer()) .SetValueSerializer(new Utf8StringSerializer()) .OpenOrCreate(); - - // atomic (thread-safe) on single mutable-segment. - zoneTree.Upsert(39, "Hello Zone Tree!"); - - // atomic across all segments - zoneTree.TryAtomicAddOrUpdate(39, "a", - bool (ref string x) => - { - x += "b"; - return true; - }); + +// Atomic (thread-safe) operation on single mutable-segment. +zoneTree.Upsert(39, "Hello ZoneTree!"); + +// Atomic operation across all segments. +zoneTree.TryAtomicAddOrUpdate(39, "a", (ref string x) => +{ + x += "b"; + return true; +}); ``` -## How to maintain LSM Tree? -Big LSM Trees require maintenance tasks. ZoneTree provides the IZoneTreeMaintenance interface to give you full power on maintenance tasks. -It also comes with a default maintainer to let you focus on your business logic without wasting time with LSM details. -You can start using the default maintainer like in the following sample code. -Note: For small data you don't need a maintainer. -```c# - var dataPath = "data/mydatabase"; - - // 1. Create your ZoneTree - using var zoneTree = new ZoneTreeFactory() + +--- + +## Maintaining the LSM Tree + +Large-scale LSM Trees require periodic maintenance to ensure optimal performance and resource utilization. ZoneTree provides the `IZoneTreeMaintenance` interface to facilitate comprehensive maintenance tasks. + +**Example Usage:** + +```csharp +using Tenray.ZoneTree; +using Tenray.ZoneTree.Maintenance; + +var dataPath = "data/mydatabase"; + +using var zoneTree = new ZoneTreeFactory() .SetComparer(new Int32ComparerAscending()) .SetDataDirectory(dataPath) .SetKeySerializer(new Int32Serializer()) .SetValueSerializer(new Utf8StringSerializer()) .OpenOrCreate(); - - using var maintainer = zoneTree.CreateMaintainer(); - maintainer.EnableJobForCleaningInactiveCaches = true; - // 2. Read/Write data - zoneTree.Upsert(39, "Hello ZoneTree!"); +using var maintainer = zoneTree.CreateMaintainer(); +maintainer.EnableJobForCleaningInactiveCaches = true; - // 3. Wait for background threads. - maintainer.WaitForBackgroundThreads(); +// Perform read/write operations. +zoneTree.Upsert(39, "Hello ZoneTree!"); + +// Wait for background maintenance tasks to complete. +maintainer.WaitForBackgroundThreads(); ``` -## How to delete keys? -In Log-Structured Merge (LSM) trees, deletions are managed by upserting a key/value pair with a deletion marker. The actual removal of the data occurs during the compaction stage. In ZoneTree, by default, the system assumes that the default values indicate deletion. However, you can customize this behavior by defining a specific deletion flag, such as using -1 for integer values or completely disable deletion by calling DisableDeletion method. +**Note:** For smaller datasets, maintenance tasks may not be necessary. The default maintainer allows developers to focus on core business logic without delving into LSM tree intricacies. -### Custom Deletion Flag -If you need more control over how deletions are handled, you can define a custom structure to represent your values and their deletion status. For example: +--- -```c# -[StructLayout(LayoutKind.Sequential)] -struct MyDeletableValueType { - int Number; - bool IsDeleted; -} -``` +## Handling Deletions + +In **Log-Structured Merge (LSM) trees**, deletions are managed by upserting a key/value pair with a **deletion marker**. Actual data removal occurs during the **compaction** stage. + +By default, ZoneTree assumes that **default values** indicate deletion. This behavior can be customized by defining specific deletion flags or disabling deletions entirely using the `DisableDeletion` method. -This struct allows you to include a boolean flag indicating whether a value is deleted. You can then use this custom type as the value type in your ZoneTree. +### Using an Integer Deletion Flag -### Configuring Deletion Markers -ZoneTree provides flexibility in managing the tree size by allowing you to configure how deletion markers are set and identified. Below are examples of how you can configure these markers for your database: +In this example, `-1` is used as the deletion marker for integer values: -#### Example 1: Using an Integer Deletion Flag -In this example, -1 is used as the deletion marker for integer values: +```csharp +using Tenray.ZoneTree; -```c# using var zoneTree = new ZoneTreeFactory() - // Additional stuff goes here - .SetIsValueDeletedDelegate((in int x) => x == -1) - .SetMarkValueDeletedDelegate((ref int x) => x = -1) - .OpenOrCreate(); + .SetIsValueDeletedDelegate((in int x) => x == -1) + .SetMarkValueDeletedDelegate((ref int x) => x = -1) + .OpenOrCreate(); + +// Deleting a key by setting its value to -1 +zoneTree.Upsert(42, -1); ``` -#### Example 2: Using a Custom Struct for Deletion -Alternatively, if you're using a custom struct to manage deletions, you can configure ZoneTree to recognize and mark deletions as follows: +### Using a Custom Struct for Deletion + +For more control, define a custom structure to represent values and their deletion status: + +```csharp +using System.Runtime.InteropServices; +using Tenray.ZoneTree; + +[StructLayout(LayoutKind.Sequential)] +struct MyDeletableValueType +{ + public int Number; + public bool IsDeleted; +} -```c# using var zoneTree = new ZoneTreeFactory() - // Additional stuff goes here - .SetIsValueDeletedDelegate((in MyDeletableValueType x) => x.IsDeleted) - .SetMarkValueDeletedDelegate((ref MyDeletableValueType x) => x.IsDeleted = true) - .OpenOrCreate(); + .SetIsValueDeletedDelegate((in MyDeletableValueType x) => x.IsDeleted) + .SetMarkValueDeletedDelegate((ref MyDeletableValueType x) => x.IsDeleted = true) + .OpenOrCreate(); + +// Deleting a key by setting the IsDeleted flag +zoneTree.Upsert(42, new MyDeletableValueType { Number = 0, IsDeleted = true }); ``` -You can also use built in generic [Deletable<TValue>](/docs/ZoneTree/api/Tenray.ZoneTree.PresetTypes.Deletable-1.html) for deletion. +--- + +## Data Iteration + +ZoneTree provides efficient mechanisms to iterate over data both **forward** and **backward**, with equal performance in both directions. Iterators also support **seek** operations for quick access to specific keys. + +### Forward and Backward Iteration -## How to iterate over data? +```csharp +using Tenray.ZoneTree; +using Tenray.ZoneTree.Collections; -Iteration is possible in both directions, forward and backward. -Unlike other LSM tree implementations, iteration performance is equal in both directions. -The following sample shows how to do the iteration. -```c# - using var zoneTree = new ZoneTreeFactory() - // Additional stuff goes here +using var zoneTree = new ZoneTreeFactory() .OpenOrCreate(); - using var iterator = zoneTree.CreateIterator(); - while(iterator.Next()) { + +// Forward iteration +using var iterator = zoneTree.CreateIterator(); +while (iterator.Next()) +{ var key = iterator.CurrentKey; var value = iterator.CurrentValue; - } - - using var reverseIterator = zoneTree.CreateReverseIterator(); - while(reverseIterator.Next()) { + // Process key and value +} + +// Backward iteration +using var reverseIterator = zoneTree.CreateReverseIterator(); +while (reverseIterator.Next()) +{ var key = reverseIterator.CurrentKey; var value = reverseIterator.CurrentValue; - } + // Process key and value +} ``` -## How to iterate starting with a key (Seekable Iterator)? +### Seekable Iterator + +The `ZoneTreeIterator` supports the `Seek()` method to jump to any record with **O(log(n))** complexity, useful for prefix searches and range queries. -ZoneTreeIterator provides Seek() method to jump into any record with in O(log(n)) complexity. -That is useful for doing prefix search with forward-iterator or with backward-iterator. -```c# - using var zoneTree = new ZoneTreeFactory() - // Additional stuff goes here +```csharp +using Tenray.ZoneTree; +using Tenray.ZoneTree.Collections; + +using var zoneTree = new ZoneTreeFactory() .OpenOrCreate(); - using var iterator = zoneTree.CreateIterator(); - // iterator jumps into the first record starting with "SomePrefix" in O(log(n)) complexity. - iterator.Seek("SomePrefix"); - - //iterator.Next() complexity is O(1) - while(iterator.Next()) { - var key = iterator.CurrentKey; - var value = iterator.CurrentValue; - } + +using var iterator = zoneTree.CreateIterator(); + +// Jump to the first record starting with "SomePrefix" +if (iterator.Seek("SomePrefix")) +{ + do + { + var key = iterator.CurrentKey; + var value = iterator.CurrentValue; + // Process key and value + } + while (iterator.Next()); +} ``` +--- ## Transaction Support -ZoneTree supports Optimistic Transactions. It is proud to announce that the ZoneTree is ACID-compliant. Of course, you can use non-transactional API for the scenarios where eventual consistency is sufficient. -ZoneTree supports 3 way of doing transactions. -1. Fluent Transactions with ready to use retry capability. -2. Classical Transaction API. -3. Exceptionless Transaction API. +ZoneTree supports **Optimistic Transactions**, ensuring **ACID compliance** while offering flexibility through various transaction models: + +1. **Fluent Transactions**: Provides an intuitive, chainable API with built-in retry capabilities. +2. **Classical Transactions**: Traditional approach with explicit control over transaction lifecycle. +3. **Exceptionless Transactions**: Allows transaction management without relying on exceptions for control flow. -The following sample shows how to do the transactions with ZoneTree Fluent Transaction API. +### Fluent Transactions + +```csharp +using System.Threading.Tasks; +using Tenray.ZoneTree; +using Tenray.ZoneTree.Transaction; -```c# using var zoneTree = new ZoneTreeFactory() - // Additional stuff goes here .OpenOrCreateTransactional(); -using var transaction = - zoneTree - .BeginFluentTransaction() - .Do((tx) => zoneTree.UpsertNoThrow(tx, 3, 9)) - .Do((tx) => - { - if (zoneTree.TryGetNoThrow(tx, 3, out var value).IsAborted) - return TransactionResult.Aborted(); - if (zoneTree.UpsertNoThrow(tx, 3, 21).IsAborted) - return TransactionResult.Aborted(); - return TransactionResult.Success(); - }) - .SetRetryCountForPendingTransactions(100) - .SetRetryCountForAbortedTransactions(10); - await transaction.CommitAsync(); + +using var transaction = zoneTree + .BeginFluentTransaction() + .Do(tx => zoneTree.UpsertNoThrow(tx, 3, 9)) + .Do(tx => + { + if (zoneTree.TryGetNoThrow(tx, 3, out var value).IsAborted) + return TransactionResult.Aborted(); + + if (zoneTree.UpsertNoThrow(tx, 3, 21).IsAborted) + return TransactionResult.Aborted(); + + return TransactionResult.Success(); + }) + .SetRetryCountForPendingTransactions(100) + .SetRetryCountForAbortedTransactions(10); + +await transaction.CommitAsync(); ``` -The following sample shows traditional way of doing transactions with ZoneTree. -```c# - using var zoneTree = new ZoneTreeFactory() - // Additional stuff goes here +### Classical Transactions + +```csharp +using System; +using System.Threading; +using Tenray.ZoneTree; +using Tenray.ZoneTree.Transaction; + +using var zoneTree = new ZoneTreeFactory() .OpenOrCreateTransactional(); - try - { - var txId = zoneTree.BeginTransaction(); - zoneTree.TryGet(txId, 3, out var value); - zoneTree.Upsert(txId, 3, 9); - var result = zoneTree.Prepare(txId); - while (result.IsPendingTransactions) { - Thread.Sleep(100); - result = zoneTree.Prepare(txId); - } - zoneTree.Commit(txId); - } - catch(TransactionAbortedException e) - { - //retry or cancel - } + +try +{ + var txId = zoneTree.BeginTransaction(); + + zoneTree.TryGet(txId, 3, out var value); + zoneTree.Upsert(txId, 3, 9); + + var result = zoneTree.Prepare(txId); + while (result.IsPendingTransactions) + { + Thread.Sleep(100); + result = zoneTree.Prepare(txId); + } + + zoneTree.Commit(txId); +} +catch (TransactionAbortedException) +{ + // Handle aborted transaction (retry or cancel) +} +``` + +### Exceptionless Transactions + +```csharp +using System; +using System.Threading.Tasks; +using Tenray.ZoneTree.Transactional; + +public async Task ExecuteTransactionWithRetryAsync(ZoneTreeFactory zoneTreeFactory, int maxRetries = 3) +{ + using var zoneTree = zoneTreeFactory.OpenOrCreateTransactional(); + var transactionId = zoneTree.BeginTransaction(); + int retryCount = 0; + + while (retryCount <= maxRetries) + { + var result = zoneTree.UpsertNoThrow(transactionId, 1, 100); + if (result.IsAborted) + { + // Abort the transaction and return false on failure. + return false; + } + + result = zoneTree.UpsertNoThrow(transactionId, 2, 200); + if (result.IsAborted) + { + // Abort the transaction and return false on failure. + return false; + } + + var prepareResult = zoneTree.PrepareNoThrow(transactionId); + if (prepareResult.IsAborted) + { + // Abort the transaction and return false on failure. + return false; + } + + if (prepareResult.IsPendingTransactions) + { + // Optionally wait or handle pending transactions + await Task.Delay(100); // Simple delay before retrying + retryCount++; + continue; // Retry the transaction + } + + if (prepareResult.IsReadyToCommit) + { + var commitResult = zoneTree.CommitNoThrow(transactionId); + if (commitResult.IsAborted) + { + // Abort the transaction and return false on failure. + return false; + } + // Transaction committed successfully + return true; + } + } + + // Return false if retries are exhausted without a successful commit. + return false; +} ``` -## Features -| ZoneTree Features | -| --------------------------------------------------------------------------------------------- | -| Works with .NET primitives, structs and classes. | -| High Speed and Low Memory consumption. | -| Crash Resilience | -| Optimum disk space utilization. | -| WAL and DiskSegment data compression. | -| Very fast load/unload. | -| Standard read/upsert/delete functions. | -| Optimistic Transaction Support | -| Atomic Read Modify Update | -| Can work in memory. | -| Can work with any disk device including cloud devices. | -| Supports optimistic transactions. | -| Supports Atomicity, Consistency, Isolation, Durability. | -| Supports Read Committed Isolation. | -| 4 different modes for write ahead log. | -| Audit support with incremental transaction log backup. | -| Live backup. | -| Configurable amount of data that can stay in memory. | -| Partially (with sparse arrays) or completely load/unload data on disk to/from memory. | -| Forward/Backward iteration. | -| Allow optional dirty reads. | -| Embeddable. | -| Optimized for SSDs. | -| Exceptionless Transaction API. | -| Fluent Transaction API with ready to use retry capabilities. | -| Easy Maintenance. | -| Configurable LSM merger. | -| Transparent and simple implementation that reveals your database's internals. | -| Fully open-source with unrestrictive MIT license. | -| Transaction Log compaction. | -| Analyze / control transactions. | -| Concurrency Control with minimum overhead by novel separation of Concurrency Stamps and Data. | -| TTL support. | -| Use your custom serializer for keys and values. | -| Use your custom comparer. | -| MultipleDiskSegments Mode to enable dividing data files into configurable sized chunks. | -| Snapshot iterators. | - -## I want to contribute. What can I do? -I appreciate any contribution to the project. -These are the things I do think we need at the moment: -1. Write tests / benchmarks. -2. Write documentation. -3. Feature requests & bug fixes. -4. Performance improvements. +--- + +## Feature Highlights + +| Feature | Description | +| ----------------------------------------------- | ------------------------------------------------------------------------------------ | +| **.NET Compatibility** | Works seamlessly with .NET primitives, structs, and classes. | +| **High Performance and Low Memory Consumption** | Optimized algorithms ensure fast operations with minimal resource usage. | +| **Crash Resilience** | Robust mechanisms protect data integrity against unexpected failures. | +| **Efficient Disk Space Utilization** | Optimized storage strategies minimize disk space requirements. | +| **Data Compression** | Supports WAL and DiskSegment data compression for efficient storage. | +| **Fast Load/Unload** | Quickly load and unload large datasets as needed. | +| **Standard CRUD Operations** | Provides intuitive and straightforward Create, Read, Update, Delete functionalities. | +| **Optimistic Transactions** | Supports concurrent operations with minimal locking overhead. | +| **Atomic Read-Modify-Update** | Ensures data consistency during complex update operations. | +| **In-Memory and Disk Storage** | Flexibly operate entirely in memory or persist data to various storage backends. | +| **Cloud Storage Support** | Compatible with cloud-based storage solutions for scalable deployments. | +| **ACID Compliance** | Guarantees Atomicity, Consistency, Isolation, and Durability across transactions. | +| **Multiple WAL Modes** | Choose from four different WAL modes to balance performance and durability. | +| **Configurable Memory Usage** | Adjust the amount of data retained in memory based on application needs. | +| **Partial and Complete Data Loading** | Load data partially (with sparse arrays) or completely to and from disk. | +| **Bidirectional Iteration** | Efficiently iterate over data both forward and backward. | +| **Optional Dirty Reads** | Allow for faster reads when strict consistency is not required. | +| **Embeddable Design** | Integrate ZoneTree directly into applications without external dependencies. | +| **SSD Optimization** | Tailored for optimal performance on solid-state drives. | +| **Exceptionless Transaction API** | Manage transactions smoothly without relying on exceptions for control flow. | +| **Fluent Transaction API** | Utilize an intuitive, chainable transaction interface with retry capabilities. | +| **Easy Maintenance** | Simplified maintenance processes ensure consistent performance. | +| **Configurable LSM Merger** | Customize merge operations to suit specific workload patterns. | +| **Transparent Implementation** | Clear and straightforward codebase reveals internal workings for easy understanding. | +| **Open-Source with MIT License** | Freely use, modify, and distribute under a permissive license. | +| **Transaction Log Compaction** | Efficiently manage and reduce the size of transaction logs. | +| **Transaction Analysis and Control** | Analyze and manage transactions for improved performance and reliability. | +| **Efficient Concurrency Control** | Minimal overhead through innovative separation of concurrency stamps and data. | +| **Time-To-Live (TTL) Support** | Automatically expire data after a specified duration. | +| **Custom Serializer and Comparer Support** | Implement custom logic for data serialization and comparison. | +| **Multiple Disk Segments Mode** | Divide data files into configurable chunks for better manageability and performance. | +| **Snapshot Iterators** | Create consistent snapshots for data analysis and backup purposes. | + +--- + +## ZoneTree.FullTextSearch + +[ZoneTree.FullTextSearch](https://www.nuget.org/packages/ZoneTree.FullTextSearch/) is an extension library built upon ZoneTree, providing a **high-performance** and **flexible full-text search engine** for .NET applications. + +**Key Features Include:** + +- **Fast and Efficient Indexing**: Handles large volumes of text data with impressive speed. +- **Advanced Query Support**: Enables complex search queries with support for Boolean operators and faceted search. +- **Customizable Components**: Allows integration of custom tokenizers, stemmers, and normalizers to suit specific application needs. +- **Scalable Architecture**: Designed to scale seamlessly with growing data and usage demands. + +For more information and detailed documentation, visit the [ZoneTree.FullTextSearch GitHub Repository](https://github.com/koculu/ZoneTree.FullTextSearch). + +--- + +## Documentation + +Explore comprehensive guides and API references to get the most out of ZoneTree: + +- **[Introduction](https://tenray.io/docs/ZoneTree/guide/introduction.html)** +- **[Quick Start Guide](https://tenray.io/docs/ZoneTree/guide/quick-start.html)** +- **[API Documentation](https://tenray.io/docs/ZoneTree/api/Tenray.ZoneTree.html)** +- **[Tuning ZoneTree](https://tenray.io/docs/ZoneTree/guide/tuning-disk-segment.html)** +- **[Features Overview](https://tenray.io/docs/ZoneTree/guide/features.html)** +- **[Terminology](https://tenray.io/docs/ZoneTree/guide/terminology.html)** +- **[Performance Details](https://tenray.io/docs/ZoneTree/guide/performance.html)** + +--- + +## Contributing + +Contributions are highly appreciated and welcomed! Here’s how you can help: + +1. **Write Tests and Benchmarks**: Improve code reliability and performance analysis. +2. **Enhance Documentation**: Help others understand and utilize ZoneTree effectively. +3. **Submit Feature Requests and Bug Reports**: Share ideas and report issues to refine ZoneTree further. +4. **Optimize Performance**: Contribute optimizations and improvements to existing functionalities. + +Please follow the guidelines outlined in **[CONTRIBUTING.md](https://github.com/koculu/ZoneTree/blob/main/.github/CONTRIBUTING.md)** to get started. + +--- + +## License + +ZoneTree is licensed under the **[MIT License](https://github.com/koculu/ZoneTree?tab=MIT-1-ov-file#readme)**, allowing for flexible use and distribution.