feat(storage): introduce merge group metrics and corrects some config (…

…#18576)

grafana/risingwave-dev-dashboard.dashboard.py

@@ -3254,13 +3254,18 @@ def section_hummock_manager(outer_panels):
                     ],
                 ),
                 panels.timeseries_count(
-                    "Move State Table Count",
+                    "Compaction Group Schedule",
                     "The times of move_state_table occurs",
                     [
                         panels.target(
                             f"sum({table_metric('storage_move_state_table_count')}) by (group)",
                             "move table cg{{group}}",
                         ),
+
+                        panels.target(
+                            f"sum({table_metric('storage_merge_compaction_group_count')}) by (group)",
+                            "merge_compaction_group_count cg-{{group}}",
+                        ),
                     ],
                 ),
                 panels.timeseries_count(

src/common/src/config.rs

@@ -298,8 +298,11 @@ pub struct MetaConfig {
     #[serde(default = "default::meta::periodic_tombstone_reclaim_compaction_interval_sec")]
     pub periodic_tombstone_reclaim_compaction_interval_sec: u64,
 
-    #[serde(default = "default::meta::periodic_split_compact_group_interval_sec")]
-    pub periodic_split_compact_group_interval_sec: u64,
+    #[serde(
+        default = "default::meta::periodic_scheduling_compaction_group_interval_sec",
+        alias = "periodic_split_compact_group_interval_sec"
+    )]
+    pub periodic_scheduling_compaction_group_interval_sec: u64,
 
     #[serde(default = "default::meta::move_table_size_limit")]
     pub move_table_size_limit: u64,
@@ -308,6 +311,7 @@ pub struct MetaConfig {
     pub split_group_size_limit: u64,
 
     #[serde(default = "default::meta::cut_table_size_limit")]
+    #[deprecated]
     pub cut_table_size_limit: u64,
 
     #[serde(default, flatten)]
@@ -1399,7 +1403,7 @@ pub mod default {
             1800 // 30mi
         }
 
-        pub fn periodic_split_compact_group_interval_sec() -> u64 {
+        pub fn periodic_scheduling_compaction_group_interval_sec() -> u64 {
             10 // 10s
         }
 
@@ -2625,4 +2629,25 @@ mod tests {
             );
         }
     }
+
+    #[test]
+    fn test_meta_configs_backward_compatibility() {
+        // Test periodic_space_reclaim_compaction_interval_sec
+        {
+            let config: RwConfig = toml::from_str(
+                r#"
+            [meta]
+            periodic_split_compact_group_interval_sec = 1
+            "#,
+            )
+            .unwrap();
+
+            assert_eq!(
+                config
+                    .meta
+                    .periodic_scheduling_compaction_group_interval_sec,
+                1
+            );
+        }
+    }
 }

src/config/docs.md

@@ -54,8 +54,8 @@ This page is automatically generated by `./risedev generate-example-config`
 | parallelism_control_trigger_period_sec | The period of parallelism control trigger. | 10 |
 | partition_vnode_count | Count of partition in split group. Meta will assign this value to every new group when it splits from default-group by automatically. Each partition contains aligned data of `VirtualNode::COUNT / partition_vnode_count` consecutive virtual-nodes of one state table. | 16 |
 | periodic_compaction_interval_sec | Schedule compaction for all compaction groups with this interval. | 60 |
+| periodic_scheduling_compaction_group_interval_sec |  | 10 |
 | periodic_space_reclaim_compaction_interval_sec | Schedule `space_reclaim` compaction for all compaction groups with this interval. | 3600 |
-| periodic_split_compact_group_interval_sec |  | 10 |
 | periodic_tombstone_reclaim_compaction_interval_sec |  | 600 |
 | periodic_ttl_reclaim_compaction_interval_sec | Schedule `ttl_reclaim` compaction for all compaction groups with this interval. | 1800 |
 | split_group_size_limit |  | 68719476736 |

src/config/example.toml

@@ -39,7 +39,7 @@ backend = "Mem"
 periodic_space_reclaim_compaction_interval_sec = 3600
 periodic_ttl_reclaim_compaction_interval_sec = 1800
 periodic_tombstone_reclaim_compaction_interval_sec = 600
-periodic_split_compact_group_interval_sec = 10
+periodic_scheduling_compaction_group_interval_sec = 10
 move_table_size_limit = 10737418240
 split_group_size_limit = 68719476736
 cut_table_size_limit = 1073741824

src/meta/node/src/lib.rs

@@ -407,9 +407,9 @@ pub fn start(
                 periodic_tombstone_reclaim_compaction_interval_sec: config
                     .meta
                     .periodic_tombstone_reclaim_compaction_interval_sec,
-                periodic_split_compact_group_interval_sec: config
+                periodic_scheduling_compaction_group_interval_sec: config
                     .meta
-                    .periodic_split_compact_group_interval_sec,
+                    .periodic_scheduling_compaction_group_interval_sec,
                 split_group_size_limit: config.meta.split_group_size_limit,
                 min_table_split_size: config.meta.move_table_size_limit,
                 table_write_throughput_threshold: config.meta.table_write_throughput_threshold,
@@ -429,7 +429,6 @@ pub fn start(
                     .compaction_task_max_heartbeat_interval_secs,
                 compaction_task_max_progress_interval_secs,
                 compaction_config: Some(config.meta.compaction_config),
-                cut_table_size_limit: config.meta.cut_table_size_limit,
                 hybrid_partition_node_count: config.meta.hybrid_partition_vnode_count,
                 event_log_enabled: config.meta.event_log_enabled,
                 event_log_channel_max_size: config.meta.event_log_channel_max_size,

src/meta/service/src/hummock_service.rs

@@ -608,7 +608,7 @@ impl HummockManagerService for HummockServiceImpl {
             periodic_space_reclaim_compaction_interval_sec,
             periodic_ttl_reclaim_compaction_interval_sec,
             periodic_tombstone_reclaim_compaction_interval_sec,
-            periodic_split_compact_group_interval_sec,
+            periodic_scheduling_compaction_group_interval_sec,
             split_group_size_limit,
             min_table_split_size,
             do_not_config_object_storage_lifecycle,

src/meta/src/hummock/manager/compaction/compaction_group_schedule.rs

@@ -258,6 +258,11 @@ impl HummockManager {
         drop(compaction_guard);
         self.report_compact_tasks(canceled_tasks).await?;
 
+        self.metrics
+            .merge_compaction_group_count
+            .with_label_values(&[&left_group_id.to_string()])
+            .inc();
+
         Ok(())
     }
 

src/meta/src/hummock/manager/timer_task.rs

@@ -145,21 +145,22 @@ impl HummockManager {
                 Box::pin(tombstone_reclaim_trigger),
             ];
 
-            let periodic_check_split_group_interval_sec = hummock_manager
+            let periodic_scheduling_compaction_group_interval_sec = hummock_manager
                 .env
                 .opts
-                .periodic_split_compact_group_interval_sec;
+                .periodic_scheduling_compaction_group_interval_sec;
 
-            if periodic_check_split_group_interval_sec > 0 {
-                let mut split_group_trigger_interval = tokio::time::interval(Duration::from_secs(
-                    periodic_check_split_group_interval_sec,
-                ));
-                split_group_trigger_interval
+            if periodic_scheduling_compaction_group_interval_sec > 0 {
+                let mut scheduling_compaction_group_trigger_interval = tokio::time::interval(
+                    Duration::from_secs(periodic_scheduling_compaction_group_interval_sec),
+                );
+                scheduling_compaction_group_trigger_interval
                     .set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay);
 
-                let split_group_trigger = IntervalStream::new(split_group_trigger_interval)
-                    .map(|_| HummockTimerEvent::GroupSchedule);
-                triggers.push(Box::pin(split_group_trigger));
+                let group_scheduling_trigger =
+                    IntervalStream::new(scheduling_compaction_group_trigger_interval)
+                        .map(|_| HummockTimerEvent::GroupSchedule);
+                triggers.push(Box::pin(group_scheduling_trigger));
             }
 
             let event_stream = select_all(triggers);
@@ -169,8 +170,8 @@ impl HummockManager {
             let shutdown_rx_shared = shutdown_rx.shared();
 
             tracing::info!(
-                "Hummock timer task tracing [GroupSplit interval {} sec] [CheckDeadTask interval {} sec] [Report interval {} sec] [CompactionHeartBeat interval {} sec]",
-                    periodic_check_split_group_interval_sec, CHECK_PENDING_TASK_PERIOD_SEC, STAT_REPORT_PERIOD_SEC, COMPACTION_HEARTBEAT_PERIOD_SEC
+                "Hummock timer task [GroupScheduling interval {} sec] [CheckDeadTask interval {} sec] [Report interval {} sec] [CompactionHeartBeat interval {} sec]",
+                periodic_scheduling_compaction_group_interval_sec, CHECK_PENDING_TASK_PERIOD_SEC, STAT_REPORT_PERIOD_SEC, COMPACTION_HEARTBEAT_PERIOD_SEC
             );
 
             loop {

src/meta/src/manager/env.rs

@@ -227,8 +227,8 @@ pub struct MetaOpts {
     /// Schedule `tombstone_reclaim_compaction` for all compaction groups with this interval.
     pub periodic_tombstone_reclaim_compaction_interval_sec: u64,
 
-    /// Schedule `split_compaction_group` for all compaction groups with this interval.
-    pub periodic_split_compact_group_interval_sec: u64,
+    /// Schedule `periodic_scheduling_compaction_group_interval_sec` for all compaction groups with this interval.
+    pub periodic_scheduling_compaction_group_interval_sec: u64,
 
     /// The size limit to split a large compaction group.
     pub split_group_size_limit: u64,
@@ -249,9 +249,6 @@ pub struct MetaOpts {
     pub compaction_task_max_progress_interval_secs: u64,
     pub compaction_config: Option<CompactionConfig>,
 
-    /// The size limit to split a state-table to independent sstable.
-    pub cut_table_size_limit: u64,
-
     /// hybird compaction group config
     ///
     /// `hybrid_partition_vnode_count` determines the granularity of vnodes in the hybrid compaction group for SST alignment.
@@ -334,7 +331,7 @@ impl MetaOpts {
             telemetry_enabled: false,
             periodic_ttl_reclaim_compaction_interval_sec: 60,
             periodic_tombstone_reclaim_compaction_interval_sec: 60,
-            periodic_split_compact_group_interval_sec: 60,
+            periodic_scheduling_compaction_group_interval_sec: 60,
             split_group_size_limit: 5 * 1024 * 1024 * 1024,
             min_table_split_size: 2 * 1024 * 1024 * 1024,
             compact_task_table_size_partition_threshold_low: 128 * 1024 * 1024,
@@ -346,7 +343,6 @@ impl MetaOpts {
             compaction_task_max_heartbeat_interval_secs: 0,
             compaction_task_max_progress_interval_secs: 1,
             compaction_config: None,
-            cut_table_size_limit: 1024 * 1024 * 1024,
             hybrid_partition_node_count: 4,
             event_log_enabled: false,
             event_log_channel_max_size: 1,

src/meta/src/rpc/metrics.rs

@@ -199,6 +199,9 @@ pub struct MetaMetrics {
     /// Write throughput of commit epoch for each stable
     pub table_write_throughput: IntCounterVec,
 
+    /// The number of compaction groups that have been triggered to move
+    pub merge_compaction_group_count: IntCounterVec,
+
     // ********************************** Auto Schema Change ************************************
     pub auto_schema_change_failure_cnt: LabelGuardedIntCounterVec<2>,
     pub auto_schema_change_success_cnt: LabelGuardedIntCounterVec<2>,
@@ -729,6 +732,14 @@ impl MetaMetrics {
         let compaction_event_loop_iteration_latency =
             register_histogram_with_registry!(opts, registry).unwrap();
 
+        let merge_compaction_group_count = register_int_counter_vec_with_registry!(
+            "storage_merge_compaction_group_count",
+            "Count of trigger merge compaction group",
+            &["group"],
+            registry
+        )
+        .unwrap();
+
         Self {
             grpc_latency,
             barrier_latency,
@@ -802,6 +813,7 @@ impl MetaMetrics {
             auto_schema_change_failure_cnt,
             auto_schema_change_success_cnt,
             auto_schema_change_latency,
+            merge_compaction_group_count,
         }
     }