feat: rotating logs (#14008)

* feat: rotating logs * feat: rotating logs * feat: rotating logs * feat: rotating logs * feat: rotating logs * feat: rotating logs --------- Co-authored-by: BohuTANG <[email protected]>
databendlabs · Dec 14, 2023 · 9fd1304 · 9fd1304
1 parent 04a8586
commit 9fd1304
Show file tree

Hide file tree

Showing 11 changed files with 62 additions and 25 deletions.
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/src/binaries/metabench/main.rs b/src/binaries/metabench/main.rs
@@ -89,6 +89,7 @@ async fn main() {
             level: config.log_level.clone(),
             dir: "./.databend/logs".to_string(),
             format: "text".to_string(),
+            limit: 48,
         },
         stderr: StderrConfig {
             on: true,

diff --git a/src/binaries/metactl/main.rs b/src/binaries/metactl/main.rs
@@ -118,6 +118,7 @@ async fn main() -> anyhow::Result<()> {
             level: config.log_level.clone(),
             dir: ".databend/logs".to_string(),
             format: "text".to_string(),
+            limit: 48,
         },
         ..Default::default()
     };

diff --git a/src/common/tracing/Cargo.toml b/src/common/tracing/Cargo.toml
@@ -32,5 +32,5 @@ serde = { workspace = true }
 serde_json = { workspace = true }
 tonic = { workspace = true }
 tracing = { version = "0.1.37", optional = true }
-tracing-appender = "0.2.2"
+tracing-appender = "0.2.3"
 tracing-subscriber = { version = "0.3.17", features = ["env-filter", "json", "valuable"], optional = true }
diff --git a/src/common/tracing/src/config.rs b/src/common/tracing/src/config.rs
@@ -36,6 +36,7 @@ impl Config {
                 level: "DEBUG".to_string(),
                 dir: "./.databend/logs".to_string(),
                 format: "text".to_string(),
+                limit: 48,
             },
             stderr: StderrConfig {
                 on: true,
@@ -53,6 +54,7 @@ pub struct FileConfig {
     pub level: String,
     pub dir: String,
     pub format: String,
+    pub limit: usize,
 }
 
 impl Display for FileConfig {
@@ -72,6 +74,7 @@ impl Default for FileConfig {
             level: "INFO".to_string(),
             dir: "./.databend/logs".to_string(),
             format: "json".to_string(),
+            limit: 48,
         }
     }
 }

diff --git a/src/common/tracing/src/init.rs b/src/common/tracing/src/init.rs
@@ -76,14 +76,26 @@ pub fn init_logging(
     mut labels: BTreeMap<String, String>,
 ) -> Vec<Box<dyn Drop + Send + Sync + 'static>> {
     let mut guards: Vec<Box<dyn Drop + Send + Sync + 'static>> = Vec::new();
+    let log_name = name;
+    let trace_name = match labels.get("node_id") {
+        None => name.to_string(),
+        Some(node_id) => format!(
+            "{}@{}",
+            name,
+            if node_id.len() >= 7 {
+                &node_id[0..7]
+            } else {
+                &node_id
+            }
+        ),
+    };
     // use name as service name if not specified
     if !labels.contains_key("service") {
-        labels.insert("service".to_string(), name.to_string());
+        labels.insert("service".to_string(), trace_name.to_string());
     }
 
     // Initialize tracing reporter
     if cfg.tracing.on {
-        let name = name.to_string();
         let otlp_endpoint = cfg.tracing.otlp_endpoint.clone();
 
         let (reporter_rt, otlp_reporter) = std::thread::spawn(|| {
@@ -108,10 +120,10 @@ pub fn init_logging(
                     .expect("initialize otlp exporter"),
                     opentelemetry::trace::SpanKind::Server,
                     Cow::Owned(opentelemetry::sdk::Resource::new([
-                        opentelemetry::KeyValue::new("service.name", name.clone()),
+                        opentelemetry::KeyValue::new("service.name", trace_name.clone()),
                     ])),
                     opentelemetry::InstrumentationLibrary::new(
-                        name,
+                        trace_name,
                         None::<&'static str>,
                         None::<&'static str>,
                         None,
@@ -140,7 +152,8 @@ pub fn init_logging(
 
     // File logger
     if cfg.file.on {
-        let (normal_log_file, flush_guard) = new_file_log_writer(&cfg.file.dir, name);
+        let (normal_log_file, flush_guard) =
+            new_file_log_writer(&cfg.file.dir, log_name, cfg.file.limit);
         guards.push(Box::new(flush_guard));
         let dispatch = fern::Dispatch::new()
             .level(cfg.file.level.parse().unwrap_or(LevelFilter::Info))
@@ -189,7 +202,8 @@ pub fn init_logging(
     // Query logger
     if cfg.query.on {
         if !cfg.query.dir.is_empty() {
-            let (query_log_file, flush_guard) = new_file_log_writer(&cfg.query.dir, name);
+            let (query_log_file, flush_guard) =
+                new_file_log_writer(&cfg.query.dir, log_name, cfg.file.limit);
             guards.push(Box::new(flush_guard));
             query_logger = query_logger.chain(Box::new(query_log_file) as Box<dyn Write + Send>);
         }
@@ -205,7 +219,8 @@ pub fn init_logging(
     // Profile logger
     if cfg.profile.on {
         if !cfg.profile.dir.is_empty() {
-            let (profile_log_file, flush_guard) = new_file_log_writer(&cfg.profile.dir, name);
+            let (profile_log_file, flush_guard) =
+                new_file_log_writer(&cfg.profile.dir, log_name, cfg.file.limit);
             guards.push(Box::new(flush_guard));
             profile_logger =
                 profile_logger.chain(Box::new(profile_log_file) as Box<dyn Write + Send>);

diff --git a/src/common/tracing/src/loggers.rs b/src/common/tracing/src/loggers.rs
@@ -40,8 +40,14 @@ use tracing_appender::rolling::Rotation;
 pub(crate) fn new_file_log_writer(
     dir: &str,
     name: impl ToString,
+    max_files: usize,
 ) -> (BufWriter<NonBlocking>, WorkerGuard) {
-    let rolling = RollingFileAppender::new(Rotation::HOURLY, dir, name.to_string());
+    let rolling = RollingFileAppender::builder()
+        .rotation(Rotation::HOURLY)
+        .filename_prefix(name.to_string())
+        .max_log_files(max_files)
+        .build(dir)
+        .expect("failed to initialize rolling file appender");
     let (non_blocking, flush_guard) = tracing_appender::non_blocking(rolling);
     let buffered_non_blocking = BufWriter::with_capacity(64 * 1024 * 1024, non_blocking);
 

diff --git a/src/meta/service/src/configs/outer_v0.rs b/src/meta/service/src/configs/outer_v0.rs
@@ -266,6 +266,7 @@ pub struct ConfigViaEnv {
     pub metasrv_log_file_level: String,
     pub metasrv_log_file_dir: String,
     pub metasrv_log_file_format: String,
+    pub metasrv_log_file_limit: usize,
     pub metasrv_log_stderr_on: bool,
     pub metasrv_log_stderr_level: String,
     pub metasrv_log_stderr_format: String,
@@ -312,6 +313,7 @@ impl From<Config> for ConfigViaEnv {
             metasrv_log_file_level: cfg.log.file.file_level,
             metasrv_log_file_dir: cfg.log.file.file_dir,
             metasrv_log_file_format: cfg.log.file.file_format,
+            metasrv_log_file_limit: cfg.log.file.file_limit,
             metasrv_log_stderr_on: cfg.log.stderr.stderr_on,
             metasrv_log_stderr_level: cfg.log.stderr.stderr_level,
             metasrv_log_stderr_format: cfg.log.stderr.stderr_format,
@@ -376,6 +378,7 @@ impl Into<Config> for ConfigViaEnv {
                 file_level: self.metasrv_log_file_level,
                 file_dir: self.metasrv_log_file_dir,
                 file_format: self.metasrv_log_file_format,
+                file_limit: self.metasrv_log_file_limit,
             },
             stderr: StderrLogConfig {
                 stderr_on: self.metasrv_log_stderr_on,
@@ -629,6 +632,11 @@ pub struct FileLogConfig {
     #[clap(long = "log-file-format", default_value = "json")]
     #[serde(rename = "format")]
     pub file_format: String,
+
+    /// Log file max
+    #[clap(long = "log-file-limit", default_value = "48")]
+    #[serde(rename = "limit")]
+    pub file_limit: usize,
 }
 
 impl Default for FileLogConfig {
@@ -645,6 +653,7 @@ impl Into<InnerFileLogConfig> for FileLogConfig {
             level: self.file_level,
             dir: self.file_dir,
             format: self.file_format,
+            limit: self.file_limit,
         }
     }
 }
@@ -656,6 +665,7 @@ impl From<InnerFileLogConfig> for FileLogConfig {
             file_level: inner.level,
             file_dir: inner.dir,
             file_format: inner.format,
+            file_limit: inner.limit,
         }
     }
 }

diff --git a/src/query/config/src/config.rs b/src/query/config/src/config.rs
@@ -1935,6 +1935,11 @@ pub struct FileLogConfig {
     #[clap(long = "log-file-format", value_name = "VALUE", default_value = "json")]
     #[serde(rename = "format")]
     pub file_format: String,
+
+    /// Log file max
+    #[clap(long = "log-file-limit", value_name = "VALUE", default_value = "48")]
+    #[serde(rename = "limit")]
+    pub file_limit: usize,
 }
 
 impl Default for FileLogConfig {
@@ -1952,6 +1957,7 @@ impl TryInto<InnerFileLogConfig> for FileLogConfig {
             level: self.file_level,
             dir: self.file_dir,
             format: self.file_format,
+            limit: self.file_limit,
         })
     }
 }
@@ -1963,6 +1969,7 @@ impl From<InnerFileLogConfig> for FileLogConfig {
             file_level: inner.level,
             file_dir: inner.dir,
             file_format: inner.format,
+            file_limit: inner.limit,
         }
     }
 }

diff --git a/src/query/service/src/global_services.rs b/src/query/service/src/global_services.rs
@@ -56,15 +56,7 @@ impl GlobalServices {
     #[async_backtrace::framed]
     pub async fn init_with(config: &InnerConfig) -> Result<()> {
         // app name format: node_id[0..7]@cluster_id
-        let app_name_shuffle = format!(
-            "databend-query-{}@{}",
-            if config.query.node_id.len() >= 7 {
-                &config.query.node_id[0..7]
-            } else {
-                &config.query.node_id
-            },
-            config.query.cluster_id
-        );
+        let app_name_shuffle = format!("databend-query-{}", config.query.cluster_id);
 
         // The order of initialization is very important
         // 1. global config init.

diff --git a/src/query/service/tests/it/storages/testdata/configs_table_basic.txt b/src/query/service/tests/it/storages/testdata/configs_table_basic.txt
@@ -23,6 +23,7 @@ DB.Table: 'system'.'configs', Table: configs-table_id:1, ver:0, Engine: SystemCo
 | 'log'     | 'file.dir'                                 | './.databend/logs'                                             | ''       |
 | 'log'     | 'file.format'                              | 'text'                                                         | ''       |
 | 'log'     | 'file.level'                               | 'DEBUG'                                                        | ''       |
+| 'log'     | 'file.limit'                               | '48'                                                           | ''       |
 | 'log'     | 'file.on'                                  | 'true'                                                         | ''       |
 | 'log'     | 'level'                                    | 'DEBUG'                                                        | ''       |
 | 'log'     | 'log_dir'                                  | 'null'                                                         | ''       |