From 183216ae3023815d01c7cdc63583df6f63710d4d Mon Sep 17 00:00:00 2001 From: dantengsky Date: Thu, 21 Sep 2023 14:12:09 +0800 Subject: [PATCH 1/9] ci: use nextest in CI --- .config/nextest.toml | 7 +++++++ .github/actions/test_unit/action.yml | 3 ++- 2 files changed, 9 insertions(+), 1 deletion(-) create mode 100644 .config/nextest.toml diff --git a/.config/nextest.toml b/.config/nextest.toml new file mode 100644 index 000000000000..23bbc9d77a2e --- /dev/null +++ b/.config/nextest.toml @@ -0,0 +1,7 @@ +[test-groups] +serial-integration = { max-threads = 1 } + + +[[profile.default.overrides]] +filter = 'package(databend-meta)' +test-group = 'serial-integration' diff --git a/.github/actions/test_unit/action.yml b/.github/actions/test_unit/action.yml index dfebb339f065..cff17adb7e7a 100644 --- a/.github/actions/test_unit/action.yml +++ b/.github/actions/test_unit/action.yml @@ -10,7 +10,8 @@ runs: bypass_env_vars: RUSTFLAGS,RUSTDOCFLAGS,RUST_TEST_THREADS,RUST_LOG,RUST_BACKTRACE - shell: bash - run: cargo -Zgitoxide=fetch,shallow-index,shallow-deps test + run: | + cargo -Zgitoxide=fetch,shallow-index,shallow-deps nextest run --no-fail-fast env: RUST_TEST_THREADS: "8" RUST_LOG: ERROR From 732cf93a21ce7ae497dda63cdc703a724c21054e Mon Sep 17 00:00:00 2001 From: dantengsky Date: Sun, 8 Oct 2023 13:10:22 +0800 Subject: [PATCH 2/9] tweak unit tests, adapts to nextest --- .config/nextest.toml | 7 ------- .../tests/it/grpc/metasrv_grpc_export.rs | 21 ++++++++++++++++--- src/meta/service/tests/it/tests/service.rs | 9 ++++++-- 3 files changed, 25 insertions(+), 12 deletions(-) delete mode 100644 .config/nextest.toml diff --git a/.config/nextest.toml b/.config/nextest.toml deleted file mode 100644 index 23bbc9d77a2e..000000000000 --- a/.config/nextest.toml +++ /dev/null @@ -1,7 +0,0 @@ -[test-groups] -serial-integration = { max-threads = 1 } - - -[[profile.default.overrides]] -filter = 'package(databend-meta)' -test-group = 'serial-integration' diff --git a/src/meta/service/tests/it/grpc/metasrv_grpc_export.rs b/src/meta/service/tests/it/grpc/metasrv_grpc_export.rs index 192f82c24df0..a7db0853a15a 100644 --- a/src/meta/service/tests/it/grpc/metasrv_grpc_export.rs +++ b/src/meta/service/tests/it/grpc/metasrv_grpc_export.rs @@ -36,6 +36,8 @@ async fn test_export() -> anyhow::Result<()> { // Thus in this test we'll gonna trigger a snapshot manually. let (tc, _addr) = crate::tests::start_metasrv().await?; + let config_id = tc.config.raft_config.config_id.clone(); + let grpc_api_advertise_addr = tc.config.grpc_api_advertise_address().unwrap(); let client = tc.grpc_client().await?; @@ -92,20 +94,33 @@ async fn test_export() -> anyhow::Result<()> { // The addresses are built from random number. // Wash them. - let lines = lines + let want = want .iter() .map(|x| { Regex::new(r"29\d\d\d") .unwrap() - .replace_all(x, "29000") + .replace_all(x, config_id.clone()) .to_string() }) .map(|x| { Regex::new(r"test-29\d\d\d") .unwrap() - .replace_all(&x, "test-29000") + .replace_all(&x, format!("test-{}", config_id)) + .to_string() + }) + .map(|x| { + Regex::new(r#""grpc_api_advertise_address":"127\.0\.0\.1:\d+""#) + .unwrap() + .replace_all( + &x, + format!(r#""grpc_api_advertise_address":"{}""#, grpc_api_advertise_addr), + ) .to_string() }) + .collect::>(); + + let lines = lines + .iter() .map(|x| { Regex::new(r"\d{13}") .unwrap() diff --git a/src/meta/service/tests/it/tests/service.rs b/src/meta/service/tests/it/tests/service.rs index e326488363cc..48ed1d546b69 100644 --- a/src/meta/service/tests/it/tests/service.rs +++ b/src/meta/service/tests/it/tests/service.rs @@ -21,7 +21,6 @@ use std::time::Duration; use anyhow::Result; use async_trait::async_trait; use common_base::base::tokio; -use common_base::base::GlobalSequence; use common_base::base::Stoppable; use common_meta_client::ClientHandle; use common_meta_client::MetaGrpcClient; @@ -91,7 +90,13 @@ pub async fn start_metasrv_cluster(node_ids: &[NodeId]) -> anyhow::Result u32 { - 29000u32 + (GlobalSequence::next() as u32) + use std::net::Ipv4Addr; + use std::net::SocketAddrV4; + use std::net::TcpListener; + let loopback = Ipv4Addr::new(127, 0, 0, 1); + let socket = SocketAddrV4::new(loopback, 0); + let listener = TcpListener::bind(socket).unwrap(); + listener.local_addr().unwrap().port() as u32 } pub struct MetaSrvTestContext { From a23c5077540af3002d4b944661cdeb632ebdf732 Mon Sep 17 00:00:00 2001 From: dantengsky Date: Sun, 8 Oct 2023 13:30:05 +0800 Subject: [PATCH 3/9] fmt code --- src/meta/service/tests/it/grpc/metasrv_grpc_export.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/meta/service/tests/it/grpc/metasrv_grpc_export.rs b/src/meta/service/tests/it/grpc/metasrv_grpc_export.rs index a7db0853a15a..3e5cd3cd422e 100644 --- a/src/meta/service/tests/it/grpc/metasrv_grpc_export.rs +++ b/src/meta/service/tests/it/grpc/metasrv_grpc_export.rs @@ -113,7 +113,10 @@ async fn test_export() -> anyhow::Result<()> { .unwrap() .replace_all( &x, - format!(r#""grpc_api_advertise_address":"{}""#, grpc_api_advertise_addr), + format!( + r#""grpc_api_advertise_address":"{}""#, + grpc_api_advertise_addr + ), ) .to_string() }) From 77dcc9af8cb7e96865665b3ed6001f0e826a4c42 Mon Sep 17 00:00:00 2001 From: dantengsky Date: Sun, 8 Oct 2023 14:20:23 +0800 Subject: [PATCH 4/9] hide nextest progress bar --- .github/actions/test_unit/action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/test_unit/action.yml b/.github/actions/test_unit/action.yml index cff17adb7e7a..d79751f76d0d 100644 --- a/.github/actions/test_unit/action.yml +++ b/.github/actions/test_unit/action.yml @@ -11,7 +11,7 @@ runs: - shell: bash run: | - cargo -Zgitoxide=fetch,shallow-index,shallow-deps nextest run --no-fail-fast + cargo -Zgitoxide=fetch,shallow-index,shallow-deps nextest run --no-fail-fast --hide-progress-bar env: RUST_TEST_THREADS: "8" RUST_LOG: ERROR From 9bbb5fc2c158a17862374a6fdc6d37e26226f54c Mon Sep 17 00:00:00 2001 From: dantengsky Date: Sun, 8 Oct 2023 14:24:11 +0800 Subject: [PATCH 5/9] make lint --- src/meta/service/tests/it/grpc/metasrv_grpc_export.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/meta/service/tests/it/grpc/metasrv_grpc_export.rs b/src/meta/service/tests/it/grpc/metasrv_grpc_export.rs index 3e5cd3cd422e..7e654d4a47be 100644 --- a/src/meta/service/tests/it/grpc/metasrv_grpc_export.rs +++ b/src/meta/service/tests/it/grpc/metasrv_grpc_export.rs @@ -122,12 +122,13 @@ async fn test_export() -> anyhow::Result<()> { }) .collect::>(); + // Wash the time_ms fields let lines = lines .iter() .map(|x| { Regex::new(r"\d{13}") .unwrap() - .replace_all(&x, "1111111111111") + .replace_all(x, "1111111111111") .to_string() }) .collect::>(); From 6b821a2854426611290d2f7727aef104a489abe4 Mon Sep 17 00:00:00 2001 From: dantengsky Date: Sun, 8 Oct 2023 15:06:37 +0800 Subject: [PATCH 6/9] Revert "fmt code" This reverts commit a23c5077540af3002d4b944661cdeb632ebdf732. --- src/meta/service/tests/it/grpc/metasrv_grpc_export.rs | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/meta/service/tests/it/grpc/metasrv_grpc_export.rs b/src/meta/service/tests/it/grpc/metasrv_grpc_export.rs index 7e654d4a47be..e66c2945f14b 100644 --- a/src/meta/service/tests/it/grpc/metasrv_grpc_export.rs +++ b/src/meta/service/tests/it/grpc/metasrv_grpc_export.rs @@ -113,10 +113,7 @@ async fn test_export() -> anyhow::Result<()> { .unwrap() .replace_all( &x, - format!( - r#""grpc_api_advertise_address":"{}""#, - grpc_api_advertise_addr - ), + format!(r#""grpc_api_advertise_address":"{}""#, grpc_api_advertise_addr), ) .to_string() }) From 1b83bdb011ebebea1fc05da7ce802cef16565ee7 Mon Sep 17 00:00:00 2001 From: dantengsky Date: Sun, 8 Oct 2023 15:08:37 +0800 Subject: [PATCH 7/9] Revert "tweak unit tests, adapts to nextest" This reverts commit 732cf93a21ce7ae497dda63cdc703a724c21054e. --- .config/nextest.toml | 7 ++++++ .../tests/it/grpc/metasrv_grpc_export.rs | 22 +++---------------- src/meta/service/tests/it/tests/service.rs | 9 ++------ 3 files changed, 12 insertions(+), 26 deletions(-) create mode 100644 .config/nextest.toml diff --git a/.config/nextest.toml b/.config/nextest.toml new file mode 100644 index 000000000000..23bbc9d77a2e --- /dev/null +++ b/.config/nextest.toml @@ -0,0 +1,7 @@ +[test-groups] +serial-integration = { max-threads = 1 } + + +[[profile.default.overrides]] +filter = 'package(databend-meta)' +test-group = 'serial-integration' diff --git a/src/meta/service/tests/it/grpc/metasrv_grpc_export.rs b/src/meta/service/tests/it/grpc/metasrv_grpc_export.rs index e66c2945f14b..d6f891e7545e 100644 --- a/src/meta/service/tests/it/grpc/metasrv_grpc_export.rs +++ b/src/meta/service/tests/it/grpc/metasrv_grpc_export.rs @@ -36,8 +36,6 @@ async fn test_export() -> anyhow::Result<()> { // Thus in this test we'll gonna trigger a snapshot manually. let (tc, _addr) = crate::tests::start_metasrv().await?; - let config_id = tc.config.raft_config.config_id.clone(); - let grpc_api_advertise_addr = tc.config.grpc_api_advertise_address().unwrap(); let client = tc.grpc_client().await?; @@ -94,34 +92,20 @@ async fn test_export() -> anyhow::Result<()> { // The addresses are built from random number. // Wash them. - let want = want + let lines = lines .iter() .map(|x| { Regex::new(r"29\d\d\d") .unwrap() - .replace_all(x, config_id.clone()) + .replace_all(x, "29000") .to_string() }) .map(|x| { Regex::new(r"test-29\d\d\d") .unwrap() - .replace_all(&x, format!("test-{}", config_id)) - .to_string() - }) - .map(|x| { - Regex::new(r#""grpc_api_advertise_address":"127\.0\.0\.1:\d+""#) - .unwrap() - .replace_all( - &x, - format!(r#""grpc_api_advertise_address":"{}""#, grpc_api_advertise_addr), - ) + .replace_all(&x, "test-29000") .to_string() }) - .collect::>(); - - // Wash the time_ms fields - let lines = lines - .iter() .map(|x| { Regex::new(r"\d{13}") .unwrap() diff --git a/src/meta/service/tests/it/tests/service.rs b/src/meta/service/tests/it/tests/service.rs index 48ed1d546b69..e326488363cc 100644 --- a/src/meta/service/tests/it/tests/service.rs +++ b/src/meta/service/tests/it/tests/service.rs @@ -21,6 +21,7 @@ use std::time::Duration; use anyhow::Result; use async_trait::async_trait; use common_base::base::tokio; +use common_base::base::GlobalSequence; use common_base::base::Stoppable; use common_meta_client::ClientHandle; use common_meta_client::MetaGrpcClient; @@ -90,13 +91,7 @@ pub async fn start_metasrv_cluster(node_ids: &[NodeId]) -> anyhow::Result u32 { - use std::net::Ipv4Addr; - use std::net::SocketAddrV4; - use std::net::TcpListener; - let loopback = Ipv4Addr::new(127, 0, 0, 1); - let socket = SocketAddrV4::new(loopback, 0); - let listener = TcpListener::bind(socket).unwrap(); - listener.local_addr().unwrap().port() as u32 + 29000u32 + (GlobalSequence::next() as u32) } pub struct MetaSrvTestContext { From 1697ba6ab2064851b524d93bc650884464c9907a Mon Sep 17 00:00:00 2001 From: dantengsky Date: Sun, 8 Oct 2023 15:09:20 +0800 Subject: [PATCH 8/9] serialize execution databend-meta --- .config/nextest.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/.config/nextest.toml b/.config/nextest.toml index 23bbc9d77a2e..17f46bb3de72 100644 --- a/.config/nextest.toml +++ b/.config/nextest.toml @@ -5,3 +5,4 @@ serial-integration = { max-threads = 1 } [[profile.default.overrides]] filter = 'package(databend-meta)' test-group = 'serial-integration' + From cd8a52c20b4d76c8c108cc0f98b5b56f984c11bb Mon Sep 17 00:00:00 2001 From: dantengsky Date: Sun, 8 Oct 2023 15:26:13 +0800 Subject: [PATCH 9/9] make lint --- src/meta/service/tests/it/grpc/metasrv_grpc_export.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/meta/service/tests/it/grpc/metasrv_grpc_export.rs b/src/meta/service/tests/it/grpc/metasrv_grpc_export.rs index d6f891e7545e..192f82c24df0 100644 --- a/src/meta/service/tests/it/grpc/metasrv_grpc_export.rs +++ b/src/meta/service/tests/it/grpc/metasrv_grpc_export.rs @@ -109,7 +109,7 @@ async fn test_export() -> anyhow::Result<()> { .map(|x| { Regex::new(r"\d{13}") .unwrap() - .replace_all(x, "1111111111111") + .replace_all(&x, "1111111111111") .to_string() }) .collect::>();