From 0688fe530c95b482a442e3b7d48a940b9f906b6e Mon Sep 17 00:00:00 2001 From: Darach Ennis Date: Fri, 12 Apr 2024 18:02:58 +0200 Subject: [PATCH] Resolve flaky test harness tests in command and integration suites Signed-off-by: Darach Ennis --- tremor-cli/src/test/assert.rs | 77 +++++++++++++++++++ tremor-cli/src/test/assert/contains.txt | 2 + tremor-cli/tests/cli/command.yml | 11 +++ .../integration/elastic-verify-gd/assert.yaml | 54 ++++++++++++- 4 files changed, 141 insertions(+), 3 deletions(-) create mode 100644 tremor-cli/src/test/assert/contains.txt diff --git a/tremor-cli/src/test/assert.rs b/tremor-cli/src/test/assert.rs index c15acefe45..103ed0be29 100644 --- a/tremor-cli/src/test/assert.rs +++ b/tremor-cli/src/test/assert.rs @@ -104,6 +104,8 @@ pub(crate) struct FileBasedAssert { pub(crate) contains: Option>, pub(crate) doesnt_contain: Option>, pub(crate) equals_file: Option, + pub(crate) is_empty: Option, + pub(crate) is_not_empty: Option, } pub(crate) type Asserts = Vec; @@ -329,6 +331,7 @@ fn process_equals_file( }) } +#[allow(clippy::too_many_lines)] pub(crate) fn process_filebased_asserts( prefix: &str, stdout_path: &Path, @@ -345,6 +348,8 @@ pub(crate) fn process_filebased_asserts( contains: None, doesnt_contain: None, equals_file: None, + is_empty: None, + is_not_empty: None, .. } => { stats.skip(); @@ -355,6 +360,8 @@ pub(crate) fn process_filebased_asserts( contains, doesnt_contain, equals_file, + is_empty, + is_not_empty, .. } => { let file = match source { @@ -394,9 +401,79 @@ pub(crate) fn process_filebased_asserts( &mut counter, )?); } + + if let Some(is_empty) = is_empty { + stats.assert(); + counter += 1; + let condition = (std::fs::metadata(&file)?.len() == 0) == *is_empty; + stats.report(condition, &file); + status::assert_has( + prefix, + &format!("Assert {counter}"), + &format!(" File `{}` is empty", &file), + None, + condition, + )?; + elements.push(report::TestElement { + description: format!("File `{file}` is empty"), + info: None, + hidden: false, + keyword: report::KeywordKind::Predicate, + result: report::ResultKind { + status: if condition { + report::StatusKind::Passed + } else { + report::StatusKind::Failed + }, + duration: 0, + }, + }); + } + + if let Some(is_not_empty) = is_not_empty { + stats.assert(); + counter += 1; + let condition = (std::fs::metadata(&file)?.len() != 0) == *is_not_empty; + stats.report(condition, &file); + status::assert_has( + prefix, + &format!("Assert {counter}"), + &format!(" File `{}` is not empty", &file), + None, + condition, + )?; + elements.push(report::TestElement { + description: format!("File `{file}` is not empty"), + info: None, + hidden: false, + keyword: report::KeywordKind::Predicate, + result: report::ResultKind { + status: if condition { + report::StatusKind::Passed + } else { + report::StatusKind::Failed + }, + duration: 0, + }, + }); + } } } } Ok((stats, elements)) } + +#[cfg(test)] +mod test { + use super::*; + + #[test] + fn test_contains() -> Result<()> { + let file = "src/test/assert/contains.txt"; + let contains = vec!["snot".to_string(), "badger".to_string()]; + let result = file_contains(file, &contains, None)?; + assert!(result); + Ok(()) + } +} diff --git a/tremor-cli/src/test/assert/contains.txt b/tremor-cli/src/test/assert/contains.txt new file mode 100644 index 0000000000..308c583160 --- /dev/null +++ b/tremor-cli/src/test/assert/contains.txt @@ -0,0 +1,2 @@ +snot +badger diff --git a/tremor-cli/tests/cli/command.yml b/tremor-cli/tests/cli/command.yml index eb25851675..1b06ef98c1 100644 --- a/tremor-cli/tests/cli/command.yml +++ b/tremor-cli/tests/cli/command.yml @@ -199,6 +199,17 @@ suites: - "Error:" - "3 | emit event.baz" - " | ^^^ Trying to access a non existing event key `baz`\n\n" + - name: setup for tremor new creates + command: rm -rf test + tags: + - run + - new + status: 0 + expects: + - source: stdout + is_empty: true + - source: stderr + is_empty: true - name: tremor new creates command: tremor new test tags: diff --git a/tremor-cli/tests/integration/elastic-verify-gd/assert.yaml b/tremor-cli/tests/integration/elastic-verify-gd/assert.yaml index 2b3d8290e4..f3415942f1 100644 --- a/tremor-cli/tests/integration/elastic-verify-gd/assert.yaml +++ b/tremor-cli/tests/integration/elastic-verify-gd/assert.yaml @@ -2,10 +2,58 @@ status: 0 name: elastic-verify-gd asserts: - source: err.log - equals_file: expected_err.json + contains: + - | + {"action":"index","correlation":"tons","index":"my_little_index","payload":{"action":null,"cause":"this one has a different type underneath the same key and thus will fail elastic","snot":"tons","tremor":12},"success":false} + - source: err.log + contains: + - | + {"action":"index","correlation":"baz","index":"my_little_index","payload":{"action":null,"cause":"this one has a different type underneath the same key and thus will fail elastic","snot":"baz","tremor":12},"success":false} + - source: ok.log + contains: + - | + {"action":"index","correlation":"badger","index":"my_little_index","payload":{"action":null,"snot":"badger"},"success":true} + - source: ok.log + contains: + - | + {"action":"index","correlation":"foo","index":"my_little_index","payload":{"action":null,"snot":"foo"},"success":true} + - source: ok.log + contains: + - | + {"action":"index","correlation":"racoon","index":"my_little_index","payload":{"action":null,"snot":"racoon","tremor":[true,true,true,false]},"success":true} + - source: ok.log + contains: + - | + {"action":"index","correlation":"bar","index":"my_little_index","payload":{"action":null,"snot":"bar","tremor":[true,true,true,false]},"success":true} + - source: ok.log + contains: + - | + {"action":"delete","correlation":"badger","index":"my_little_index","payload":{"action":"delete","doc_id":"badger","snot":"badger"},"success":true} + - source: ok.log + contains: + - | + {"action":"delete","correlation":"fop","index":"my_little_index","payload":{"action":"delete","doc_id":"badger","snot":"fop"},"success":true} + - source: ok.log + contains: + - | + {"action":"index","correlation":"badger","index":"my_little_index","payload":{"action":"index","doc_id":"badger","snot":"badger"},"success":true} + - source: ok.log + contains: + - | + {"action":"index","correlation":"snoop","index":"my_little_index","payload":{"action":"index","doc_id":"badger","snot":"snoop"},"success":true} + - source: ok.log + contains: + - | + {"action":"update","correlation":"badger","index":"my_little_index","payload":{"action":"update","doc_id":"badger","snot":"badger"},"success":true} - source: ok.log - equals_file: expected_ok.json + contains: + - | + {"action":"update","correlation":"dogg","index":"my_little_index","payload":{"action":"update","doc_id":"badger","snot":"dogg"},"success":true} + - source: ok.log + contains: + - | + {"action":"index","correlation":"badger","index":"my_little_index","payload":{"action":null,"snot":"badger"},"success":true} - source: fg.err.log contains: - | - All required CB events received. \ No newline at end of file + All required CB events received.