From 20bc3f2ac19a2e6313257e064f57db3db29fa295 Mon Sep 17 00:00:00 2001 From: Thomas Patzke Date: Mon, 14 Oct 2024 22:59:42 +0200 Subject: [PATCH 1/2] TextQueryBackend optionally allows special characters in startswith, endswith, and contains expressions --- sigma/conversion/base.py | 19 +++++++--- tests/test_conversion_base.py | 68 +++++++++++++++++++++++++++++++++++ 2 files changed, 82 insertions(+), 5 deletions(-) diff --git a/sigma/conversion/base.py b/sigma/conversion/base.py index 4833c1e7..3887ae62 100644 --- a/sigma/conversion/base.py +++ b/sigma/conversion/base.py @@ -807,8 +807,11 @@ class variables. If this is not sufficient, the respective methods can be implem # String matching operators. if none is appropriate eq_token is used. startswith_expression: ClassVar[Optional[str]] = None + startswith_expression_allow_special: ClassVar[bool] = False endswith_expression: ClassVar[Optional[str]] = None + endswith_expression_allow_special: ClassVar[bool] = False contains_expression: ClassVar[Optional[str]] = None + contains_expression_allow_special: ClassVar[bool] = False wildcard_match_expression: ClassVar[Optional[str]] = ( None # Special expression if wildcards can't be matched with the eq_token operator. ) @@ -1309,9 +1312,10 @@ def convert_condition_field_eq_val_str( self.startswith_expression is not None # 'startswith' operator is defined in backend and cond.value.endswith(SpecialChars.WILDCARD_MULTI) # String ends with wildcard - and not cond.value[ - :-1 - ].contains_special() # Remainder of string doesn't contains special characters + and ( + self.startswith_expression_allow_special + or not cond.value[:-1].contains_special() + ) # Remainder of string doesn't contains special characters or it's allowed ): expr = ( self.startswith_expression @@ -1320,7 +1324,9 @@ def convert_condition_field_eq_val_str( elif ( # Same as above but for 'endswith' operator: string starts with wildcard and doesn't contains further special characters self.endswith_expression is not None and cond.value.startswith(SpecialChars.WILDCARD_MULTI) - and not cond.value[1:].contains_special() + and ( + self.endswith_expression_allow_special or not cond.value[1:].contains_special() + ) ): expr = self.endswith_expression value = cond.value[1:] @@ -1328,7 +1334,10 @@ def convert_condition_field_eq_val_str( self.contains_expression is not None and cond.value.startswith(SpecialChars.WILDCARD_MULTI) and cond.value.endswith(SpecialChars.WILDCARD_MULTI) - and not cond.value[1:-1].contains_special() + and ( + self.contains_expression_allow_special + or not cond.value[1:-1].contains_special() + ) ): expr = self.contains_expression value = cond.value[1:-1] diff --git a/tests/test_conversion_base.py b/tests/test_conversion_base.py index 1f483425..9ed48f63 100644 --- a/tests/test_conversion_base.py +++ b/tests/test_conversion_base.py @@ -328,6 +328,29 @@ def test_convert_value_str_startswith_further_wildcard(test_backend): ) +def test_convert_value_str_startswith_further_wildcard_allowed(test_backend, monkeypatch): + monkeypatch.setattr(test_backend, "startswith_expression_allow_special", True) + assert ( + test_backend.convert( + SigmaCollection.from_yaml( + """ + title: Test + status: test + logsource: + category: test_category + product: test_product + detection: + sel: + fieldA|startswith: "va*lue" + field A|startswith: "va*lue" + condition: sel + """ + ) + ) + == ['mappedA startswith "va*lue" and \'field A\' startswith "va*lue"'] + ) + + def test_convert_value_str_startswith_expression_not_defined(test_backend, monkeypatch): monkeypatch.setattr(test_backend, "startswith_expression", None) assert ( @@ -416,6 +439,29 @@ def test_convert_value_str_endswith_further_wildcard(test_backend): ) +def test_convert_value_str_endswith_further_wildcard_allowed(test_backend, monkeypatch): + monkeypatch.setattr(test_backend, "endswith_expression_allow_special", True) + assert ( + test_backend.convert( + SigmaCollection.from_yaml( + """ + title: Test + status: test + logsource: + category: test_category + product: test_product + detection: + sel: + fieldA|endswith: "va*lue" + field A|endswith: "va*lue" + condition: sel + """ + ) + ) + == ['mappedA endswith "va*lue" and \'field A\' endswith "va*lue"'] + ) + + def test_convert_value_str_endswith_expression_not_defined(test_backend, monkeypatch): monkeypatch.setattr(test_backend, "endswith_expression", None) assert ( @@ -503,6 +549,28 @@ def test_convert_value_str_contains_further_wildcard(test_backend): ) +def test_convert_value_str_contains_further_wildcard_allowed(test_backend, monkeypatch): + monkeypatch.setattr(test_backend, "contains_expression_allow_special", True) + assert ( + test_backend.convert( + SigmaCollection.from_yaml( + """ + title: Test + status: test + logsource: + category: test_category + product: test_product + detection: + sel: + fieldA|contains: "va*lue" + condition: sel + """ + ) + ) + == ['mappedA contains "va*lue"'] + ) + + def test_convert_value_str_wildcard_to_regex(test_backend, monkeypatch): monkeypatch.setattr(test_backend, "wildcard_match_expression", '{field} match "{regex}"') assert ( From a81d690af8993864b68fe4588f90132c88284b64 Mon Sep 17 00:00:00 2001 From: Thomas Patzke Date: Mon, 14 Oct 2024 23:26:38 +0200 Subject: [PATCH 2/2] Optionally allow special characters in cased startswith, endswith, and contains expressions --- sigma/conversion/base.py | 20 ++++-- tests/test_conversion_base.py | 129 ++++++++++++++++++++++++++++++++++ 2 files changed, 144 insertions(+), 5 deletions(-) diff --git a/sigma/conversion/base.py b/sigma/conversion/base.py index 3887ae62..7f99fb7e 100644 --- a/sigma/conversion/base.py +++ b/sigma/conversion/base.py @@ -842,8 +842,11 @@ class variables. If this is not sufficient, the respective methods can be implem # Case sensitive string matching operators similar to standard string matching. If not provided, # case_sensitive_match_expression is used. case_sensitive_startswith_expression: ClassVar[Optional[str]] = None + case_sensitive_startswith_expression_allow_special: ClassVar[bool] = False case_sensitive_endswith_expression: ClassVar[Optional[str]] = None + case_sensitive_endswith_expression_allow_special: ClassVar[bool] = False case_sensitive_contains_expression: ClassVar[Optional[str]] = None + case_sensitive_contains_expression_allow_special: ClassVar[bool] = False # CIDR expressions: define CIDR matching if backend has native support. Else pySigma expands # CIDR values into string wildcard matches. @@ -1369,9 +1372,10 @@ def convert_condition_field_eq_val_str_case_sensitive( self.case_sensitive_startswith_expression is not None # 'startswith' operator is defined in backend and cond.value.endswith(SpecialChars.WILDCARD_MULTI) # String ends with wildcard - and not cond.value[ - :-1 - ].contains_special() # Remainder of string doesn't contains special characters + and ( + self.case_sensitive_startswith_expression_allow_special + or not cond.value[:-1].contains_special() + ) # Remainder of string doesn't contains special characters or it's allowed ): expr = ( self.case_sensitive_startswith_expression @@ -1380,7 +1384,10 @@ def convert_condition_field_eq_val_str_case_sensitive( elif ( # Same as above but for 'endswith' operator: string starts with wildcard and doesn't contains further special characters self.case_sensitive_endswith_expression is not None and cond.value.startswith(SpecialChars.WILDCARD_MULTI) - and not cond.value[1:].contains_special() + and ( + self.case_sensitive_endswith_expression_allow_special + or not cond.value[1:].contains_special() + ) ): expr = self.case_sensitive_endswith_expression value = cond.value[1:] @@ -1388,7 +1395,10 @@ def convert_condition_field_eq_val_str_case_sensitive( self.case_sensitive_contains_expression is not None and cond.value.startswith(SpecialChars.WILDCARD_MULTI) and cond.value.endswith(SpecialChars.WILDCARD_MULTI) - and not cond.value[1:-1].contains_special() + and ( + self.case_sensitive_contains_expression_allow_special + or not cond.value[1:-1].contains_special() + ) ): expr = self.case_sensitive_contains_expression value = cond.value[1:-1] diff --git a/tests/test_conversion_base.py b/tests/test_conversion_base.py index 9ed48f63..abf08885 100644 --- a/tests/test_conversion_base.py +++ b/tests/test_conversion_base.py @@ -351,6 +351,49 @@ def test_convert_value_str_startswith_further_wildcard_allowed(test_backend, mon ) +def test_convert_value_str_startswith_cased_further_wildcard(test_backend): + assert ( + test_backend.convert( + SigmaCollection.from_yaml( + """ + title: Test + status: test + logsource: + category: test_category + product: test_product + detection: + sel: + "field|startswith|cased": "va*lue" + condition: sel + """ + ) + ) + == ['field casematch "va*lue*"'] + ) + + +def test_convert_value_str_startswith_cased_further_wildcard_allowed(test_backend, monkeypatch): + monkeypatch.setattr(test_backend, "case_sensitive_startswith_expression_allow_special", True) + assert ( + test_backend.convert( + SigmaCollection.from_yaml( + """ + title: Test + status: test + logsource: + category: test_category + product: test_product + detection: + sel: + "field|startswith|cased": "va*lue" + condition: sel + """ + ) + ) + == ['field startswith_cased "va*lue"'] + ) + + def test_convert_value_str_startswith_expression_not_defined(test_backend, monkeypatch): monkeypatch.setattr(test_backend, "startswith_expression", None) assert ( @@ -462,6 +505,49 @@ def test_convert_value_str_endswith_further_wildcard_allowed(test_backend, monke ) +def test_convert_value_str_endswith_cased_further_wildcard(test_backend): + assert ( + test_backend.convert( + SigmaCollection.from_yaml( + """ + title: Test + status: test + logsource: + category: test_category + product: test_product + detection: + sel: + "field|endswith|cased": "va*lue" + condition: sel + """ + ) + ) + == ['field casematch "*va*lue"'] + ) + + +def test_convert_value_str_endswith_cased_further_wildcard_allowed(test_backend, monkeypatch): + monkeypatch.setattr(test_backend, "case_sensitive_endswith_expression_allow_special", True) + assert ( + test_backend.convert( + SigmaCollection.from_yaml( + """ + title: Test + status: test + logsource: + category: test_category + product: test_product + detection: + sel: + "field|endswith|cased": "va*lue" + condition: sel + """ + ) + ) + == ['field endswith_cased "va*lue"'] + ) + + def test_convert_value_str_endswith_expression_not_defined(test_backend, monkeypatch): monkeypatch.setattr(test_backend, "endswith_expression", None) assert ( @@ -571,6 +657,49 @@ def test_convert_value_str_contains_further_wildcard_allowed(test_backend, monke ) +def test_convert_value_str_contains_cased_further_wildcard(test_backend): + assert ( + test_backend.convert( + SigmaCollection.from_yaml( + """ + title: Test + status: test + logsource: + category: test_category + product: test_product + detection: + sel: + "field|contains|cased": "va*lue" + condition: sel + """ + ) + ) + == ['field casematch "*va*lue*"'] + ) + + +def test_convert_value_str_contains_cased_further_wildcard_allowed(test_backend, monkeypatch): + monkeypatch.setattr(test_backend, "case_sensitive_contains_expression_allow_special", True) + assert ( + test_backend.convert( + SigmaCollection.from_yaml( + """ + title: Test + status: test + logsource: + category: test_category + product: test_product + detection: + sel: + "field|contains|cased": "va*lue" + condition: sel + """ + ) + ) + == ['field contains_cased "va*lue"'] + ) + + def test_convert_value_str_wildcard_to_regex(test_backend, monkeypatch): monkeypatch.setattr(test_backend, "wildcard_match_expression", '{field} match "{regex}"') assert (