Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Allow special characters in startswith, endswith, and contains expressions #292

Merged
merged 2 commits into from
Oct 14, 2024
Merged

Allow special characters in startswith, endswith, and contains expressions #292

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
20bc3f2
TextQueryBackend optionally allows special characters in startswith, …
thomaspatzke Oct 14, 2024
a81d690
Optionally allow special characters in cased startswith, endswith, an…
thomaspatzke Oct 14, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
39 changes: 29 additions & 10 deletions sigma/conversion/base.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -807,8 +807,11 @@ class variables. If this is not sufficient, the respective methods can be implem

# String matching operators. if none is appropriate eq_token is used.
startswith_expression: ClassVar[Optional[str]] = None
startswith_expression_allow_special: ClassVar[bool] = False
endswith_expression: ClassVar[Optional[str]] = None
endswith_expression_allow_special: ClassVar[bool] = False
contains_expression: ClassVar[Optional[str]] = None
contains_expression_allow_special: ClassVar[bool] = False
wildcard_match_expression: ClassVar[Optional[str]] = (
None # Special expression if wildcards can't be matched with the eq_token operator.
)
Expand Down Expand Up @@ -839,8 +842,11 @@ class variables. If this is not sufficient, the respective methods can be implem
# Case sensitive string matching operators similar to standard string matching. If not provided,
# case_sensitive_match_expression is used.
case_sensitive_startswith_expression: ClassVar[Optional[str]] = None
case_sensitive_startswith_expression_allow_special: ClassVar[bool] = False
case_sensitive_endswith_expression: ClassVar[Optional[str]] = None
case_sensitive_endswith_expression_allow_special: ClassVar[bool] = False
case_sensitive_contains_expression: ClassVar[Optional[str]] = None
case_sensitive_contains_expression_allow_special: ClassVar[bool] = False

# CIDR expressions: define CIDR matching if backend has native support. Else pySigma expands
# CIDR values into string wildcard matches.
Expand Down Expand Up @@ -1309,9 +1315,10 @@ def convert_condition_field_eq_val_str(
self.startswith_expression
is not None # 'startswith' operator is defined in backend
and cond.value.endswith(SpecialChars.WILDCARD_MULTI) # String ends with wildcard
and not cond.value[
:-1
].contains_special() # Remainder of string doesn't contains special characters
and (
self.startswith_expression_allow_special
or not cond.value[:-1].contains_special()
) # Remainder of string doesn't contains special characters or it's allowed
):
expr = (
self.startswith_expression
Expand All @@ -1320,15 +1327,20 @@ def convert_condition_field_eq_val_str(
elif ( # Same as above but for 'endswith' operator: string starts with wildcard and doesn't contains further special characters
self.endswith_expression is not None
and cond.value.startswith(SpecialChars.WILDCARD_MULTI)
and not cond.value[1:].contains_special()
and (
self.endswith_expression_allow_special or not cond.value[1:].contains_special()
)
):
expr = self.endswith_expression
value = cond.value[1:]
elif ( # contains: string starts and ends with wildcard
self.contains_expression is not None
and cond.value.startswith(SpecialChars.WILDCARD_MULTI)
and cond.value.endswith(SpecialChars.WILDCARD_MULTI)
and not cond.value[1:-1].contains_special()
and (
self.contains_expression_allow_special
or not cond.value[1:-1].contains_special()
)
):
expr = self.contains_expression
value = cond.value[1:-1]
Expand Down Expand Up @@ -1360,9 +1372,10 @@ def convert_condition_field_eq_val_str_case_sensitive(
self.case_sensitive_startswith_expression
is not None # 'startswith' operator is defined in backend
and cond.value.endswith(SpecialChars.WILDCARD_MULTI) # String ends with wildcard
and not cond.value[
:-1
].contains_special() # Remainder of string doesn't contains special characters
and (
self.case_sensitive_startswith_expression_allow_special
or not cond.value[:-1].contains_special()
) # Remainder of string doesn't contains special characters or it's allowed
):
expr = (
self.case_sensitive_startswith_expression
Expand All @@ -1371,15 +1384,21 @@ def convert_condition_field_eq_val_str_case_sensitive(
elif ( # Same as above but for 'endswith' operator: string starts with wildcard and doesn't contains further special characters
self.case_sensitive_endswith_expression is not None
and cond.value.startswith(SpecialChars.WILDCARD_MULTI)
and not cond.value[1:].contains_special()
and (
self.case_sensitive_endswith_expression_allow_special
or not cond.value[1:].contains_special()
)
):
expr = self.case_sensitive_endswith_expression
value = cond.value[1:]
elif ( # contains: string starts and ends with wildcard
self.case_sensitive_contains_expression is not None
and cond.value.startswith(SpecialChars.WILDCARD_MULTI)
and cond.value.endswith(SpecialChars.WILDCARD_MULTI)
and not cond.value[1:-1].contains_special()
and (
self.case_sensitive_contains_expression_allow_special
or not cond.value[1:-1].contains_special()
)
):
expr = self.case_sensitive_contains_expression
value = cond.value[1:-1]
Expand Down
197 changes: 197 additions & 0 deletions tests/test_conversion_base.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -328,6 +328,72 @@ def test_convert_value_str_startswith_further_wildcard(test_backend):
)


def test_convert_value_str_startswith_further_wildcard_allowed(test_backend, monkeypatch):
monkeypatch.setattr(test_backend, "startswith_expression_allow_special", True)
assert (
test_backend.convert(
SigmaCollection.from_yaml(
"""
title: Test
status: test
logsource:
category: test_category
product: test_product
detection:
sel:
fieldA|startswith: "va*lue"
field A|startswith: "va*lue"
condition: sel
"""
)
)
== ['mappedA startswith "va*lue" and \'field A\' startswith "va*lue"']
)


def test_convert_value_str_startswith_cased_further_wildcard(test_backend):
assert (
test_backend.convert(
SigmaCollection.from_yaml(
"""
title: Test
status: test
logsource:
category: test_category
product: test_product
detection:
sel:
"field|startswith|cased": "va*lue"
condition: sel
"""
)
)
== ['field casematch "va*lue*"']
)


def test_convert_value_str_startswith_cased_further_wildcard_allowed(test_backend, monkeypatch):
monkeypatch.setattr(test_backend, "case_sensitive_startswith_expression_allow_special", True)
assert (
test_backend.convert(
SigmaCollection.from_yaml(
"""
title: Test
status: test
logsource:
category: test_category
product: test_product
detection:
sel:
"field|startswith|cased": "va*lue"
condition: sel
"""
)
)
== ['field startswith_cased "va*lue"']
)


def test_convert_value_str_startswith_expression_not_defined(test_backend, monkeypatch):
monkeypatch.setattr(test_backend, "startswith_expression", None)
assert (
Expand Down Expand Up @@ -416,6 +482,72 @@ def test_convert_value_str_endswith_further_wildcard(test_backend):
)


def test_convert_value_str_endswith_further_wildcard_allowed(test_backend, monkeypatch):
monkeypatch.setattr(test_backend, "endswith_expression_allow_special", True)
assert (
test_backend.convert(
SigmaCollection.from_yaml(
"""
title: Test
status: test
logsource:
category: test_category
product: test_product
detection:
sel:
fieldA|endswith: "va*lue"
field A|endswith: "va*lue"
condition: sel
"""
)
)
== ['mappedA endswith "va*lue" and \'field A\' endswith "va*lue"']
)


def test_convert_value_str_endswith_cased_further_wildcard(test_backend):
assert (
test_backend.convert(
SigmaCollection.from_yaml(
"""
title: Test
status: test
logsource:
category: test_category
product: test_product
detection:
sel:
"field|endswith|cased": "va*lue"
condition: sel
"""
)
)
== ['field casematch "*va*lue"']
)


def test_convert_value_str_endswith_cased_further_wildcard_allowed(test_backend, monkeypatch):
monkeypatch.setattr(test_backend, "case_sensitive_endswith_expression_allow_special", True)
assert (
test_backend.convert(
SigmaCollection.from_yaml(
"""
title: Test
status: test
logsource:
category: test_category
product: test_product
detection:
sel:
"field|endswith|cased": "va*lue"
condition: sel
"""
)
)
== ['field endswith_cased "va*lue"']
)


def test_convert_value_str_endswith_expression_not_defined(test_backend, monkeypatch):
monkeypatch.setattr(test_backend, "endswith_expression", None)
assert (
Expand Down Expand Up @@ -503,6 +635,71 @@ def test_convert_value_str_contains_further_wildcard(test_backend):
)


def test_convert_value_str_contains_further_wildcard_allowed(test_backend, monkeypatch):
monkeypatch.setattr(test_backend, "contains_expression_allow_special", True)
assert (
test_backend.convert(
SigmaCollection.from_yaml(
"""
title: Test
status: test
logsource:
category: test_category
product: test_product
detection:
sel:
fieldA|contains: "va*lue"
condition: sel
"""
)
)
== ['mappedA contains "va*lue"']
)


def test_convert_value_str_contains_cased_further_wildcard(test_backend):
assert (
test_backend.convert(
SigmaCollection.from_yaml(
"""
title: Test
status: test
logsource:
category: test_category
product: test_product
detection:
sel:
"field|contains|cased": "va*lue"
condition: sel
"""
)
)
== ['field casematch "*va*lue*"']
)


def test_convert_value_str_contains_cased_further_wildcard_allowed(test_backend, monkeypatch):
monkeypatch.setattr(test_backend, "case_sensitive_contains_expression_allow_special", True)
assert (
test_backend.convert(
SigmaCollection.from_yaml(
"""
title: Test
status: test
logsource:
category: test_category
product: test_product
detection:
sel:
"field|contains|cased": "va*lue"
condition: sel
"""
)
)
== ['field contains_cased "va*lue"']
)


def test_convert_value_str_wildcard_to_regex(test_backend, monkeypatch):
monkeypatch.setattr(test_backend, "wildcard_match_expression", '{field} match "{regex}"')
assert (
Expand Down
Loading