From 5ebdfc43f1e109a58cdced2ba10496e336899a21 Mon Sep 17 00:00:00 2001 From: Casey McGinley Date: Fri, 13 Sep 2024 12:03:14 -0700 Subject: [PATCH 01/17] typing and linting fixes --- .../DetectionTestingManager.py | 2 +- contentctl/actions/test.py | 62 +++++++++---------- 2 files changed, 31 insertions(+), 33 deletions(-) diff --git a/contentctl/actions/detection_testing/DetectionTestingManager.py b/contentctl/actions/detection_testing/DetectionTestingManager.py index 5ad5e117..bb02d74c 100644 --- a/contentctl/actions/detection_testing/DetectionTestingManager.py +++ b/contentctl/actions/detection_testing/DetectionTestingManager.py @@ -28,7 +28,7 @@ @dataclass(frozen=False) class DetectionTestingManagerInputDto: - config: Union[test,test_servers] + config: Union[test, test_servers] detections: List[Detection] views: list[DetectionTestingView] diff --git a/contentctl/actions/test.py b/contentctl/actions/test.py index 716ecd71..9a5ec510 100644 --- a/contentctl/actions/test.py +++ b/contentctl/actions/test.py @@ -1,38 +1,29 @@ from dataclasses import dataclass from typing import List +import pathlib -from contentctl.objects.config import test_common -from contentctl.objects.enums import DetectionTestingMode, DetectionStatus, AnalyticsType +from contentctl.objects.config import test_servers +from contentctl.objects.config import test as test_ +from contentctl.objects.enums import DetectionTestingMode from contentctl.objects.detection import Detection - -from contentctl.input.director import DirectorOutputDto - from contentctl.actions.detection_testing.DetectionTestingManager import ( DetectionTestingManager, DetectionTestingManagerInputDto, ) - - from contentctl.actions.detection_testing.infrastructures.DetectionTestingInfrastructure import ( DetectionTestingManagerOutputDto, ) - - from contentctl.actions.detection_testing.views.DetectionTestingViewWeb import ( DetectionTestingViewWeb, ) - from contentctl.actions.detection_testing.views.DetectionTestingViewCLI import ( DetectionTestingViewCLI, ) - from contentctl.actions.detection_testing.views.DetectionTestingViewFile import ( DetectionTestingViewFile, ) - from contentctl.objects.integration_test import IntegrationTest -import pathlib MAXIMUM_CONFIGURATION_TIME_SECONDS = 600 @@ -40,8 +31,8 @@ @dataclass(frozen=True) class TestInputDto: detections: List[Detection] - config: test_common - + config: test_ | test_servers + class Test: def filter_tests(self, input_dto: TestInputDto) -> None: @@ -52,7 +43,7 @@ def filter_tests(self, input_dto: TestInputDto) -> None: Args: input_dto (TestInputDto): A configuration of the test and all of the tests to be run. - """ + """ if not input_dto.config.enable_integration_testing: # Skip all integraiton tests if integration testing is not enabled: @@ -61,7 +52,6 @@ def filter_tests(self, input_dto: TestInputDto) -> None: if isinstance(test, IntegrationTest): test.skip("TEST SKIPPED: Skipping all integration tests") - def execute(self, input_dto: TestInputDto) -> bool: output_dto = DetectionTestingManagerOutputDto() @@ -92,7 +82,11 @@ def execute(self, input_dto: TestInputDto) -> bool: print(f"MODE: [{mode}] - Test [{len(input_dto.detections)}] detections") if mode in [DetectionTestingMode.changes.value, DetectionTestingMode.selected.value]: files_string = '\n- '.join( - [str(pathlib.Path(detection.file_path).relative_to(input_dto.config.path)) for detection in input_dto.detections] + [ + str( + pathlib.Path(detection.file_path).relative_to(input_dto.config.path) + ) for detection in input_dto.detections + ] ) print(f"Detections:\n- {files_string}") @@ -102,44 +96,48 @@ def execute(self, input_dto: TestInputDto) -> bool: try: summary_results = file.getSummaryObject() summary = summary_results.get("summary", {}) + if not isinstance(summary, dict): + raise ValueError( + f"Summary in results was an unexpected type ({type(summary)}): {summary}" + ) - print(f"Test Summary (mode: {summary.get('mode','Error')})") - print(f"\tSuccess : {summary.get('success',False)}") + print(f"Test Summary (mode: {summary.get('mode', 'Error')})") + print(f"\tSuccess : {summary.get('success', False)}") print( - f"\tSuccess Rate : {summary.get('success_rate','ERROR')}" + f"\tSuccess Rate : {summary.get('success_rate', 'ERROR')}" ) print( - f"\tTotal Detections : {summary.get('total_detections','ERROR')}" + f"\tTotal Detections : {summary.get('total_detections', 'ERROR')}" ) print( - f"\tTotal Tested Detections : {summary.get('total_tested_detections','ERROR')}" + f"\tTotal Tested Detections : {summary.get('total_tested_detections', 'ERROR')}" ) print( - f"\t Passed Detections : {summary.get('total_pass','ERROR')}" + f"\t Passed Detections : {summary.get('total_pass', 'ERROR')}" ) print( - f"\t Failed Detections : {summary.get('total_fail','ERROR')}" + f"\t Failed Detections : {summary.get('total_fail', 'ERROR')}" ) print( - f"\tSkipped Detections : {summary.get('total_skipped','ERROR')}" + f"\tSkipped Detections : {summary.get('total_skipped', 'ERROR')}" ) print( "\tProduction Status :" ) print( - f"\t Production Detections : {summary.get('total_production','ERROR')}" + f"\t Production Detections : {summary.get('total_production', 'ERROR')}" ) print( - f"\t Experimental Detections : {summary.get('total_experimental','ERROR')}" + f"\t Experimental Detections : {summary.get('total_experimental', 'ERROR')}" ) print( - f"\t Deprecated Detections : {summary.get('total_deprecated','ERROR')}" + f"\t Deprecated Detections : {summary.get('total_deprecated', 'ERROR')}" ) print( - f"\tManually Tested Detections : {summary.get('total_manual','ERROR')}" + f"\tManually Tested Detections : {summary.get('total_manual', 'ERROR')}" ) print( - f"\tUntested Detections : {summary.get('total_untested','ERROR')}" + f"\tUntested Detections : {summary.get('total_untested', 'ERROR')}" ) print(f"\tTest Results File : {file.getOutputFilePath()}") print( @@ -147,7 +145,7 @@ def execute(self, input_dto: TestInputDto) -> bool: "detection types (e.g. Correlation), but there may be overlap between these\n" "categories." ) - return summary_results.get("summary", {}).get("success", False) + return summary.get("success", False) except Exception as e: print(f"Error determining if whole test was successful: {str(e)}") From 336cd7b5fb96766bd3dcd52ba8e791210fac176e Mon Sep 17 00:00:00 2001 From: Casey McGinley Date: Mon, 7 Oct 2024 20:43:57 -0700 Subject: [PATCH 02/17] initial commit; adding validation against CMS index --- .../DetectionTestingManager.py | 2 +- .../DetectionTestingInfrastructure.py | 46 +- .../views/DetectionTestingViewCLI.py | 2 + .../objects/content_versioning_service.py | 473 ++++++++++++++++++ contentctl/objects/correlation_search.py | 10 +- contentctl/output/conf_output.py | 4 +- 6 files changed, 515 insertions(+), 22 deletions(-) create mode 100644 contentctl/objects/content_versioning_service.py diff --git a/contentctl/actions/detection_testing/DetectionTestingManager.py b/contentctl/actions/detection_testing/DetectionTestingManager.py index bb02d74c..4f04b202 100644 --- a/contentctl/actions/detection_testing/DetectionTestingManager.py +++ b/contentctl/actions/detection_testing/DetectionTestingManager.py @@ -90,7 +90,7 @@ def sigint_handler(signum, frame): result = future.result() except Exception as e: self.output_dto.terminate = True - print(f"Error setting up container: {str(e)}") + print(f"Error setting up instance: {str(e)}") # Start and wait for all tests to run if not self.output_dto.terminate: diff --git a/contentctl/actions/detection_testing/infrastructures/DetectionTestingInfrastructure.py b/contentctl/actions/detection_testing/infrastructures/DetectionTestingInfrastructure.py index ca28dce8..2841b3a5 100644 --- a/contentctl/actions/detection_testing/infrastructures/DetectionTestingInfrastructure.py +++ b/contentctl/actions/detection_testing/infrastructures/DetectionTestingInfrastructure.py @@ -11,14 +11,13 @@ from ssl import SSLEOFError, SSLZeroReturnError from sys import stdout from shutil import copyfile -from typing import Union, Optional +from typing import Union, Optional, Callable -from pydantic import BaseModel, PrivateAttr, Field, dataclasses +from pydantic import BaseModel, PrivateAttr, Field, dataclasses, computed_field import requests # type: ignore import splunklib.client as client # type: ignore from splunklib.binding import HTTPError # type: ignore from splunklib.results import JSONResultsReader, Message # type: ignore -import splunklib.results from urllib3 import disable_warnings import urllib.parse @@ -34,6 +33,7 @@ from contentctl.objects.test_group import TestGroup from contentctl.objects.base_test_result import TestResultStatus from contentctl.objects.correlation_search import CorrelationSearch, PbarData +from contentctl.objects.content_versioning_service import ContentVersioningService from contentctl.helper.utils import Utils from contentctl.actions.detection_testing.progress_bar import ( format_pbar_string, @@ -60,6 +60,8 @@ class CleanupTestGroupResults(BaseModel): class ContainerStoppedException(Exception): pass + + class CannotRunBaselineException(Exception): # Support for testing detections with baselines # does not currently exist in contentctl. @@ -125,18 +127,19 @@ def setup(self): ) self.start_time = time.time() + setup_functions: list[tuple[Callable[[], None | client.Service], str]] = [ + (self.start, "Starting"), + (self.get_conn, "Waiting for App Installation"), + (self.configure_conf_file_datamodels, "Configuring Datamodels"), + (self.create_replay_index, f"Create index '{self.sync_obj.replay_index}'"), + (self.configure_imported_roles, "Configuring Roles"), + (self.configure_delete_indexes, "Configuring Indexes"), + (self.configure_hec, "Configuring HEC"), + ] + setup_functions = setup_functions + self.content_versioning_service.setup_functions + setup_functions.append((self.wait_for_ui_ready, "Finishing Setup")) try: - for func, msg in [ - (self.start, "Starting"), - (self.get_conn, "Waiting for App Installation"), - (self.configure_conf_file_datamodels, "Configuring Datamodels"), - (self.create_replay_index, f"Create index '{self.sync_obj.replay_index}'"), - (self.configure_imported_roles, "Configuring Roles"), - (self.configure_delete_indexes, "Configuring Indexes"), - (self.configure_hec, "Configuring HEC"), - (self.wait_for_ui_ready, "Finishing Setup") - ]: - + for func, msg in setup_functions: self.format_pbar_string( TestReportingType.SETUP, self.get_name(), @@ -149,13 +152,23 @@ def setup(self): except Exception as e: self.pbar.write(str(e)) self.finish() - return + raise self.format_pbar_string(TestReportingType.SETUP, self.get_name(), "Finished Setup!") def wait_for_ui_ready(self): self.get_conn() + @computed_field + @property + def content_versioning_service(self) -> ContentVersioningService: + return ContentVersioningService( + global_config=self.global_config, + infrastructure=self.infrastructure, + service=self.get_conn(), + detections=self.sync_obj.inputQueue + ) + def configure_hec(self): self.hec_channel = str(uuid.uuid4()) try: @@ -1198,7 +1211,7 @@ def delete_attack_data(self, attack_data_files: list[TestAttackData]): job = self.get_conn().jobs.create(splunk_search, **kwargs) results_stream = job.results(output_mode="json") # TODO: should we be doing something w/ this reader? - _ = splunklib.results.JSONResultsReader(results_stream) + _ = JSONResultsReader(results_stream) except Exception as e: raise ( @@ -1393,6 +1406,7 @@ def hec_raw_replay( def status(self): pass + # TODO (cmcginley): the finish function doesn't actually stop execution def finish(self): self.pbar.bar_format = f"Finished running tests on instance: [{self.get_name()}]" self.pbar.update() diff --git a/contentctl/actions/detection_testing/views/DetectionTestingViewCLI.py b/contentctl/actions/detection_testing/views/DetectionTestingViewCLI.py index 1675fce4..a5ec8a24 100644 --- a/contentctl/actions/detection_testing/views/DetectionTestingViewCLI.py +++ b/contentctl/actions/detection_testing/views/DetectionTestingViewCLI.py @@ -51,6 +51,8 @@ def showStatus(self, interval: int = 1): while True: summary = self.getSummaryObject() + # TODO (cmcginley): there's a 1-off error here I think (we show one more than we + # actually have during testing) total = len( summary.get("tested_detections", []) + summary.get("untested_detections", []) diff --git a/contentctl/objects/content_versioning_service.py b/contentctl/objects/content_versioning_service.py new file mode 100644 index 00000000..e752d081 --- /dev/null +++ b/contentctl/objects/content_versioning_service.py @@ -0,0 +1,473 @@ +import time +import uuid +import json +import re +from typing import Any, Callable +from functools import cached_property + +from pydantic import BaseModel, PrivateAttr, computed_field +import splunklib.client as splunklib # type: ignore +from splunklib.binding import HTTPError, ResponseReader # type: ignore +from splunklib.data import Record # type: ignore + +from contentctl.objects.config import test_common, Infrastructure +from contentctl.objects.detection import Detection +from contentctl.objects.correlation_search import ResultIterator, get_logger + +# TODO (cmcginley): remove this logger +logger = get_logger() + + +class ContentVersioningService(BaseModel): + global_config: test_common + infrastructure: Infrastructure + service: splunklib.Service + detections: list[Detection] + + _cms_main_job: splunklib.Job | None = PrivateAttr(default=None) + + class Config: + arbitrary_types_allowed = True + + @computed_field + @property + def setup_functions(self) -> list[tuple[Callable[[], None], str]]: + return [ + (self.activate_versioning, "Activating Content Versioning"), + (self.wait_for_cms_main, "Waiting for CMS Parser"), + (self.validate_content_against_cms, "Validating Against CMS"), + ] + + def _query_content_versioning_service(self, method: str, body: dict[str, Any] = {}) -> Record: + """ + Queries the SA-ContentVersioning service. Output mode defaults to JSON. + + :param method: HTTP request method (e.g. GET) + :type method: str + :param body: the payload/data/body of the request + :type body: dict[str, Any] + + :returns: a splunklib Record object (wrapper around dict) indicating the response + :rtype: :class:`splunklib.data.Record` + """ + # Add output mode to body + if "output_mode" not in body: + body["output_mode"] = "json" + + # Query the content versioning service + try: + response = self.service.request( # type: ignore + method=method, + path_segment="configs/conf-feature_flags/general", + body=body, + app="SA-ContentVersioning" + ) + except HTTPError as e: + # Raise on any HTTP errors + raise HTTPError( + f"Error querying content versioning service: {e}" + ) from e + + return response + + @property + def is_versioning_activated(self) -> bool: + """ + Indicates whether the versioning service is activated or not + + :returns: a bool indicating if content versioning is activated or not + :rtype: bool + """ + # Query the SA-ContentVersioning service for versioning status + response = self._query_content_versioning_service(method="GET") + + # Grab the response body and check for errors + if "body" not in response: + raise KeyError( + f"Cannot retrieve versioning status, 'body' was not found in JSON response: {response}" + ) + body: Any = response["body"] # type: ignore + if not isinstance(body, ResponseReader): + raise ValueError( + "Cannot retrieve versioning status, value at 'body' in JSON response had an unexpected" + f" type: expected '{ResponseReader}', received '{type(body)}'" + ) + + # Read the JSON and parse it into a dictionary + json_ = body.readall() + try: + data = json.loads(json_) + except json.JSONDecodeError as e: + raise ValueError(f"Unable to parse response body as JSON: {e}") from e + + # Find the versioning_activated field and report any errors + try: + for entry in data["entry"]: + if entry["name"] == "general": + return bool(int(entry["content"]["versioning_activated"])) + except KeyError as e: + raise KeyError( + "Cannot retrieve versioning status, unable to versioning status using the expected " + f"keys: {e}" + ) from e + raise ValueError( + "Cannot retrieve versioning status, unable to find an entry matching 'general' in the " + "response." + ) + + def activate_versioning(self) -> None: + """ + Activate the content versioning service + """ + # TODO (cmcginley): add conditional logging s.t. this check only happens when integration + # testing is enabled AND ES is at least version 8.0, AND mode is `all` + # Post to the SA-ContentVersioning service to set versioning status + self._query_content_versioning_service( + method="POST", + body={ + "versioning_activated": True + } + ) + + # Confirm versioning has been enabled + if not self.is_versioning_activated: + raise Exception("Something went wrong, content versioning is still disabled.") + + @computed_field + @cached_property + def cms_fields(self) -> list[str]: + """ + Property listing the fields we want to pull from the cms_main index + + :returns: a list of strings, the fields we want + :rtype: list[str] + """ + return [ + "app_name", + f"action.{self.global_config.app.label.lower()}.full_search_name", + "detection_id", + "version", + "action.correlationsearch.label", + "sourcetype" + ] + + @property + def is_cms_parser_enabled(self) -> bool: + """ + Indicates whether the cms_parser mod input is enabled or not. + + :returns: a bool indicating if cms_parser mod input is activated or not + :rtype: bool + """ + # Get the data input entity + cms_parser = self.service.input("data/inputs/cms_parser/main") # type: ignore + + # Convert the 'disabled' field to an int, then a bool, and then invert to be 'enabled' + return not bool(int(cms_parser.content["disabled"])) # type: ignore + + def force_cms_parser(self) -> None: + """ + Force the cms_parser to being it's run being disabling and re-enabling it. + """ + # Get the data input entity + cms_parser = self.service.input("data/inputs/cms_parser/main") # type: ignore + + # Disable and re-enable + cms_parser.disable() + cms_parser.enable() + + # Confirm the cms_parser is enabled + if not self.is_cms_parser_enabled: + raise Exception("Something went wrong, cms_parser is still disabled.") + + def wait_for_cms_main(self) -> None: + """ + Checks the cms_main index until it has the expected number of events, or it times out. + """ + # Force the cms_parser to start parsing our savedsearches.conf + self.force_cms_parser() + + # Set counters and limits for out exp. backoff timer + elapsed_sleep_time = 0 + num_tries = 0 + time_to_sleep = 2**num_tries + max_sleep = 300 + + # Loop until timeout + while elapsed_sleep_time < max_sleep: + # Sleep, and add the time to the elapsed counter + logger.info(f"Waiting {time_to_sleep} for cms_parser to finish") + time.sleep(time_to_sleep) + elapsed_sleep_time += time_to_sleep + logger.info( + f"Checking cms_main (attempt #{num_tries + 1} - {elapsed_sleep_time} seconds elapsed of " + f"{max_sleep} max)" + ) + + # Check if the number of CMS events matches or exceeds the number of detections + if self.get_num_cms_events() >= len(self.detections): + logger.info( + f"Found {self.get_num_cms_events(use_cache=True)} events in cms_main which " + f"meets or exceeds the expected {len(self.detections)}." + ) + break + + # Update the number of times we've tried, and increment the time to sleep + num_tries += 1 + time_to_sleep = 2**num_tries + + # If the computed time to sleep will exceed max_sleep, adjust appropriately + if (elapsed_sleep_time + time_to_sleep) > max_sleep: + time_to_sleep = max_sleep - elapsed_sleep_time + + def _query_cms_main(self, use_cache: bool = False) -> splunklib.Job: + """ + Queries the cms_main index, optionally appending the provided query suffix. + + :param use_cache: a flag indicating whether the cached job should be returned + :type use_cache: bool + + :returns: a search Job entity + :rtype: :class:`splunklib.client.Job` + """ + # Use the cached job if asked to do so + if use_cache: + if self._cms_main_job is not None: + return self._cms_main_job + raise Exception( + "Attempting to return a cached job against the cms_main index, but no job has been" + " cached yet." + ) + + # Construct the query looking for CMS events matching the content app name + query = ( + f"search index=cms_main app_name=\"{self.global_config.app.appid}\" | " + f"fields {', '.join(self.cms_fields)}" + ) + logger.debug(f"Query on cms_main: {query}") + + # Get the job as a blocking operation, set the cache, and return + self._cms_main_job = self.service.search(query, exec_mode="blocking") # type: ignore + return self._cms_main_job + + def get_num_cms_events(self, use_cache: bool = False) -> int: + """ + Gets the number of matching events in the cms_main index + + :param use_cache: a flag indicating whether the cached job should be returned + :type use_cache: bool + + :returns: the count of matching events + :rtype: int + """ + # Query the cms_main index + job = self._query_cms_main(use_cache=use_cache) + + # Convert the result count to an int + return int(job["resultCount"]) + + def validate_content_against_cms(self) -> None: + """ + Using the cms_main index, validate content against the index to ensure our + savedsearches.conf is compatible with ES content versioning features. **NOTE**: while in + the future, this function may validate more types of content, currently, we only validate + detections against the cms_main index. + """ + # Get the cached job and result count + result_count = self.get_num_cms_events(use_cache=True) + job = self._query_cms_main(use_cache=True) + + # Create a running list of validation errors + exceptions: list[Exception] = [] + + # Generate an error for the count mismatch + if result_count != len(self.detections): + msg = ( + f"Expected {len(self.detections)} matching events in cms_main, but " + f"found {result_count}." + ) + logger.error(msg) + exceptions.append(Exception(msg)) + logger.info( + f"Expecting {len(self.detections)} matching events in cms_main, " + f"found {result_count}." + ) + + # Init some counters and a mapping of detections to their names + count = 100 + offset = 0 + remaining_detections = {x.name: x for x in self.detections} + matched_detections: dict[str, Detection] = {} + + # Iterate over the results until we've gone through them all + while offset < result_count: + iterator = ResultIterator( + job.results( # type: ignore + output_mode="json", + count=count, + offset=offset + ) + ) + + # Iterate over the currently fetched results + for cms_event in iterator: + # Increment the offset for each result + offset += 1 + + # Get the name of the search in the CMS event and attempt to use pattern matching + # to strip the prefix and suffix used for the savedsearches.conf name so we can + # compare to the detection + cms_entry_name = cms_event["sourcetype"] + logger.info(f"{offset}: Matching cms_main entry '{cms_entry_name}' against detections") + ptrn = re.compile(r"^" + self.global_config.app.label + r" - (?P.+) - Rule$") + match = ptrn.match(cms_event["sourcetype"]) + + # Report any errors extracting the detection name from the longer rule name + if match is None: + msg = ( + f"Entry in cms_main ('{cms_entry_name}') did not match the expected naming " + "scheme; cannot compare to our detections." + ) + logger.error(msg) + exceptions.append(Exception(msg)) + continue + + # Extract the detection name if matching was successful + cms_entry_name = match.group("cms_entry_name") + + # If CMS entry name matches one of the detections already matched, we've got an + # unexpected repeated entry + if cms_entry_name in matched_detections: + msg = ( + f"Detection '{cms_entry_name}' appears more than once in the cms_main " + "index." + ) + logger.error(msg) + exceptions.append(Exception(msg)) + continue + + # Iterate over the detections and compare the CMS entry name against each + result_matches_detection = False + for detection_name in remaining_detections: + # If we find a match, break this loop, set the found flag and move the detection + # from those that still need to matched to those already matched + if cms_entry_name == detection_name: + logger.info( + f"{offset}: Succesfully matched cms_main entry against detection " + f"('{detection_name}')!" + ) + + # Validate other fields of the cms_event against the detection + exception = self.validate_detection_against_cms_event( + cms_event, + remaining_detections[detection_name] + ) + + # Save the exception if validation failed + if exception is not None: + exceptions.append(exception) + + # Delete the matched detection and move it to the matched list + result_matches_detection = True + matched_detections[detection_name] = remaining_detections[detection_name] + del remaining_detections[detection_name] + break + + # Generate an exception if we couldn't match the CMS main entry to a detection + if result_matches_detection is False: + msg = ( + f"Could not match entry in cms_main for ('{cms_entry_name}') against any " + "of the expected detections." + ) + logger.error(msg) + exceptions.append(Exception(msg)) + + # If we have any remaining detections, they could not be matched against an entry in + # cms_main and there may have been a parsing issue with savedsearches.conf + if len(remaining_detections) > 0: + # Generate exceptions for the unmatched detections + for detection_name in remaining_detections: + msg = ( + f"Detection '{detection_name}' not found in cms_main; there may be an " + "issue with savedsearches.conf" + ) + logger.error(msg) + exceptions.append(Exception(msg)) + + # Raise exceptions as a group + if len(exceptions) > 0: + raise ExceptionGroup( + "1 or more issues validating our detections against the cms_main index", + exceptions + ) + + # Else, we've matched/validated all detections against cms_main + logger.info("Matched and validated all detections against cms_main!") + + def validate_detection_against_cms_event( + self, + cms_event: dict[str, Any], + detection: Detection + ) -> Exception | None: + """ + Given an event from the cms_main index and the matched detection, compare fields and look + for any inconsistencies + + :param cms_event: The event from the cms_main index + :type cms_event: dict[str, Any] + :param detection: The matched detection + :type detection: :class:`contentctl.objects.detection.Detection` + + :return: The generated exception, or None + :rtype: Exception | None + """ + # TODO (cmcginley): validate additional fields between the cms_event and the detection + + cms_uuid = uuid.UUID(cms_event["detection_id"]) + full_search_key = f"action.{self.global_config.app.label.lower()}.full_search_name" + rule_name_from_detection = f"{self.global_config.app.label} - {detection.name} - Rule" + + # Compare the UUIDs + if cms_uuid != detection.id: + msg = ( + f"UUID in cms_event ('{cms_uuid}') does not match UUID in detection " + f"('{detection.id}'): {detection.name}" + ) + logger.error(msg) + return Exception(msg) + elif cms_event["version"] != f"{detection.version}-1": + # Compare the versions (we append '-1' to the detection version to be in line w/ the + # internal representation in ES) + msg = ( + f"Version in cms_event ('{cms_event['version']}') does not match version in " + f"detection ('{detection.version}-1'): {detection.name}" + ) + logger.error(msg) + return Exception(msg) + elif cms_event[full_search_key] != rule_name_from_detection: + # Compare the full search name + msg = ( + f"Full search name in cms_event ('{cms_event[full_search_key]}') " + f"does not match detection name ('{detection.name}')" + ) + logger.error(msg) + return Exception(msg) + elif cms_event["action.correlationsearch.label"] != f"{self.global_config.app.label} - {detection.name} - Rule": + # Compare the correlation search label + msg = ( + f"Correlation search label in cms_event " + f"('{cms_event['action.correlationsearch.label']}') does not match detection name " + f"('{detection.name}')" + ) + logger.error(msg) + return Exception(msg) + elif cms_event["sourcetype"] != f"{self.global_config.app.label} - {detection.name} - Rule": + # Compare the full search name + msg = ( + f"Sourcetype in cms_event ('{cms_event[f'sourcetype']}') does not match detection " + f"name ('{detection.name}')" + ) + logger.error(msg) + return Exception(msg) + + return None diff --git a/contentctl/objects/correlation_search.py b/contentctl/objects/correlation_search.py index a0b25da9..173e7d1b 100644 --- a/contentctl/objects/correlation_search.py +++ b/contentctl/objects/correlation_search.py @@ -31,8 +31,9 @@ from contentctl.objects.observable import Observable +# TODO (cmcginley): disable logging # Suppress logging by default; enable for local testing -ENABLE_LOGGING = False +ENABLE_LOGGING = True LOG_LEVEL = logging.DEBUG LOG_PATH = "correlation_search.log" @@ -144,13 +145,14 @@ def __init__(self, response_reader: ResponseReader) -> None: def __iter__(self) -> "ResultIterator": return self - def __next__(self) -> dict: + def __next__(self) -> dict[str, Any]: # Use a reader for JSON format so we can iterate over our results for result in self.results_reader: # log messages, or raise if error if isinstance(result, Message): # convert level string to level int - level_name = result.type.strip().upper() + level_name: str = result.type.strip().upper() # type: ignore + # TODO (cmcginley): this method is deprecated; replace with our own enum level: int = logging.getLevelName(level_name) # log message at appropriate level and raise if needed @@ -161,7 +163,7 @@ def __next__(self) -> dict: # if dict, just return elif isinstance(result, dict): - return result + return result # type: ignore # raise for any unexpected types else: diff --git a/contentctl/output/conf_output.py b/contentctl/output/conf_output.py index cf4574ae..7132ef0e 100644 --- a/contentctl/output/conf_output.py +++ b/contentctl/output/conf_output.py @@ -72,7 +72,9 @@ def writeAppConf(self)->set[pathlib.Path]: [self.config.app])) return written_files - + # TODO (cmcginley): we could have a discrepancy between detections tested and those delivered + # based on the jinja2 template + # {% if (detection.type == 'TTP' or detection.type == 'Anomaly' or detection.type == 'Hunting' or detection.type == 'Correlation') %} def writeObjects(self, objects: list, type: SecurityContentType = None) -> set[pathlib.Path]: written_files:set[pathlib.Path] = set() if type == SecurityContentType.detections: From 2c87187a40647a670df727404049554d6ca0e4e0 Mon Sep 17 00:00:00 2001 From: Casey McGinley Date: Tue, 8 Oct 2024 00:13:30 -0700 Subject: [PATCH 03/17] added logic for checking when to run cms testing --- .../DetectionTestingManager.py | 38 +++++-- .../DetectionTestingInfrastructure.py | 102 +++++++++++++++--- .../objects/content_versioning_service.py | 54 ++++++---- 3 files changed, 152 insertions(+), 42 deletions(-) diff --git a/contentctl/actions/detection_testing/DetectionTestingManager.py b/contentctl/actions/detection_testing/DetectionTestingManager.py index 4f04b202..a1d659ca 100644 --- a/contentctl/actions/detection_testing/DetectionTestingManager.py +++ b/contentctl/actions/detection_testing/DetectionTestingManager.py @@ -65,7 +65,7 @@ def sigint_handler(signum, frame): print("*******************************") signal.signal(signal.SIGINT, sigint_handler) - + with concurrent.futures.ThreadPoolExecutor( max_workers=len(self.input_dto.config.test_instances), ) as instance_pool, concurrent.futures.ThreadPoolExecutor( @@ -73,11 +73,19 @@ def sigint_handler(signum, frame): ) as view_runner, concurrent.futures.ThreadPoolExecutor( max_workers=len(self.input_dto.config.test_instances), ) as view_shutdowner: + # Capture any errors for reporting at the end after all threads have been gathered + errors: dict[str, list[Exception]] = { + "INSTANCE SETUP ERRORS": [], + "TESTING ERRORS": [], + "ERRORS DURING VIEW SHUTDOWN": [], + "ERRORS DURING VIEW EXECUTION": [], + } # Start all the views future_views = { view_runner.submit(view.setup): view for view in self.input_dto.views } + # Configure all the instances future_instances_setup = { instance_pool.submit(instance.setup): instance @@ -87,10 +95,10 @@ def sigint_handler(signum, frame): # Wait for all instances to be set up for future in concurrent.futures.as_completed(future_instances_setup): try: - result = future.result() + _ = future.result() except Exception as e: self.output_dto.terminate = True - print(f"Error setting up instance: {str(e)}") + errors["INSTANCE SETUP ERRORS"].append(e) # Start and wait for all tests to run if not self.output_dto.terminate: @@ -102,10 +110,10 @@ def sigint_handler(signum, frame): # Wait for execution to finish for future in concurrent.futures.as_completed(future_instances_execute): try: - result = future.result() + _ = future.result() except Exception as e: self.output_dto.terminate = True - print(f"Error running in container: {str(e)}") + errors["TESTING ERRORS"].append(e) self.output_dto.terminate = True @@ -115,16 +123,28 @@ def sigint_handler(signum, frame): } for future in concurrent.futures.as_completed(future_views_shutdowner): try: - result = future.result() + _ = future.result() except Exception as e: - print(f"Error stopping view: {str(e)}") + errors["ERRORS DURING VIEW SHUTDOWN"].append(e) # Wait for original view-related threads to complete for future in concurrent.futures.as_completed(future_views): try: - result = future.result() + _ = future.result() except Exception as e: - print(f"Error running container: {str(e)}") + errors["ERRORS DURING VIEW EXECUTION"].append(e) + + # Log any errors + for error_type in errors: + if len(errors[error_type]) > 0: + print() + print(f"[{error_type}]:") + for error in errors[error_type]: + print(f"\t❌ {str(error)}") + if isinstance(error, ExceptionGroup): + for suberror in error.exceptions: # type: ignore + print(f"\t\t❌ {str(suberror)}") # type: ignore + print() return self.output_dto diff --git a/contentctl/actions/detection_testing/infrastructures/DetectionTestingInfrastructure.py b/contentctl/actions/detection_testing/infrastructures/DetectionTestingInfrastructure.py index 2841b3a5..2757665a 100644 --- a/contentctl/actions/detection_testing/infrastructures/DetectionTestingInfrastructure.py +++ b/contentctl/actions/detection_testing/infrastructures/DetectionTestingInfrastructure.py @@ -20,8 +20,9 @@ from splunklib.results import JSONResultsReader, Message # type: ignore from urllib3 import disable_warnings import urllib.parse +from semantic_version import Version # type: ignore -from contentctl.objects.config import test_common, Infrastructure +from contentctl.objects.config import test_common, Infrastructure, All from contentctl.objects.enums import PostTestBehavior, AnalyticsType from contentctl.objects.detection import Detection from contentctl.objects.base_test import BaseTest @@ -42,6 +43,9 @@ TestingStates ) +# The app name of ES; needed to check ES version +ES_APP_NAME = "SplunkEnterpriseSecuritySuite" + class SetupTestGroupResults(BaseModel): exception: Union[Exception, None] = None @@ -127,17 +131,27 @@ def setup(self): ) self.start_time = time.time() + + # Init the list of setup functions we always need setup_functions: list[tuple[Callable[[], None | client.Service], str]] = [ (self.start, "Starting"), (self.get_conn, "Waiting for App Installation"), (self.configure_conf_file_datamodels, "Configuring Datamodels"), (self.create_replay_index, f"Create index '{self.sync_obj.replay_index}'"), + (self.check_for_es_install, "Checking for ES Install"), (self.configure_imported_roles, "Configuring Roles"), (self.configure_delete_indexes, "Configuring Indexes"), (self.configure_hec, "Configuring HEC"), ] - setup_functions = setup_functions + self.content_versioning_service.setup_functions + + # Add any setup functions only applicable to content versioning validation + if self.should_test_content_versioning: + setup_functions = setup_functions + self.content_versioning_service.setup_functions + + # Add the final setup function setup_functions.append((self.wait_for_ui_ready, "Finishing Setup")) + + # Execute and report on each setup function try: for func, msg in setup_functions: self.format_pbar_string( @@ -150,9 +164,11 @@ def setup(self): self.check_for_teardown() except Exception as e: - self.pbar.write(str(e)) + msg = f"[{self.get_name()}]: {str(e)}" self.finish() - raise + if isinstance(e, ExceptionGroup): + raise ExceptionGroup(msg, e.exceptions) from e # type: ignore + raise Exception(msg) from e self.format_pbar_string(TestReportingType.SETUP, self.get_name(), "Finished Setup!") @@ -162,6 +178,14 @@ def wait_for_ui_ready(self): @computed_field @property def content_versioning_service(self) -> ContentVersioningService: + """ + A computed field returning a handle to the content versioning service, used by ES to + version detections. We use this model to validate that all detections have been installed + compatibly with ES versioning. + + :return: a handle to the content versioning service on the instance + :rtype: :class:`contentctl.objects.content_versioning_service.ContentVersioningService` + """ return ContentVersioningService( global_config=self.global_config, infrastructure=self.infrastructure, @@ -169,6 +193,57 @@ def content_versioning_service(self) -> ContentVersioningService: detections=self.sync_obj.inputQueue ) + @property + def should_test_content_versioning(self) -> bool: + """ + Indicates whether we should test content versioning. Content versioning + should be tested when integration testing is enabled, the mode is all, and ES is at least + version 8.0.0. + + :return: a bool indicating whether we should test content versioning + :rtype: bool + """ + es_version = self.es_version + return ( + self.global_config.enable_integration_testing + and isinstance(self.global_config.mode, All) + and es_version is not None + and es_version >= Version("8.0.0") + ) + + @property + def es_version(self) -> Version | None: + """ + Returns the version of Enterprise Security installed on the instance; None if not installed. + + :return: the version of ES, as a semver aware object + :rtype: :class:`semantic_version.Version` + """ + if not self.es_installed: + return None + return Version(self.get_conn().apps[ES_APP_NAME]["version"]) # type: ignore + + @property + def es_installed(self) -> bool: + """ + Indicates whether ES is installed on the instance. + + :return: a bool indicating whether ES is installed or not + :rtype: bool + """ + return ES_APP_NAME in self.get_conn().apps + + def check_for_es_install(self) -> None: + """ + Validating function which raises an error if Enterprise Security is not installed and + integration testing is enabled. + """ + if not self.es_installed and self.global_config.enable_integration_testing: + raise Exception( + "Enterprise Security does not appear to be installed on this instance and " + "integration testing is enabled." + ) + def configure_hec(self): self.hec_channel = str(uuid.uuid4()) try: @@ -282,25 +357,22 @@ def configure_imported_roles( ): indexes.append(self.sync_obj.replay_index) indexes_encoded = ";".join(indexes) + + # Include ES roles if installed + if self.es_installed: + imported_roles = imported_roles + enterprise_security_roles try: self.get_conn().roles.post( self.infrastructure.splunk_app_username, - imported_roles=imported_roles + enterprise_security_roles, + imported_roles=imported_roles, srchIndexesAllowed=indexes_encoded, srchIndexesDefault=self.sync_obj.replay_index, ) return except Exception as e: - self.pbar.write( - f"Enterprise Security Roles do not exist:'{enterprise_security_roles}: {str(e)}" - ) - - self.get_conn().roles.post( - self.infrastructure.splunk_app_username, - imported_roles=imported_roles, - srchIndexesAllowed=indexes_encoded, - srchIndexesDefault=self.sync_obj.replay_index, - ) + msg = f"Error configuring roles: {str(e)}" + self.pbar.write(msg) + raise Exception(msg) from e def configure_delete_indexes(self, indexes: list[str] = ["_*", "*"]): indexes.append(self.sync_obj.replay_index) diff --git a/contentctl/objects/content_versioning_service.py b/contentctl/objects/content_versioning_service.py index e752d081..d6203acc 100644 --- a/contentctl/objects/content_versioning_service.py +++ b/contentctl/objects/content_versioning_service.py @@ -17,21 +17,42 @@ # TODO (cmcginley): remove this logger logger = get_logger() +# TODO (cmcginley): would it be better for this to only run on one instance? Or to consolidate +# error reporting at least? + class ContentVersioningService(BaseModel): + """ + A model representing the content versioning service used in ES 8.0.0+. This model can be used + to validate that detections have been installed in a way that is compatible with content + versioning. + """ + + # The global contentctl config global_config: test_common + + # The instance specific infra config infrastructure: Infrastructure + + # The splunklib service service: splunklib.Service + + # The list of detections detections: list[Detection] + # The cached job on the splunk instance of the cms events _cms_main_job: splunklib.Job | None = PrivateAttr(default=None) class Config: + # We need to allow arbitrary type for the splunklib service arbitrary_types_allowed = True @computed_field @property def setup_functions(self) -> list[tuple[Callable[[], None], str]]: + """ + Returns the list of setup functions needed for content versioning testing + """ return [ (self.activate_versioning, "Activating Content Versioning"), (self.wait_for_cms_main, "Waiting for CMS Parser"), @@ -56,7 +77,7 @@ def _query_content_versioning_service(self, method: str, body: dict[str, Any] = # Query the content versioning service try: - response = self.service.request( # type: ignore + response = self.service.request( # type: ignore method=method, path_segment="configs/conf-feature_flags/general", body=body, @@ -119,8 +140,6 @@ def activate_versioning(self) -> None: """ Activate the content versioning service """ - # TODO (cmcginley): add conditional logging s.t. this check only happens when integration - # testing is enabled AND ES is at least version 8.0, AND mode is `all` # Post to the SA-ContentVersioning service to set versioning status self._query_content_versioning_service( method="POST", @@ -325,7 +344,7 @@ def validate_content_against_cms(self) -> None: # Report any errors extracting the detection name from the longer rule name if match is None: msg = ( - f"Entry in cms_main ('{cms_entry_name}') did not match the expected naming " + f"[{cms_entry_name}]: Entry in cms_main did not match the expected naming " "scheme; cannot compare to our detections." ) logger.error(msg) @@ -339,7 +358,7 @@ def validate_content_against_cms(self) -> None: # unexpected repeated entry if cms_entry_name in matched_detections: msg = ( - f"Detection '{cms_entry_name}' appears more than once in the cms_main " + f"[{cms_entry_name}]: Detection appears more than once in the cms_main " "index." ) logger.error(msg) @@ -376,7 +395,7 @@ def validate_content_against_cms(self) -> None: # Generate an exception if we couldn't match the CMS main entry to a detection if result_matches_detection is False: msg = ( - f"Could not match entry in cms_main for ('{cms_entry_name}') against any " + f"[{cms_entry_name}]: Could not match entry in cms_main against any " "of the expected detections." ) logger.error(msg) @@ -388,7 +407,7 @@ def validate_content_against_cms(self) -> None: # Generate exceptions for the unmatched detections for detection_name in remaining_detections: msg = ( - f"Detection '{detection_name}' not found in cms_main; there may be an " + f"[{detection_name}]: Detection not found in cms_main; there may be an " "issue with savedsearches.conf" ) logger.error(msg) @@ -430,8 +449,8 @@ def validate_detection_against_cms_event( # Compare the UUIDs if cms_uuid != detection.id: msg = ( - f"UUID in cms_event ('{cms_uuid}') does not match UUID in detection " - f"('{detection.id}'): {detection.name}" + f"[{detection.name}]: UUID in cms_event ('{cms_uuid}') does not match UUID in " + f"detection ('{detection.id}')" ) logger.error(msg) return Exception(msg) @@ -439,33 +458,32 @@ def validate_detection_against_cms_event( # Compare the versions (we append '-1' to the detection version to be in line w/ the # internal representation in ES) msg = ( - f"Version in cms_event ('{cms_event['version']}') does not match version in " - f"detection ('{detection.version}-1'): {detection.name}" + f"[{detection.name}]: Version in cms_event ('{cms_event['version']}') does not " + f"match version in detection ('{detection.version}-1')" ) logger.error(msg) return Exception(msg) elif cms_event[full_search_key] != rule_name_from_detection: # Compare the full search name msg = ( - f"Full search name in cms_event ('{cms_event[full_search_key]}') " - f"does not match detection name ('{detection.name}')" + f"[{detection.name}]: Full search name in cms_event " + f"('{cms_event[full_search_key]}') does not match detection name" ) logger.error(msg) return Exception(msg) elif cms_event["action.correlationsearch.label"] != f"{self.global_config.app.label} - {detection.name} - Rule": # Compare the correlation search label msg = ( - f"Correlation search label in cms_event " - f"('{cms_event['action.correlationsearch.label']}') does not match detection name " - f"('{detection.name}')" + f"[{detection.name}]: Correlation search label in cms_event " + f"('{cms_event['action.correlationsearch.label']}') does not match detection name" ) logger.error(msg) return Exception(msg) elif cms_event["sourcetype"] != f"{self.global_config.app.label} - {detection.name} - Rule": # Compare the full search name msg = ( - f"Sourcetype in cms_event ('{cms_event[f'sourcetype']}') does not match detection " - f"name ('{detection.name}')" + f"[{detection.name}]: Sourcetype in cms_event ('{cms_event[f'sourcetype']}') does " + f"not match detection name" ) logger.error(msg) return Exception(msg) From 27cc4785277d7723d9a92d247aa8042b09993f9b Mon Sep 17 00:00:00 2001 From: Casey McGinley Date: Tue, 8 Oct 2024 11:15:14 -0700 Subject: [PATCH 04/17] migrated logging to utils --- contentctl/helper/utils.py | 57 ++++++++++++++++++ contentctl/objects/config.py | 2 - .../objects/content_versioning_service.py | 59 ++++++++++++------- contentctl/objects/correlation_search.py | 57 +++++------------- 4 files changed, 109 insertions(+), 66 deletions(-) diff --git a/contentctl/helper/utils.py b/contentctl/helper/utils.py index 261ecb64..4dbbf51e 100644 --- a/contentctl/helper/utils.py +++ b/contentctl/helper/utils.py @@ -6,6 +6,7 @@ import string from timeit import default_timer import pathlib +import logging from typing import Union, Tuple import tqdm @@ -485,3 +486,59 @@ def getPercent(numerator: float, denominator: float, decimal_places: int) -> str ratio = numerator / denominator percent = ratio * 100 return Utils.getFixedWidth(percent, decimal_places) + "%" + + @staticmethod + def get_logger( + name: str, + log_level: int, + log_path: str, + enable_logging: bool + ) -> logging.Logger: + """ + Gets a logger instance for the given name; logger is configured if not already configured. + The NullHandler is used to suppress loggging when running in production so as not to + conflict w/ contentctl's larger pbar-based logging. The StreamHandler is enabled by setting + enable_logging to True (useful for debugging/testing locally) + + :param name: the logger name + :type name: str + :param log_level: the logging level (e.g. `logging.Debug`) + :type log_level: int + :param log_path: the path for the log file + :type log_path: str + :param enable_logging: a flag indicating whether logging should be redirected from null to + the stream handler + :type enable_logging: bool + + :return: a logger + :rtype: :class:`logging.Logger` + """ + # get logger for module + logger = logging.getLogger(name) + + # set propagate to False if not already set as such (needed to that we do not flow up to any + # root loggers) + if logger.propagate: + logger.propagate = False + + # if logger has no handlers, it needs to be configured for the first time + if not logger.hasHandlers(): + # set level + logger.setLevel(log_level) + + # if logging enabled, use a StreamHandler; else, use the NullHandler to suppress logging + handler: logging.Handler + if enable_logging: + handler = logging.FileHandler(log_path) + else: + handler = logging.NullHandler() + + # Format our output + formatter = logging.Formatter('%(asctime)s - %(levelname)s:%(name)s - %(message)s') + handler.setFormatter(formatter) + + # Set handler level and add to logger + handler.setLevel(log_level) + logger.addHandler(handler) + + return logger diff --git a/contentctl/objects/config.py b/contentctl/objects/config.py index 0b262c55..8ef9b30a 100644 --- a/contentctl/objects/config.py +++ b/contentctl/objects/config.py @@ -318,8 +318,6 @@ class inspect(build): f"or CLI invocation appropriately] {validate.model_fields['enrichments'].description}" ) ) - # TODO (cmcginley): wording should change here if we want to be able to download any app from - # Splunkbase previous_build: str | None = Field( default=None, description=( diff --git a/contentctl/objects/content_versioning_service.py b/contentctl/objects/content_versioning_service.py index d6203acc..a6dc6ab4 100644 --- a/contentctl/objects/content_versioning_service.py +++ b/contentctl/objects/content_versioning_service.py @@ -2,20 +2,25 @@ import uuid import json import re +import logging from typing import Any, Callable from functools import cached_property -from pydantic import BaseModel, PrivateAttr, computed_field +from pydantic import BaseModel, PrivateAttr, computed_field, Field import splunklib.client as splunklib # type: ignore from splunklib.binding import HTTPError, ResponseReader # type: ignore from splunklib.data import Record # type: ignore from contentctl.objects.config import test_common, Infrastructure from contentctl.objects.detection import Detection -from contentctl.objects.correlation_search import ResultIterator, get_logger +from contentctl.objects.correlation_search import ResultIterator +from contentctl.helper.utils import Utils -# TODO (cmcginley): remove this logger -logger = get_logger() +# TODO (cmcginley): suppress logging +# Suppress logging by default; enable for local testing +ENABLE_LOGGING = True +LOG_LEVEL = logging.DEBUG +LOG_PATH = "content_versioning_service.log" # TODO (cmcginley): would it be better for this to only run on one instance? Or to consolidate # error reporting at least? @@ -40,6 +45,16 @@ class ContentVersioningService(BaseModel): # The list of detections detections: list[Detection] + # The logger to use (logs all go to a null pipe unless ENABLE_LOGGING is set to True, so as not + # to conflict w/ tqdm) + logger: logging.Logger = Field(default_factory=lambda: Utils.get_logger( + __name__, + LOG_LEVEL, + LOG_PATH, + ENABLE_LOGGING + ) + ) + # The cached job on the splunk instance of the cms events _cms_main_job: splunklib.Job | None = PrivateAttr(default=None) @@ -215,17 +230,17 @@ def wait_for_cms_main(self) -> None: # Loop until timeout while elapsed_sleep_time < max_sleep: # Sleep, and add the time to the elapsed counter - logger.info(f"Waiting {time_to_sleep} for cms_parser to finish") + self.logger.info(f"Waiting {time_to_sleep} for cms_parser to finish") time.sleep(time_to_sleep) elapsed_sleep_time += time_to_sleep - logger.info( + self.logger.info( f"Checking cms_main (attempt #{num_tries + 1} - {elapsed_sleep_time} seconds elapsed of " f"{max_sleep} max)" ) # Check if the number of CMS events matches or exceeds the number of detections if self.get_num_cms_events() >= len(self.detections): - logger.info( + self.logger.info( f"Found {self.get_num_cms_events(use_cache=True)} events in cms_main which " f"meets or exceeds the expected {len(self.detections)}." ) @@ -263,7 +278,7 @@ def _query_cms_main(self, use_cache: bool = False) -> splunklib.Job: f"search index=cms_main app_name=\"{self.global_config.app.appid}\" | " f"fields {', '.join(self.cms_fields)}" ) - logger.debug(f"Query on cms_main: {query}") + self.logger.debug(f"Query on cms_main: {query}") # Get the job as a blocking operation, set the cache, and return self._cms_main_job = self.service.search(query, exec_mode="blocking") # type: ignore @@ -305,9 +320,9 @@ def validate_content_against_cms(self) -> None: f"Expected {len(self.detections)} matching events in cms_main, but " f"found {result_count}." ) - logger.error(msg) + self.logger.error(msg) exceptions.append(Exception(msg)) - logger.info( + self.logger.info( f"Expecting {len(self.detections)} matching events in cms_main, " f"found {result_count}." ) @@ -337,7 +352,7 @@ def validate_content_against_cms(self) -> None: # to strip the prefix and suffix used for the savedsearches.conf name so we can # compare to the detection cms_entry_name = cms_event["sourcetype"] - logger.info(f"{offset}: Matching cms_main entry '{cms_entry_name}' against detections") + self.logger.info(f"{offset}: Matching cms_main entry '{cms_entry_name}' against detections") ptrn = re.compile(r"^" + self.global_config.app.label + r" - (?P.+) - Rule$") match = ptrn.match(cms_event["sourcetype"]) @@ -347,7 +362,7 @@ def validate_content_against_cms(self) -> None: f"[{cms_entry_name}]: Entry in cms_main did not match the expected naming " "scheme; cannot compare to our detections." ) - logger.error(msg) + self.logger.error(msg) exceptions.append(Exception(msg)) continue @@ -361,7 +376,7 @@ def validate_content_against_cms(self) -> None: f"[{cms_entry_name}]: Detection appears more than once in the cms_main " "index." ) - logger.error(msg) + self.logger.error(msg) exceptions.append(Exception(msg)) continue @@ -371,7 +386,7 @@ def validate_content_against_cms(self) -> None: # If we find a match, break this loop, set the found flag and move the detection # from those that still need to matched to those already matched if cms_entry_name == detection_name: - logger.info( + self.logger.info( f"{offset}: Succesfully matched cms_main entry against detection " f"('{detection_name}')!" ) @@ -398,7 +413,7 @@ def validate_content_against_cms(self) -> None: f"[{cms_entry_name}]: Could not match entry in cms_main against any " "of the expected detections." ) - logger.error(msg) + self.logger.error(msg) exceptions.append(Exception(msg)) # If we have any remaining detections, they could not be matched against an entry in @@ -410,7 +425,7 @@ def validate_content_against_cms(self) -> None: f"[{detection_name}]: Detection not found in cms_main; there may be an " "issue with savedsearches.conf" ) - logger.error(msg) + self.logger.error(msg) exceptions.append(Exception(msg)) # Raise exceptions as a group @@ -421,7 +436,7 @@ def validate_content_against_cms(self) -> None: ) # Else, we've matched/validated all detections against cms_main - logger.info("Matched and validated all detections against cms_main!") + self.logger.info("Matched and validated all detections against cms_main!") def validate_detection_against_cms_event( self, @@ -452,7 +467,7 @@ def validate_detection_against_cms_event( f"[{detection.name}]: UUID in cms_event ('{cms_uuid}') does not match UUID in " f"detection ('{detection.id}')" ) - logger.error(msg) + self.logger.error(msg) return Exception(msg) elif cms_event["version"] != f"{detection.version}-1": # Compare the versions (we append '-1' to the detection version to be in line w/ the @@ -461,7 +476,7 @@ def validate_detection_against_cms_event( f"[{detection.name}]: Version in cms_event ('{cms_event['version']}') does not " f"match version in detection ('{detection.version}-1')" ) - logger.error(msg) + self.logger.error(msg) return Exception(msg) elif cms_event[full_search_key] != rule_name_from_detection: # Compare the full search name @@ -469,7 +484,7 @@ def validate_detection_against_cms_event( f"[{detection.name}]: Full search name in cms_event " f"('{cms_event[full_search_key]}') does not match detection name" ) - logger.error(msg) + self.logger.error(msg) return Exception(msg) elif cms_event["action.correlationsearch.label"] != f"{self.global_config.app.label} - {detection.name} - Rule": # Compare the correlation search label @@ -477,7 +492,7 @@ def validate_detection_against_cms_event( f"[{detection.name}]: Correlation search label in cms_event " f"('{cms_event['action.correlationsearch.label']}') does not match detection name" ) - logger.error(msg) + self.logger.error(msg) return Exception(msg) elif cms_event["sourcetype"] != f"{self.global_config.app.label} - {detection.name} - Rule": # Compare the full search name @@ -485,7 +500,7 @@ def validate_detection_against_cms_event( f"[{detection.name}]: Sourcetype in cms_event ('{cms_event[f'sourcetype']}') does " f"not match detection name" ) - logger.error(msg) + self.logger.error(msg) return Exception(msg) return None diff --git a/contentctl/objects/correlation_search.py b/contentctl/objects/correlation_search.py index 173e7d1b..3cefd383 100644 --- a/contentctl/objects/correlation_search.py +++ b/contentctl/objects/correlation_search.py @@ -29,53 +29,15 @@ from contentctl.objects.risk_event import RiskEvent from contentctl.objects.notable_event import NotableEvent from contentctl.objects.observable import Observable +from contentctl.helper.utils import Utils -# TODO (cmcginley): disable logging # Suppress logging by default; enable for local testing -ENABLE_LOGGING = True +ENABLE_LOGGING = False LOG_LEVEL = logging.DEBUG LOG_PATH = "correlation_search.log" -def get_logger() -> logging.Logger: - """ - Gets a logger instance for the module; logger is configured if not already configured. The - NullHandler is used to suppress loggging when running in production so as not to conflict w/ - contentctl's larger pbar-based logging. The StreamHandler is enabled by setting ENABLE_LOGGING - to True (useful for debugging/testing locally) - """ - # get logger for module - logger = logging.getLogger(__name__) - - # set propagate to False if not already set as such (needed to that we do not flow up to any - # root loggers) - if logger.propagate: - logger.propagate = False - - # if logger has no handlers, it needs to be configured for the first time - if not logger.hasHandlers(): - # set level - logger.setLevel(LOG_LEVEL) - - # if logging enabled, use a StreamHandler; else, use the NullHandler to suppress logging - handler: logging.Handler - if ENABLE_LOGGING: - handler = logging.FileHandler(LOG_PATH) - else: - handler = logging.NullHandler() - - # Format our output - formatter = logging.Formatter('%(asctime)s - %(levelname)s:%(name)s - %(message)s') - handler.setFormatter(formatter) - - # Set handler level and add to logger - handler.setLevel(LOG_LEVEL) - logger.addHandler(handler) - - return logger - - class SavedSearchKeys(str, Enum): """ Various keys into the SavedSearch content @@ -140,7 +102,12 @@ def __init__(self, response_reader: ResponseReader) -> None: ) # get logger - self.logger: logging.Logger = get_logger() + self.logger: logging.Logger = Utils.get_logger( + __name__, + LOG_LEVEL, + LOG_PATH, + ENABLE_LOGGING + ) def __iter__(self) -> "ResultIterator": return self @@ -222,7 +189,13 @@ class CorrelationSearch(BaseModel): # The logger to use (logs all go to a null pipe unless ENABLE_LOGGING is set to True, so as not # to conflict w/ tqdm) - logger: logging.Logger = Field(default_factory=get_logger) + logger: logging.Logger = Field(default_factory=lambda: Utils.get_logger( + __name__, + LOG_LEVEL, + LOG_PATH, + ENABLE_LOGGING + ) + ) # The search name (e.g. "ESCU - Windows Modify Registry EnableLinkedConnections - Rule") name: Optional[str] = None From 35ece19ba5b8aa75cbfbbd19ab4cfb3e7a8c52af Mon Sep 17 00:00:00 2001 From: Casey McGinley Date: Tue, 8 Oct 2024 15:08:27 -0700 Subject: [PATCH 05/17] writing out setup complete to terminal --- .../infrastructures/DetectionTestingInfrastructure.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/contentctl/actions/detection_testing/infrastructures/DetectionTestingInfrastructure.py b/contentctl/actions/detection_testing/infrastructures/DetectionTestingInfrastructure.py index 497cf9fb..ab30dd50 100644 --- a/contentctl/actions/detection_testing/infrastructures/DetectionTestingInfrastructure.py +++ b/contentctl/actions/detection_testing/infrastructures/DetectionTestingInfrastructure.py @@ -170,7 +170,14 @@ def setup(self): raise ExceptionGroup(msg, e.exceptions) from e # type: ignore raise Exception(msg) from e - self.format_pbar_string(TestReportingType.SETUP, self.get_name(), "Finished Setup!") + self.pbar.write( + self.format_pbar_string( + TestReportingType.SETUP, + self.get_name(), + "Finished Setup!", + set_pbar=False + ) + ) def wait_for_ui_ready(self): self.get_conn() From 21520bc693e44e13b16e02b9c546f420e6ddf9a2 Mon Sep 17 00:00:00 2001 From: Casey McGinley Date: Tue, 8 Oct 2024 16:22:59 -0700 Subject: [PATCH 06/17] setup functions need to be run before we can check if content versioning should be enforced --- .../DetectionTestingInfrastructure.py | 25 ++++++++++++------- ...DetectionTestingInfrastructureContainer.py | 1 - 2 files changed, 16 insertions(+), 10 deletions(-) diff --git a/contentctl/actions/detection_testing/infrastructures/DetectionTestingInfrastructure.py b/contentctl/actions/detection_testing/infrastructures/DetectionTestingInfrastructure.py index ab30dd50..e7bfffde 100644 --- a/contentctl/actions/detection_testing/infrastructures/DetectionTestingInfrastructure.py +++ b/contentctl/actions/detection_testing/infrastructures/DetectionTestingInfrastructure.py @@ -133,7 +133,7 @@ def setup(self): self.start_time = time.time() # Init the list of setup functions we always need - setup_functions: list[tuple[Callable[[], None | client.Service], str]] = [ + primary_setup_functions: list[tuple[Callable[[], None | client.Service], str]] = [ (self.start, "Starting"), (self.get_conn, "Waiting for App Installation"), (self.configure_conf_file_datamodels, "Configuring Datamodels"), @@ -142,18 +142,13 @@ def setup(self): (self.configure_imported_roles, "Configuring Roles"), (self.configure_delete_indexes, "Configuring Indexes"), (self.configure_hec, "Configuring HEC"), + (self.wait_for_ui_ready, "Finishing Primary Setup") ] - # Add any setup functions only applicable to content versioning validation - if self.should_test_content_versioning: - setup_functions = setup_functions + self.content_versioning_service.setup_functions - - # Add the final setup function - setup_functions.append((self.wait_for_ui_ready, "Finishing Setup")) - # Execute and report on each setup function try: - for func, msg in setup_functions: + # Run the primary setup functions + for func, msg in primary_setup_functions: self.format_pbar_string( TestReportingType.SETUP, self.get_name(), @@ -163,6 +158,18 @@ def setup(self): func() self.check_for_teardown() + # Run any setup functions only applicable to content versioning validation + if self.should_test_content_versioning: + for func, msg in self.content_versioning_service.setup_functions: + self.format_pbar_string( + TestReportingType.SETUP, + self.get_name(), + msg, + update_sync_status=True, + ) + func() + self.check_for_teardown() + except Exception as e: msg = f"[{self.get_name()}]: {str(e)}" self.finish() diff --git a/contentctl/actions/detection_testing/infrastructures/DetectionTestingInfrastructureContainer.py b/contentctl/actions/detection_testing/infrastructures/DetectionTestingInfrastructureContainer.py index f5887033..e3405553 100644 --- a/contentctl/actions/detection_testing/infrastructures/DetectionTestingInfrastructureContainer.py +++ b/contentctl/actions/detection_testing/infrastructures/DetectionTestingInfrastructureContainer.py @@ -47,7 +47,6 @@ def get_docker_client(self): raise (Exception(f"Failed to get docker client: {str(e)}")) def check_for_teardown(self): - try: container: docker.models.containers.Container = self.get_docker_client().containers.get(self.get_name()) except Exception as e: From 84e89c4c15d51fdf6774b7058ac602ad143589d8 Mon Sep 17 00:00:00 2001 From: Casey McGinley Date: Thu, 24 Oct 2024 11:52:25 -0700 Subject: [PATCH 07/17] missing comma --- .../infrastructures/DetectionTestingInfrastructure.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contentctl/actions/detection_testing/infrastructures/DetectionTestingInfrastructure.py b/contentctl/actions/detection_testing/infrastructures/DetectionTestingInfrastructure.py index a7223abe..b41a5a3d 100644 --- a/contentctl/actions/detection_testing/infrastructures/DetectionTestingInfrastructure.py +++ b/contentctl/actions/detection_testing/infrastructures/DetectionTestingInfrastructure.py @@ -148,7 +148,7 @@ def setup(self): (self.get_conn, "Waiting for App Installation"), (self.configure_conf_file_datamodels, "Configuring Datamodels"), (self.create_replay_index, f"Create index '{self.sync_obj.replay_index}'"), - (self.get_all_indexes, "Getting all indexes from server") + (self.get_all_indexes, "Getting all indexes from server"), (self.check_for_es_install, "Checking for ES Install"), (self.configure_imported_roles, "Configuring Roles"), (self.configure_delete_indexes, "Configuring Indexes"), From 84a0d1bccd6818bfb47c51376a51cc6f3dac4b57 Mon Sep 17 00:00:00 2001 From: Casey McGinley Date: Thu, 24 Oct 2024 14:43:33 -0700 Subject: [PATCH 08/17] logging the beginning of content versioning validation --- .../infrastructures/DetectionTestingInfrastructure.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/contentctl/actions/detection_testing/infrastructures/DetectionTestingInfrastructure.py b/contentctl/actions/detection_testing/infrastructures/DetectionTestingInfrastructure.py index b41a5a3d..0f56048b 100644 --- a/contentctl/actions/detection_testing/infrastructures/DetectionTestingInfrastructure.py +++ b/contentctl/actions/detection_testing/infrastructures/DetectionTestingInfrastructure.py @@ -171,6 +171,14 @@ def setup(self): # Run any setup functions only applicable to content versioning validation if self.should_test_content_versioning: + self.pbar.write( + self.format_pbar_string( + TestReportingType.SETUP, + self.get_name(), + "Beginning Content Versioning Validation...", + set_pbar=False + ) + ) for func, msg in self.content_versioning_service.setup_functions: self.format_pbar_string( TestReportingType.SETUP, From c8496947414c41843379f79248b16a80f155728f Mon Sep 17 00:00:00 2001 From: Casey McGinley Date: Tue, 29 Oct 2024 12:39:33 -0700 Subject: [PATCH 09/17] adjusting for some of the changes in build and ES 8.0 --- .../objects/content_versioning_service.py | 136 +++++++++++------- 1 file changed, 85 insertions(+), 51 deletions(-) diff --git a/contentctl/objects/content_versioning_service.py b/contentctl/objects/content_versioning_service.py index a6dc6ab4..6ebc064c 100644 --- a/contentctl/objects/content_versioning_service.py +++ b/contentctl/objects/content_versioning_service.py @@ -16,6 +16,14 @@ from contentctl.objects.correlation_search import ResultIterator from contentctl.helper.utils import Utils +# TODO (cmcginley): +# - [x] version naming scheme seems to have changed from X - X to X.X +# - [x] sourcetype no longer holds detection name but instead is stash_common_detection_model +# - [ ] action.escu.full_search_name no longer available +# - [ ] check to see if we can get "name" +# - [ ] move strings to enums +# - [ ] additionally, timeout for cms_parser seems to need more time + # TODO (cmcginley): suppress logging # Suppress logging by default; enable for local testing ENABLE_LOGGING = True @@ -55,6 +63,15 @@ class ContentVersioningService(BaseModel): ) ) + def model_post_init(self, __context: Any) -> None: + super().model_post_init(__context) + + # Log instance details + self.logger.info( + f"[{self.infrastructure.instance_name} ({self.infrastructure.instance_address})] " + "Initing ContentVersioningService" + ) + # The cached job on the splunk instance of the cms events _cms_main_job: splunklib.Job | None = PrivateAttr(default=None) @@ -167,6 +184,10 @@ def activate_versioning(self) -> None: if not self.is_versioning_activated: raise Exception("Something went wrong, content versioning is still disabled.") + self.logger.info( + f"[{self.infrastructure.instance_name}] Versioning service successfully activated" + ) + @computed_field @cached_property def cms_fields(self) -> list[str]: @@ -178,7 +199,6 @@ def cms_fields(self) -> list[str]: """ return [ "app_name", - f"action.{self.global_config.app.label.lower()}.full_search_name", "detection_id", "version", "action.correlationsearch.label", @@ -214,6 +234,10 @@ def force_cms_parser(self) -> None: if not self.is_cms_parser_enabled: raise Exception("Something went wrong, cms_parser is still disabled.") + self.logger.info( + f"[{self.infrastructure.instance_name}] cms_parser successfully toggled to force run" + ) + def wait_for_cms_main(self) -> None: """ Checks the cms_main index until it has the expected number of events, or it times out. @@ -225,27 +249,36 @@ def wait_for_cms_main(self) -> None: elapsed_sleep_time = 0 num_tries = 0 time_to_sleep = 2**num_tries - max_sleep = 300 + max_sleep = 480 # Loop until timeout while elapsed_sleep_time < max_sleep: # Sleep, and add the time to the elapsed counter - self.logger.info(f"Waiting {time_to_sleep} for cms_parser to finish") + self.logger.info( + f"[{self.infrastructure.instance_name}] Waiting {time_to_sleep} for cms_parser to " + "finish" + ) time.sleep(time_to_sleep) elapsed_sleep_time += time_to_sleep self.logger.info( - f"Checking cms_main (attempt #{num_tries + 1} - {elapsed_sleep_time} seconds elapsed of " - f"{max_sleep} max)" + f"[{self.infrastructure.instance_name}] Checking cms_main (attempt #{num_tries + 1}" + f" - {elapsed_sleep_time} seconds elapsed of {max_sleep} max)" ) # Check if the number of CMS events matches or exceeds the number of detections if self.get_num_cms_events() >= len(self.detections): self.logger.info( - f"Found {self.get_num_cms_events(use_cache=True)} events in cms_main which " + f"[{self.infrastructure.instance_name}] Found " + f"{self.get_num_cms_events(use_cache=True)} events in cms_main which " f"meets or exceeds the expected {len(self.detections)}." ) break - + else: + self.logger.info( + f"[{self.infrastructure.instance_name}] Found " + f"{self.get_num_cms_events(use_cache=True)} matching events in cms_main; " + f"expecting {len(self.detections)}. Continuing to wait..." + ) # Update the number of times we've tried, and increment the time to sleep num_tries += 1 time_to_sleep = 2**num_tries @@ -278,7 +311,7 @@ def _query_cms_main(self, use_cache: bool = False) -> splunklib.Job: f"search index=cms_main app_name=\"{self.global_config.app.appid}\" | " f"fields {', '.join(self.cms_fields)}" ) - self.logger.debug(f"Query on cms_main: {query}") + self.logger.debug(f"[{self.infrastructure.instance_name}] Query on cms_main: {query}") # Get the job as a blocking operation, set the cache, and return self._cms_main_job = self.service.search(query, exec_mode="blocking") # type: ignore @@ -317,14 +350,14 @@ def validate_content_against_cms(self) -> None: # Generate an error for the count mismatch if result_count != len(self.detections): msg = ( - f"Expected {len(self.detections)} matching events in cms_main, but " - f"found {result_count}." + f"[{self.infrastructure.instance_name}] Expected {len(self.detections)} matching " + f"events in cms_main, but found {result_count}." ) self.logger.error(msg) exceptions.append(Exception(msg)) self.logger.info( - f"Expecting {len(self.detections)} matching events in cms_main, " - f"found {result_count}." + f"[{self.infrastructure.instance_name}] Expecting {len(self.detections)} matching " + f"events in cms_main, found {result_count}." ) # Init some counters and a mapping of detections to their names @@ -351,30 +384,34 @@ def validate_content_against_cms(self) -> None: # Get the name of the search in the CMS event and attempt to use pattern matching # to strip the prefix and suffix used for the savedsearches.conf name so we can # compare to the detection - cms_entry_name = cms_event["sourcetype"] - self.logger.info(f"{offset}: Matching cms_main entry '{cms_entry_name}' against detections") - ptrn = re.compile(r"^" + self.global_config.app.label + r" - (?P.+) - Rule$") - match = ptrn.match(cms_event["sourcetype"]) + cms_entry_name = cms_event["action.correlationsearch.label"] + self.logger.info( + f"[{self.infrastructure.instance_name}] {offset}: Matching cms_main entry " + f"'{cms_entry_name}' against detections" + ) + ptrn = re.compile(r"^" + self.global_config.app.label + r" - (?P.+) - Rule$") + match = ptrn.match(cms_event["action.correlationsearch.label"]) # Report any errors extracting the detection name from the longer rule name if match is None: msg = ( - f"[{cms_entry_name}]: Entry in cms_main did not match the expected naming " - "scheme; cannot compare to our detections." + f"[{self.infrastructure.instance_name}] [{cms_entry_name}]: Entry in " + "cms_main did not match the expected naming scheme; cannot compare to our " + "detections." ) self.logger.error(msg) exceptions.append(Exception(msg)) continue # Extract the detection name if matching was successful - cms_entry_name = match.group("cms_entry_name") + stripped_cms_entry_name = match.group("stripped_cms_entry_name") # If CMS entry name matches one of the detections already matched, we've got an # unexpected repeated entry - if cms_entry_name in matched_detections: + if stripped_cms_entry_name in matched_detections: msg = ( - f"[{cms_entry_name}]: Detection appears more than once in the cms_main " - "index." + f"[{self.infrastructure.instance_name}] [{stripped_cms_entry_name}]: Detection " + f"appears more than once in the cms_main index." ) self.logger.error(msg) exceptions.append(Exception(msg)) @@ -385,10 +422,10 @@ def validate_content_against_cms(self) -> None: for detection_name in remaining_detections: # If we find a match, break this loop, set the found flag and move the detection # from those that still need to matched to those already matched - if cms_entry_name == detection_name: + if stripped_cms_entry_name == detection_name: self.logger.info( - f"{offset}: Succesfully matched cms_main entry against detection " - f"('{detection_name}')!" + f"[{self.infrastructure.instance_name}] {offset}: Succesfully matched " + f"cms_main entry against detection ('{detection_name}')!" ) # Validate other fields of the cms_event against the detection @@ -410,8 +447,8 @@ def validate_content_against_cms(self) -> None: # Generate an exception if we couldn't match the CMS main entry to a detection if result_matches_detection is False: msg = ( - f"[{cms_entry_name}]: Could not match entry in cms_main against any " - "of the expected detections." + f"[{self.infrastructure.instance_name}] [{stripped_cms_entry_name}]: Could not " + "match entry in cms_main against any of the expected detections." ) self.logger.error(msg) exceptions.append(Exception(msg)) @@ -422,8 +459,8 @@ def validate_content_against_cms(self) -> None: # Generate exceptions for the unmatched detections for detection_name in remaining_detections: msg = ( - f"[{detection_name}]: Detection not found in cms_main; there may be an " - "issue with savedsearches.conf" + f"[{self.infrastructure.instance_name}] [{detection_name}]: Detection not " + "found in cms_main; there may be an issue with savedsearches.conf" ) self.logger.error(msg) exceptions.append(Exception(msg)) @@ -436,7 +473,10 @@ def validate_content_against_cms(self) -> None: ) # Else, we've matched/validated all detections against cms_main - self.logger.info("Matched and validated all detections against cms_main!") + self.logger.info( + f"[{self.infrastructure.instance_name}] Matched and validated all detections against " + "cms_main!" + ) def validate_detection_against_cms_event( self, @@ -458,47 +498,41 @@ def validate_detection_against_cms_event( # TODO (cmcginley): validate additional fields between the cms_event and the detection cms_uuid = uuid.UUID(cms_event["detection_id"]) - full_search_key = f"action.{self.global_config.app.label.lower()}.full_search_name" rule_name_from_detection = f"{self.global_config.app.label} - {detection.name} - Rule" # Compare the UUIDs if cms_uuid != detection.id: msg = ( - f"[{detection.name}]: UUID in cms_event ('{cms_uuid}') does not match UUID in " - f"detection ('{detection.id}')" + f"[{self.infrastructure.instance_name}] [{detection.name}]: UUID in cms_event " + f"('{cms_uuid}') does not match UUID in detection ('{detection.id}')" ) self.logger.error(msg) return Exception(msg) - elif cms_event["version"] != f"{detection.version}-1": - # Compare the versions (we append '-1' to the detection version to be in line w/ the + elif cms_event["version"] != f"{detection.version}.1": + # Compare the versions (we append '.1' to the detection version to be in line w/ the # internal representation in ES) msg = ( - f"[{detection.name}]: Version in cms_event ('{cms_event['version']}') does not " - f"match version in detection ('{detection.version}-1')" - ) - self.logger.error(msg) - return Exception(msg) - elif cms_event[full_search_key] != rule_name_from_detection: - # Compare the full search name - msg = ( - f"[{detection.name}]: Full search name in cms_event " - f"('{cms_event[full_search_key]}') does not match detection name" + f"[{self.infrastructure.instance_name}] [{detection.name}]: Version in cms_event " + f"('{cms_event['version']}') does not match version in detection " + f"('{detection.version}.1')" ) self.logger.error(msg) return Exception(msg) - elif cms_event["action.correlationsearch.label"] != f"{self.global_config.app.label} - {detection.name} - Rule": + elif cms_event["action.correlationsearch.label"] != rule_name_from_detection: # Compare the correlation search label msg = ( - f"[{detection.name}]: Correlation search label in cms_event " - f"('{cms_event['action.correlationsearch.label']}') does not match detection name" + f"[{self.infrastructure.instance_name}][{detection.name}]: Correlation search " + f"label in cms_event ('{cms_event['action.correlationsearch.label']}') does not " + "match detection name" ) self.logger.error(msg) return Exception(msg) - elif cms_event["sourcetype"] != f"{self.global_config.app.label} - {detection.name} - Rule": + elif cms_event["sourcetype"] != "stash_common_detection_model": # Compare the full search name msg = ( - f"[{detection.name}]: Sourcetype in cms_event ('{cms_event[f'sourcetype']}') does " - f"not match detection name" + f"[{self.infrastructure.instance_name}] [{detection.name}]: Unexpected sourcetype " + f"in cms_event ('{cms_event[f'sourcetype']}'); expected " + "'stash_common_detection_model'" ) self.logger.error(msg) return Exception(msg) From 4a5cc2faa7488b538da05fb190289429d118f2c8 Mon Sep 17 00:00:00 2001 From: Casey McGinley Date: Tue, 29 Oct 2024 15:40:07 -0700 Subject: [PATCH 10/17] adding error filtering --- .../objects/content_versioning_service.py | 18 ++++++++++----- contentctl/objects/correlation_search.py | 22 ++++++++++++++++--- 2 files changed, 32 insertions(+), 8 deletions(-) diff --git a/contentctl/objects/content_versioning_service.py b/contentctl/objects/content_versioning_service.py index 6ebc064c..fc9eb08e 100644 --- a/contentctl/objects/content_versioning_service.py +++ b/contentctl/objects/content_versioning_service.py @@ -19,10 +19,12 @@ # TODO (cmcginley): # - [x] version naming scheme seems to have changed from X - X to X.X # - [x] sourcetype no longer holds detection name but instead is stash_common_detection_model -# - [ ] action.escu.full_search_name no longer available -# - [ ] check to see if we can get "name" +# - [x] action.escu.full_search_name no longer available +# - [x] check to see if we can get "name" # - [ ] move strings to enums -# - [ ] additionally, timeout for cms_parser seems to need more time +# - [x] additionally, timeout for cms_parser seems to need more time +# - [ ] validate multi-line fields -> search, description, action.notable.param.rule_description, +# action.notable.param.drilldown_searches # TODO (cmcginley): suppress logging # Suppress logging by default; enable for local testing @@ -366,14 +368,20 @@ def validate_content_against_cms(self) -> None: remaining_detections = {x.name: x for x in self.detections} matched_detections: dict[str, Detection] = {} + # Create a filter for a specific memory error we're ok ignoring + sub_second_order_pattern = re.compile( + r".*Events might not be returned in sub-second order due to search memory limits.*" + ) + # Iterate over the results until we've gone through them all while offset < result_count: iterator = ResultIterator( - job.results( # type: ignore + response_reader=job.results( # type: ignore output_mode="json", count=count, offset=offset - ) + ), + error_filters=[sub_second_order_pattern] ) # Iterate over the currently fetched results diff --git a/contentctl/objects/correlation_search.py b/contentctl/objects/correlation_search.py index 8fa3d534..8a27ebf3 100644 --- a/contentctl/objects/correlation_search.py +++ b/contentctl/objects/correlation_search.py @@ -1,6 +1,7 @@ import logging import time import json +import re from typing import Any from enum import Enum from functools import cached_property @@ -34,7 +35,7 @@ # Suppress logging by default; enable for local testing -ENABLE_LOGGING = False +ENABLE_LOGGING = True LOG_LEVEL = logging.DEBUG LOG_PATH = "correlation_search.log" @@ -93,15 +94,25 @@ class ResultIterator: Given a ResponseReader, constructs a JSONResultsReader and iterates over it; when Message instances are encountered, they are logged if the message is anything other than "error", in which case an error is raised. Regular results are returned as expected + :param response_reader: a ResponseReader object - :param logger: a Logger object + :type response_reader: :class:`splunklib.binding.ResponseReader` + :param error_filters: set of re Patterns used to filter out errors we're ok ignoring + :type error_filters: list[:class:`re.Pattern[str]`] """ - def __init__(self, response_reader: ResponseReader) -> None: + def __init__( + self, + response_reader: ResponseReader, + error_filters: list[re.Pattern[str]] = [] + ) -> None: # init the results reader self.results_reader: JSONResultsReader = JSONResultsReader( response_reader ) + # the list of patterns for errors to ignore + self.error_filters: list[re.Pattern[str]] = error_filters + # get logger self.logger: logging.Logger = Utils.get_logger( __name__, @@ -127,6 +138,11 @@ def __next__(self) -> dict[str, Any]: message = f"SPLUNK: {result.message}" self.logger.log(level, message) if level == logging.ERROR: + # if the error matches any of the filters, just continue on + for filter in self.error_filters: + if filter.match(message): + continue + # if no filter was matched, raise raise ServerError(message) # if dict, just return From 1f2431f0c8fb80e254cffe4d5199a51411765727 Mon Sep 17 00:00:00 2001 From: Casey McGinley Date: Tue, 29 Oct 2024 18:44:03 -0700 Subject: [PATCH 11/17] bugfix fitlering --- contentctl/objects/correlation_search.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/contentctl/objects/correlation_search.py b/contentctl/objects/correlation_search.py index 8a27ebf3..6328a6f5 100644 --- a/contentctl/objects/correlation_search.py +++ b/contentctl/objects/correlation_search.py @@ -137,13 +137,19 @@ def __next__(self) -> dict[str, Any]: # log message at appropriate level and raise if needed message = f"SPLUNK: {result.message}" self.logger.log(level, message) + filtered = False if level == logging.ERROR: - # if the error matches any of the filters, just continue on + # if the error matches any of the filters, flag it for filter in self.error_filters: - if filter.match(message): - continue + self.logger.debug(f"Filter: {filter}; message: {message}") + if filter.match(message) is not None: + self.logger.debug(f"Error matched filter {filter}; continuing") + filtered = True + break + # if no filter was matched, raise - raise ServerError(message) + if not filtered: + raise ServerError(message) # if dict, just return elif isinstance(result, dict): From aac149c7950f153177a7f54f0e9b57d11cdbc1bb Mon Sep 17 00:00:00 2001 From: Casey McGinley Date: Wed, 30 Oct 2024 00:32:36 -0700 Subject: [PATCH 12/17] bumping timeout --- contentctl/objects/content_versioning_service.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contentctl/objects/content_versioning_service.py b/contentctl/objects/content_versioning_service.py index fc9eb08e..34534731 100644 --- a/contentctl/objects/content_versioning_service.py +++ b/contentctl/objects/content_versioning_service.py @@ -251,7 +251,7 @@ def wait_for_cms_main(self) -> None: elapsed_sleep_time = 0 num_tries = 0 time_to_sleep = 2**num_tries - max_sleep = 480 + max_sleep = 600 # Loop until timeout while elapsed_sleep_time < max_sleep: From b3c6948488257a66b224774e4f89faa7a487c86d Mon Sep 17 00:00:00 2001 From: Casey McGinley Date: Wed, 30 Oct 2024 10:26:24 -0700 Subject: [PATCH 13/17] TODO (revert): temporarily disabling some v alidations --- .../detection_abstract.py | 37 ++++++++++--------- contentctl/output/conf_writer.py | 5 ++- 2 files changed, 22 insertions(+), 20 deletions(-) diff --git a/contentctl/objects/abstract_security_content_objects/detection_abstract.py b/contentctl/objects/abstract_security_content_objects/detection_abstract.py index 34374a88..110e56ac 100644 --- a/contentctl/objects/abstract_security_content_objects/detection_abstract.py +++ b/contentctl/objects/abstract_security_content_objects/detection_abstract.py @@ -569,25 +569,26 @@ def model_post_init(self, __context: Any) -> None: # 1 of the drilldowns contains the string Drilldown.SEARCH_PLACEHOLDER. # This is presently a requirement when 1 or more drilldowns are added to a detection. # Note that this is only required for production searches that are not hunting + + # TODO (cmcginley): commenting out for testing + # if self.type == AnalyticsType.Hunting.value or self.status != DetectionStatus.production.value: + # #No additional check need to happen on the potential drilldowns. + # pass + # else: + # found_placeholder = False + # if len(self.drilldown_searches) < 2: + # raise ValueError(f"This detection is required to have 2 drilldown_searches, but only has [{len(self.drilldown_searches)}]") + # for drilldown in self.drilldown_searches: + # if DRILLDOWN_SEARCH_PLACEHOLDER in drilldown.search: + # found_placeholder = True + # if not found_placeholder: + # raise ValueError("Detection has one or more drilldown_searches, but none of them " + # f"contained '{DRILLDOWN_SEARCH_PLACEHOLDER}. This is a requirement " + # "if drilldown_searches are defined.'") - if self.type == AnalyticsType.Hunting.value or self.status != DetectionStatus.production.value: - #No additional check need to happen on the potential drilldowns. - pass - else: - found_placeholder = False - if len(self.drilldown_searches) < 2: - raise ValueError(f"This detection is required to have 2 drilldown_searches, but only has [{len(self.drilldown_searches)}]") - for drilldown in self.drilldown_searches: - if DRILLDOWN_SEARCH_PLACEHOLDER in drilldown.search: - found_placeholder = True - if not found_placeholder: - raise ValueError("Detection has one or more drilldown_searches, but none of them " - f"contained '{DRILLDOWN_SEARCH_PLACEHOLDER}. This is a requirement " - "if drilldown_searches are defined.'") - - # Update the search fields with the original search, if required - for drilldown in self.drilldown_searches: - drilldown.perform_search_substitutions(self) + # # Update the search fields with the original search, if required + # for drilldown in self.drilldown_searches: + # drilldown.perform_search_substitutions(self) #For experimental purposes, add the default drilldowns #self.drilldown_searches.extend(Drilldown.constructDrilldownsFromDetection(self)) diff --git a/contentctl/output/conf_writer.py b/contentctl/output/conf_writer.py index 4d2e0490..c5f8e2e0 100644 --- a/contentctl/output/conf_writer.py +++ b/contentctl/output/conf_writer.py @@ -115,8 +115,9 @@ def writeDashboardFiles(config:build, dashboards:list[Dashboard])->set[pathlib.P output_file_path = dashboard.getOutputFilepathRelativeToAppRoot(config) # Check that the full output path does not exist so that we are not having an # name collision with a file in app_template - if (config.getPackageDirectoryPath()/output_file_path).exists(): - raise FileExistsError(f"ERROR: Overwriting Dashboard File {output_file_path}. Does this file exist in {config.getAppTemplatePath()} AND {config.path/'dashboards'}?") + # TODO (cmcginley): commenting out for testing + # if (config.getPackageDirectoryPath()/output_file_path).exists(): + # raise FileExistsError(f"ERROR: Overwriting Dashboard File {output_file_path}. Does this file exist in {config.getAppTemplatePath()} AND {config.path/'dashboards'}?") ConfWriter.writeXmlFileHeader(output_file_path, config) dashboard.writeDashboardFile(ConfWriter.getJ2Environment(), config) From 65f3c81b37894f7f36c644917bc8c8835f0113f3 Mon Sep 17 00:00:00 2001 From: Casey McGinley Date: Thu, 21 Nov 2024 13:01:42 -0800 Subject: [PATCH 14/17] logging tracebacks in verbose mode --- .../DetectionTestingManager.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/contentctl/actions/detection_testing/DetectionTestingManager.py b/contentctl/actions/detection_testing/DetectionTestingManager.py index a1d659ca..1992de2b 100644 --- a/contentctl/actions/detection_testing/DetectionTestingManager.py +++ b/contentctl/actions/detection_testing/DetectionTestingManager.py @@ -1,4 +1,5 @@ from typing import List,Union +import traceback from contentctl.objects.config import test, test_servers, Container,Infrastructure from contentctl.actions.detection_testing.infrastructures.DetectionTestingInfrastructure import DetectionTestingInfrastructure from contentctl.actions.detection_testing.infrastructures.DetectionTestingInfrastructureContainer import DetectionTestingInfrastructureContainer @@ -98,6 +99,10 @@ def sigint_handler(signum, frame): _ = future.result() except Exception as e: self.output_dto.terminate = True + # Output the traceback if we encounter errors in verbose mode + if self.input_dto.config.verbose: + tb = traceback.format_exc() + print(tb) errors["INSTANCE SETUP ERRORS"].append(e) # Start and wait for all tests to run @@ -113,6 +118,10 @@ def sigint_handler(signum, frame): _ = future.result() except Exception as e: self.output_dto.terminate = True + # Output the traceback if we encounter errors in verbose mode + if self.input_dto.config.verbose: + tb = traceback.format_exc() + print(tb) errors["TESTING ERRORS"].append(e) self.output_dto.terminate = True @@ -125,6 +134,10 @@ def sigint_handler(signum, frame): try: _ = future.result() except Exception as e: + # Output the traceback if we encounter errors in verbose mode + if self.input_dto.config.verbose: + tb = traceback.format_exc() + print(tb) errors["ERRORS DURING VIEW SHUTDOWN"].append(e) # Wait for original view-related threads to complete @@ -132,6 +145,10 @@ def sigint_handler(signum, frame): try: _ = future.result() except Exception as e: + # Output the traceback if we encounter errors in verbose mode + if self.input_dto.config.verbose: + tb = traceback.format_exc() + print(tb) errors["ERRORS DURING VIEW EXECUTION"].append(e) # Log any errors From 57fe77517c403cdf6995678aa8a1bfa1700824ed Mon Sep 17 00:00:00 2001 From: Casey McGinley Date: Thu, 21 Nov 2024 13:10:03 -0800 Subject: [PATCH 15/17] Revert "TODO (revert): temporarily disabling some v" This reverts commit b3c6948488257a66b224774e4f89faa7a487c86d. --- .../detection_abstract.py | 37 +++++++++---------- contentctl/output/conf_writer.py | 5 +-- 2 files changed, 20 insertions(+), 22 deletions(-) diff --git a/contentctl/objects/abstract_security_content_objects/detection_abstract.py b/contentctl/objects/abstract_security_content_objects/detection_abstract.py index 738d7622..dc0350d5 100644 --- a/contentctl/objects/abstract_security_content_objects/detection_abstract.py +++ b/contentctl/objects/abstract_security_content_objects/detection_abstract.py @@ -569,26 +569,25 @@ def model_post_init(self, __context: Any) -> None: # 1 of the drilldowns contains the string Drilldown.SEARCH_PLACEHOLDER. # This is presently a requirement when 1 or more drilldowns are added to a detection. # Note that this is only required for production searches that are not hunting - - # TODO (cmcginley): commenting out for testing - # if self.type == AnalyticsType.Hunting.value or self.status != DetectionStatus.production.value: - # #No additional check need to happen on the potential drilldowns. - # pass - # else: - # found_placeholder = False - # if len(self.drilldown_searches) < 2: - # raise ValueError(f"This detection is required to have 2 drilldown_searches, but only has [{len(self.drilldown_searches)}]") - # for drilldown in self.drilldown_searches: - # if DRILLDOWN_SEARCH_PLACEHOLDER in drilldown.search: - # found_placeholder = True - # if not found_placeholder: - # raise ValueError("Detection has one or more drilldown_searches, but none of them " - # f"contained '{DRILLDOWN_SEARCH_PLACEHOLDER}. This is a requirement " - # "if drilldown_searches are defined.'") - # # Update the search fields with the original search, if required - # for drilldown in self.drilldown_searches: - # drilldown.perform_search_substitutions(self) + if self.type == AnalyticsType.Hunting.value or self.status != DetectionStatus.production.value: + #No additional check need to happen on the potential drilldowns. + pass + else: + found_placeholder = False + if len(self.drilldown_searches) < 2: + raise ValueError(f"This detection is required to have 2 drilldown_searches, but only has [{len(self.drilldown_searches)}]") + for drilldown in self.drilldown_searches: + if DRILLDOWN_SEARCH_PLACEHOLDER in drilldown.search: + found_placeholder = True + if not found_placeholder: + raise ValueError("Detection has one or more drilldown_searches, but none of them " + f"contained '{DRILLDOWN_SEARCH_PLACEHOLDER}. This is a requirement " + "if drilldown_searches are defined.'") + + # Update the search fields with the original search, if required + for drilldown in self.drilldown_searches: + drilldown.perform_search_substitutions(self) #For experimental purposes, add the default drilldowns #self.drilldown_searches.extend(Drilldown.constructDrilldownsFromDetection(self)) diff --git a/contentctl/output/conf_writer.py b/contentctl/output/conf_writer.py index a4d17e38..410ce4f6 100644 --- a/contentctl/output/conf_writer.py +++ b/contentctl/output/conf_writer.py @@ -232,9 +232,8 @@ def writeDashboardFiles(config:build, dashboards:list[Dashboard])->set[pathlib.P output_file_path = dashboard.getOutputFilepathRelativeToAppRoot(config) # Check that the full output path does not exist so that we are not having an # name collision with a file in app_template - # TODO (cmcginley): commenting out for testing - # if (config.getPackageDirectoryPath()/output_file_path).exists(): - # raise FileExistsError(f"ERROR: Overwriting Dashboard File {output_file_path}. Does this file exist in {config.getAppTemplatePath()} AND {config.path/'dashboards'}?") + if (config.getPackageDirectoryPath()/output_file_path).exists(): + raise FileExistsError(f"ERROR: Overwriting Dashboard File {output_file_path}. Does this file exist in {config.getAppTemplatePath()} AND {config.path/'dashboards'}?") ConfWriter.writeXmlFileHeader(output_file_path, config) dashboard.writeDashboardFile(ConfWriter.getJ2Environment(), config) From 390c3727bf83b5af3e50e4ed4434b542a7d8629f Mon Sep 17 00:00:00 2001 From: Casey McGinley Date: Mon, 9 Dec 2024 09:36:53 -0800 Subject: [PATCH 16/17] updating TODOs, updating query --- .../objects/content_versioning_service.py | 24 +++---------------- contentctl/objects/correlation_search.py | 2 +- 2 files changed, 4 insertions(+), 22 deletions(-) diff --git a/contentctl/objects/content_versioning_service.py b/contentctl/objects/content_versioning_service.py index 34534731..84ca7a3e 100644 --- a/contentctl/objects/content_versioning_service.py +++ b/contentctl/objects/content_versioning_service.py @@ -16,15 +16,6 @@ from contentctl.objects.correlation_search import ResultIterator from contentctl.helper.utils import Utils -# TODO (cmcginley): -# - [x] version naming scheme seems to have changed from X - X to X.X -# - [x] sourcetype no longer holds detection name but instead is stash_common_detection_model -# - [x] action.escu.full_search_name no longer available -# - [x] check to see if we can get "name" -# - [ ] move strings to enums -# - [x] additionally, timeout for cms_parser seems to need more time -# - [ ] validate multi-line fields -> search, description, action.notable.param.rule_description, -# action.notable.param.drilldown_searches # TODO (cmcginley): suppress logging # Suppress logging by default; enable for local testing @@ -310,8 +301,8 @@ def _query_cms_main(self, use_cache: bool = False) -> splunklib.Job: # Construct the query looking for CMS events matching the content app name query = ( - f"search index=cms_main app_name=\"{self.global_config.app.appid}\" | " - f"fields {', '.join(self.cms_fields)}" + f"search index=cms_main sourcetype=stash_common_detection_model " + f"app_name=\"{self.global_config.app.appid}\" | fields {', '.join(self.cms_fields)}" ) self.logger.debug(f"[{self.infrastructure.instance_name}] Query on cms_main: {query}") @@ -503,7 +494,7 @@ def validate_detection_against_cms_event( :return: The generated exception, or None :rtype: Exception | None """ - # TODO (cmcginley): validate additional fields between the cms_event and the detection + # TODO (PEX-509): validate additional fields between the cms_event and the detection cms_uuid = uuid.UUID(cms_event["detection_id"]) rule_name_from_detection = f"{self.global_config.app.label} - {detection.name} - Rule" @@ -535,14 +526,5 @@ def validate_detection_against_cms_event( ) self.logger.error(msg) return Exception(msg) - elif cms_event["sourcetype"] != "stash_common_detection_model": - # Compare the full search name - msg = ( - f"[{self.infrastructure.instance_name}] [{detection.name}]: Unexpected sourcetype " - f"in cms_event ('{cms_event[f'sourcetype']}'); expected " - "'stash_common_detection_model'" - ) - self.logger.error(msg) - return Exception(msg) return None diff --git a/contentctl/objects/correlation_search.py b/contentctl/objects/correlation_search.py index 6328a6f5..6f819d1a 100644 --- a/contentctl/objects/correlation_search.py +++ b/contentctl/objects/correlation_search.py @@ -131,7 +131,7 @@ def __next__(self) -> dict[str, Any]: if isinstance(result, Message): # convert level string to level int level_name: str = result.type.strip().upper() # type: ignore - # TODO (cmcginley): this method is deprecated; replace with our own enum + # TODO (PEX-510): this method is deprecated; replace with our own enum level: int = logging.getLevelName(level_name) # log message at appropriate level and raise if needed From 8362af754acf327ec00daeb4bd5e189ae5722a84 Mon Sep 17 00:00:00 2001 From: Casey McGinley Date: Tue, 10 Dec 2024 12:28:12 -0800 Subject: [PATCH 17/17] annotating issues --- contentctl/actions/detection_testing/DetectionTestingManager.py | 1 + .../actions/detection_testing/views/DetectionTestingViewCLI.py | 2 +- contentctl/output/conf_output.py | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/contentctl/actions/detection_testing/DetectionTestingManager.py b/contentctl/actions/detection_testing/DetectionTestingManager.py index 1992de2b..1098f088 100644 --- a/contentctl/actions/detection_testing/DetectionTestingManager.py +++ b/contentctl/actions/detection_testing/DetectionTestingManager.py @@ -67,6 +67,7 @@ def sigint_handler(signum, frame): signal.signal(signal.SIGINT, sigint_handler) + # TODO (#337): futures can be hard to maintain/debug; let's consider alternatives with concurrent.futures.ThreadPoolExecutor( max_workers=len(self.input_dto.config.test_instances), ) as instance_pool, concurrent.futures.ThreadPoolExecutor( diff --git a/contentctl/actions/detection_testing/views/DetectionTestingViewCLI.py b/contentctl/actions/detection_testing/views/DetectionTestingViewCLI.py index a5ec8a24..f3b8ebc3 100644 --- a/contentctl/actions/detection_testing/views/DetectionTestingViewCLI.py +++ b/contentctl/actions/detection_testing/views/DetectionTestingViewCLI.py @@ -51,7 +51,7 @@ def showStatus(self, interval: int = 1): while True: summary = self.getSummaryObject() - # TODO (cmcginley): there's a 1-off error here I think (we show one more than we + # TODO (#338): there's a 1-off error here I think (we show one more than we # actually have during testing) total = len( summary.get("tested_detections", []) diff --git a/contentctl/output/conf_output.py b/contentctl/output/conf_output.py index 44aa4d26..6d63d7e1 100644 --- a/contentctl/output/conf_output.py +++ b/contentctl/output/conf_output.py @@ -79,7 +79,7 @@ def writeMiscellaneousAppFiles(self)->set[pathlib.Path]: return written_files - # TODO (cmcginley): we could have a discrepancy between detections tested and those delivered + # TODO (#339): we could have a discrepancy between detections tested and those delivered # based on the jinja2 template # {% if (detection.type == 'TTP' or detection.type == 'Anomaly' or detection.type == 'Hunting' or detection.type == 'Correlation') %} def writeObjects(self, objects: list, type: SecurityContentType = None) -> set[pathlib.Path]: