Merge branch 'master' into auth

docs/guides/admin/troubleshooting.md

@@ -61,7 +61,7 @@ See [docs/learn/server-admin](docs/learn/server-admin#troubleshooting-database-s
 - Is it a specific file that causes problems?
 - What is the WebUI status of the import?
 - Try switching from WebUI to the `import_client.py` to upload the same file
-- Try to upload one of the [sample files](https://github.com/google/timesketch/blob/master/test_tools/test_events/sigma_events.csv)
+- Try to upload one of the [sample files](https://github.com/google/timesketch/blob/master/tests/test_events/sigma_events.csv)
 - If you open a Github issue for an import issue, please indicate, what type of file you try to upload and what error message / stacktrace you have
 
 ### Issues importing a CSV file
@@ -71,7 +71,7 @@ See [docs/learn/server-admin](docs/learn/server-admin#troubleshooting-database-s
 - Is there an encoding issue in the CSV file
 - If you tried to upload via web, try the import client and the other way around
 - Check the celery logs
-- Try to upload [This sample](https://github.com/google/timesketch/blob/master/test_tools/test_events/sigma_events.csv)
+- Try to upload [This sample](https://github.com/google/timesketch/blob/master/tests/test_events/sigma_events.csv)
 - If you open a Github issue, provide at least the header of your CSV and a few lines of content (please scramble PII) so it can be reproduced.
 
 ### Issues importing Plaso file

notebooks/Sigma_test_Notebook.ipynb

@@ -82,7 +82,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "url = \"https://raw.githubusercontent.com/google/timesketch/master/test_tools/test_events/sigma_events.csv\" \n",
+    "url = \"https://raw.githubusercontent.com/google/timesketch/master/tests/test_events/sigma_events.csv\" \n",
     "download = requests.get(url).content"
    ]
   },

timesketch/api/v1/resources/analysis.py

@@ -51,6 +51,7 @@
 logger = logging.getLogger("timesketch.analysis_api")
 
 
+# TODO: Filter DFIQ analyzer results from this!
 class AnalysisResource(resources.ResourceMixin, Resource):
     """Resource to get analyzer session."""
 
@@ -180,6 +181,7 @@ def get(self, sketch_id):
               * display_name: Display name of the analyzer for the UI
               * description: Description of the analyzer provided in the class
               * is_multi: Boolean indicating if the analyzer is a multi analyzer
+              * is_dfiq: Boolean indicating if the analyzer is a DFIQ analyzer
         """
         sketch = Sketch.get_with_acl(sketch_id)
         if not sketch:
@@ -188,22 +190,26 @@ def get(self, sketch_id):
             abort(
                 HTTP_STATUS_CODE_FORBIDDEN, "User does not have read access to sketch"
             )
-        analyzers = [x for x, y in analyzer_manager.AnalysisManager.get_analyzers()]
 
-        analyzers = analyzer_manager.AnalysisManager.get_analyzers()
+        include_dfiq = (
+            request.args.get("include_dfiq", default="false").lower() == "true"
+        )
+
+        analyzers = analyzer_manager.AnalysisManager.get_analyzers(
+            include_dfiq=include_dfiq
+        )
         analyzers_detail = []
         for analyzer_name, analyzer_class in analyzers:
             # TODO: update the multi_analyzer detection logic for edgecases
             # where analyzers are using custom parameters (e.g. misp)
-            multi = False
-            if len(analyzer_class.get_kwargs()) > 0:
-                multi = True
             analyzers_detail.append(
                 {
                     "name": analyzer_name,
                     "display_name": analyzer_class.DISPLAY_NAME,
                     "description": analyzer_class.DESCRIPTION,
-                    "is_multi": multi,
+                    "is_multi": len(analyzer_class.get_kwargs()) > 0,
+                    "is_dfiq": hasattr(analyzer_class, "IS_DFIQ_ANALYZER")
+                    and analyzer_class.IS_DFIQ_ANALYZER,
                 }
             )
 
@@ -266,8 +272,17 @@ def post(self, sketch_id):
         if form.get("analyzer_force_run"):
             analyzer_force_run = True
 
+        include_dfiq = False
+        if form.get("include_dfiq"):
+            include_dfiq = True
+
         analyzers = []
-        all_analyzers = [x for x, _ in analyzer_manager.AnalysisManager.get_analyzers()]
+        all_analyzers = [
+            x
+            for x, _ in analyzer_manager.AnalysisManager.get_analyzers(
+                include_dfiq=include_dfiq
+            )
+        ]
         for analyzer in analyzer_names:
             for correct_name in all_analyzers:
                 if fnmatch.fnmatch(correct_name, analyzer):
@@ -301,6 +316,7 @@ def post(self, sketch_id):
                     analyzer_kwargs=analyzer_kwargs,
                     timeline_id=timeline_id,
                     analyzer_force_run=analyzer_force_run,
+                    include_dfiq=include_dfiq,
                 )
             except KeyError as e:
                 logger.warning(

timesketch/api/v1/resources/scenarios.py

@@ -37,6 +37,7 @@
 from timesketch.models.sketch import InvestigativeQuestion
 from timesketch.models.sketch import InvestigativeQuestionApproach
 from timesketch.models.sketch import InvestigativeQuestionConclusion
+from timesketch.lib.analyzers.dfiq_plugins.manager import DFIQAnalyzerManager
 
 
 logger = logging.getLogger("timesketch.scenario_api")
@@ -58,6 +59,59 @@ def load_dfiq_from_config():
     return DFIQ(dfiq_path)
 
 
+def check_and_run_dfiq_analysis_steps(dfiq_obj, sketch, analyzer_manager=None):
+    """Checks if any DFIQ analyzers need to be executed for the given DFIQ object.
+
+    Args:
+        dfiq_obj: The DFIQ object (Scenario, Question, or Approach).
+        sketch: The sketch object associated with the DFIQ object.
+        analyzer_manager: Optional. An existing instance of DFIQAnalyzerManager.
+
+    Returns:
+        List of analyzer_session objects (can be empty) or False.
+    """
+    # Initialize the analyzer manager only once.
+    if not analyzer_manager:
+        analyzer_manager = DFIQAnalyzerManager(sketch=sketch)
+
+    analyzer_sessions = []
+    if isinstance(dfiq_obj, InvestigativeQuestionApproach):
+        session = analyzer_manager.trigger_analyzers_for_approach(approach=dfiq_obj)
+        if session:
+            analyzer_sessions.extend(session)
+    elif isinstance(dfiq_obj, InvestigativeQuestion):
+        for approach in dfiq_obj.approaches:
+            session = analyzer_manager.trigger_analyzers_for_approach(approach=approach)
+            if session:
+                analyzer_sessions.extend(session)
+    elif isinstance(dfiq_obj, Facet):
+        for question in dfiq_obj.questions:
+            result = check_and_run_dfiq_analysis_steps(
+                question, sketch, analyzer_manager
+            )
+            if result:
+                analyzer_sessions.extend(result)
+    elif isinstance(dfiq_obj, Scenario):
+        if dfiq_obj.facets:
+            for facet in dfiq_obj.facets:
+                result = check_and_run_dfiq_analysis_steps(
+                    facet, sketch, analyzer_manager
+                )
+                if result:
+                    analyzer_sessions.extend(result)
+        if dfiq_obj.questions:
+            for question in dfiq_obj.questions:
+                result = check_and_run_dfiq_analysis_steps(
+                    question, sketch, analyzer_manager
+                )
+                if result:
+                    analyzer_sessions.extend(result)
+    else:
+        return False  # Invalid DFIQ object type
+
+    return analyzer_sessions if analyzer_sessions else False
+
+
 class ScenarioTemplateListResource(resources.ResourceMixin, Resource):
     """List all scenarios available."""
 
@@ -241,9 +295,23 @@ def post(self, sketch_id):
 
                     question_sql.approaches.append(approach_sql)
 
+                    db_session.add(question_sql)
+
+                # TODO: Remove commit and check function here when questions are
+                # linked to Scenarios again!
+                # Needs a tmp commit here so we can run the analyzer on the question.
+                db_session.commit()
+                # Check if any of the questions contains analyzer approaches
+                check_and_run_dfiq_analysis_steps(question_sql, sketch)
+
         db_session.add(scenario_sql)
         db_session.commit()
 
+        # This does not work, since we don't have Scnearios linked down to
+        # Approaches anymore! We intentionally broke the link to facets to show
+        # Questions in the frontend.
+        # check_and_run_dfiq_analysis_steps(scenario_sql, sketch)
+
         return self.to_json(scenario_sql)
 
 
@@ -594,6 +662,8 @@ def post(self, sketch_id):
         db_session.add(new_question)
         db_session.commit()
 
+        check_and_run_dfiq_analysis_steps(new_question, sketch)
+
         return self.to_json(new_question)
 
 

timesketch/api/v1/resources_test.py

@@ -26,6 +26,12 @@
 from timesketch.lib.definitions import HTTP_STATUS_CODE_INTERNAL_SERVER_ERROR
 from timesketch.lib.testlib import BaseTest
 from timesketch.lib.testlib import MockDataStore
+from timesketch.lib.dfiq import DFIQ
+from timesketch.api.v1.resources import scenarios
+from timesketch.models.sketch import Scenario
+from timesketch.models.sketch import InvestigativeQuestion
+from timesketch.models.sketch import InvestigativeQuestionApproach
+from timesketch.models.sketch import Facet
 
 from timesketch.api.v1.resources import ResourceMixin
 
@@ -1403,3 +1409,126 @@ def test_system_settings_resource(self):
         response = self.client.get(self.resource_url)
         expected_response = {"DFIQ_ENABLED": False, "LLM_PROVIDER": "test"}
         self.assertEqual(response.json, expected_response)
+
+
+class ScenariosResourceTest(BaseTest):
+    """Tests the scenarios resource."""
+
+    @mock.patch("timesketch.lib.analyzers.dfiq_plugins.manager.DFIQAnalyzerManager")
+    def test_check_and_run_dfiq_analysis_steps(self, mock_analyzer_manager):
+        """Test triggering analyzers for different DFIQ objects."""
+        test_sketch = self.sketch1
+        test_user = self.user1
+        self.sketch1.set_status("ready")
+        self._commit_to_database(test_sketch)
+
+        # Load DFIQ objects
+        dfiq_obj = DFIQ("./test_data/dfiq/")
+
+        scenario = dfiq_obj.scenarios[0]
+        scenario_sql = Scenario(
+            dfiq_identifier=scenario.id,
+            uuid=scenario.uuid,
+            name=scenario.name,
+            display_name=scenario.name,
+            description=scenario.description,
+            spec_json=scenario.to_json(),
+            sketch=test_sketch,
+            user=test_user,
+        )
+
+        facet = dfiq_obj.facets[0]
+        facet_sql = Facet(
+            dfiq_identifier=facet.id,
+            uuid=facet.uuid,
+            name=facet.name,
+            display_name=facet.name,
+            description=facet.description,
+            spec_json=facet.to_json(),
+            sketch=test_sketch,
+            user=test_user,
+        )
+        scenario_sql.facets = [facet_sql]
+
+        question = dfiq_obj.questions[0]
+        question_sql = InvestigativeQuestion(
+            dfiq_identifier=question.id,
+            uuid=question.uuid,
+            name=question.name,
+            display_name=question.name,
+            description=question.description,
+            spec_json=question.to_json(),
+            sketch=test_sketch,
+            scenario=scenario_sql,
+            user=test_user,
+        )
+        facet_sql.questions = [question_sql]
+
+        approach = question.approaches[0]
+        approach_sql = InvestigativeQuestionApproach(
+            name=approach.name,
+            display_name=approach.name,
+            description=approach.description,
+            spec_json=approach.to_json(),
+            user=test_user,
+        )
+        question_sql.approaches = [approach_sql]
+
+        self._commit_to_database(approach_sql)
+        self._commit_to_database(question_sql)
+        self._commit_to_database(facet_sql)
+        self._commit_to_database(scenario_sql)
+
+        # Test without analysis step
+        result = scenarios.check_and_run_dfiq_analysis_steps(scenario_sql, test_sketch)
+        self.assertFalse(result)
+
+        result = scenarios.check_and_run_dfiq_analysis_steps(facet_sql, test_sketch)
+        self.assertFalse(result)
+
+        result = scenarios.check_and_run_dfiq_analysis_steps(approach_sql, test_sketch)
+        self.assertFalse(result)
+
+        # Add analysis step to approach
+        approach.steps.append(
+            {
+                "stage": "analysis",
+                "type": "timesketch-analyzer",
+                "value": "test_analyzer",
+            }
+        )
+        approach_sql.spec_json = approach.to_json()
+
+        # Mocking analyzer manager response.
+        mock_analyzer_manager.trigger_analyzers_for_approach.return_value = [
+            mock.MagicMock()
+        ]
+
+        # Test with analysis step
+        result = scenarios.check_and_run_dfiq_analysis_steps(
+            scenario_sql, test_sketch, mock_analyzer_manager
+        )
+        self.assertEqual(result, [mock.ANY, mock.ANY])
+        mock_analyzer_manager.trigger_analyzers_for_approach.assert_called_with(
+            approach=approach_sql
+        )
+
+        result = scenarios.check_and_run_dfiq_analysis_steps(
+            facet_sql, test_sketch, mock_analyzer_manager
+        )
+        self.assertEqual(result, [mock.ANY])
+        mock_analyzer_manager.trigger_analyzers_for_approach.assert_called_with(
+            approach=approach_sql
+        )
+
+        result = scenarios.check_and_run_dfiq_analysis_steps(
+            question_sql, test_sketch, mock_analyzer_manager
+        )
+        self.assertEqual(result, [mock.ANY])
+        mock_analyzer_manager.trigger_analyzers_for_approach.assert_called_with(
+            approach=approach_sql
+        )
+
+        # Test with invalid object
+        result = scenarios.check_and_run_dfiq_analysis_steps("invalid", test_sketch)
+        self.assertFalse(result)

timesketch/lib/analyzers/__init__.py

@@ -40,3 +40,4 @@
 
 import timesketch.lib.analyzers.authentication
 import timesketch.lib.analyzers.contrib
+import timesketch.lib.analyzers.dfiq_plugins

timesketch/lib/analyzers/dfiq_plugins/__init__.py

@@ -0,0 +1,5 @@
+"""DFIQ Analyzer module."""
+
+from timesketch.lib.analyzers.dfiq_plugins import manager as dfiq_analyzer_manager
+
+dfiq_analyzer_manager.load_dfiq_analyzers()