Removing old feature extractor analyzer (#2969)

* Prevent the same comment to be added twice for winevt skipped mappings.
* Removing the old feature extractor analyzer.
* Change features config naming dependencies.

contrib/deploy_timesketch.ps1

@@ -66,7 +66,8 @@ Write-Host "* Fetching configuration files.."
 (Invoke-webrequest -URI $GITHUB_BASE_URL/data/tags.yaml).Content | out-file timesketch\etc\timesketch\tags.yaml -encoding UTF8NoBOM
 (Invoke-webrequest -URI $GITHUB_BASE_URL/data/plaso.mappings).Content | out-file timesketch\etc\timesketch\plaso.mappings -encoding UTF8NoBOM
 (Invoke-webrequest -URI $GITHUB_BASE_URL/data/generic.mappings).Content | out-file timesketch\etc\timesketch\generic.mappings -encoding UTF8NoBOM
-(Invoke-webrequest -URI $GITHUB_BASE_URL/data/features.yaml).Content | out-file timesketch\etc\timesketch\features.yaml -encoding UTF8NoBOM
+(Invoke-webrequest -URI $GITHUB_BASE_URL/data/regex_features.yaml).Content | out-file timesketch\etc\timesketch\regex_features.yaml -encoding UTF8NoBOM
+(Invoke-webrequest -URI $GITHUB_BASE_URL/data/winevt_features.yaml).Content | out-file timesketch\etc\timesketch\winevt_features.yaml -encoding UTF8NoBOM
 (Invoke-webrequest -URI $GITHUB_BASE_URL/data/ontology.yaml).Content | out-file timesketch\etc\timesketch\ontology.yaml -encoding UTF8NoBOM
 (Invoke-webrequest -URI $GITHUB_BASE_URL/data/intelligence_tag_metadata.yaml).Content | out-file timesketch\etc\timesketch\intelligence_tag_metadata.yaml -encoding UTF8NoBOM
 (Invoke-webrequest -URI $GITHUB_BASE_URL/data/sigma_config.yaml).Content | out-file timesketch\etc\timesketch\sigma_config.yaml -encoding UTF8NoBOM

contrib/deploy_timesketch.sh

@@ -89,7 +89,8 @@ curl -s $GITHUB_BASE_URL/data/timesketch.conf > timesketch/etc/timesketch/timesk
 curl -s $GITHUB_BASE_URL/data/tags.yaml > timesketch/etc/timesketch/tags.yaml
 curl -s $GITHUB_BASE_URL/data/plaso.mappings > timesketch/etc/timesketch/plaso.mappings
 curl -s $GITHUB_BASE_URL/data/generic.mappings > timesketch/etc/timesketch/generic.mappings
-curl -s $GITHUB_BASE_URL/data/features.yaml > timesketch/etc/timesketch/features.yaml
+curl -s $GITHUB_BASE_URL/data/regex_features.yaml > timesketch/etc/timesketch/regex_features.yaml
+curl -s $GITHUB_BASE_URL/data/winevt_features.yaml > timesketch/etc/timesketch/winevt_features.yaml
 curl -s $GITHUB_BASE_URL/data/ontology.yaml > timesketch/etc/timesketch/ontology.yaml
 curl -s $GITHUB_BASE_URL/data/sigma_rule_status.csv > timesketch/etc/timesketch/sigma_rule_status.csv
 curl -s $GITHUB_BASE_URL/data/tags.yaml > timesketch/etc/timesketch/tags.yaml

data/features.yaml → data/regex_features.yaml

@@ -183,84 +183,6 @@ ssh_failed_method:
         store_as: 'authentication_method'
         re: 'Failed (?P<authentication_method>[^\s]+) for .*ssh\d'
 
-win_login_subject_username:
-        query_string: 'source_name:Microsoft-Windows-Security-Auditing AND (event_identifier:4624 OR event_identifier:4625)'
-        attribute: 'xml_string'
-        store_as: 'subject_username'
-        re: '"SubjectUserName">(?P<subject_username>[^<]+)</Data>'
-
-win_login_subject_domain:
-        query_string: 'source_name:Microsoft-Windows-Security-Auditing AND (event_identifier:4624 OR event_identifier:4625)'
-        attribute: 'xml_string'
-        store_as: 'subject_domain'
-        re: '"SubjectDomainName">(?P<subject_domain>[^<]+)</Data>'
-
-win_login_subject_logon_id:
-        query_string: 'source_name:Microsoft-Windows-Security-Auditing AND (event_identifier:4624 OR event_identifier:4625)'
-        attribute: 'xml_string'
-        store_as: 'subject_logon_id'
-        re: '"SubjectLogonId">(?P<subject_logon_id>[^<]+)</Data>'
-
-win_login_username:
-        query_string: 'source_name:Microsoft-Windows-Security-Auditing AND (event_identifier:4624 OR event_identifier:4625)'
-        attribute: 'xml_string'
-        store_as: 'username'
-        re: '"TargetUserName">(?P<username>[^<]+)</Data>'
-
-win_login_domain:
-        query_string: 'source_name:Microsoft-Windows-Security-Auditing AND (event_identifier:4624 OR event_identifier:4625)'
-        attribute: 'xml_string'
-        store_as: 'domain'
-        re: '"TargetDomainName">(?P<domain>[^<]+)</Data>'
-
-win_login_logon_id:
-        query_string: 'source_name:Microsoft-Windows-Security-Auditing AND (event_identifier:4624 OR event_identifier:4625)'
-        attribute: 'xml_string'
-        store_as: 'logon_id'
-        re: '"TargetLogonId">(?P<logon_id>[^<]+)</Data>'
-
-win_login_logon_type:
-        query_string: 'source_name:Microsoft-Windows-Security-Auditing AND (event_identifier:4624 OR event_identifier:4625)'
-        attribute: 'xml_string'
-        store_as: 'logon_type'
-        re: '"LogonType">(?P<logon_type>[^<]+)</Data>'
-
-win_login_logon_process_name:
-        query_string: 'source_name:Microsoft-Windows-Security-Auditing AND (event_identifier:4624 OR event_identifier:4625)'
-        attribute: 'xml_string'
-        store_as: 'logon_process_name'
-        re: '"LogonProcessName">(?P<logon_process_name>[^<]+)</Data>'
-
-win_login_workstation_name:
-        query_string: 'source_name:Microsoft-Windows-Security-Auditing AND (event_identifier:4624 OR event_identifier:4625)'
-        attribute: 'xml_string'
-        store_as: 'workstation_name'
-        re: '"WorkstationName">(?P<workstation_name>[^<]+)</Data>'
-
-win_login_process_id:
-        query_string: 'source_name:Microsoft-Windows-Security-Auditing AND (event_identifier:4624 OR event_identifier:4625)'
-        attribute: 'xml_string'
-        store_as: 'process_id'
-        re: '"ProcessId">(?P<process_id>[^<]+)</Data>'
-
-win_login_process_name:
-        query_string: 'source_name:Microsoft-Windows-Security-Auditing AND (event_identifier:4624 OR event_identifier:4625)'
-        attribute: 'xml_string'
-        store_as: 'process_name'
-        re: '"ProcessName">(?P<process_name>[^<]+)</Data>'
-
-win_login_ip_address:
-        query_string: 'source_name:Microsoft-Windows-Security-Auditing AND (event_identifier:4624 OR event_identifier:4625)'
-        attribute: 'xml_string'
-        store_as: 'ip_address'
-        re: '"IpAddress">(?P<ip_address>[^<]+)</Data>'
-
-win_login_port:
-        query_string: 'source_name:Microsoft-Windows-Security-Auditing AND (event_identifier:4624 OR event_identifier:4625)'
-        attribute: 'xml_string'
-        store_as: 'port'
-        re: '"IpPort">(?P<port>[^<]+)</Data>'
-
 win_bits_client_ipv4_addresses:
         query_string: 'data_type:"windows:evtx:record" AND source_name:Microsoft-Windows-Bits-Client'
         attribute: 'strings'

docker/dev/build/docker-entrypoint.sh

@@ -9,7 +9,8 @@ if [ "$1" = 'timesketch' ]; then
   # Copy config files
   mkdir /etc/timesketch
   cp /usr/local/src/timesketch/data/timesketch.conf /etc/timesketch/
-  cp /usr/local/src/timesketch/data/features.yaml /etc/timesketch/
+  cp /usr/local/src/timesketch/data/regex_features.yaml /etc/timesketch/
+  cp /usr/local/src/timesketch/data/winevt_features.yaml /etc/timesketch/
   cp /usr/local/src/timesketch/data/tags.yaml /etc/timesketch/
   cp /usr/local/src/timesketch/data/intelligence_tag_metadata.yaml /etc/timesketch/
   cp /usr/local/src/timesketch/data/plaso.mappings /etc/timesketch/

docker/e2e/Dockerfile

@@ -37,7 +37,8 @@ RUN cp /tmp/timesketch/data/timesketch.conf /etc/timesketch/
 RUN cp /tmp/timesketch/data/ontology.yaml /etc/timesketch/
 RUN cp /tmp/timesketch/data/tags.yaml /etc/timesketch/
 RUN cp /tmp/timesketch/data/intelligence_tag_metadata.yaml /etc/timesketch/
-RUN cp /tmp/timesketch/data/features.yaml /etc/timesketch/
+RUN cp /tmp/timesketch/data/regex_features.yaml /etc/timesketch/
+RUN cp /tmp/timesketch/data/winevt_features.yaml /etc/timesketch/
 RUN cp /tmp/timesketch/data/plaso.mappings /etc/timesketch/
 RUN cp /tmp/timesketch/data/generic.mappings /etc/timesketch/
 RUN cp /tmp/timesketch/data/sigma_config.yaml /etc/timesketch/

docs/developers/analyzer-development.md

@@ -50,8 +50,8 @@ of the following.
 If you just want to extract a simple feature, e.g. want to extract a hostname or
 IP that is somewhere in the message field, or inside another attribute you don't
 have to write a new analyzer, you can take advantage of the feature_extraction
-analyzer. All you need to do is to edit the `features.yaml` file found here:
-https://github.com/google/timesketch/blob/master/data/features.yaml
+analyzer. All you need to do is to edit the `regex_features.yaml` file found here:
+https://github.com/google/timesketch/blob/master/data/regex_features.yaml
 
 An example extraction entry looks like this:
 

docs/guides/analyzers/feature_extraction.md

@@ -3,7 +3,7 @@ hide:
   - footer
 ---
 The feature extraction analyzer creates attributes out of event data based on regular expressions. Different
-features can be specified in the `data/features.yaml` file.
+features can be specified in the `data/regex_features.yaml` file.
 
 Please be aware that this analyzer does *not* extract ipv4, email-addresses and similar from *all* events, but only those that match the query_string.
 

timesketch/lib/analyzers/__init__.py

@@ -20,7 +20,6 @@
 from timesketch.lib.analyzers import chain
 from timesketch.lib.analyzers import domain
 from timesketch.lib.analyzers import expert_sessionizers
-from timesketch.lib.analyzers import feature_extraction_plugin
 from timesketch.lib.analyzers import feature_extraction
 from timesketch.lib.analyzers import gcp_logging
 from timesketch.lib.analyzers import geoip