Skip to content

Commit

Permalink
Merge branch 'master' into master
Browse files Browse the repository at this point in the history
  • Loading branch information
jaegeral authored Oct 11, 2023
2 parents 0fe823c + 15a7991 commit e319e2d
Show file tree
Hide file tree
Showing 21 changed files with 628 additions and 328 deletions.
2 changes: 2 additions & 0 deletions cli_client/python/timesketch_cli_client/cli.py
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@
from timesketch_cli_client.commands import analyze
from timesketch_cli_client.commands import config
from timesketch_cli_client.commands import importer
from timesketch_cli_client.commands import intelligence
from timesketch_cli_client.commands import search
from timesketch_cli_client.commands import sketch as sketch_command
from timesketch_cli_client.commands import timelines
Expand Down Expand Up @@ -169,6 +170,7 @@ def cli(ctx, sketch, output):
cli.add_command(importer.importer)
cli.add_command(events.events_group)
cli.add_command(sigma.sigma_group)
cli.add_command(intelligence.intelligence_group)


# pylint: disable=no-value-for-parameter
Expand Down
145 changes: 145 additions & 0 deletions cli_client/python/timesketch_cli_client/commands/intelligence.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,145 @@
# Copyright 2023 Google Inc. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Commands for interacting with intelligence within a Sketch."""

import sys
import json
import click


@click.group("intelligence")
def intelligence_group():
"""Manage intelligence within a sketch."""


@intelligence_group.command("list")
@click.option(
"--header/--no-header",
default=True,
help="Include header in output. (default is to show header))",
)
@click.option(
"--columns",
default="ioc,type",
help="Comma separated list of columns to show. (default: ioc,type)",
)
@click.pass_context
def list_intelligence(ctx, header, columns):
"""List all intelligence.
Args:
ctx: Click context object.
header: Include header in output. (default is to show header)
columns: Comma separated list of columns to show. (default: ioc,type)
Other options: externalURI, tags
"""

if not columns:
columns = "ioc,type"

columns = columns.split(",")

output = ctx.obj.output_format
sketch = ctx.obj.sketch
try:
intelligence = sketch.get_intelligence_attribute()
except ValueError as e:
click.echo(e)
sys.exit(1)

if not intelligence:
click.echo("No intelligence found.")
ctx.exit(1)
if output == "json":
click.echo(json.dumps(intelligence, indent=4, sort_keys=True))
elif output == "text":
if header:
click.echo("\t".join(columns))
for entry in intelligence:
row = []
for column in columns:
if column == "tags":
row.append(",".join(entry.get(column, [])))
else:
row.append(entry.get(column, ""))
click.echo("\t".join(row))
elif output == "csv":
if header:
click.echo(",".join(columns))
for entry in intelligence:
row = []
for column in columns:
if column == "tags":
# Tags can be multiple values but they should only be
# one value on the csv so we join them with a comma
# surrounded the array by quotes
row.append(f'"{",".join(entry.get(column, []))}"')
else:
row.append(entry.get(column, ""))
click.echo(",".join(row))
else:
click.echo(f"Output format {output} not implemented.")


@intelligence_group.command("add")
@click.option(
"--ioc",
required=True,
help="Indicator Of Compromise (IOC) value.",
)
@click.option(
"--ioc-type",
required=False,
help="Type of the intelligence (ipv4, hash_sha256, hash_sha1, hash_md5, other).",
)
@click.option(
"--tags",
required=False,
help="Comma separated list of tags.",
)
@click.pass_context
def add_intelligence(ctx, ioc, tags, ioc_type="other"):
"""Add intelligence to a sketch.
A sketch can have multiple intelligence entries. Each entry consists of
an indicator, a type and a list of tags.
Reference: https://timesketch.org/guides/user/intelligence/
Args:
ctx: Click context object.
ioc: IOC value.
ioc_type: Type of the intelligence. This is defined in the ontology file.
If a string doesn't match any of the aforementioned IOC types,
the type will fall back to other.
tags: Comma separated list of tags.
"""
sketch = ctx.obj.sketch

# Create a tags dict from the comma separated list
if tags:
tags = tags.split(",")
tags = {tag: [] for tag in tags}
else:
tags = []

ioc_dict = {"ioc": ioc, "type": ioc_type, "tags": tags}
# Put the ioc in a nested object to match the format of the API
data = {"data": [ioc_dict]}
try:
sketch.add_attribute(name="intelligence", ontology="intelligence", value=data)
except ValueError as e:
click.echo(e)
sys.exit(1)
click.echo(f"Intelligence added: {ioc}")
Original file line number Diff line number Diff line change
@@ -0,0 +1,46 @@
# Copyright 2023 Google Inc. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Tests for intelligence command."""

import unittest
import mock

from click.testing import CliRunner

# pylint: disable=import-error
from timesketch_api_client import test_lib as api_test_lib

# pylint: enable=import-error

from .. import test_lib
from .intelligence import intelligence_group


class IntelligenceTest(unittest.TestCase):
"""Test Sigma CLI command."""

@mock.patch("requests.Session", api_test_lib.mock_session)
def setUp(self):
"""Setup test case."""
self.ctx = test_lib.get_cli_context()

def test_list_intelligence(self):
"""Test to list Sigma rules."""
runner = CliRunner()
result = runner.invoke(
intelligence_group,
["list"],
obj=self.ctx,
)
assert 1 is result.exit_code
3 changes: 2 additions & 1 deletion contrib/deploy_timesketch.sh
Original file line number Diff line number Diff line change
Expand Up @@ -97,7 +97,8 @@ curl -s $GITHUB_BASE_URL/data/intelligence_tag_metadata.yaml > timesketch/etc/ti
curl -s $GITHUB_BASE_URL/data/sigma_config.yaml > timesketch/etc/timesketch/sigma_config.yaml
curl -s $GITHUB_BASE_URL/data/sigma_rule_status.csv > timesketch/etc/timesketch/sigma_rule_status.csv
curl -s $GITHUB_BASE_URL/data/sigma/rules/lnx_susp_zmap.yml > timesketch/etc/timesketch/sigma/rules/lnx_susp_zmap.yml
curl -s $GITHUB_BASE_URL/data/plaso_formatters.yaml > timesketch/etc/plaso_formatters.yaml
curl -s $GITHUB_BASE_URL/data/plaso_formatters.yaml > timesketch/etc/timesketch/plaso_formatters.yaml
curl -s $GITHUB_BASE_URL/data/context_links.yaml > timesketch/etc/timesketch/context_links.yaml
curl -s $GITHUB_BASE_URL/contrib/nginx.conf > timesketch/etc/nginx.conf
echo "OK"

Expand Down
52 changes: 24 additions & 28 deletions data/features.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -121,149 +121,145 @@ ssh_client_ipv4_addresses:
query_string: 'reporter:"sshd"'
attribute: 'message'
store_as: 'client_ip'
re: '^\[sshd\] \[\d+\]: Connection from ((?:[0-9]{1,3}\.){3}[0-9]{1,3})
port \d+ on (?:[0-9]{1,3}\.){3}[0-9]{1,3} port \d+(?: rdomain ? .*)?$'
re: 'Connection from ((?:[0-9]{1,3}\.){3}[0-9]{1,3}) port \d+ on (?:[0-9]{1,3}\.){3}[0-9]{1,3} port \d+(?: rdomain ? .*)?$'

ssh_client_ipv4_addresses_2:
query_string: 'reporter:"sshd"'
attribute: 'message'
store_as: 'client_ip'
re: '\[sshd, pid: \d+\] Connection [a-z]+ by
((?:[0-9]{1,3}\.){3}[0-9]{1,3}) port \d+'
re: 'Connection [a-z]+ by ((?:[0-9]{1,3}\.){3}[0-9]{1,3}) port \d+'

ssh_host_ipv4_addresses:
query_string: 'reporter:"sshd"'
attribute: 'message'
store_as: 'host_ip'
re: '^\[sshd\] \[\d+\]: Connection from (?:[0-9]{1,3}\.){3}[0-9]{1,3}
port \d+ on ((?:[0-9]{1,3}\.){3}[0-9]{1,3}) port \d+(?: rdomain ? .*)?$'
re: '^\[sshd\] \[\d+\]: Connection from (?:[0-9]{1,3}\.){3}[0-9]{1,3} port \d+ on ((?:[0-9]{1,3}\.){3}[0-9]{1,3}) port \d+(?: rdomain ? .*)?$'

ssh_client_password_ipv4_addresses:
query_string: 'reporter:"sshd"'
attribute: 'message'
store_as: 'client_ip'
re: '^\[sshd, pid: \d+\] (?:Accepted|Failed) (?:password|publickey) for \w+
from ((?:[0-9]{1,3}\.){3}[0-9]{1,3}) port \d+'
re: '(?:Accepted|Failed) (?:password|publickey) for \w+ from ((?:[0-9]{1,3}\.){3}[0-9]{1,3}) port \d+'

ssh_disconnected_username:
query_string: 'reporter:"sshd"'
attribute: 'body'
store_as: 'username'
re: '^Disconnected\s+from user (?P<username>[^\s]+) [^\s]+ port \d+$'
re: 'Disconnected\s+from user (?P<username>[^\s]+) [^\s]+ port \d+$'

ssh_disconnected_ip_address:
query_string: 'reporter:"sshd"'
attribute: 'body'
store_as: 'ip_address'
re: '^Disconnected from user [^\s]+ (?P<ip_address>[^\s]+) port \d+$'
re: 'Disconnected from user [^\s]+ (?P<ip_address>[^\s]+) port \d+$'

ssh_disconnected_port:
query_string: 'reporter:"sshd"'
attribute: 'body'
store_as: 'port'
re: '^Disconnected from user [^\s]+ [^\s]+ port (?P<port>\d+)$'
re: 'Disconnected from user [^\s]+ [^\s]+ port (?P<port>\d+)$'

ssh_failed_username:
query_string: 'reporter:"sshd"'
attribute: 'body'
store_as: 'username'
re: '^Failed password for (?:invalid user)?\s*(?P<username>[^\s]+) from [^\s]+ port \d+ ssh\d'
re: 'Failed password for (?:invalid user)?\s*(?P<username>[^\s]+) from [^\s]+ port \d+ ssh\d'

ssh_failed_ip_address:
query_string: 'reporter:"sshd"'
attribute: 'body'
store_as: 'ip_address'
re: '^Failed password for (?:invalid user)?\s*[^\s]+ from (?P<ip_address>[^\s]+) port \d+ ssh\d'
re: 'Failed password for (?:invalid user)?\s*[^\s]+ from (?P<ip_address>[^\s]+) port \d+ ssh\d'

ssh_failed_port:
query_string: 'reporter:"sshd"'
attribute: 'body'
store_as: 'port'
re: '^Failed password for (?:invalid user)?\s*[^\s]+ from [^\s]+ port (?P<port>\d+) ssh\d'
re: 'Failed password for (?:invalid user)?\s*[^\s]+ from [^\s]+ port (?P<port>\d+) ssh\d'

ssh_failed_method:
query_string: 'reporter:"sshd"'
attribute: 'body'
store_as: 'authentication_method'
re: '^Failed (?P<authentication_method>[^\s]+) for .*ssh\d'
re: 'Failed (?P<authentication_method>[^\s]+) for .*ssh\d'

win_login_subject_username:
query_string: 'source_name:Microsoft-Windows-Security-Auditing AND (event_identifier:4624 OR event_identifier:4625)'
attribute: 'xml_string'
store_as: 'subject_username'
re: '.*"SubjectUserName">(?P<subject_username>[^<]+)</Data>'
re: '"SubjectUserName">(?P<subject_username>[^<]+)</Data>'

win_login_subject_domain:
query_string: 'source_name:Microsoft-Windows-Security-Auditing AND (event_identifier:4624 OR event_identifier:4625)'
attribute: 'xml_string'
store_as: 'subject_domain'
re: '.*"SubjectDomainName">(?P<subject_domain>[^<]+)</Data>'
re: '"SubjectDomainName">(?P<subject_domain>[^<]+)</Data>'

win_login_subject_logon_id:
query_string: 'source_name:Microsoft-Windows-Security-Auditing AND (event_identifier:4624 OR event_identifier:4625)'
attribute: 'xml_string'
store_as: 'subject_logon_id'
re: '.*"SubjectLogonId">(?P<subject_logon_id>[^<]+)</Data>'
re: '"SubjectLogonId">(?P<subject_logon_id>[^<]+)</Data>'

win_login_username:
query_string: 'source_name:Microsoft-Windows-Security-Auditing AND (event_identifier:4624 OR event_identifier:4625)'
attribute: 'xml_string'
store_as: 'username'
re: '.*"TargetUserName">(?P<username>[^<]+)</Data>'
re: '"TargetUserName">(?P<username>[^<]+)</Data>'

win_login_domain:
query_string: 'source_name:Microsoft-Windows-Security-Auditing AND (event_identifier:4624 OR event_identifier:4625)'
attribute: 'xml_string'
store_as: 'domain'
re: '.*"TargetDomainName">(?P<domain>[^<]+)</Data>'
re: '"TargetDomainName">(?P<domain>[^<]+)</Data>'

win_login_logon_id:
query_string: 'source_name:Microsoft-Windows-Security-Auditing AND (event_identifier:4624 OR event_identifier:4625)'
attribute: 'xml_string'
store_as: 'logon_id'
re: '.*"TargetLogonId">(?P<logon_id>[^<]+)</Data>'
re: '"TargetLogonId">(?P<logon_id>[^<]+)</Data>'

win_login_logon_type:
query_string: 'source_name:Microsoft-Windows-Security-Auditing AND (event_identifier:4624 OR event_identifier:4625)'
attribute: 'xml_string'
store_as: 'logon_type'
re: '.*"LogonType">(?P<logon_type>[^<]+)</Data>'
re: '"LogonType">(?P<logon_type>[^<]+)</Data>'

win_login_logon_process_name:
query_string: 'source_name:Microsoft-Windows-Security-Auditing AND (event_identifier:4624 OR event_identifier:4625)'
attribute: 'xml_string'
store_as: 'logon_process_name'
re: '.*"LogonProcessName">(?P<logon_process_name>[^<]+)</Data>'
re: '"LogonProcessName">(?P<logon_process_name>[^<]+)</Data>'

win_login_workstation_name:
query_string: 'source_name:Microsoft-Windows-Security-Auditing AND (event_identifier:4624 OR event_identifier:4625)'
attribute: 'xml_string'
store_as: 'workstation_name'
re: '.*"WorkstationName">(?P<workstation_name>[^<]+)</Data>'
re: '"WorkstationName">(?P<workstation_name>[^<]+)</Data>'

win_login_process_id:
query_string: 'source_name:Microsoft-Windows-Security-Auditing AND (event_identifier:4624 OR event_identifier:4625)'
attribute: 'xml_string'
store_as: 'process_id'
re: '.*"ProcessId">(?P<process_id>[^<]+)</Data>'
re: '"ProcessId">(?P<process_id>[^<]+)</Data>'

win_login_process_name:
query_string: 'source_name:Microsoft-Windows-Security-Auditing AND (event_identifier:4624 OR event_identifier:4625)'
attribute: 'xml_string'
store_as: 'process_name'
re: '.*"ProcessName">(?P<process_name>[^<]+)</Data>'
re: '"ProcessName">(?P<process_name>[^<]+)</Data>'

win_login_ip_address:
query_string: 'source_name:Microsoft-Windows-Security-Auditing AND (event_identifier:4624 OR event_identifier:4625)'
attribute: 'xml_string'
store_as: 'ip_address'
re: '.*"IpAddress">(?P<ip_address>[^<]+)</Data>'
re: '"IpAddress">(?P<ip_address>[^<]+)</Data>'

win_login_port:
query_string: 'source_name:Microsoft-Windows-Security-Auditing AND (event_identifier:4624 OR event_identifier:4625)'
attribute: 'xml_string'
store_as: 'port'
re: '.*"IpPort">(?P<port>[^<]+)</Data>'
re: '"IpPort">(?P<port>[^<]+)</Data>'

win_bits_client_ipv4_addresses:
query_string: 'data_type:"windows:evtx:record" AND source_name:Microsoft-Windows-Bits-Client'
Expand Down
Loading

0 comments on commit e319e2d

Please sign in to comment.