From e3db367874db7425b3a66a6bf69258973b3f1ea7 Mon Sep 17 00:00:00 2001 From: Colby Prior Date: Wed, 19 Aug 2020 13:10:13 +1000 Subject: [PATCH 01/11] Fix _docker_run to actually run thug instead of returning test case --- api_app/script_analyzers/classes.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api_app/script_analyzers/classes.py b/api_app/script_analyzers/classes.py index 31991005dd..dbafec0df7 100644 --- a/api_app/script_analyzers/classes.py +++ b/api_app/script_analyzers/classes.py @@ -279,7 +279,7 @@ def _docker_run(self, req_data, req_files=None): """ # handle in case this is a test - if hasattr(self, "is_test"): + if hasattr(self, "is_test") and getattr(self, "is_test"): # only happens in case of testing self.report["success"] = True return {} From afc3ae3a302dc31096cd4fed26d521b16b622e97 Mon Sep 17 00:00:00 2001 From: Matteo Lodi Date: Wed, 19 Aug 2020 09:34:17 +0200 Subject: [PATCH 02/11] updated docs --- README.md | 14 ++++++++------ docs/source/Advanced-Usage.md | 31 ++++++++++++++++++++++++++++++- docs/source/Installation.md | 23 ++++------------------- 3 files changed, 42 insertions(+), 26 deletions(-) diff --git a/README.md b/README.md index 599aa2e7dd..b9e3e9279e 100644 --- a/README.md +++ b/README.md @@ -40,6 +40,8 @@ Documentation about IntelOwl installation, usage, contribution can be found at h [First announcement](https://www.certego.net/en/news/new-year-new-tool-intel-owl/) +[Daily Swig Article](https://portswigger.net/daily-swig/intel-owl-osint-tool-automates-the-intel-gathering-process-using-a-single-api) + ### Free Internal Modules Available - Static Doc Analysis @@ -143,14 +145,14 @@ This project was created and will be upgraded thanks to the following organizati ### Google Summer Of Code -The project was accepted to the GSoC 2020 under the Honeynet Project!! - -Stay tuned for upcoming [new features](https://www.honeynet.org/gsoc/gsoc-2020/google-summer-of-code-2020-project-ideas/#intel-owl-improvements) developed by Eshaan Bansal ([Twitter](https://twitter.com/mask0fmydisguis)). +The project was accepted to the GSoC 2020 under the Honeynet Project!! A lot of [new features](https://www.honeynet.org/gsoc/gsoc-2020/google-summer-of-code-2020-project-ideas/#intel-owl-improvements) were developed by Eshaan Bansal ([Twitter](https://twitter.com/mask0fmydisguis)). -### About the author +Stay tuned for the upcoming GSoC 2021! Join the [Honeynet Slack chat](https://gsoc-slack.honeynet.org/) for more info. -Feel free to contact the author at any time: -Matteo Lodi ([Twitter](https://twitter.com/matte_lodi)) +### About the author and maintainers +Feel free to contact the main developers at any time: +- Matteo Lodi ([Twitter](https://twitter.com/matte_lodi)): Author and creator +- Eshaan Bansal ([Twitter](https://twitter.com/mask0fmydisguis)): Principal maintainer We also have a dedicated twitter account for the project: [@intel_owl](https://twitter.com/intel_owl). diff --git a/docs/source/Advanced-Usage.md b/docs/source/Advanced-Usage.md index e36d6b3940..e57902dbdc 100644 --- a/docs/source/Advanced-Usage.md +++ b/docs/source/Advanced-Usage.md @@ -5,6 +5,8 @@ This page includes details about some advanced features that Intel Owl provides - [Elastic Search (with Kibana)](#elastic-search) - [Django Groups & Permissions](#django-groups-permissions) - [Optional Analyzers](#optional-analyzers) +- [Authentication options](#authentication-options) +- [GKE deployment](#google-kubernetes-engine-deployment) ## Elastic Search @@ -93,4 +95,31 @@ table, th, td { In the project, you can find template file `.env_file_integrations_template`. You have to create new file named `env_file_integrations` from this. Docker services defined in the compose files added in `COMPOSE_FILE` variable present in the `.env` file are ran on `docker-compose up`. So, modify it to include only the analyzers you wish to use. -Such compose files are available under `integrations/`. \ No newline at end of file +Such compose files are available under `integrations/`. + + +## Authentication options +IntelOwl provides support for some of the most common authentication methods: +* LDAP +* GSuite (work in progress) +#### LDAP +IntelOwl leverages [Django-auth-ldap](https://github.com/django-auth-ldap/django-auth-ldap +) to perform authentication via LDAP. + +How to configure and enable LDAP on Intel Owl? + +Inside the `settings` directory you can find a file called `ldap_config_template.py`. This file provides an example of configuration. +Copy that file into the same directory with the name `ldap_config.py`. +Then change the values with your LDAP configuration. + +For more details on how to configure this file, check the [official documentation](https://django-auth-ldap.readthedocs.io/en/latest/) of the django-auth-ldap library. + +Once you have done that, you have to set the environment variable `LDAP_ENABLED` as `True` in the environment configuration file `env_file_app`. +Finally, you can restart the application. + + + +## Google Kubernetes Engine deployment +Refer to the following blog post for an example on how to deploy IntelOwl on Google Kubernetes Engine: + +https://mostwanted002.cf/post/intel-owl-gke/ diff --git a/docs/source/Installation.md b/docs/source/Installation.md index c09d2fce49..eca1f25cac 100644 --- a/docs/source/Installation.md +++ b/docs/source/Installation.md @@ -30,6 +30,8 @@ docker exec -ti intel_owl_uwsgi python3 manage.py createsuperuser # now the app is running on localhost:80 ``` +Also, there is a [youtube video](https://www.youtube.com/watch?v=GuEhqQJSQAs) that may help in the installation process. + ## Deployment The project leverages docker-compose for a classic server deployment. So, you need [docker](https://docs.docker.com/get-docker/) and [docker-compose](https://docs.docker.com/compose/install/) installed in your machine. @@ -150,25 +152,8 @@ For a full description of the available keys, check the [Usage](./Usage.md) page > Some analyzers are kept optional and can easily be enabled. Refer to [this](https://intelowl.readthedocs.io/en/stable/Advanced-Usage.html#optional-analyzers) part of the docs. -#### Authentication options -IntelOwl provides support for some of the most common authentication methods: -* LDAP -* GSuite (work in progress) -##### LDAP -IntelOwl leverages [Django-auth-ldap](https://github.com/django-auth-ldap/django-auth-ldap -) to perform authentication via LDAP. - -How to configure and enable LDAP on Intel Owl? - -Inside the `settings` directory you can find a file called `ldap_config_template.py`. This file provides an example of configuration. -Copy that file into the same directory with the name `ldap_config.py`. -Then change the values with your LDAP configuration. - -For more details on how to configure this file, check the [official documentation](https://django-auth-ldap.readthedocs.io/en/latest/) of the django-auth-ldap library. - -Once you have done that, you have to set the environment variable `LDAP_ENABLED` as `True` in the environment configuration file `env_file_app`. -Finally, you can restart the application. - +### Authentication options (optional) +> Refer to [this](https://intelowl.readthedocs.io/en/stable/Advanced-Usage.html#authentication-options) part of the docs. ### Rebuilding the project If you make some code changes and you like to rebuild the project, launch the following command from the project directory: From 16e1ba439e90f90bb5bc6571563e0c42d2ec22f2 Mon Sep 17 00:00:00 2001 From: Matteo Lodi Date: Wed, 19 Aug 2020 12:37:14 +0200 Subject: [PATCH 03/11] adjusts to LDAP documentation --- docker-compose-for-tests.yml | 2 +- docs/source/Advanced-Usage.md | 5 ++++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/docker-compose-for-tests.yml b/docker-compose-for-tests.yml index c7de0e6311..3e0d19edc7 100644 --- a/docker-compose-for-tests.yml +++ b/docker-compose-for-tests.yml @@ -68,7 +68,7 @@ services: container_name: intel_owl_celery_worker restart: unless-stopped stop_grace_period: 3m - command: /usr/local/bin/celery -A intel_owl.celery worker --uid www-data --gid www-data --pidfile="/tmp/%n.pid" --time-limit=1000 + command: /usr/local/bin/celery -A intel_owl.celery worker --uid www-data --gid www-data --pidfile="/tmp/%n.pid" --time-limit=10000 volumes: - ./configuration/analyzer_config.json:/opt/deploy/configuration/analyzer_config.json - generic_logs:/var/log/intel_owl diff --git a/docs/source/Advanced-Usage.md b/docs/source/Advanced-Usage.md index e57902dbdc..a673539834 100644 --- a/docs/source/Advanced-Usage.md +++ b/docs/source/Advanced-Usage.md @@ -115,7 +115,10 @@ Then change the values with your LDAP configuration. For more details on how to configure this file, check the [official documentation](https://django-auth-ldap.readthedocs.io/en/latest/) of the django-auth-ldap library. Once you have done that, you have to set the environment variable `LDAP_ENABLED` as `True` in the environment configuration file `env_file_app`. -Finally, you can restart the application. +Finally, you can rebuild and restart the application by specifying the testing docker-compose file: +``` +docker-compose -f docker-compose-for-tests.yml up --build` +``` From 3390d2921d466cc4d2a711b0d40fae875b8bf120 Mon Sep 17 00:00:00 2001 From: Eshaan Bansal Date: Tue, 18 Aug 2020 21:49:50 +0530 Subject: [PATCH 04/11] added pulsedive for all types of observables --- api_app/script_analyzers/classes.py | 6 +- .../observable_analyzers/pulsedive.py | 92 +++++++++++++++++++ .../observable_analyzers/shodan.py | 16 ++-- configuration/analyzer_config.json | 10 ++ env_file_app_template | 1 + intel_owl/tasks.py | 18 ++++ tests/test_api.py | 2 + tests/test_observables.py | 9 +- tests/utils.py | 17 ++++ 9 files changed, 157 insertions(+), 14 deletions(-) create mode 100644 api_app/script_analyzers/observable_analyzers/pulsedive.py diff --git a/api_app/script_analyzers/classes.py b/api_app/script_analyzers/classes.py index dbafec0df7..ffeca92432 100644 --- a/api_app/script_analyzers/classes.py +++ b/api_app/script_analyzers/classes.py @@ -3,7 +3,7 @@ import logging import requests import json -from abc import ABC, abstractmethod +from abc import ABCMeta, abstractmethod from api_app.exceptions import ( AnalyzerRunNotImplemented, @@ -15,7 +15,7 @@ logger = logging.getLogger(__name__) -class BaseAnalyzerMixin(ABC): +class BaseAnalyzerMixin(metaclass=ABCMeta): """ Abstract Base class for Analyzers. Never inherit from this branch, @@ -177,7 +177,7 @@ def after_run(self): ) -class DockerBasedAnalyzer(ABC): +class DockerBasedAnalyzer(metaclass=ABCMeta): """ Abstract class for a docker based analyzer (integration). Inherit this branch along with either one of ObservableAnalyzer or FileAnalyzer diff --git a/api_app/script_analyzers/observable_analyzers/pulsedive.py b/api_app/script_analyzers/observable_analyzers/pulsedive.py new file mode 100644 index 0000000000..addf3ecb34 --- /dev/null +++ b/api_app/script_analyzers/observable_analyzers/pulsedive.py @@ -0,0 +1,92 @@ +import logging +import requests +import time + +from api_app.exceptions import AnalyzerRunException +from api_app.script_analyzers.classes import ObservableAnalyzer +from intel_owl import secrets + + +logger = logging.getLogger(__name__) + + +class Pulsedive(ObservableAnalyzer): + base_url: str = "https://pulsedive.com/api" + max_tries: int = 10 + poll_distance: int = 10 + + def set_config(self, additional_config_params): + self.api_key_name = additional_config_params.get( + "api_key_name", "PULSEDIVE_API_KEY" + ) + self.__api_key = secrets.get_secret(self.api_key_name) + active_scan = additional_config_params.get("active_scan", True) + self.probe = 1 if active_scan else 0 + + def run(self): + result = {} + if not self.__api_key: + warning = f"No API key retrieved with name: {self.api_key_name}" + logger.info( + f"{warning}. Continuing without API key..." f" <- {self.__repr__()}" + ) + self.report["errors"].append(warning) + else: + default_param = f"&key={self.__api_key}" + + # headers = {"Key": self.__api_key, "Accept": "application/json"} + # 1. query to info.php to check if the indicator is already in the database + params = f"indicator={self.observable_name}" + if self.__api_key: + params += default_param + resp = requests.get(f"{self.base_url}/info.php?{params}") + resp.raise_for_status() + result = resp.json() + e = result.get("error", None) + if e == "Indicator not found.": + # 2. submit new scan to analyze.php + params = f"value={self.observable_name}&probe={self.probe}" + if self.__api_key: + params += default_param + headers = { + "Content-Type": "application/x-www-form-urlencoded; charset=UTF-8" + } + resp = requests.post( + f"{self.base_url}/analyze.php", data=params, headers=headers + ) + resp.raise_for_status() + qid = resp.json().get("qid", None) + # 3. retrieve result using qid after waiting for 10 seconds + params = f"qid={qid}" + if self.__api_key: + params += default_param + result = self.__poll_for_result(params) + if result.get("data", None): + result = result["data"] + + return result + + def __poll_for_result(self, params): + result = {} + url = f"{self.base_url}/analyze.php?{params}" + obj_repr = self.__repr__() + for chance in range(self.max_tries): + logger.info( + f"polling request #{chance+1} for observable: {self.observable_name}" + f" <- {obj_repr}" + ) + time.sleep(self.poll_distance) + resp = requests.get(url) + resp.raise_for_status() + resp_json = resp.json() + status = resp_json.get("status", None) + if status == "done": + result = resp_json + break + elif status == "processing": + continue + else: + err = resp_json.get("error", "Report not found.") + raise AnalyzerRunException(err) + + return result diff --git a/api_app/script_analyzers/observable_analyzers/shodan.py b/api_app/script_analyzers/observable_analyzers/shodan.py index 2feffb5cd4..8f21f7602a 100644 --- a/api_app/script_analyzers/observable_analyzers/shodan.py +++ b/api_app/script_analyzers/observable_analyzers/shodan.py @@ -1,6 +1,6 @@ import requests -from api_app.exceptions import AnalyzerRunException +from api_app.exceptions import AnalyzerRunException, AnalyzerConfigurationException from api_app.script_analyzers import classes from intel_owl import secrets @@ -10,12 +10,14 @@ class Shodan(classes.ObservableAnalyzer): def set_config(self, additional_config_params): self.analysis_type = additional_config_params.get("shodan_analysis", "search") - api_key_name = additional_config_params.get("api_key_name", "SHODAN_KEY") - self.__api_key = secrets.get_secret(api_key_name) + self.api_key_name = additional_config_params.get("api_key_name", "SHODAN_KEY") + self.__api_key = secrets.get_secret(self.api_key_name) def run(self): if not self.__api_key: - raise AnalyzerRunException("no api key retrieved") + raise AnalyzerConfigurationException( + f"No API key retrieved with name: {self.api_key_name}." + ) if self.analysis_type == "search": params = {"key": self.__api_key, "minify": True} @@ -26,9 +28,9 @@ def run(self): } uri = f"labs/honeyscore/{self.observable_name}" else: - raise AnalyzerRunException( - f"not supported observable type {self.observable_classification}." - "Supported is IP" + raise AnalyzerConfigurationException( + f"analysis type: '{self.analysis_type}' not suported." + "Supported are: 'search', 'honeyscore'." ) try: diff --git a/configuration/analyzer_config.json b/configuration/analyzer_config.json index 8a9a21caf8..0b096f6178 100644 --- a/configuration/analyzer_config.json +++ b/configuration/analyzer_config.json @@ -897,5 +897,15 @@ "enable_awis": true, "user_agent": "nexuschrome18" } + }, + "Pulsedive_Active_IOC": { + "type": "observable", + "observable_supported": ["ip", "domain", "url", "hash"], + "description": "Scan indicators and retrieve results from Pulsedive's API", + "python_module": "pulsedive_run", + "additional_config_params": { + "api_key_name": "PULSEDIVE_API_KEY", + "active_scan": true + } } } diff --git a/env_file_app_template b/env_file_app_template index eaef58112f..ae46b6e418 100644 --- a/env_file_app_template +++ b/env_file_app_template @@ -41,6 +41,7 @@ CENSYS_API_ID= CENSYS_API_SECRET= ONYPHE_KEY= GREYNOISE_API_KEY= +PULSEDIVE_API_KEY= # Test tokens TEST_JOB_ID=1 diff --git a/intel_owl/tasks.py b/intel_owl/tasks.py index 81bee6f48f..49bef4ad6d 100644 --- a/intel_owl/tasks.py +++ b/intel_owl/tasks.py @@ -51,6 +51,7 @@ securitytrails, cymru, tranco, + pulsedive, ) from api_app import crons @@ -713,6 +714,23 @@ def tranco_run( ).start() +@shared_task(soft_time_limit=100) +def pulsedive_run( + analyzer_name, + job_id, + observable_name, + observable_classification, + additional_config_params, +): + pulsedive.Pulsedive( + analyzer_name, + job_id, + observable_name, + observable_classification, + additional_config_params, + ).start() + + @shared_task(soft_time_limit=500) def peframe_run( analyzer_name, job_id, filepath, filename, md5, additional_config_params diff --git a/tests/test_api.py b/tests/test_api.py index 152b544051..5658ba2413 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -126,6 +126,7 @@ def test_send_analysis_request_domain(self): "Threatminer_Subdomains", "ONYPHE", "URLhaus", + "Pulsedive_Active_IOC", ] observable_name = os.environ.get("TEST_DOMAIN", "google.com") md5 = hashlib.md5(observable_name.encode("utf-8")).hexdigest() @@ -165,6 +166,7 @@ def test_send_analysis_request_ip(self): "ONYPHE", "HoneyDB_Scan_Twitter", "HoneyDB_Get", + "Pulsedive_Active_IOC", ] observable_name = os.environ.get("TEST_IP", "8.8.8.8") md5 = hashlib.md5(observable_name.encode("utf-8")).hexdigest() diff --git a/tests/test_observables.py b/tests/test_observables.py index 74469f3888..6d1fe7ccda 100644 --- a/tests/test_observables.py +++ b/tests/test_observables.py @@ -43,6 +43,7 @@ thug_url, ) from .utils import ( + CommonTestCases, MockResponseNoOp, mocked_requests, ) @@ -73,7 +74,7 @@ def mocked_pypdns(*args, **kwargs): @mock_connections(patch("requests.get", side_effect=mocked_requests)) @mock_connections(patch("requests.post", side_effect=mocked_requests)) -class IPAnalyzersTests(TestCase): +class IPAnalyzersTests(CommonTestCases, TestCase): def setUp(self): params = { "source": "test", @@ -345,7 +346,7 @@ def active_dns_classic_reverse(self, mock_get=None, mock_post=None): @mock_connections(patch("requests.get", side_effect=mocked_requests)) @mock_connections(patch("requests.post", side_effect=mocked_requests)) -class DomainAnalyzersTests(TestCase): +class DomainAnalyzersTests(CommonTestCases, TestCase): def setUp(self): params = { "source": "test", @@ -590,7 +591,7 @@ def test_thug_url(self, mock_get=None, mock_post=None): @mock_connections(patch("requests.get", side_effect=mocked_requests)) @mock_connections(patch("requests.post", side_effect=mocked_requests)) -class URLAnalyzersTests(TestCase): +class URLAnalyzersTests(CommonTestCases, TestCase): def setUp(self): params = { "source": "test", @@ -712,7 +713,7 @@ def test_thug_url(self, mock_get=None, mock_post=None): @mock_connections(patch("requests.get", side_effect=mocked_requests)) @mock_connections(patch("requests.post", side_effect=mocked_requests)) -class HashAnalyzersTests(TestCase): +class HashAnalyzersTests(CommonTestCases, TestCase): def setUp(self): params = { "source": "test", diff --git a/tests/utils.py b/tests/utils.py index 4b5a3b3929..e81ca59c3f 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -1,4 +1,5 @@ # utils.py: useful utils for mocking requests and responses for testing +from api_app.script_analyzers.observable_analyzers import pulsedive # class for mocking responses @@ -40,3 +41,19 @@ def mocked_docker_analyzer_get(*args, **kwargs): def mocked_docker_analyzer_post(*args, **kwargs): return MockResponse({"key": "test", "status": "running"}, 202) + + +class CommonTestCases: + """ + Tests which are common for all types of observables. + """ + + def test_pulsevide(self, mock_get=None, mock_post=None): + report = pulsedive.Pulsedive( + "Pulsedive_Active_IOC", + self.job_id, + self.observable_name, + self.observable_classification, + {}, + ).start() + self.assertEqual(report.get("success", False), True) From f236d8b175dfd6e5312cb0f9178a98cad7117940 Mon Sep 17 00:00:00 2001 From: Eshaan Bansal Date: Tue, 18 Aug 2020 22:57:05 +0530 Subject: [PATCH 05/11] refactor tests/ code to reduce duplicate code --- tests/mock_utils.py | 52 +++++ tests/test_files.py | 2 +- tests/test_observables.py | 399 ++++---------------------------------- tests/utils.py | 178 +++++++++++++---- 4 files changed, 238 insertions(+), 393 deletions(-) create mode 100644 tests/mock_utils.py diff --git a/tests/mock_utils.py b/tests/mock_utils.py new file mode 100644 index 0000000000..b676e80ef6 --- /dev/null +++ b/tests/mock_utils.py @@ -0,0 +1,52 @@ +# mock_utils.py: useful utils for mocking requests and responses for testing +from intel_owl import settings + + +# class for mocking responses +class MockResponse: + def __init__(self, json_data, status_code): + self.json_data = json_data + self.status_code = status_code + self.text = "" + self.content = b"" + + def json(self): + return self.json_data + + def raise_for_status(self): + pass + + +# a mock response class that has no operation +class MockResponseNoOp: + def __init__(self, json_data, status_code): + pass + + def search(self, **kwargs): + return {} + + def query(self, val): + return {} + + +# it is optional to mock requests +def mock_connections(decorator): + return decorator if settings.MOCK_CONNECTIONS else lambda x: x + + +def mocked_requests(*args, **kwargs): + return MockResponse({}, 200) + + +def mocked_requests_noop(*args, **kwargs): + return MockResponseNoOp({}, 200) + + +def mocked_docker_analyzer_get(*args, **kwargs): + return MockResponse( + {"key": "test", "returncode": 0, "report": {"test": "This is a test."}}, 200 + ) + + +def mocked_docker_analyzer_post(*args, **kwargs): + return MockResponse({"key": "test", "status": "running"}, 202) diff --git a/tests/test_files.py b/tests/test_files.py index 78689a462e..e44f7a6274 100644 --- a/tests/test_files.py +++ b/tests/test_files.py @@ -28,7 +28,7 @@ from api_app.script_analyzers.observable_analyzers import vt3_get from api_app.models import Job -from .utils import ( +from .mock_utils import ( MockResponse, mocked_requests, mocked_docker_analyzer_get, diff --git a/tests/test_observables.py b/tests/test_observables.py index 6d1fe7ccda..1b1e058095 100644 --- a/tests/test_observables.py +++ b/tests/test_observables.py @@ -1,7 +1,6 @@ # for observable analyzers, if can customize the behavior based on: # DISABLE_LOGGING_TEST to True -> logging disabled # MOCK_CONNECTIONS to True -> connections to external analyzers are faked -import hashlib import logging import os from unittest import skipIf @@ -9,44 +8,40 @@ from django.test import TestCase -from api_app.models import Job from api_app.script_analyzers.observable_analyzers import ( abuseipdb, censys, shodan, - fortiguard, maxmind, greynoise, - googlesf, - otx, talos, tor, circl_pssl, circl_pdns, robtex, - vt2_get, - ha_get, - vt3_get, - misp, dnsdb, honeydb, hunter, mb_get, - onyphe, threatminer, - urlhaus, active_dns, auth0, securitytrails, cymru, tranco, - thug_url, ) -from .utils import ( - CommonTestCases, +from .mock_utils import ( MockResponseNoOp, + mock_connections, mocked_requests, ) +from .utils import ( + CommonTestCases_observables, + CommonTestCases_ip_url_domain, + CommonTestCases_ip_domain_hash, + CommonTestCases_url_domain, +) + from intel_owl import settings logger = logging.getLogger(__name__) @@ -55,15 +50,6 @@ logging.disable(logging.CRITICAL) -# it is optional to mock requests -def mock_connections(decorator): - return decorator if settings.MOCK_CONNECTIONS else lambda x: x - - -def mocked_pymisp(*args, **kwargs): - return MockResponseNoOp({}, 200) - - def mocked_pypssl(*args, **kwargs): return MockResponseNoOp({}, 200) @@ -74,9 +60,15 @@ def mocked_pypdns(*args, **kwargs): @mock_connections(patch("requests.get", side_effect=mocked_requests)) @mock_connections(patch("requests.post", side_effect=mocked_requests)) -class IPAnalyzersTests(CommonTestCases, TestCase): - def setUp(self): - params = { +class IPAnalyzersTests( + CommonTestCases_ip_domain_hash, + CommonTestCases_ip_url_domain, + CommonTestCases_observables, + TestCase, +): + @staticmethod + def get_params(): + return { "source": "test", "is_sample": False, "observable_name": os.environ.get("TEST_IP", "8.8.8.8"), @@ -84,14 +76,6 @@ def setUp(self): "force_privacy": False, "analyzers_requested": ["test"], } - params["md5"] = hashlib.md5( - params["observable_name"].encode("utf-8") - ).hexdigest() - test_job = Job(**params) - test_job.save() - self.job_id = test_job.id - self.observable_name = test_job.observable_name - self.observable_classification = test_job.observable_classification def test_abuseipdb(self, mock_get=None, mock_post=None): report = abuseipdb.AbuseIPDB( @@ -204,22 +188,6 @@ def test_greynoise(self, mock_get=None, mock_post=None): ).start() self.assertEqual(report.get("success", False), True) - def test_gsf(self, mock_get=None, mock_post=None): - report = googlesf.GoogleSF( - "GoogleSafeBrowsing", - self.job_id, - self.observable_name, - self.observable_classification, - {}, - ).start() - self.assertEqual(report.get("success", False), True) - - def test_otx(self, mock_get=None, mock_post=None): - report = otx.OTX( - "OTX", self.job_id, self.observable_name, self.observable_classification, {} - ).start() - self.assertEqual(report.get("success", False), True) - def test_talos(self, mock_get=None, mock_post=None): report = talos.Talos( "TalosReputation", @@ -281,57 +249,6 @@ def test_dnsdb(self, mock_get=None, mock_post=None): ).start() self.assertEqual(report.get("success", False), True) - def test_vt2_get(self, mock_get=None, mock_post=None): - report = vt2_get.VirusTotalv2( - "VT_v2_Get", - self.job_id, - self.observable_name, - self.observable_classification, - {}, - ).start() - self.assertEqual(report.get("success", False), True) - - def test_ha_get(self, mock_get=None, mock_post=None): - report = ha_get.HybridAnalysisGet( - "HybridAnalysis_Get_Observable", - self.job_id, - self.observable_name, - self.observable_classification, - {}, - ).start() - self.assertEqual(report.get("success", False), True) - - def test_vt3_get(self, mock_get=None, mock_post=None): - report = vt3_get.VirusTotalv3( - "VT_v3_Get", - self.job_id, - self.observable_name, - self.observable_classification, - {}, - ).start() - self.assertEqual(report.get("success", False), True) - - @mock_connections(patch("pymisp.ExpandedPyMISP", side_effect=mocked_pymisp)) - def test_misp_first(self, mock_get=None, mock_post=None, mock_pymisp=None): - report = misp.MISP( - "MISP_FIRST", - self.job_id, - self.observable_name, - self.observable_classification, - {"api_key_name": "FIRST_MISP_API", "url_key_name": "FIRST_MISP_URL"}, - ).start() - self.assertEqual(report.get("success", False), True) - - def test_onyphe(self, mock_get=None, mock_post=None): - report = onyphe.Onyphe( - "ONYPHE", - self.job_id, - self.observable_name, - self.observable_classification, - {}, - ).start() - self.assertEqual(report.get("success", False), True) - def active_dns_classic_reverse(self, mock_get=None, mock_post=None): report = active_dns.active_dns.ActiveDNS( "ActiveDNS_Classic_reverse", @@ -346,9 +263,16 @@ def active_dns_classic_reverse(self, mock_get=None, mock_post=None): @mock_connections(patch("requests.get", side_effect=mocked_requests)) @mock_connections(patch("requests.post", side_effect=mocked_requests)) -class DomainAnalyzersTests(CommonTestCases, TestCase): - def setUp(self): - params = { +class DomainAnalyzersTests( + CommonTestCases_ip_domain_hash, + CommonTestCases_ip_url_domain, + CommonTestCases_url_domain, + CommonTestCases_observables, + TestCase, +): + @staticmethod + def get_params(): + return { "source": "test", "is_sample": False, "observable_name": os.environ.get("TEST_DOMAIN", "www.google.com"), @@ -356,24 +280,6 @@ def setUp(self): "force_privacy": False, "analyzers_requested": ["test"], } - params["md5"] = hashlib.md5( - params["observable_name"].encode("utf-8") - ).hexdigest() - test_job = Job(**params) - test_job.save() - self.job_id = test_job.id - self.observable_name = test_job.observable_name - self.observable_classification = test_job.observable_classification - - def test_fortiguard(self, mock_get=None, mock_post=None): - report = fortiguard.Fortiguard( - "Fortiguard", - self.job_id, - self.observable_name, - self.observable_classification, - {}, - ).start() - self.assertEqual(report.get("success", False), True) def test_tranco(self, mock_get=None, mock_post=None): report = tranco.Tranco( @@ -415,22 +321,6 @@ def test_threatminer_domain(self, mock_get=None, mock_post=None): ).start() self.assertEqual(report.get("success", False), True) - def test_gsf(self, mock_get=None, mock_post=None): - report = googlesf.GoogleSF( - "GoogleSafeBrowsing", - self.job_id, - self.observable_name, - self.observable_classification, - {}, - ).start() - self.assertEqual(report.get("success", False), True) - - def test_otx(self, mock_get=None, mock_post=None): - report = otx.OTX( - "OTX", self.job_id, self.observable_name, self.observable_classification, {} - ).start() - self.assertEqual(report.get("success", False), True) - @mock_connections(patch("pypdns.PyPDNS", side_effect=mocked_pypdns)) def test_circl_pdns(self, mock_get=None, mock_post=None, sessions_get=None): report = circl_pdns.CIRCL_PDNS( @@ -462,67 +352,6 @@ def test_dnsdb(self, mock_get=None, mock_post=None): ).start() self.assertEqual(report.get("success", False), True) - def test_vt2_get(self, mock_get=None, mock_post=None): - report = vt2_get.VirusTotalv2( - "VT_v2_Get", - self.job_id, - self.observable_name, - self.observable_classification, - {}, - ).start() - self.assertEqual(report.get("success", False), True) - - def test_ha_get(self, mock_get=None, mock_post=None): - report = ha_get.HybridAnalysisGet( - "HybridAnalysis_Get_Observable", - self.job_id, - self.observable_name, - self.observable_classification, - {}, - ).start() - self.assertEqual(report.get("success", False), True) - - def test_vt3_get(self, mock_get=None, mock_post=None): - report = vt3_get.VirusTotalv3( - "VT_v3_Get", - self.job_id, - self.observable_name, - self.observable_classification, - {}, - ).start() - self.assertEqual(report.get("success", False), True) - - @mock_connections(patch("pymisp.ExpandedPyMISP", side_effect=mocked_pymisp)) - def test_misp_first(self, mock_get=None, mock_post=None, mock_pymisp=None): - report = misp.MISP( - "MISP_FIRST", - self.job_id, - self.observable_name, - self.observable_classification, - {"api_key_name": "FIRST_MISP_API", "url_key_name": "FIRST_MISP_URL"}, - ).start() - self.assertEqual(report.get("success", False), True) - - def test_onyphe(self, mock_get=None, mock_post=None): - report = onyphe.Onyphe( - "ONYPHE", - self.job_id, - self.observable_name, - self.observable_classification, - {}, - ).start() - self.assertEqual(report.get("success", False), True) - - def test_urlhaus(self, mock_get=None, mock_post=None): - report = urlhaus.URLHaus( - "URLhaus", - self.job_id, - self.observable_name, - self.observable_classification, - {}, - ).start() - self.assertEqual(report.get("success", False), True) - def test_active_dns(self, mock_get=None, mock_post=None): # Google google_report = active_dns.ActiveDNS( @@ -577,23 +406,18 @@ def test_cloudFlare_malware(self, mock_get=None, mock_post=None): self.assertEqual(report.get("success", False), True, f"report: {report}") - def test_thug_url(self, mock_get=None, mock_post=None): - additional_params = {"test": True} - report = thug_url.ThugUrl( - "Thug_URL_Info", - self.job_id, - self.observable_name, - self.observable_classification, - additional_params, - ).start() - self.assertEqual(report.get("success", False), True) - @mock_connections(patch("requests.get", side_effect=mocked_requests)) @mock_connections(patch("requests.post", side_effect=mocked_requests)) -class URLAnalyzersTests(CommonTestCases, TestCase): - def setUp(self): - params = { +class URLAnalyzersTests( + CommonTestCases_ip_url_domain, + CommonTestCases_url_domain, + CommonTestCases_observables, + TestCase, +): + @staticmethod + def get_params(): + return { "source": "test", "is_sample": False, "observable_name": os.environ.get( @@ -603,40 +427,6 @@ def setUp(self): "force_privacy": False, "analyzers_requested": ["test"], } - params["md5"] = hashlib.md5( - params["observable_name"].encode("utf-8") - ).hexdigest() - test_job = Job(**params) - test_job.save() - self.job_id = test_job.id - self.observable_name = test_job.observable_name - self.observable_classification = test_job.observable_classification - - def test_fortiguard(self, mock_get=None, mock_post=None): - report = fortiguard.Fortiguard( - "Fortiguard", - self.job_id, - self.observable_name, - self.observable_classification, - {}, - ).start() - self.assertEqual(report.get("success", False), True) - - def test_gsf(self, mock_get=None, mock_post=None): - report = googlesf.GoogleSF( - "GoogleSafeBrowsing", - self.job_id, - self.observable_name, - self.observable_classification, - {}, - ).start() - self.assertEqual(report.get("success", False), True) - - def test_otx(self, mock_get=None, mock_post=None): - report = otx.OTX( - "OTX", self.job_id, self.observable_name, self.observable_classification, {} - ).start() - self.assertEqual(report.get("success", False), True) @mock_connections(patch("pypdns.PyPDNS", side_effect=mocked_pypdns)) def test_circl_pdns(self, mock_get=None, mock_post=None, sessions_get=None): @@ -659,63 +449,15 @@ def test_robtex_fdns(self, mock_get=None, mock_post=None): ).start() self.assertEqual(report.get("success", False), True) - def test_vt2_get(self, mock_get=None, mock_post=None): - report = vt2_get.VirusTotalv2( - "VT_v2_Get", - self.job_id, - self.observable_name, - self.observable_classification, - {}, - ).start() - self.assertEqual(report.get("success", False), True) - - def test_vt3_get(self, mock_get=None, mock_post=None): - report = vt3_get.VirusTotalv3( - "VT_v3_Get", - self.job_id, - self.observable_name, - self.observable_classification, - {}, - ).start() - self.assertEqual(report.get("success", False), True) - - def test_onyphe(self, mock_get=None, mock_post=None): - report = onyphe.Onyphe( - "ONYPHE", - self.job_id, - self.observable_name, - self.observable_classification, - {}, - ).start() - self.assertEqual(report.get("success", False), True) - - def test_urlhaus(self, mock_get=None, mock_post=None): - report = urlhaus.URLHaus( - "URLhaus", - self.job_id, - self.observable_name, - self.observable_classification, - {}, - ).start() - self.assertEqual(report.get("success", False), True) - - def test_thug_url(self, mock_get=None, mock_post=None): - additional_params = {"test": True} - report = thug_url.ThugUrl( - "Thug_URL_Info", - self.job_id, - self.observable_name, - self.observable_classification, - additional_params, - ).start() - self.assertEqual(report.get("success", False), True) - @mock_connections(patch("requests.get", side_effect=mocked_requests)) @mock_connections(patch("requests.post", side_effect=mocked_requests)) -class HashAnalyzersTests(CommonTestCases, TestCase): - def setUp(self): - params = { +class HashAnalyzersTests( + CommonTestCases_ip_domain_hash, CommonTestCases_observables, TestCase +): + @staticmethod + def get_params(): + return { "source": "test", "is_sample": False, "observable_name": os.environ.get( @@ -725,61 +467,6 @@ def setUp(self): "force_privacy": False, "analyzers_requested": ["test"], } - params["md5"] = hashlib.md5( - params["observable_name"].encode("utf-8") - ).hexdigest() - test_job = Job(**params) - test_job.save() - self.job_id = test_job.id - self.observable_name = test_job.observable_name - self.observable_classification = test_job.observable_classification - - def test_otx(self, mock_get=None, mock_post=None): - report = otx.OTX( - "OTX", self.job_id, self.observable_name, self.observable_classification, {} - ).start() - self.assertEqual(report.get("success", False), True) - - def test_vt2_get(self, mock_get=None, mock_post=None): - report = vt2_get.VirusTotalv2( - "VT_v2_Get", - self.job_id, - self.observable_name, - self.observable_classification, - {}, - ).start() - self.assertEqual(report.get("success", False), True) - - def test_ha_get(self, mock_get=None, mock_post=None): - report = ha_get.HybridAnalysisGet( - "HybridAnalysis_Get_Observable", - self.job_id, - self.observable_name, - self.observable_classification, - {}, - ).start() - self.assertEqual(report.get("success", False), True) - - def test_vt3_get(self, mock_get=None, mock_post=None): - report = vt3_get.VirusTotalv3( - "VT_v3_Get", - self.job_id, - self.observable_name, - self.observable_classification, - {}, - ).start() - self.assertEqual(report.get("success", False), True) - - @mock_connections(patch("pymisp.ExpandedPyMISP", side_effect=mocked_pymisp)) - def test_misp_first(self, mock_get=None, mock_post=None, mock_pymisp=None): - report = misp.MISP( - "MISP_FIRST", - self.job_id, - self.observable_name, - self.observable_classification, - {"api_key_name": "FIRST_MISP_API", "url_key_name": "FIRST_MISP_URL"}, - ).start() - self.assertEqual(report.get("success", False), True) def test_mb_get(self, mock_get=None, mock_post=None): report = mb_get.MB_GET( diff --git a/tests/utils.py b/tests/utils.py index e81ca59c3f..6648bb45ec 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -1,59 +1,165 @@ -# utils.py: useful utils for mocking requests and responses for testing -from api_app.script_analyzers.observable_analyzers import pulsedive - - -# class for mocking responses -class MockResponse: - def __init__(self, json_data, status_code): - self.json_data = json_data - self.status_code = status_code - self.text = "" - self.content = b"" +import hashlib +from abc import ABCMeta +from unittest.mock import patch + +from api_app.models import Job +from api_app.script_analyzers.observable_analyzers import ( + pulsedive, + otx, + vt2_get, + vt3_get, + misp, + onyphe, + ha_get, + thug_url, + urlhaus, + googlesf, + fortiguard, +) + +from .mock_utils import mock_connections, mocked_requests_noop + + +# Abstract Base classes constructed for most common occuring combinations +# to avoid duplication of code +class CommonTestCases_observables(metaclass=ABCMeta): + """ + Includes tests which are common for all types of observables. + """ - def json(self): - return self.json_data + def setUp(self): + params = self.get_params() + params["md5"] = hashlib.md5( + params["observable_name"].encode("utf-8") + ).hexdigest() + test_job = Job(**params) + test_job.save() + self.job_id = test_job.id + self.observable_name = test_job.observable_name + self.observable_classification = test_job.observable_classification + + def test_vt3_get(self, mock_get=None, mock_post=None): + report = vt3_get.VirusTotalv3( + "VT_v3_Get", + self.job_id, + self.observable_name, + self.observable_classification, + {}, + ).start() + self.assertEqual(report.get("success", False), True) - def raise_for_status(self): - pass + def test_vt2_get(self, mock_get=None, mock_post=None): + report = vt2_get.VirusTotalv2( + "VT_v2_Get", + self.job_id, + self.observable_name, + self.observable_classification, + {}, + ).start() + self.assertEqual(report.get("success", False), True) + def test_otx(self, mock_get=None, mock_post=None): + report = otx.OTX( + "OTX", self.job_id, self.observable_name, self.observable_classification, {} + ).start() + self.assertEqual(report.get("success", False), True) -# a mock response class that has no operation -class MockResponseNoOp: - def __init__(self, json_data, status_code): - pass + def test_pulsevide(self, mock_get=None, mock_post=None): + report = pulsedive.Pulsedive( + "Pulsedive_Active_IOC", + self.job_id, + self.observable_name, + self.observable_classification, + {}, + ).start() + self.assertEqual(report.get("success", False), True) - def search(self, **kwargs): - return {} + @mock_connections(patch("pymisp.ExpandedPyMISP", side_effect=mocked_requests_noop)) + def test_misp_first(self, *args): + report = misp.MISP( + "MISP_FIRST", + self.job_id, + self.observable_name, + self.observable_classification, + {"api_key_name": "FIRST_MISP_API", "url_key_name": "FIRST_MISP_URL"}, + ).start() + self.assertEqual(report.get("success", False), True) - def query(self, val): - return {} +class CommonTestCases_ip_url_domain(metaclass=ABCMeta): + """ + Tests which are common for IP, URL, domain types. + """ -def mocked_requests(*args, **kwargs): - return MockResponse({}, 200) + def test_gsf(self, mock_get=None, mock_post=None): + report = googlesf.GoogleSF( + "GoogleSafeBrowsing", + self.job_id, + self.observable_name, + self.observable_classification, + {}, + ).start() + self.assertEqual(report.get("success", False), True) + def test_onyphe(self, mock_get=None, mock_post=None): + report = onyphe.Onyphe( + "ONYPHE", + self.job_id, + self.observable_name, + self.observable_classification, + {}, + ).start() + self.assertEqual(report.get("success", False), True) -def mocked_docker_analyzer_get(*args, **kwargs): - return MockResponse( - {"key": "test", "returncode": 0, "report": {"test": "This is a test."}}, 200 - ) +class CommonTestCases_ip_domain_hash(metaclass=ABCMeta): + """ + Tests which are common for IP, domain, hash types. + """ -def mocked_docker_analyzer_post(*args, **kwargs): - return MockResponse({"key": "test", "status": "running"}, 202) + def test_ha_get(self, mock_get=None, mock_post=None): + report = ha_get.HybridAnalysisGet( + "HybridAnalysis_Get_Observable", + self.job_id, + self.observable_name, + self.observable_classification, + {}, + ).start() + self.assertEqual(report.get("success", False), True) -class CommonTestCases: +class CommonTestCases_url_domain(metaclass=ABCMeta): """ - Tests which are common for all types of observables. + Tests which are common for URL and Domain types. """ - def test_pulsevide(self, mock_get=None, mock_post=None): - report = pulsedive.Pulsedive( - "Pulsedive_Active_IOC", + def test_fortiguard(self, mock_get=None, mock_post=None): + report = fortiguard.Fortiguard( + "Fortiguard", self.job_id, self.observable_name, self.observable_classification, {}, ).start() self.assertEqual(report.get("success", False), True) + + def test_urlhaus(self, mock_get=None, mock_post=None): + report = urlhaus.URLHaus( + "URLhaus", + self.job_id, + self.observable_name, + self.observable_classification, + {}, + ).start() + self.assertEqual(report.get("success", False), True) + + def test_thug_url(self, mock_get=None, mock_post=None): + additional_params = {"test": True} + report = thug_url.ThugUrl( + "Thug_URL_Info", + self.job_id, + self.observable_name, + self.observable_classification, + additional_params, + ).start() + self.assertEqual(report.get("success", False), True) From ec2fae6fc6d21311d0e7e86ee909e09bb0041416 Mon Sep 17 00:00:00 2001 From: Eshaan Bansal Date: Tue, 18 Aug 2020 23:08:31 +0530 Subject: [PATCH 06/11] update docs, write about pulsedive --- docs/source/Installation.md | 54 ++++++++++++++++++------------------- docs/source/Usage.md | 14 +++++----- 2 files changed, 35 insertions(+), 33 deletions(-) diff --git a/docs/source/Installation.md b/docs/source/Installation.md index eca1f25cac..081603abaf 100644 --- a/docs/source/Installation.md +++ b/docs/source/Installation.md @@ -61,36 +61,36 @@ In the project you can find a template file named `env_file_app_template`. You have to create a new file named `env_file_app` from that template and modify it with your own configuration. REQUIRED variables to run the image: -* DB_HOST, DB_PORT, DB_USER, DB_PASSWORD: PostgreSQL configuration +* `DB_HOST`, `DB_PORT`, `DB_USER`, `DB_PASSWORD`: PostgreSQL configuration Strongly recommended variable to set: -* DJANGO_SECRET: random 50 chars key, must be unique. If you do not provide one, Intel Owl will automatically set a new secret on every run. +* `DJANGO_SECRET`: random 50 chars key, must be unique. If you do not provide one, Intel Owl will automatically set a new secret on every run. Optional variables needed to enable specific analyzers: -* ABUSEIPDB_KEY: AbuseIPDB API key -* AUTH0_KEY: Auth0 API Key -* SECURITYTRAILS_KEY: Securitytrails API Key -* SHODAN_KEY: Shodan API key -* HUNTER_API_KEY: Hunter.io API key -* GSF_KEY: Google Safe Browsing API key -* OTX_KEY: Alienvault OTX API key -* CIRCL_CREDENTIALS: CIRCL PDNS credentials in the format: `user|pass` -* VT_KEY: VirusTotal API key -* HA_KEY: HybridAnalysis API key -* INTEZER_KEY: Intezer API key -* FIRST_MISP_API: FIRST MISP API key -* FIRST_MISP_URL: FIRST MISP URL -* MISP_KEY: your own MISP instance key -* MISP_URL your own MISP instance URL -* CUCKOO_URL: your cuckoo instance URL -* HONEYDB_API_ID & HONEYDB_API_KEY: HoneyDB API credentials -* CENSYS_API_ID & CENSYS_API_SECRET: Censys credentials -* ONYPHE_KEY: Onyphe.io's API Key -* GREYNOISE_API_KEY: GreyNoise API ([docs](https://docs.greynoise.io)) +* `ABUSEIPDB_KEY`: AbuseIPDB API key +* `AUTH0_KEY`: Auth0 API Key +* `SECURITYTRAILS_KEY`: Securitytrails API Key +* `SHODAN_KEY`: Shodan API key +* `HUNTER_API_KEY`: Hunter.io API key +* `GSF_KEY`: Google Safe Browsing API key +* `OTX_KEY`: Alienvault OTX API key +* `CIRCL_CREDENTIALS`: CIRCL PDNS credentials in the format: `user|pass` +* `VT_KEY`: VirusTotal API key +* `HA_KEY`: HybridAnalysis API key +* `INTEZER_KEY`: Intezer API key +* `FIRST_MISP_API`: FIRST MISP API key +* `FIRST_MISP_URL`: FIRST MISP URL +* `MISP_KEY`: your own MISP instance key +* `MISP_URL`: your own MISP instance URL +* `CUCKOO_URL`: your cuckoo instance URL +* `HONEYDB_API_ID` & `HONEYDB_API_KEY`: HoneyDB API credentials +* `CENSYS_API_ID` & `CENSYS_API_SECRET`: Censys credentials +* `ONYPHE_KEY`: Onyphe.io's API Key +* `GREYNOISE_API_KEY`: GreyNoise API ([docs](https://docs.greynoise.io)) Advanced additional configuration: -* OLD_JOBS_RETENTION_DAYS: Database retention, default 3 days. Change this if you want to keep your old analysis longer in the database. -* PYINTELOWL_TOKEN_LIFETIME: Token Lifetime for requests coming from the [PyIntelOwl](https://github.com/intelowlproject/pyintelowl) library, default to 7 days. It will expire only if unused. Increment this if you plan to use these tokens rarely. +* `OLD_JOBS_RETENTION_DAYS`: Database retention, default 3 days. Change this if you want to keep your old analysis longer in the database. +* `PYINTELOWL_TOKEN_LIFETIME`: Token Lifetime for requests coming from the [PyIntelOwl](https://github.com/intelowlproject/pyintelowl) library, default to 7 days. It will expire only if unused. Increment this if you plan to use these tokens rarely. ### Database configuration (required) Before running the project, you must populate the basic configuration for PostgreSQL. @@ -98,9 +98,9 @@ In the project you can find a template file named `env_file_postgres_template`. You have to create a new file named `env_file_postgres` from that template and modify it with your own configuration. Required variables (we need to insert some of the values we have put in the previous configuration): -* POSTGRES_PASSWORD (same as DB_PASSWORD) -* POSTGRES_USER (same as DB_USER) -* POSTGRES_DB -> default `intel_owl_db` +* `POSTGRES_PASSWORD` (same as DB_PASSWORD) +* `POSTGRES_USER` (same as DB_USER) +* `POSTGRES_DB` -> default `intel_owl_db` If you prefer to use an external PostgreSQL instance, you should just remove the relative image from the `docker-compose.yml` file and provide the configuration to connect to your controlled instance/s. diff --git a/docs/source/Usage.md b/docs/source/Usage.md index 93410bcd3a..997336d69e 100644 --- a/docs/source/Usage.md +++ b/docs/source/Usage.md @@ -7,15 +7,15 @@ There are multiple ways to interact with the Intel Owl APIs, 1. IntelOwl-ng (Web Interface) -- Inbuilt Web interface with dashboard, visualizations of analysis data, easy to use forms for requesting -new analysis, tags management and more features. -- Built with Angular 9 and available on [Github](https://github.com/intelowlproject/intelowl-ng). + - Inbuilt Web interface with dashboard, visualizations of analysis data, easy to use forms for requesting + new analysis, tags management and more features + - Built with Angular 9 and available on [Github](https://github.com/intelowlproject/intelowl-ng). 2. pyIntelOwl (CLI/Library) -- Official client that is available at: [PyIntelOwl](https://github.com/intelowlproject/pyintelowl). -- Can be used as a library for your own python projects or, -- directly via the command line interface + - Official client that is available at: [PyIntelOwl](https://github.com/intelowlproject/pyintelowl), + - Can be used as a library for your own python projects or, + - directly via the command line interface. ### Tokens creation The server authentication is managed by API keys. So, if you want to interact with Intel Owl, you have to create one or more unprivileged users from the Django Admin Interface and then generate a token for those users. @@ -110,11 +110,13 @@ The following is the list of the available analyzers you can run out-of-the-box: * `Cymru_Hash_Registry_Get_Observable`: Check if a particular hash is available in the malware hash registry of Team Cymru * `Tranco`: Check if a domain is in the latest Tranco ranking top sites list * `Thug_URL_Info_*`: Perform hybrid dynamic/static analysis on a URL using [Thug low-interaction honeyclient](https://thug-honeyclient.readthedocs.io/) +* `Pulsedive_Active_IOC`: Scan indicators and retrieve results from [Pulsedive's API](https://pulsedive.com/api/). ## Analyzers customization You can create new analyzers based on already existing modules by changing the configuration values (`analyzer_config.json`). The following are all the keys that you can change without touching the source code: +* `disabled`: you can choose to disable certain analyzers, then they won't appear in the dropdown list and won't run if requested. * `leaks_info`: if set, in the case you specify via the API that a resource is sensitive, the specific analyzer won't be executed * `external_service`: if set, in the case you specify via the API to exclude external services, the specific analyzer won't be executed * `supported_filetypes`: can be populated as a list. If set, if you ask to analyze a file with a different mimetype from the ones you specified, it won't be executed From 4b65bc4b32162f7b7cc059abe80f3032a7b0a66a Mon Sep 17 00:00:00 2001 From: Eshaan Bansal Date: Wed, 19 Aug 2020 21:32:11 +0530 Subject: [PATCH 07/11] update Dockerfile to use Python 3.7, add quark analyzer --- .dockerignore | 14 ++++++++++++-- Dockerfile | 3 ++- .../file_analyzers/quark_engine.py | 13 +++++++++++++ api_app/script_analyzers/yara_repo_downloader.sh | 6 +++++- configuration/analyzer_config.json | 6 ++++++ intel_owl/tasks.py | 10 ++++++++++ requirements.txt | 3 ++- tests/test_files.py | 7 +++++++ 8 files changed, 57 insertions(+), 5 deletions(-) create mode 100644 api_app/script_analyzers/file_analyzers/quark_engine.py diff --git a/.dockerignore b/.dockerignore index 79968b3a95..625e37619f 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,10 +1,20 @@ .gitignore +.git .vscode +.lgtm.yml +.travis.yml __pycache__ +venv/ .env env_file_app +env_file_app_template env_file_postgres +env_file_postgres_template env_file_integrations -venv/ +env_file_integrations_template +env_file_app_travis +docs/ +integrations/ settings/ldap_config.py -docker-compose-override.yml \ No newline at end of file +docker-compose* +*.quark.log \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index 1d41e3420f..c035b33034 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.6 +FROM python:3.7 ENV PYTHONUNBUFFERED 1 ENV DJANGO_SETTINGS_MODULE intel_owl.settings @@ -35,6 +35,7 @@ RUN touch ${LOG_PATH}/django/api_app.log ${LOG_PATH}/django/api_app_errors.log \ # this is cause stringstifer creates this directory during the build and cause celery to crash && rm -rf /root/.local +# download yara rules and quark engine rules RUN api_app/script_analyzers/yara_repo_downloader.sh # this is because botocore points to legacy endpoints diff --git a/api_app/script_analyzers/file_analyzers/quark_engine.py b/api_app/script_analyzers/file_analyzers/quark_engine.py new file mode 100644 index 0000000000..09280d312b --- /dev/null +++ b/api_app/script_analyzers/file_analyzers/quark_engine.py @@ -0,0 +1,13 @@ +from api_app.script_analyzers.classes import FileAnalyzer + + +class QuarkEngine(FileAnalyzer): + def run(self): + from quark.report import Report + + # new report object + report = Report() + # start analysis + report.analysis(self.filepath, "/opt/deploy/quark-rules") + # return json report + return report.get_report("json") diff --git a/api_app/script_analyzers/yara_repo_downloader.sh b/api_app/script_analyzers/yara_repo_downloader.sh index ae13e46baa..708bc27907 100755 --- a/api_app/script_analyzers/yara_repo_downloader.sh +++ b/api_app/script_analyzers/yara_repo_downloader.sh @@ -40,5 +40,9 @@ git clone --depth 1 https://github.com/Neo23x0/signature-base.git cd /opt/deploy/yara/signature-base/yara rm generic_anomalies.yar general_cloaking.yar thor_inverse_matches.yar yara_mixed_ext_vars.yar thor-webshells.yar +# Download rules for quark-engine analyzer +cd /opt/deploy +svn export https://github.com/quark-engine/quark-engine/tags/v20.08/quark/rules quark-rules + # chown directories -chown -R www-data:www-data /opt/deploy/yara \ No newline at end of file +chown -R www-data:www-data /opt/deploy/yara /opt/deploy/quark-rules \ No newline at end of file diff --git a/configuration/analyzer_config.json b/configuration/analyzer_config.json index 0b096f6178..486d63edb6 100644 --- a/configuration/analyzer_config.json +++ b/configuration/analyzer_config.json @@ -730,6 +730,12 @@ "python_module": "apkid_run", "description": "APKiD identifies many compilers, packers, obfuscators, and other weird stuff from an APK or DEX file." }, + "Quark_Engine_APK": { + "type": "file", + "supported_filetypes": ["application/zip", "application/java-archive", "application/vnd.android.package-archive", "application/x-dex"], + "python_module": "quark_engine_run", + "description": "An Obfuscation-Neglect Android Malware Scoring System" + }, "PEframe_Scan": { "type": "file", "supported_filetypes": ["application/x-dosexec"], diff --git a/intel_owl/tasks.py b/intel_owl/tasks.py index 49bef4ad6d..ea6f88c80d 100644 --- a/intel_owl/tasks.py +++ b/intel_owl/tasks.py @@ -19,6 +19,7 @@ capa_info, boxjs_scan, apkid, + quark_engine, ) from api_app.script_analyzers.observable_analyzers import ( abuseipdb, @@ -787,3 +788,12 @@ def apkid_run(analyzer_name, job_id, filepath, filename, md5, additional_config_ apkid.APKiD( analyzer_name, job_id, filepath, filename, md5, additional_config_params ).start() + + +@shared_task(soft_time_limit=120) +def quark_engine_run( + analyzer_name, job_id, filepath, filename, md5, additional_config_params +): + quark_engine.QuarkEngine( + analyzer_name, job_id, filepath, filename, md5, additional_config_params + ).start() diff --git a/requirements.txt b/requirements.txt index eaf33d3f65..a7978fe84a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -77,4 +77,5 @@ zipp==0.6.0 djangorestframework-simplejwt==4.4.0 djangorestframework-guardian==0.3.0 flake8==3.8.2 -black==19.10b0 \ No newline at end of file +black==19.10b0 +quark-engine==20.8 \ No newline at end of file diff --git a/tests/test_files.py b/tests/test_files.py index e44f7a6274..f89db64367 100644 --- a/tests/test_files.py +++ b/tests/test_files.py @@ -24,6 +24,7 @@ capa_info, boxjs_scan, apkid, + quark_engine, ) from api_app.script_analyzers.observable_analyzers import vt3_get @@ -420,6 +421,12 @@ def test_apkid(self, mock_get=None, mock_post=None): ).start() self.assertEqual(report.get("success", False), True) + def test_quark_engine(self, mock_get=None, mock_post=None): + report = quark_engine.QuarkEngine( + "Quark_Engine_APK", self.job_id, self.filepath, self.filename, self.md5, {}, + ).start() + self.assertEqual(report.get("success", False), True) + def _generate_test_job_with_file(params, filename): test_file = f"{settings.PROJECT_LOCATION}/test_files/{filename}" From 5c60ec3d38bc7e4cc1a3fa4ae6b9c57c823dfe97 Mon Sep 17 00:00:00 2001 From: Eshaan Bansal Date: Wed, 19 Aug 2020 21:54:27 +0530 Subject: [PATCH 08/11] update docs, write about quark --- docs/source/Advanced-Usage.md | 6 ++++++ docs/source/Usage.md | 3 +++ 2 files changed, 9 insertions(+) diff --git a/docs/source/Advanced-Usage.md b/docs/source/Advanced-Usage.md index a673539834..a25fb278bf 100644 --- a/docs/source/Advanced-Usage.md +++ b/docs/source/Advanced-Usage.md @@ -69,26 +69,32 @@ table, th, td { Name Analyzers + Description PEframe PEframe_Scan + performs static analysis on Portable Executable malware and generic suspicious file Thug Thug_URL_Info_*, Thug_HTML_Info_* + performs hybrid dynamic/static analysis on a URL or HTML page. FireEye Capa Capa_Info + detects capabilities in executable files Box-JS BoxJS_Scan_JavaScript + tool for studying JavaScript malware APK Analyzers APKiD_Scan_APK_DEX_JAR + identifies many compilers, packers, obfuscators, and other weird stuff from an APK or DEX file diff --git a/docs/source/Usage.md b/docs/source/Usage.md index 997336d69e..838707311e 100644 --- a/docs/source/Usage.md +++ b/docs/source/Usage.md @@ -62,6 +62,7 @@ The following is the list of the available analyzers you can run out-of-the-box: * `Capa_Info`: [Capa](https://github.com/fireeye/capa) detects capabilities in executable files * `BoxJS_Scan_Javascript`: [Box-JS](https://github.com/CapacitorSet/box-js) is a tool for studying JavaScript malware. * `APKiD_Scan_APK_DEX_JAR`: [APKiD](https://github.com/rednaga/APKiD) identifies many compilers, packers, obfuscators, and other weird stuff from an APK or DEX file. +* `Quark_Engine_APK`: [Quark Engine](https://github.com/quark-engine/quark-engine) is an Obfuscation-Neglect Android Malware Scoring System. #### Observable analyzers (ip, domain, url, hash) * `VirusTotal_v3_Get_Observable`: search an observable in the VirusTotal DB @@ -112,6 +113,8 @@ The following is the list of the available analyzers you can run out-of-the-box: * `Thug_URL_Info_*`: Perform hybrid dynamic/static analysis on a URL using [Thug low-interaction honeyclient](https://thug-honeyclient.readthedocs.io/) * `Pulsedive_Active_IOC`: Scan indicators and retrieve results from [Pulsedive's API](https://pulsedive.com/api/). +#### [Additional analyzers](https://intelowl.readthedocs.io/en/develop/Advanced-Usage.html#optional-analyzers) that can be enabled per your wish. + ## Analyzers customization You can create new analyzers based on already existing modules by changing the configuration values (`analyzer_config.json`). From 41913878b1ce86ebad17648c742438e771bf8c4e Mon Sep 17 00:00:00 2001 From: Eshaan Bansal Date: Wed, 19 Aug 2020 23:17:40 +0530 Subject: [PATCH 09/11] increase max_length for file_mimetype, index some fields --- api_app/models.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/api_app/models.py b/api_app/models.py index bc0a97b0e5..8ea568bd6b 100644 --- a/api_app/models.py +++ b/api_app/models.py @@ -28,13 +28,18 @@ def __str__(self): class Job(models.Model): + class Meta: + indexes = [ + models.Index(fields=["md5", "status",]), + ] + source = models.CharField(max_length=50, blank=False, default="none") is_sample = models.BooleanField(blank=False, default=False) md5 = models.CharField(max_length=32, blank=False) observable_name = models.CharField(max_length=512, blank=True) observable_classification = models.CharField(max_length=12, blank=True) file_name = models.CharField(max_length=50, blank=True) - file_mimetype = models.CharField(max_length=50, blank=True) + file_mimetype = models.CharField(max_length=80, blank=True) status = models.CharField( max_length=32, blank=False, choices=STATUS, default="pending" ) From c691033ac95d05e7062a3c2c2f136126d94dbcf8 Mon Sep 17 00:00:00 2001 From: Matteo Lodi Date: Thu, 20 Aug 2020 09:09:28 +0200 Subject: [PATCH 10/11] little adjusts + bumped to v.1.4.0 --- .env | 5 +++-- .gitignore | 2 +- docs/source/Advanced-Usage.md | 2 +- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/.env b/.env index b4e4cadc59..ab552507b3 100644 --- a/.env +++ b/.env @@ -2,10 +2,10 @@ ### All services specified in all compose-files in variable COMPOSE_FILE will be built/ran ### By default, when you use `docker-compose up` only docker-compose.yml is read ### For each additional integration, the location of it's docker-compose.<>.yml file should be appended to -### the COMPOSE_FILE variable each seperated with ':'. If you are on windows, replace all ':' with ';'. +### the COMPOSE_FILE variable each separated with ':'. If you are on windows, replace all ':' with ';'. ### Reference to Docker's official Docs: https://docs.docker.com/compose/reference/envvars/#compose_file#compose_file -INTELOWL_TAG_VERSION=v1.3.1 +INTELOWL_TAG_VERSION=v1.4.0 ###### Default (Production) ###### @@ -25,3 +25,4 @@ COMPOSE_FILE=docker-compose.yml ###### For travis ###### #COMPOSE_FILE=docker-compose-for-tests.yml:./integrations/docker-compose-for-tests.peframe.yml:./integrations/docker-compose-for-tests.thug.yml:./integrations/docker-compose-for-tests.capa.yml:./integrations/docker-compose-for-tests.boxjs.yml:./integrations/docker-compose-for-tests.apk.yml + diff --git a/.gitignore b/.gitignore index e4b7abb2be..43d688979d 100644 --- a/.gitignore +++ b/.gitignore @@ -7,4 +7,4 @@ env_file_postgres env_file_integrations venv/ settings/ldap_config.py -docker-compose-override.yml +docker-compose-override.yml \ No newline at end of file diff --git a/docs/source/Advanced-Usage.md b/docs/source/Advanced-Usage.md index a673539834..7aaf98bf25 100644 --- a/docs/source/Advanced-Usage.md +++ b/docs/source/Advanced-Usage.md @@ -108,7 +108,7 @@ IntelOwl leverages [Django-auth-ldap](https://github.com/django-auth-ldap/django How to configure and enable LDAP on Intel Owl? -Inside the `settings` directory you can find a file called `ldap_config_template.py`. This file provides an example of configuration. +Inside the `intel_owl` directory you can find a file called `ldap_config_template.py`. This file provides an example of configuration. Copy that file into the same directory with the name `ldap_config.py`. Then change the values with your LDAP configuration. From 4da4118b0e5fdfbce99fcfcee51f41f8a2e0b7fe Mon Sep 17 00:00:00 2001 From: Matteo Lodi Date: Thu, 20 Aug 2020 10:28:05 +0200 Subject: [PATCH 11/11] fix DNS analyzers --- .../observable_analyzers/active_dns.py | 20 ++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/api_app/script_analyzers/observable_analyzers/active_dns.py b/api_app/script_analyzers/observable_analyzers/active_dns.py index 39e440e991..05e41c7c58 100644 --- a/api_app/script_analyzers/observable_analyzers/active_dns.py +++ b/api_app/script_analyzers/observable_analyzers/active_dns.py @@ -68,6 +68,7 @@ def __handle_activedns_error(self, err: str): self.report["success"] = False def __doh_google(self): + result = {} if self.observable_classification == "domain": try: authority_answer = "" @@ -94,7 +95,7 @@ def __doh_google(self): f"observable: {self.observable_name} active_dns query" f" retrieved no valid A answer: {answers}" ) - self.report["report"] = { + result = { "name": self.observable_name, "resolution": ip, "authoritative_answer": authority_answer, @@ -107,8 +108,10 @@ def __doh_google(self): self.__handle_activedns_error( "cannot analyze something different from type: domain" ) + return result def __doh_cloudflare(self): + result = {} if self.observable_classification == "domain": try: client = requests.session() @@ -131,7 +134,7 @@ def __doh_cloudflare(self): else "NXDOMAIN" ) - self.report["report"] = { + result = { "name": self.observable_name, "resolution": result_data, } @@ -143,8 +146,10 @@ def __doh_cloudflare(self): self.__handle_activedns_error( "cannot analyze something different from type: domain" ) + return result def __doh_cloudflare_malware(self): + result = {} if self.observable_classification == "domain": try: result = {"name": self.observable_name} @@ -169,6 +174,8 @@ def __doh_cloudflare_malware(self): # known as malicious if resolution == "0.0.0.0": result["is_malicious"] = True + else: + result["is_malicious"] = False else: logger.warning( f"no Answer key retrieved for {self.observable_name}" @@ -176,7 +183,6 @@ def __doh_cloudflare_malware(self): ) result["no_answer"] = True - self.report["report"] = result except requests.exceptions.RequestException as err: self.__handle_activedns_error( f"observable_name:{self.observable_name}, RequestException {err}" @@ -185,6 +191,7 @@ def __doh_cloudflare_malware(self): self.__handle_activedns_error( "cannot analyze something different from type: domain" ) + return result def __classic_dns(self): result = {} @@ -193,11 +200,14 @@ def __classic_dns(self): resolutions = [] try: domains = socket.gethostbyaddr(self.observable_name) - resolutions = domains[2] + resolutions = domains[0] except (socket.gaierror, socket.herror): logger.info( f"no resolution found for observable {self.observable_name}" ) + logger.info( + f"resolution {resolutions} found for observable {self.observable_name}" + ) result = {"name": self.observable_name, "resolutions": resolutions} elif self.observable_classification == "domain": try: @@ -208,4 +218,4 @@ def __classic_dns(self): else: self.__handle_activedns_error("not analyzable") - self.report["report"] = result + return result