diff --git a/.env b/.env index c0429bb527..6d2b85dece 100644 --- a/.env +++ b/.env @@ -5,7 +5,7 @@ ### the COMPOSE_FILE variable each separated with ':'. If you are on windows, replace all ':' with ';'. ### Reference to Docker's official Docs: https://docs.docker.com/compose/reference/envvars/#compose_file#compose_file -INTELOWL_TAG_VERSION=v1.6.1 +INTELOWL_TAG_VERSION=v1.7.0 ###### Default (Production) ###### diff --git a/.github/FUNDING.yml b/.github/FUNDING.yml index dc2dd42024..0833f21951 100644 --- a/.github/FUNDING.yml +++ b/.github/FUNDING.yml @@ -1 +1,2 @@ liberapay: intelowlproject +custom: ["https://xscode.com/intelowlproject/IntelOwl"] diff --git a/README.md b/README.md index d5a0ed04e3..ea75712e3d 100644 --- a/README.md +++ b/README.md @@ -1,118 +1,53 @@ -![Intel Owl](static_intel/intel_owl.jpeg) +Intel Owl [![Language grade: Python](https://img.shields.io/lgtm/grade/python/g/intelowlproject/IntelOwl.svg?logo=lgtm&logoWidth=18)](https://lgtm.com/projects/g/intelowlproject/IntelOwl/context:python) [![CodeFactor](https://www.codefactor.io/repository/github/intelowlproject/intelowl/badge)](https://www.codefactor.io/repository/github/intelowlproject/intelowl) [![Build Status](https://travis-ci.com/intelowlproject/IntelOwl.svg?branch=master)](https://travis-ci.org/intelowlproject/IntelOwl) [![Code style: black](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/psf/black) -# Intel Owl +Get Support
+_For urgent issues and priority support, visit [https://xscode.com/intelowlproject/IntelOwl](https://xscode.com/intelowlproject/IntelOwl)._ -Do you want to get **threat intelligence data** about a file, an IP or a domain? +# Intel Owl -Do you want to get this kind of data from multiple sources at the same time using **a single API request**? +Do you want to get **threat intelligence data** about a malware, an IP or a domain? Do you want to get this kind of data from multiple sources at the same time using **a single API request**? You are in the right place! -This application is built to **scale out** and to **speed up the retrieval of threat info**. - -It can be integrated easily in your stack of security tools to automate common jobs usually performed, for instance, by SOC analysts manually. - -Intel Owl is composed of **analyzers** that can be run to retrieve data from external sources (like VirusTotal or AbuseIPDB) or to generate intel from internal analyzers (like Yara or Oletools) - -This solution is for everyone who needs a single point to query for info about a specific file or observable (domain, IP, URL, hash). +Intel Owl is an Open Source Intelligence, or OSINT solution to get threat intelligence data about a specific file, an IP or a domain from a single API at scale. It integrates a number of analyzers available online and is for everyone who needs a single point to query for info about a specific file or observable. -Main features: +### Features -- full django-python application -- easily and completely customizable, both the APIs and the analyzers -- clone the project, set up the configuration and you are ready to run -- Official frontend client: **[IntelOwl-ng](https://github.com/intelowlproject/IntelOwl-ng)** provides features such as dashboard, visualizations of analysis data, easy to use forms for requesting new analysis, etc. +- Provides enrichment of threat intel for malware as well as observables (IP, Domain, URL and hash). +- This application is built to **scale out** and to **speed up the retrieval of threat info**. +- It can be integrated easily in your stack of security tools ([pyintelowl](https://github.com/intelowlproject/pyintelowl)) to automate common jobs usually performed, for instance, by SOC analysts manually. +- Intel Owl is composed of **analyzers** that can be run to retrieve data from external sources (like VirusTotal or AbuseIPDB) or to generate intel from internal analyzers (like Yara or Oletools) +- API written in Django and Python 3.7. +- Inbuilt frontend client: **[IntelOwl-ng](https://github.com/intelowlproject/IntelOwl-ng)** provides features such as dashboard, visualizations of analysis data, easy to use forms for requesting new analysis, etc. [Live Demo](https://intelowlclient.firebaseapp.com/). -### Documentation +## Documentation [![Documentation Status](https://readthedocs.org/projects/intelowl/badge/?version=latest)](https://intelowl.readthedocs.io/en/latest/?badge=latest) -Documentation about IntelOwl installation, usage, contribution can be found at https://intelowl.readthedocs.io/. - -### Blog posts - -[Daily Swig Article](https://portswigger.net/daily-swig/intel-owl-osint-tool-automates-the-intel-gathering-process-using-a-single-api) - -[Honeynet Blog: v1.0.0 Announcement](https://www.honeynet.org/?p=7558) - -[Certego Blog: First announcement](https://www.certego.net/en/news/new-year-new-tool-intel-owl/) - -### Free Internal Modules Available +Documentation about IntelOwl installation, usage, configuration and contribution can be found at https://intelowl.readthedocs.io/. -- Static Document Analysis -- Static RTF Analysis -- Static PDF Analysis -- Static PE Analysis -- Static Generic File Analysis -- Strings analysis with ML -- PE Emulation with Speakeasy -- PE Signature verification -- PE Capabilities Extraction -- Emulated Javascript Analysis -- Android Malware Analysis +## Blog posts -**Free modules that require additional configuration**: +To know more about the project and it's growth over time, you may be interested in reading the following: -- Cuckoo (requires at least one working Cuckoo instance) -- MISP (requires at least one working MISP instance) -- Yara (Community, Neo23x0, Intezer and McAfee rules are already available. There's the chance to add your own rules) +- [Intel Owl on Daily Swig](https://portswigger.net/daily-swig/intel-owl-osint-tool-automates-the-intel-gathering-process-using-a-single-api) +- [Honeynet: v1.0.0 Announcement](https://www.honeynet.org/?p=7558) +- [Certego Blog: First announcement](https://www.certego.net/en/news/new-year-new-tool-intel-owl/) -### External Services Available +## Available services or analyzers -##### required paid or trial API key +You can see the full list of all available analyzers, [here](https://intelowl.readthedocs.io/en/latest/Usage.html#available-analyzers). -- GreyNoise v2 +| Inbuilt modules | External Services | Free modules that require additional configuration | +|- |- |- | +| - Static Document, RTF, PDF, PE, Generic File Analysis
- Strings analysis with ML
- PE Emulation with Speakeasy
- PE Signature verification
- PE Capabilities Extraction
- Emulated Javascript Analysis
- Android Malware Analysis
- SPF and DMARC Validator
- more... | - GreyNoise v2
- Intezer Scan
- VirusTotal v2+v3
- HybridAnalysis
- Censys.io
- Shodan
- AlienVault OTX
- Threatminer
- Abuse.ch
- many more.. | - Cuckoo (requires at least one working Cuckoo instance)
- MISP (requires at least one working MISP instance)
- Yara (Community, Neo23x0, Intezer and McAfee rules are already available. There's the chance to add your own rules) | -##### required paid or free API key - -- VirusTotal v2 + v3 -- HybridAnalysis -- Intezer -- Farsight DNSDB -- Hunter.io - Email Hunting -- ONYPHE -- Censys.io -- SecurityTrails -- Intelligence X -- Pulsedive API (works w/o API key as well) - -##### required free API key - -- GoogleSafeBrowsing -- AbuseIPDB -- Shodan -- HoneyDB -- AlienVault OTX -- MaxMind -- Auth0 - -##### needed access request - -- CIRCL PassiveDNS + PassiveSSL - -##### without api key - -- Fortiguard URL Analyzer -- GreyNoise Alpha API v1 -- Talos Reputation -- Tor Project -- Robtex -- Threatminer -- Abuse.ch MalwareBazaar -- Abuse.ch URLhaus -- Team Cymru Malware Hash Registry -- Tranco Rank -- Google DoH -- CloudFlare DoH Classic -- CloudFlare DoH Malware -- Classic DNS resolution - -### Legal notice +## Legal notice You as a user of this project must review, accept and comply with the license terms of each downloaded/installed package listed below. By proceeding with the @@ -142,21 +77,20 @@ license terms. [Quark-Engine](https://github.com/quark-engine/quark-engine) [IntelX](https://intelx.io/terms-of-service) -### Acknowledgments +## Acknowledgments This project was created and will be upgraded thanks to the following organizations: Certego Logo Honeynet.org logo - -### Google Summer Of Code +#### Google Summer Of Code The project was accepted to the GSoC 2020 under the Honeynet Project!! A lot of [new features](https://www.honeynet.org/gsoc/gsoc-2020/google-summer-of-code-2020-project-ideas/#intel-owl-improvements) were developed by Eshaan Bansal ([Twitter](https://twitter.com/mask0fmydisguis)). Stay tuned for the upcoming GSoC 2021! Join the [Honeynet Slack chat](https://gsoc-slack.honeynet.org/) for more info. -### About the author and maintainers +## About the author and maintainers Feel free to contact the main developers at any time: - Matteo Lodi ([Twitter](https://twitter.com/matte_lodi)): Author and creator diff --git a/api_app/script_analyzers/file_analyzers/unpac_me.py b/api_app/script_analyzers/file_analyzers/unpac_me.py new file mode 100644 index 0000000000..c4737debea --- /dev/null +++ b/api_app/script_analyzers/file_analyzers/unpac_me.py @@ -0,0 +1,99 @@ +import requests +import logging +from api_app.script_analyzers.classes import FileAnalyzer +from api_app.exceptions import AnalyzerRunException +import time +from intel_owl import secrets +from typing import Dict + +logger = logging.getLogger(__name__) + + +class UnpacMe(FileAnalyzer): + base_url: str = "https://api.unpac.me/api/v1/" + + def set_config(self, additional_config_params): + self.api_key_name = additional_config_params.get( + "api_key_name", "UNPAC_ME_API_KEY" + ) + private = additional_config_params.get("private", False) + self.private = "private" if private else "public" + self.__api_key = secrets.get_secret(self.api_key_name) + # max no. of tries when polling for result + self.max_tries = additional_config_params.get("max_tries", 30) + # interval b/w HTTP requests when polling + self.poll_distance = 5 + + def run(self): + if not self.__api_key: + raise AnalyzerRunException( + f"No API key retrieved with name: {self.api_key_name}" + ) + self.headers = {"Authorization": "Key %s" % self.__api_key} + unpac_id = self._upload() + logger.info(f"md5 {self.md5} job {self.job_id} uploaded id {unpac_id}") + for chance in range(self.max_tries): + time.sleep(self.poll_distance) + logger.info( + f"unpacme polling, try n.{chance + 1}." + f" job_id {self.job_id}. starting the query" + ) + status = self._get_status(unpac_id) + if status == "fail": + logger.error(f"md5 {self.md5} job {self.job_id} analysis has failed") + raise AnalyzerRunException("failed analysis") + if status != "complete": + logger.info( + f"md5 {self.md5} job {self.job_id} id {unpac_id} status {status}" + ) + continue + return self._get_report(unpac_id) + + def _req_with_checks(self, url, files=None, post=False): + try: + if post: + r = requests.post( + self.base_url + url, files=files, headers=self.headers + ) + else: + headers = self.headers if self.private == "private" else {} + r = requests.get(self.base_url + url, files=files, headers=headers) + r.raise_for_status() + except requests.exceptions.HTTPError as e: + logger.error( + f"md5 {self.md5} job {self.job_id} url {url} has http error {str(e)}" + ) + if post: + raise AnalyzerRunException("Monthly quota exceeded!") + raise AnalyzerRunException(e) + except requests.exceptions.Timeout as e: + logger.error( + f"md5 {self.md5} job {self.job_id} url {url} has timeout error {str(e)}" + ) + raise AnalyzerRunException(e) + except requests.exceptions.RequestException as e: + logger.error( + f"md5 {self.md5} job {self.job_id} url {url} failed with error {str(e)}" + ) + raise AnalyzerRunException(e) + return r + + def _upload(self) -> str: + with open(self.filepath, "rb") as f: + file_data = f.read() + files = {"file": (self.filename, file_data)} + r = self._req_with_checks("private/upload", files=files, post=True) + response = r.json() + if "id" not in response: + raise AnalyzerRunException( + f"md5 {self.md5} job {self.job_id} function upload id not in response" + ) + return response["id"] + + def _get_status(self, unpac_me_id) -> str: + response = self._req_with_checks(f"{self.private}/status/{unpac_me_id}") + return response.json().get("status", False) + + def _get_report(self, unpac_me_id) -> Dict: + response = self._req_with_checks(f"{self.private}/results/{unpac_me_id}") + return response.json() diff --git a/api_app/script_analyzers/observable_analyzers/checkdmarc.py b/api_app/script_analyzers/observable_analyzers/checkdmarc.py new file mode 100644 index 0000000000..37053eb964 --- /dev/null +++ b/api_app/script_analyzers/observable_analyzers/checkdmarc.py @@ -0,0 +1,31 @@ +import subprocess +import json +from shutil import which + +from api_app.script_analyzers import classes +from api_app.exceptions import AnalyzerRunException + + +class CheckDMARC(classes.ObservableAnalyzer): + check_command: str = "checkdmarc" + + def run(self): + if not which(self.check_command): + self.report["success"] = False + raise AnalyzerRunException("checkdmarc not installed!") + + process = subprocess.Popen( + [self.check_command, self.observable_name], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + process.wait() + stdout, stderr = process.communicate() + + dmarc_info = stdout.decode("utf-8"), stderr + + dmarc_str = dmarc_info[0] + + dmarc_json = json.loads(dmarc_str) + + return dmarc_json diff --git a/api_app/script_analyzers/observable_analyzers/cymru.py b/api_app/script_analyzers/observable_analyzers/cymru.py index 9fddfebeab..78c03a0eb2 100644 --- a/api_app/script_analyzers/observable_analyzers/cymru.py +++ b/api_app/script_analyzers/observable_analyzers/cymru.py @@ -20,15 +20,14 @@ def run(self): # reference: https://team-cymru.com/community-services/mhr/ # if the resolution works, this means that the file is reported # as malware by Cymru - resolutions = [] + domains = None try: query_to_perform = f"{self.observable_name}.malware.hash.cymru.com" domains = socket.gethostbyaddr(query_to_perform) - resolutions = domains[2] except (socket.gaierror, socket.herror): logger.info(f"observable {self.observable_name} not found in HMR DB") - if resolutions: + if domains: results["found"] = True - results["resolution_data"] = resolutions + results["resolution_data"] = domains[2] return results diff --git a/api_app/script_analyzers/observable_analyzers/talos.py b/api_app/script_analyzers/observable_analyzers/talos.py index 3e2dcfce6d..446eb874a9 100644 --- a/api_app/script_analyzers/observable_analyzers/talos.py +++ b/api_app/script_analyzers/observable_analyzers/talos.py @@ -32,7 +32,7 @@ def run(self): def updater(): try: logger.info("starting download of db from talos") - url = "https://www.talosintelligence.com/documents/ip-blacklist" + url = "https://snort.org/downloads/ip-block-list" r = requests.get(url) r.raise_for_status() diff --git a/api_app/script_analyzers/observable_analyzers/vt2_get.py b/api_app/script_analyzers/observable_analyzers/vt2_get.py index 01cf6f2cd6..0d0879012f 100644 --- a/api_app/script_analyzers/observable_analyzers/vt2_get.py +++ b/api_app/script_analyzers/observable_analyzers/vt2_get.py @@ -18,30 +18,37 @@ def run(self): f"No API key retrieved with name: {self.api_key_name}" ) - return vt_get_report( + resp = vt_get_report( self.__api_key, self.observable_name, self.observable_classification ) + resp_code = resp.get("response_code", 1) + verbose_msg = resp.get("verbose_msg", "") + if resp_code == -1 or "Invalid resource" in verbose_msg: + self.report["errors"].append(verbose_msg) + raise AnalyzerRunException(f"response code {resp_code}. response: {resp}") + return resp -def vt_get_report(api_key, observable_name, observable_classification): + +def vt_get_report(api_key, observable_name, obs_clsfn): params = {"apikey": api_key} - if observable_classification == "domain": + if obs_clsfn == "domain": params["domain"] = observable_name uri = "domain/report" - elif observable_classification == "ip": + elif obs_clsfn == "ip": params["ip"] = observable_name uri = "ip-address/report" - elif observable_classification == "url": + elif obs_clsfn == "url": params["resource"] = observable_name uri = "url/report" - elif observable_classification == "hash": + elif obs_clsfn == "hash": params["resource"] = observable_name params["allinfo"] = 1 uri = "file/report" else: raise AnalyzerRunException( - "not supported observable type {}. Supported are: hash, ip, domain and url" - "".format(observable_classification) + f"not supported observable type {obs_clsfn}. " + "Supported are: hash, ip, domain and url." ) try: @@ -49,8 +56,5 @@ def vt_get_report(api_key, observable_name, observable_classification): response.raise_for_status() except requests.RequestException as e: raise AnalyzerRunException(e) - result = response.json() - response_code = result.get("response_code", 1) - if response_code == -1: - raise AnalyzerRunException(f"response code -1. result:{result}") - return result + + return response.json() diff --git a/api_app/script_analyzers/observable_analyzers/whoisxmlapi.py b/api_app/script_analyzers/observable_analyzers/whoisxmlapi.py new file mode 100644 index 0000000000..de32031eb7 --- /dev/null +++ b/api_app/script_analyzers/observable_analyzers/whoisxmlapi.py @@ -0,0 +1,31 @@ +import requests + +from api_app.exceptions import AnalyzerRunException +from api_app.script_analyzers import classes +from intel_owl import secrets + + +class Whoisxmlapi(classes.ObservableAnalyzer): + url: str = "https://www.whoisxmlapi.com/whoisserver/WhoisService" + + def set_config(self, additional_config_params): + self.api_key_name = additional_config_params.get( + "api_key_name", "WHOISXMLAPI_KEY" + ) + self.__api_key = secrets.get_secret(self.api_key_name) + + def run(self): + if not self.__api_key: + raise AnalyzerRunException( + f"No API key retrieved with name: {self.api_key_name}" + ) + + params = { + "apiKey": self.__api_key, + "domainName": self.observable_name, + "outputFormat": "JSON", + } + response = requests.get(self.url, params=params) + response.raise_for_status() + + return response.json() diff --git a/configuration/analyzer_config.json b/configuration/analyzer_config.json index 4340d1c113..9a8436d381 100644 --- a/configuration/analyzer_config.json +++ b/configuration/analyzer_config.json @@ -76,6 +76,12 @@ "python_module": "boxjs_run", "description": "A tool for studying JavaScript malware" }, + "CheckDMARC": { + "type": "observable", + "observable_supported": ["domain"], + "description": "An SPF and DMARC DNS records validator", + "python_module": "checkdmarc_run" + }, "CIRCLPassiveDNS": { "type": "observable", "observable_supported": ["domain", "url"], @@ -792,6 +798,17 @@ "description": "check if a domain is in the last Tranco ranking top sites list", "python_module": "tor_run" }, + "UnpacMe_EXE_Unpacker": { + "type": "file", + "supported_filetypes": ["application/x-dosexec"], + "description": "UnpacMe unpacker", + "python_module": "unpac_me_run", + "additional_config_params": { + "api_key_name": "UNPAC_ME_API_KEY", + "private": false, + "max_tries": 30 + } + }, "URLhaus": { "type": "observable", "observable_supported": ["domain", "url"], @@ -902,6 +919,17 @@ "api_key_name": "VT_KEY" } }, + "Whoisxmlapi": { + "type": "observable", + "observable_supported": ["ip", "domain"], + "external_service": true, + "description": "the WHOIS record data, of a domain name, an IP address, or an email address", + "requires_configuration": true, + "python_module": "whoisxmlapi_run", + "additional_config_params": { + "api_key_name": "WHOISXMLAPI_KEY" + } + }, "Yara_Scan_Community": { "type": "file", "description": "scan a file with community yara rules", diff --git a/docs/source/Installation.md b/docs/source/Installation.md index 200801e73a..5b546293ba 100644 --- a/docs/source/Installation.md +++ b/docs/source/Installation.md @@ -90,6 +90,7 @@ Optional variables needed to enable specific analyzers: * `ONYPHE_KEY`: Onyphe.io's API Key * `GREYNOISE_API_KEY`: GreyNoise API ([docs](https://docs.greynoise.io)) * `INTELX_API_KEY`: IntelligenceX API ([docs](https://intelx.io/product)) +* `UNPAC_ME_API_KEY`: UnpacMe API ([docs](https://api.unpac.me/)) Advanced additional configuration: * `OLD_JOBS_RETENTION_DAYS`: Database retention, default 3 days. Change this if you want to keep your old analysis longer in the database. diff --git a/docs/source/Usage.md b/docs/source/Usage.md index ccc9d9bc0a..d315e187be 100644 --- a/docs/source/Usage.md +++ b/docs/source/Usage.md @@ -68,6 +68,7 @@ The following is the list of the available analyzers you can run out-of-the-box: * `APKiD_Scan_APK_DEX_JAR`: [APKiD](https://github.com/rednaga/APKiD) identifies many compilers, packers, obfuscators, and other weird stuff from an APK or DEX file. * `Quark_Engine_APK`: [Quark Engine](https://github.com/quark-engine/quark-engine) is an Obfuscation-Neglect Android Malware Scoring System. * `IntelX_Phonebook`: [IntelligenceX](https://intelx.io/) is a search engine and data archive. Fetches emails, urls, domains associated with an observable. +* `UnpacMe_EXE_Unpacker`: [UnpacMe](https://www.unpac.me/) is an automated malware unpacking service #### Observable analyzers (ip, domain, url, hash) * `VirusTotal_v3_Get_Observable`: search an observable in the VirusTotal DB @@ -117,6 +118,8 @@ The following is the list of the available analyzers you can run out-of-the-box: * `Tranco`: Check if a domain is in the latest Tranco ranking top sites list * `Thug_URL_Info_*`: Perform hybrid dynamic/static analysis on a URL using [Thug low-interaction honeyclient](https://thug-honeyclient.readthedocs.io/) * `Pulsedive_Active_IOC`: Scan indicators and retrieve results from [Pulsedive's API](https://pulsedive.com/api/). +* `CheckDMARC`: An SPF and DMARC DNS records validator for domains. +* `Whoisxmlapi`: Fetch WHOIS record data, of a domain name, an IP address, or an email address. #### [Additional analyzers](https://intelowl.readthedocs.io/en/develop/Advanced-Usage.html#optional-analyzers) that can be enabled per your wish. diff --git a/docs/source/conf.py b/docs/source/conf.py index f2b1cbd39a..6a65ec8588 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -22,7 +22,7 @@ author = "Matteo Lodi" # The full version, including alpha/beta/rc tags -release = "1.6.1" +release = "1.7.0" # -- General configuration --------------------------------------------------- diff --git a/env_file_app_template b/env_file_app_template index fcf0ceed60..8c71448980 100644 --- a/env_file_app_template +++ b/env_file_app_template @@ -43,6 +43,8 @@ ONYPHE_KEY= GREYNOISE_API_KEY= PULSEDIVE_API_KEY= INTELX_API_KEY= +WHOISXMLAPI_KEY= +UNPAC_ME_API_KEY= # Test tokens TEST_JOB_ID=1 diff --git a/env_file_app_travis b/env_file_app_travis index d300d0727a..c014ee89dc 100644 --- a/env_file_app_travis +++ b/env_file_app_travis @@ -5,6 +5,16 @@ DB_PORT=5432 DB_USER=user DB_PASSWORD=password +# Config variables +OLD_JOBS_RETENTION_DAYS=2 +PYINTELOWL_TOKEN_LIFETIME=7 + +# Elastic Search Configuration +ELASTICSEARCH_ENABLED=False +ELASTICSEARCH_HOST= +ELASTICSEARCH_NO_OF_SHARDS=1 +ELASTICSEARCH_NO_OF_REPLICAS=0 + SHODAN_KEY=test ABUSEIPDB_KEY=test AUTH0_KEY=test @@ -27,6 +37,8 @@ CENSYS_API_SECRET=test ONYPHE_KEY=test GREYNOISE_API_KEY=test INTELX_API_KEY=test +WHOISXMLAPI_KEY=test +UNPAC_ME_API_KEY=test # Test tokens TEST_JOB_ID=1 diff --git a/intel_owl/tasks.py b/intel_owl/tasks.py index f0edfc7dfc..b62182439d 100644 --- a/intel_owl/tasks.py +++ b/intel_owl/tasks.py @@ -21,6 +21,7 @@ boxjs_scan, apkid, quark_engine, + unpac_me, ) from api_app.script_analyzers.observable_analyzers import ( abuseipdb, @@ -55,6 +56,8 @@ tranco, pulsedive, intelx, + whoisxmlapi, + checkdmarc, ) from api_app import crons @@ -825,3 +828,46 @@ def quark_engine_run( quark_engine.QuarkEngine( analyzer_name, job_id, filepath, filename, md5, additional_config_params ).start() + + +@shared_task(soft_time_limit=400) +def unpac_me_run( + analyzer_name, job_id, filepath, filename, md5, additional_config_params +): + unpac_me.UnpacMe( + analyzer_name, job_id, filepath, filename, md5, additional_config_params + ).start() + + +@shared_task(soft_time_limit=30) +def whoisxmlapi_run( + analyzer_name, + job_id, + observable_name, + observable_classification, + additional_config_params, +): + whoisxmlapi.Whoisxmlapi( + analyzer_name, + job_id, + observable_name, + observable_classification, + additional_config_params, + ).start() + + +@shared_task(soft_time_limit=30) +def checkdmarc_run( + analyzer_name, + job_id, + observable_name, + observable_classification, + additional_config_params, +): + checkdmarc.CheckDMARC( + analyzer_name, + job_id, + observable_name, + observable_classification, + additional_config_params, + ).start() diff --git a/requirements.txt b/requirements.txt index 0f9af9f940..925572c5c7 100644 --- a/requirements.txt +++ b/requirements.txt @@ -8,6 +8,7 @@ celery==4.4.7 certifi==2019.6.16 cffi==1.12.3 chardet==3.0.4 +checkdmarc==4.3.1 colorama==0.3.7 colorclass==2.2.0 configparser==3.8.1 @@ -37,10 +38,10 @@ more-itertools==7.2.0 msoffcrypto-tool==4.10.1 numpy==1.17.1 olefile==0.46 -git+git://github.com/mlodic/oletools.git +git+git://github.com/mlodic/oletools.git@aef91b95cc9e870da2dd4205f9fc8a933e2ea2dd OTXv2==1.5.9 peepdf==0.4.2 -git+git://github.com/mlodic/pefile.git +https://github.com/mlodic/pefile/archive/master.tar.gz pluggy==0.13.0 psycopg2-binary==2.8.4 py==1.8.0 diff --git a/static_intel/xscode-banner.png b/static_intel/xscode-banner.png new file mode 100644 index 0000000000..d473560011 Binary files /dev/null and b/static_intel/xscode-banner.png differ diff --git a/tests/test_files.py b/tests/test_files.py index 04e2ddd120..60ae179ab1 100644 --- a/tests/test_files.py +++ b/tests/test_files.py @@ -26,6 +26,7 @@ boxjs_scan, apkid, quark_engine, + unpac_me, ) from api_app.script_analyzers.observable_analyzers import vt3_get @@ -49,6 +50,14 @@ def mock_connections(decorator): return decorator if settings.MOCK_CONNECTIONS else lambda x: x +def mocked_unpacme_post(*args, **kwargs): + return MockResponse({"id": "test"}, 200) + + +def mocked_unpacme_get(*args, **kwargs): + return MockResponse({"id": "test", "status": "complete"}, 200) + + def mocked_vt_get(*args, **kwargs): return MockResponse({"data": {"attributes": {"status": "completed"}}}, 200) @@ -193,6 +202,19 @@ def test_yara_exe(self): ).start() self.assertEqual(report.get("success", False), True) + @mock_connections(patch("requests.get", side_effect=mocked_unpacme_get)) + @mock_connections(patch("requests.post", side_effect=mocked_unpacme_post)) + def test_unpacme_exe(self, mock_get=None, mock_post=None): + report = unpac_me.UnpacMe( + "UnpacMe_EXE_Unpacker", + self.job_id, + self.filepath, + self.filename, + self.md5, + {}, + ).start() + self.assertEqual(report.get("success", False), True) + @mock_connections(patch("requests.get", side_effect=mocked_vt_get)) @mock_connections(patch("requests.post", side_effect=mocked_vt_post)) def test_vt3_scan_exe(self, mock_get=None, mock_post=None): diff --git a/tests/test_observables.py b/tests/test_observables.py index 1b1e058095..1a669fffad 100644 --- a/tests/test_observables.py +++ b/tests/test_observables.py @@ -29,6 +29,8 @@ securitytrails, cymru, tranco, + whoisxmlapi, + checkdmarc, ) from .mock_utils import ( MockResponseNoOp, @@ -260,6 +262,16 @@ def active_dns_classic_reverse(self, mock_get=None, mock_post=None): self.assertEqual(report.get("success", False), True, f"report: {report}") + def test_whoisxmlapi_ip(self, mock_get=None, mock_post=None): + report = whoisxmlapi.Whoisxmlapi( + "Whoisxmlapi_ip", + self.job_id, + self.observable_name, + self.observable_classification, + {}, + ).start() + self.assertEqual(report.get("success", False), True) + @mock_connections(patch("requests.get", side_effect=mocked_requests)) @mock_connections(patch("requests.post", side_effect=mocked_requests)) @@ -406,6 +418,27 @@ def test_cloudFlare_malware(self, mock_get=None, mock_post=None): self.assertEqual(report.get("success", False), True, f"report: {report}") + def test_whoisxmlapi_domain(self, mock_get=None, mock_post=None): + report = whoisxmlapi.Whoisxmlapi( + "Whoisxmlapi_domain", + self.job_id, + self.observable_name, + self.observable_classification, + {}, + ).start() + self.assertEqual(report.get("success", False), True) + + def test_checkdmarc(self, mock_get=None, mock_post=None): + report = checkdmarc.CheckDMARC( + "CheckDMARC", + self.job_id, + self.observable_name, + self.observable_classification, + {}, + ).start() + + self.assertEqual(report.get("success", False), True) + @mock_connections(patch("requests.get", side_effect=mocked_requests)) @mock_connections(patch("requests.post", side_effect=mocked_requests)) diff --git a/tests/utils.py b/tests/utils.py index 424374bc12..072ef7ccc1 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -45,7 +45,7 @@ def test_vt3_get(self, mock_get=None, mock_post=None): self.job_id, self.observable_name, self.observable_classification, - {}, + {"max_tries": 1}, ).start() self.assertEqual(report.get("success", False), True)