From 0c795fa7515c978497d6e59779cd5e7b12e0c0e1 Mon Sep 17 00:00:00 2001 From: Jared Atkinson Date: Mon, 4 Dec 2017 08:35:58 +0100 Subject: [PATCH] Uploaded the wrong Docker stuff --- ACE-Docker/README.md | 32 +++-- ACE-Docker/ace-mssql-linux/import-data.sh | 23 --- ACE-Docker/ace-nginx/.DS_Store | Bin 0 -> 6148 bytes ACE-Docker/ace-nginx/README.md | 34 +++++ ACE-Docker/ace-nginx/entrypoint.sh | 7 +- ACE-Docker/ace-nginx/nginx.conf | 2 +- ACE-Docker/ace-rabbitmq/README.md | 21 +++ ACE-Docker/ace-rabbitmq/ace-cache.py | 125 ++++++++++++++++ ACE-Docker/ace-rabbitmq/ace-entrypoint.sh | 8 ++ ACE-Docker/ace-rabbitmq/ace-lookup.py | 135 ++++++++++++++++++ ACE-Docker/ace-rabbitmq/ace.py | 89 ------------ ACE-Docker/ace-rabbitmq/dockerfile | 13 +- ACE-Docker/ace-sql/.DS_Store | Bin 0 -> 6148 bytes ACE-Docker/ace-sql/README.md | 16 +++ .../{ace-mssql-linux => ace-sql}/ace.sql | 8 +- .../{ace-mssql-linux => ace-sql}/dockerfile | 2 +- ACE-Docker/ace-sql/import-data.sh | 23 +++ ACE-Docker/ace.env | 5 +- ACE-Docker/docker-compose.yml | 15 +- ACE-Docker/settings.sh | 63 ++++++++ ACE-Docker/start.sh | 74 ++++++++++ 21 files changed, 541 insertions(+), 154 deletions(-) delete mode 100644 ACE-Docker/ace-mssql-linux/import-data.sh create mode 100644 ACE-Docker/ace-nginx/.DS_Store create mode 100644 ACE-Docker/ace-nginx/README.md create mode 100644 ACE-Docker/ace-rabbitmq/README.md create mode 100644 ACE-Docker/ace-rabbitmq/ace-cache.py create mode 100644 ACE-Docker/ace-rabbitmq/ace-entrypoint.sh create mode 100644 ACE-Docker/ace-rabbitmq/ace-lookup.py delete mode 100644 ACE-Docker/ace-rabbitmq/ace.py create mode 100644 ACE-Docker/ace-sql/.DS_Store create mode 100644 ACE-Docker/ace-sql/README.md rename ACE-Docker/{ace-mssql-linux => ace-sql}/ace.sql (91%) rename ACE-Docker/{ace-mssql-linux => ace-sql}/dockerfile (92%) create mode 100644 ACE-Docker/ace-sql/import-data.sh create mode 100644 ACE-Docker/settings.sh create mode 100644 ACE-Docker/start.sh diff --git a/ACE-Docker/README.md b/ACE-Docker/README.md index aa4af75..5361900 100644 --- a/ACE-Docker/README.md +++ b/ACE-Docker/README.md @@ -5,21 +5,25 @@ This project focuses on simplifying ACE's deployment process as much as possible ## Components -### ace-app -ASP.NET Core Web Application. This is the main component of the Automated Collection and Enrichment Platform. The ACE web app allows for user, credential, and computer management, as well as, script management and tasking. +### [specterops/ace-mssql-linux](https://hub.docker.com/r/specterops/ace-mssql-linux/) +MSSQL Server. This database provides a backend to keep track of all of the data ACE needs to do its job. This includes User, Credential, Computer, Script, and Schedules. -### ace-mssql-linux -ACE leverages a MSSQL database on the backend to keep track of all the data it needs to do its job. +### [specterops/ace-rabbitmq](https://hub.docker.com/r/specterops/ace-rabbitmq/) +RabbitMQ Messaging System. ACE's enrichment pipeline is built on a robust messaging system that guides each scan result through data enrichments, like Virus Total hash lookups, all the way to ingestion into a SIEM. -### ace-rabbitmq - - -### ace-web +### [specterops/ace-nginx](https://hub.docker.com/r/specterops/ace-nginx/) +NGINX HTTP(S) Reverse Proxy. Proxy's access to the ACE Web Application and provides SSL Certificates for those connections. ## Getting Started -``` -sudo git clone https://github.com/Invoke-IR/ACE.git -cd ACE/ACE-Docker -docker-compose build -docker-compose up -d -``` \ No newline at end of file +Our goal is to make provisioning ACE as simple as possible, so we wrote a small batch script to get things set up. Follow the steps, on a Linux or OSX machine, below and you should be in business: +* Install Docker +* If on Linux, Install Docker Compose +* Adjust Docker preferences to allow containers to use 4GBs of RAM (Docker -> Preferences -> Advanced -> Memory) +* Download this repository +* Execute start.sh + +start.sh is a simple shell script that accomplishes the remaining set up steps. Below is a list of tasks accomplished by start.sh: +* Create SSL certificate +* Add SSL Thumbprint to the ACE Web Application's appsettings.json file +* Build ACE Docker images with Docker Compose +* Start ACE Docker containers \ No newline at end of file diff --git a/ACE-Docker/ace-mssql-linux/import-data.sh b/ACE-Docker/ace-mssql-linux/import-data.sh deleted file mode 100644 index 496d9c6..0000000 --- a/ACE-Docker/ace-mssql-linux/import-data.sh +++ /dev/null @@ -1,23 +0,0 @@ -/opt/mssql/bin/sqlservr & - -#wait for the SQL Server to come up -sleep 45s - -# Create Unique API Key -echo ">> Generating ACE API Key" - -apikey=$(cat /proc/sys/kernel/random/uuid) -sed -i -e 's/\[APIKEY\]/'"$apikey"'/g' /usr/src/ace/ace.sql - -#run the setup script to create the DB and the schema in the DB -/opt/mssql-tools/bin/sqlcmd -S localhost -U sa -P $1 -Q "CREATE DATABASE ACEWebService" -/opt/mssql-tools/bin/sqlcmd -S localhost -U sa -P $1 -d ACEWebService -i /usr/src/ace/ace.sql - -echo ">> ########################################################" -echo ">> # ACE Admin APIKey: $apikey" -echo ">> # SQL Server SA Password: $1" -echo ">> ########################################################" - -while true; do -sleep 300 -done \ No newline at end of file diff --git a/ACE-Docker/ace-nginx/.DS_Store b/ACE-Docker/ace-nginx/.DS_Store new file mode 100644 index 0000000000000000000000000000000000000000..98ebfa7f60a494509c78fce0b00311e58b347547 GIT binary patch literal 6148 zcmeHK%Sr?>5UkccEUUsEJ>~~o@aSz=)`Mq%K=!4}4vzQ$z0H^St6KeVadeI%BHfTw zC+SKu1(T!!$n3XA~^SrkQ`9@FmLp}h#+CyLFn!adg5pvJ0aqQBUsdmmwm1`BMt z>)rk@>Si&on>oi-oL82UH#2&`5o2%h$oLoBk-f3j1P{3BGUf0KbHvDNi+{$1t2S(} zyBwU6Gx@Zdu9%hzqynixDv%2Ni~_u~)rJR-S*HT2Kq|0PK+lK5qF4pCj&^jgvJ`+g zV7D24eJ(4f1Xh8qBS&cBRH9QQQVelA`%A=Cfvuy{A(4DYto)I@h^@}~i-kif$E;I< zRG_PX=DxJC-v8(PWqOl*3&|=KNCp0@0y3Cg&ZhjjxLdz{r{1-N<%&g3<63oS^gE9L iHuM}hwm}~+>NBniY#ntL?Ke6x9|9&wR;j=*DDV!n-#1?X literal 0 HcmV?d00001 diff --git a/ACE-Docker/ace-nginx/README.md b/ACE-Docker/ace-nginx/README.md new file mode 100644 index 0000000..9aa389f --- /dev/null +++ b/ACE-Docker/ace-nginx/README.md @@ -0,0 +1,34 @@ +Built on [nginx](https://hub.docker.com/_/nginx/), this image provides an SSL proxy for the [ACE Web Application](https://github.com/Invoke-IR/ACE/tree/master/ACE-WebService). + +ACE relies on SSL for two important features: +* Encryption - Data sent to and from the ACE Web Application is encrypted +* Authentication - Certificate pinning is used to provide server side authentication to avoid Man-in-the-Middle attacks. + +## Using this Image +The ACE Nginx can be run in a couple different ways. +### Standalone +If you are running ACE in a test/development/standalone deployment, then you can simply run the container as shown below. +``` +docker run --name ace-nginx -p 80:80 -p 443:443 -d specterops/ace-nginx +``` +### Clustered/Redundant +If you plan on running ACE in a Kubernetes cluster with replication, you want to maintain the same SSL certificates in all instances of the specterops/ace-nginx image. This can be achieved through the use of Volumes. + +Simply create a docker volume (it can be named "certs" or whatever you choose). +``` +docker volume create --name certs +``` + +Then run your container(s) with the -v flag, linking your newly created volume to "/etc/nginx/certs". The volume will ensure a consistent SSL certificate across all ace-nginx instances. +``` +docker run --name ace-nginx -v certs:/etc/nginx/certs -p 80:80 -p 443:443 -d specterops/ace-nginx +``` + +### Get SSL Certificate Thumbprint +The .NET WebClient does not trust self-signed SSL Certificates by default. The ACE PowerShell module bypasses this limitation by using certificate pinning, where the PowerShell script compares the user supplied SSL Thumbprint to that returned by the target server. If the Thumbprints match, then the server is authenticated and the request is allowed. The SSL Thumbprint is output at container runtime and can be found with the following command: +``` +docker logs ace-nginx +################################################################ +# ACE SSL Thumbprint: 3179CC1A0A0E20477260BFB8D559F35240297E6B # +################################################################ +``` \ No newline at end of file diff --git a/ACE-Docker/ace-nginx/entrypoint.sh b/ACE-Docker/ace-nginx/entrypoint.sh index 77066da..6306074 100644 --- a/ACE-Docker/ace-nginx/entrypoint.sh +++ b/ACE-Docker/ace-nginx/entrypoint.sh @@ -1,5 +1,8 @@ #!/bin/sh +# Add Environment Variable to nginx.conf +sed -i -e 's/\[WEBSERVICE_IP\]/'"$WEBSERVICE_IP"'/g' /etc/nginx/nginx.conf + # Check if /etc/nginx/certs directory exits if [ ! -d /etc/nginx/certs ]; then mkdir /etc/nginx/certs @@ -12,6 +15,4 @@ fi # Get and output SSL Thumbprint fingerprint=$(openssl x509 -in /etc/nginx/certs/server.crt -noout -fingerprint | sed 's/SHA1 Fingerprint=//g' | sed 's/://g') -echo "################################################################" -echo "# ACE SSL Thumbprint: $fingerprint #" -echo "################################################################" \ No newline at end of file +echo "\"Thumbprint\": \"$fingerprint\"," \ No newline at end of file diff --git a/ACE-Docker/ace-nginx/nginx.conf b/ACE-Docker/ace-nginx/nginx.conf index 0062e66..0af2e06 100644 --- a/ACE-Docker/ace-nginx/nginx.conf +++ b/ACE-Docker/ace-nginx/nginx.conf @@ -8,7 +8,7 @@ http { # Act as Load Balancer for 4 nodes upstream web.ace.local { - server 172.18.0.4:80; + server [WEBSERVICE_IP]:5000; # server dockernginxkestrel_core-app_2:80; # server dockernginxkestrel_core-app_3:80; # server dockernginxkestrel_core-app_4:80; diff --git a/ACE-Docker/ace-rabbitmq/README.md b/ACE-Docker/ace-rabbitmq/README.md new file mode 100644 index 0000000..6d1e122 --- /dev/null +++ b/ACE-Docker/ace-rabbitmq/README.md @@ -0,0 +1,21 @@ +Built on [RabbitMQ](https://hub.docker.com/_/rabbitmq/), this images provides the backend database used by the [ACE RabbitMQ Server](https://github.com/Invoke-IR/ACE/tree/master/ACE-RabbitMQ). + +## Requirements +* This image requires Docker Engine 1.8+ in any of their supported platforms. +* Requires the following environment flags +* RABBITMQ_DEFAULT_USER= +* RABBITMQ_DEFAULT_PASS= +* APIKEY= + +## Using this Image +### Run +``` +docker run --name ace-rabbitmq -e 'RABBITMQ_DEFAULT_USER=yourUsername' -e 'RABBITMQ_DEFAULT_PASS=yourPassword' -e 'APIKEY=yourVirusTotalPublicAPIKey' -p 5672:5672 -p 15672:15672 -d specterops/ace-rabbitmq +``` +# For Persistence +If you desire your RabbitMQ data and setting to persist between containers, you need to create a docker volume ```docker volume create rabbitmq``` then add ```-v rabbitmq:/var/lib/rabbitmq``` to the docker run command + +### Environment Variables +* **RABBITMQ_DEFAULT_USER** Username for RabbitMQ server. Will be used to connect to server and log into management interface. +* **RABBITMQ_DEFAULT_PASS** Password for RabbitMQ server. Will be used to connect to server and log into management interface. +* **APIKEY** Public VirusTotal API key. Allows for lookups of hashes on VirusTotal \ No newline at end of file diff --git a/ACE-Docker/ace-rabbitmq/ace-cache.py b/ACE-Docker/ace-rabbitmq/ace-cache.py new file mode 100644 index 0000000..5fb941f --- /dev/null +++ b/ACE-Docker/ace-rabbitmq/ace-cache.py @@ -0,0 +1,125 @@ +#!/usr/bin/env python +import json +import sys +import pika +import requests +from argparse import ArgumentParser +from json import dumps + +# Our local cache of hashes. Each of the consumers checks this dictionary first +# before doing a lookup against VirusTotal to save time and API queries +cachedEntries = {} + +class CachedConsumer(object): + """A consumer that receives hashes and queries the VirusTotal api + to find if VirusTotal has any matching hashes, and how many positive + (malicious) results for that hash. + """ + EXCHANGE = 'ace_exchange' + EXCHANGE_TYPE = 'topic' + + def __init__(self, connection): + """Create a new instance of LookupConsumer, passing in the API key to use. + + :param connection connection: A pika connection object. + """ + self._connection = connection + self._channel = None + + def consume_message(self, channel, method, properties, body): + """Consume a message from channel. This function is passed as a callback + to basic_consume. After checking the body of the message, the consumer checks the + cache and either publish the cached entry, or perform a lookup and add the result + to the cache. + """ + self._channel = channel + message = json.loads(body) # parse the JSON results from the message + newRoutingKey = "" + if 'SHA256Hash' in message and message['SHA256Hash'] is not None: + sha256hash = message['SHA256Hash'] # assign the value temporarily instead of doing a lookup each time + if sha256hash in cachedEntries: #hash is cached + print "Hash is cached" + message[u"VTRecordExists"] = cachedEntries[sha256hash][u"VTRecordExists"] + if u"VTPositives" in cachedEntries[sha256hash]: + message[u"VTPositives"] = cachedEntries[sha256hash][u"VTPositives"] + enrichment,newRoutingKey = method.routing_key.split(".",1) + self.publish_message(method, message, newRoutingKey) + elif u'VTRecordExists' in message: #needs to be cached + print "Adding hash to cache" + cachedEntries[sha256hash] = {} + cachedEntries[sha256hash][u"VTRecordExists"] = message[u"VTRecordExists"] + if u'VTPositives' in message: + cachedEntries[sha256hash][u'VTPositives'] = message[u'VTPositives'] + enrichment,newRoutingKey = method.routing_key.split(".",1) + self.publish_message(method, message, newRoutingKey) + else: #send for lookup + print "sending to VT" + newRoutingKey = "lookup." + method.routing_key + self.publish_message(method, message, newRoutingKey) + self._connection.sleep(1) + elif message['SHA256Hash'] is None: + print "Hash is null" + enrichment,newRoutingKey = method.routing_key.split(".",1) + self.publish_message(method, message, newRoutingKey) + + def publish_message(self, method, message, routingKey): + """Publish a message to the channel with the new routing key after enrichment. + """ + body = json.dumps(message) + channel = self._channel + channel.basic_ack(delivery_tag = method.delivery_tag) + channel.basic_publish(exchange=self.EXCHANGE, routing_key=routingKey,body=body, properties=pika.BasicProperties(delivery_mode = 2,)) + +def main(): + parser = ArgumentParser() + parser.add_argument( + '-s', '--Server', dest='rabbitmq_server', default='', + help='[MANDATORY] RabbitMQ server hostname or IP address') + parser.add_argument( + '-u', '--User', dest='rabbitmq_user', default='', + help='[OPTIONAL] RabbitMQ username') + parser.add_argument( + '-p', '--Password', dest='rabbitmq_password', default='', + help='[OPTIONAL] RabbitMQ password') + + args = parser.parse_args() + try: + if (args.rabbitmq_password != '' and args.rabbitmq_user != ''): + creds = pika.PlainCredentials(args.rabbitmq_user, args.rabbitmq_password) + connection = pika.BlockingConnection(pika.ConnectionParameters(host=args.rabbitmq_server, + credentials=creds)) + elif (args.rabbitmq_server != ''): + connection = pika.BlockingConnection(pika.ConnectionParameters(host=args.rabbitmq_server)) + else: + print("Must provide command line parameters, run 'python ACE_RabbitMQ.py -h' for help") + return + channel = connection.channel() + except: + print("Issue connecting to RabbitMQ,") + + channel.exchange_declare(exchange='ace_exchange',exchange_type='topic', durable=True) + + channel.queue_declare(queue='siem', durable=True) + channel.queue_declare(queue='cached_hash', durable=True) + channel.queue_declare(queue='lookup', durable=True) + channel.queue_declare(queue='status', durable=True) + + channel.queue_bind(exchange='ace_exchange', queue='siem', routing_key='siem') + channel.queue_bind(exchange='ace_exchange', queue='cached_hash', routing_key='hash.#') + channel.queue_bind(exchange='ace_exchange', queue='lookup', routing_key='lookup.hash.#') + channel.queue_bind(exchange='ace_exchange', queue='status', routing_key='status') + channel.basic_qos(prefetch_count=1) + + + print("Waiting for messages") + + cacheConsume = CachedConsumer(connection) + + channel.basic_consume(cacheConsume.consume_message, queue='cached_hash') + + channel.start_consuming() + + connection.close() + +if __name__ == '__main__': + main() \ No newline at end of file diff --git a/ACE-Docker/ace-rabbitmq/ace-entrypoint.sh b/ACE-Docker/ace-rabbitmq/ace-entrypoint.sh new file mode 100644 index 0000000..b06c7cb --- /dev/null +++ b/ACE-Docker/ace-rabbitmq/ace-entrypoint.sh @@ -0,0 +1,8 @@ +#!/bin/bash +python /root/ace-lookup.py -s 127.0.0.1 -u $RABBITMQ_DEFAULT_USER -p $RABBITMQ_DEFAULT_PASS -k $APIKEY & +python /root/ace-cache.py -s 127.0.0.1 -u $RABBITMQ_DEFAULT_USER -p $RABBITMQ_DEFAULT_PASS & + +echo "\"RabbitMQUserName\": \"$RABBITMQ_DEFAULT_USER\"," +echo "\"RabbitMQPassword\": \"$RABBITMQ_DEFAULT_PASS\"," + +while true; do :; sleep 600; done \ No newline at end of file diff --git a/ACE-Docker/ace-rabbitmq/ace-lookup.py b/ACE-Docker/ace-rabbitmq/ace-lookup.py new file mode 100644 index 0000000..4f7908c --- /dev/null +++ b/ACE-Docker/ace-rabbitmq/ace-lookup.py @@ -0,0 +1,135 @@ +#!/usr/bin/env python +import json +import sys +import pika +import requests +from argparse import ArgumentParser +from json import dumps + +class VTConsumer(object): + """A consumer that receives hashes and queries the VirusTotal api + to find if VirusTotal has any matching hashes, and how many positive + (malicious) results for that hash. + """ + EXCHANGE = 'ace_exchange' + EXCHANGE_TYPE = 'topic' + + def __init__(self, api_key, connection): + """Create a new instance of VTConsumer, passing in the API key to use. + + :param str api_key: The VirusTotal API key to use. + :param connection connection: A pika connection object. + """ + self._apikey = api_key + self._connection = connection + self._channel = None + + def consume_message(self, channel, method, properties, body): + """Consume a message from channel. This function is passed as a callback + to basic_consume. After checking the body of the message, the consumer checks the + cache and either publish the cached entry, or perform a lookup and add the result + to the cache. + """ + self._channel = channel + message = json.loads(body) # parse the JSON results from the message + entry = {} + sha256hash = message['SHA256Hash'] + entry = self.lookup_hash(sha256hash) + print entry + if u'VTRecordExists' in entry: + message[u"VTRecordExists"] = entry[u"VTRecordExists"] + if u'VTPositives' in entry: + message[u'VTPositives'] = entry[u'VTPositives'] + self.publish_message(method, message) + + def lookup_hash(self, sha256hash): + """Perform a lookup against VirusTotal for a given hash. + + :param str vt_hash: A SHA256Hash to check against the VirusTotal API. + """ + params = { 'apikey': self._apikey, 'resource': sha256hash } + headers = {"Accept-Encoding": "gzip, deflate", "User-Agent" : "gzip, VirusTotal ACE Enrichment Consumer v0.1"} + response = requests.get('https://www.virustotal.com/vtapi/v2/file/report', params=params, headers=headers) + if response.status_code == 204: + self._connection.sleep(60) + response = requests.get('https://www.virustotal.com/vtapi/v2/file/report', params=params, headers=headers) + json_response = response.json() + if json_response['response_code'] == 1: + new_record = {} + new_record[u"VTRecordExists"] = u"True" + new_record[u"VTPositives"] = json_response['positives'] + elif json_response['response_code'] == 0: + new_record = {} + new_record[u"VTRecordExists"] = u"False" + elif json_response['response_code'] == -2: + new_record = {} + new_record[u"VTRecordExists"] = u"False" + return new_record + + def publish_message(self, method, message): + """Publish a message to the channel with the new routing key after enrichment. + """ + enrichment,newRoutingKey = method.routing_key.split(".",1) + body = json.dumps(message) + channel = self._channel + channel.basic_ack(delivery_tag = method.delivery_tag) + channel.basic_publish(exchange=self.EXCHANGE, routing_key=newRoutingKey,body=body, properties=pika.BasicProperties(delivery_mode = 2,)) + +def main(): + parser = ArgumentParser() + parser.add_argument( + '-s', '--Server', dest='rabbitmq_server', default='', + help='[MANDATORY] RabbitMQ server hostname or IP address') + parser.add_argument( + '-u', '--User', dest='rabbitmq_user', default='', + help='[OPTIONAL] RabbitMQ username') + parser.add_argument( + '-p', '--Password', dest='rabbitmq_password', default='', + help='[OPTIONAL] RabbitMQ password') + parser.add_argument( + '-k', '--APIKey', dest='VTAPIKey', default='', + help='[MANDATORY] VirusTotal API Key') + + args = parser.parse_args() + try: + if (args.VTAPIKey == ''): + print("Must provide command line parameters, run 'python ACE_RabbitMQ.py -h' for help") + return + if (args.rabbitmq_password != '' and args.rabbitmq_user != ''): + creds = pika.PlainCredentials(args.rabbitmq_user, args.rabbitmq_password) + connection = pika.BlockingConnection(pika.ConnectionParameters(host=args.rabbitmq_server, + credentials=creds)) + elif (args.rabbitmq_server != ''): + connection = pika.BlockingConnection(pika.ConnectionParameters(host=args.rabbitmq_server)) + else: + print("Must provide command line parameters, run 'python ACE_RabbitMQ.py -h' for help") + return + channel = connection.channel() + except: + print("Issue connecting to RabbitMQ,") + + channel.exchange_declare(exchange='ace_exchange',exchange_type='topic', durable=True) + + channel.queue_declare(queue='siem', durable=True) + channel.queue_declare(queue='cached_hash', durable=True) + channel.queue_declare(queue='lookup', durable=True) + channel.queue_declare(queue='status', durable=True) + + channel.queue_bind(exchange='ace_exchange', queue='siem', routing_key='siem') + channel.queue_bind(exchange='ace_exchange', queue='cached_hash', routing_key='hash.#') + channel.queue_bind(exchange='ace_exchange', queue='lookup', routing_key='lookup.hash.#') + channel.queue_bind(exchange='ace_exchange', queue='status', routing_key='status') + channel.basic_qos(prefetch_count=1) + + + print("Waiting for messages") + + consumer = VTConsumer(args.VTAPIKey, connection) + channel.basic_consume(consumer.consume_message, queue='lookup') + + channel.start_consuming() + + connection.close() + +if __name__ == '__main__': + main() \ No newline at end of file diff --git a/ACE-Docker/ace-rabbitmq/ace.py b/ACE-Docker/ace-rabbitmq/ace.py deleted file mode 100644 index a91d00d..0000000 --- a/ACE-Docker/ace-rabbitmq/ace.py +++ /dev/null @@ -1,89 +0,0 @@ -#!/usr/bin/env python -import json -import pika -import requests -import sys -import time -from argparse import ArgumentParser -from json import dumps - - -def main(): - def hashlookupconsumer(ch, method, properties, body): - message = json.loads(body) - try: - params = {'apikey': '[VIRUSTOTAL-APIKEY]', 'resource': message['SHA256Hash']} - headers = {"Accept-Encoding": "gzip, deflate", "User-Agent" : "gzip, My Python requests library example client or username"} - response = requests.get('https://www.virustotal.com/vtapi/v2/file/report', params=params, headers=headers) - if response.status_code == 204: - time.sleep(60) - response = requests.get('https://www.virustotal.com/vtapi/v2/file/report', params=params, headers=headers) - json_response = response.json() - if json_response['response_code'] == 1: - message[u"VTRecordExists"] = u"True" - message[u"VTPositives"] = json_response['positives'] - print message - else: - message[u"VTRecordExists"] = u"False" - except KeyError: - print("No hash field") - enrichment,newRoutingKey = method.routing_key.split(".",1) - print("Routing Key:" + newRoutingKey) - channel.basic_ack(delivery_tag = method.delivery_tag) - #Need to strip the front off and enrich here and error catching - channel.basic_publish(exchange='ace_exchange', - routing_key=newRoutingKey, - body=body) - parser = ArgumentParser() - parser.add_argument( - '-s', '--Server', dest='rabbitmq_server', default='', - help='[MANDATORY] RabbitMQ server hostname or IP address') - parser.add_argument( - '-u', '--User', dest='rabbitmq_user', default='', - help='[OPTIONAL] RabbitMQ username') - parser.add_argument( - '-p', '--Password', dest='rabbitmq_password', default='', - help='[OPTIONAL] RabbitMQ password') - - args = parser.parse_args() - - try: - if (args.rabbitmq_password != '' and args.rabbitmq_user != ''): - creds = pika.PlainCredentials(args.rabbitmq_user, args.rabbitmq_password) - connection = pika.BlockingConnection(pika.ConnectionParameters(host=args.rabbitmq_server, - credentials=creds)) - elif (args.rabbitmq_server != ''): - connection = pika.BlockingConnection(pika.ConnectionParameters(host=args.rabbitmq_server)) - else: - print "Must provide command line parameters, run 'python ACE_RabbitMQ.py -h' for help" - return - channel = connection.channel() - except: - print "Issue connecting to RabbitMQ," - - channel.exchange_declare(exchange='ace_exchange',exchange_type='topic') - - channel.queue_declare(queue='file_output') - channel.queue_declare(queue='siem') - channel.queue_declare(queue='pre_hash') - - channel.queue_bind(exchange='ace_exchange', - queue='siem', - routing_key='siem') - channel.queue_bind(exchange='ace_exchange', - queue='file_output', - routing_key='file') - channel.queue_bind(exchange='ace_exchange', - queue='pre_hash', - routing_key='hash.#') - - print("Waiting for messages") - - channel.basic_consume(hashlookupconsumer, queue='pre_hash') - - channel.start_consuming() - - connection.close() - -if __name__ == '__main__': - main() \ No newline at end of file diff --git a/ACE-Docker/ace-rabbitmq/dockerfile b/ACE-Docker/ace-rabbitmq/dockerfile index 23d2209..2e3c7e2 100644 --- a/ACE-Docker/ace-rabbitmq/dockerfile +++ b/ACE-Docker/ace-rabbitmq/dockerfile @@ -1,13 +1,18 @@ FROM rabbitmq:3-management MAINTAINER Jared Atkinson +ADD ace-entrypoint.sh /root/ace-entrypoint.sh +ADD ace-cache.py /root/ace-cache.py +ADD ace-lookup.py /root/ace-lookup.py RUN \ - apt-get update -y \ + chmod +x /root/ace-entrypoint.sh \ + && chmod +x /root/ace-cache.py \ + && chmod +x /root/ace-lookup.py \ + && apt-get update -y \ && apt-get upgrade -y \ && apt-get dist-upgrade -y \ && apt-get install -y python2.7 python-pip \ && pip install pika requests -ADD ace.py /root/ace.py CMD \ - /usr/local/bin/docker-entrypoint.sh rabbitmq-server & \ + /usr/local/bin/docker-entrypoint.sh rabbitmq-server > /dev/null & \ sleep 30 \ - && python /root/ace.py -s 127.0.0.1 -u $RABBITMQ_DEFAULT_USER -p $RABBITMQ_DEFAULT_PASS \ No newline at end of file + && /root/ace-entrypoint.sh \ No newline at end of file diff --git a/ACE-Docker/ace-sql/.DS_Store b/ACE-Docker/ace-sql/.DS_Store new file mode 100644 index 0000000000000000000000000000000000000000..10bf1dd0f7e85589fff72523cdde64cfcf07951b GIT binary patch literal 6148 zcmeHKyGjH>5Ukb%4+XhQ!wF3kj7^r~42=B&?}_5d>Z$quj$f(Oj}vd*P(-8~rfYV( zYi6flXSM;z_;qm(i~tPT6h%8dro+9Vok-p#ip}wiC)8JUGoRHx6aB>@efK>~F~<^T z`11bEj1kHrN5qqNGuyyE7HF7%PsSQ6+@Z$O_q^fYt;0EF)&m~NTjK?%oMB3w+KRUw z?nQ@di>6BjQh`(;6-WhsOab26YQqD^^r=89kP3V%pyxwjQ>+48M>{&`tOOtqIBdqg z))LAofmLAZ$OuiGN_473i6Ktsc!|6!uyu4gB#IA-l{dwU*y^0WSURL~OrHv*0^15` z?MoZ!{eQ_%rnktqlys>;D)3(wkiq17GUo2$ZvD1Ty=x2G4V#+gwc4Svzx5Nq2R%oQ eE$H#0edbkxt)tGO<3=aukAMl1E*1C%1wH^a*f&T3 literal 0 HcmV?d00001 diff --git a/ACE-Docker/ace-sql/README.md b/ACE-Docker/ace-sql/README.md new file mode 100644 index 0000000..6dbb7da --- /dev/null +++ b/ACE-Docker/ace-sql/README.md @@ -0,0 +1,16 @@ +Built on [microsoft/mssql-server-linux](https://hub.docker.com/r/microsoft/mssql-server-linux/), this images provides the backend database used by the [ACE Web Application](https://github.com/Invoke-IR/ACE/tree/master/ACE-WebService). + +## Requirements +* This image requires Docker Engine 1.8+ in any of their supported platforms. +* At least 3.25 GB of RAM. Make sure to assign enough memory to the Docker VM if you're running on Docker for Mac or Windows. +* Requires the following environment flags +* SA_PASSWORD= +* A strong system administrator (SA) password: At least 8 characters including uppercase, lowercase letters, base-10 digits and/or non-alphanumeric symbols. + +## Using this Image +### Run +``` +docker run --name ace-sql -e 'SA_PASSWORD=yourStrong(!)Password' -p 1433:1433 -d specterops/ace-sql +``` +### Environment Variables +* **SA_PASSWORD** is the database system administrator (userid = 'sa') password used to connect to SQL Server once the container is running. Important note: This password needs to include at least 8 characters of at least three of these four categories: uppercase letters, lowercase letters, numbers and non-alphanumeric symbols. \ No newline at end of file diff --git a/ACE-Docker/ace-mssql-linux/ace.sql b/ACE-Docker/ace-sql/ace.sql similarity index 91% rename from ACE-Docker/ace-mssql-linux/ace.sql rename to ACE-Docker/ace-sql/ace.sql index e071eab..4e6bb2c 100644 --- a/ACE-Docker/ace-mssql-linux/ace.sql +++ b/ACE-Docker/ace-sql/ace.sql @@ -41,19 +41,15 @@ CREATE TABLE [dbo].[Scripts] ( [LastUpdateTime] DATETIME2 (7) NOT NULL, [Name] NVARCHAR (MAX) NOT NULL, [Uri] NVARCHAR (MAX) NOT NULL, - [Enrichment] NVARCHAR (MAX) DEFAULT (N'') NOT NULL, - [Output] NVARCHAR (MAX) DEFAULT (N'') NOT NULL, CONSTRAINT [PK_Scripts] PRIMARY KEY CLUSTERED ([Id] ASC) ); CREATE TABLE [dbo].[Downloads] ( [Id] UNIQUEIDENTIFIER NOT NULL, - [AccessedTime] DATETIME2 (7) NOT NULL, - [BornTime] DATETIME2 (7) NOT NULL, [ComputerName] NVARCHAR (MAX) NOT NULL, + [Content] sql_variant NOT NULL, [DownloadTime] DATETIME2 (7) NOT NULL, [FullPath] NVARCHAR (MAX) NOT NULL, - [ModifiedTime] DATETIME2 (7) NOT NULL, [Name] NVARCHAR (MAX) NOT NULL, CONSTRAINT [PK_Downloads] PRIMARY KEY CLUSTERED ([Id] ASC) ); @@ -90,4 +86,4 @@ CREATE TABLE [dbo].[Users] ( CONSTRAINT [PK_Users] PRIMARY KEY CLUSTERED ([Id] ASC) ); -INSERT INTO [dbo].[Users] ([Id], [ApiKey], [FirstName], [IsAdmin], [LastName], [UserName]) VALUES (N'334d89c9-da7a-43e8-a648-5dc8b22019ed', N'A40E94D3-E463-4321-9E12-C1AE5D1A6525', N'Admin', 1, N'Admin', N'admin') +INSERT INTO [dbo].[Users] ([Id], [ApiKey], [FirstName], [IsAdmin], [LastName], [UserName]) VALUES (N'334d89c9-da7a-43e8-a648-5dc8b22019ed', N'[APIKEY]', N'Admin', 1, N'Admin', N'admin') diff --git a/ACE-Docker/ace-mssql-linux/dockerfile b/ACE-Docker/ace-sql/dockerfile similarity index 92% rename from ACE-Docker/ace-mssql-linux/dockerfile rename to ACE-Docker/ace-sql/dockerfile index 2d972a8..ad10482 100644 --- a/ACE-Docker/ace-mssql-linux/dockerfile +++ b/ACE-Docker/ace-sql/dockerfile @@ -7,7 +7,7 @@ ENV ACCEPT_EULA Y RUN mkdir -p /usr/src/ace WORKDIR /usr/src/ace -#COPY entrypoint.sh /usr/src/ace +# Copy files to container COPY import-data.sh /usr/src/ace COPY ace.sql /usr/src/ace diff --git a/ACE-Docker/ace-sql/import-data.sh b/ACE-Docker/ace-sql/import-data.sh new file mode 100644 index 0000000..3eb9977 --- /dev/null +++ b/ACE-Docker/ace-sql/import-data.sh @@ -0,0 +1,23 @@ +/opt/mssql/bin/sqlservr > /dev/null & + +#wait for the SQL Server to come up +sleep 45s + +# Create Unique API Key +apikey=$(cat /proc/sys/kernel/random/uuid) +startacesweep=$(cat /proc/sys/kernel/random/uuid) +downloadacefile=$(cat /proc/sys/kernel/random/uuid) +sed -i -e 's/\[APIKEY\]/'"$apikey"'/g' /usr/src/ace/ace.sql + +#run the setup script to create the DB and the schema in the DB +/opt/mssql-tools/bin/sqlcmd -S localhost -U sa -P $SA_PASSWORD -Q "CREATE DATABASE ACEWebService" > /dev/null +/opt/mssql-tools/bin/sqlcmd -S localhost -U sa -P $SA_PASSWORD -d ACEWebService -i /usr/src/ace/ace.sql > /dev/null + +echo "\"ApiKey\": \"$apikey\"," +echo "\"StartAceSweep\": \"$startacesweep\"," +echo "\"DownloadAceFile\": \"$downloadacefile\"" +echo "\"DefaultConnection\": \"Server=sql.ace.local;Database=ACEWebService;User Id=sa;Password=$SA_PASSWORD;MultipleActiveResultSets=true\"" + +while true; do +sleep 300 +done \ No newline at end of file diff --git a/ACE-Docker/ace.env b/ACE-Docker/ace.env index f81af64..2f93183 100644 --- a/ACE-Docker/ace.env +++ b/ACE-Docker/ace.env @@ -1,4 +1,5 @@ -ACCEPT_EULA=Y SA_PASSWORD=P@ssw0rd! RABBITMQ_DEFAULT_USER=ace -RABBITMQ_DEFAULT_PASS=P@ssw0rd! \ No newline at end of file +RABBITMQ_DEFAULT_PASS=P@ssw0rd! +APIKEY=YOURAPIKEYHERE +WEBSERVICE_IP=192.168.92.152 \ No newline at end of file diff --git a/ACE-Docker/docker-compose.yml b/ACE-Docker/docker-compose.yml index 7740342..91d2bdc 100644 --- a/ACE-Docker/docker-compose.yml +++ b/ACE-Docker/docker-compose.yml @@ -5,10 +5,6 @@ networks: ipam: config: - subnet: 172.18.0.0/16 - -volumes: - certs: - services: ace-rabbitmq: image: specterops/ace-rabbitmq @@ -23,9 +19,8 @@ services: ports: - 5672:5672 - 15672:15672 - ace-sql: - image: specterops/ace-mssql-linux + image: specterops/ace-sql container_name: ace-sql env_file: ./ace.env hostname: ace-sql @@ -36,17 +31,15 @@ services: ipv4_address: 172.18.0.3 ports: - 1433:1433 - - ace-web: + ace-nginx: image: specterops/ace-nginx container_name: ace-nginx + env_file: ./ace.env hostname: ace-nginx - volumes: - - certs:/etc/nginx/certs networks: ace: aliases: - - web.ace.local + - nginx.ace.local ipv4_address: 172.18.0.4 ports: - "80:80" diff --git a/ACE-Docker/settings.sh b/ACE-Docker/settings.sh new file mode 100644 index 0000000..308eb93 --- /dev/null +++ b/ACE-Docker/settings.sh @@ -0,0 +1,63 @@ +# Get IP Address +unameOut="$(uname -s)" +case "${unameOut}" in + Linux*) ip=$(/sbin/ifconfig eth0 | grep 'inet addr:' | cut -d: -f2 | awk '{print $1}');; + Darwin*) ip=$(ifconfig en0 | grep inet | grep -v inet6 | cut -d ' ' -f2);; + CYGWIN*) ip=Cygwin;; + MINGW*) ip=MinGw;; + *) ip="UNKNOWN:${unameOut}" +esac + +# Write appsettings.Production.json to screen +clear +echo "" +echo "" +echo "==========================================================" +echo "| appsettings.Production.json |" +echo "==========================================================" +echo "" +echo "{" +echo " \"Logging\": {" +echo " \"IncludeScopes\": false," +echo " \"LogLevel\": {" +echo " \"Default\": \"Debug\"," +echo " \"System\": \"Information\"," +echo " \"Microsoft\": \"Information\"" +echo " }" +echo " }," +echo "" +echo " \"AppSettings\": {" +echo " \"RabbitMQServer\": \"$ip\"," +echo " $(docker logs ace-rabbitmq | grep UserName)" +echo " $(docker logs ace-rabbitmq | grep Password)" +echo " $(docker logs ace-nginx | grep Thumbprint)" +echo " $(docker logs ace-sql | grep ApiKey)" +echo " $(docker logs ace-sql | grep StartAceSweep)" +echo " $(docker logs ace-sql | grep DownloadAceFile)" +echo " }," +echo "" +echo " \"ConnectionStrings\": {" +echo " $(docker logs ace-sql | grep DefaultConnection | sed s/sql.ace.local/$ip/)" +echo " }" +echo "}" +echo "" +echo "==========================================================" +echo "" +echo "" + +echo "===============================================================" +echo "| Thank you for provisioning ACE with Docker!! |" +echo "| Please use the following information to interact with ACE |" +echo "===============================================================" +echo "" +echo " \$settings = @{" +echo " Uri = 'https://$ip'" +IFS='"' read -r -a array <<< "$(docker logs ace-sql | grep Api)" +echo " ApiKey = '${array[3]}'" +IFS='"' read -r -a array <<< "$(docker logs ace-nginx)" +echo " Thumbprint = '${array[3]}'" +echo " }" +echo "" +echo "==============================================================" +echo "" +echo "" \ No newline at end of file diff --git a/ACE-Docker/start.sh b/ACE-Docker/start.sh new file mode 100644 index 0000000..d4b2936 --- /dev/null +++ b/ACE-Docker/start.sh @@ -0,0 +1,74 @@ +# Get directory of script and change to it +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +cd $DIR + +# Build Docker Images and Start Containers +docker-compose build +docker-compose up -d + +# Get IP Address +unameOut="$(uname -s)" +case "${unameOut}" in + Linux*) ip=$(/sbin/ifconfig eth0 | grep 'inet addr:' | cut -d: -f2 | awk '{print $1}');; + Darwin*) ip=$(ifconfig en0 | grep inet | grep -v inet6 | cut -d ' ' -f2);; + CYGWIN*) ip=Cygwin;; + MINGW*) ip=MinGw;; + *) ip="UNKNOWN:${unameOut}" +esac + +sleep 60 + +# Write appsettings.Production.json to screen +clear +echo "" +echo "" +echo "==========================================================" +echo "| appsettings.Production.json |" +echo "==========================================================" +echo "" +echo "{" +echo " \"Logging\": {" +echo " \"IncludeScopes\": false," +echo " \"LogLevel\": {" +echo " \"Default\": \"Debug\"," +echo " \"System\": \"Information\"," +echo " \"Microsoft\": \"Information\"" +echo " }" +echo " }," +echo "" +echo " \"AppSettings\": {" +echo " \"RabbitMQServer\": \"$ip\"," +echo " $(docker logs ace-rabbitmq | grep UserName)" +echo " $(docker logs ace-rabbitmq | grep Password)" +echo " $(docker logs ace-nginx | grep Thumbprint)" +echo " $(docker logs ace-sql | grep ApiKey)" +echo " $(docker logs ace-sql | grep StartAceSweep)" +echo " $(docker logs ace-sql | grep DownloadAceFile)" +echo " }," +echo "" +echo " \"ConnectionStrings\": {" +echo " $(docker logs ace-sql | grep DefaultConnection | sed s/sql.ace.local/$ip/)" +echo " }" +echo "}" +echo "" +echo "==========================================================" +echo "" +echo "" + +# Provide configuration details for PowerShell Module +echo "===============================================================" +echo "| Thank you for provisioning ACE with Docker!! |" +echo "| Please use the following information to interact with ACE |" +echo "===============================================================" +echo "" +echo " \$settings = @{" +echo " Uri = 'https://$ip'" +IFS='"' read -r -a array <<< "$(docker logs ace-sql | grep Api)" +echo " ApiKey = '${array[3]}'" +IFS='"' read -r -a array <<< "$(docker logs ace-nginx | grep Thumbprint)" +echo " Thumbprint = '${array[3]}'" +echo " }" +echo "" +echo "==============================================================" +echo "" +echo "" \ No newline at end of file