OWTF-IG comprehensive update on Enumeration and Discovery (owtf#1283)

* Added dirsearch, capabilities for nmap, zip, unzip for SecLists

* Added capabilities for container in compose file

* Added dirsearch and SecLists entries

* Force rebuild of new container (for new versions)

* Fixing dirsearch virtual env issues. Typo in resources.cfg

* Move dirsearch installation from Dockerfile to install script

* Fixing tools custom installation

* Allow user to build owtf container with and without permissive network capabilities

* Adjust tools' configurations and running commands

* Fixing requests package version conflict between owtf and dirsearch

Makefile

@@ -93,9 +93,14 @@ docker-run:
 	@echo "--> Running the Docker development image"
 	docker run -it -p 8009:8009 -p 8008:8008 -p 8010:8010 -v $(current_dir):/owtf owtf/owtf /bin/bash
 
-compose:
-	@echo "--> Running the Docker Compose setup"
-	docker-compose -f docker/docker-compose.dev.yml up
+### Options to allow docker to have permissive network capabilities, allowing it to run tools such as nmap
+compose-safe:
+	@echo "--> Running the Docker Compose setup with network capabilties for container"
+	docker-compose -f docker/docker-compose.dev.yml up --build
+
+compose-unsafe:
+	@echo "--> Running the Docker Compose setup without network capabilties for container"
+	docker-compose -f docker/docker-compose.dev.unsafe.yml up --build
 
 ### DEBIAN PACKAGING
 

docker/Dockerfile

@@ -19,16 +19,20 @@ ENV PYTHONDONTWRITEBYTECODE 1
 ENV PYCURL_SSL_LIBRARY openssl
 
 # Install dependencies and clean up in the same layer to reduce image size
-RUN apt-get -y update && \
-    apt-get -y install --no-install-recommends git xvfb xserver-xephyr libxml2-dev libxslt-dev libssl-dev zlib1g-dev gcc python-all-dev \
+RUN apt -y update && \
+    apt -y install --no-install-recommends git xvfb xserver-xephyr libxml2-dev libxslt-dev libssl-dev zlib1g-dev gcc python-all-dev libcap2-bin zip unzip \
     postgresql-server-dev-all postgresql-client postgresql-client-common \
     postgresql libcurl4-openssl-dev proxychains tor ca-certificates libpq-dev  \
     libxslt1-dev libldap2-dev libsasl2-dev libffi-dev net-tools lsof locales-all \
     make build-essential libssl-dev zlib1g-dev libbz2-dev libreadline-dev libsqlite3-dev wget curl llvm libncurses5-dev xz-utils tk-dev libxml2-dev libxmlsec1-dev libffi-dev liblzma-dev \
     lbd theharvester tlssled nikto dnsrecon nmap whatweb skipfish dirbuster metasploit-framework wpscan wapiti hydra metagoofil o-saft amass && \
-    apt-get clean && \
+    apt -y install httpx-toolkit nuclei subfinder && \
+    apt clean && \
     rm -rf /var/lib/apt/lists/*
 
+# Nmap needs capabilities to run on docker
+RUN setcap cap_net_admin,cap_net_raw=eip $(which nmap)
+
 # Stage 2: User Setup
 FROM base AS owtf_setup
 
@@ -66,6 +70,7 @@ ENV PATH="$VIRTUAL_ENV/bin:$PATH"
 RUN pip install --upgrade pip
 RUN pip install setuptools==57.5.0 cffi wheel
 
+
 #Installing python dependencies
 RUN pip install -r ${HOME}/requirements/base.txt
 

docker/docker-compose.dev.unsafe.yml

@@ -0,0 +1,32 @@
+version: "3.3"
+services:
+  owtf:
+    cap_add:
+      - CAP_NET_RAW
+      - CAP_NET_ADMIN
+    restart: always
+    build:
+      context: ..
+      dockerfile: docker/Dockerfile 
+    command: ["/usr/bin/wait-for-it.sh", "db:5432", "--", "owtf"]
+    environment:
+      - DOCKER=1
+      - POSTGRES_USER=owtf_db_user
+      - POSTGRES_PASSWORD=jgZKW33Q+HZk8rqylZxaPg1lbuNGHJhgzsq3gBKV32g=
+      - POSTGRES_DB=owtf_db
+    ports:
+      - 8008:8008
+      - 8010:8010
+      - 8009:8009
+    depends_on:
+      - db
+    volumes:
+      - ..:/owtf
+  db:
+    image: postgres:alpine
+    ports:
+      - 5432:5432
+    environment:
+      - POSTGRES_USER=owtf_db_user
+      - POSTGRES_PASSWORD=jgZKW33Q+HZk8rqylZxaPg1lbuNGHJhgzsq3gBKV32g=
+      - POSTGRES_DB=owtf_db

docker/docker-compose.dev.yml

@@ -4,7 +4,7 @@ services:
     restart: always
     build:
       context: ..
-      dockerfile: docker/Dockerfile
+      dockerfile: docker/Dockerfile 
     command: ["/usr/bin/wait-for-it.sh", "db:5432", "--", "owtf"]
     environment:
       - DOCKER=1

owtf/data/conf/general.yaml

@@ -78,6 +78,18 @@ TOOLS:
     value: ~/.owtf/tools/restricted/testssl/
   - config: TOOL_GCPBUCKETBRUTE
     value: ~/.owtf/tools/restricted/gcpbucketbrute/
+  - config: TOOL_DIRSEARCH
+    value: ~/.owtf/tools/restricted/dirsearch/dirsearch-master/dirsearch.py
+  - config: TOOL_AMASS
+    value: /usr/bin/amass
+  - config: TOOL_NUCLEI 
+    value: /usr/bin/nuclei
+  - config: TOOL_HTTPX
+    value: /usr/bin/httpx-toolkit
+  - config: TOOL_GAU
+    value: ~/.owtf/tools/restricted/gau/gau
+
+
 
 DICTIONARIES:
 # ************************* Password brute-force dictionaries / defaults ******************************
@@ -113,6 +125,9 @@ DICTIONARIES:
     value: /usr/share/doc/python-impacket-doc/examples/samrdump.py
 # ************************** Directory Brute-forcing dictionaries **************************
 
+# SecList
+  - config: DICT_SECLISTS
+    value: ~/.owtf/dictionaries/restricted/seclists/SecLists-master
 # Generic all-in-one dictionary that combines svndigger with raft files of mixedcase:
   - config: DICT_GENERIC_DIRBUSTER
     value: "@@@FRAMEWORK_DIR@@@/dictionaries/restricted/combined/filtered_combined_mixedcase.txt"