Isolates the OWTF webapp from the complete project (owtf#1277)

* resolves issue owtf#1241 * fix: implement http.cookies * fix:bug with decode * fix: worker logs text * fix: worker logs handling * Revert "fix: worker logs handling" This reverts commit 502608e. * Revert "fix: worker logs text" This reverts commit 8463f3d. * patch:spacings * fix:deprecated dependencies * feat:frontend dockerfile done * chore:frontend bugs fixed * chore:changed docker compose * chore:fixing cors and output file issue * chore: optimizing docker build for owtf-backend * chore:implemented script to setup webapp * fix:fixed formatting issues * feat:frontend dockerfile done * chore:frontend bugs fixed * chore:changed docker compose * chore:fixing cors and output file issue * chore: optimizing docker build for owtf-backend * chore:implemented script to setup webapp * fix:fixed formatting issues * fix:dependency issue * patch:refactored the code * OWTF Kubernetes Deployment Manifests (owtf#1285) * Add kubernetes manifests * Update README.md * Update deploy-script.sh * OWTF-IG comprehensive update on Enumeration and Discovery (owtf#1283) * Added dirsearch, capabilities for nmap, zip, unzip for SecLists * Added capabilities for container in compose file * Added dirsearch and SecLists entries * Force rebuild of new container (for new versions) * Fixing dirsearch virtual env issues. Typo in resources.cfg * Move dirsearch installation from Dockerfile to install script * Fixing tools custom installation * Allow user to build owtf container with and without permissive network capabilities * Adjust tools' configurations and running commands * Fixing requests package version conflict between owtf and dirsearch * feat:frontend dockerfile done * chore:frontend bugs fixed * chore:changed docker compose * chore:fixing cors and output file issue * chore: optimizing docker build for owtf-backend * chore:implemented script to setup webapp * fix:fixed formatting issues * resolves issue owtf#1241 * fix: implement http.cookies * fix:bug with decode * fix: worker logs text * fix: worker logs handling * Revert "fix: worker logs handling" This reverts commit 502608e. * Revert "fix: worker logs text" This reverts commit 8463f3d. * fix:deprecated dependencies * feat:frontend dockerfile done * chore:frontend bugs fixed * chore:changed docker compose * chore:fixing cors and output file issue * chore: optimizing docker build for owtf-backend * chore:implemented script to setup webapp * fix:fixed formatting issues * fix:dependency issue * patch:refactored the code --------- Co-authored-by: Pratham <[email protected]> Co-authored-by: Rahul Surwade <[email protected]> Co-authored-by: Vinh Pham Ngoc Thanh <[email protected]>
Pratham1812 · Aug 20, 2024 · 8e7096c · 8e7096c
1 parent 402f8f8
commit 8e7096c
Show file tree

Hide file tree

Showing 36 changed files with 48,376 additions and 20,895 deletions.
diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
@@ -46,7 +46,6 @@ jobs:
         run: |
           sudo apt update
           sudo apt -y install libffi-dev golang postgresql-client libgnutls28-dev libyaml-dev python3-yaml python3-pip libcurl4-openssl-dev libssl-dev python3-venv
-          pip3 install pyopenssl
       - name: install
         run: |
           echo "Running install base requirements"

diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -4,4 +4,5 @@ repos:
     hooks:
       - id: black
         args: [--line-length=120, --safe]
-        python_version: python3.11
+
+        python_version: python3.11
diff --git a/Makefile b/Makefile
@@ -68,11 +68,16 @@ install-requirements: install-python-requirements install-ui-requirements instal
 
 web:
 ifdef OWTF_ENV
-	cd owtf/webapp && yarn run ${OWTF_ENV}
+	cd owtf/webapp && yarn run start
 else
-	@echo "--> No environment specified. Usage: make OWTF_ENV={dev, prod} web"
+	@echo "--> No environment specified. Usage: make web OWTF_ENV={dev}"
 endif
 
+setup-web:
+	@echo "--> Setting up the webapp on http://localhost:8019"
+	cd scripts && ./setup_webapp.sh
+
+
 post-install:
 	@echo "--> Installing dictionaries and tools"
 	python3 scripts/install_tools.py

diff --git a/docker/Dockerfile → docker/Dockerfile.backend b/docker/Dockerfile → docker/Dockerfile.backend
@@ -63,7 +63,7 @@ USER owtf
 
 #Create and activating a python virtualenv
 ENV VIRTUAL_ENV="${HOME}/owtf-venv"
-RUN python3.9 -m venv $VIRTUAL_ENV
+RUN python3.11 -m venv $VIRTUAL_ENV
 ENV PATH="$VIRTUAL_ENV/bin:$PATH"
 
 # Ensure pip and setuptools are at their latest versions

diff --git a/docker/Dockerfile.frontend b/docker/Dockerfile.frontend
@@ -0,0 +1,15 @@
+# Stage 1: Build the React app
+FROM node:18 AS builder
+WORKDIR /owtf-webapp
+COPY ./owtf/webapp/package.json ./owtf/webapp/yarn.lock ./
+RUN yarn install --silent
+COPY ./owtf/webapp .
+RUN yarn build
+# # Stage 2: Serve the built React app with Nginx
+FROM nginx:alpine
+COPY ./owtf/webapp/owtf.conf /etc/nginx/conf.d/default.conf
+# Copy the built React app to Nginx's web server directory
+COPY --from=builder /owtf-webapp/build /usr/share/nginx/html
+EXPOSE 8019
+CMD ["nginx", "-g", "daemon off;"]
+# Start Nginx when the container runs
diff --git a/docker/docker-compose.dev.yml b/docker/docker-compose.dev.yml
@@ -1,10 +1,13 @@
-version: "3.3"
+version: "3.9"
+networks:
+  backend: {}
 services:
-  owtf:
+  owtf-backend:
+    container_name: owtf-backend
     restart: always
     build:
       context: ..
-      dockerfile: docker/Dockerfile 
+      dockerfile: docker/Dockerfile.backend
     command: ["/usr/bin/wait-for-it.sh", "db:5432", "--", "owtf"]
     environment:
       - DOCKER=1
@@ -16,9 +19,24 @@ services:
       - 8010:8010
       - 8009:8009
     depends_on:
+      - owtf-frontend
       - db
     volumes:
       - ..:/owtf
+    networks:
+      - backend
+  owtf-frontend:
+    container_name: owtf-frontend
+    restart: always
+    build:
+      context: ..
+      dockerfile: docker/Dockerfile.frontend
+    ports:
+      - 8019:8019
+    volumes:
+      - ..:/owtf
+    networks:
+      - backend
   db:
     image: postgres:alpine
     ports:
@@ -27,3 +45,5 @@ services:
       - POSTGRES_USER=owtf_db_user
       - POSTGRES_PASSWORD=jgZKW33Q+HZk8rqylZxaPg1lbuNGHJhgzsq3gBKV32g=
       - POSTGRES_DB=owtf_db
+    networks: 
+      - backend
diff --git a/owtf/api/handlers/api_token.py b/owtf/api/handlers/api_token.py
@@ -51,6 +51,7 @@ def get(self):
         try:
             token = self.request.headers.get("Authorization").split()[1]
             payload = jwt.decode(token, JWT_SECRET_KEY, options=JWT_OPTIONS, algorithms=[JWT_ALGORITHM])
+
             user_id = payload.get("user_id", None)
             if not user_id:
                 raise APIError(400, "Invalid User Id")

diff --git a/owtf/api/handlers/auth.py b/owtf/api/handlers/auth.py
@@ -26,6 +26,7 @@
     SMTP_PORT,
     SERVER_ADDR,
     SERVER_PORT,
+    FRONTEND_SERVER_PORT,
 )
 from owtf.db.session import Session
 from uuid import uuid4
@@ -340,7 +341,7 @@ def post(self):
             + ", <br/><br/>"
             """ 
             Click here """
-            + "http://{}:{}".format(SERVER_ADDR, str(SERVER_PORT))
+            + "http://{}:{}".format(SERVER_ADDR, str(FRONTEND_SERVER_PORT))
             + "/email-verify/"
             + email_confirmation_dict["key_value"]
             + """ to activate your account (Link will expire in 1 hour).

diff --git a/owtf/api/handlers/base.py b/owtf/api/handlers/base.py
@@ -19,6 +19,7 @@
     USE_SENTRY,
     SERVER_ADDR,
     SESSION_COOKIE_NAME,
+    ALLOWED_ORIGINS,
 )
 from owtf.utils.strings import utf8
 
@@ -42,16 +43,30 @@ class SentryHandler(SentryMixin, RequestHandler):
 
 
 class BaseRequestHandler(RequestHandler):
+    CORS_ORIGIN = ALLOWED_ORIGINS[1]
+
     def set_default_headers(self):
+        # Security headers
         self.add_header("X-OWTF-Version", __version__)
         self.add_header("X-Frame-Options", "SAMEORIGIN")
         self.add_header("X-XSS-Protection", "1; mode=block")
         self.add_header("X-Content-Type-Options", "nosniff")
         self.add_header("Referrer-Policy", "strict-origin-when-cross-origin")
+
+        # CORS headers
+        self.set_header("Access-Control-Allow-Origin", self.CORS_ORIGIN)
+        self.set_header("Access-Control-Allow-Methods", "GET,POST,PUT,DELETE,OPTIONS")
+        self.set_header("Access-Control-Request-Credentials", "true")
+        self.set_header("Access-Control-Allow-Headers", "Authorization,Content-Type")
+        # Caching headers
         self.add_header("Cache-Control", "no-cache,no-store,max-age=0,must-revalidate")
         self.add_header("Pragma", "no-cache")
         self.add_header("Expires", "-1")
 
+    def options(self, *args, **kwargs):
+        self.set_status(204)
+        self.finish()
+
 
 class APIRequestHandler(BaseRequestHandler):
     def initialize(self):

diff --git a/owtf/api/handlers/config.py b/owtf/api/handlers/config.py
@@ -14,7 +14,7 @@
 class ConfigurationHandler(APIRequestHandler):
     """Update framework settings and tool paths."""
 
-    SUPPORTED_METHODS = ["GET", "PATCH"]
+    SUPPORTED_METHODS = ["GET", "PATCH", "OPTIONS"]
 
     def get(self):
         """Return all configuration items.

diff --git a/owtf/api/handlers/jwtauth.py b/owtf/api/handlers/jwtauth.py
@@ -5,6 +5,7 @@
 from owtf.models.user_login_token import UserLoginToken
 import jwt
 from owtf.settings import JWT_SECRET_KEY, JWT_OPTIONS, JWT_ALGORITHM
+
 from owtf.db.session import Session
 
 

diff --git a/owtf/api/handlers/misc.py b/owtf/api/handlers/misc.py
@@ -30,10 +30,6 @@ def get(self):
 class ProgressBarHandler(APIRequestHandler):
     SUPPORTED_METHODS = ["GET", "POST", "PUT", "PATCH", "DELETE"]
 
-    def set_default_headers(self):
-        self.add_header("Access-Control-Allow-Origin", "*")
-        self.add_header("Access-Control-Allow-Methods", "GET, POST, DELETE")
-
     def get(self):
         try:
             self.write(plugin_count_output(self.session))
@@ -95,3 +91,4 @@ def delete(self, error_id=None):
             self.finish()
         except exceptions.InvalidErrorReference:
             raise tornado.web.HTTPError(400)
+
diff --git a/owtf/api/handlers/transactions.py b/owtf/api/handlers/transactions.py
@@ -242,21 +242,21 @@ def get(self, target_id=None):
             logging.warn(e.parameter)
             raise tornado.web.HTTPError(400)
 
-    @tornado.web.asynchronous
+    @tornado.gen.coroutine
     def post(self, target_url):
         raise tornado.web.HTTPError(405)
 
-    @tornado.web.asynchronous
+    @tornado.gen.coroutine
     def put(self):
         raise tornado.web.HTTPError(405)
 
-    @tornado.web.asynchronous
+    @tornado.gen.coroutine
     def patch(self):
         # TODO: allow modification of urls from the ui, may be adjusting scope etc.. but i don't understand
         # it's use yet ;)
         raise tornado.web.HTTPError(405)  # @UndefinedVariable
 
-    @tornado.web.asynchronous
+    @tornado.gen.coroutine
     def delete(self, target_id=None):
         # TODO: allow deleting of urls from the ui
         raise tornado.web.HTTPError(405)  # @UndefinedVariable

diff --git a/owtf/api/handlers/work.py b/owtf/api/handlers/work.py
@@ -44,10 +44,6 @@ class WorkerHandler(APIRequestHandler):
 
     SUPPORTED_METHODS = ["GET", "POST", "DELETE", "OPTIONS"]
 
-    def set_default_headers(self):
-        self.add_header("Access-Control-Allow-Origin", "*")
-        self.add_header("Access-Control-Allow-Methods", "GET, POST, DELETE")
-
     def get(self, worker_id=None, action=None):
         """Get all workers by ID.
 

diff --git a/owtf/api/main.py b/owtf/api/main.py
@@ -22,7 +22,6 @@
 
 
 class APIServer(OWTFProcess):
-
     def pseudo_run(self):
         api_v1 = Application(handlers=API_v1_HANDLERS, debug=DEBUG, autoreload=False, gzip=True)
 
@@ -49,7 +48,7 @@ def pseudo_run(self):
             port = int(SERVER_PORT)
             address = SERVER_ADDR
             self.server.bind(port, address=address)
-            logging.warning("Starting API and UI server at http://{}:{}".format(SERVER_ADDR, str(SERVER_PORT)))
+            logging.warning("Starting API server at http://{}:{}".format(SERVER_ADDR, str(SERVER_PORT)))
             self.logger.disable_console_logging()
             tornado.options.parse_command_line(
                 args=["dummy_arg", "--log_file_prefix={}".format(SERVER_LOG), "--logging=info"]

diff --git a/owtf/api/routes.py b/owtf/api/routes.py
@@ -14,6 +14,7 @@
 from owtf.api.handlers.report import ReportExportHandler
 from owtf.api.handlers.session import OWTFSessionHandler
 from owtf.api.handlers.targets import TargetConfigHandler, TargetConfigSearchHandler, TargetSeverityChartHandler
+from owtf.files.handlers import StaticFileHandler
 from owtf.utils.file import get_dir_worker_logs
 from owtf.api.handlers.transactions import (
     TransactionDataHandler,
@@ -105,11 +106,9 @@
 ]
 
 UI_HANDLERS = [
-    tornado.web.url(r"/static/(.*)", tornado.web.StaticFileHandler, {"path": STATIC_ROOT}),
+    tornado.web.url(r"/static/(.*)", StaticFileHandler, {"path": STATIC_ROOT}),
     tornado.web.url(r"/debug/health/?$", HealthCheckHandler),
-    tornado.web.url(
-        r"/logs/(.*)", tornado.web.StaticFileHandler, {"path": get_dir_worker_logs()}, name="logs_files_url"
-    ),
+    tornado.web.url(r"/logs/(.*)", StaticFileHandler, {"path": get_dir_worker_logs()}, name="logs_files_url"),
     tornado.web.url(r"/output_files/(.*)", FileRedirectHandler, name="file_redirect_url"),
-    tornado.web.url(r"^/(?!api|debug|static|output_files|logs)(.*)$", IndexHandler),
+    tornado.web.url(r"^/(?!api|debug|static|output_files)(.*)$", IndexHandler),
 ]
diff --git a/owtf/files/handlers.py b/owtf/files/handlers.py
@@ -14,12 +14,16 @@
 import tornado
 import tornado.template
 import tornado.web
+from owtf.api.handlers.base import BaseRequestHandler
 
-__all__ = ["StaticFileHandler"]
+__all__ = ["StaticFileHandler", "OutputFileHandler"]
 
 
-class StaticFileHandler(tornado.web.StaticFileHandler):
+class StaticFileHandler(tornado.web.StaticFileHandler, BaseRequestHandler):
+    pass
 
+
+class OutputFileHandler(tornado.web.StaticFileHandler):
     def set_default_headers(self):
         self.add_header("Access-Control-Allow-Origin", "*")
         self.add_header("Access-Control-Allow-Methods", "GET")

diff --git a/owtf/files/routes.py b/owtf/files/routes.py
@@ -5,10 +5,10 @@
 """
 import tornado.web
 
-from owtf.files.handlers import StaticFileHandler
+from owtf.files.handlers import OutputFileHandler
 from owtf.utils.file import get_dir_worker_logs, get_output_dir_target
 
 HANDLERS = [
-    tornado.web.url(r"/logs/(.*)", StaticFileHandler, {"path": get_dir_worker_logs()}, name="logs_files_url"),
-    tornado.web.url(r"/(.*)", StaticFileHandler, {"path": get_output_dir_target()}, name="output_files_url"),
+    tornado.web.url(r"/logs/(.*)", OutputFileHandler, {"path": get_dir_worker_logs()}, name="logs_files_url"),
+    tornado.web.url(r"/(.*)", OutputFileHandler, {"path": get_output_dir_target()}, name="output_files_url"),
 ]
diff --git a/owtf/managers/config.py b/owtf/managers/config.py
@@ -32,7 +32,7 @@ def load_config_file(file_path, fallback_file_path):
         # check if the config file exists
         abort_framework("Config file not found at: {}".format(file_path))
     try:
-        config_map = yaml.load(FileOperations.open(file_path, "r"))
+        config_map = yaml.safe_load(FileOperations.open(file_path, "r"))
         return config_map
     except yaml.YAMLError:
         abort_framework("Error parsing config file at: {}".format(file_path))

diff --git a/owtf/settings.py b/owtf/settings.py
@@ -35,13 +35,14 @@
     DATABASE_PASS = "jgZKW33Q+HZk8rqylZxaPg1lbuNGHJhgzsq3gBKV32g="
     DATABASE_NAME = "owtf_db"
     DATABASE_USER = "owtf_db_user"
-    DATABASE_IP = os.environ.get("POSTGRES_HOST","127.0.0.1")
+    DATABASE_IP = os.environ.get("POSTGRES_HOST", "127.0.0.1")
     DATABASE_PORT = 5432
 
 # API and UI Server
 SERVER_ADDR = "0.0.0.0"
 SERVER_PORT = 8009
 FILE_SERVER_PORT = 8010
+FRONTEND_SERVER_PORT = 8019
 
 # Default API version
 DEFAULT_API_VERSION = "v1"
@@ -53,8 +54,8 @@
 SESSION_COOKIE_NAME = "owtf-session"
 
 # CORS settings. Fine grained, do not override if possible.
-SIMPLE_HEADERS = ["accept", "accept-language", "content-language"]
-ALLOWED_ORIGINS = ["http:/localhost:8009", "http://localhost:8010"]
+SIMPLE_HEADERS = ["accept", "accept-language", "content-language", "authorization"]
+ALLOWED_ORIGINS = ["http://localhost:8010", "http://localhost:8019"]
 ALLOWED_METHODS = ["GET", "POST", "DELETE"]
 SEND_CREDENTIALS = False