diff --git a/.devcontainer/triage-portal/Dockerfile b/.devcontainer/triage-portal/Dockerfile new file mode 100644 index 00000000..809be5eb --- /dev/null +++ b/.devcontainer/triage-portal/Dockerfile @@ -0,0 +1,30 @@ +# [Choice] Python version (use -bullseye variants on local arm64/Apple Silicon): 3, 3.10, 3.9, 3.8, 3.7, 3.6, 3-bullseye, 3.10-bullseye, 3.9-bullseye, 3.8-bullseye, 3.7-bullseye, 3.6-bullseye, 3-buster, 3.10-buster, 3.9-buster, 3.8-buster, 3.7-buster, 3.6-buster +ARG VARIANT=3-bullseye +FROM mcr.microsoft.com/vscode/devcontainers/python:0-${VARIANT} + +ENV PYTHONUNBUFFERED 1 + +# [Choice] Node.js version: none, lts/*, 16, 14, 12, 10 +ARG NODE_VERSION="none" +RUN if [ "${NODE_VERSION}" != "none" ]; then su vscode -c "umask 0002 && . /usr/local/share/nvm/nvm.sh && nvm install ${NODE_VERSION} 2>&1"; fi + +# Install OSS Gadget +# License: MIT +ARG OSSGADGET_VERSION="0.1.307" +RUN cd /opt && \ + wget -q https://github.com/microsoft/OSSGadget/releases/download/v${OSSGADGET_VERSION}/OSSGadget_linux_${OSSGADGET_VERSION}.tar.gz -O OSSGadget.tar.gz && \ + tar zxvf OSSGadget.tar.gz && \ + rm OSSGadget.tar.gz && \ + mv OSSGadget_linux_${OSSGADGET_VERSION} OSSGadget + +# [Optional] If your requirements rarely change, uncomment this section to add them to the image. +# COPY requirements.txt /tmp/pip-tmp/ +# RUN pip3 --disable-pip-version-check --no-cache-dir install -r /tmp/pip-tmp/requirements.txt \ +# && rm -rf /tmp/pip-tmp + +# [Optional] Uncomment this section to install additional OS packages. +# RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \ +# && apt-get -y install --no-install-recommends + + + diff --git a/.devcontainer/triage-portal/devcontainer.json b/.devcontainer/triage-portal/devcontainer.json new file mode 100644 index 00000000..d8879322 --- /dev/null +++ b/.devcontainer/triage-portal/devcontainer.json @@ -0,0 +1,68 @@ +// For format details, see https://aka.ms/devcontainer.json. For config options, see the README at: +// https://github.com/microsoft/vscode-dev-containers/tree/v0.203.0/containers/python-3-postgres +// Update the VARIANT arg in docker-compose.yml to pick a Python version +{ + "name": "Python 3 & PostgreSQL", + "dockerComposeFile": "docker-compose.yml", + "service": "app", + "workspaceFolder": "/workspaces", + // Set *default* container specific settings.json values on container create. + "settings": { + "sqltools.connections": [ + { + "name": "Container database", + "driver": "PostgreSQL", + "previewLimit": 50, + "server": "localhost", + "port": 5432, + "database": "triage", + "username": "triage_user", + "password": "triage_password" + } + ], + "python.pythonPath": "/usr/local/bin/python", + "python.languageServer": "Pylance", + "python.linting.enabled": true, + "python.linting.pylintEnabled": true, + "python.formatting.autopep8Path": "/usr/local/py-utils/bin/autopep8", + "python.formatting.blackPath": "/usr/local/py-utils/bin/black", + "python.formatting.yapfPath": "/usr/local/py-utils/bin/yapf", + "python.linting.banditPath": "/usr/local/py-utils/bin/bandit", + "python.linting.flake8Path": "/usr/local/py-utils/bin/flake8", + "python.linting.mypyPath": "/usr/local/py-utils/bin/mypy", + "python.linting.pycodestylePath": "/usr/local/py-utils/bin/pycodestyle", + "python.linting.pydocstylePath": "/usr/local/py-utils/bin/pydocstyle", + "python.linting.pylintPath": "/usr/local/py-utils/bin/pylint", + "python.testing.pytestPath": "/usr/local/py-utils/bin/pytest", + "python.defaultInterpreterPath": "${workspaceFolder}/alpha-omega/omega/triage-portal/.venv/bin/python" + }, + // Add the IDs of extensions you want installed when the container is created. + "extensions": [ + "ms-python.python", + "ms-python.vscode-pylance", + "mtxr.sqltools", + "mtxr.sqltools-driver-pg", + "GitHub.copilot" + ], + // Use 'forwardPorts' to make a list of ports inside the container available locally. + "forwardPorts": [ + 8000 + ], + "portsAttributes": { + "8000": { + "label": "Triage Portal (Django)", + "protocol": "http", + "onAutoForward": "notify", + "requireLocalPort": false, + "elevateIfNeeded": false + } + }, + // Use 'postCreateCommand' to run commands after the container is created. + "postCreateCommand": "bash alpha-omega/.devcontainer/triage-portal/postcreate-initialize.sh", + "remoteEnv": { + "DJANGO_SETTINGS_MODULE": "core.settings", + "PYTHONPATH": "/workspaces/alpha-omega/omega/triage-portal/src" + }, + // Comment out connect as root instead. More info: https://aka.ms/vscode-remote/containers/non-root. + "remoteUser": "vscode" +} \ No newline at end of file diff --git a/.devcontainer/triage-portal/docker-compose.yml b/.devcontainer/triage-portal/docker-compose.yml new file mode 100644 index 00000000..2fb9b54c --- /dev/null +++ b/.devcontainer/triage-portal/docker-compose.yml @@ -0,0 +1,54 @@ +version: '3.8' + +services: + app: + build: + context: .. + dockerfile: triage-portal/Dockerfile + args: + # Update 'VARIANT' to pick a version of Python: 3, 3.10, 3.9, 3.8, 3.7, 3.6 + # Append -bullseye or -buster to pin to an OS version. + # Use -bullseye variants on local arm64/Apple Silicon. + VARIANT: 3-bullseye + # Optional Node.js version to install + NODE_VERSION: "lts/*" + + volumes: + - ..:/workspace:cached + init: true + + # Overrides default command so things don't shut down after the process ends. + command: sleep infinity + + # Runs app on the same network as the database container, allows "forwardPorts" in devcontainer.json function. + network_mode: service:db + + # Uncomment the next line to use a non-root user for all processes. + # user: vscode + + # Use "forwardPorts" in **devcontainer.json** to forward an app port locally. + # (Adding the "ports" property to this file will not forward from a Codespace.) + + db: + image: postgres:latest + restart: unless-stopped + volumes: + - postgres-data:/var/lib/postgresql/data + environment: + POSTGRES_USER: triage_user + POSTGRES_DB: triage + POSTGRES_PASSWORD: triage_password + + redis: + image: redis:latest + restart: unless-stopped + network_mode: service:db + volumes: + - redis-data:/data + + # Add "forwardPorts": ["5432"] to **devcontainer.json** to forward PostgreSQL locally. + # (Adding the "ports" property to this file will not forward from a Codespace.) + +volumes: + postgres-data: + redis-data: diff --git a/.devcontainer/triage-portal/postcreate-initialize.sh b/.devcontainer/triage-portal/postcreate-initialize.sh new file mode 100644 index 00000000..5b444ecd --- /dev/null +++ b/.devcontainer/triage-portal/postcreate-initialize.sh @@ -0,0 +1,49 @@ +#!/bin/bash + +ROOT="/workspaces/alpha-omega/omega/triage-portal" +cd "$ROOT" + +# Create and activate the virtual environment +echo "Creating virtual environment." +python -mvenv .venv +source .venv/bin/activate + +# Install Python dependencies +echo "Installing Python (back-end) dependencies." +cd $ROOT/src +python -m pip install --upgrade pip +pip install wheel +pip install -r ./requirements.txt + +# Install JavaScript dependencies +echo "Installing JavaScript (front-end) dependencies." +cd $ROOT/src +npm i -g yarn +yarn + +# Update default environment +echo "Creating development environment." +cd $ROOT/src +cp .env-template .env +SECRET_KEY=$(python -c "import secrets; print(secrets.token_hex(64))") +sed -i "s/%RANDOM_STRING%/$SECRET_KEY/" .env +unset SECRET_KEY + +# Create working directories +echo "Creating working directories." +mkdir $ROOT/logs + +# Set up database +echo "Setting up datatbase." +cd $ROOT/src +python manage.py migrate +python manage.py makemigrations +python manage.py migrate triage + +# Create superuser +DJANGO_SUPERUSER_USERNAME="admin" \ +DJANGO_SUPERUSER_PASSWORD="admin" \ +DJANGO_SUPERUSER_EMAIL="nobody@localhost" \ +python manage.py createsuperuser --noinput + +echo "Initialization completed." diff --git a/.gitignore b/.gitignore index b3ddf9e7..ca962725 100644 --- a/.gitignore +++ b/.gitignore @@ -1,8 +1,12 @@ /.vs *.env *.pyc -**/.mypy_cache +/omega/analyzer/worker/results +/venv/* +**/.venv **/venv/ *.db +*.pem oss-gadget.log -*.pem \ No newline at end of file +**/.idea/workspace.xml +**/.idea/tasks.xml \ No newline at end of file diff --git a/.vscode/launch.json b/.vscode/launch.json new file mode 100644 index 00000000..de9ab8b3 --- /dev/null +++ b/.vscode/launch.json @@ -0,0 +1,20 @@ +{ + // Use IntelliSense to learn about possible attributes. + // Hover to view descriptions of existing attributes. + // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 + "version": "0.2.0", + "configurations": [ + { + "name": "Triage Portal - Start Server (Debug)", + "type": "python", + "request": "launch", + "program": "src/manage.py", + "cwd": "${workspaceFolder}/omega/triage-portal", + "args": [ + "runserver", + "0.0.0.0:8001" + ], + "django": true + } + ] +} \ No newline at end of file diff --git a/.vscode/project.code-workspace b/.vscode/project.code-workspace new file mode 100644 index 00000000..09906959 --- /dev/null +++ b/.vscode/project.code-workspace @@ -0,0 +1,11 @@ +{ + "folders": [ + { + "name": "triage-portal", + "path": "/workspaces/alpha-omega/omega/triage-portal" + }, + { + "path": "../.devcontainer" + } + ] +} \ No newline at end of file diff --git a/omega/triage-portal/.gitignore b/omega/triage-portal/.gitignore new file mode 100644 index 00000000..476f27cd --- /dev/null +++ b/omega/triage-portal/.gitignore @@ -0,0 +1,144 @@ +# Derived from https://github.com/github/gitignore/blob/218a941be92679ce67d0484547e3e142b2f5f6f0/Python.gitignore + +# Static Files (brought in via Yarn) +src/triage/static/triage/resources +logs/ + +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ +cover/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +.pybuilder/ +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +# For a library or package, you might want to ignore these files since the code is +# intended to run in multiple environments; otherwise, check them in: +# .python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# pytype static type analyzer +.pytype/ + +# Cython debug symbols +cython_debug/ \ No newline at end of file diff --git a/omega/triage-portal/.vscode/launch.json b/omega/triage-portal/.vscode/launch.json new file mode 100644 index 00000000..bfb02916 --- /dev/null +++ b/omega/triage-portal/.vscode/launch.json @@ -0,0 +1,19 @@ +{ + // Use IntelliSense to learn about possible attributes. + // Hover to view descriptions of existing attributes. + // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 + "version": "0.2.0", + "configurations": [ + { + "name": "Python: Django", + "type": "python", + "request": "launch", + "program": "src/manage.py", + "args": [ + "runserver", + "0.0.0.0:8001" + ], + "django": true + } + ] +} \ No newline at end of file diff --git a/omega/triage-portal/.vscode/settings.json b/omega/triage-portal/.vscode/settings.json new file mode 100644 index 00000000..a08338d8 --- /dev/null +++ b/omega/triage-portal/.vscode/settings.json @@ -0,0 +1,42 @@ +{ + "files.exclude": { + "**/__pycache__": true + }, + "python.formatting.provider": "black", + "python.formatting.blackArgs": [ + "--line-length", + "100", + "--force-exclude", + "\\.html" + ], + "editor.rulers": [ + 100 + ], + "editor.inlineSuggest.enabled": true, + "html.format.enable": false, + "python.terminal.activateEnvInCurrentTerminal": true, + "terminal.integrated.env.linux": { + "DJANGO_SETTINGS_MODULE": "core.settings", + "PYTHONPATH": "${workspaceFolder}/omega/triage-portal/portal/src" + }, + "python.defaultInterpreterPath": "${workspaceFolder}/omega/triage-portal/.venv/bin/python", + "python.linting.enabled": false, + "python.linting.pylintPath": "pylint", + "python.linting.pylintEnabled": false, + "python.linting.pylintArgs": [ + "--init-hook=import sys;sys.path.append('${workspaceFolder}/omage/triage-portal/src')", + "--load-plugins", + "pylint_django", + "--django-settings-module", + "core.settings" + ], + "python.linting.banditEnabled": false, + "python.linting.banditPath": "/usr/local/py-utils/bin/bandit", + "[python]": { + "editor.codeActionsOnSave": { + "source.organizeImports": true + } + }, + "editor.bracketPairColorization.enabled": true, + "html.format.templating": true +} \ No newline at end of file diff --git a/omega/triage-portal/.vscode/tasks.json b/omega/triage-portal/.vscode/tasks.json new file mode 100644 index 00000000..5b72615a --- /dev/null +++ b/omega/triage-portal/.vscode/tasks.json @@ -0,0 +1,93 @@ +{ + // See https://go.microsoft.com/fwlink/?LinkId=733558 + // for the documentation about the tasks.json format + "version": "2.0.0", + "tasks": [ + { + "label": "PyLint: Scan entire project", + "type": "shell", + "options": { + "cwd": "${workspaceFolder}/omega/triage-portal/.venv/bin/" + }, + "command": "${workspaceFolder}/omega/triage-portal/.venv/bin/pylint", + "args": [ + "--msg-template", + "\"{path}:{line}:{column}:{category}:{symbol} - {msg}\"", + { + "value": "--init-hook=\"import sys;sys.path.append('${workspaceFolder}/omega/triage-portal/src')\"", + "quoting": "strong" + }, + "--load-plugins", + "pylint_django", + "--django-settings-module", + "core.settings", + "--exit-zero", + "${workspaceFolder}/omega/triage-portal/src/triage" + ], + "presentation": { + "reveal": "never", + "panel": "shared" + }, + "problemMatcher": { + "owner": "python", + "fileLocation": [ + "absolute" + ], + "pattern": { + "regexp": "^(.+):(\\d+):(\\d+):(\\w+):(.*)$", + "file": 1, + "line": 2, + "column": 3, + "severity": 4, + "message": 5 + } + } + }, + { + "label": "Django: Make Migrations (Triage)", + "type": "shell", + "command": "${config:python.defaultInterpreterPath}", + "args": [ + "manage.py", + "makemigrations", + "triage" + ], + "problemMatcher": [], + "options": { + "cwd": "${workspaceFolder}/omega/triage-portal/src" + }, + }, + { + "label": "Django: Migrate Database (Triage)", + "type": "shell", + "command": "${config:python.defaultInterpreterPath}", + "args": [ + "manage.py", + "migrate" + ], + "problemMatcher": [], + "options": { + "cwd": "${workspaceFolder}/omega/triage-portal/src" + }, + }, + { + "label": "Redis: Clear Local Cache", + "type": "shell", + "presentation": { + "reveal": "never", + "panel": "shared" + }, + "command": "${config:python.defaultInterpreterPath}", + "args": [ + "manage.py", + "shell", + "-c", + "from django.core.cache import cache; cache.clear()" + ], + "problemMatcher": [], + "options": { + "cwd": "${workspaceFolder}/omega/triage-portal/src" + }, + }, + ] +} \ No newline at end of file diff --git a/omega/triage-portal/LICENSE b/omega/triage-portal/LICENSE new file mode 100644 index 00000000..912e1c2e --- /dev/null +++ b/omega/triage-portal/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2021 Open Source Security Foundation + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/omega/triage-portal/README.md b/omega/triage-portal/README.md new file mode 100644 index 00000000..c3efcb85 --- /dev/null +++ b/omega/triage-portal/README.md @@ -0,0 +1,30 @@ +# Omega Triage Portal + +The Omega Triage Portal is a web-application that can help manage automated vulnerability reports. +It was designed for scale, (hundreds of thousands of projects, many millions of findings), +but may also be useful at lower scale. + +**The Portal is in early development, and is not ready for general use.** + +## Getting Started + +This extension can be used from GitHub Codespaces: + +[![Open in GitHub Codespaces](https://github.com/codespaces/badge.svg)](https://github.com/codespaces/new?machine=basicLinux32gb&repo=426394209&ref=scovetta%2Fadd-triage-portal&location=WestUs2&devcontainer_path=.devcontainer%2Ftriage-portal%2Fdevcontainer.json) + +Once loaded, open the `.vscode/project.code-workspace` file and then click the `Open Workspace` +button. A new widow will open. This is needed because VS Code launch settings are nested +within the omega/triage-portal folder. + +You can then run the Django launch task to start the application. Navigate to + and enter the default credentials (admin/admin), then +navigate back to . + +## Contributing + +TBD + +## Security + +See [SECURITY.md](https://github.com/ossf/alpha-omega/blob/main/SECURITY.md). + diff --git a/omega/triage-portal/src/.env-template b/omega/triage-portal/src/.env-template new file mode 100644 index 00000000..8ac1d4e2 --- /dev/null +++ b/omega/triage-portal/src/.env-template @@ -0,0 +1,27 @@ +BASE_URL='https://localhost' + +# Re-generate using `python -c "import secrets; print(secrets.token_hex(64))"` +SECRET_KEY = "%RANDOM_STRING%" + +DEBUG=True + +# Database +DATABASE_ENGINE='django.db.backends.postgresql_psycopg2' +DATABASE_NAME='triage' +DATABASE_USER='triage_user' +DATABASE_PASSWORD='triage_password' +DATABASE_HOST='db' +DATABASE_PORT=5432 + +# Cache +ENABLE_CACHE=True +CACHE_USE_REDIS=True +CACHE_REDIS_CONNECTION="redis://127.0.0.1:6379/1" +CACHE_REDIS_PASSWORD='' + +#APPINSIGHTS_IKEY = '' + +TOOLSHED_BLOB_STORAGE_URL = "" +TOOLSHED_BLOB_STORAGE_CONTAINER = "" + +OSSGADGET_PATH="/opt/OSSGadget" \ No newline at end of file diff --git a/omega/triage-portal/src/.yarnrc b/omega/triage-portal/src/.yarnrc new file mode 100644 index 00000000..6a06e36a --- /dev/null +++ b/omega/triage-portal/src/.yarnrc @@ -0,0 +1 @@ +--modules-folder triage/static/triage/resources diff --git a/omega/triage-portal/src/core/__init__.py b/omega/triage-portal/src/core/__init__.py new file mode 100644 index 00000000..91d1f8c3 --- /dev/null +++ b/omega/triage-portal/src/core/__init__.py @@ -0,0 +1,24 @@ +"""Basic helper functions""" + +import os +from django.core.exceptions import ImproperlyConfigured +import django + +def get_env_variable(var_name, optional=False): + """ + Retrieve an environment variable. Any failures will cause an exception + to be thrown. + """ + try: + return os.environ[var_name] + except KeyError as ex: + if optional: + return False + raise ImproperlyConfigured( + f"Error: You must set the {var_name} environment variable." + ) from ex + + +def to_bool(option: str) -> bool: + """Convert a string to a boolean.""" + return option and option.lower().strip() in ["true", "1"] diff --git a/omega/triage-portal/src/core/asgi.py b/omega/triage-portal/src/core/asgi.py new file mode 100644 index 00000000..9fbf1e38 --- /dev/null +++ b/omega/triage-portal/src/core/asgi.py @@ -0,0 +1,16 @@ +""" +ASGI config for core project. + +It exposes the ASGI callable as a module-level variable named ``application``. + +For more information on this file, see +https://docs.djangoproject.com/en/3.2/howto/deployment/asgi/ +""" + +import os + +from django.core.asgi import get_asgi_application + +os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'core.settings') + +application = get_asgi_application() diff --git a/omega/triage-portal/src/core/settings.py b/omega/triage-portal/src/core/settings.py new file mode 100644 index 00000000..b895a5de --- /dev/null +++ b/omega/triage-portal/src/core/settings.py @@ -0,0 +1,225 @@ +import os +from pathlib import Path + +from django.core.exceptions import ImproperlyConfigured + +from core import get_env_variable, to_bool + +# Build paths inside the project like this: BASE_DIR / 'subdir'. +BASE_DIR = Path(__file__).resolve().parent.parent + +# Read environment variables from a file +try: + import dotenv + + dotenv.read_dotenv(os.path.join(BASE_DIR, ".env")) +except Exception: + raise ImproperlyConfigured("A .env file was not found. Environment variables are not set.") + +SECRET_KEY = get_env_variable("SECRET_KEY") +DEBUG = to_bool(get_env_variable("DEBUG")) + +INTERNAL_IPS = [ +] + +ALLOWED_HOSTS = ['*'] + +# Application definition + +INSTALLED_APPS = [ + "django.contrib.admin", + "django.contrib.auth", + "django.contrib.contenttypes", + "django.contrib.sessions", + "django.contrib.messages", + "django.contrib.staticfiles", + "taggit", + "triage", + "debug_toolbar", +] + +MIDDLEWARE = [ + "debug_toolbar.middleware.DebugToolbarMiddleware", + "django.middleware.security.SecurityMiddleware", + "django.contrib.sessions.middleware.SessionMiddleware", + "django.middleware.common.CommonMiddleware", + "django.middleware.csrf.CsrfViewMiddleware", + "django.contrib.auth.middleware.AuthenticationMiddleware", + "django.contrib.messages.middleware.MessageMiddleware", + "django.middleware.clickjacking.XFrameOptionsMiddleware", +] + +ROOT_URLCONF = "core.urls" + +TEMPLATES = [ + { + "BACKEND": "django.template.backends.django.DjangoTemplates", + "DIRS": [], + "APP_DIRS": True, + "OPTIONS": { + "context_processors": [ + "django.template.context_processors.debug", + "django.template.context_processors.request", + "django.contrib.auth.context_processors.auth", + "django.contrib.messages.context_processors.messages", + ], + }, + }, +] + +WSGI_APPLICATION = "core.wsgi.application" + + +# Database +# https://docs.djangoproject.com/en/3.2/ref/settings/#databases + +DATABASES = { + "default": { + "ENGINE": get_env_variable("DATABASE_ENGINE"), + "NAME": get_env_variable("DATABASE_NAME"), + "USER": get_env_variable("DATABASE_USER"), + "PASSWORD": get_env_variable("DATABASE_PASSWORD"), + "HOST": get_env_variable("DATABASE_HOST"), + "PORT": get_env_variable("DATABASE_PORT"), + "OPTIONS": {"options": "-c statement_timeout=5000"}, + } +} + + +# Password validation +# https://docs.djangoproject.com/en/3.2/ref/settings/#auth-password-validators + +AUTH_PASSWORD_VALIDATORS = [ + { + "NAME": "django.contrib.auth.password_validation.UserAttributeSimilarityValidator", + }, + { + "NAME": "django.contrib.auth.password_validation.MinimumLengthValidator", + }, + { + "NAME": "django.contrib.auth.password_validation.CommonPasswordValidator", + }, + { + "NAME": "django.contrib.auth.password_validation.NumericPasswordValidator", + }, +] + +# Internationalization +# https://docs.djangoproject.com/en/3.2/topics/i18n/ + +LANGUAGE_CODE = "en-us" +TIME_ZONE = "America/Los_Angeles" +USE_I18N = True +USE_L10N = True +USE_TZ = True + +# Static files (CSS, JavaScript, Images) +# https://docs.djangoproject.com/en/3.2/howto/static-files/ + +STATIC_URL = "/static/" + +# Default primary key field type +# https://docs.djangoproject.com/en/3.2/ref/settings/#default-auto-field +DEFAULT_AUTO_FIELD = "django.db.models.BigAutoField" + +# Set up caching +DEFAULT_CACHE_TIMEOUT = 60 * 30 # 30 minutes +CACHES = {} +if to_bool(get_env_variable("ENABLE_CACHE")): + if to_bool(get_env_variable("CACHE_USE_REDIS")): + CACHES = { + "default": { + "BACKEND": "django_redis.cache.RedisCache", + "LOCATION": get_env_variable("CACHE_REDIS_CONNECTION"), + "OPTIONS": { + "CLIENT_CLASS": "django_redis.client.DefaultClient", + "TIMEOUT": DEFAULT_CACHE_TIMEOUT, + "PASSWORD": get_env_variable("CACHE_REDIS_PASSWORD"), + }, + } + } + else: + CACHES = { + "default": { + "BACKEND": "django.core.cache.backends.locmem.LocMemCache", + "LOCATION": "unique-snowflake", + "TIMEOUT": DEFAULT_CACHE_TIMEOUT, + }, + } + +# Configure application logging +LOGGING = { + "version": 1, + "disable_existing_loggers": False, + "formatters": { + "verbose": { + "format": u"[%(asctime)s] %(levelname)s [%(name)s.%(funcName)s:%(lineno)s] %(message)s", + "datefmt": "%d/%b/%Y %H:%M:%S", + }, + "simple": {"format": u"%(levelname)s %(message)s"}, + }, + "handlers": { + "console": {"class": "logging.StreamHandler", "formatter": "verbose", "level": "WARNING"}, + #'appinsights': { + # 'class': 'applicationinsights.django.LoggingHandler', + # 'level': 'INFO', + # }, + "file": { + "level": "DEBUG", + "class": "logging.handlers.RotatingFileHandler", + "filename": os.path.join(BASE_DIR, "..", "logs", "omega-triage.log"), + "maxBytes": 1024 * 1024 * 50, # 50 MB + "backupCount": 5, + "formatter": "verbose", + "encoding": "utf-8", + }, + "database-log": { + "level": "DEBUG", + "class": "logging.handlers.RotatingFileHandler", + "filename": os.path.join(BASE_DIR, "..", "logs", "database.log"), + "maxBytes": 1024 * 1024 * 50, # 50 MB + "backupCount": 1, + "formatter": "verbose", + "encoding": "utf-8", + }, + }, + "loggers": { + "": { + "handlers": ["file", "console"], + "level": "WARNING", + "propagate": True, + }, + "django": { + "level": "WARNING", + "handlers": ["console", "file"], + "propagate": False, + }, + "django.db": { + "level": "DEBUG", + "handlers": ["database-log"], + "propagate": True, + }, + "triage": { + "handlers": ["console", "file"], + "level": "DEBUG", + "propagate": False, + }, + }, +} + +TOOLSHED_BLOB_STORAGE_CONTAINER_SECRET = get_env_variable("TOOLSHED_BLOB_STORAGE_CONTAINER") +TOOLSHED_BLOB_STORAGE_URL_SECRET = get_env_variable("TOOLSHED_BLOB_STORAGE_URL") + +OSSGADGET_PATH = get_env_variable("OSSGADGET_PATH") + +AUTH_USER_MODEL = "auth.User" # pylint: disable=hard-coded-auth-user + +# File Storage Providers for files, attachments, etc. +FILE_STORAGE_PROVIDERS = { + "default": { + "provider": "triage.util.content_managers.file_manager.FileManager", + "args": { + "root_path": "/home/vscode/omega-fs" + } + } +} \ No newline at end of file diff --git a/omega/triage-portal/src/core/urls.py b/omega/triage-portal/src/core/urls.py new file mode 100644 index 00000000..03ada66a --- /dev/null +++ b/omega/triage-portal/src/core/urls.py @@ -0,0 +1,17 @@ +""" +Main URL Configuration +""" +from django.contrib import admin +from django.urls import include, path + +from core.settings import DEBUG + +urlpatterns = [ + path("", include("triage.urls")), +] + +if DEBUG: + urlpatterns += [ + path("admin/", admin.site.urls), + path("__debug__/", include("debug_toolbar.urls")), + ] diff --git a/omega/triage-portal/src/core/wsgi.py b/omega/triage-portal/src/core/wsgi.py new file mode 100644 index 00000000..135b7a61 --- /dev/null +++ b/omega/triage-portal/src/core/wsgi.py @@ -0,0 +1,16 @@ +""" +WSGI config for core project. + +It exposes the WSGI callable as a module-level variable named ``application``. + +For more information on this file, see +https://docs.djangoproject.com/en/3.2/howto/deployment/wsgi/ +""" + +import os + +from django.core.wsgi import get_wsgi_application + +os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'core.settings') + +application = get_wsgi_application() diff --git a/omega/triage-portal/src/manage.py b/omega/triage-portal/src/manage.py new file mode 100755 index 00000000..f2a662cf --- /dev/null +++ b/omega/triage-portal/src/manage.py @@ -0,0 +1,22 @@ +#!/usr/bin/env python +"""Django's command-line utility for administrative tasks.""" +import os +import sys + + +def main(): + """Run administrative tasks.""" + os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'core.settings') + try: + from django.core.management import execute_from_command_line + except ImportError as exc: + raise ImportError( + "Couldn't import Django. Are you sure it's installed and " + "available on your PYTHONPATH environment variable? Did you " + "forget to activate a virtual environment?" + ) from exc + execute_from_command_line(sys.argv) + + +if __name__ == '__main__': + main() diff --git a/omega/triage-portal/src/package.json b/omega/triage-portal/src/package.json new file mode 100644 index 00000000..6fec9438 --- /dev/null +++ b/omega/triage-portal/src/package.json @@ -0,0 +1,24 @@ +{ + "dependencies": { + "@fortawesome/fontawesome-free": "^5.7.2", + "@popperjs/core": "^2.10.1", + "ace-builds": "^1.4.2", + "add": "^2.0.6", + "blueimp-file-upload": "^10.1.0", + "bootstrap": "^5.1.0", + "datatables.net-bs5": "^1.11.3", + "datatables.net-fixedheader-bs5": "^3.2.0", + "datatables.net-keytable-bs5": "^2.6.4", + "datatables.net-select": "^1.3.0", + "inconsolata": "^0.0.2", + "jquery": "^3.3.1", + "jquery-contextmenu": "^2.8.0", + "js-yaml": "^3.13.0", + "jstree": "^3.3.11", + "jstree-bootstrap-theme": "^1.0.1", + "select2": "^4.1.0-rc.0", + "source-code-pro": "^2.30.2", + "tablesorter": "^2.31.1", + "yarn": "^1.13.0" + } +} diff --git a/omega/triage-portal/src/pyproject.toml b/omega/triage-portal/src/pyproject.toml new file mode 100644 index 00000000..53ce723d --- /dev/null +++ b/omega/triage-portal/src/pyproject.toml @@ -0,0 +1,536 @@ +[tool.isort] +profile = "black" +known_first_party = ["triage", "core"] + +[tool.pylint.main] +# Analyse import fallback blocks. This can be used to support both Python 2 and 3 +# compatible code, which means that the block might have code that exists only in +# one or another interpreter, leading to false positives when analysed. +# analyse-fallback-blocks = + +# Always return a 0 (non-error) status code, even if lint errors are found. This +# is primarily useful in continuous integration scripts. +# exit-zero = + +# A comma-separated list of package or module names from where C extensions may +# be loaded. Extensions are loading into the active Python interpreter and may +# run arbitrary code. +# extension-pkg-allow-list = + +# A comma-separated list of package or module names from where C extensions may +# be loaded. Extensions are loading into the active Python interpreter and may +# run arbitrary code. (This is an alternative name to extension-pkg-allow-list +# for backward compatibility.) +# extension-pkg-whitelist = + +# Return non-zero exit code if any of these messages/categories are detected, +# even if score is above --fail-under value. Syntax same as enable. Messages +# specified are enabled, while categories only check already-enabled messages. +# fail-on = + +# Specify a score threshold under which the program will exit with error. +fail-under = 10 + +# Interpret the stdin as a python script, whose filename needs to be passed as +# the module_or_package argument. +# from-stdin = + +# Files or directories to be skipped. They should be base names, not paths. +ignore = ["CVS"] + +# Add files or directories matching the regular expressions patterns to the +# ignore-list. The regex matches against paths and can be in Posix or Windows +# format. Because '\' represents the directory delimiter on Windows systems, it +# can't be used as an escape character. +# ignore-paths = + +# Files or directories matching the regular expression patterns are skipped. The +# regex matches against base names, not paths. The default value ignores Emacs +# file locks +ignore-patterns = ["^\\.#"] + +# List of module names for which member attributes should not be checked (useful +# for modules/projects where namespaces are manipulated during runtime and thus +# existing member attributes cannot be deduced by static analysis). It supports +# qualified module names, as well as Unix pattern matching. +# ignored-modules = + +# Python code to execute, usually for sys.path manipulation such as +# pygtk.require(). +init-hook = 'import sys; sys.path.append("/workspaces/alpha-omega/omega/triage-portal/src"); sys.path.append("/workspaces/alpha-omega/omega/triage-portal/.venv/lib/python3.11/site-packages")' + +# Use multiple processes to speed up Pylint. Specifying 0 will auto-detect the +# number of processors available to use, and will cap the count on Windows to +# avoid hangs. +jobs = 1 + +# Control the amount of potential inferred values when inferring a single object. +# This can help the performance when dealing with large functions or complex, +# nested conditions. +limit-inference-results = 100 + +# List of plugins (as comma separated values of python module names) to load, +# usually to register additional checkers. +load-plugins = [ + "pylint_django" +] + +# Pickle collected data for later comparisons. +persistent = true + +# Minimum Python version to use for version dependent checks. Will default to the +# version used to run pylint. +py-version = "3.11" + +# Discover python modules and packages in the file system subtree. +# recursive = + +# When enabled, pylint would attempt to guess common misconfiguration and emit +# user-friendly hints instead of false-positive error messages. +suggestion-mode = true + +# Allow loading of arbitrary C extensions. Extensions are imported into the +# active Python interpreter and may run arbitrary code. +# unsafe-load-any-extension = + +[tool.pylint.basic] +# Naming style matching correct argument names. +argument-naming-style = "snake_case" + +# Regular expression matching correct argument names. Overrides argument-naming- +# style. If left empty, argument names will be checked with the set naming style. +# argument-rgx = + +# Naming style matching correct attribute names. +attr-naming-style = "snake_case" + +# Regular expression matching correct attribute names. Overrides attr-naming- +# style. If left empty, attribute names will be checked with the set naming +# style. +# attr-rgx = + +# Bad variable names which should always be refused, separated by a comma. +bad-names = ["foo", "bar", "baz", "toto", "tutu", "tata"] + +# Bad variable names regexes, separated by a comma. If names match any regex, +# they will always be refused +# bad-names-rgxs = + +# Naming style matching correct class attribute names. +class-attribute-naming-style = "any" + +# Regular expression matching correct class attribute names. Overrides class- +# attribute-naming-style. If left empty, class attribute names will be checked +# with the set naming style. +# class-attribute-rgx = + +# Naming style matching correct class constant names. +class-const-naming-style = "UPPER_CASE" + +# Regular expression matching correct class constant names. Overrides class- +# const-naming-style. If left empty, class constant names will be checked with +# the set naming style. +# class-const-rgx = + +# Naming style matching correct class names. +class-naming-style = "PascalCase" + +# Regular expression matching correct class names. Overrides class-naming-style. +# If left empty, class names will be checked with the set naming style. +# class-rgx = + +# Naming style matching correct constant names. +const-naming-style = "UPPER_CASE" + +# Regular expression matching correct constant names. Overrides const-naming- +# style. If left empty, constant names will be checked with the set naming style. +# const-rgx = + +# Minimum line length for functions/classes that require docstrings, shorter ones +# are exempt. +docstring-min-length = -1 + +# Naming style matching correct function names. +function-naming-style = "snake_case" + +# Regular expression matching correct function names. Overrides function-naming- +# style. If left empty, function names will be checked with the set naming style. +# function-rgx = + +# Good variable names which should always be accepted, separated by a comma. +good-names = ["f", "i", "j", "k", "ex", "Run", "_"] + +# Good variable names regexes, separated by a comma. If names match any regex, +# they will always be accepted +# good-names-rgxs = + +# Include a hint for the correct naming format with invalid-name. +# include-naming-hint = + +# Naming style matching correct inline iteration names. +inlinevar-naming-style = "any" + +# Regular expression matching correct inline iteration names. Overrides +# inlinevar-naming-style. If left empty, inline iteration names will be checked +# with the set naming style. +# inlinevar-rgx = + +# Naming style matching correct method names. +method-naming-style = "snake_case" + +# Regular expression matching correct method names. Overrides method-naming- +# style. If left empty, method names will be checked with the set naming style. +# method-rgx = + +# Naming style matching correct module names. +module-naming-style = "snake_case" + +# Regular expression matching correct module names. Overrides module-naming- +# style. If left empty, module names will be checked with the set naming style. +# module-rgx = + +# Colon-delimited sets of names that determine each other's naming style when the +# name regexes allow several styles. +# name-group = + +# Regular expression which should only match function or class names that do not +# require a docstring. +no-docstring-rgx = "^_" + +# List of decorators that produce properties, such as abc.abstractproperty. Add +# to this list to register other decorators that produce valid properties. These +# decorators are taken in consideration only for invalid-name. +property-classes = ["abc.abstractproperty"] + +# Regular expression matching correct type variable names. If left empty, type +# variable names will be checked with the set naming style. +# typevar-rgx = + +# Naming style matching correct variable names. +variable-naming-style = "snake_case" + +# Regular expression matching correct variable names. Overrides variable-naming- +# style. If left empty, variable names will be checked with the set naming style. +# variable-rgx = + +[tool.pylint.classes] +# Warn about protected attribute access inside special methods +# check-protected-access-in-special-methods = + +# List of method names used to declare (i.e. assign) instance attributes. +defining-attr-methods = ["__init__", "__new__", "setUp", "__post_init__"] + +# List of member names, which should be excluded from the protected access +# warning. +exclude-protected = ["_asdict", "_fields", "_replace", "_source", "_make"] + +# List of valid names for the first argument in a class method. +valid-classmethod-first-arg = ["cls"] + +# List of valid names for the first argument in a metaclass class method. +valid-metaclass-classmethod-first-arg = ["cls"] + +[tool.pylint.design] +# List of regular expressions of class ancestor names to ignore when counting +# public methods (see R0903) +# exclude-too-few-public-methods = + +# List of qualified class names to ignore when counting class parents (see R0901) +# ignored-parents = + +# Maximum number of arguments for function / method. +max-args = 5 + +# Maximum number of attributes for a class (see R0902). +max-attributes = 7 + +# Maximum number of boolean expressions in an if statement (see R0916). +max-bool-expr = 5 + +# Maximum number of branch for function / method body. +max-branches = 12 + +# Maximum number of locals for function / method body. +max-locals = 15 + +# Maximum number of parents for a class (see R0901). +max-parents = 7 + +# Maximum number of public methods for a class (see R0904). +max-public-methods = 20 + +# Maximum number of return / yield for function / method body. +max-returns = 6 + +# Maximum number of statements in function / method body. +max-statements = 50 + +# Minimum number of public methods for a class (see R0903). +min-public-methods = 2 + +[tool.pylint.exceptions] +# Exceptions that will emit a warning when caught. +overgeneral-exceptions = ["BaseException", "Exception"] + +[tool.pylint.format] +# Expected format of line ending, e.g. empty (any line ending), LF or CRLF. +# expected-line-ending-format = + +# Regexp for a line that is allowed to be longer than the limit. +ignore-long-lines = "^\\s*(# )??$" + +# Number of spaces of indent required inside a hanging or continued line. +indent-after-paren = 4 + +# String used as indentation unit. This is usually " " (4 spaces) or "\t" (1 +# tab). +indent-string = " " + +# Maximum number of characters on a single line. +max-line-length = 100 + +# Maximum number of lines in a module. +max-module-lines = 1000 + +# Allow the body of a class to be on the same line as the declaration if body +# contains single statement. +# single-line-class-stmt = + +# Allow the body of an if to be on the same line as the test if there is no else. +# single-line-if-stmt = + +[tool.pylint.imports] +# List of modules that can be imported at any level, not just the top level one. +# allow-any-import-level = + +# Allow wildcard imports from modules that define __all__. +# allow-wildcard-with-all = + +# Deprecated modules which should not be used, separated by a comma. +# deprecated-modules = + +# Output a graph (.gv or any supported image format) of external dependencies to +# the given file (report RP0402 must not be disabled). +# ext-import-graph = + +# Output a graph (.gv or any supported image format) of all (i.e. internal and +# external) dependencies to the given file (report RP0402 must not be disabled). +# import-graph = + +# Output a graph (.gv or any supported image format) of internal dependencies to +# the given file (report RP0402 must not be disabled). +# int-import-graph = + +# Force import order to recognize a module as part of the standard compatibility +# libraries. +# known-standard-library = + +# Force import order to recognize a module as part of a third party library. +known-third-party = ["enchant"] + +# Couples of modules and preferred modules, separated by a comma. +# preferred-modules = + +[tool.pylint.logging] +# The type of string formatting that logging methods do. `old` means using % +# formatting, `new` is for `{}` formatting. +logging-format-style = "old" + +# Logging modules to check that the string format arguments are in logging +# function parameter format. +logging-modules = ["logging"] + +[tool.pylint."messages control"] +# Only show warnings with the listed confidence levels. Leave empty to show all. +# Valid levels: HIGH, CONTROL_FLOW, INFERENCE, INFERENCE_FAILURE, UNDEFINED. +confidence = ["HIGH", "CONTROL_FLOW", "INFERENCE", "INFERENCE_FAILURE", "UNDEFINED"] + +# Disable the message, report, category or checker with the given id(s). You can +# either give multiple identifiers separated by comma (,) or put this option +# multiple times (only on the command line, not in the configuration file where +# it should appear only once). You can also use "--disable=all" to disable +# everything first and then re-enable specific checks. For example, if you want +# to run only the similarities checker, you can use "--disable=all +# --enable=similarities". If you want to run only the classes checker, but have +# no Warning level messages displayed, use "--disable=all --enable=classes +# --disable=W". +disable = ["raw-checker-failed", "bad-inline-option", "locally-disabled", "file-ignored", "suppressed-message", "useless-suppression", "deprecated-pragma", "use-symbolic-message-instead"] + +# Enable the message, report, category or checker with the given id(s). You can +# either give multiple identifier separated by comma (,) or put this option +# multiple time (only on the command line, not in the configuration file where it +# should appear only once). See also the "--disable" option for examples. +enable = ["c-extension-no-member"] + +[tool.pylint.method_args] +# List of qualified names (i.e., library.method) which require a timeout +# parameter e.g. 'requests.api.get,requests.api.post' +timeout-methods = ["requests.api.delete", "requests.api.get", "requests.api.head", "requests.api.options", "requests.api.patch", "requests.api.post", "requests.api.put", "requests.api.request"] + +[tool.pylint.miscellaneous] +# List of note tags to take in consideration, separated by a comma. +notes = ["FIXME", "XXX", "TODO"] + +# Regular expression of note tags to take in consideration. +# notes-rgx = + +[tool.pylint.refactoring] +# Maximum number of nested blocks for function / method body +max-nested-blocks = 5 + +# Complete name of functions that never returns. When checking for inconsistent- +# return-statements if a never returning function is called then it will be +# considered as an explicit return statement and no message will be printed. +never-returning-functions = ["sys.exit", "argparse.parse_error"] + +[tool.pylint.reports] +# Python expression which should return a score less than or equal to 10. You +# have access to the variables 'fatal', 'error', 'warning', 'refactor', +# 'convention', and 'info' which contain the number of messages in each category, +# as well as 'statement' which is the total number of statements analyzed. This +# score is used by the global evaluation report (RP0004). +evaluation = "max(0, 0 if fatal else 10.0 - ((float(5 * error + warning + refactor + convention) / statement) * 10))" + +# Template used to display messages. This is a python new-style format string +# used to format the message information. See doc for all details. +# msg-template = + +# Set the output format. Available formats are text, parseable, colorized, json +# and msvs (visual studio). You can also give a reporter class, e.g. +# mypackage.mymodule.MyReporterClass. +# output-format = + +# Tells whether to display a full report or only the messages. +# reports = + +# Activate the evaluation score. +score = true + +[tool.pylint.similarities] +# Comments are removed from the similarity computation +ignore-comments = true + +# Docstrings are removed from the similarity computation +ignore-docstrings = true + +# Imports are removed from the similarity computation +ignore-imports = true + +# Signatures are removed from the similarity computation +ignore-signatures = true + +# Minimum lines number of a similarity. +min-similarity-lines = 4 + +[tool.pylint.spelling] +# Limits count of emitted suggestions for spelling mistakes. +max-spelling-suggestions = 4 + +# Spelling dictionary name. Available dictionaries: none. To make it work, +# install the 'python-enchant' package. +# spelling-dict = + +# List of comma separated words that should be considered directives if they +# appear at the beginning of a comment and should not be checked. +spelling-ignore-comment-directives = "fmt: on,fmt: off,noqa:,noqa,nosec,isort:skip,mypy:" + +# List of comma separated words that should not be checked. +# spelling-ignore-words = + +# A path to a file that contains the private dictionary; one word per line. +# spelling-private-dict-file = + +# Tells whether to store unknown words to the private dictionary (see the +# --spelling-private-dict-file option) instead of raising a message. +# spelling-store-unknown-words = + +[tool.pylint.string] +# This flag controls whether inconsistent-quotes generates a warning when the +# character used as a quote delimiter is used inconsistently within a module. +# check-quote-consistency = + +# This flag controls whether the implicit-str-concat should generate a warning on +# implicit string concatenation in sequences defined over several lines. +# check-str-concat-over-line-jumps = + +[tool.pylint.typecheck] +# List of decorators that produce context managers, such as +# contextlib.contextmanager. Add to this list to register other decorators that +# produce valid context managers. +contextmanager-decorators = ["contextlib.contextmanager"] + +# List of members which are set dynamically and missed by pylint inference +# system, and so shouldn't trigger E1101 when accessed. Python regular +# expressions are accepted. +# generated-members = + +# Tells whether missing members accessed in mixin class should be ignored. A +# class is considered mixin if its name matches the mixin-class-rgx option. +# Tells whether to warn about missing members when the owner of the attribute is +# inferred to be None. +ignore-none = true + +# This flag controls whether pylint should warn about no-member and similar +# checks whenever an opaque object is returned when inferring. The inference can +# return multiple potential results while evaluating a Python object, but some +# branches might not be evaluated, which results in partial inference. In that +# case, it might be useful to still emit no-member and other checks for the rest +# of the inferred objects. +ignore-on-opaque-inference = true + +# List of symbolic message names to ignore for Mixin members. +ignored-checks-for-mixins = ["no-member", "not-async-context-manager", "not-context-manager", "attribute-defined-outside-init"] + +# List of class names for which member attributes should not be checked (useful +# for classes with dynamically set attributes). This supports the use of +# qualified names. +ignored-classes = ["optparse.Values", "thread._local", "_thread._local", "argparse.Namespace"] + +# Show a hint with possible names when a member name was not found. The aspect of +# finding the hint is based on edit distance. +missing-member-hint = true + +# The minimum edit distance a name should have in order to be considered a +# similar match for a missing member name. +missing-member-hint-distance = 1 + +# The total number of similar names that should be taken in consideration when +# showing a hint for a missing member. +missing-member-max-choices = 1 + +# Regex pattern to define which classes are considered mixins. +mixin-class-rgx = ".*[Mm]ixin" + +# List of decorators that change the signature of a decorated function. +# signature-mutators = + +[tool.pylint.variables] +# List of additional names supposed to be defined in builtins. Remember that you +# should avoid defining new builtins when possible. +# additional-builtins = + +# Tells whether unused global variables should be treated as a violation. +allow-global-unused-variables = true + +# List of names allowed to shadow builtins +# allowed-redefined-builtins = + +# List of strings which can identify a callback function by name. A callback name +# must start or end with one of those strings. +callbacks = ["cb_", "_cb"] + +# A regular expression matching the name of dummy variables (i.e. expected to not +# be used). +dummy-variables-rgx = "_+$|(_[a-zA-Z0-9_]*[a-zA-Z0-9]+?$)|dummy|^ignored_|^unused_" + +# Argument names that match this expression will be ignored. +ignored-argument-names = "_.*|^ignored_|^unused_" + +# Tells whether we should check for unused import in __init__ files. +# init-import = + +# List of qualified module names which can have objects that can redefine +# builtins. +redefining-builtins-modules = ["six.moves", "past.builtins", "future.builtins", "builtins", "io"] + + diff --git a/omega/triage-portal/src/requirements.txt b/omega/triage-portal/src/requirements.txt new file mode 100644 index 00000000..0f26f945 --- /dev/null +++ b/omega/triage-portal/src/requirements.txt @@ -0,0 +1,74 @@ +asgiref==3.5.2 +astroid==2.12.13 +azure-core==1.26.1 +azure-storage-blob==12.14.1 +bandit==1.7.4 +black==22.10.0 +certifi==2022.9.24 +cffi==1.15.1 +charset-normalizer==2.1.1 +click==8.1.3 +cryptography==38.0.4 +Deprecated==1.2.13 +dill==0.3.6 +Django==4.1.3 +django-dotenv==1.4.2 +django-redis==5.2.0 +django-taggit==3.1.0 +django_debug_toolbar==3.8.1 +dodgy==0.2.1 +flake8==5.0.4 +flake8-polyfill==1.0.2 +gitdb==4.0.10 +GitPython==3.1.29 +idna==3.4 +isodate==0.6.1 +isort==5.10.1 +lazy-object-proxy==1.6.0 +Markdown==3.4.1 +mccabe==0.7.0 +msrest==0.7.1 +mypy-extensions==0.4.3 +oauthlib==3.2.2 +packageurl-python==0.10.4 +packaging==21.3 +pathspec==0.10.2 +pbr==5.8.0 +pep8-naming==0.10.0 +platformdirs==2.5.4 +poetry-semver==0.1.0 +prospector==1.8.2 +psycopg2==2.9.5 +pycodestyle==2.9.1 +pycparser==2.21 +pydocstyle==6.1.1 +pyflakes==2.5.0 +pylint==2.15.7 +pylint-celery==0.3 +pylint-common==0.2.5 +pylint-django==2.5.3 +pylint-flask==0.6 +pylint-plugin-utils==0.7 +pyparsing==3.0.9 +python-magic==0.4.27 +pytz==2022.6 +PyYAML==6.0 +redis==3.5.3 +regex==2022.10.31 +requests==2.28.1 +requests-oauthlib==1.3.1 +requirements-detector==1.0.3 +setoptconf==0.3.0 +setoptconf-tmp==0.3.1 +six==1.16.0 +smmap==5.0.0 +snowballstemmer==2.2.0 +sqlparse==0.4.3 +stevedore==4.1.1 +toml==0.10.2 +tomli==2.0.1 +tomlkit==0.11.6 +typing_extensions==4.4.0 +urllib3==1.26.13 +wrapt==1.14.1 +zstd==1.5.2.6 diff --git a/omega/triage-portal/src/triage/__init__.py b/omega/triage-portal/src/triage/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/omega/triage-portal/src/triage/apps.py b/omega/triage-portal/src/triage/apps.py new file mode 100644 index 00000000..d7fce052 --- /dev/null +++ b/omega/triage-portal/src/triage/apps.py @@ -0,0 +1,61 @@ +# -*- coding: utf-8 -*- +"""This module configures application-level settings for the Triage Portal.""" + +import logging +import mimetypes + +from django.apps import AppConfig +from django.contrib import admin + +from core.settings import DEBUG + +logger = logging.getLogger(__name__) + + +class TriageConfig(AppConfig): + """ + Application configuration for Omega/Triage . + + This class gets called when Django is initialized, and takes care of + one-time initialization. + """ + + default_auto_field = "django.db.models.BigAutoField" + name = "triage" + verbose_name = "Triage" + + _is_init_completed = False + + def ready(self): + if self._is_init_completed: + return True # Only run once + logger.debug("TriageConfig initializing.") + + if DEBUG: + self._register_models_admin_config() + + mimetypes.init() + + self._is_init_completed = True + + return True + + def _register_models_admin_config(self): + """Registers all Triage models in the Django admin interface.""" + models = self.apps.get_models() + num_registered = 0 + + class TriageModelAdmin(admin.ModelAdmin): + """Custom model admin to show additional fields.""" + + readonly_fields = ("uuid",) + + for model in filter(lambda m: not admin.site.is_registered(m), models): + if model.__module__.startswith("triage.") and hasattr(model, "uuid"): + admin.site.register(model, TriageModelAdmin) + else: + admin.site.register(model, admin.ModelAdmin) + + num_registered += 1 + + logger.debug("Registered %d models to admin module.", num_registered) diff --git a/omega/triage-portal/src/triage/management/__init__.py b/omega/triage-portal/src/triage/management/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/omega/triage-portal/src/triage/management/commands/__init__.py b/omega/triage-portal/src/triage/management/commands/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/omega/triage-portal/src/triage/management/commands/clear_all_findings.py b/omega/triage-portal/src/triage/management/commands/clear_all_findings.py new file mode 100644 index 00000000..c72183dc --- /dev/null +++ b/omega/triage-portal/src/triage/management/commands/clear_all_findings.py @@ -0,0 +1,22 @@ +import logging + +from django.core.management.base import BaseCommand + +from triage.models import File, FileContent, Finding, Project, Tool + +logger = logging.getLogger(__name__) + + +class Command(BaseCommand): + help = "Clears all findings from the repository" + + def handle(self, *args, **options): + """Handle the 'clear_all_findings' command.""" + + Finding.objects.all().delete() + Project.objects.all().delete() + Tool.objects.all().delete() + File.objects.all().delete() + FileContent.objects.all().delete() + + print("Operation complete.") diff --git a/omega/triage-portal/src/triage/management/commands/import_toolshed.py b/omega/triage-portal/src/triage/management/commands/import_toolshed.py new file mode 100644 index 00000000..86b704a9 --- /dev/null +++ b/omega/triage-portal/src/triage/management/commands/import_toolshed.py @@ -0,0 +1,172 @@ +import json +import logging +import os +import re +from functools import lru_cache + +from azure.storage.blob import BlobProperties, BlobServiceClient +from django.contrib.auth import get_user_model +from django.core.management.base import BaseCommand +from packageurl import PackageURL + +from core.settings import ( + TOOLSHED_BLOB_STORAGE_CONTAINER_SECRET, + TOOLSHED_BLOB_STORAGE_URL_SECRET, +) +from triage.models import ProjectVersion, Scan, Tool +from triage.util.finding_importers.file_importer import FileImporter +from triage.util.finding_importers.sarif_importer import SARIFImporter + +logger = logging.getLogger(__name__) + + +class Command(BaseCommand): + help = "Imports a project from the Toolshed repository" + + def add_arguments(self, parser): + """Assembles arguments to the command.""" + parser.add_argument("--package", required=False, type=str, help="URL (PackageURL format)") + parser.add_argument( + "--import-all", required=False, action="store_true", help="SARIF file to load" + ) + parser.add_argument( + "--maximum", + required=False, + type=int, + help="Maximum number of entries to import", + default=0, + ) + + @classmethod + def prefix_for_package_url(self, package_url_str: str) -> str: + try: + package_url = PackageURL.from_string(package_url_str) + if package_url.namespace: + prefix = f"{package_url.type}/{package_url.namespace}/{package_url.name}/{package_url.version}" + else: + prefix = f"{package_url.type}/{package_url.name}/{package_url.version}" + except ValueError: + prefix = None + + return prefix + + def handle(self, *args, **options): + """Handle the 'import sarif' command.""" + package = options.get("package") + if not package and not options.get("import_all"): + raise ValueError("Must specify either --package or --import-all") + + sarif_importer = SARIFImporter() + + self.initialize_toolshed() + user = get_user_model().objects.get(pk=1) + scan_map = {} + + if package: + prefix = Command.prefix_for_package_url(package) + blobs = self.container.list_blobs(name_starts_with=prefix) + elif options.get("import_all"): + package_url = None + blobs = self.container.list_blobs( + name_starts_with="npm", + ) # TODO: temporary + else: + raise ValueError("Must specify either --package or --import-all") + + num_imported = 0 + for blob in blobs: # type: BlobProperties + print(f"Importing {blob.name}") + num_imported += 1 + if num_imported > options.get("maximum", 0) > 0: + logger.info("Maximum number of entries reached") + break + + package_url = Command.filename_to_package_url(blob.name) + + # match = re.match(r".*/tool-([^\.]+)\.sarif", blob.name, re.IGNORECASE) + # if not match: + # logger.info(f"Skipping {blob.name}") + # continue + # tool_name = match.group(1) + project_version = ProjectVersion.get_or_create_from_package_url(package_url, user) + + # Each tool result is a separate scan + # scan = Scan( + # project_version=project_version, + # tool=Tool.objects.get_or_create(name=tool_name)[0], + # created_by=user, + # updated_by=user, + # ) + # scan.save() + + file_importer = FileImporter() + + if blob.name.endswith(".sarif"): + logger.debug("Importing %s", blob.name) + try: + blob_contents = self.container.download_blob(blob.name).content_as_text() + sarif = json.loads(blob_contents) + SARIFImporter.import_sarif_file(package_url, sarif, user) + FileImporter.import_file( + f"tool/{os.path.basename(blob.name)}", + blob.name, + blob_contents.encode("utf-8", errors="ignore"), + user, + ) + except Exception as msg: + logger.error("Unable to import %s: %s", blob.name, msg, exc_info=True) + continue + + elif "/reference-binaries/" in blob.name: + print(f"Importing {blob.name}") + logger.debug("Importing %s", blob.name) + try: + blob_contents = self.container.download_blob(blob.name).content_as_bytes() + FileImporter.import_file( + f"src/{blob.name}", blob.name, blob_contents, project_version + ) + except Exception as msg: + logger.error("Unable to import %s: %s", blob.name, msg, exc_info=True) + continue + + @classmethod + @lru_cache + def filename_to_package_url(cls, filename): + """Convert a filename to a PackageURL.""" + print(filename) + match = re.match( + r"^(?P[^/]+)/(?P[^/]+)/(?P[^/]+)/(?P[^/]+)/(reference-binaries|summary-|tool-|admin-).*$", + filename, + ) + print(f"Match={bool(match)}") + if not match: + match = re.match( + r"^(?P[^/]+)/(?P[^/]+)/(?P[^/]+)/(reference-binaries|summary-|tool-|admin-).*$", + filename, + ) + if not match: + logger.debug("Unable to parse filename [%s]", filename) + return None + + try: + package_url = PackageURL( + type=match.group("type"), + namespace=match.group("namespace") if "namespace" in match.groupdict() else None, + name=match.group("name"), + version=match.group("version"), + ) + print(package_url) + return package_url + except Exception as msg: + logger.debug("Unable to create PackageURL from %s: %s", filename, msg) + return None + + def initialize_toolshed(self): + if not TOOLSHED_BLOB_STORAGE_URL_SECRET or not TOOLSHED_BLOB_STORAGE_CONTAINER_SECRET: + raise ValueError("TOOLSHED_BLOB_STORAGE_URL and TOOLSHED_BLOB_CONTAINER must be set") + + self.blob_service = BlobServiceClient(TOOLSHED_BLOB_STORAGE_URL_SECRET) + + self.container = self.blob_service.get_container_client( + TOOLSHED_BLOB_STORAGE_CONTAINER_SECRET + ) diff --git a/omega/triage-portal/src/triage/migrations/0001_initial.py b/omega/triage-portal/src/triage/migrations/0001_initial.py new file mode 100644 index 00000000..f1ae4db4 --- /dev/null +++ b/omega/triage-portal/src/triage/migrations/0001_initial.py @@ -0,0 +1,186 @@ +# Generated by Django 3.2.9 on 2021-11-26 06:47 + +from django.conf import settings +from django.db import migrations, models +import django.db.models.deletion +import uuid + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [ + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ] + + operations = [ + migrations.CreateModel( + name='Project', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('created_at', models.DateTimeField(auto_now_add=True)), + ('updated_at', models.DateTimeField(auto_now=True)), + ('uuid', models.UUIDField(db_index=True, default=uuid.uuid4, editable=False)), + ('name', models.CharField(db_index=True, max_length=1024)), + ('package_url', models.CharField(blank=True, db_index=True, max_length=1024, null=True)), + ('metadata', models.JSONField(null=True)), + ('created_by', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='+', to=settings.AUTH_USER_MODEL)), + ('updated_by', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='+', to=settings.AUTH_USER_MODEL)), + ], + options={ + 'abstract': False, + }, + ), + migrations.CreateModel( + name='ProjectVersion', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('created_at', models.DateTimeField(auto_now_add=True)), + ('updated_at', models.DateTimeField(auto_now=True)), + ('uuid', models.UUIDField(db_index=True, default=uuid.uuid4, editable=False)), + ('package_url', models.CharField(blank=True, db_index=True, max_length=1024, null=True)), + ('metadata', models.JSONField(null=True)), + ('created_by', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='+', to=settings.AUTH_USER_MODEL)), + ('project', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='triage.project')), + ('updated_by', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='+', to=settings.AUTH_USER_MODEL)), + ], + options={ + 'abstract': False, + }, + ), + migrations.CreateModel( + name='TriageRule', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('event', models.CharField(choices=[('FN', 'On New Finding'), ('FM', 'On Modified Finding')], default='FN', max_length=2)), + ('condition', models.TextField(blank=True, max_length=2048, null=True)), + ('action', models.TextField(blank=True, max_length=2048, null=True)), + ('active', models.BooleanField(db_index=True, default=True)), + ('priority', models.PositiveSmallIntegerField(default=1000)), + ('type', models.CharField(choices=[('PY', 'Python Function')], default='PY', max_length=2)), + ], + ), + migrations.CreateModel( + name='Tool', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('created_at', models.DateTimeField(auto_now_add=True)), + ('updated_at', models.DateTimeField(auto_now=True)), + ('uuid', models.UUIDField(db_index=True, default=uuid.uuid4, editable=False)), + ('name', models.CharField(max_length=128)), + ('version', models.CharField(blank=True, max_length=64)), + ('type', models.CharField(choices=[('NS', 'Not Specified'), ('MA', 'Manual'), ('SA', 'Static Analysis'), ('OT', 'Other')], default='NS', max_length=2)), + ('active', models.BooleanField(default=True)), + ('created_by', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='+', to=settings.AUTH_USER_MODEL)), + ('updated_by', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='+', to=settings.AUTH_USER_MODEL)), + ], + options={ + 'abstract': False, + }, + ), + migrations.CreateModel( + name='Scan', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('created_at', models.DateTimeField(auto_now_add=True)), + ('updated_at', models.DateTimeField(auto_now=True)), + ('uuid', models.UUIDField(db_index=True, default=uuid.uuid4, editable=False)), + ('artifact_uuid', models.UUIDField(blank=True, editable=False, null=True)), + ('active', models.BooleanField(default=True)), + ('created_dt', models.DateTimeField(auto_now_add=True)), + ('created_by', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='+', to=settings.AUTH_USER_MODEL)), + ('project_version', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='triage.projectversion')), + ('tool', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='triage.tool')), + ('updated_by', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='+', to=settings.AUTH_USER_MODEL)), + ], + options={ + 'abstract': False, + }, + ), + migrations.CreateModel( + name='Note', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('created_at', models.DateTimeField(auto_now_add=True)), + ('updated_at', models.DateTimeField(auto_now=True)), + ('content', models.TextField()), + ('created_by', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='+', to=settings.AUTH_USER_MODEL)), + ('updated_by', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='+', to=settings.AUTH_USER_MODEL)), + ], + options={ + 'abstract': False, + }, + ), + migrations.CreateModel( + name='Finding', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('created_at', models.DateTimeField(auto_now_add=True)), + ('updated_at', models.DateTimeField(auto_now=True)), + ('uuid', models.UUIDField(db_index=True, default=uuid.uuid4, editable=False)), + ('title', models.CharField(max_length=1024)), + ('file_path', models.CharField(max_length=2048)), + ('file_line', models.PositiveIntegerField(blank=True, null=True)), + ('impact_usage', models.PositiveBigIntegerField(blank=True, null=True)), + ('impact_context', models.PositiveBigIntegerField(blank=True, null=True)), + ('analyst_impact', models.PositiveBigIntegerField(blank=True, null=True)), + ('confidence', models.CharField(choices=[('NS', 'Not Specified'), ('VL', 'Very Low'), ('L', 'Low'), ('M', 'Medium'), ('H', 'High'), ('VH', 'Very High')], default='NS', max_length=2)), + ('severity_level', models.CharField(choices=[('NS', 'Not Specified'), ('NO', 'None'), ('IN', 'Informational'), ('VL', 'Very Low'), ('L', 'Low'), ('M', 'Medium'), ('H', 'High'), ('VH', 'Very High')], default='NS', max_length=2)), + ('analyst_severity_level', models.CharField(choices=[('NS', 'Not Specified'), ('NO', 'None'), ('IN', 'Informational'), ('VL', 'Very Low'), ('L', 'Low'), ('M', 'Medium'), ('H', 'High'), ('VH', 'Very High')], default='NS', max_length=2)), + ('state', models.CharField(choices=[('N', 'New'), ('A', 'Active'), ('R', 'Resolved'), ('D', 'Deleted'), ('CL', 'Closed'), ('NS', 'Not Specified')], default='N', max_length=2)), + ('assigned_dt', models.DateTimeField(auto_now_add=True)), + ('assigned_to', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, to=settings.AUTH_USER_MODEL)), + ('created_by', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='+', to=settings.AUTH_USER_MODEL)), + ('scan', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='triage.scan')), + ('updated_by', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='+', to=settings.AUTH_USER_MODEL)), + ], + options={ + 'abstract': False, + }, + ), + migrations.CreateModel( + name='Filter', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('created_at', models.DateTimeField(auto_now_add=True)), + ('updated_at', models.DateTimeField(auto_now=True)), + ('name', models.CharField(max_length=128)), + ('definition', models.TextField()), + ('active', models.BooleanField(default=True)), + ('priority', models.PositiveSmallIntegerField(default=1000)), + ('created_by', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='+', to=settings.AUTH_USER_MODEL)), + ('project', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, to='triage.project')), + ('updated_by', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='+', to=settings.AUTH_USER_MODEL)), + ], + options={ + 'abstract': False, + }, + ), + migrations.CreateModel( + name='Case', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('created_at', models.DateTimeField(auto_now_add=True)), + ('updated_at', models.DateTimeField(auto_now=True)), + ('uuid', models.UUIDField(db_index=True, default=uuid.uuid4, editable=False)), + ('state', models.CharField(choices=[('N', 'New'), ('R', 'Reported'), ('A', 'Active'), ('RS', 'Resolved'), ('D', 'Deleted'), ('CL', 'Closed'), ('NS', 'Not Specified')], default='N', max_length=2)), + ('title', models.CharField(max_length=1024)), + ('description', models.TextField(blank=True, null=True)), + ('assigned_dt', models.DateTimeField(auto_now_add=True)), + ('reported_to', models.CharField(blank=True, max_length=2048, null=True)), + ('reporting_partner', models.CharField(blank=True, choices=[('N', 'None'), ('GS', 'GitHub Security Lab'), ('CT', 'CERT'), ('MS', 'MSRC'), ('NS', 'Not Specified')], default='NS', max_length=2, null=True)), + ('report_foreign_reference', models.CharField(blank=True, max_length=1024, null=True)), + ('resolved_target_dt', models.DateTimeField(blank=True, null=True)), + ('resolved_dt', models.DateTimeField(blank=True, null=True)), + ('assigned_to', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='assigned_cases', to=settings.AUTH_USER_MODEL)), + ('created_by', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='+', to=settings.AUTH_USER_MODEL)), + ('findings', models.ManyToManyField(related_name='cases', to='triage.Finding')), + ('notes', models.ManyToManyField(related_name='cases', to='triage.Note')), + ('updated_by', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='+', to=settings.AUTH_USER_MODEL)), + ], + options={ + 'abstract': False, + }, + ), + ] diff --git a/omega/triage-portal/src/triage/migrations/0002_auto_20211126_0713.py b/omega/triage-portal/src/triage/migrations/0002_auto_20211126_0713.py new file mode 100644 index 00000000..2127e7be --- /dev/null +++ b/omega/triage-portal/src/triage/migrations/0002_auto_20211126_0713.py @@ -0,0 +1,22 @@ +# Generated by Django 3.2.9 on 2021-11-26 07:13 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('triage', '0001_initial'), + ] + + operations = [ + migrations.RemoveField( + model_name='scan', + name='artifact_uuid', + ), + migrations.AddField( + model_name='scan', + name='artifact_url_base', + field=models.CharField(blank=True, max_length=1024, null=True), + ), + ] diff --git a/omega/triage-portal/src/triage/migrations/0003_auto_20211127_0020.py b/omega/triage-portal/src/triage/migrations/0003_auto_20211127_0020.py new file mode 100644 index 00000000..805c6b6f --- /dev/null +++ b/omega/triage-portal/src/triage/migrations/0003_auto_20211127_0020.py @@ -0,0 +1,32 @@ +# Generated by Django 3.2.9 on 2021-11-27 00:20 + +from django.db import migrations, models +import django.db.models.deletion +import uuid + + +class Migration(migrations.Migration): + + dependencies = [ + ('triage', '0002_auto_20211126_0713'), + ] + + operations = [ + migrations.AlterField( + model_name='case', + name='state', + field=models.CharField(choices=[('N', 'New'), ('A', 'Active'), ('R', 'Resolved'), ('D', 'Deleted'), ('CL', 'Closed'), ('NS', 'Not Specified')], default='N', max_length=2), + ), + migrations.CreateModel( + name='ToolDefect', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('uuid', models.UUIDField(db_index=True, default=uuid.uuid4, editable=False)), + ('title', models.CharField(max_length=1024)), + ('state', models.CharField(choices=[('N', 'New'), ('A', 'Active'), ('R', 'Resolved'), ('D', 'Deleted'), ('CL', 'Closed'), ('NS', 'Not Specified')], default='N', max_length=2)), + ('finding', models.ManyToManyField(to='triage.Finding')), + ('notes', models.ManyToManyField(to='triage.Note')), + ('tool', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='triage.tool')), + ], + ), + ] diff --git a/omega/triage-portal/src/triage/migrations/0004_tooldefect_assigned_to.py b/omega/triage-portal/src/triage/migrations/0004_tooldefect_assigned_to.py new file mode 100644 index 00000000..9251b693 --- /dev/null +++ b/omega/triage-portal/src/triage/migrations/0004_tooldefect_assigned_to.py @@ -0,0 +1,21 @@ +# Generated by Django 3.2.9 on 2021-11-27 00:25 + +from django.conf import settings +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ('triage', '0003_auto_20211127_0020'), + ] + + operations = [ + migrations.AddField( + model_name='tooldefect', + name='assigned_to', + field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, to=settings.AUTH_USER_MODEL), + ), + ] diff --git a/omega/triage-portal/src/triage/migrations/0005_tool_friendly_name.py b/omega/triage-portal/src/triage/migrations/0005_tool_friendly_name.py new file mode 100644 index 00000000..f17b0ea3 --- /dev/null +++ b/omega/triage-portal/src/triage/migrations/0005_tool_friendly_name.py @@ -0,0 +1,18 @@ +# Generated by Django 3.2.9 on 2021-11-27 01:32 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('triage', '0004_tooldefect_assigned_to'), + ] + + operations = [ + migrations.AddField( + model_name='tool', + name='friendly_name', + field=models.CharField(blank=True, max_length=128, null=True), + ), + ] diff --git a/omega/triage-portal/src/triage/migrations/0006_remove_filter_definition_remove_filter_name_and_more.py b/omega/triage-portal/src/triage/migrations/0006_remove_filter_definition_remove_filter_name_and_more.py new file mode 100644 index 00000000..26c5ae80 --- /dev/null +++ b/omega/triage-portal/src/triage/migrations/0006_remove_filter_definition_remove_filter_name_and_more.py @@ -0,0 +1,47 @@ +# Generated by Django 4.0 on 2021-12-07 22:37 + +from django.db import migrations, models +import taggit.managers + + +class Migration(migrations.Migration): + + dependencies = [ + ('taggit', '0004_alter_taggeditem_content_type_alter_taggeditem_tag'), + ('triage', '0005_tool_friendly_name'), + ] + + operations = [ + migrations.RemoveField( + model_name='filter', + name='definition', + ), + migrations.RemoveField( + model_name='filter', + name='name', + ), + migrations.RemoveField( + model_name='filter', + name='project', + ), + migrations.AddField( + model_name='filter', + name='action', + field=models.TextField(blank=True, null=True), + ), + migrations.AddField( + model_name='filter', + name='condition', + field=models.TextField(blank=True, null=True), + ), + migrations.AddField( + model_name='filter', + name='title', + field=models.CharField(blank=True, max_length=255, null=True), + ), + migrations.AddField( + model_name='finding', + name='tags', + field=taggit.managers.TaggableManager(help_text='A comma-separated list of tags.', through='taggit.TaggedItem', to='taggit.Tag', verbose_name='Tags'), + ), + ] diff --git a/omega/triage-portal/src/triage/migrations/0007_filter_uuid.py b/omega/triage-portal/src/triage/migrations/0007_filter_uuid.py new file mode 100644 index 00000000..d0b0d8ed --- /dev/null +++ b/omega/triage-portal/src/triage/migrations/0007_filter_uuid.py @@ -0,0 +1,19 @@ +# Generated by Django 4.0 on 2021-12-08 02:22 + +from django.db import migrations, models +import uuid + + +class Migration(migrations.Migration): + + dependencies = [ + ('triage', '0006_remove_filter_definition_remove_filter_name_and_more'), + ] + + operations = [ + migrations.AddField( + model_name='filter', + name='uuid', + field=models.UUIDField(db_index=True, default=uuid.uuid4, editable=False), + ), + ] diff --git a/omega/triage-portal/src/triage/migrations/0008_alter_finding_managers_remove_finding_analyst_impact_and_more.py b/omega/triage-portal/src/triage/migrations/0008_alter_finding_managers_remove_finding_analyst_impact_and_more.py new file mode 100644 index 00000000..451e5edc --- /dev/null +++ b/omega/triage-portal/src/triage/migrations/0008_alter_finding_managers_remove_finding_analyst_impact_and_more.py @@ -0,0 +1,37 @@ +# Generated by Django 4.0 on 2021-12-12 01:22 + +from django.db import migrations, models +import django.db.models.manager + + +class Migration(migrations.Migration): + + dependencies = [ + ('triage', '0007_filter_uuid'), + ] + + operations = [ + migrations.AlterModelManagers( + name='finding', + managers=[ + ('active_findings', django.db.models.manager.Manager()), + ], + ), + migrations.RemoveField( + model_name='finding', + name='analyst_impact', + ), + migrations.RemoveField( + model_name='finding', + name='impact_context', + ), + migrations.RemoveField( + model_name='finding', + name='impact_usage', + ), + migrations.AddField( + model_name='finding', + name='estimated_impact', + field=models.PositiveIntegerField(blank=True, null=True), + ), + ] diff --git a/omega/triage-portal/src/triage/migrations/0009_tooldefect_priority_tooldefect_tags.py b/omega/triage-portal/src/triage/migrations/0009_tooldefect_priority_tooldefect_tags.py new file mode 100644 index 00000000..aa3c7753 --- /dev/null +++ b/omega/triage-portal/src/triage/migrations/0009_tooldefect_priority_tooldefect_tags.py @@ -0,0 +1,25 @@ +# Generated by Django 4.0 on 2021-12-13 21:00 + +from django.db import migrations, models +import taggit.managers + + +class Migration(migrations.Migration): + + dependencies = [ + ('taggit', '0004_alter_taggeditem_content_type_alter_taggeditem_tag'), + ('triage', '0008_alter_finding_managers_remove_finding_analyst_impact_and_more'), + ] + + operations = [ + migrations.AddField( + model_name='tooldefect', + name='priority', + field=models.PositiveSmallIntegerField(default=0), + ), + migrations.AddField( + model_name='tooldefect', + name='tags', + field=taggit.managers.TaggableManager(help_text='A comma-separated list of tags.', through='taggit.TaggedItem', to='taggit.Tag', verbose_name='Tags'), + ), + ] diff --git a/omega/triage-portal/src/triage/migrations/0010_tooldefect_description.py b/omega/triage-portal/src/triage/migrations/0010_tooldefect_description.py new file mode 100644 index 00000000..bdb02c25 --- /dev/null +++ b/omega/triage-portal/src/triage/migrations/0010_tooldefect_description.py @@ -0,0 +1,18 @@ +# Generated by Django 4.0 on 2021-12-13 21:07 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('triage', '0009_tooldefect_priority_tooldefect_tags'), + ] + + operations = [ + migrations.AddField( + model_name='tooldefect', + name='description', + field=models.TextField(blank=True, null=True), + ), + ] diff --git a/omega/triage-portal/src/triage/migrations/0011_filter_last_executed.py b/omega/triage-portal/src/triage/migrations/0011_filter_last_executed.py new file mode 100644 index 00000000..dc788156 --- /dev/null +++ b/omega/triage-portal/src/triage/migrations/0011_filter_last_executed.py @@ -0,0 +1,18 @@ +# Generated by Django 4.0 on 2021-12-19 05:05 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('triage', '0010_tooldefect_description'), + ] + + operations = [ + migrations.AddField( + model_name='filter', + name='last_executed', + field=models.DateTimeField(blank=True, null=True), + ), + ] diff --git a/omega/triage-portal/src/triage/migrations/0012_filecontent_remove_finding_file_path_file_and_more.py b/omega/triage-portal/src/triage/migrations/0012_filecontent_remove_finding_file_path_file_and_more.py new file mode 100644 index 00000000..bc69d624 --- /dev/null +++ b/omega/triage-portal/src/triage/migrations/0012_filecontent_remove_finding_file_path_file_and_more.py @@ -0,0 +1,48 @@ +# Generated by Django 4.0 on 2021-12-23 06:23 + +from django.db import migrations, models +import django.db.models.deletion +import uuid + + +class Migration(migrations.Migration): + + dependencies = [ + ('triage', '0011_filter_last_executed'), + ] + + operations = [ + migrations.CreateModel( + name='FileContent', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('hash', models.BinaryField(db_index=True, max_length=32)), + ('content_type', models.CharField(blank=True, db_index=True, max_length=64, null=True)), + ('data', models.BinaryField(blank=True, null=True)), + ], + ), + migrations.RemoveField( + model_name='finding', + name='file_path', + ), + migrations.CreateModel( + name='File', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('uuid', models.UUIDField(db_index=True, default=uuid.uuid4, editable=False)), + ('name', models.CharField(db_index=True, max_length=512)), + ('path', models.CharField(db_index=True, max_length=4096)), + ('content', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='triage.filecontent')), + ], + ), + migrations.AddField( + model_name='finding', + name='file', + field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, to='triage.file'), + ), + migrations.AddField( + model_name='scan', + name='files', + field=models.ManyToManyField(blank=True, to='triage.File'), + ), + ] diff --git a/omega/triage-portal/src/triage/migrations/0013_projectversion_files.py b/omega/triage-portal/src/triage/migrations/0013_projectversion_files.py new file mode 100644 index 00000000..c1fb4d87 --- /dev/null +++ b/omega/triage-portal/src/triage/migrations/0013_projectversion_files.py @@ -0,0 +1,18 @@ +# Generated by Django 4.0 on 2021-12-23 07:05 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('triage', '0012_filecontent_remove_finding_file_path_file_and_more'), + ] + + operations = [ + migrations.AddField( + model_name='projectversion', + name='files', + field=models.ManyToManyField(blank=True, to='triage.File'), + ), + ] diff --git a/omega/triage-portal/src/triage/migrations/0014_remove_finding_scan_finding_normalized_title_and_more.py b/omega/triage-portal/src/triage/migrations/0014_remove_finding_scan_finding_normalized_title_and_more.py new file mode 100644 index 00000000..28c426e9 --- /dev/null +++ b/omega/triage-portal/src/triage/migrations/0014_remove_finding_scan_finding_normalized_title_and_more.py @@ -0,0 +1,48 @@ +# Generated by Django 4.0 on 2021-12-23 23:25 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('triage', '0013_projectversion_files'), + ] + + operations = [ + migrations.RemoveField( + model_name='finding', + name='scan', + ), + migrations.AddField( + model_name='finding', + name='normalized_title', + field=models.CharField(blank=True, max_length=1024, null=True), + ), + migrations.AddField( + model_name='finding', + name='project_version', + field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, to='triage.projectversion'), + ), + migrations.AddField( + model_name='finding', + name='tool', + field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, to='triage.tool'), + ), + migrations.AlterField( + model_name='file', + name='content', + field=models.ForeignKey(editable=False, on_delete=django.db.models.deletion.CASCADE, to='triage.filecontent'), + ), + migrations.AlterField( + model_name='projectversion', + name='files', + field=models.ManyToManyField(blank=True, editable=False, to='triage.File'), + ), + migrations.AlterField( + model_name='tool', + name='version', + field=models.CharField(blank=True, max_length=64, null=True), + ), + ] diff --git a/omega/triage-portal/src/triage/migrations/0015_alter_finding_analyst_severity_level_and_more.py b/omega/triage-portal/src/triage/migrations/0015_alter_finding_analyst_severity_level_and_more.py new file mode 100644 index 00000000..983b3a2e --- /dev/null +++ b/omega/triage-portal/src/triage/migrations/0015_alter_finding_analyst_severity_level_and_more.py @@ -0,0 +1,43 @@ +# Generated by Django 4.0 on 2021-12-24 16:40 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('triage', '0014_remove_finding_scan_finding_normalized_title_and_more'), + ] + + operations = [ + migrations.AlterField( + model_name='finding', + name='analyst_severity_level', + field=models.CharField(choices=[('NS', 'Not Specified'), ('NO', 'None'), ('IN', 'Informational'), ('VL', 'Very Low'), ('L', 'Low'), ('M', 'Medium'), ('H', 'High'), ('VH', 'Very High')], db_index=True, default='NS', max_length=2), + ), + migrations.AlterField( + model_name='finding', + name='normalized_title', + field=models.CharField(blank=True, db_index=True, max_length=1024, null=True), + ), + migrations.AlterField( + model_name='finding', + name='severity_level', + field=models.CharField(choices=[('NS', 'Not Specified'), ('NO', 'None'), ('IN', 'Informational'), ('VL', 'Very Low'), ('L', 'Low'), ('M', 'Medium'), ('H', 'High'), ('VH', 'Very High')], db_index=True, default='NS', max_length=2), + ), + migrations.AlterField( + model_name='finding', + name='state', + field=models.CharField(choices=[('N', 'New'), ('A', 'Active'), ('R', 'Resolved'), ('D', 'Deleted'), ('CL', 'Closed'), ('NS', 'Not Specified')], db_index=True, default='N', max_length=2), + ), + migrations.AlterField( + model_name='finding', + name='title', + field=models.CharField(db_index=True, max_length=1024), + ), + migrations.AlterField( + model_name='projectversion', + name='files', + field=models.ManyToManyField(blank=True, to='triage.File'), + ), + ] diff --git a/omega/triage-portal/src/triage/migrations/0016_rename_finding_tooldefect_findings.py b/omega/triage-portal/src/triage/migrations/0016_rename_finding_tooldefect_findings.py new file mode 100644 index 00000000..c55c3216 --- /dev/null +++ b/omega/triage-portal/src/triage/migrations/0016_rename_finding_tooldefect_findings.py @@ -0,0 +1,18 @@ +# Generated by Django 4.0 on 2021-12-25 03:58 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('triage', '0015_alter_finding_analyst_severity_level_and_more'), + ] + + operations = [ + migrations.RenameField( + model_name='tooldefect', + old_name='finding', + new_name='findings', + ), + ] diff --git a/omega/triage-portal/src/triage/migrations/0017_tooldefect_created_at_tooldefect_created_by_and_more.py b/omega/triage-portal/src/triage/migrations/0017_tooldefect_created_at_tooldefect_created_by_and_more.py new file mode 100644 index 00000000..70c3277b --- /dev/null +++ b/omega/triage-portal/src/triage/migrations/0017_tooldefect_created_at_tooldefect_created_by_and_more.py @@ -0,0 +1,117 @@ +# Generated by Django 4.0 on 2021-12-25 18:36 + +from django.db import migrations, models +import django.db.models.deletion +import django.utils.timezone + + +class Migration(migrations.Migration): + + dependencies = [ + ('auth', '0012_alter_user_first_name_max_length'), + ('triage', '0016_rename_finding_tooldefect_findings'), + ] + + operations = [ + migrations.AddField( + model_name='tooldefect', + name='created_at', + field=models.DateTimeField(auto_now_add=True, default=django.utils.timezone.now), + preserve_default=False, + ), + migrations.AddField( + model_name='tooldefect', + name='created_by', + field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='+', to='auth.user'), + ), + migrations.AddField( + model_name='tooldefect', + name='updated_at', + field=models.DateTimeField(auto_now=True), + ), + migrations.AddField( + model_name='tooldefect', + name='updated_by', + field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='+', to='auth.user'), + ), + migrations.AlterField( + model_name='case', + name='created_by', + field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='+', to='auth.user'), + ), + migrations.AlterField( + model_name='case', + name='updated_by', + field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='+', to='auth.user'), + ), + migrations.AlterField( + model_name='filter', + name='created_by', + field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='+', to='auth.user'), + ), + migrations.AlterField( + model_name='filter', + name='updated_by', + field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='+', to='auth.user'), + ), + migrations.AlterField( + model_name='finding', + name='created_by', + field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='+', to='auth.user'), + ), + migrations.AlterField( + model_name='finding', + name='updated_by', + field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='+', to='auth.user'), + ), + migrations.AlterField( + model_name='note', + name='created_by', + field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='+', to='auth.user'), + ), + migrations.AlterField( + model_name='note', + name='updated_by', + field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='+', to='auth.user'), + ), + migrations.AlterField( + model_name='project', + name='created_by', + field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='+', to='auth.user'), + ), + migrations.AlterField( + model_name='project', + name='updated_by', + field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='+', to='auth.user'), + ), + migrations.AlterField( + model_name='projectversion', + name='created_by', + field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='+', to='auth.user'), + ), + migrations.AlterField( + model_name='projectversion', + name='updated_by', + field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='+', to='auth.user'), + ), + migrations.AlterField( + model_name='scan', + name='created_by', + field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='+', to='auth.user'), + ), + migrations.AlterField( + model_name='scan', + name='updated_by', + field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='+', to='auth.user'), + ), + migrations.AlterField( + model_name='tool', + name='created_by', + field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='+', to='auth.user'), + ), + migrations.AlterField( + model_name='tool', + name='updated_by', + field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='+', to='auth.user'), + ), + ] diff --git a/omega/triage-portal/src/triage/migrations/0018_alter_tooldefect_managers_and_more.py b/omega/triage-portal/src/triage/migrations/0018_alter_tooldefect_managers_and_more.py new file mode 100644 index 00000000..5b91984e --- /dev/null +++ b/omega/triage-portal/src/triage/migrations/0018_alter_tooldefect_managers_and_more.py @@ -0,0 +1,39 @@ +# Generated by Django 4.0 on 2021-12-31 06:49 + +from django.db import migrations, models +import django.db.models.manager + + +class Migration(migrations.Migration): + + dependencies = [ + ('triage', '0017_tooldefect_created_at_tooldefect_created_by_and_more'), + ] + + operations = [ + migrations.AlterModelManagers( + name='tooldefect', + managers=[ + ('active_tool_defects', django.db.models.manager.Manager()), + ], + ), + migrations.RenameField( + model_name='case', + old_name='report_foreign_reference', + new_name='reporting_reference', + ), + migrations.RenameField( + model_name='case', + old_name='resolved_dt', + new_name='resolved_actual_dt', + ), + migrations.RemoveField( + model_name='case', + name='assigned_dt', + ), + migrations.AlterField( + model_name='case', + name='reporting_partner', + field=models.CharField(blank=True, choices=[('N', 'None'), ('GS', 'GitHub Security Lab'), ('CT', 'CERT'), ('MS', 'Microsoft Security Response Center'), ('NS', 'Not Specified')], default='NS', max_length=2, null=True), + ), + ] diff --git a/omega/triage-portal/src/triage/migrations/0019_case_reported_dt.py b/omega/triage-portal/src/triage/migrations/0019_case_reported_dt.py new file mode 100644 index 00000000..dac70a07 --- /dev/null +++ b/omega/triage-portal/src/triage/migrations/0019_case_reported_dt.py @@ -0,0 +1,18 @@ +# Generated by Django 4.0 on 2022-01-01 05:46 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('triage', '0018_alter_tooldefect_managers_and_more'), + ] + + operations = [ + migrations.AddField( + model_name='case', + name='reported_dt', + field=models.DateTimeField(blank=True, null=True), + ), + ] diff --git a/omega/triage-portal/src/triage/migrations/0020_attachment_case_attachments.py b/omega/triage-portal/src/triage/migrations/0020_attachment_case_attachments.py new file mode 100644 index 00000000..9aa4e5fb --- /dev/null +++ b/omega/triage-portal/src/triage/migrations/0020_attachment_case_attachments.py @@ -0,0 +1,27 @@ +# Generated by Django 4.0 on 2022-01-01 06:25 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('triage', '0019_case_reported_dt'), + ] + + operations = [ + migrations.CreateModel( + name='Attachment', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('filename', models.CharField(max_length=1024)), + ('content', models.BinaryField()), + ('content_type', models.CharField(max_length=255)), + ], + ), + migrations.AddField( + model_name='case', + name='attachments', + field=models.ManyToManyField(related_name='cases', to='triage.Attachment'), + ), + ] diff --git a/omega/triage-portal/src/triage/migrations/0021_attachment_uuid.py b/omega/triage-portal/src/triage/migrations/0021_attachment_uuid.py new file mode 100644 index 00000000..e4715926 --- /dev/null +++ b/omega/triage-portal/src/triage/migrations/0021_attachment_uuid.py @@ -0,0 +1,19 @@ +# Generated by Django 4.0 on 2022-01-01 08:14 + +from django.db import migrations, models +import uuid + + +class Migration(migrations.Migration): + + dependencies = [ + ('triage', '0020_attachment_case_attachments'), + ] + + operations = [ + migrations.AddField( + model_name='attachment', + name='uuid', + field=models.UUIDField(db_index=True, default=uuid.uuid4, editable=False), + ), + ] diff --git a/omega/triage-portal/src/triage/migrations/0022_wikiarticle_alter_note_options_wikiarticlerevision_and_more.py b/omega/triage-portal/src/triage/migrations/0022_wikiarticle_alter_note_options_wikiarticlerevision_and_more.py new file mode 100644 index 00000000..592f693f --- /dev/null +++ b/omega/triage-portal/src/triage/migrations/0022_wikiarticle_alter_note_options_wikiarticlerevision_and_more.py @@ -0,0 +1,50 @@ +# Generated by Django 4.0 on 2022-01-01 23:58 + +from django.db import migrations, models +import django.db.models.deletion +import uuid + + +class Migration(migrations.Migration): + + dependencies = [ + ('auth', '0012_alter_user_first_name_max_length'), + ('triage', '0021_attachment_uuid'), + ] + + operations = [ + migrations.CreateModel( + name='WikiArticle', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('uuid', models.UUIDField(db_index=True, default=uuid.uuid4, editable=False)), + ('slug', models.SlugField(max_length=1024)), + ], + ), + migrations.AlterModelOptions( + name='note', + options={'ordering': ['-created_at']}, + ), + migrations.CreateModel( + name='WikiArticleRevision', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('created_at', models.DateTimeField(auto_now_add=True)), + ('updated_at', models.DateTimeField(auto_now=True)), + ('title', models.CharField(max_length=1024)), + ('content', models.TextField(blank=True, null=True)), + ('state', models.CharField(choices=[('N', 'New'), ('A', 'Active'), ('R', 'Resolved'), ('D', 'Deleted'), ('CL', 'Closed'), ('NS', 'Not Specified')], default='N', max_length=2)), + ('created_by', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='+', to='auth.user')), + ('parent', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='revisions', to='triage.wikiarticle')), + ('updated_by', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='+', to='auth.user')), + ], + options={ + 'abstract': False, + }, + ), + migrations.AddField( + model_name='wikiarticle', + name='current', + field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, to='triage.wikiarticlerevision'), + ), + ] diff --git a/omega/triage-portal/src/triage/migrations/0023_rename_parent_wikiarticlerevision_article_and_more.py b/omega/triage-portal/src/triage/migrations/0023_rename_parent_wikiarticlerevision_article_and_more.py new file mode 100644 index 00000000..5e820e28 --- /dev/null +++ b/omega/triage-portal/src/triage/migrations/0023_rename_parent_wikiarticlerevision_article_and_more.py @@ -0,0 +1,24 @@ +# Generated by Django 4.0 on 2022-01-02 04:18 + +from django.db import migrations, models +import uuid + + +class Migration(migrations.Migration): + + dependencies = [ + ('triage', '0022_wikiarticle_alter_note_options_wikiarticlerevision_and_more'), + ] + + operations = [ + migrations.RenameField( + model_name='wikiarticlerevision', + old_name='parent', + new_name='article', + ), + migrations.AddField( + model_name='wikiarticlerevision', + name='uuid', + field=models.UUIDField(db_index=True, default=uuid.uuid4, editable=False), + ), + ] diff --git a/omega/triage-portal/src/triage/migrations/0024_wikiarticlerevision_change_comment_and_more.py b/omega/triage-portal/src/triage/migrations/0024_wikiarticlerevision_change_comment_and_more.py new file mode 100644 index 00000000..9f8e2158 --- /dev/null +++ b/omega/triage-portal/src/triage/migrations/0024_wikiarticlerevision_change_comment_and_more.py @@ -0,0 +1,28 @@ +# Generated by Django 4.0 on 2022-01-02 04:59 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('triage', '0023_rename_parent_wikiarticlerevision_article_and_more'), + ] + + operations = [ + migrations.AddField( + model_name='wikiarticlerevision', + name='change_comment', + field=models.CharField(blank=True, max_length=512, null=True), + ), + migrations.AlterField( + model_name='wikiarticle', + name='slug', + field=models.SlugField(unique=True), + ), + migrations.AlterField( + model_name='wikiarticlerevision', + name='state', + field=models.CharField(choices=[('N', 'New'), ('A', 'Active'), ('R', 'Resolved'), ('D', 'Deleted'), ('CL', 'Closed'), ('NS', 'Not Specified')], default='A', max_length=2), + ), + ] diff --git a/omega/triage-portal/src/triage/migrations/0025_alter_wikiarticle_options_and_more.py b/omega/triage-portal/src/triage/migrations/0025_alter_wikiarticle_options_and_more.py new file mode 100644 index 00000000..555f171b --- /dev/null +++ b/omega/triage-portal/src/triage/migrations/0025_alter_wikiarticle_options_and_more.py @@ -0,0 +1,37 @@ +# Generated by Django 4.0 on 2022-01-03 19:31 + +from django.db import migrations, models +import django.db.models.manager + + +class Migration(migrations.Migration): + + dependencies = [ + ('triage', '0024_wikiarticlerevision_change_comment_and_more'), + ] + + operations = [ + migrations.AlterModelOptions( + name='wikiarticle', + options={'ordering': ['slug']}, + ), + migrations.AlterModelOptions( + name='wikiarticlerevision', + options={'ordering': ['-created_at']}, + ), + migrations.AlterModelManagers( + name='wikiarticle', + managers=[ + ('active_wiki_articles', django.db.models.manager.Manager()), + ], + ), + migrations.RemoveField( + model_name='wikiarticlerevision', + name='state', + ), + migrations.AddField( + model_name='wikiarticle', + name='state', + field=models.CharField(choices=[('N', 'New'), ('A', 'Active'), ('R', 'Resolved'), ('D', 'Deleted'), ('CL', 'Closed'), ('NS', 'Not Specified')], default='A', max_length=2), + ), + ] diff --git a/omega/triage-portal/src/triage/migrations/0026_alter_attachment_uuid_alter_case_uuid_and_more.py b/omega/triage-portal/src/triage/migrations/0026_alter_attachment_uuid_alter_case_uuid_and_more.py new file mode 100644 index 00000000..b2ab4728 --- /dev/null +++ b/omega/triage-portal/src/triage/migrations/0026_alter_attachment_uuid_alter_case_uuid_and_more.py @@ -0,0 +1,98 @@ +# Generated by Django 4.1.3 on 2022-12-15 18:00 + +from django.db import migrations, models +import uuid + + +class Migration(migrations.Migration): + + dependencies = [ + ("triage", "0025_alter_wikiarticle_options_and_more"), + ] + + operations = [ + migrations.AlterField( + model_name="attachment", + name="uuid", + field=models.UUIDField( + db_index=True, default=uuid.uuid4, editable=False, unique=True + ), + ), + migrations.AlterField( + model_name="case", + name="uuid", + field=models.UUIDField( + db_index=True, default=uuid.uuid4, editable=False, unique=True + ), + ), + migrations.AlterField( + model_name="file", + name="uuid", + field=models.UUIDField( + db_index=True, default=uuid.uuid4, editable=False, unique=True + ), + ), + migrations.AlterField( + model_name="filter", + name="uuid", + field=models.UUIDField( + db_index=True, default=uuid.uuid4, editable=False, unique=True + ), + ), + migrations.AlterField( + model_name="finding", + name="uuid", + field=models.UUIDField( + db_index=True, default=uuid.uuid4, editable=False, unique=True + ), + ), + migrations.AlterField( + model_name="project", + name="uuid", + field=models.UUIDField( + db_index=True, default=uuid.uuid4, editable=False, unique=True + ), + ), + migrations.AlterField( + model_name="projectversion", + name="uuid", + field=models.UUIDField( + db_index=True, default=uuid.uuid4, editable=False, unique=True + ), + ), + migrations.AlterField( + model_name="scan", + name="uuid", + field=models.UUIDField( + db_index=True, default=uuid.uuid4, editable=False, unique=True + ), + ), + migrations.AlterField( + model_name="tool", + name="uuid", + field=models.UUIDField( + db_index=True, default=uuid.uuid4, editable=False, unique=True + ), + ), + migrations.AlterField( + model_name="tooldefect", + name="uuid", + field=models.UUIDField( + db_index=True, default=uuid.uuid4, editable=False, unique=True + ), + ), + migrations.AlterField( + model_name="wikiarticle", + name="uuid", + field=models.UUIDField( + db_index=True, default=uuid.uuid4, editable=False, unique=True + ), + ), + migrations.AlterField( + model_name="wikiarticlerevision", + name="uuid", + field=models.UUIDField( + db_index=True, default=uuid.uuid4, editable=False, unique=True + ), + ), + ] diff --git a/omega/triage-portal/src/triage/migrations/0027_file_file_type.py b/omega/triage-portal/src/triage/migrations/0027_file_file_type.py new file mode 100644 index 00000000..eacaee88 --- /dev/null +++ b/omega/triage-portal/src/triage/migrations/0027_file_file_type.py @@ -0,0 +1,23 @@ +# Generated by Django 4.1.3 on 2022-12-18 05:33 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("triage", "0026_alter_attachment_uuid_alter_case_uuid_and_more"), + ] + + operations = [ + migrations.AddField( + model_name="file", + name="file_type", + field=models.CharField( + choices=[("S", "Source Code"), ("R", "Scan Result"), ("U", "Unknown")], + db_index=True, + default="U", + max_length=64, + ), + ), + ] diff --git a/omega/triage-portal/src/triage/migrations/0028_remove_file_content_file_content_type_and_more.py b/omega/triage-portal/src/triage/migrations/0028_remove_file_content_file_content_type_and_more.py new file mode 100644 index 00000000..1d098abc --- /dev/null +++ b/omega/triage-portal/src/triage/migrations/0028_remove_file_content_file_content_type_and_more.py @@ -0,0 +1,37 @@ +# Generated by Django 4.1.3 on 2022-12-18 07:30 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("triage", "0027_file_file_type"), + ] + + operations = [ + migrations.RemoveField( + model_name="file", + name="content", + ), + migrations.AddField( + model_name="file", + name="content_type", + field=models.CharField(blank=True, db_index=True, max_length=64, null=True), + ), + migrations.AddField( + model_name="file", + name="storage_key", + field=models.CharField(blank=True, db_index=True, max_length=64, null=True), + ), + migrations.AlterField( + model_name="file", + name="file_type", + field=models.CharField( + choices=[("S", "Source Code"), ("R", "Scan Result"), ("U", "Unknown")], + db_index=True, + default="U", + max_length=16, + ), + ), + ] diff --git a/omega/triage-portal/src/triage/migrations/0029_rename_storage_key_file_file_key.py b/omega/triage-portal/src/triage/migrations/0029_rename_storage_key_file_file_key.py new file mode 100644 index 00000000..e41fde0a --- /dev/null +++ b/omega/triage-portal/src/triage/migrations/0029_rename_storage_key_file_file_key.py @@ -0,0 +1,18 @@ +# Generated by Django 4.1.3 on 2022-12-18 07:35 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ("triage", "0028_remove_file_content_file_content_type_and_more"), + ] + + operations = [ + migrations.RenameField( + model_name="file", + old_name="storage_key", + new_name="file_key", + ), + ] diff --git a/omega/triage-portal/src/triage/migrations/__init__.py b/omega/triage-portal/src/triage/migrations/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/omega/triage-portal/src/triage/models/File.py b/omega/triage-portal/src/triage/models/File.py new file mode 100644 index 00000000..306a992d --- /dev/null +++ b/omega/triage-portal/src/triage/models/File.py @@ -0,0 +1,65 @@ +import hashlib +import base64 +import logging +import uuid + +from django.db import models +from django.utils.translation import gettext_lazy as _ + +from core import settings +from triage.models import BaseTimestampedModel, BaseUserTrackedModel, WorkItemState + +logger = logging.getLogger(__name__) + + +FILE_TYPES = ( + ("source", _("Source")), + ("binary", _("Binary")), + ("other", _("Other")), + ("unknown", _("Unknown")), + ("", _("Unknown")), +) + + +class File(models.Model): + """ + Represents a file that is associated with an analyzed project. + """ + + class FileType(models.TextChoices): + SOURCE_CODE = "S", _("Source Code") + SCAN_RESULT = "R", _("Scan Result") + UNKNOWN = "U", _("Unknown") + + uuid = models.UUIDField( + default=uuid.uuid4, editable=False, db_index=True, unique=True + ) + name = models.CharField(max_length=512, db_index=True) + path = models.CharField(max_length=4096, db_index=True) + file_type = models.CharField( + max_length=16, db_index=True, choices=FileType.choices, default=FileType.UNKNOWN + ) + content_type = models.CharField(max_length=64, db_index=True, null=True, blank=True) + file_key = models.CharField(max_length=64, db_index=True, null=True, blank=True) + + def __str__(self): + return str(self.path) + + +class FileContent(models.Model): + """Represents file content.""" + + hash = models.BinaryField(max_length=32, db_index=True) + content_type = models.CharField(max_length=64, db_index=True, null=True, blank=True) + data = models.BinaryField(null=True, blank=True) + + def __str__(self): + return base64.b64encode(self.hash).decode("ascii") + + @classmethod + def generate_hash(cls, data: bytes, encode=False) -> str | bytes: + """Generates a hash for the given data.""" + digest = hashlib.sha256(data, usedforsecurity=True).digest() + if encode: + return base64.b64encode(digest).decode("ascii") + return digest diff --git a/omega/triage-portal/src/triage/models/__init__.py b/omega/triage-portal/src/triage/models/__init__.py new file mode 100644 index 00000000..96c33759 --- /dev/null +++ b/omega/triage-portal/src/triage/models/__init__.py @@ -0,0 +1,30 @@ +# -*- coding: utf-8 -*- +""" +This file is required so that individual modules can be referenced from files within +this directory. +""" + +from triage.models.attachment import Attachment + +# import triage.models.project +# import triage.models.tool_defect +from triage.models.base import BaseTimestampedModel, BaseUserTrackedModel, WorkItemState +from triage.models.case import Case +from triage.models.File import File, FileContent +from triage.models.filter import Filter +from triage.models.finding import Finding +from triage.models.note import Note +from triage.models.project import Project, ProjectVersion +from triage.models.scan import Scan +from triage.models.tool import Tool +from triage.models.tool_defect import ToolDefect +from triage.models.triage import TriageRule +from triage.models.wiki import WikiArticle, WikiArticleRevision + +# class File(models.Model): +# uuid = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) +# scan = models.ForeignKey(Scan, on_delete=models.CASCADE)# + +# content = models.FileField(upload_to="file_archive") +# content_hash = models.CharField(max_length=128, db_index=True) +# path = models.CharField(max_length=4096) diff --git a/omega/triage-portal/src/triage/models/attachment.py b/omega/triage-portal/src/triage/models/attachment.py new file mode 100644 index 00000000..3a48e556 --- /dev/null +++ b/omega/triage-portal/src/triage/models/attachment.py @@ -0,0 +1,19 @@ +import logging +import uuid + +from django.db import models + +logger = logging.getLogger(__name__) + + +class Attachment(models.Model): + uuid = models.UUIDField(default=uuid.uuid4, editable=False, db_index=True, unique=True) + filename = models.CharField(max_length=1024) + content = models.BinaryField() + content_type = models.CharField(max_length=255) + + def __str__(self): + return self.filename + + def get_absolute_url(self): + return f"/attachment/{self.uuid}" diff --git a/omega/triage-portal/src/triage/models/base.py b/omega/triage-portal/src/triage/models/base.py new file mode 100644 index 00000000..16c80644 --- /dev/null +++ b/omega/triage-portal/src/triage/models/base.py @@ -0,0 +1,91 @@ +from django.contrib.auth.models import User +from django.db import models +from django.utils.translation import gettext_lazy as _ + +from core import settings + + +class BaseTimestampedModel(models.Model): + """A mixin that adds a created/updated date field to a model.""" + + created_at = models.DateTimeField(auto_now_add=True) + updated_at = models.DateTimeField(auto_now=True) + + class Meta: + abstract = True + + +class BaseUserTrackedModel(models.Model): + """A mixin that adds a created/updated by field to a model.""" + + created_by = models.ForeignKey( + settings.AUTH_USER_MODEL, on_delete=models.CASCADE, related_name="+", null=True, blank=True + ) + updated_by = models.ForeignKey( + settings.AUTH_USER_MODEL, on_delete=models.CASCADE, related_name="+", null=True, blank=True + ) + + class Meta: + abstract = True + + +class WorkItemState(models.TextChoices): + """ + This class contains work item state fields, which can be used as choice values + in other models. + """ + + NEW = "N", _("New") + ACTIVE = "A", _("Active") + RESOLVED = "R", _("Resolved") + DELETED = "D", _("Deleted") + CLOSED = "CL", _("Closed") + NOT_SPECIFIED = "NS", _("Not Specified") + + @classmethod + def parse(cls, state: str, strict: bool = False) -> "WorkItemState": + """Convert a string into a WorkItemState. + + Args: + state (str): The string to parse. + strict (bool): If True, then only parse strict equality (case-insensitive) values. + + Returns: + WorkItemState: The WorkItemState corresponding to the string. + If the string is not a valid WorkItemState, returns WorkItemState.NOT_SPECIFIED. + + If strict is False (default), then this method maps related strings to a close + approximation, so "very high" and "critical" are both mapped to "VERY_HIGH", etc. + """ + if state is None or not isinstance(state, str): + return cls.NOT_SPECIFIED + state = state.lower().strip() + if strict: + if state == "new": + return cls.NEW + if state == "active": + return cls.ACTIVE + if state == "resolved": + return cls.RESOLVED + if state == "deleted": + return cls.DELETED + if state == "closed": + return cls.CLOSED + if state == "not specified": + return cls.NOT_SPECIFIED + if state == "none": + return cls.NOT_SPECIFIED + else: + if state in ["new", "n"]: + return cls.NEW + if state in ["active", "a"]: + return cls.ACTIVE + if state in ["resolved", "r"]: + return cls.RESOLVED + if state in ["deleted", "d"]: + return cls.DELETED + if state in ["closed", "c", "cl"]: + return cls.CLOSED + if state in ["not specified", "ns", "none"]: + return cls.NOT_SPECIFIED + return cls.NOT_SPECIFIED diff --git a/omega/triage-portal/src/triage/models/case.py b/omega/triage-portal/src/triage/models/case.py new file mode 100644 index 00000000..38ffc786 --- /dev/null +++ b/omega/triage-portal/src/triage/models/case.py @@ -0,0 +1,56 @@ +import logging +import uuid + +from django.db import models +from django.utils.translation import gettext_lazy as _ + +from core import settings +from triage.models import BaseTimestampedModel, BaseUserTrackedModel, WorkItemState + +logger = logging.getLogger(__name__) + + +class Case(BaseTimestampedModel, BaseUserTrackedModel): + """ + Represents a case that is being reported to a maintainer for a fix. + """ + + class CasePartner(models.TextChoices): + NONE = "N", _("None") + GITHUB_SECURITY_LAB = "GS", _("GitHub Security Lab") + CERT = "CT", _("CERT") + MSRC = "MS", _("Microsoft Security Response Center") + NOT_SPECIFIED = "NS", _("Not Specified") + + uuid = models.UUIDField(default=uuid.uuid4, editable=False, db_index=True, unique=True) + findings = models.ManyToManyField("Finding", related_name="cases") + state = models.CharField(max_length=2, choices=WorkItemState.choices, default=WorkItemState.NEW) + title = models.CharField(max_length=1024) + description = models.TextField(null=True, blank=True) + assigned_to = models.ForeignKey( + settings.AUTH_USER_MODEL, + null=True, + blank=True, + related_name="assigned_cases", + on_delete=models.SET_NULL, + ) + reported_to = models.CharField(max_length=2048, null=True, blank=True) + reported_dt = models.DateTimeField(null=True, blank=True) + reporting_partner = models.CharField( + max_length=2, + choices=CasePartner.choices, + default=CasePartner.NOT_SPECIFIED, + null=True, + blank=True, + ) + reporting_reference = models.CharField(max_length=1024, null=True, blank=True) + resolved_target_dt = models.DateTimeField(null=True, blank=True) + resolved_actual_dt = models.DateTimeField(null=True, blank=True) + notes = models.ManyToManyField("Note", related_name="cases") + attachments = models.ManyToManyField("Attachment", related_name="cases") + + def __str__(self): + return self.title + + def get_absolute_url(self): + return f"/case/{self.uuid}" diff --git a/omega/triage-portal/src/triage/models/filter.py b/omega/triage-portal/src/triage/models/filter.py new file mode 100644 index 00000000..ea47ef81 --- /dev/null +++ b/omega/triage-portal/src/triage/models/filter.py @@ -0,0 +1,157 @@ +import datetime +import logging +import uuid +from typing import Any + +from django.contrib.auth.models import User +from django.core.exceptions import ValidationError +from django.db import models +from django.utils import timezone +from django.utils.translation import gettext_lazy as _ +from wrapt import synchronized + +from triage.models import BaseTimestampedModel, BaseUserTrackedModel + +logger = logging.getLogger(__name__) + + +class Filter(BaseTimestampedModel, BaseUserTrackedModel): + """ + Represents a filter that is automatically applied to a Finding. + + The purpose of this is to allow users to improve the quality of findings in a + programmatic way. + + All filters are run automatically when a Finding is created or modified. + If a filter is modified, it will be re-run on all Findings. + """ + + class FilterEvent(models.TextChoices): + ON_FINDING_NEW = "FN", _("On New Finding") + ON_FINDING_MODIFIED = "FM", _("On Modified Finding") + + class RuleType(models.TextChoices): + PYTHON_FUNCTION = "PY", _("Python Function") + + uuid = models.UUIDField(default=uuid.uuid4, editable=False, db_index=True, unique=True) + title = models.CharField(max_length=255, null=True, blank=True) + condition = models.TextField(null=True, blank=True) + action = models.TextField(null=True, blank=True) + active = models.BooleanField(default=True) + priority = models.PositiveSmallIntegerField(default=1000) + last_executed = models.DateTimeField(null=True, blank=True) + + def __str__(self): + return self.title + + def clean(self): + """ + Validate that the condition and action are valid Python code. + """ + if not Filter.get_filter_function(self.condition, "condition"): + raise ValidationError(_("Condition is not valid Python code.")) + + if not Filter.get_filter_function(self.action, "action"): + raise ValidationError(_("Action is not valid Python code.")) + + if not self.title: + raise ValidationError(_("Title is required.")) + + if self.priority < 0 or self.priority > 1000: + raise ValidationError(_("Priority must be between 0 and 1000.")) + + @classmethod + def get_filter_function(cls, function_body: str, type: str) -> Any | None: + """ + Creates a dynamic function with the body provided, of the type provided. + In addition to sanity checking, this function also ensures that the function + is "safe" using the is_safe_function call. + + Args: + function_body: The body of the function to create. + type: The type of function to create, either "condition" or "action". + + Returns: + The function created, or None if the function body is invalid. + """ + if not function_body or not type: + return None + + try: + function_str = ( + f"def {type}(finding):\n" + + "\n return_value = None\n" + + "\n from triage.models import Finding\n" + + "\n".join([" " + line for line in function_body.splitlines()]) + + "\n return return_value" + ) + logger.debug(function_str) + if Filter.is_safe_function(function_str): + return compile(function_str, type, "exec") + else: + raise Exception("Function is not safe.") + except Exception as msg: + logger.warning("Invalid %s function: %s", type, msg) + return None + + @classmethod + def is_safe_function(self, code): + """ + Helper method to check if a function is safe, meaning whether it uses unsafe + functionality, like exec, eval, imports, or other potentially dangerous strings. + + Since dynamic function creation is inherently dangerous, we don't expect this + to be fool-proof, but it should be good enough for most cases. + + TODO: We need to actually implement this function. + """ + import ast + + body = ast.parse(code) + return True + + @classmethod + def execute_all(cls): + """ + Execute all filters. + """ + for filter in Filter.objects.filter(active=True).order_by("priority"): + filter.execute() + + @synchronized + def execute(self): + """ + Execute the filter's condition and action. + """ + if not self.active: + logger.info("Filter #%d is not active.", self.pk) + return + + from triage.models import Finding + + try: + condition_bytecode = Filter.get_filter_function(self.condition, "condition") + exec(condition_bytecode, globals()) + + action_bytecode = Filter.get_filter_function(self.action, "action") + exec(action_bytecode, globals()) + + for finding in Finding.active_findings.all(): + try: + if condition(finding): # type: ignore (dynamic variable) + logger.debug( + "Executing filter action %s on finding %s", self.title, finding + ) + action(finding) # type: ignore (dynamic valiable) + except Exception as msg: + logger.error( + "Error executing filter action %s on finding %s: %s", + self.title, + finding, + msg, + ) + except Exception as msg: + logger.exception("Error executing filter %s: %s", self.title, msg) + + self.last_executed = timezone.now() + self.save() diff --git a/omega/triage-portal/src/triage/models/finding.py b/omega/triage-portal/src/triage/models/finding.py new file mode 100644 index 00000000..b7b2a97d --- /dev/null +++ b/omega/triage-portal/src/triage/models/finding.py @@ -0,0 +1,188 @@ +import logging +import os +import uuid + +from django.contrib.auth.models import User +from django.db import models +from django.utils.translation import gettext_lazy as _ +from taggit.managers import TaggableManager + +from triage.models import BaseTimestampedModel, BaseUserTrackedModel, WorkItemState + +logger = logging.getLogger(__name__) + + +class ActiveFindingsManager(models.Manager): + def get_queryset(self): + return ( + super() + .get_queryset() + .filter( + state__in=[WorkItemState.NEW, WorkItemState.ACTIVE, WorkItemState.NOT_SPECIFIED] + ) + ) + + +class Finding(BaseTimestampedModel, BaseUserTrackedModel): + class ConfidenceLevel(models.TextChoices): + NOT_SPECIFIED = "NS", _("Not Specified") + VERY_LOW = "VL", _("Very Low") + LOW = "L", _("Low") + MEDIUM = "M", _("Medium") + HIGH = "H", _("High") + VERY_HIGH = "VH", _("Very High") + + class SeverityLevel(models.TextChoices): + NOT_SPECIFIED = "NS", _("Not Specified") + NONE = "NO", _("None") + INFORMATIONAL = "IN", _("Informational") + VERY_LOW = "VL", _("Very Low") + LOW = "L", _("Low") + MEDIUM = "M", _("Medium") + HIGH = "H", _("High") + VERY_HIGH = "VH", _("Very High") + + @classmethod + def parse(cls, severity: str, strict: bool = False) -> "Finding.SeverityLevel": + """Convert a string into a SeverityLevel. + + Args: + severity (str): The string to parse. + strict (bool): If True, then only parse strict equality (case-insensitive) values. + + Returns: + SeverityLevel: The SeverityLevel corresponding to the string. + If the string is not a valid SeverityLevel, returns SeverityLevel.NOT_SPECIFIED. + + If strict is False (default), then this method maps related strings to a close + approximation, so "very high" and "critical" are both mapped to "VERY_HIGH", etc. + """ + if severity is None or not isinstance(severity, str): + return cls.NOT_SPECIFIED + severity = severity.lower().strip() + if strict: + if severity == "very_high": + return cls.VERY_HIGH + if severity == "high": + return cls.HIGH + if severity == "medium": + return cls.MEDIUM + if severity == "low": + return cls.LOW + if severity == "very_low": + return cls.VERY_LOW + if severity == "informational": + return cls.INFORMATIONAL + if severity == "none": + return cls.NONE + else: + if severity in ["critical", "fatal", "very high", "very_high", "veryhigh", "vh"]: + return cls.VERY_HIGH + if severity in ["important", "error", "high", "h"]: + return cls.HIGH + if severity in ["moderate", "warn", "warning", "medium", "m"]: + return cls.MEDIUM + if severity in ["low", "l"]: + return cls.LOW + if severity in ["defense-in-depth", "verylow", "very_low", "very low"]: + return cls.VERY_LOW + if severity in ["info", "informational"]: + return cls.INFORMATIONAL + if severity in ["fp", "false positive", "none"]: + return cls.NONE + return cls.NOT_SPECIFIED + + uuid = models.UUIDField(default=uuid.uuid4, editable=False, db_index=True, unique=True) + project_version = models.ForeignKey( + "ProjectVersion", on_delete=models.CASCADE, null=True, blank=True + ) + + title = models.CharField(max_length=1024, db_index=True) + normalized_title = models.CharField(max_length=1024, null=True, blank=True, db_index=True) + + file = models.ForeignKey("File", null=True, blank=True, on_delete=models.SET_NULL) + file_line = models.PositiveIntegerField(null=True, blank=True) + + # Impact showing how important a finding is to the larger community. + # The larger the number, the higher the impact. Used for sorting. + estimated_impact = models.PositiveIntegerField(null=True, blank=True) + + # Confidence showing how certain a finding is. + confidence = models.CharField( + max_length=2, choices=ConfidenceLevel.choices, default=ConfidenceLevel.NOT_SPECIFIED + ) + + # Severity showing how important a finding is to the security quality of a project. + severity_level = models.CharField( + max_length=2, + choices=SeverityLevel.choices, + default=SeverityLevel.NOT_SPECIFIED, + db_index=True, + ) + analyst_severity_level = models.CharField( + max_length=2, + choices=SeverityLevel.choices, + default=SeverityLevel.NOT_SPECIFIED, + db_index=True, + ) + + state = models.CharField( + max_length=2, choices=WorkItemState.choices, default=WorkItemState.NEW, db_index=True + ) + + tool = models.ForeignKey( + "Tool", null=True, blank=True, on_delete=models.SET_NULL, db_index=True + ) + + # Who the finding is currently assigned to + assigned_to = models.ForeignKey(User, null=True, blank=True, on_delete=models.SET_NULL) + assigned_dt = models.DateTimeField(auto_now_add=True) + + tags = TaggableManager() + + active_findings = ActiveFindingsManager() + objects = models.Manager() + + def __str__(self): + return f"{self.normalized_title} in {self.file}:{self.file_line}" + + def get_absolute_url(self): + return f"/findings/{self.uuid}" + + @property + def get_filename_display(self): + """Render the filename or a placeholder where one does not exist.""" + if self.file: + return self.file.name or "-" + else: + return "-" + + @property + def get_calculated_severity(self): + """Gets the best severity level (analyst estimate taking precedence)""" + if self.analyst_severity_level == Finding.SeverityLevel.NOT_SPECIFIED: + return self.severity_level + return self.analyst_severity_level + + @property + def get_severity_display(self): + """Gets the best severity level (analyst estimate taking precedence)""" + if self.analyst_severity_level == Finding.SeverityLevel.NOT_SPECIFIED: + return self.get_severity_level_display() + return self.get_analyst_severity_level_display() + + @property + def get_impact_display(self): + """Gets the best impact level (analyst estimate taking precedence)""" + if self.estimated_impact is not None: + return self.estimated_impact + else: + return None + + def get_source_code(self): + """Retrieve source code pertaining to this finding.""" + if self.file: + return self.file.content + + logger.debug("No source code available (file is empty)") + return None diff --git a/omega/triage-portal/src/triage/models/note.py b/omega/triage-portal/src/triage/models/note.py new file mode 100644 index 00000000..e177c44e --- /dev/null +++ b/omega/triage-portal/src/triage/models/note.py @@ -0,0 +1,17 @@ +import logging + +from django.db import models + +from triage.models import BaseTimestampedModel, BaseUserTrackedModel + +logger = logging.getLogger(__name__) + + +class Note(BaseTimestampedModel, BaseUserTrackedModel): + content = models.TextField() + + def __str__(self): + return self.content + + class Meta: + ordering = ["-created_at"] diff --git a/omega/triage-portal/src/triage/models/project.py b/omega/triage-portal/src/triage/models/project.py new file mode 100644 index 00000000..d51846c7 --- /dev/null +++ b/omega/triage-portal/src/triage/models/project.py @@ -0,0 +1,74 @@ +import logging +import uuid + +from django.contrib.auth.models import User +from django.db import models +from django.utils.translation import gettext_lazy as _ +from packageurl import PackageURL + +from triage.models import BaseTimestampedModel, BaseUserTrackedModel +from triage.models.base import BaseTimestampedModel, BaseUserTrackedModel, WorkItemState +from triage.util.azure_blob_storage import ( + AzureBlobStorageAccessor, + ToolshedBlobStorageAccessor, +) +from triage.util.general import modify_purl + +logger = logging.getLogger(__name__) + + +class Project(BaseTimestampedModel, BaseUserTrackedModel): + """An abstract project undergoing analysis.""" + + uuid = models.UUIDField(default=uuid.uuid4, editable=False, db_index=True, unique=True) + name = models.CharField(max_length=1024, db_index=True) + package_url = models.CharField(max_length=1024, null=True, blank=True, db_index=True) + metadata = models.JSONField(null=True) + + def __str__(self): + return self.name + + def get_absolute_url(self): + return f"/projects/{self.uuid}" + + +class ProjectVersion(BaseTimestampedModel, BaseUserTrackedModel): + """A version of a project.""" + + uuid = models.UUIDField(default=uuid.uuid4, editable=False, db_index=True, unique=True) + project = models.ForeignKey("Project", on_delete=models.CASCADE) + package_url = models.CharField(max_length=1024, null=True, blank=True, db_index=True) + files = models.ManyToManyField("File", blank=True, editable=True) + metadata = models.JSONField(null=True) + + def __str__(self): + return self.package_url + + def get_absolute_url(self): + return f"/projects/{self.project.uuid}/{self.uuid}" + + @classmethod + def get_or_create_from_package_url( + cls, package_url: PackageURL, created_by: User + ) -> "ProjectVersion": + """Retrieves or create a PackageVersion from the given package_url.""" + if package_url is None: + raise ValueError("'package_url' cannot be None.") + + package_url_no_version = modify_purl(package_url, version=None) + + if package_url.namespace: + package_name = f"{package_url.namespace}/{package_url.name}" + else: + package_name = package_url.name + + project, _ = Project.objects.get_or_create( + package_url=str(package_url_no_version), + defaults={"name": package_name, "created_by": created_by, "updated_by": created_by}, + ) + project_version, _ = cls.objects.get_or_create( + project=project, + package_url=package_url, + defaults={"created_by": created_by, "updated_by": created_by}, + ) + return project_version diff --git a/omega/triage-portal/src/triage/models/scan.py b/omega/triage-portal/src/triage/models/scan.py new file mode 100644 index 00000000..643a38f6 --- /dev/null +++ b/omega/triage-portal/src/triage/models/scan.py @@ -0,0 +1,88 @@ +import logging +import uuid +from typing import Optional + +from django.db import models +from django.utils.translation import gettext_lazy as _ +from packageurl import PackageURL + +from triage.models import BaseTimestampedModel, BaseUserTrackedModel +from triage.util.azure_blob_storage import ( + AzureBlobStorageAccessor, + ToolshedBlobStorageAccessor, +) +from triage.util.general import modify_purl +from triage.util.source_viewer.viewer import SourceViewer + +logger = logging.getLogger(__name__) + + +class Scan(BaseTimestampedModel, BaseUserTrackedModel): + """A scan of a project version.""" + + uuid = models.UUIDField(default=uuid.uuid4, editable=False, db_index=True, unique=True) + project_version = models.ForeignKey("ProjectVersion", on_delete=models.CASCADE) + tool = models.ForeignKey("Tool", on_delete=models.CASCADE) + + artifact_url_base = models.CharField(max_length=1024, null=True, blank=True) + active = models.BooleanField(default=True) + + files = models.ManyToManyField("File", blank=True) + created_dt = models.DateTimeField(auto_now_add=True) + + def __str__(self): + return f"An execution of {self.tool} against {self.project_version}" + + def get_absolute_url(self): + return f"/scan/{self.uuid}" + + def get_file_content(self, filename: str) -> Optional[str]: + if filename is None: + return None + + accessor = ToolshedBlobStorageAccessor(self) + for _filename in accessor.get_all_files(): + print(f"Comparing {_filename} to {filename}") + if _filename == filename: + return accessor.get_file_content(_filename) + return None + + def get_source_files(self) -> list: + """Retreives the source code for this scan.""" + accessor = ToolshedBlobStorageAccessor(self) + return accessor.get_source_files() + + def get_source_code(self, filename: str) -> Optional[str]: + """Retreives the source code for a file.""" + + viewer = SourceViewer(self.project_version.package_url) + res = viewer.get_file(filename) + if res: + return res.get("content") + return None + + def get_file_list(self) -> list: + """Retreives a list of files in the scan.""" + viewer = SourceViewer(self.project_version.package_url) + return viewer.get_file_list() + + def get_blob_list(self) -> list: + """Retreives a list of blobs in the scan.""" + purl = PackageURL.from_string(self.project_version.package_url) + if purl.namespace: + prefix = f"{purl.type}/{purl.namespace}/{purl.name}/{purl.version}" + else: + prefix = f"{purl.type}/{purl.name}/{purl.version}" + + accessor = AzureBlobStorageAccessor(prefix) + return accessor.get_blob_list() + + def get_file_contents(self, filename) -> Optional[str]: + """Retreives the contents of a file.""" + accessor = ToolshedBlobStorageAccessor(self) + return accessor.get_file_contents(filename) + + def get_package_contents(self, filename) -> Optional[str]: + """Retreives the contents of a file.""" + accessor = ToolshedBlobStorageAccessor(self) + return accessor.get_package_contents(filename) diff --git a/omega/triage-portal/src/triage/models/tool.py b/omega/triage-portal/src/triage/models/tool.py new file mode 100644 index 00000000..1098485e --- /dev/null +++ b/omega/triage-portal/src/triage/models/tool.py @@ -0,0 +1,41 @@ +import logging +import uuid + +from django.db import models +from django.utils.translation import gettext_lazy as _ + +from triage.models.base import BaseTimestampedModel, BaseUserTrackedModel, WorkItemState + +logger = logging.getLogger(__name__) + + +class Tool(BaseTimestampedModel, BaseUserTrackedModel): + """A tool used to create a finding.""" + + class ToolType(models.TextChoices): + NOT_SPECIFIED = "NS", _("Not Specified") + MANUAL = "MA", _("Manual") + STATIC_ANALYSIS = "SA", _("Static Analysis") + OTHER = "OT", _("Other") + + uuid = models.UUIDField(default=uuid.uuid4, editable=False, db_index=True, unique=True) + name = models.CharField(max_length=128) + friendly_name = models.CharField(max_length=128, blank=True, null=True) + version = models.CharField(max_length=64, null=True, blank=True) + type = models.CharField(max_length=2, choices=ToolType.choices, default=ToolType.NOT_SPECIFIED) + active = models.BooleanField(default=True) + + def __str__(self) -> str: + parts = [] + if self.friendly_name: + parts.append(self.friendly_name) + else: + parts.append(self.name) + if self.version: + parts.append(self.version) + return " ".join(parts) + + def save(self, *args, **kwargs) -> None: + if self.friendly_name is None: + self.friendly_name = self.name + super().save(*args, **kwargs) diff --git a/omega/triage-portal/src/triage/models/tool_defect.py b/omega/triage-portal/src/triage/models/tool_defect.py new file mode 100644 index 00000000..2ea1c30a --- /dev/null +++ b/omega/triage-portal/src/triage/models/tool_defect.py @@ -0,0 +1,52 @@ +import logging +import uuid + +from django.contrib.auth.models import User +from django.core.cache import cache +from django.db import models +from django.utils.translation import gettext_lazy as _ +from taggit.managers import TaggableManager + +from triage.models import ( + BaseTimestampedModel, + BaseUserTrackedModel, + Note, + Tool, + WorkItemState, +) + +logger = logging.getLogger(__name__) + + +class ActiveToolDefectsManager(models.Manager): + def get_queryset(self): + return ( + super() + .get_queryset() + .filter( + state__in=[WorkItemState.NEW, WorkItemState.ACTIVE, WorkItemState.NOT_SPECIFIED] + ) + ) + + +class ToolDefect(BaseTimestampedModel, BaseUserTrackedModel): + """ + A tool defect is a defect that is filed against a tool. + """ + + tool = models.ForeignKey(Tool, on_delete=models.CASCADE) + uuid = models.UUIDField(default=uuid.uuid4, editable=False, db_index=True, unique=True) + title = models.CharField(max_length=1024) + description = models.TextField(null=True, blank=True) + findings = models.ManyToManyField("Finding") + state = models.CharField(choices=WorkItemState.choices, max_length=2, default=WorkItemState.NEW) + assigned_to = models.ForeignKey(User, null=True, blank=True, on_delete=models.SET_NULL) + priority = models.PositiveSmallIntegerField(default=0) + notes = models.ManyToManyField(Note) + tags = TaggableManager() + + active_tool_defects = ActiveToolDefectsManager() + objects = models.Manager() + + def __str__(self): + return self.title diff --git a/omega/triage-portal/src/triage/models/triage.py b/omega/triage-portal/src/triage/models/triage.py new file mode 100644 index 00000000..528c0288 --- /dev/null +++ b/omega/triage-portal/src/triage/models/triage.py @@ -0,0 +1,38 @@ +import logging + +from django.db import models +from django.utils.translation import gettext_lazy as _ + +from triage.util.general import modify_purl + +logger = logging.getLogger(__name__) + + +class TriageRule(models.Model): + """ + ??? + def applies(finding) -> bool + def action(finding) -> void + + if applies(f): action(f) + """ + + class TriageEvent(models.TextChoices): + ON_FINDING_NEW = "FN", _("On New Finding") + ON_FINDING_MODIFIED = "FM", _("On Modified Finding") + + class RuleType(models.TextChoices): + PYTHON_FUNCTION = "PY", _("Python Function") + + event = models.CharField( + max_length=2, choices=TriageEvent.choices, default=TriageEvent.ON_FINDING_NEW + ) + condition = models.TextField(max_length=2048, null=True, blank=True) + action = models.TextField(max_length=2048, null=True, blank=True) + + active = models.BooleanField(db_index=True, default=True) + priority = models.PositiveSmallIntegerField(default=1000) + + type = models.CharField( + max_length=2, choices=RuleType.choices, default=RuleType.PYTHON_FUNCTION + ) diff --git a/omega/triage-portal/src/triage/models/wiki.py b/omega/triage-portal/src/triage/models/wiki.py new file mode 100644 index 00000000..52ec2fa8 --- /dev/null +++ b/omega/triage-portal/src/triage/models/wiki.py @@ -0,0 +1,77 @@ +import logging +import uuid + +from django.db import models + +from triage.models import BaseTimestampedModel, BaseUserTrackedModel, WorkItemState + +logger = logging.getLogger(__name__) + + +class WikiArticleRevision(BaseTimestampedModel, BaseUserTrackedModel): + uuid = models.UUIDField(default=uuid.uuid4, editable=False, db_index=True, unique=True) + article = models.ForeignKey("WikiArticle", on_delete=models.CASCADE, related_name="revisions") + title = models.CharField(max_length=1024) + content = models.TextField(null=True, blank=True) + change_comment = models.CharField(max_length=512, null=True, blank=True) + + def save(self, *args, **kwargs): + super().save(*args, **kwargs) + self.article.current = self + self.article.save() + + def __str__(self): + return self.title + + def get_absolute_url(self): + return f"/wiki/{self.article.slug}/{self.uuid}" + + def get_absolute_edit_url(self): + return f"/wiki/{self.article.slug}/{self.uuid}/edit" + + class Meta: + ordering = ["-created_at"] + + +class ActiveWikiArticleManager(models.Manager): + def get_queryset(self): + return ( + super() + .get_queryset() + .filter( + state__in=[WorkItemState.NEW, WorkItemState.ACTIVE, WorkItemState.NOT_SPECIFIED] + ) + ) + + +class WikiArticle(models.Model): + uuid = models.UUIDField(default=uuid.uuid4, editable=False, db_index=True, unique=True) + slug = models.SlugField(unique=True) + state = models.CharField( + max_length=2, choices=WorkItemState.choices, default=WorkItemState.ACTIVE + ) + current = models.ForeignKey( + WikiArticleRevision, on_delete=models.CASCADE, null=True, blank=True + ) + + active_wiki_articles = ActiveWikiArticleManager() + objects = models.Manager() + + def __str__(self): + if self.current: + return self.current.title + else: + return "(No article)" + + def get_absolute_url(self): + return f"/wiki/{self.slug}" + + def get_absolute_edit_url(self): + return f"/wiki/{self.slug}/edit" + + @property + def versions(self): + return WikiArticleRevision.objects.filter(article=self).order_by("-created_at") + + class Meta: + ordering = ["slug"] diff --git a/omega/triage-portal/src/triage/static/triage/images/icon-nvd.nist.gov.png b/omega/triage-portal/src/triage/static/triage/images/icon-nvd.nist.gov.png new file mode 100644 index 00000000..de118d1b Binary files /dev/null and b/omega/triage-portal/src/triage/static/triage/images/icon-nvd.nist.gov.png differ diff --git a/omega/triage-portal/src/triage/static/triage/images/icon-searchcode.com.png b/omega/triage-portal/src/triage/static/triage/images/icon-searchcode.com.png new file mode 100644 index 00000000..2f7e71cc Binary files /dev/null and b/omega/triage-portal/src/triage/static/triage/images/icon-searchcode.com.png differ diff --git a/omega/triage-portal/src/triage/static/triage/images/icon-sourcegraph.com.svg b/omega/triage-portal/src/triage/static/triage/images/icon-sourcegraph.com.svg new file mode 100644 index 00000000..a5f3cb0f --- /dev/null +++ b/omega/triage-portal/src/triage/static/triage/images/icon-sourcegraph.com.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/omega/triage-portal/src/triage/static/triage/images/icon-vscode.png b/omega/triage-portal/src/triage/static/triage/images/icon-vscode.png new file mode 100644 index 00000000..37a08476 Binary files /dev/null and b/omega/triage-portal/src/triage/static/triage/images/icon-vscode.png differ diff --git a/omega/triage-portal/src/triage/static/triage/images/nvd b/omega/triage-portal/src/triage/static/triage/images/nvd new file mode 100644 index 00000000..a6415a12 Binary files /dev/null and b/omega/triage-portal/src/triage/static/triage/images/nvd differ diff --git a/omega/triage-portal/src/triage/static/triage/omega.js b/omega/triage-portal/src/triage/static/triage/omega.js new file mode 100644 index 00000000..3c915c71 --- /dev/null +++ b/omega/triage-portal/src/triage/static/triage/omega.js @@ -0,0 +1,296 @@ +$(document).ready(function () { + /* Initialize Bootstrap Components */ + $('[data-toggle="popover"]').popover(); + $('[data-toggle="tooltip"]').tooltip(); + + /* Add CSRF token to AJAX requests */ + $.ajaxSetup({ + 'timeout': 15000, + 'beforeSend': function (jqXHR, settings) { + if (!/^(GET|HEAD|OPTIONS|TRACE)$/.test(settings.type) && !this.crossDomain) { + jqXHR.setRequestHeader("X-CSRFToken", getCookie('csrftoken')); + } + } + }); + + /* + * Initialize the DataTable (finding list) + */ + $('#finding_list').DataTable({ + select: { + style: 'os', + info: false + }, + scrollResize: true, + scrollCollapse: true, + scrollY: '100', + lengthChange: false, + paging: false, + info: false, + searching: false, + order: [ + [0, 'asc'], + [1, 'asc'] + ], + columnDefs: [ + { 'searchable': false, 'targets': [] }, + ], + initComplete: function (settings, json) { + $('#finding_list').on('select.dt', function (e, dt, type, indexes) { + if (type === 'row' && indexes.length === 1) { + let row = $('#finding_list').DataTable().rows(indexes).nodes().to$(); + let finding_uuid = row.data('finding-uuid'); + document.location.href = `/findings/${finding_uuid}`; + } + }); + } + }); + + // Initialize the ACE editor + initialize_editor(); + + // Auto-open single-child nodes + $("#data").on("open_node.jstree", function (e, data) { + try { + if (data.node.children.length == 1) { + $('#data').jstree().open_node(data.node.children[0]) + } + } catch (e) { + console.log(`Error: ${e}`); + } + }); +}) + +// General Purpose Helper Functions +let getCookie = function (name) { + var cookieValue = null; + if (document.cookie && document.cookie !== '') { + var cookies = document.cookie.split(';'); + for (var i = 0; i < cookies.length; i++) { + var cookie = jQuery.trim(cookies[i]); + // Does this cookie string begin with the name we want? + if (cookie.substring(0, name.length + 1) === (name + '=')) { + cookieValue = decodeURIComponent(cookie.substring(name.length + 1)); + break; + } + } + } + return cookieValue; +} + +const load_source_code = function(options) { + $('#finding_center').css('opacity', '0.20'); + $.ajax({ + 'url': '/api/findings/get_source_code', + 'method': 'GET', + 'data': { + 'file_uuid': options['file_uuid'] + }, + 'dataType': 'json', + 'success': function ({ file_contents, file_name, status }, textStatus, jqXHR) { + let editor = ace.edit("editor"); + editor.getSession().setValue(atob(file_contents)); + var get_mode_filename = (file_name) => { + if (file_name.indexOf('.sarif')) { + return `${file_name}.json`; + } + return file_name; + } + let mode = ace.require("ace/ext/modelist").getModeForPath(get_mode_filename(file_name)).mode; + editor.session.setMode(mode); + editor.resize(); + // Show the editor if needed + $('#editor-container').removeClass('d-none'); + + // Set the editor title + const file_line = $(document.body).data('current_finding').file_line; + if (file_line !== undefined && options['first'] === true) { + $('#file_path').text(file_name + ":" + file_line); + } else { + $('#file_path').text(file_name); + } + + //var path_abbrev = path; + //if (path_abbrev.length > 150) { + // path_abbrev = '...' + path_abbrev.substring(path_abbrev.length - 150, path_abbrev.length); + //} + //$('#editor-title .text').text(path_abbrev).attr('title', path); + const file_path = $(document.body).data('current_finding').file_path; + var finding_title = ($(document.body).data('current_finding').finding_title || '').substring(0, 40); + if (file_path !== undefined && file_line !== undefined && options['first'] === true) { + let session = ace.edit('editor').getSession(); + session.clearAnnotations(); + session.setAnnotations([{ + row: file_line - 1, + column: 0, + text: finding_title, + type: 'error' + }]); + } else { + ace.edit('editor').getSession().clearAnnotations(); + } + + $(window).trigger('resize'); + $('#editor').css('height', $(window).height() - $('#editor').offset().top - 10); + + if (options['first'] === true) { + window.setTimeout(function() { + ace.edit('editor').scrollToLine(file_line, true, false); + }, 50); + } + }, + 'error': function (jqXHR, textStatus, errorThrown) { + ace.edit('editor').getSession().clearAnnotations(); + ace.edit('editor').getSession().setMode('ace/mode/text') + set_editor_text(`Error ${jqXHR.status}: ${jqXHR.responseJSON.message}.`); + }, + 'complete': function () { + $('#finding_center').css('opacity', '1.0'); + } + }); +}; +const initialize_editor = function () { + try { + let editor = ace.edit("editor"), + session = editor.getSession(); + editor.setOptions({ + useWorker: false + }); + editor.setShowPrintMargin(false); + editor.setTheme("ace/theme/cobalt"); + editor.setReadOnly(true) + editor.setOptions({ + 'fontFamily': 'Inconsolata', + 'fontSize': localStorage.getItem('last-used-editor-font-size') || '1.1rem', + }); + } catch (e) { + console.log(e); + } +} + +const set_editor_text = function (text) { + let editor = ace.edit("editor"); + editor.getSession().setValue(text); + editor.resize(); +} + +const load_file_listing = function (options, callback) { + $.ajax({ + 'url': '/api/findings/get_files', + 'method': 'GET', + 'data': options, + 'success': function (data, textStatus, jqXHR) { + if ($('#data').jstree(true)) { + $('#data').jstree(true).destroy(); + } + $('#data').jstree({ + 'core': { + 'data': data.data, + 'multiple': false, + 'themes': { + 'dblclick_toggle': false, + 'icons': true, + 'name': 'proton', + 'responsive': true + } + }, + 'animation': 40, + 'plugins': ['sort', 'contextmenu'], + 'sort': function (a, b) { + a1 = this.get_node(a); + b1 = this.get_node(b); + if (a1.children.length === 0 && b1.children.length === 0) { + return a1.text.localeCompare(b1.text); + } else if (a1.children.length === 0) { + return 1; + } else if (b1.children.length === 0) { + return -1; + } else { + return a1.text.localeCompare(b1.text); + } + }, + 'contextmenu': { + items: function (node) { + var tree = $('#data').jstree(true); + if (node.id == '#') { + return {}; + } + return { + "Download": { + "separator_before": false, + "separator_after": false, + "label": "Download", + "icon": "fa fa-download", + "_class": "file_tree_context_menu_item", + "action": function (obj) { + var node = tree.get_node(obj.reference); + if (node.children.length === 0) { + document.location.href = `/api/findings/download_file?file_uuid=${node.original.file_uuid}`; + } else { + //document.location.href = `/api/findings/download_file?finding_uuid=${options.finding_uuid}&file_path=${node.id}&recursive=true`; + alert('not implemented'); + } + } + } + } + } + } + }); + $('#data').on({ + "loaded.jstree": function (event, data) { + $(this).jstree("open_node", $(this).find('li:first')); + }, + "changed.jstree": function (event, data) { + if (data && data.node && data.node.children && data.node.children.length === 0) { + const original_file_uuid = $.data(document.body, 'current_finding').file_uuid; + if (data.node.original.file_uuid != original_file_uuid) { + $('#finding_center').removeClass('col-lg-8').addClass('col-lg-10'); + $('#finding_right').remove(); + } + load_source_code({ + 'file_uuid': data.node.original.file_uuid, + 'first': data.event === undefined // Event is undefined when changed is called manually upon paage load (@magic) + }); + } + } + }); + } + }); +} + +const beautify_source_code = () => { + const beautify = ace.require("ace/ext/beautify"); + const editor = ace.edit("editor"); + if (!!beautify && !!editor) { + beautify.beautify(editor.session); + } +}; + +const toggle_word_wrap = () => { + const session = ace.edit('editor').getSession(); + session.setUseWrapMode(!session.getUseWrapMode()); +} + +const change_font_size = (size) => { + const editor = ace.edit('editor'); + let fontSize = editor.getFontSize(); + if (fontSize.indexOf('rem') > -1) { + fontSize = fontSize.replace('rem', ''); + fontSize = parseFloat(fontSize) * size; + fontSize = fontSize + 'rem'; + } else if (fontSize.indexOf('px') > -1) { + fontSize = fontSize.replace('px', ''); + fontSize = parseFloat(fontSize) * size; + fontSize = fontSize + 'px'; + } + editor.setFontSize(fontSize); + localStorage.setItem('last-used-editor-font-size', fontSize); +} + +const IS_SUCCESS = (data) => { + if (!data) return false; + const status = data.status + ''; + if (status === 'success') return true; + if (status === 'ok' || status.startsWith('ok')) return true; + return false; +} \ No newline at end of file diff --git a/omega/triage-portal/src/triage/templates/triage/base.html b/omega/triage-portal/src/triage/templates/triage/base.html new file mode 100644 index 00000000..dbc46fa4 --- /dev/null +++ b/omega/triage-portal/src/triage/templates/triage/base.html @@ -0,0 +1,155 @@ +{% load static %} + + + + + + + + + + + + + + + + + + + {% block style_include %} + {% endblock style_include %} + + {% block header %} + {% endblock header %} + + + + {{ page_title|default:"Omega | Open Source Security Foundation" }} + + + {% block full_body %} + + +
+ {% block body %} + {% endblock body %} +
+ + {% endblock full_body %} + + {% block footer %} + {% endblock footer %} + + + + + + + + + + + + + + + + + + + + + + + + {% block javascript_include %} + {% endblock javascript_include %} + + \ No newline at end of file diff --git a/omega/triage-portal/src/triage/templates/triage/case_list.html b/omega/triage-portal/src/triage/templates/triage/case_list.html new file mode 100644 index 00000000..afea0f04 --- /dev/null +++ b/omega/triage-portal/src/triage/templates/triage/case_list.html @@ -0,0 +1,82 @@ +{% extends "./base.html" %} + +{% block body %} + + +
+
+ +
+
+ + + + +
+
+
+
+
+
+

Cases

+ {% if cases %} + + + + + + + + + + + + + + {% for case in cases %} + + + + + + + + + {% empty %} + + + + {% endfor %} + +
TitleAssigned ToStateReporting PartnerReportedResolution Estimate
{{ case.title }}{{ case.assigned_to.get_full_name|default:'Unassigned' }}{{ case.get_state_display }}{{ case.get_reporting_partner_display }}{{ case.reported_dt|date:"SHORT_DATE_FORMAT" }}{{ case.resolved_target_dt|date:"SHORT_DATE_FORMAT" }}
No cases found.
+ {% else %} + There are no cases available. Try expanding your query. + {% endif %} +
+
+{% endblock body %} + +{% block javascript %} + /* Handle row clicks */ + $('tr.data_row').on('click', (e) => { + let case_uuid = $(e.target).closest('tr').data('case_uuid') + if (case_uuid !== undefined) { + document.location.href = `/cases/${case_uuid}`; + } + }); +{% endblock %} \ No newline at end of file diff --git a/omega/triage-portal/src/triage/templates/triage/case_show copy.html b/omega/triage-portal/src/triage/templates/triage/case_show copy.html new file mode 100644 index 00000000..53a89fc3 --- /dev/null +++ b/omega/triage-portal/src/triage/templates/triage/case_show copy.html @@ -0,0 +1,63 @@ +{% extends "./base.html" %} + +{% block body %} + {% block main %} + + +
+
+ +
+
+ + + + +
+
+
+
+
+
+

Active Cases

+ + + + + + + + + + + {% for case in cases %} + + + + + + + {% empty %} + + + + {% endfor %} + +
Case IDTitleAssigned ToStatus
{{ case.pk }}{{ case.title }}{{ case.assigned_to }}{{ case.get_status_display }}
No cases found.
+
+
+ + {% include "triage/case_part_edit.html" %} + {% endblock main %} +{% endblock body %} \ No newline at end of file diff --git a/omega/triage-portal/src/triage/templates/triage/case_show.html b/omega/triage-portal/src/triage/templates/triage/case_show.html new file mode 100644 index 00000000..f7bec1d7 --- /dev/null +++ b/omega/triage-portal/src/triage/templates/triage/case_show.html @@ -0,0 +1,207 @@ +{% extends "./base.html" %} +{% load gravatar %} +{% block body %} +
+
+ {% csrf_token %} + + + {% if error_messages %} + + {% endif %} + +
+
+ {% if case %} + + {% else %} + + {% endif %} + New + Back to List +
+ + {% if case %} +

Modify Case

+ {% else %} +

Add Case

+ {% endif %} +
+ +
+
+
+
+
+ + +
+
+ + +
+
+
+
+
+
+ +
+ +
+
+ To associate a finding with this case, navigate to the finding and select "Add to Case" and choose this case. +
+
+
+
+ +
+ +
+
+
+
+
+
+
+
+ + +
+
+ + +
+
+ + +
+
+ + +
+
+ + +
+
+ + +
+
+ + +
+
+ + +
+
+
Notes
+ + + + + {% include "triage/widgets/notes.html" with notes=case.notes.all %} + +
+
+
+{% endblock %} + +{% block javascript %} + $(document).ready(function() { + $('.upload_container') + .on('drop', dropHandler) + .on('dragover', dragOver) + .on('dragleave', dragEnd); + }); + function dragEnd(e) { + e.preventDefault(); + e.stopPropagation(); + $(e.target).closest('.upload_container').css('border', ''); + } + function dragOver(e) { + e.preventDefault(); + e.stopPropagation(); + e.originalEvent.dataTransfer.dropEffect = 'copy'; + $(e.target).closest('.upload_container').css('border', '2px dashed #ccc'); + } + function dropHandler(e) { + e.stopPropagation(); + e.preventDefault(); + const container = $(e.target).closest('.upload_container'); + container.css('border', ''); + + var files = e.originalEvent.dataTransfer.files; + var formData = new FormData(); + for (var i = 0; i < files.length; i++) { + formData.append('attachment', files[i]); + } + formData.append('target_type', container.data('target_type')); + formData.append('target_uuid', container.data('target_uuid')); + $.ajax({ + url: '/api/upload', + type: 'POST', + data: formData, + processData: false, + contentType: false, + success: function(data) { + for (let attachment of data.attachments) { + $('#case_attachments').append( + '
  • ' + + '' + + ' ' + + attachment.filename + + '' + + '
    • ' + ) + } + }, + error: function(data) { + console.log(data); + } + }); + } +{% endblock %} \ No newline at end of file diff --git a/omega/triage-portal/src/triage/templates/triage/filter_list.html b/omega/triage-portal/src/triage/templates/triage/filter_list.html new file mode 100644 index 00000000..9a3299e6 --- /dev/null +++ b/omega/triage-portal/src/triage/templates/triage/filter_list.html @@ -0,0 +1,86 @@ +{% extends "./base.html" %} + +{% block body %} + {% block main %} + + +
    +
    + +
    +
    + + + + +
    +
    +
    +
    +
    +
    +
    + New Filter +
    +

    Filters

    + {% if filters %} + + + + + + + + + + + + + {% for filter in filters %} + + + + + + + + + {% empty %} + + + + {% endfor %} + +
    TitleActivePriorityLast ExecutedLast UpdatedActions
    {{ filter.title }}{{ filter.active }}{{ filter.priority }}{{ filter.last_executed|date:"SHORT_DATETIME_FORMAT" }}{{ filter.updated_at|date:"SHORT_DATETIME_FORMAT" }} + Execute +
    No filters found.
    + {% else %} + There are no filters available. Try expanding your query. + {% endif %} +
    +
    + {% endblock main %} +{% endblock body %} + +{% block javascript %} + /* Handle clicks of the "New Tool Finding" button */ + $('tr.data_row').on('click', (e) => { + let filter_uuid = $(e.target).closest('tr').data('filter_uuid') + if (filter_uuid !== undefined) { + document.location.href = `/filter/${filter_uuid}`; + } + }); +{% endblock %} diff --git a/omega/triage-portal/src/triage/templates/triage/filter_show.html b/omega/triage-portal/src/triage/templates/triage/filter_show.html new file mode 100644 index 00000000..af9e7543 --- /dev/null +++ b/omega/triage-portal/src/triage/templates/triage/filter_show.html @@ -0,0 +1,140 @@ +{% extends "./base.html" %} + +{% block body %} + {% block main %} + + + +
    +
    +
    + {% csrf_token %} + + + {% if error_messages %} + + {% endif %} + +
    +
    + {% if filter %} + + {% else %} + + {% endif %} + New + Delete + Back to List +
    + {% if filter %} +

    Modify Filter

    + {% else %} +

    Add Filter

    + {% endif %} +
    + +
    +
    +
    + + +
    +
    + + +
    +
    + + + 0=highest priority, 1000=lowest priority +
    +
    + +
    + +
    +
    + + +
    +
    +
    +
    +
    + + {% endblock %} +{% endblock %} + +{% block javascript %} + +// Source: https://stackoverflow.com/a/19513428/1384352 +// Hook up ACE editor to all textareas with data-editor attribute +$(function() { + $('textarea[data-editor]').each(function() { + var textarea = $(this); + var mode = textarea.data('editor') || 'text'; + var editDiv = $('
    ', { + position: 'absolute', + width: '100%', + height: textarea.height(), + }).insertBefore(textarea); + textarea.css('display', 'none'); + var editor = ace.edit(editDiv[0]); + editor.setShowPrintMargin(false); + editor.renderer.setShowGutter(textarea.data('gutter')); + editor.getSession().setValue(textarea.val()); + editor.getSession().setMode("ace/mode/" + mode); + editor.setOptions({ + 'fontFamily': 'Inconsolata', + 'fontSize': localStorage.getItem('last-used-editor-font-size') || '1.1rem', + tabSize: 2, + useSoftTabs: true + }); + editor.setTheme("ace/theme/cobalt"); + + // copy back to textarea on form submit... + textarea.closest('form').on('submit', () => { + textarea.val(editor.getSession().getValue()); + }); + }); + $('a#delete_filter').on('click', (e) => { + e.preventDefault(); + if (confirm('Are you sure you want to delete this filter? This action cannot be undone.')) { + $.ajax({ + url: 'filter/delete', + type: 'POST', + data: { + filter_uuid: '{{ filter.uuid }}' + }, + success: function(data) { + window.location.href = '/filter'; + }, + error: function(data) { + alert('Error deleting filter: ' + data.responseText); + } + }); + } + }); +}); +{% endblock javascript %} diff --git a/omega/triage-portal/src/triage/templates/triage/findings_list.html b/omega/triage-portal/src/triage/templates/triage/findings_list.html new file mode 100644 index 00000000..36f602cf --- /dev/null +++ b/omega/triage-portal/src/triage/templates/triage/findings_list.html @@ -0,0 +1,111 @@ +{% extends "./base.html" %} + +{% block body %} +{% block main %} + + +
    +
    + +
    +
    + + + + + +
    + +
    +
    +
    +
    +
    + + + + + + + + + + + + + {% for finding in findings %} + + + + + + + + + {% endfor %} + +
    PackageIssueFilenameToolSeverityState
    + {% include "triage/widgets/project_name_pretty.html" with project=finding.project_version only %} + {{ finding.normalized_title|truncatechars:100 }}{{ finding.get_filename_display }}:{{ finding.file_line }}{{ finding.tool.name }}{{ finding.get_severity_display }}{{ finding.get_state_display }}
    + +
    + +
    + {% endblock main %} +{% endblock body %} + +{% block javascript %} + /* Handle clicks of the "New Tool Finding" button */ + $('#action_new_tooling_bug').on('click', (e) => { + let finding_uuid = $('#finding_list').data('finding_uuid'); + if (finding_uuid !== undefined) { + document.location.href = `/tool_defect/new?finding_uuid=${finding_uuid}`; + } + }); +{% endblock %} \ No newline at end of file diff --git a/omega/triage-portal/src/triage/templates/triage/findings_show copy.html b/omega/triage-portal/src/triage/templates/triage/findings_show copy.html new file mode 100644 index 00000000..d112f4f6 --- /dev/null +++ b/omega/triage-portal/src/triage/templates/triage/findings_show copy.html @@ -0,0 +1,175 @@ +{% extends "./base.html" %} + +{% block body %} + {% block main %} + +
    +
    + +
    +
    + + + + +
    +
    +
    +
    + + + + + +
    +
    + + + + + + + + + + {% for finding in findings %} + + + + + + {% endfor %} + +
    PackageIssueFilename
    {{ finding.scan.project_version }}{{ finding.title }}{{ finding.get_filename_display }}
    +
    + +
    + + +
    +
    + +
    +
    +
    +
    +
    +

    + +

    +
    +
    +
    +
    +
    +
    +
    +

    + +

    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
      +
    • +
      +

      Assignee

      + {% if finding.assigned_to %} + {{ finding.assigned_to }} + {% else %} + No one—assign yourself + {% endif %} +
      • +
    • +
      +

      Severity

      + {{ finding.get_severity_display }} +
      • +
    • A third item
      • +
    • A fourth item
      • +
    • And a fifth one
      • +
    +
    +
    + Triage Updates +
    +
    +
    +
    Severity
    + +
    + +
    +
    True/False Positive
    + +
    + +
    +
    Notes
    + +
    +
    +
    +
    +
    + {% endblock main %} +{% endblock body %} + +{% block javascript %} + /* Handle clicks of the "New Tool Finding" button */ + $('#action_new_tooling_bug').on('click', (e) => { + let finding_uuid = $('#finding_list').data('finding_uuid'); + if (finding_uuid !== undefined) { + document.location.href = `/tool_defect/new?finding_uuid=${finding_uuid}`; + } + }); +{% endblock %} \ No newline at end of file diff --git a/omega/triage-portal/src/triage/templates/triage/findings_show.html b/omega/triage-portal/src/triage/templates/triage/findings_show.html new file mode 100644 index 00000000..452e2614 --- /dev/null +++ b/omega/triage-portal/src/triage/templates/triage/findings_show.html @@ -0,0 +1,368 @@ +{% extends "./base.html" %} + +{% load gravatar %} + +{% block body %} + +
    +
    +
    + {{ finding.title }} + #{{ finding.pk }} +
    +
    + +
    +
    +
    + + +
    +   + {{ finding.file_path }} + {% if finding.file_line %} + :{{finding.file_line }} + {% endif %} + +
    +
    +
    +
    +
    +
    +
    + + +
    +
      +
    • +

      Details

      +
      +
      Tool:
      +
      {{ finding.tool.friendly_name }} {{ finding.tool.version }}
      +
      +
      • + +
    • + +
        +
      • + Choose an assignee +
        • +
      • + + + + {% for assignee in assignee_list %} + + {% endfor %} + +
        • +
      +

      Assignee

      +
      + {% if finding.assigned_to %} + {{ finding.assigned_to.username }} + {{ finding.assigned_to.first_name }} {{ finding.assigned_to.last_name }} + {% else %} + No one—assign yourself + {% endif %} +
      +
      • +
    • + +
        +
      • +
        +   New +
        + Add to Case +
        • +
      • + + + {% for case in case_list %} + + {% endfor %} + +
        • +
      +

      Severity

      + +
      • + +
    • + +
        +
      • + Update Impact +
        • +
      • + + Values range from 0 (no impact) to 32,767 (highest possible impact), leave blank for unknown.. +
        • +
      +

      Estimated Impact

      + {{ finding.estimated_impact }} +
      • +
    • + +
        +
      • + Choose a label +
        • +
      • + + + {% for label in label_list %} + + {% endfor %} + +
        • +
      +

      Labels

      + {% for label in finding.labels %} + {{ label }} + {% empty %} + No labels + {% endfor %} +
      • +
    • +

      Notes

      + +
      • +
    • + +
      • +
    +
    +
    +{% endblock body %} + +{% block javascript %} + /* Stash some page-level data */ + $(document.body).data('current_finding', { + 'finding_uuid': '{{ finding.uuid }}', + 'project_version_uuid': '{{ finding.project_version.uuid }}', + 'file_line': '{{ finding.file_line }}', + 'file_path': '{{ finding.file.path }}', + 'file_uuid': '{{ finding.file.uuid }}', + 'finding_title': '{{ finding.title }}' + }); + + /* Handle clicks of the "New Tool Finding" button */ + $('#action_new_tooling_bug').on('click', (e) => { + let finding_uuid = $('#finding_list').data('finding_uuid'); + if (finding_uuid !== undefined) { + document.location.href = `/tool_defect/new?finding_uuid=${finding_uuid}`; + } + }); + + /** + * Responds to the "assign self" link. + */ + $('#assign_self').on('click', (e) => { + const finding_uuid = '{{ finding.uuid }}'; + $.ajax({ + url: `/api/1/findings/update`, + data: { + 'finding_uuid': finding_uuid, + 'assigned_to': '$self' + }, + type: 'POST', + success: (data) => { + if (IS_SUCCESS(data)) { + $('#assigned_to_content').clear(); + const anchor = $(''); + anchor.attr('href', '#'); + anchor.text(data.assigned_to); + $('#assigned_to_content').append(anchor); + } + else { + console.log('Error: Unsuccessful response from server.'); + } + } + }); + }); + + $('#assigned_to').on('change', (e) => { + const finding_uuid = '{{ finding.uuid }}'; + $.ajax({ + url: `/api/1/findings/update`, + data: { + 'finding_uuid': finding_uuid, + 'assigned_to': $(e.target).val() + }, + type: 'POST', + success: (data) => { + if (IS_SUCCESS(data)) { + $('#assigned_to_content').clear(); + const anchor = $(''); + anchor.attr('href', '#'); + anchor.text(data.assigned_to); + $('#assigned_to_content').append(anchor); + } + else { + console.log('Error: Unsuccessful response from server.'); + } + } + }); + }); + $('#estimated_impact').on('keypress', (e) => { + if (e.keyCode === 13) { + e.preventDefault(); + $('#estimated_impact').trigger('change'); + } + }); + $('#estimated_impact').on('change', (e) => { + const finding_uuid = '{{ finding.uuid }}'; + $.ajax({ + url: `/api/1/findings/update`, + data: { + 'finding_uuid': finding_uuid, + 'estimated_impact': $(e.target).val() + }, + type: 'POST', + success: (data) => { + if (IS_SUCCESS(data)) { + $('#estimated_impact_content').text($(e.target).val()); + } + else { + console.log('Error: Unsuccessful response from server.'); + } + } + }); + $('ul.dropdown-menu.show').hide(); + }); + + $(document).ready(() => { + load_file_listing({"project_version_uuid": "{{ finding.project_version.uuid }}"}); + $('#data').on({ + "loaded.jstree": function (event, data) { + $('#data').jstree('select_node', '{{ finding.file.path }}', false); + + } + }); + + /* Resize the file and editor */ + $(window).on('resize', (e) => { + $('#editor').css('height', $(window).height() - $('#editor').offset().top - 10); + $('#data').css('height', $(window).height() - $('#data').offset().top - 10); + }); + $(window).trigger('resize'); + }); + + var update_finding = (options) => { + $.ajax({ + url: `/api/1/findings/update`, + type: 'POST', + data: options, + success: (data) => { + console.log(data); + } + }); + }; + + $('#severity_button_group input').on('click', (e) => { + let severity = $(e.target).val(); + update_finding({ + 'finding_uuid': '{{ finding.uuid }}', + 'analyst_severity_level': severity + }); + }); + +{% endblock %} \ No newline at end of file diff --git a/omega/triage-portal/src/triage/templates/triage/findings_upload.html b/omega/triage-portal/src/triage/templates/triage/findings_upload.html new file mode 100644 index 00000000..cdf0cec4 --- /dev/null +++ b/omega/triage-portal/src/triage/templates/triage/findings_upload.html @@ -0,0 +1,38 @@ +{% extends "./base.html" %} + +{% block body %} + {% block main %} + + +{% if status == "ok" %} + +{% endif %} + + +
    + +
    +
    +

    Upload SARIF

    +
    + {% csrf_token %} +
    + + +
    +
    + + +
    +
    + +
    +
    +
    +
    +
    + {% endblock main %} +{% endblock body %} \ No newline at end of file diff --git a/omega/triage-portal/src/triage/templates/triage/home.html b/omega/triage-portal/src/triage/templates/triage/home.html new file mode 100644 index 00000000..bb406d16 --- /dev/null +++ b/omega/triage-portal/src/triage/templates/triage/home.html @@ -0,0 +1,209 @@ +{% extends "./base.html" %} +{% load static %} +{% block body %} + {% block main %} + +
    +
    +
    +
    +
    +

    Omega

    +

    + Omega is a security analysis triage and management suite created and + maintained by the Open Source Security Foundation. +

    + +
    +
    +
    +
    +
    +
    +
    +
    Latest Findings
    +

    Access the latest findings from the last 24 hours.

    +     View Findings +
    +
    + Last updated {{ finding_last_updated|timesince }} ago +
    +
    +
    + +
    +
    +
    +
    Active Cases
    +

    View all reported but unfixed vulnerabilities.

    + View Active Cases +
    +
    + Last updated {{ case_last_updated|timesince }} ago +
    +
    +
    + +
    +
    +
    +
    Vulnerability Search
    +

    + Search for a vulnerability by its CVE or description. Service provided by third-parties. +

    + +
    + + + +
    +
    +

    +
    +
    +
    +
    + + +
    +
    +
    +
    +
    +
    Code Search
    +

    + Code search is provided by third-parties. +

    + +
    + + + +
    +
    +

    +
    +
    +
    + +
    +
    +
    +
    My Work
    +

    +

      +
    • + My Cases + {{ my_work.num_cases|floatformat:"g" }} +
      • +
    • + My Findings + {{ my_work.num_findings|floatformat:"g" }} +
      • +
    • + My Tool Defects + {{ my_work.num_tool_defects|floatformat:"g" }} +
      • +
    +

    +
    +
    +
    + +
    +
    +
    +
    Metrics
    +

    +

      +
    • + Total Findings + {{ metrics.num_findings|floatformat:"g" }} +
      • +
    • + Active Findings + {{ metrics.num_active_findings|floatformat:"g" }} +
      • +
    • + New This Week + {{ metrics.num_new_findings|floatformat:"g" }} +
      • +
    +

    +
    +
    +
    +
    + +
    +
    +
    + + {% endblock main %} +{% endblock body %} + +{% block javascript %} + /* Handle clicks of the "Code Search" buttons */ + $('.code_search').on('click', (e) => { + e.preventDefault(); + const query = $('#code_search_query').val(); + if (!query || query.trim() === '') { + return; + } + + const target = $(e.target).closest('button,a').data('target'); + switch (target) { + case 'github': + window.open('https://cs.github.com/?q=' + encodeURIComponent(query), '_omega_cs', 'noopener,noreferrer'); + break; + case 'sourcegraph': + window.open('https://sourcegraph.com/search?patternType=literal&q=context:global+' + encodeURIComponent(query), '_omega_cs', 'noopener,noreferrer'); + break; + case 'searchcode': + window.open('https://searchcode.com/codesearch?q=' + encodeURIComponent(query), '_omega_cs', 'noopener,noreferrer'); + break; + case 'debian': + window.open('https://codesearch.debian.net/search?literal=1&q=' + encodeURIComponent(query), '_omega_cs', 'noopener,noreferrer'); + break; + default: + } + }); + + /* Handle clicks of the "Vulnerability Search" buttons */ + $('.vulnerability_search').on('click', (e) => { + e.preventDefault(); + const query = $('#vulnerability_query').val(); + if (!query || query.trim() === '') { + return; + } + + const target = $(e.target).closest('button,a').data('target'); + switch (target) { + case 'nvd.nist.gov': + window.open('https://nvd.nist.gov/vuln/search/results?form_type=Basic&results_type=overview&search_type=all&isCpeNameSearch=false&query=' + encodeURIComponent(query), '_omega_vs', 'noopener,noreferrer'); + break; + case 'cve.circl.lu': + $('
    ').appendTo('body').submit().remove(); + break; + case 'exploit-db.com': + window.open('https://www.exploit-db.com/search?q=' + encodeURIComponent(query), '_omega_vs', 'noopener,noreferrer'); + break; + default: + } + }); +{% endblock %} diff --git a/omega/triage-portal/src/triage/templates/triage/tool_defect_list.html b/omega/triage-portal/src/triage/templates/triage/tool_defect_list.html new file mode 100644 index 00000000..4997a48f --- /dev/null +++ b/omega/triage-portal/src/triage/templates/triage/tool_defect_list.html @@ -0,0 +1,77 @@ +{% extends "./base.html" %} + +{% block body %} + {% block main %} + + +
    +
    + +
    +
    + + + + +
    +
    +
    +
    +
    +
    +

    Tool Defects

    + + + + + + + + + + + + + {% for tool_defect in tool_defects %} + + + + + + + + + {% empty %} + + + + {% endfor %} + +
    Tool Defect IDTitleToolAssigned ToStatusCreated
    {{ tool_defect.pk }}{{ tool_defect.title }}{{ tool_defect.tool.name }}{{ tool_defect.assigned_to|default:"No one" }}{{ tool_defect.get_state_display }}{{ tool_defect.created_at|timesince }} ago
    No defects found.
    +
    +
    + + {% endblock main %} +{% endblock body %} + +{% block javascript %} + /* Handle row clicks */ + $('tr.data_row').on('click', (e) => { + let tool_defect_uuid = $(e.target).closest('tr').data('tool_defect_uuid') + if (tool_defect_uuid !== undefined) { + document.location.href = `/tool_defect/${tool_defect_uuid}`; + } + }); +{% endblock %} \ No newline at end of file diff --git a/omega/triage-portal/src/triage/templates/triage/tool_defect_new.html b/omega/triage-portal/src/triage/templates/triage/tool_defect_new.html new file mode 100644 index 00000000..a9110ef9 --- /dev/null +++ b/omega/triage-portal/src/triage/templates/triage/tool_defect_new.html @@ -0,0 +1,8 @@ +{% extends "./base.html" %} + +{% block body %} + {% block main %} +
    + {% include "triage/tool_defect_part_edit.html" %} + {% endblock main %} +{% endblock body %} \ No newline at end of file diff --git a/omega/triage-portal/src/triage/templates/triage/tool_defect_show.html b/omega/triage-portal/src/triage/templates/triage/tool_defect_show.html new file mode 100644 index 00000000..783fff0e --- /dev/null +++ b/omega/triage-portal/src/triage/templates/triage/tool_defect_show.html @@ -0,0 +1,123 @@ +{% extends "./base.html" %} + +{% load gravatar %} + +{% block body %} + {% block main %} +
    +
    + {% csrf_token %} + {% if tool_defect %} + + + {% else %} + + {% endif %} + +
    +
    + {% if tool_defect %} + + {% else %} + + {% endif %} + Back to List +
    + + {% if tool_defect %} +

    Edit Tool Defect

    + {% else %} +

    Add Tool Defect

    + {% endif %} +
    + +
    +
    +
    + + +
    +
    + + +
    +
    + + +
    +
    + + +
    +
    +
    + +
    + + {% include "triage/widgets/notes.html" with notes=tool_defect.notes.all %} + + + +
    +
    +
    + +
    + + {% if tool_defect.findings or finding %} + + + + + + + + + + + + {% if finding %} + + + + + + + + + {% endif %} + {% for finding in tool_defect.findings.all %} + + + + + + + + {% endfor %} + +
    TitleProjectStateSeverityActions
    {{ finding.normalized_title }}{{ finding.project_version.package_url }}{{ finding.get_state_display }}{{ finding.get_severity_display }} + +    + +
    {{ finding.normalized_title }}{{ finding.project_version.package_url }}{{ finding.get_state_display }}{{ finding.get_severity_display }} + +    + +
    + {% else %} +

    No findings associated with this defect.

    + {% endif %} +
    +
    + + {% endblock main %} +{% endblock body %} \ No newline at end of file diff --git a/omega/triage-portal/src/triage/templates/triage/widgets/icon_for_file.html b/omega/triage-portal/src/triage/templates/triage/widgets/icon_for_file.html new file mode 100644 index 00000000..fbc382b5 --- /dev/null +++ b/omega/triage-portal/src/triage/templates/triage/widgets/icon_for_file.html @@ -0,0 +1,25 @@ +{% spaceless %} + {% with ext4=filename|slice:"-4:" ext3=filename|slice:"-3:" ext2=filename|slice:"-2" %} + {% if ext3 == "pdf" %} + + {% elif ext3 == "doc" or ext4 == "docx" %} + + {% elif ext3 == "zip" or ext3 == "tar" %} + + {% elif ext3 == "xls" or ext4 == "xlsx" %} + + {% elif ext3 == "ppt" or ext4 == "pptx" %} + + {% elif ext3 == "mp3" or ext3 == "wav" or ext3 == "ogg" or ext4 == "flac" %} + + {% elif ext3 == "mp4" or ext3 == "avi" or ext3 == "mkv" or ext3 == "mov" or ext3 == "wmv" or ext3 == "mpg" or ext3 == "mpeg" %} + + {% elif ext3 == "gif" or ext3 == "jpg" or ext3 == "png" or ext3 == "ico" %} + + {% elif ext3 == "txt" or ext2 == "md" or ext3 == "rst" or ext2 == "py" or ext2 == "sh" or ext3 == "xml" or ext4 == "html" or ext3 == "css" or ext2 == "js" %} + + {% else %} + + {% endif %} + {% endwith %} +{% endspaceless %} \ No newline at end of file diff --git a/omega/triage-portal/src/triage/templates/triage/widgets/notes.html b/omega/triage-portal/src/triage/templates/triage/widgets/notes.html new file mode 100644 index 00000000..192481c5 --- /dev/null +++ b/omega/triage-portal/src/triage/templates/triage/widgets/notes.html @@ -0,0 +1,13 @@ +{% if notes %} + {% load gravatar %} + {% for note in notes %} +
    + +
    +
    + {{ note.created_by.get_full_name }} + commented {{ note.created_at|date:"SHORT_DATETIME_FORMAT" }} +
    {{ note.content }}
    +
    + {% endfor %} +{% endif %} \ No newline at end of file diff --git a/omega/triage-portal/src/triage/templates/triage/widgets/project_name_pretty.html b/omega/triage-portal/src/triage/templates/triage/widgets/project_name_pretty.html new file mode 100644 index 00000000..3f66807f --- /dev/null +++ b/omega/triage-portal/src/triage/templates/triage/widgets/project_name_pretty.html @@ -0,0 +1,30 @@ +{% load project_helpers %} +{% parse_package_url project.package_url %} +{% spaceless %} + {% if package_url.type == "npm" %} + + {% elif package_url.type == "gem" %} + + {% elif package_url.type == "nuget" %} + + {% elif package_url.type == "pypi" %} + + {% elif package_url.type == "github" %} + + {% elif package_url.type == "ubuntu" %} + + {% else %} +
    {{ package_url.type }}
    + {% endif %} + + + {% if package_url.namespace %} + {{ package_url.namespace }}/{{ package_url.name }} + {% else %} + {{ package_url.name }} + {% endif %} + + {{ package_url.version }} + + +{% endspaceless %} \ No newline at end of file diff --git a/omega/triage-portal/src/triage/templates/triage/wiki_edit.html b/omega/triage-portal/src/triage/templates/triage/wiki_edit.html new file mode 100644 index 00000000..ff08b687 --- /dev/null +++ b/omega/triage-portal/src/triage/templates/triage/wiki_edit.html @@ -0,0 +1,95 @@ +{% extends "./base.html" %} +{% load gravatar %} +{% block body %} +
    + {% csrf_token %} + + +
    +
    +
    + {% if wiki_article %} + + {% else %} + + {% endif %} + New + Cancel +
    + + {% if wiki_article %} +

    Edit Wiki Article

    + {% else %} +

    Add Wiki Article

    + {% endif %} +
    + + {% if wiki_article.versions %} +
    + {% else %} +
    + {% endif %} +
    +
    + + +
    +
    + + +
    +
    + + +
    +
    + + +
    +
    + +
    + + +
    +
    +
    +
    + {% if wiki_article.versions %} +
    +
    + + +
    +
    + {% endif %} +
    + +{% endblock body %} + +{% block javascript %} +$('#minor-edit').on('click', (e) => { + $('#wiki_article_change_comment').val('Minor edit'); +}); +{% endblock javascript %} \ No newline at end of file diff --git a/omega/triage-portal/src/triage/templates/triage/wiki_list.html b/omega/triage-portal/src/triage/templates/triage/wiki_list.html new file mode 100644 index 00000000..3390be70 --- /dev/null +++ b/omega/triage-portal/src/triage/templates/triage/wiki_list.html @@ -0,0 +1,70 @@ +{% extends "./base.html" %} + +{% block body %} + + +
    +
    + +
    +
    + + + + +
    +
    +
    +
    +
    +
    +

    Wiki Articles

    + {% if wiki_articles %} + + + + + + + + + + + {% for wiki_article in wiki_articles %} + + + + + + + {% empty %} + + + + {% endfor %} + +
    TitleStateLast UpdatedLast Updated by
    {{ wiki_article.current.title }}{{ wiki_article.get_state_display }}{{ wiki_article.current.updated_at|date:"SHORT_DATETIME_FORMAT" }}{{ wiki_article.current.updated_by.get_full_name }}
    No wiki articles found.
    + {% else %} + There are no wiki articles available. Try expanding your query. + {% endif %} +
    +
    +{% endblock body %} + +{% block javascript %} + /* Handle row clicks */ + $('tr.data_row').on('click', (e) => { + let slug = $(e.target).closest('tr').data('wiki_article_slug') + if (slug !== undefined) { + document.location.href = `/wiki/${slug}`; + } + }); +{% endblock %} \ No newline at end of file diff --git a/omega/triage-portal/src/triage/templates/triage/wiki_show.html b/omega/triage-portal/src/triage/templates/triage/wiki_show.html new file mode 100644 index 00000000..9edf86ef --- /dev/null +++ b/omega/triage-portal/src/triage/templates/triage/wiki_show.html @@ -0,0 +1,24 @@ +{% extends "./base.html" %} +{% load wiki %} +{% block body %} +
    + {% csrf_token %} + + +
    +
    +
    + Edit + New + View All +
    + +

    {{ wiki_article.current.title }}

    +
    + +
    + {{ wiki_article.current.content|wiki_markdown|safe }} +
    +
    +
    +{% endblock body %} \ No newline at end of file diff --git a/omega/triage-portal/src/triage/templatetags/__init__.py b/omega/triage-portal/src/triage/templatetags/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/omega/triage-portal/src/triage/templatetags/gravatar.py b/omega/triage-portal/src/triage/templatetags/gravatar.py new file mode 100644 index 00000000..003ace87 --- /dev/null +++ b/omega/triage-portal/src/triage/templatetags/gravatar.py @@ -0,0 +1,21 @@ +import hashlib +import logging + +from django import template + +register = template.Library() +logger = logging.getLogger(__name__) + + +@register.filter +def gravatar(user, size=128): + """Convert the user's email address passed in into a gravatar URL.""" + root = "https://gravatar.com/avatar/{0}?r=g&d=mp&s={1}&{2}" + if user and user.email: + email = user.email.strip().lower().encode("utf-8") + # MD5 is mandated by the Gravatar spec - gravatar.com + hash_value = hashlib.md5(email).hexdigest() # noqa # nosec + return root.format(hash_value, str(size), "") + else: + # Mystery Person + return root.format("", str(size), "f=y") diff --git a/omega/triage-portal/src/triage/templatetags/project_helpers.py b/omega/triage-portal/src/triage/templatetags/project_helpers.py new file mode 100644 index 00000000..7d50b088 --- /dev/null +++ b/omega/triage-portal/src/triage/templatetags/project_helpers.py @@ -0,0 +1,11 @@ +from django import template +from packageurl import PackageURL + +register = template.Library() + + +@register.simple_tag(takes_context=True) +def parse_package_url(context, package_url): + package_url_obj = PackageURL.from_string(package_url) + context.update({"package_url": package_url_obj.to_dict()}) + return "" diff --git a/omega/triage-portal/src/triage/templatetags/wiki.py b/omega/triage-portal/src/triage/templatetags/wiki.py new file mode 100644 index 00000000..080ad9be --- /dev/null +++ b/omega/triage-portal/src/triage/templatetags/wiki.py @@ -0,0 +1,22 @@ +import hashlib +import logging + +import markdown +from django import template +from markdown.extensions.wikilinks import WikiLinkExtension + +register = template.Library() +logger = logging.getLogger(__name__) + + +@register.filter +def wiki_markdown(context): + if not context: + return "" + try: + extensions = [WikiLinkExtension(base_url="/wiki/", end_url="")] + result = markdown.markdown(context, extensions=extensions) + return result + except Exception as msg: + logger.warn("Error in wiki_markdown: %s" % msg) + return context diff --git a/omega/triage-portal/src/triage/urls.py b/omega/triage-portal/src/triage/urls.py new file mode 100644 index 00000000..f5e37d08 --- /dev/null +++ b/omega/triage-portal/src/triage/urls.py @@ -0,0 +1,50 @@ +# -*- coding: utf-8 -*- +"""This module URL patterns specific to the Triage Portal.""" + +from django.urls import path + +from triage.views import attachments, cases, filters, findings, home, tool_defect, wiki + +urlpatterns = [ + # Cases + path("cases/", cases.show_case), + path("cases/new", cases.new_case), + path("cases/save", cases.save_case), + path("cases/", cases.show_cases), + # Tooling Defects + path("tool_defect/", tool_defect.show_tool_defect), + path("tool_defect/new", tool_defect.show_add_tool_defect), + path("tool_defect/save", tool_defect.save_tool_defect), + path("tool_defect/", tool_defect.show_tool_defects), + # Findings + path("api/findings/add_archive", findings.api_add_scan_archive), + path("api/findings/get_files", findings.api_get_files), + path("api/findings/get_source_code", findings.api_get_source_code), + path("api/findings/download_file", findings.api_download_file), + path("api/upload", findings.api_upload_attachment), + # path("api/findings/get_file_list", findings.api_get_file_list), + path("api/findings/get_blob_list", findings.api_get_blob_list), + path("api/1/findings/update", findings.api_update_finding), + # Attachments + path("attachment/", attachments.download_attachment), + path("findings/", findings.show_finding_by_uuid), + path("findings/upload", findings.show_upload), + path("findings/", findings.show_findings), + # Filters + path("filter/", filters.show_filter), + path("filter/new", filters.new_filter), + path("filter/save", filters.save_filter), + path("filter/execute", filters.execute_filter), + path("filter/delete", filters.delete_filter), + path("filter/", filters.show_filters), + # Wiki + path("wiki/special:list", wiki.show_wiki_article_list), + path("wiki/save", wiki.save_wiki_article), + path("wiki/", wiki.show_wiki_article), + path("wiki//", wiki.show_wiki_article_revision), + path("wiki//edit", wiki.edit_wiki_article), + path("wiki///edit", wiki.edit_wiki_article_revision), + path("wiki/", wiki.home), + # Default (Home) + path("", home.home), +] diff --git a/omega/triage-portal/src/triage/util/azure_blob_storage.py b/omega/triage-portal/src/triage/util/azure_blob_storage.py new file mode 100644 index 00000000..0228be10 --- /dev/null +++ b/omega/triage-portal/src/triage/util/azure_blob_storage.py @@ -0,0 +1,225 @@ +import io +import logging +import os +import tarfile +import urllib.parse +import uuid +from typing import List, Optional + +from azure.storage.blob import BlobClient, BlobServiceClient +from django.core.cache import cache +from packageurl import PackageURL + +import triage +from core.settings import ( + DEFAULT_CACHE_TIMEOUT, + TOOLSHED_BLOB_STORAGE_CONTAINER_SECRET, + TOOLSHED_BLOB_STORAGE_URL_SECRET, +) +from triage.util.source_viewer.pathsimilarity import PathSimilarity +from triage.util.source_viewer.viewer import SourceViewer + +logger = logging.getLogger(__name__) + + +class AzureBlobStorageAccessor: + """ + This class is used to access blob stored in the Toolshed container. To use + it, pass in a name prefix, which is usually {type}/{name}/{version} or + {type}/{namespace}/{name}/{version}. + + Example: + >>> blob_storage = AzureBlobStorageAccessor('npm/left-pad/1.3.0') + >>> blob_storage.get_blob_list() + >>> blob_storage.get_tool_contents('tool-codeql-results.json') + >>> blob_storage.get_package_contents(') + >>> blob_storage.get_blob_contents('tool-codeql-results.json') + """ + + def __init__(self, name_prefix: str): + """Initialize AzureBlobStorageAccessor.""" + if not name_prefix or not name_prefix.strip(): + raise ValueError("name_prefix cannot be empty") + + if not TOOLSHED_BLOB_STORAGE_URL_SECRET or not TOOLSHED_BLOB_STORAGE_CONTAINER_SECRET: + raise ValueError("TOOLSHED_BLOB_STORAGE_URL and TOOLSHED_BLOB_CONTAINER must be set") + + self.blob_service = BlobServiceClient(TOOLSHED_BLOB_STORAGE_URL_SECRET) + self.container = self.blob_service.get_container_client( + TOOLSHED_BLOB_STORAGE_CONTAINER_SECRET + ) + self.name_prefix = name_prefix + + def get_blob_list(self) -> List[dict]: + """Get list of blobs in the Toolshed container.""" + try: + cache_key = f"AzureBlobStorageAccessor[name_prefix={self.name_prefix}].blob_list" + if cache.has_key(cache_key): + return cache.get(cache_key) + else: + data = list( + map( + lambda b: { + "full_path": b.name, + "relative_path": b.name[len(self.name_prefix) + 1 :], + }, + self.container.list_blobs(name_starts_with=self.name_prefix), + ) + ) + cache.set(cache_key, data, timeout=DEFAULT_CACHE_TIMEOUT) + return data + except: + logger.exception("Failed to get blob list") + return [] + + def get_blob_contents(self, blob_name: str) -> str | bytes: + """Load blob contents from Toolshed.""" + try: + blob = self.container.get_blob_client(blob_name) + if blob.exists(): + logger.info("Blob exists, downloading: %s", blob_name) + return blob.download_blob().readall() + else: + logger.warning("Blob %s does not exist.", blob_name) + return None + except: + logger.exception("Failed to get blob contents") + return None + + +class ToolshedBlobStorageAccessor: + def __init__(self, scan: "triage.models.Scan"): + if not scan: + raise ValueError("scan cannot be empty") + self.scan = scan + self.package_url = PackageURL.from_string(scan.project_version.package_url) + name_prefix = self.get_toolshed_prefix(self.package_url) + if not name_prefix: + raise ValueError("Invalid package_url") + + self.blob_accessor = AzureBlobStorageAccessor(name_prefix) + + def get_toolshed_prefix(self, package_url: PackageURL): + if not package_url: + return None + + if package_url.namespace: + parts = [package_url.type, package_url.namespace, package_url.name, package_url.version] + else: + parts = [package_url.type, package_url.name, package_url.version] + + # Escape reserved URL characters + prefix = "/".join(map(urllib.parse.quote, parts)) + + # TODO: Add validation based on https://docs.microsoft.com/en-us/rest/api/storageservices/naming-and-referencing-containers--blobs--and-metadata + return prefix + + def get_tool_files(self, path_prefix="/tools"): + """Retrieve all tool findings files from Toolshed.""" + results = [] # type: List[dict] + for blob in self.blob_accessor.get_blob_list(): + results.append(os.path.join(path_prefix, blob.get("relative_path"))) + return results + + def get_package_files(self, path_prefix="/package"): + """Retreive all package file contents from any available source.""" + results = [] + for blob in self.blob_accessor.get_blob_list(): + if blob.get("relative_path").startswith("reference-binaries"): + if blob.get("relative_path").endswith(".tgz"): + contents = self.blob_accessor.get_blob_contents(blob.get("full_path")) + tar = tarfile.open(fileobj=io.BytesIO(contents), mode="r") + for member in tar.getmembers(): + results.append(os.path.join(path_prefix, member.name)) + + if not results: + viewer = SourceViewer(self.package_url) + viewer.load_if_needed() + for filename in viewer.get_file_list(): + results.append(os.path.join(path_prefix, filename)) + return results + + def get_package_contents(self, filename): + """Retrieve package file contents from Toolshed.""" + cache_key = f"Storage[purl={self.package_url}].filename={filename}" + if cache.has_key(cache_key): + return cache.get(cache_key) + + try: + logger.info("Attempting to retrieve file contents for %s", filename) + if filename.startswith("package/"): + filename = filename[len("package/") :] + clean_filename = self.clean_filename(filename) + + for blob in self.blob_accessor.get_blob_list(): + if not blob.get("relative_path").startswith("reference-binaries") or not blob.get( + "relative_path" + ).endswith(".tgz"): + continue + contents = self.blob_accessor.get_blob_contents(blob.get("full_path")) + tar = tarfile.open(fileobj=io.BytesIO(contents), mode="r") + for member in tar.getmembers(): + if member.name == clean_filename: + contents = tar.extractfile(member).read() + logger.info("Content length: %d bytes", len(contents)) + cache.set(cache_key, contents, timeout=DEFAULT_CACHE_TIMEOUT) + return contents + + # No exact match, try for fuzzy ones + member_names = [t.name for t in tar.getmembers()] + most_similar = PathSimilarity.find_most_similar_path(member_names, clean_filename) + if most_similar: + logger.info("Most similar path: %s", most_similar) + contents = tar.extractfile(most_similar).read() + logger.info("Content length: %d bytes", len(contents)) + cache.set(cache_key, contents, timeout=DEFAULT_CACHE_TIMEOUT) + return contents + + logger.warning("File %s not found in package", filename) + cache.set(cache_key, "", timeout=DEFAULT_CACHE_TIMEOUT) + return None + except: + logger.exception("Failed to get blob contents") + cache.set(cache_key, "", timeout=DEFAULT_CACHE_TIMEOUT) + return None + + def get_file_contents(self, filename): + """Retrieve contents of a file from the Toolshed.""" + cache_key = f"Storage[purl={self.package_url}].filename={filename}" + if cache.has_key(cache_key): + return cache.get(cache_key) + try: + logger.info("Attempting to retrieve file contents for %s", filename) + if filename.startswith("tools/"): + filename = filename[len("tools/") :] + clean_filename = self.clean_filename(filename) + full_path = os.path.join(self.get_toolshed_prefix(self.package_url), filename) + contents = self.blob_accessor.get_blob_contents(full_path) + logger.info("Content length: %d bytes", len(contents)) + cache.set(cache_key, contents, timeout=DEFAULT_CACHE_TIMEOUT) + return contents + except: + logger.exception("Failed to get blob contents") + cache.set(cache_key, "", timeout=DEFAULT_CACHE_TIMEOUT) + return None + + def get_all_files(self): + """Retrieve all files about the scan.""" + return self.get_tool_files() + self.get_package_files() + self.get_intermediate_files() + + def get_intermediate_files(self): + return [] + + def clean_filename(self, filename: str) -> Optional[str]: + if not filename: + return None + + if filename.startswith("pkg:"): + return None + + if filename.startswith("/opt/"): + parts = filename.split("/")[3:] + parts = parts[parts.index("src") + 1 :] + return "/".join(parts) + + return filename diff --git a/omega/triage-portal/src/triage/util/content_managers/__init__.py b/omega/triage-portal/src/triage/util/content_managers/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/omega/triage-portal/src/triage/util/content_managers/base_manager.py b/omega/triage-portal/src/triage/util/content_managers/base_manager.py new file mode 100644 index 00000000..64823ac9 --- /dev/null +++ b/omega/triage-portal/src/triage/util/content_managers/base_manager.py @@ -0,0 +1,2 @@ +class BaseManager: + pass \ No newline at end of file diff --git a/omega/triage-portal/src/triage/util/content_managers/file_manager.py b/omega/triage-portal/src/triage/util/content_managers/file_manager.py new file mode 100644 index 00000000..a224e72a --- /dev/null +++ b/omega/triage-portal/src/triage/util/content_managers/file_manager.py @@ -0,0 +1,94 @@ +from typing import Tuple +import logging +import os +import zstd +import hashlib + +from .base_manager import BaseManager +from core.settings import FILE_STORAGE_PROVIDERS + +logger = logging.getLogger(__name__) + +class FileManager(BaseManager): + """Manages file content based on the file system.""" + + # The compression algorithm to use, or None for no compression + compressor = 'zstd' # type: str | None + + def __init__(self, **kwargs): + """Initiizes a new FileManager object.""" + + # Get the root file system path (either provided or through config) + root_path = str(kwargs.get('root_path', '')) + if not root_path: + default_manager = FILE_STORAGE_PROVIDERS.get("default") + if not default_manager: + raise EnvironmentError( + "Missing configuration value for FILE_STORAGE_PROVIDERS.default" + ) + root_path = str(default_manager.get("args", {}).get("root_path", '')) + + # Create path if necessary + if not os.path.exists(root_path): + os.makedirs(root_path, exist_ok=True) + + logger.debug("Initiizing a FileManager with root path: %s", root_path) + self.root_path = root_path + + def compress(self, filename: str, content: bytes) -> Tuple[str, bytes]: + """Compresses content using the configured compressor.""" + if self.compressor == 'zstd': + return (filename + '.zst', zstd.compress(content)) # pylint: disable=c-extension-no-member + + # No compression + return (filename, content) + + def decompress(self, filename: str, content: bytes) -> bytes: + """Decompresses content using the appropriate decompressor.""" + if filename.endswith('.zst'): + return zstd.decompress(content) # pylint: disable=c-extension-no-member + return content + + def get_file(self, file_key: str) -> bytes | None: + """Retrieve a file from the file system.""" + logger.debug("Looking for file with key: %s", file_key) + path = self.find_file_by_key(file_key) + if path and os.path.isfile(path): + with open(path, "rb") as f: + return self.decompress(path, f.read()) + return None + + def add_file(self, content: bytes, path: str, exist_ok: bool = True) -> str: + """Adds a file to the file system.""" + file_key = hashlib.sha256(content).hexdigest() + logger.debug("Adding file with key: %s", file_key) + + path = self._get_full_path(file_key) + if any(os.path.isfile(_path) for _path in [path, path + '.zst']): + if exist_ok: + return file_key + raise ValueError(f"File with key {file_key} already exists.") + + # Make sure the directory exists + os.makedirs(os.path.dirname(path), exist_ok=True) + + filename, content = self.compress(path, content) + with open(filename, 'wb') as f: + f.write(content) + + return file_key + + def find_file_by_key(self, file_key: str) -> str | None: + """Find a file by its key.""" + path = self._get_full_path(file_key) + for _path in [path + '.zst', path]: + if os.path.isfile(_path): + return _path + return None + + def _get_full_path(self, uuid: str) -> str: + if not uuid: + raise ValueError("UUID cannot be empty.") + prefix_1 = uuid[0:3] + prefix_2 = uuid[0:5] + return os.path.join(self.root_path, prefix_1, prefix_2, uuid) diff --git a/omega/triage-portal/src/triage/util/finding_importers/archive_importer.py b/omega/triage-portal/src/triage/util/finding_importers/archive_importer.py new file mode 100644 index 00000000..326b910a --- /dev/null +++ b/omega/triage-portal/src/triage/util/finding_importers/archive_importer.py @@ -0,0 +1,145 @@ +import json +import io +import logging +import os +import tarfile +import zipfile + +import magic +from triage.models import File, FileContent, ProjectVersion +from triage.util.finding_importers.sarif_importer import SARIFImporter +from django.contrib.auth.models import AbstractBaseUser, AnonymousUser +from triage.util.content_managers.file_manager import FileManager +from core import settings + +logger = logging.getLogger(__name__) + + +class ArchiveImporter: + """Imports an archive of files into the database. + + The archive can contain source code (within a 'reference-source' directory), + scan results (including SARIF), and other files.""" + + def __init__(self): + self.storage_manager = FileManager() + + def import_archive( + self, + filename: str, + archive: bytes, + project_version: ProjectVersion, + user: AbstractBaseUser | AnonymousUser | None = None, + ): + """ + Adds a file to the database. + """ + logger.debug("Importing archive: %s", filename) + + for file_info in self.extract_archive(filename, archive): + logger.debug("Processing file: %s", file_info.get("name")) + + if "/reference-binaries/" in file_info.get("name"): + logger.debug( + "File was in reference source directory, saving as source code." + ) + + # Extract each file, save it to the database + for source_file_info in self.extract_archive( + file_info.get("name"), file_info.get("content") + ): + logger.debug("Saving source code: %s", source_file_info.get("name")) + self.add_file( + source_file_info.get("content"), + source_file_info.get("name"), + project_version, + File.FileType.SOURCE_CODE, + save=False, + ) + + elif file_info.get("name").endswith("sarif"): + logger.debug("File was a SARIF file, saving as scan.") + self.add_file( + file_info.get("content"), + file_info.get("name"), + project_version, + File.FileType.SCAN_RESULT, + save=False, + ) + + logger.debug("File was a SARIF file, saving as scan.") + sarif_content = json.loads(file_info.get("content")) + SARIFImporter.import_sarif_file(sarif_content, project_version, user) + + else: + logger.debug("File was generic file, saving as a scan result.") + self.add_file( + file_info.get("content"), + file_info.get("name"), + project_version, + File.FileType.SCAN_RESULT, + save=False, + ) + + project_version.save() + + def add_file( + self, + content: bytes, + path: str, + project_version: ProjectVersion, + file_type: str, + save: bool = True, + ): + """Adds a file to storage.""" + mime_type = magic.from_buffer(content, mime=True) + + file_key = self.storage_manager.add_file(content, path) + # Files are unique by content, path, etc. + file = File.objects.get_or_create( + name=os.path.basename(path), + path=path, + content_type=mime_type, + file_key=file_key, + file_type=file_type, + )[0] + + project_version.files.add(file) + if save: + project_version.save() + + def extract_archive(self, file_path: str, file_content: bytes) -> dict: + """ + Extracts contents of an archive file. + """ + if file_path.endswith(".tgz") or file_path.endswith(".tar.gz"): + logger.debug("Detected archive type: tar.gz (%s)", file_path) + with tarfile.open(fileobj=io.BytesIO(file_content), mode="r") as tar: + for member in tar.getmembers(): + if member.isfile(): + content = tar.extractfile(member) + yield { + "name": member.name, + "path": member.name, + "size": member.size, + "content": content.read() if content is not None else None, + } + elif file_path.endswith(".zip"): + logger.debug("Detected archive type: zip (%s)", file_path) + with zipfile.ZipFile(io.BytesIO(file_content), mode="r") as zip_obj: + for member in zip_obj.infolist(): + if not member.is_dir(): # ZipInfo does not have is_file() + yield { + "name": member.filename, + "path": member.filename, + "size": member.file_size, + "content": zip_obj.read(member), + } + else: + logger.debug("File %s is not an archive", file_path) + return { + "name": file_path, + "path": file_path, + "size": len(file_content), + "content": file_content, + } diff --git a/omega/triage-portal/src/triage/util/finding_importers/file_importer.py b/omega/triage-portal/src/triage/util/finding_importers/file_importer.py new file mode 100644 index 00000000..bf68b3bd --- /dev/null +++ b/omega/triage-portal/src/triage/util/finding_importers/file_importer.py @@ -0,0 +1,73 @@ +import hashlib +import io +import logging +import mimetypes +import os +import tarfile +import zipfile + +from triage.models import File, FileContent, ProjectVersion, Scan + +logger = logging.getLogger(__name__) + + +class FileImporter: + @classmethod + def import_file( + cls, root: str, file_path: str, file_content: bytes, target: Scan | ProjectVersion + ) -> bool: + """ + Adds a file to the database. + """ + if root is None: + root = "" + + for file_info in cls.extract_archive(file_path, file_content): + logger.debug("Saving file %s", file_info.get("name")) + mime_type = mimetypes.guess_type(file_info.get("name"), strict=False)[0] + if mime_type is None: + mime_type = "application/octet-stream" + + content_hash = hashlib.sha256(file_info.get("content")).digest() + file_content = FileContent.objects.get_or_create( + hash=content_hash, + defaults={"content_type": mime_type, "data": file_info.get("content")}, + )[0] + file = File.objects.get_or_create( + name=os.path.basename(file_info.get("name")), + path=file_info.get("name"), + content=file_content, + )[0] + + if isinstance(target, (Scan, ProjectVersion)): + target.files.add(file) + + @classmethod + def extract_archive(cls, file_path: str, file_content: bytes) -> dict: + """ + Extracts contents of an archive file. + """ + if file_path.endswith(".tgz"): + logger.debug("Detected archive type: tar.gz (%s)", file_path) + with tarfile.open(fileobj=io.BytesIO(file_content), mode="r") as tar: + for member in tar.getmembers(): + if member.isfile(): + content = tar.extractfile(member) + yield { + "name": member.name, + "size": member.size, + "content": content.read() if content is not None else None, + } + elif file_path.endswith(".zip"): + logger.debug("Detected archive type: zip (%s)", file_path) + with zipfile.ZipFile(io.BytesIO(file_content), mode="r") as zip: + for member in zip.infolist(): + if member.is_file(): + yield { + "name": member.filename, + "size": member.file_size, + "content": zip.read(member), + } + else: + logger.debug("File %s is not an archive", file_path) + return {"name": file_path, "size": len(file_content), "content": file_content} diff --git a/omega/triage-portal/src/triage/util/finding_importers/sarif_importer.py b/omega/triage-portal/src/triage/util/finding_importers/sarif_importer.py new file mode 100644 index 00000000..bd2e3368 --- /dev/null +++ b/omega/triage-portal/src/triage/util/finding_importers/sarif_importer.py @@ -0,0 +1,223 @@ +# -*- coding: utf-8 -*- +"""This module provides support for import SARIF files into the Triage Portal's data model.""" + +import hashlib +import json +import logging +import os +import re +import uuid +from typing import Optional, Type + +from django.contrib.auth import get_user_model +from django.contrib.auth.models import AbstractBaseUser +from packageurl import PackageURL + +from triage.models import Finding, ProjectVersion, Scan, Tool, WorkItemState, File +from triage.util.general import get_complex + +logger = logging.getLogger(__name__) + + +class SARIFImporter: + """ + This class handles importing SARIF files into the database. + """ + + @classmethod + def import_sarif_file( + cls, sarif: dict, project_version: ProjectVersion, user: AbstractBaseUser | None + ) -> bool: + """ + Imports a SARIF file containing tool findings into the database. + + Args: + package_url: The PackageURL to attach all findings to. This PackageURL must + contain a version. + sarif: The SARIF content (as a dict) to import. + file_archive: The file archive containing the SARIF file. + + Returns: + True if the SARIF content was successfully imported, False otherwise. + """ + if sarif is None: + raise ValueError("The sarif content must not be None.") + + if sarif.get("version") != "2.1.0": + raise ValueError("Only SARIF version 2.1.0 is supported.") + + if project_version is None: + raise ValueError("The project version must not be None.") + + if user is None: + user = get_user_model().objects.get(id=1) # TODO: Fix this hardcoding + + num_imported = 0 + processed = set() # Reduce duplicates + + # First load all of the rules + for run in sarif.get("runs", []): + tool_name = get_complex(run, "tool.driver.name") + tool_version = get_complex(run, "tool.driver.version") + tool = Tool.objects.get_or_create( + name=tool_name, + version=tool_version, + defaults={ + "created_by": user, + "updated_by": user, + "type": Tool.ToolType.STATIC_ANALYSIS, + }, + )[0] + + logger.debug("Processing run for tool: %s", tool) + + rule_description_map = {} + for rule in get_complex(run, "tool.driver.rules"): + rule_id = get_complex(rule, "id") + rule_description = get_complex(rule, "shortDescription.text") + if rule_id and rule_description: + rule_description_map[rule_id] = rule_description + + for result in run.get("results", []): + rule_id = result.get("ruleId") + logger.debug("Saving result for rule #%s", rule_id) + + message = get_complex(result, "message.text") + level = get_complex(result, "level") + for location in get_complex(result, "locations"): + artifact_location = get_complex( + location, "physicalLocation.artifactLocation" + ) + + src_root = get_complex(artifact_location, "uriBaseId", "%SRCROOT%") + if str(src_root).upper() not in ["%SRCROOT%", "SRCROOT"]: + continue + + uri = get_complex(artifact_location, "uri") + + # Ensure we only insert the same message once + key = { + "title": message, + "path": uri, + "line_number": get_complex( + location, "physicalLocation.region.startLine" + ), + } + key = hashlib.sha256(json.dumps(key).encode("utf-8")).digest() + + if key not in processed: + logger.debug("New key for issue %s, adding.", message) + processed.add(key) + + file_path = get_complex(artifact_location, "uri") + file_path = cls.normalize_file_path(file_path) + + file = cls.get_most_likely_source(project_version, file_path) + if not file: + logger.debug("File not found, skipping.") + continue + + # Create the issue + finding = Finding() + finding.title = message + finding.normalized_title = cls.normalize_title(message) + finding.state = WorkItemState.NEW + finding.file = file + finding.tool = tool + finding.project_version = project_version + + finding.file_line = get_complex( + location, "physicalLocation.region.startLine", None + ) + finding.severity_level = Finding.SeverityLevel.parse(level) + finding.analyst_severity_level = ( + Finding.SeverityLevel.NOT_SPECIFIED + ) + finding.confidence = Finding.ConfidenceLevel.NOT_SPECIFIED + + finding.created_by = user + finding.updated_by = user + + if Finding.objects.filter( + title=finding.title, + file=finding.file, + file_line=finding.file_line, + project_version=finding.project_version, + ).exists(): + logger.debug("Duplicate finding, skipping.") + continue + + finding.save() + + num_imported += 1 + + if num_imported: + logger.debug("SARIF file successfully imported.") + return True + else: + logger.debug("SARIF file processed, but no issues were found.") + return False + + @classmethod + def normalize_file_path(cls, path): + """Normalizes a file path to be relative to the root.""" + logger.debug("normalize_file_path(%s)", path) + try: + result = path + if path.split("/")[2] == "package": + result = "/".join(path.split("/")[2:]) + logger.debug("Normalizing file path [%s] -> [%s]", path, result) + return result + except: + return path + + @classmethod + def normalize_title(cls, title): + norm = { + r"^Bracket object notation with user input is present.*": "Bracket object notation", + r"^Object injection via bracket notation.*": "Object injection", + r"^`ref` usage found.*": "Use of `ref`", + } + for regex, replacement in norm.items(): + if re.match(regex, title, re.IGNORECASE): + return replacement + return title + + @classmethod + def get_most_likely_source( + cls, project_version: ProjectVersion, file_path: str + ) -> File | None: + """Returns the most likely source file for a given issue.""" + + possible_files = project_version.files.filter( + path__endswith=os.path.basename(file_path) + ) + if not possible_files: + logger.debug("No files found for path %s, skipping.", file_path) + return None + + if len(possible_files) == 1: + logger.debug( + "Only one possible file found for path %s, using that one.", file_path + ) + return possible_files.first() + + # Let's make up a shortest-suffix algorithm (why not?!) + file_path = file_path.strip(os.path.sep) + parts = file_path.split(os.path.sep) + best_option = None + + # Iterate through increasingly large suffixes of the path, and see which files + # end with it. Only count the first one found at each level, since we have no + # other way to distinguish between them. + for i in range(len(parts) - 1, -1, -1): + target = os.path.sep + os.path.sep.join(parts[i:]) + logger.debug("New target: [%s]", target) + + for possible_file in possible_files: + if possible_file.path.endswith(target): + logger.debug("Best option is now [%s]", possible_file.path) + best_option = possible_file + break + + return best_option diff --git a/omega/triage-portal/src/triage/util/general.py b/omega/triage-portal/src/triage/util/general.py new file mode 100644 index 00000000..d31d8ae8 --- /dev/null +++ b/omega/triage-portal/src/triage/util/general.py @@ -0,0 +1,63 @@ +import logging +from datetime import datetime + +from django.utils import timezone +from django.utils.dateparse import parse_date as django_parse_date +from packageurl import PackageURL + +logger = logging.getLogger(__name__) + + +def get_complex(obj, key, default_value: str | None = ""): + """Get a value from the dictionary d by nested.key.value. + If keys contain periods, then use key=['a','b','c'] instead.""" + if not obj or not isinstance(obj, dict): + return default_value + _data = obj + try: + parts = key.split(".") if isinstance(key, str) else key + + for inner_key in parts: + _data = _data[inner_key] + return _data + except Exception: # pylint: disable=broad-except + return default_value + + +def modify_purl(purl: PackageURL, **kwargs) -> PackageURL: + """Modify a PackageURL by adding or replacing values in kwargs.""" + return PackageURL(**(purl.to_dict() | kwargs)) + + +def strtobool(value: str, default: bool) -> bool: + """Convert a string representation of truth to True or False. + True values are 'y', 'yes', 't', 'true', 'on', and '1'; + False values are 'n', 'no', 'f', 'false', 'off', and '0'. + Raises ValueError if the string is anything else. + """ + if isinstance(value, bool): + return value + value = str(value).lower() + if value in ("y", "yes", "t", "true", "on", "1"): + return True + elif value in ("n", "no", "f", "false", "off", "0"): + return False + return default + + +def parse_date(date_str: str) -> datetime | None: + """Converts a date string to a timezone-aware datetime object.""" + if date_str: + try: + parsed = django_parse_date(date_str) + if parsed: + parsed_dt = datetime(parsed.year, parsed.month, parsed.day) + if parsed_dt: + return timezone.make_aware(parsed_dt) + except Exception as msg: # pylint: disable=broad-except + logger.warning("Failed to parse date: %s", msg) + return None + +def clamp(value, min_value, max_value): + """Clamp a value between a minimum and maximum value.""" + return max(min_value, min(float(value), max_value)) diff --git a/omega/triage-portal/src/triage/util/search_parser.py b/omega/triage-portal/src/triage/util/search_parser.py new file mode 100644 index 00000000..f074619f --- /dev/null +++ b/omega/triage-portal/src/triage/util/search_parser.py @@ -0,0 +1,257 @@ +import datetime +import logging + +import django.db.models.base +import pyparsing as pp +from django.db.models import Model, Q +from django.utils import timezone + +from triage.models import Finding, WikiArticle, WorkItemState + +logger = logging.getLogger(__name__) + + +def parse_query_to_Q(model: Model, query: str) -> Q: + """ + Parse a query string into a Q object. + """ + + # Define the grammar + assigned_to_clause = pp.Group( + pp.Keyword("assigned_to").suppress() + + pp.Literal(":").suppress() + + pp.Word(pp.alphanums).setResultsName("username") + ).setResultsName("assigned_to") + + priority_clause = pp.Group( + pp.Keyword("priority").suppress() + + pp.Literal(":").suppress() + + pp.one_of(["<", ">", "<=", ">=", "==", "!="]).setResultsName("op") + + pp.Word(pp.nums).setResultsName("value") + ).setResultsName("priority") + + severity_clause = pp.Group( + pp.Keyword("severity").suppress() + + pp.Literal(":").suppress() + + pp.delimited_list( + pp.one_of( + [ + "critical", + "very high", + "vh", + "veryhigh", + "high", + "h", + "medium", + "m", + "low", + "very low", + "very_low", + "verylow", + "vl", + "informational", + "unknown", + ] + ) + ) + ).setResultsName("severity") + + updated_dt_clause = pp.Group( + pp.Keyword("updated").suppress() + + pp.Literal(":").suppress() + + pp.one_of(["<", ">", "<=", ">=", "==", "!="]).setResultsName("op") + + pp.pyparsing_common.iso8601_date("datetime") + ).setResultsName("updated_dt") + + created_dt_clause = pp.Group( + pp.Keyword("created").suppress() + + pp.Literal(":").suppress() + + pp.one_of(["<", ">", "<=", ">=", "==", "!="]).setResultsName("op") + + ( + pp.pyparsing_common.iso8601_date("datetime") + ^ ( + pp.one_of(["@today"]).setResultsName("anchor") + + pp.one_of(["+", "-"]).setResultsName("anchor_op") + + pp.Word(pp.nums).setResultsName("anchor_value") + ) + ) + ).setResultsName("created_dt") + + state_clause = pp.Group( + pp.Keyword("state").suppress() + + pp.Literal(":").suppress() + + pp.delimited_list( + pp.one_of( + [str(c[0]) for c in WorkItemState.choices] + + [str(c[1]) for c in WorkItemState.choices], + caseless=True, + ) + ).setResultsName("states") + ).setResultsName("state") + + purl_clause = pp.Group( + pp.Keyword("purl").suppress() + + pp.Literal(":").suppress() + + pp.Word(pp.alphanums + ":@/?=-.").setResultsName("purl") + ).setResultsName("purl") + + other_clause = pp.Word(pp.printables).setResultsName("text_search") + + available_attributes = [ + getattr(model, key).field.name + for key in dir(model) + if isinstance(getattr(model, key), django.db.models.query_utils.DeferredAttribute) + ] + + parser_elements = [] + if "assigned_to" in available_attributes: + parser_elements.append(assigned_to_clause) + if "severity_level" in available_attributes: + parser_elements.append(severity_clause) + if "created_at" in available_attributes: + parser_elements.append(created_dt_clause) + if "updated_at" in available_attributes: + parser_elements.append(updated_dt_clause) + if "priority" in available_attributes: + parser_elements.append(priority_clause) + if "state" in available_attributes: + parser_elements.append(state_clause) + if "package_url" in available_attributes: + parser_elements.append(purl_clause) + if model == Finding: # Special case for foreign key + parser_elements.append(purl_clause) + + parser_elements.append(other_clause) + parser_elements = list(dict.fromkeys(parser_elements)) # Unique only + + clause = parser_elements[0] + + if len(parser_elements) > 1: + for element in parser_elements[1:]: + clause |= element + + full_clause = pp.OneOrMore(clause) + + # Parse the query + try: + results = full_clause.parse_string(query) + except: + logger.error("Failed to parse query: %s", query) + return None + + # Assemble the Q object + q = Q() + if results.assigned_to: + q = q & Q(assigned_to__username=results.assigned_to.username) + + if results.severity: + severities = map(Finding.SeverityLevel.parse, results.severity.asList()) + q = q & Q(severity_level__in=list(severities)) + + if results.state: + states = map(WorkItemState.parse, results.state.asList()) + q = q & Q(state__in=list(states)) + + # Handle updated:$op$date or updated:$op@today[+-]$num + if results.updated_dt: + if results.updated_dt.datetime: + target = results.updated_dt.datetime + elif results.updated_dt.anchor: + if results.updated_dt.anchor == "@today": + target = timezone.now() + if results.updated_dt.anchor_op == "-": + target -= datetime.timedelta(days=int(results.updated_dt.anchor_value)) + elif results.updated_dt.anchor_op == "+": + target += datetime.timedelta(days=int(results.updated_dt.anchor_value)) + else: + raise ValueError("Unknown anchor: %s" % results.updated_dt.anchor) + else: + raise ValueError("Unknown updated_dt: %s" % results.updated_dt) + + if results.updated_dt.op == "<": + q = q & Q(updated_at__lt=target) + elif results.updated_dt.op == ">": + q = q & Q(updated_at__gt=target) + elif results.updated_dt.op == "<=": + q = q & Q(updated_at__lte=target) + elif results.updated_dt.op == ">=": + q = q & Q(updated_at__gte=target) + elif results.updated_dt.op == "==": + q = q & Q(updated_at__exact=target) + elif results.updated_dt.op == "!=": + q = q & ~Q(created_at__exact=target) + else: + logger.warning("Unknown updated_dt op: %s", results.updated_dt.op) + + if results.created_dt: + if results.created_dt.datetime: + target = results.created_dt.datetime + elif results.created_dt.anchor: + if results.created_dt.anchor == "@today": + target = timezone.now() + if results.created_dt.anchor_op == "-": + target -= datetime.timedelta(days=int(results.created_dt.anchor_value)) + elif results.created_dt.anchor_op == "+": + target += datetime.timedelta(days=int(results.created_dt.anchor_value)) + else: + raise ValueError("Unknown anchor: %s" % results.created_dt.anchor) + else: + raise ValueError("Unknown created_dt: %s" % results.created_dt) + + if results.created_dt.op == "<": + q = q & Q(created_at__lt=target) + elif results.created_dt.op == ">": + q = q & Q(created_at__gt=target) + elif results.created_dt.op == "<=": + q = q & Q(created_at__lte=target) + elif results.created_dt.op == ">=": + q = q & Q(created_at__gte=target) + elif results.created_dt.op == "==": + q = q & Q(created_at__exact=target) + elif results.created_dt.op == "!=": + q = q & ~Q(created_at__exact=target) + else: + logger.warning("Unknown created_dt op: %s", results.created_dt.op) + + if results.priority: + if results.priority.op == "<": + q = q & Q(priority__lt=results.priority.value) + elif results.priority.op == ">": + q = q & Q(priority__gt=results.priority.value) + elif results.priority.op == "<=": + q = q & Q(priority__lte=results.priority.value) + elif results.priority.op == ">=": + q = q & Q(priority__gte=results.priority.value) + elif results.priority.op == "==": + q = q & Q(priority__exact=results.priority.value) + elif results.priority.op == "!=": + q = q & ~Q(priority__exact=results.priority.value) + else: + logger.warning("Unknown priority op: %s", results.priority.op) + + if results.purl: + if "project_version" in available_attributes: + q = q & ( + Q(project_version__project__package_url=results.purl.purl) + | Q(project_version__package_url=results.purl.purl) + ) + if "package_url" in available_attributes: + q = q & Q(package_url=results.purl.purl) + + # Handle full text search a little differently + if results.text_search: + text_qq = Q() + if "project_version" in available_attributes: + text_qq |= Q(project_version__project__name__icontains=results.text_search) + if "title" in available_attributes: + text_qq |= Q(title__icontains=results.text_search) + if "description" in available_attributes: + text_qq |= Q(description__icontains=results.text_search) + + if model == WikiArticle: # @HACK: Special case for foreign key + text_qq |= Q(current__content__icontains=results.text_search) + + q = q & text_qq + + logger.debug("Query: %s", q) + return q diff --git a/omega/triage-portal/src/triage/util/source_viewer/__init__.py b/omega/triage-portal/src/triage/util/source_viewer/__init__.py new file mode 100644 index 00000000..b81a6ff6 --- /dev/null +++ b/omega/triage-portal/src/triage/util/source_viewer/__init__.py @@ -0,0 +1,130 @@ +import logging + +from django.db.models.query import QuerySet + +logger = logging.getLogger(__name__) + +""" +Example: +input: +["/foo", "/foo/bar", "/foo/bar/baz"] +output: +[{ + id: "/foo" + full_path: "/foo" +] + +""" + + +def path_to_graph(files: QuerySet, package_url, separator="/", root=None): + """ + Converts a list of paths into a graph suitable for jstree. + + Args: + paths: list of paths + separator: path separator + root: root directory to pin the graph to + + Returns: + a list of dictionaries containing the relevant + fields for jstree. + """ + if not files: + return [] + + result = [] + seen_nids = set() + if root: + result.append( + { + "id": root, + "full_path": "#", + "text": root, + "parent": "#", + "package_url": None, + "path": "/", + "file_id": None, + "icon": "fa fa-folder", + } + ) + else: + root = "#" + + for file in files: + path = file.path + if not isinstance(path, str) or not path or path.startswith("pkg:"): + logger.debug("Ignoring invalid path [%s]", path) + continue + + if not path.startswith(separator): + path = separator + path + + path_parts = path.split(separator)[1:] + + logging.debug(f"Analyzing: %s", path_parts) + for (part_id, part) in enumerate(path_parts): + if part_id == 0: + parent_id = root + node_id = part + else: + parent_id = separator.join(path_parts[:part_id]) + node_id = separator.join(path_parts[: (part_id + 1)]) + node_name = part + + if node_name and node_id not in seen_nids: + result.append( + { + "id": node_id, + "full_path": node_id, + "text": node_name, + "parent": parent_id, + "package_url": package_url, + "file_uuid": file.uuid, + "li_attr": {"package_url": package_url}, + "path": node_id, + "icon": get_icon_for_path(node_name, part_id == len(path_parts)), + } + ) + seen_nids.add(node_id) + return result + + +def get_icon_for_path(path: str, is_leaf_node: bool) -> str: + # if not is_leaf_node: + # return "fa fa-folder" + + icon_map = { + "application/javascript": "fa fa-code", + "text/x-python": "fa fa-code", + "application/json": "fa fa-code", + "text/html": "fab fa-html5", + "text/css": "fab fa-css3", + "text/markdown": "fab fa-markdown", + "text/plain": "fas fa-file-alt", + "application/pdf": "fas fa-file-pdf", + "application/zip": "far fa-file-archive", + "application/x-tar": "far fa-file-archive", + "text/csv": "fas fa-file-csv", + } + extension_map = { + ".cs": "fa fa-code", + ".log": "far fa-file-alt", + ".gz": "fa fa-file-archive", + ".error": "fas fa-exclamation-triangle", + ".sarif": "fas fa-bug", + } + import mimetypes + + for mime_type, css in icon_map.items(): + if mimetypes.guess_type(path)[0] == mime_type: + return css + + for extension, css in extension_map.items(): + if path.endswith(extension): + return css + + if "." not in path: + return "far fa-folder-open" + + return "fa fa-file-alt" # fallback, default diff --git a/omega/triage-portal/src/triage/util/source_viewer/pathsimilarity.py b/omega/triage-portal/src/triage/util/source_viewer/pathsimilarity.py new file mode 100644 index 00000000..07be428f --- /dev/null +++ b/omega/triage-portal/src/triage/util/source_viewer/pathsimilarity.py @@ -0,0 +1,134 @@ +import math +import os +import logging +from typing import Optional, List + +logger = logging.getLogger(__name__) + +class PathSimilarity: + def __init__(self): + raise NotImplementedError("This class is not intended to be instantiated.") + + @classmethod + def _normalize_path(cls, path: str) -> str: + """ + Attempts to normalize a path to aid searching. The results + are not necessarily valid paths (i.e. case insensitivity). + """ + if not path: + return None + path = path.replace("\\", "/") + + if not path.startswith("/"): + path = "/" + path + + if path.endswith("/"): + path = path[:-1] + + path = path.strip().lower() + return path + + + @classmethod + def get_path_similarity(cls, path1: str, path2: str) -> float: + """Estimates how similar the two paths are. + + Args: + path1: The first path to compare. + path2: The second path to compare. + + Returns: + A float between 0 and 1 indicating how similar the two paths are. + """ + logger.debug('get_path_similarity(%s, %s)', path1, path2) + if path1 == path2: + return 1.0 + + path1 = cls._normalize_path(path1) + path2 = cls._normalize_path(path2) + + if ( + not path1 # Invalid + or not path2 # Invalid + or path1.startswith("pkg:") # Not a path + or path2.startswith("pkg:") # Not a path + ): + return 0.0 + + # The paths must share the same basename + if os.path.basename(path1) != os.path.basename(path2): + return 0.0 + + # If one is a suffix of the other, then it's the best we can do. + if path1.endswith(path2) or path2.endswith(path1): + return 0.80 + + longest_suffix = cls.get_longest_common_suffix(path1, path2) + if longest_suffix: + suffix_dirs = longest_suffix.count('/') + 1 + min_common_dirs = min(path1.count('/'), path2.count('/')) + 1 + if suffix_dirs == min_common_dirs: + return 0.90 + else: + return min(math.sqrt(suffix_dirs / min_common_dirs), 0.90) + else: + return 0.0 + + @classmethod + def get_longest_common_suffix(cls, path1: str, path2: str) -> str: + """ + Calculate the longest common suffix of two strings. + Since these strings are going to be relatively small (path lengths), + we'll use a simple naiive algorithm. + + A common suffix is defined as the longest string that is a suffix of + both strings, but is also the start of a filename or directory. This + means that "foo.txt" is the common suffix of "/bar/foo.txt" and "/quux/foo.txt", + but is not the common suffix of "/barfoo.txt" and "/quux/foo.txt". + + Args: + path1: The first path to compare. + path2: The second path to compare. + + Returns: + The longest common suffix of the two paths, or None if there + is no common suffix. + """ + # Simplify the algorithm to reduce copy/paste. + cases = [(path1, path2), (path2, path1)] + longest_common_suffix = None + + for case in cases: + for index in range(len(case[0]), 0, -1): + subpath = case[0][index:] # Progressively longer suffixes + + is_suffix = case[1].endswith(subpath) + is_dir = subpath.startswith('/') or (index > 0 and case[0][index-1] == '/') + is_longest = index > 0 and case[0][index-1] == '/' + + if is_suffix and is_dir and is_longest: + longest_common_suffix = subpath + + return longest_common_suffix + + @classmethod + def find_most_similar_path(cls, target_paths: List[str], path: str) -> Optional[str]: + """ + Finds the path in the list that is most similar to the given path. + The similarity is calculated using the get_path_similarity function. + + Args: + target_paths: The list of paths to compare against. + path: The path to compare. + + Returns: + The path in the list that is most similar to the given path, or None + """ + best_similarity = 0.0 + best_path = None + for target in target_paths: + similarity = cls.get_path_similarity(target, path) + if similarity > best_similarity: + best_similarity = similarity + best_path = target + return best_path diff --git a/omega/triage-portal/src/triage/util/source_viewer/viewer.py b/omega/triage-portal/src/triage/util/source_viewer/viewer.py new file mode 100644 index 00000000..2fcc1636 --- /dev/null +++ b/omega/triage-portal/src/triage/util/source_viewer/viewer.py @@ -0,0 +1,96 @@ +import logging +import os +import shutil +import subprocess +import tempfile +from typing import Callable, List + +from django.core.cache import cache + +from core.settings import OSSGADGET_PATH +from triage.util.source_viewer.pathsimilarity import PathSimilarity + +logger = logging.getLogger(__name__) + +""" +Usage: +u = SourceViewer("pkg:npm/left-pad@1.3.0") +file = u.get_file("/index.js") +files = u.find_files(u => u.name == "index.js") +file = u.get_files() +""" + + +class SourceViewer: + def __init__(self, package_url): + self.package_url = str(package_url) + + def load_if_needed(self): + """Call OSS-Download to retrieve the package, extract it, and the load it into memory.""" + if cache.get(f"sv_{self.package_url}_exists") == 1: + return + + logger.debug("Loading source for package %s", self.package_url) + + with tempfile.TemporaryDirectory() as temp_directory: + res = subprocess.run( + ["./oss-download", "-e", "-x", temp_directory, self.package_url], + capture_output=True, + cwd=OSSGADGET_PATH, + ) + if res.returncode != 0 or not os.listdir(temp_directory): + logger.debug("Failed to load source for package %s", self.package_url) + raise Exception("Failed to download package") + + cache_updates = { + f"sv_{self.package_url}_exists": 1, + f"sv_{self.package_url}_files": set(), + } + for root, dirs, files in os.walk(temp_directory): + for file in files: + full_path = os.path.join(root, file) + relative_path = full_path[len(temp_directory) + 1 :].replace("\\", "/") + cache_updates[f"sv_{self.package_url}_files"].add(relative_path) + with open(full_path, "rb") as f: + file_cache_key = f"sv_{self.package_url}_{relative_path}" + cache_updates[file_cache_key] = f.read() + logger.debug( + "Adding %d entries to cache", len(cache_updates[f"sv_{self.package_url}_files"]) + ) + cache.set_many(cache_updates, timeout=60 * 60 * 8) + self._is_loaded = True + + def get_file(self, file_path: str) -> dict: + logger.debug("get_file(%s)", file_path) + if not file_path: + return None + + self.load_if_needed() + + target_path = PathSimilarity.find_most_similar_path(self.get_file_list(), file_path) + if target_path: + return { + "path": target_path, + "content": cache.get(f"sv_{self.package_url}_{target_path}"), + } + else: + return None + + def get_file_list(self): + self.load_if_needed() + return cache.get(f"sv_{self.package_url}_files") + + def get_files(self): + return self.find_files(lambda u, c: True) + + def find_files(self, lambda_filter: Callable[[str], str]) -> List[dict]: + self.load_if_needed() + + file_list = cache.get(f"sv_{self.package_url}_files") + for file_path in file_list: + print(file_path) + if lambda_filter(file_path): + yield { + "path": file_path, + "content": cache.get(f"sv_{self.package_url}_{file_path}"), + } diff --git a/omega/triage-portal/src/triage/views/attachments.py b/omega/triage-portal/src/triage/views/attachments.py new file mode 100644 index 00000000..b43807a1 --- /dev/null +++ b/omega/triage-portal/src/triage/views/attachments.py @@ -0,0 +1,27 @@ +import uuid + +from django.contrib.auth import get_user_model +from django.contrib.auth.decorators import login_required +from django.http import HttpRequest, HttpResponse +from django.shortcuts import get_object_or_404 +from django.views.decorators.http import require_http_methods + +from triage.models import Attachment + + +@login_required +@require_http_methods(["GET"]) +def download_attachment(request: HttpRequest, attachment_uuid: uuid.UUID) -> HttpResponse: + """Downloads an attachment + + Params: + attachment_uuid: UUID of the attachment to download. + """ + attachment = get_object_or_404(Attachment, uuid=attachment_uuid) + return HttpResponse( + attachment.content, + content_type=attachment.content_type, + headers={ + "Content-Disposition": f"attachment; filename={attachment.filename}", + }, + ) diff --git a/omega/triage-portal/src/triage/views/cases.py b/omega/triage-portal/src/triage/views/cases.py new file mode 100644 index 00000000..10f5f556 --- /dev/null +++ b/omega/triage-portal/src/triage/views/cases.py @@ -0,0 +1,122 @@ +import json +import os +import uuid +from base64 import b64encode +from typing import Any, List +from uuid import UUID + +from django.contrib.auth import get_user_model +from django.contrib.auth.decorators import login_required +from django.http import ( + HttpRequest, + HttpResponse, + HttpResponseBadRequest, + HttpResponseForbidden, + HttpResponseNotFound, + HttpResponseRedirect, + JsonResponse, +) +from django.shortcuts import get_object_or_404, render +from django.views.decorators.csrf import csrf_exempt +from django.views.decorators.http import require_http_methods +from packageurl import PackageURL + +from triage.models import Case, Project, ProjectVersion, WorkItemState +from triage.util.azure_blob_storage import ToolshedBlobStorageAccessor +from triage.util.finding_importers.sarif_importer import SARIFImporter +from triage.util.general import parse_date +from triage.util.search_parser import parse_query_to_Q +from triage.util.source_viewer import path_to_graph +from triage.util.source_viewer.viewer import SourceViewer + + +@login_required +@require_http_methods(["GET"]) +def show_cases(request: HttpRequest) -> HttpResponse: + """Shows cases based on a query. + + Params: + q: query to search for, or all findings if not provided + """ + query = request.GET.get("q", "").strip() + cases = Case.objects.all() # Default + if query: + query_object = parse_query_to_Q(Case, query) + if query_object: + cases = cases.filter(query_object) + + context = { + "query": query, + "cases": cases, + "case_states": WorkItemState.choices, + "reporting_partner": Case.CasePartner.choices, + } + + return render(request, "triage/case_list.html", context) + +@login_required +@require_http_methods(["GET"]) +def show_case(request: HttpRequest, case_uuid: UUID) -> HttpResponse: + """Shows a case.""" + case = get_object_or_404(Case, uuid=str(case_uuid)) + context = { + "case": case, + "case_states": WorkItemState.choices, + "reporting_partners": Case.CasePartner.choices, + "users": get_user_model().objects.all(), + } + return render(request, "triage/case_show.html", context) + + +@login_required +@require_http_methods(["GET"]) +def new_case(request: HttpRequest) -> HttpResponse: + """Shows the new case form.""" + context = { + "case_states": WorkItemState.choices, + "reporting_partners": Case.CasePartner.choices, + "users": get_user_model().objects.all(), + } + return render(request, "triage/case_show.html", context) + + +@login_required +@require_http_methods(["POST"]) +def save_case(request: HttpRequest) -> HttpResponse: + """Saves a case.""" + case_uuid = request.POST.get("case_uuid") + if not case_uuid: + case = Case() + case.created_by = request.user + else: + case = get_object_or_404(Case, uuid=case_uuid) + + case.title = request.POST.get("title") + case.state = request.POST.get("state") + case.description = request.POST.get("description") + + assigned_to = request.POST.get("assigned_to") + if assigned_to: + case.assigned_to = get_user_model().objects.get(username=assigned_to) + else: + case.assigned_to = None + + case.reported_to = request.POST.get("reported_to") + case.reporting_partner = request.POST.get("reporting_partner") + case.reporting_reference = request.POST.get("reporting_reference") + case.reported_dt = parse_date(request.POST.get("reported_dt")) + case.resolved_target_dt = parse_date(request.POST.get("resolved_target_dt")) + case.resolved_actual_dt = parse_date(request.POST.get("resolved_actual_dt")) + case.updated_by = request.user + + if not case.created_by: + case.created_by = request.user + + case.full_clean() + case.save() + + note_text = request.POST.get("note_text") + if note_text and note_text.strip(): + case.notes.create(content=note_text, created_by=request.user, updated_by=request.user) + + return HttpResponseRedirect(f"/cases/{case.uuid}") diff --git a/omega/triage-portal/src/triage/views/filters.py b/omega/triage-portal/src/triage/views/filters.py new file mode 100644 index 00000000..766913ad --- /dev/null +++ b/omega/triage-portal/src/triage/views/filters.py @@ -0,0 +1,106 @@ +from uuid import UUID + +from django.contrib.auth.decorators import login_required +from django.core.exceptions import ValidationError +from django.http import ( + HttpRequest, + HttpResponse, + HttpResponseBadRequest, + HttpResponseNotFound, + HttpResponseRedirect, + JsonResponse, +) +from django.shortcuts import get_object_or_404, redirect, render +from django.views.decorators.http import require_http_methods + +from triage.models import Filter +from triage.util.general import strtobool +from triage.util.search_parser import parse_query_to_Q + + +@require_http_methods(["GET"]) +def show_filters(request: HttpRequest) -> HttpResponse: + """Shows filters based on a query. + + Params: + q: query to search for, or all findings if not provided + """ + query = request.GET.get("q", "").strip() + filters = Filter.objects.all() # Default + if query: + query_object = parse_query_to_Q(Filter, query) + if query_object: + filters = filters.filter(query_object) + context = {"query": query, "filters": filters} + return render(request, "triage/filter_list.html", context) + + +@require_http_methods(["GET"]) +def new_filter(request: HttpRequest) -> HttpResponse: + """Show a form to create a new filter.""" + return render(request, "triage/filter_show.html") + + +@require_http_methods(["GET"]) +def show_filter(request: HttpRequest, filter_uuid: UUID) -> HttpResponse: + """Show a filter.""" + if filter_uuid: + filter = Filter.objects.get(uuid=str(filter_uuid)) + return render(request, "triage/filter_show.html", {"filter": filter}) + else: + return HttpResponseNotFound() + + +@login_required +@require_http_methods(["GET"]) +def execute_filter(request: HttpRequest) -> JsonResponse: + """Execute a filter.""" + filter_uuid = request.GET.get("filter_uuid") + if filter_uuid: + filter = Filter.objects.get(uuid=str(filter_uuid)) + filter.execute() + return JsonResponse({"status": "success"}) + else: + return HttpResponseBadRequest() + + +@login_required +@require_http_methods(["POST"]) +def delete_filter(request: HttpRequest) -> HttpResponse: + """Delete a filter.""" + filter_uuid = request.POST.get("filter_uuid") + if filter_uuid: + filter = get_object_or_404(Filter, uuid=str(filter_uuid)) + filter.delete() + return HttpResponseRedirect("/filter") + else: + return HttpResponseBadRequest() + + +@login_required +@require_http_methods(["POST"]) +def save_filter(request: HttpRequest) -> HttpResponse: + """Edit a filter.""" + filter_uuid = request.POST.get("filter_uuid") + if filter_uuid: + filter = Filter.objects.get(uuid=filter_uuid) + else: + filter = Filter() + filter.created_by = request.user + + filter.title = request.POST.get("title") + filter.condition = request.POST.get("condition") + filter.action = request.POST.get("action") + filter.active = strtobool(request.POST.get("active"), True) + filter.priority = int(request.POST.get("priority")) + filter.updated_by = request.user + + try: + filter.full_clean() + except ValidationError as e: + return render( + request, "triage/filter_edit.html", {"filter": filter, "error_messages": e.messages} + ) + filter.save() + + return HttpResponseRedirect(f"/filter/{filter.uuid}") diff --git a/omega/triage-portal/src/triage/views/findings.py b/omega/triage-portal/src/triage/views/findings.py new file mode 100644 index 00000000..691c4a30 --- /dev/null +++ b/omega/triage-portal/src/triage/views/findings.py @@ -0,0 +1,320 @@ +import json +import logging +import os +from base64 import b64encode + +from django.contrib.auth import get_user_model +from django.contrib.auth.decorators import login_required +from django.core.paginator import Paginator +from django.http import (HttpRequest, HttpResponse, HttpResponseBadRequest, + JsonResponse) +from django.shortcuts import get_object_or_404, redirect, render +from django.views.decorators.cache import cache_page +from django.views.decorators.csrf import csrf_exempt +from django.views.decorators.http import require_http_methods +from packageurl import PackageURL +from triage.models import (Case, File, Finding, + ProjectVersion, WorkItemState) +from triage.util.content_managers.file_manager import FileManager +from triage.util.finding_importers.archive_importer import ArchiveImporter +from triage.util.finding_importers.sarif_importer import SARIFImporter +from triage.util.general import clamp +from triage.util.search_parser import parse_query_to_Q +from triage.util.source_viewer import path_to_graph + +logger = logging.getLogger(__name__) + + +@login_required +def show_findings(request: HttpRequest) -> HttpResponse: + """Shows findings based on a query. + + Params: + q: query to search for, or all findings if not provided + """ + query = request.GET.get("q", "").strip() + page_size = clamp(request.GET.get("page_size", 20), 10, 500) + page = clamp(request.GET.get("page", 1), 1, 1000) + + findings = Finding.active_findings.all() + + if query: + findings = Finding.objects.exclude(state=WorkItemState.DELETED) + query_object = parse_query_to_Q(Finding, query) + if query_object: + findings = findings.filter(query_object) + + findings = findings.select_related("project_version", "tool", "file") + findings = findings.order_by("-project_version__package_url", "title", "created_at") + paginator = Paginator(findings, page_size) + page_object = paginator.get_page(page) + + query_string = request.GET.copy() + if "page" in query_string: + query_string.pop("page", None) + + context = { + "query": query, + "findings": page_object, + "params": query_string.urlencode(), + } + + return render(request, "triage/findings_list.html", context) + + +@login_required +@require_http_methods(["GET", "POST"]) +def show_upload(request: HttpRequest) -> HttpResponse: + """Show the upload form for findings (SARIF, etc.)""" + if request.method == "GET": + return render(request, "triage/findings_upload.html") + + if request.method == "POST": + package_url = request.POST.get("package_url") + files = request.FILES.getlist("sarif_file[]") + + if not files: + return HttpResponseBadRequest("No files provided") + + for file in files: + try: + importer = SARIFImporter.import_sarif_file( + package_url, json.load(file), request.user + ) + except: # pylint: disable=bare-except + logger.warning("Failed to import SARIF file", exc_info=True) + + return redirect("/findings/upload?status=success") + + +@login_required +def show_finding_by_uuid(request: HttpRequest, finding_uuid) -> HttpResponse: + finding = get_object_or_404(Finding, uuid=finding_uuid) + from django.contrib.auth.models import \ + User # pylint: disable=import-outside-toplevel + + assignee_list = User.objects.all() + context = {"finding": finding, "assignee_list": assignee_list} + return render(request, "triage/findings_show.html", context) + + +@login_required +@require_http_methods(["POST"]) +def api_update_finding(request: HttpRequest) -> JsonResponse: + """Updates a Finding.""" + + finding_uuid = request.POST.get("finding_uuid") + finding = get_object_or_404(Finding, uuid=finding_uuid) + # if not finding.can_edit(request.user): + # return HttpResponseForbidden() + + # Modify only these fields, if provided + permitted_fields = [ + "analyst_impact", + "confidence", + "analyst_severity_level", + "assigned_to", + "estimated_impact", + ] + is_modified = False + for field in permitted_fields: + if field in request.POST: + value = request.POST.get(field) + if field == "assigned_to": + if value == "$self": # Special case: set to current user + value = request.user + if value == "$clear": # Special case: clear the field + value = None + else: + value = get_user_model().objects.filter(username=value).first() + if value is None: + continue # No action, invalid user passed in + + if getattr(finding, field) != value: + setattr(finding, field, value) + is_modified = True + + if is_modified: + finding.save() + return JsonResponse({"status": "ok"}) + else: + return JsonResponse({"status": "ok, not modified"}) + + +@login_required +@cache_page(60 * 30) +def api_get_source_code(request: HttpRequest) -> JsonResponse: + """Returns the source code for a finding.""" + file_uuid = request.GET.get("file_uuid") + if file_uuid: + file = File.objects.filter(uuid=file_uuid).first() + if file and file.file_key: + file_manager = FileManager() + content = file_manager.get_file(file.file_key) + if content is not None: + return JsonResponse( + { + "file_contents": b64encode(content).decode("utf-8"), + "file_name": file.path, + "status": "ok", + } + ) + logger.info("Source code not found for %s", file_uuid) + return JsonResponse( + {"status": "error", "message": "File not found"}, status=404 + ) + +@login_required +@cache_page(60 * 5) +def api_get_files(request: HttpRequest) -> JsonResponse: + """Returns a list of files related to a finding.""" + project_version_uuid = request.GET.get("project_version_uuid") + project_version = get_object_or_404(ProjectVersion, uuid=project_version_uuid) + + source_graph = path_to_graph( + project_version.files.all(), + project_version.package_url, + separator="/", + root=str(project_version.package_url), + ) + + return JsonResponse({"data": source_graph, "status": "ok"}) + + +# @login_required +# def api_get_file_list(request: HttpRequest) -> JsonResponse: +# """Returns a list of files in a project.""" +# finding_uuid = request.GET.get("finding_uuid") +# finding = get_object_or_404(Finding, uuid=finding_uuid) +# source_code = finding.scan.get_file_list() +# source_graph = path_to_graph(source_code, finding.scan.project_version.package_url)# +# +# return JsonResponse({"data": source_graph, "status": "ok"}) + + +@login_required +def api_download_file(request: HttpRequest) -> HttpResponse: + """Returns a list of files in a project.""" + return HttpResponse("Not implemented.") + + +@login_required +def api_get_blob_list(request: HttpRequest) -> JsonResponse: + """Returns a list of files in a project.""" + finding_uuid = request.GET.get("finding_uuid") + finding = get_object_or_404(Finding, uuid=finding_uuid) + source_code = finding.scan.get_blob_list() + source_graph = path_to_graph( + map(lambda s: s.get("relative_path"), source_code), + finding.scan.project_version.package_url, + ) + + return JsonResponse({"data": source_graph, "status": "ok"}) + + +@csrf_exempt +@require_http_methods(["POST"]) +def api_add_scan_archive(request: HttpRequest) -> JsonResponse: + """Inserts data into the database. + + Required: + - scan_artifact => an archive of the content analyzed + - package_url => the package URL (must include version) + - scan_type => the type of scan (e.g. "toolshed") + - replace => replace existing scan if it exists (default: false) + """ + scan_artifact = request.FILES.get("scan_artifact") + if scan_artifact is None: + return JsonResponse({"error": "No scan_artifact provided"}) + + try: + package_url = PackageURL.from_string(request.POST.get("package_url")) + if package_url.version is None: + raise ValueError("Missing version") + except ValueError: + return JsonResponse({"error": "Invalid or missing package url"}) + + if request.user.is_anonymous: + user = get_user_model().objects.get(id=1) + else: + user = request.user + + project_version = ProjectVersion.get_or_create_from_package_url(package_url, user) + + archive_importer = ArchiveImporter() + archive_importer.import_archive( + scan_artifact.name, scan_artifact.read(), project_version, user + ) + + return JsonResponse({"success": True}) + +@csrf_exempt +@require_http_methods(["POST"]) +def api_add_artifact(request: HttpRequest) -> JsonResponse: + """Inserts data into the database. + + Required: + - artifact_type => "scan" or "package" + - action => "add" or "replace" (only used for package) + - package_url => the package URL (must include version) + - artifact => file contents to import + """ + action = request.FILES.get("action") + if action not in ["add", "update"]: + return JsonResponse({"error": "Invalid action"}) + + artifact_type = request.FILES.get("artifact_type") + if artifact_type is None: + return JsonResponse({"error": "No artifact type provided"}) + + try: + package_url = PackageURL.from_string(request.POST.get("package_url")) + if package_url.version is None: + raise ValueError("Missing version") + except ValueError: + return JsonResponse({"error": "Invalid or missing package url"}) + + # TODO: Make this better + user = get_user_model().objects.get(pk=1) + + if artifact_type == "sarif": + sarif = request.FILES.get("sarif") + sarif_content = json.load(sarif) + SARIFImporter.import_sarif_file(package_url, sarif_content, user) + + # Source code files are usually archives that contain source code or other + # artifacts. They'll be automatically extracted during the import process. They + # are associated with a ProjectVersion. + if "source_code" in request.FILES: + source_code = request.FILES.get("source_code") + source_code_content = source_code.read() + FileImporter.import_artifact(package_url, source_code_content, user) + + return JsonResponse({"success": True}) + + +def api_upload_attachment(request: HttpRequest) -> JsonResponse: + """Handles uploads (attachments)""" + target_type = request.POST.get("target_type") + target_uuid = request.POST.get("target_uuid") + if target_uuid is None or target_uuid == "": + return JsonResponse({"error": "No target_uuid provided"}) + + if target_type == "case": + obj = get_object_or_404(Case, uuid=target_uuid) + else: + return JsonResponse({"error": "Invalid target_type"}) + + attachments = request.FILES.getlist("attachment") + results = [] + for attachment in attachments: + new_attachment = obj.attachments.create( + filename=attachment.name, + content_type=attachment.content_type, + content=attachment.read(), + ) + results.append( + {"filename": new_attachment.filename, "uuid": new_attachment.uuid} + ) + + return JsonResponse({"success": True, "attachments": results}) diff --git a/omega/triage-portal/src/triage/views/home.py b/omega/triage-portal/src/triage/views/home.py new file mode 100644 index 00000000..458ca525 --- /dev/null +++ b/omega/triage-portal/src/triage/views/home.py @@ -0,0 +1,39 @@ +from datetime import timedelta + +from django.contrib.auth.decorators import login_required +from django.http import HttpRequest, HttpResponse +from django.shortcuts import render +from django.utils import timezone + +from triage.models import Case, Finding, ToolDefect + + +@login_required +def home(request: HttpRequest) -> HttpResponse: + most_recent_finding = Finding.objects.all().order_by("-updated_at") + finding_last_updated = most_recent_finding.first().created_at if most_recent_finding else None + + most_recent_case = Case.objects.all().order_by("-updated_at") + case_last_updated = most_recent_case.first().created_at if most_recent_case else None + + my_work = { + "num_cases": Case.objects.filter(assigned_to=request.user).count(), + "num_findings": Finding.objects.filter(assigned_to=request.user).count(), + "num_tool_defects": ToolDefect.objects.filter(assigned_to=request.user).count(), + } + metrics = { + "num_findings": Finding.objects.count(), + "num_active_findings": Finding.active_findings.count(), + "num_new_findings": Finding.active_findings.filter( + created_at__gt=timezone.now() - timedelta(days=7) + ).count(), + } + context = { + "finding_last_updated": finding_last_updated, + "case_last_updated": case_last_updated, + "my_work": my_work, + "user": request.user, + "metrics": metrics, + "last_week": timezone.now() - timedelta(days=7), + } + return render(request, "triage/home.html", context) diff --git a/omega/triage-portal/src/triage/views/tool_defect.py b/omega/triage-portal/src/triage/views/tool_defect.py new file mode 100644 index 00000000..b3267eeb --- /dev/null +++ b/omega/triage-portal/src/triage/views/tool_defect.py @@ -0,0 +1,121 @@ +import json +import os +import uuid +from base64 import b64encode +from typing import Any, List + +from django.contrib.auth.decorators import login_required +from django.http import ( + HttpRequest, + HttpResponse, + HttpResponseBadRequest, + HttpResponseForbidden, + HttpResponseNotFound, + HttpResponseRedirect, + JsonResponse, +) +from django.shortcuts import get_object_or_404, render +from django.views.decorators.cache import never_cache +from django.views.decorators.csrf import csrf_exempt +from django.views.decorators.http import require_http_methods +from packageurl import PackageURL + +from triage.models import ( + Case, + Finding, + Note, + Project, + ProjectVersion, + Tool, + ToolDefect, + WorkItemState, +) +from triage.util.azure_blob_storage import ToolshedBlobStorageAccessor +from triage.util.finding_importers.sarif_importer import SARIFImporter +from triage.util.search_parser import parse_query_to_Q +from triage.util.source_viewer import path_to_graph +from triage.util.source_viewer.viewer import SourceViewer + + +@login_required +def show_tool_defects(request: HttpRequest) -> HttpResponse: + """Shows tool_defectsbased on a query. + + Params: + q: query to search for, or all findings if not provided + """ + query = request.GET.get("q", "").strip() + tool_defects = ToolDefect.active_tool_defects.all() + + if query: + tool_defects = ToolDefect.objects.exclude(state=WorkItemState.DELETED) + query_object = parse_query_to_Q(ToolDefect, query) + if query_object: + tool_defects = tool_defects.filter(query_object) + + context = { + "query": query, + "tool_defects": tool_defects, + "tool_defect_states": WorkItemState.choices, + } + + return render(request, "triage/tool_defect_list.html", context) + + +@login_required +@never_cache +def show_tool_defect(request: HttpRequest, tool_defect_uuid: uuid.UUID) -> HttpResponse: + """Shows a tool defect.""" + tool_defect = get_object_or_404(ToolDefect, uuid=tool_defect_uuid) + context = { + "tool_defect": tool_defect, + "tools": Tool.objects.filter(active=True), + "tool_defect_states": WorkItemState.choices, + } + return render(request, "triage/tool_defect_show.html", context) + + +@login_required +def show_add_tool_defect(request: HttpRequest) -> HttpResponse: + """Shows the add tool defect form.""" + finding_uuid = request.GET.get("finding_uuid") + finding = get_object_or_404(Finding, uuid=finding_uuid) + c = { + "tools": Tool.objects.filter(active=True), + "tool_defect_states": WorkItemState.choices, + "finding": finding, + } + return render(request, "triage/tool_defect_show.html", c) + + +@login_required +@require_http_methods(["POST"]) +def save_tool_defect(request: HttpRequest) -> HttpResponse: + """Saves a tool defect.""" + + action = request.POST.get("action") + if action == "create": + tool_defect = ToolDefect() + else: + tool_defect = get_object_or_404(ToolDefect, uuid=request.POST["uuid"]) + + tool_defect.tool = get_object_or_404(Tool, uuid=request.POST["tool"]) + tool_defect.title = request.POST["title"] + tool_defect.state = request.POST["state"] + tool_defect.description = request.POST["description"] + note_content = request.POST.get("note_content") + tool_defect.save() + + finding_uuid = request.POST.get("finding_uuid"); + if finding_uuid: + finding = get_object_or_404(Finding, uuid=finding_uuid) + tool_defect.findings.add(finding) + tool_defect.save() + + if note_content and note_content.strip(): + note = Note(content=note_content, created_by=request.user, updated_by=request.user) + note.save() + tool_defect.notes.add(note) + tool_defect.save() + + return HttpResponseRedirect(f"/tool_defect/{tool_defect.uuid}") diff --git a/omega/triage-portal/src/triage/views/wiki.py b/omega/triage-portal/src/triage/views/wiki.py new file mode 100644 index 00000000..6f52b5a2 --- /dev/null +++ b/omega/triage-portal/src/triage/views/wiki.py @@ -0,0 +1,161 @@ +import json +import os +import uuid +from base64 import b64encode +from typing import Any, List +from uuid import UUID + +from django.contrib.auth import get_user_model +from django.contrib.auth.decorators import login_required +from django.http import ( + HttpRequest, + HttpResponse, + HttpResponseBadRequest, + HttpResponseForbidden, + HttpResponseNotFound, + HttpResponseRedirect, + JsonResponse, +) +from django.shortcuts import get_object_or_404, redirect, render +from django.utils.text import slugify +from django.views.decorators.cache import never_cache +from django.views.decorators.csrf import csrf_exempt +from django.views.decorators.http import require_http_methods +from packageurl import PackageURL + +from triage.models import ( + Case, + Project, + ProjectVersion, + WikiArticle, + WikiArticleRevision, + WorkItemState, +) +from triage.util.azure_blob_storage import ToolshedBlobStorageAccessor +from triage.util.finding_importers.sarif_importer import SARIFImporter +from triage.util.general import parse_date +from triage.util.search_parser import parse_query_to_Q +from triage.util.source_viewer import path_to_graph +from triage.util.source_viewer.viewer import SourceViewer + + +@login_required +@require_http_methods(["GET"]) +def home(request: HttpRequest) -> HttpResponse: + return HttpResponseRedirect("/wiki/home") + + +@login_required +@require_http_methods(["GET"]) +@never_cache +def show_wiki_article_list(request: HttpRequest) -> HttpResponse: + """Shows all wiki articles.""" + wiki_articles = WikiArticle.active_wiki_articles.all() + + query = request.GET.get("q", "").strip() + wiki_articles = WikiArticle.objects.all() # Default + if query: + query_object = parse_query_to_Q(WikiArticle, query) + if query_object: + wiki_articles = wiki_articles.filter(query_object) + + context = {"query": query, "wiki_articles": wiki_articles} + return render(request, "triage/wiki_list.html", context) + + +@login_required +@require_http_methods(["GET"]) +def show_wiki_article( + request: HttpRequest, slug: str, template: str = "triage/wiki_show.html" +) -> HttpResponse: + """Shows a wiki article (current revision).""" + article = WikiArticle.objects.filter(slug=slug).first() + if article: + context = { + "wiki_article": article, + "wiki_article_revision": article.current, + "wiki_article_states": WorkItemState.choices, + } + return render(request, template, context) + else: + if slug == "new": + slug = "" + context = { + "wiki_article": { + "slug": slug, + "wiki_article_states": WorkItemState.choices, + } + } + return render(request, "triage/wiki_edit.html", context) + + +def edit_wiki_article(request: HttpRequest, slug: str) -> HttpResponse: + return show_wiki_article(request, slug, "triage/wiki_edit.html") + + +@login_required +@require_http_methods(["GET"]) +def show_wiki_article_revision( + request: HttpRequest, + slug: str, + wiki_article_revision_uuid: UUID, + template: str = "triage/wiki_show.html", +) -> HttpResponse: + """Shows a wiki article (current revision).""" + article_revision = get_object_or_404( + WikiArticleRevision, article__slug=slug, uuid=wiki_article_revision_uuid + ) + context = { + "wiki_article": article_revision.article, + "wiki_article_revision": article_revision, + "wiki_article_states": WorkItemState.choices, + } + return render(request, template, context) + + +def edit_wiki_article_revision( + request: HttpRequest, slug: str, wiki_article_revision_uuid: UUID +) -> HttpResponse: + return show_wiki_article_revision( + request, slug, wiki_article_revision_uuid, "triage/wiki_edit.html" + ) + + +@login_required +@require_http_methods(["POST"]) +def save_wiki_article(request: HttpRequest) -> HttpResponse: + """Saves a wiki article. + + Required fields: + wiki_article_uuid: UUID of the wiki article to update. If not provided, a new + wiki article will be created. + title: Title of the wiki article. + content: Content of the wiki article. + state: State of the wiki article. + change_comment: Comment for the change. + """ + wiki_article_uuid = request.POST.get("wiki_article_uuid") + if not wiki_article_uuid: + wiki_article = WikiArticle() + else: + wiki_article = get_object_or_404(WikiArticle, uuid=wiki_article_uuid) + + slug = request.POST.get("slug") + if not slug: + slug = slugify(request.POST.get("title")) + wiki_article.state = request.POST.get("state") + wiki_article.slug = slug + wiki_article.save() + + wiki_article_revision = WikiArticleRevision( + title=request.POST.get("title"), + content=request.POST.get("content"), + article=wiki_article, + change_comment=request.POST.get("change_comment"), + created_by=request.user, + updated_by=request.user, + ) + wiki_article_revision.full_clean() + wiki_article_revision.save() + + return redirect(wiki_article) diff --git a/omega/triage-portal/src/yarn.lock b/omega/triage-portal/src/yarn.lock new file mode 100644 index 00000000..56392877 --- /dev/null +++ b/omega/triage-portal/src/yarn.lock @@ -0,0 +1,187 @@ +# THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY. +# yarn lockfile v1 + + +"@fortawesome/fontawesome-free@^5.7.2": + version "5.15.4" + resolved "https://registry.yarnpkg.com/@fortawesome/fontawesome-free/-/fontawesome-free-5.15.4.tgz#ecda5712b61ac852c760d8b3c79c96adca5554e5" + integrity sha512-eYm8vijH/hpzr/6/1CJ/V/Eb1xQFW2nnUKArb3z+yUWv7HTwj6M7SP957oMjfZjAHU6qpoNc2wQvIxBLWYa/Jg== + +"@popperjs/core@^2.10.1": + version "2.10.2" + resolved "https://registry.yarnpkg.com/@popperjs/core/-/core-2.10.2.tgz#0798c03351f0dea1a5a4cabddf26a55a7cbee590" + integrity sha512-IXf3XA7+XyN7CP9gGh/XB0UxVMlvARGEgGXLubFICsUMGz6Q+DU+i4gGlpOxTjKvXjkJDJC8YdqdKkDj9qZHEQ== + +ace-builds@^1.4.2: + version "1.4.13" + resolved "https://registry.yarnpkg.com/ace-builds/-/ace-builds-1.4.13.tgz#186f42d3849ebcc6a48b93088a058489897514c1" + integrity sha512-SOLzdaQkY6ecPKYRDDg+MY1WoGgXA34cIvYJNNoBMGGUswHmlauU2Hy0UL96vW0Fs/LgFbMUjD+6vqzWTldIYQ== + +add@^2.0.6: + version "2.0.6" + resolved "https://registry.yarnpkg.com/add/-/add-2.0.6.tgz#248f0a9f6e5a528ef2295dbeec30532130ae2235" + integrity sha1-JI8Kn25aUo7yKV2+7DBTITCuIjU= + +argparse@^1.0.7: + version "1.0.10" + resolved "https://registry.yarnpkg.com/argparse/-/argparse-1.0.10.tgz#bcd6791ea5ae09725e17e5ad988134cd40b3d911" + integrity sha512-o5Roy6tNG4SL/FOkCAN6RzjiakZS25RLYFrcMttJqbdd8BWrnA+fGz57iN5Pb06pvBGvl5gQ0B48dJlslXvoTg== + dependencies: + sprintf-js "~1.0.2" + +blueimp-canvas-to-blob@3: + version "3.29.0" + resolved "https://registry.yarnpkg.com/blueimp-canvas-to-blob/-/blueimp-canvas-to-blob-3.29.0.tgz#d965f06cb1a67fdae207a2be56683f55ef531466" + integrity sha512-0pcSSGxC0QxT+yVkivxIqW0Y4VlO2XSDPofBAqoJ1qJxgH9eiUDLv50Rixij2cDuEfx4M6DpD9UGZpRhT5Q8qg== + +blueimp-file-upload@^10.1.0: + version "10.32.0" + resolved "https://registry.yarnpkg.com/blueimp-file-upload/-/blueimp-file-upload-10.32.0.tgz#897c91f813ccf4b6cd14db2da3da237ce433e04e" + integrity sha512-3WMJw5Cbfz94Adl1OeyH+rRpGwHiNHzja+CR6aRWPoAtwrUwvP5gXKo0XdX+sdPE+iCU63Xmba88hoHQmzY8RQ== + optionalDependencies: + blueimp-canvas-to-blob "3" + blueimp-load-image "5" + blueimp-tmpl "3" + +blueimp-load-image@5: + version "5.16.0" + resolved "https://registry.yarnpkg.com/blueimp-load-image/-/blueimp-load-image-5.16.0.tgz#16b763f57e6725f8865517bca8eb7c3dc7d41e09" + integrity sha512-3DUSVdOtlfNRk7moRZuTwDmA3NnG8KIJuLcq3c0J7/BIr6X3Vb/EpX3kUH1joxUhmoVF4uCpDfz7wHkz8pQajA== + +blueimp-tmpl@3: + version "3.20.0" + resolved "https://registry.yarnpkg.com/blueimp-tmpl/-/blueimp-tmpl-3.20.0.tgz#bed897db362c70d3740e0ad1020ce84bfa15ceaa" + integrity sha512-g6ln9L+VX8ZA4WA8mgKMethYH+5teroJ2uOkCvcthy9Y9d9LrQ42OAMn+r3ECKu9CB+xe9GOChlIUJBSxwkI6g== + +bootstrap@^5.1.0: + version "5.1.3" + resolved "https://registry.yarnpkg.com/bootstrap/-/bootstrap-5.1.3.tgz#ba081b0c130f810fa70900acbc1c6d3c28fa8f34" + integrity sha512-fcQztozJ8jToQWXxVuEyXWW+dSo8AiXWKwiSSrKWsRB/Qt+Ewwza+JWoLKiTuQLaEPhdNAJ7+Dosc9DOIqNy7Q== + +datatables.net-bs5@>=1.10.25, datatables.net-bs5@^1.11.3: + version "1.11.3" + resolved "https://registry.yarnpkg.com/datatables.net-bs5/-/datatables.net-bs5-1.11.3.tgz#939d0e66fbf518718a519534a88fc88cd29405b0" + integrity sha512-u0tosKUR1XNpXzxOOt2NInnNYayt7GQoG+OM1xPRhdkZ7ZBD4oNF8S0aKve8yvSUq/ZwTMh4WJeh80GdmrJAdQ== + dependencies: + datatables.net ">=1.10.25" + jquery ">=1.7" + +datatables.net-fixedheader-bs5@^3.2.0: + version "3.2.0" + resolved "https://registry.yarnpkg.com/datatables.net-fixedheader-bs5/-/datatables.net-fixedheader-bs5-3.2.0.tgz#a5b180a34dded5e526fd7d8ba855b493af76f996" + integrity sha512-OT7RpaDWaOYzDScZwFk8KSsfIJVkGN6BLlpQy/KceDOHSwO7BQShETO1l7svQpoTOX9edMP3BQIxF03swUHMAA== + dependencies: + datatables.net-bs5 ">=1.10.25" + datatables.net-fixedheader ">=3.1.9" + jquery ">=1.7" + +datatables.net-fixedheader@>=3.1.9: + version "3.2.0" + resolved "https://registry.yarnpkg.com/datatables.net-fixedheader/-/datatables.net-fixedheader-3.2.0.tgz#5d4891942ecf2a66ff2dfb1be54802cfa6c5927e" + integrity sha512-p8GfXhOLBv94uFmgUSKthNw1Sd9szBMqEHHe0DwRsbBR/4WworYzaeysy45HPoBh1tL6abWZC7gW6I4nLGtzlQ== + dependencies: + datatables.net ">=1.10.25" + jquery ">=1.7" + +datatables.net-keytable-bs5@^2.6.4: + version "2.6.4" + resolved "https://registry.yarnpkg.com/datatables.net-keytable-bs5/-/datatables.net-keytable-bs5-2.6.4.tgz#1adaabb5af4099f2c97dc55aa2f9d68cc31cc2a0" + integrity sha512-SHDgrWTHOtrPeuBHepla+GKx0SwpDc00M9zd0WlKtjpAK89TQBO5h0n4xKgycvjbBEA0XVAzyyW0ufE/OklELg== + dependencies: + datatables.net-bs5 ">=1.10.25" + datatables.net-keytable ">=2.6.2" + jquery ">=1.7" + +datatables.net-keytable@>=2.6.2: + version "2.6.4" + resolved "https://registry.yarnpkg.com/datatables.net-keytable/-/datatables.net-keytable-2.6.4.tgz#0cb355441a87d1c7aa6d230641157d17829892a2" + integrity sha512-84ldg8S3rAzHKJw4inz/a4lWUWaVhrSrH16+zO9sxUdBgT+rTTjx7aN5d3JwrdroEYB3z2uWfd0MOxNlNXeuUQ== + dependencies: + datatables.net ">=1.10.25" + jquery ">=1.7" + +datatables.net-select@^1.3.0: + version "1.3.3" + resolved "https://registry.yarnpkg.com/datatables.net-select/-/datatables.net-select-1.3.3.tgz#9259f70bf257e5d32198f38424d005808b78dc6d" + integrity sha512-M4e9Qx790IPt+tc+CLgk7gPram3i+M2OmhIkhIpp7RcZ2Ay4App4ouQZcEx3j1MTRIWxtOz47xjpWrwVfJ23YQ== + dependencies: + datatables.net "^1.10.15" + jquery ">=1.7" + +datatables.net@>=1.10.25, datatables.net@^1.10.15: + version "1.11.3" + resolved "https://registry.yarnpkg.com/datatables.net/-/datatables.net-1.11.3.tgz#80e691036efcd62467558ee64c07dd566cb761b4" + integrity sha512-VMj5qEaTebpNurySkM6jy6sGpl+s6onPK8xJhYr296R/vUBnz1+id16NVqNf9z5aR076OGcpGHCuiTuy4E05oQ== + dependencies: + jquery ">=1.7" + +esprima@^4.0.0: + version "4.0.1" + resolved "https://registry.yarnpkg.com/esprima/-/esprima-4.0.1.tgz#13b04cdb3e6c5d19df91ab6987a8695619b0aa71" + integrity sha512-eGuFFw7Upda+g4p+QHvnW0RyTX/SVeJBDM/gCtMARO0cLuT2HcEKnTPvhjV6aGeqrCB/sbNop0Kszm0jsaWU4A== + +inconsolata@^0.0.2: + version "0.0.2" + resolved "https://registry.yarnpkg.com/inconsolata/-/inconsolata-0.0.2.tgz#baa1433f43d42aa1edb26bd9ee87738e929250eb" + integrity sha1-uqFDP0PUKqHtsmvZ7odzjpKSUOs= + +jquery-contextmenu@^2.8.0: + version "2.9.2" + resolved "https://registry.yarnpkg.com/jquery-contextmenu/-/jquery-contextmenu-2.9.2.tgz#f9dc362e45871dda2e50fa45de2243e917446ced" + integrity sha512-6S6sH/08owDStC/7zNwcN366yR0ydX6PmMB0RnjLRQOp7Nc/rqwEHglshfHrrw2kdTev97GXwRXrayDUmToIOw== + dependencies: + jquery "^3.5.0" + +jquery@>=1.2.6, jquery@>=1.7, jquery@>=1.9.1, jquery@^3.3.1, jquery@^3.5.0: + version "3.6.0" + resolved "https://registry.yarnpkg.com/jquery/-/jquery-3.6.0.tgz#c72a09f15c1bdce142f49dbf1170bdf8adac2470" + integrity sha512-JVzAR/AjBvVt2BmYhxRCSYysDsPcssdmTFnzyLEts9qNwmjmu4JTAMYubEfwVOSwpQ1I1sKKFcxhZCI2buerfw== + +js-yaml@^3.13.0: + version "3.14.1" + resolved "https://registry.yarnpkg.com/js-yaml/-/js-yaml-3.14.1.tgz#dae812fdb3825fa306609a8717383c50c36a0537" + integrity sha512-okMH7OXXJ7YrN9Ok3/SXrnu4iX9yOk+25nqX4imS2npuvTYDmo/QEZoqwZkYaIDk3jVvBOTOIEgEhaLOynBS9g== + dependencies: + argparse "^1.0.7" + esprima "^4.0.0" + +jstree-bootstrap-theme@^1.0.1: + version "1.0.1" + resolved "https://registry.yarnpkg.com/jstree-bootstrap-theme/-/jstree-bootstrap-theme-1.0.1.tgz#7d5edc73a846e8da7f94f57a1cc5ddee9d9eab4b" + integrity sha1-fV7cc6hG6Np/lPV6HMXd7p2eq0s= + dependencies: + jquery ">=1.9.1" + +jstree@^3.3.11: + version "3.3.12" + resolved "https://registry.yarnpkg.com/jstree/-/jstree-3.3.12.tgz#cf206bc85dcf4a4664ed6617eaae3bd5983d8601" + integrity sha512-vHNLWkUr02ZYH7RcIckvhtLUtneWCVEtIKpIp2G9WtRh01ITv18EoNtNQcFG3ozM+oK6wp1Z300gSLXNQWCqGA== + dependencies: + jquery ">=1.9.1" + +select2@^4.1.0-rc.0: + version "4.1.0-rc.0" + resolved "https://registry.yarnpkg.com/select2/-/select2-4.1.0-rc.0.tgz#ba3cd3901dda0155e1c0219ab41b74ba51ea22d8" + integrity sha512-Hr9TdhyHCZUtwznEH2CBf7967mEM0idtJ5nMtjvk3Up5tPukOLXbHUNmh10oRfeNIhj+3GD3niu+g6sVK+gK0A== + +source-code-pro@^2.30.2: + version "2.38.0" + resolved "https://registry.yarnpkg.com/source-code-pro/-/source-code-pro-2.38.0.tgz#85c57689f7386bb9d0515fb00ba4845bfb7b485b" + integrity sha512-JMXu7l3XrLREG17eEwY66ANG9716WTu6OeNvZfRKQKANEvbSERDZjk5AYTHeV6owQNPQTeiiW3ri2Ou93XFGvg== + +sprintf-js@~1.0.2: + version "1.0.3" + resolved "https://registry.yarnpkg.com/sprintf-js/-/sprintf-js-1.0.3.tgz#04e6926f662895354f3dd015203633b857297e2c" + integrity sha1-BOaSb2YolTVPPdAVIDYzuFcpfiw= + +tablesorter@^2.31.1: + version "2.31.3" + resolved "https://registry.yarnpkg.com/tablesorter/-/tablesorter-2.31.3.tgz#94c33234ba0e5d9efc5ba4e48651010a396c8b64" + integrity sha512-ueEzeKiMajDcCWnUoT1dOeNEaS1OmPh9+8J0O2Sjp3TTijMygH74EA9QNJiNkLJqULyNU0RhbKY26UMUq9iurA== + dependencies: + jquery ">=1.2.6" + +yarn@^1.13.0: + version "1.22.17" + resolved "https://registry.yarnpkg.com/yarn/-/yarn-1.22.17.tgz#bf910747d22497b573131f7341c0e1d15c74036c" + integrity sha512-H0p241BXaH0UN9IeH//RT82tl5PfNraVpSpEoW+ET7lmopNC61eZ+A+IDvU8FM6Go5vx162SncDL8J1ZjRBriQ==