Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Migrate code from ossf/omega-triage-portal to alpha-omega. #55

Merged
merged 22 commits into from
Jan 4, 2023
Merged
Show file tree
Hide file tree
Changes from 8 commits
Commits
Show all changes
22 commits
Select commit Hold shift + click to select a range
a74a81f
Migrate code from ossf/omega-triage-portal to alpha-omega.
scovetta Dec 4, 2022
f52c02b
Move dev container to root of repo.
scovetta Dec 4, 2022
7550d2c
Fiddle with Codespaces to get it work in dir.
scovetta Dec 4, 2022
26f6818
Upgrade dependencies, tweak codespaces config.
scovetta Dec 5, 2022
824985e
Fix home page view bug when database is empty.
scovetta Dec 5, 2022
855741c
Improve triage portal readme.
scovetta Dec 5, 2022
89b305a
Tweaked requirements to get around incompatible semvers.
scovetta Dec 5, 2022
c3d0c36
Updated T-P Readme for missed markdown bracket
Cyber-JiuJiteria Dec 8, 2022
9e005ab
Update UUID fields to be unique explicitly.
scovetta Dec 15, 2022
5661941
Add support for prioritity "not-equals".
scovetta Dec 15, 2022
4b42ce1
Added support for != to created/updated date.
scovetta Dec 15, 2022
2129da2
Remove some unused code.
scovetta Dec 15, 2022
3c60a47
Fix linting config
scovetta Dec 17, 2022
c10fe85
Starting refactor of file storage.
scovetta Dec 18, 2022
79dd75d
Add zstd and magic modules.
scovetta Dec 18, 2022
c5c25c2
Remove null bytes from FileManager source code
scovetta Dec 18, 2022
e281db5
Clean up (remove SCIM refs, add comments)
scovetta Dec 18, 2022
fae8d9c
Improvements to SARIF import (finding right file)
scovetta Dec 19, 2022
3d2b003
Minor UI improvements, move file uploads to $HOME.
scovetta Dec 19, 2022
39b17ec
Minor fixes to tool defects.
scovetta Dec 21, 2022
b828951
Clean up TODO comment
scovetta Jan 4, 2023
9736427
Merge branch 'main' into scovetta/add-triage-portal
scovetta Jan 4, 2023
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
30 changes: 30 additions & 0 deletions .devcontainer/triage-portal/Dockerfile
Original file line number Diff line number Diff line change
@@ -0,0 +1,30 @@
# [Choice] Python version (use -bullseye variants on local arm64/Apple Silicon): 3, 3.10, 3.9, 3.8, 3.7, 3.6, 3-bullseye, 3.10-bullseye, 3.9-bullseye, 3.8-bullseye, 3.7-bullseye, 3.6-bullseye, 3-buster, 3.10-buster, 3.9-buster, 3.8-buster, 3.7-buster, 3.6-buster
ARG VARIANT=3-bullseye
FROM mcr.microsoft.com/vscode/devcontainers/python:0-${VARIANT}

ENV PYTHONUNBUFFERED 1

# [Choice] Node.js version: none, lts/*, 16, 14, 12, 10
ARG NODE_VERSION="none"
RUN if [ "${NODE_VERSION}" != "none" ]; then su vscode -c "umask 0002 && . /usr/local/share/nvm/nvm.sh && nvm install ${NODE_VERSION} 2>&1"; fi

# Install OSS Gadget
# License: MIT
ARG OSSGADGET_VERSION="0.1.307"
RUN cd /opt && \
wget -q https://github.com/microsoft/OSSGadget/releases/download/v${OSSGADGET_VERSION}/OSSGadget_linux_${OSSGADGET_VERSION}.tar.gz -O OSSGadget.tar.gz && \
tar zxvf OSSGadget.tar.gz && \
rm OSSGadget.tar.gz && \
mv OSSGadget_linux_${OSSGADGET_VERSION} OSSGadget

# [Optional] If your requirements rarely change, uncomment this section to add them to the image.
# COPY requirements.txt /tmp/pip-tmp/
# RUN pip3 --disable-pip-version-check --no-cache-dir install -r /tmp/pip-tmp/requirements.txt \
# && rm -rf /tmp/pip-tmp

# [Optional] Uncomment this section to install additional OS packages.
# RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \
# && apt-get -y install --no-install-recommends <your-package-list-here>



68 changes: 68 additions & 0 deletions .devcontainer/triage-portal/devcontainer.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,68 @@
// For format details, see https://aka.ms/devcontainer.json. For config options, see the README at:
// https://github.com/microsoft/vscode-dev-containers/tree/v0.203.0/containers/python-3-postgres
// Update the VARIANT arg in docker-compose.yml to pick a Python version
{
"name": "Python 3 & PostgreSQL",
"dockerComposeFile": "docker-compose.yml",
"service": "app",
"workspaceFolder": "/workspaces",
// Set *default* container specific settings.json values on container create.
"settings": {
"sqltools.connections": [
{
"name": "Container database",
"driver": "PostgreSQL",
"previewLimit": 50,
"server": "localhost",
"port": 5432,
"database": "triage",
"username": "triage_user",
"password": "triage_password"
}
],
"python.pythonPath": "/usr/local/bin/python",
"python.languageServer": "Pylance",
"python.linting.enabled": true,
"python.linting.pylintEnabled": true,
"python.formatting.autopep8Path": "/usr/local/py-utils/bin/autopep8",
"python.formatting.blackPath": "/usr/local/py-utils/bin/black",
"python.formatting.yapfPath": "/usr/local/py-utils/bin/yapf",
"python.linting.banditPath": "/usr/local/py-utils/bin/bandit",
"python.linting.flake8Path": "/usr/local/py-utils/bin/flake8",
"python.linting.mypyPath": "/usr/local/py-utils/bin/mypy",
"python.linting.pycodestylePath": "/usr/local/py-utils/bin/pycodestyle",
"python.linting.pydocstylePath": "/usr/local/py-utils/bin/pydocstyle",
"python.linting.pylintPath": "/usr/local/py-utils/bin/pylint",
"python.testing.pytestPath": "/usr/local/py-utils/bin/pytest",
"python.defaultInterpreterPath": "${workspaceFolder}/alpha-omega/omega/triage-portal/.venv/bin/python"
},
// Add the IDs of extensions you want installed when the container is created.
"extensions": [
"ms-python.python",
"ms-python.vscode-pylance",
"mtxr.sqltools",
"mtxr.sqltools-driver-pg",
"GitHub.copilot"
],
// Use 'forwardPorts' to make a list of ports inside the container available locally.
"forwardPorts": [
8000
],
"portsAttributes": {
"8000": {
"label": "Triage Portal (Django)",
"protocol": "http",
"onAutoForward": "notify",
"requireLocalPort": false,
"elevateIfNeeded": false
}
},
// Use 'postCreateCommand' to run commands after the container is created.
"postCreateCommand": "bash alpha-omega/.devcontainer/triage-portal/postcreate-initialize.sh",
"remoteEnv": {
"DJANGO_SETTINGS_MODULE": "core.settings",
"PYTHONPATH": "/workspaces/alpha-omega/omega/triage-portal/src"
},
// Comment out connect as root instead. More info: https://aka.ms/vscode-remote/containers/non-root.
"remoteUser": "vscode"
}
54 changes: 54 additions & 0 deletions .devcontainer/triage-portal/docker-compose.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,54 @@
version: '3.8'

services:
app:
build:
context: ..
dockerfile: triage-portal/Dockerfile
args:
# Update 'VARIANT' to pick a version of Python: 3, 3.10, 3.9, 3.8, 3.7, 3.6
# Append -bullseye or -buster to pin to an OS version.
# Use -bullseye variants on local arm64/Apple Silicon.
VARIANT: 3-bullseye
# Optional Node.js version to install
NODE_VERSION: "lts/*"

volumes:
- ..:/workspace:cached
init: true

# Overrides default command so things don't shut down after the process ends.
command: sleep infinity

# Runs app on the same network as the database container, allows "forwardPorts" in devcontainer.json function.
network_mode: service:db

# Uncomment the next line to use a non-root user for all processes.
# user: vscode

# Use "forwardPorts" in **devcontainer.json** to forward an app port locally.
# (Adding the "ports" property to this file will not forward from a Codespace.)

db:
image: postgres:latest
restart: unless-stopped
volumes:
- postgres-data:/var/lib/postgresql/data
environment:
POSTGRES_USER: triage_user
POSTGRES_DB: triage
POSTGRES_PASSWORD: triage_password

redis:
image: redis:latest
restart: unless-stopped
network_mode: service:db
volumes:
- redis-data:/data

# Add "forwardPorts": ["5432"] to **devcontainer.json** to forward PostgreSQL locally.
# (Adding the "ports" property to this file will not forward from a Codespace.)

volumes:
postgres-data:
redis-data:
49 changes: 49 additions & 0 deletions .devcontainer/triage-portal/postcreate-initialize.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,49 @@
#!/bin/bash

ROOT="/workspaces/alpha-omega/omega/triage-portal"
cd "$ROOT"

# Create and activate the virtual environment
echo "Creating virtual environment."
python -mvenv .venv
source .venv/bin/activate

# Install Python dependencies
echo "Installing Python (back-end) dependencies."
cd $ROOT/src
python -m pip install --upgrade pip
pip install wheel
pip install -r ./requirements.txt

# Install JavaScript dependencies
echo "Installing JavaScript (front-end) dependencies."
cd $ROOT/src
npm i -g yarn
yarn

# Update default environment
echo "Creating development environment."
cd $ROOT/src
cp .env-template .env
SECRET_KEY=$(python -c "import secrets; print(secrets.token_hex(64))")
sed -i "s/%RANDOM_STRING%/$SECRET_KEY/" .env
unset SECRET_KEY

# Create working directories
echo "Creating working directories."
mkdir $ROOT/logs

# Set up database
echo "Setting up datatbase."
cd $ROOT/src
python manage.py migrate
python manage.py makemigrations
python manage.py migrate triage

# Create superuser
DJANGO_SUPERUSER_USERNAME="admin" \
DJANGO_SUPERUSER_PASSWORD="admin" \
DJANGO_SUPERUSER_EMAIL="nobody@localhost" \
python manage.py createsuperuser --noinput

echo "Initialization completed."
3 changes: 2 additions & 1 deletion .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -2,4 +2,5 @@
*.env
*.pyc
omega/analyzer/worker/results
venv/*
venv/*
/**/.venv/*
11 changes: 11 additions & 0 deletions .vscode/project.code-workspace
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
{
"folders": [
{
"name": "triage-portal",
"path": "/workspaces/alpha-omega/omega/triage-portal"
},
{
"path": "../.devcontainer"
}
]
}
144 changes: 144 additions & 0 deletions omega/triage-portal/.gitignore
Original file line number Diff line number Diff line change
@@ -0,0 +1,144 @@
# Derived from https://github.com/github/gitignore/blob/218a941be92679ce67d0484547e3e142b2f5f6f0/Python.gitignore

# Static Files (brought in via Yarn)
src/triage/static/triage/resources
logs/

# Byte-compiled / optimized / DLL files
__pycache__/
*.py[cod]
*$py.class

# C extensions
*.so

# Distribution / packaging
.Python
build/
develop-eggs/
dist/
downloads/
eggs/
.eggs/
lib/
lib64/
parts/
sdist/
var/
wheels/
share/python-wheels/
*.egg-info/
.installed.cfg
*.egg
MANIFEST

# PyInstaller
# Usually these files are written by a python script from a template
# before PyInstaller builds the exe, so as to inject date/other infos into it.
*.manifest
*.spec

# Installer logs
pip-log.txt
pip-delete-this-directory.txt

# Unit test / coverage reports
htmlcov/
.tox/
.nox/
.coverage
.coverage.*
.cache
nosetests.xml
coverage.xml
*.cover
*.py,cover
.hypothesis/
.pytest_cache/
cover/

# Translations
*.mo
*.pot

# Django stuff:
*.log
local_settings.py
db.sqlite3
db.sqlite3-journal

# Flask stuff:
instance/
.webassets-cache

# Scrapy stuff:
.scrapy

# Sphinx documentation
docs/_build/

# PyBuilder
.pybuilder/
target/

# Jupyter Notebook
.ipynb_checkpoints

# IPython
profile_default/
ipython_config.py

# pyenv
# For a library or package, you might want to ignore these files since the code is
# intended to run in multiple environments; otherwise, check them in:
# .python-version

# pipenv
# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
# However, in case of collaboration, if having platform-specific dependencies or dependencies
# having no cross-platform support, pipenv may install dependencies that don't work, or not
# install all needed dependencies.
#Pipfile.lock

# PEP 582; used by e.g. github.com/David-OConnor/pyflow
__pypackages__/

# Celery stuff
celerybeat-schedule
celerybeat.pid

# SageMath parsed files
*.sage.py

# Environments
.env
.venv
env/
venv/
ENV/
env.bak/
venv.bak/

# Spyder project settings
.spyderproject
.spyproject

# Rope project settings
.ropeproject

# mkdocs documentation
/site

# mypy
.mypy_cache/
.dmypy.json
dmypy.json

# Pyre type checker
.pyre/

# pytype static type analyzer
.pytype/

# Cython debug symbols
cython_debug/
19 changes: 19 additions & 0 deletions omega/triage-portal/.vscode/launch.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,19 @@
{
// Use IntelliSense to learn about possible attributes.
// Hover to view descriptions of existing attributes.
// For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
"version": "0.2.0",
"configurations": [
{
"name": "Python: Django",
"type": "python",
"request": "launch",
"program": "src/manage.py",
"args": [
"runserver",
"0.0.0.0:8001"
],
"django": true
}
]
}
Loading