diff --git a/myenv/bin/Activate.ps1 b/myenv/bin/Activate.ps1 new file mode 100644 index 0000000..b49d77b --- /dev/null +++ b/myenv/bin/Activate.ps1 @@ -0,0 +1,247 @@ +<# +.Synopsis +Activate a Python virtual environment for the current PowerShell session. + +.Description +Pushes the python executable for a virtual environment to the front of the +$Env:PATH environment variable and sets the prompt to signify that you are +in a Python virtual environment. Makes use of the command line switches as +well as the `pyvenv.cfg` file values present in the virtual environment. + +.Parameter VenvDir +Path to the directory that contains the virtual environment to activate. The +default value for this is the parent of the directory that the Activate.ps1 +script is located within. + +.Parameter Prompt +The prompt prefix to display when this virtual environment is activated. By +default, this prompt is the name of the virtual environment folder (VenvDir) +surrounded by parentheses and followed by a single space (ie. '(.venv) '). + +.Example +Activate.ps1 +Activates the Python virtual environment that contains the Activate.ps1 script. + +.Example +Activate.ps1 -Verbose +Activates the Python virtual environment that contains the Activate.ps1 script, +and shows extra information about the activation as it executes. + +.Example +Activate.ps1 -VenvDir C:\Users\MyUser\Common\.venv +Activates the Python virtual environment located in the specified location. + +.Example +Activate.ps1 -Prompt "MyPython" +Activates the Python virtual environment that contains the Activate.ps1 script, +and prefixes the current prompt with the specified string (surrounded in +parentheses) while the virtual environment is active. + +.Notes +On Windows, it may be required to enable this Activate.ps1 script by setting the +execution policy for the user. You can do this by issuing the following PowerShell +command: + +PS C:\> Set-ExecutionPolicy -ExecutionPolicy RemoteSigned -Scope CurrentUser + +For more information on Execution Policies: +https://go.microsoft.com/fwlink/?LinkID=135170 + +#> +Param( + [Parameter(Mandatory = $false)] + [String] + $VenvDir, + [Parameter(Mandatory = $false)] + [String] + $Prompt +) + +<# Function declarations --------------------------------------------------- #> + +<# +.Synopsis +Remove all shell session elements added by the Activate script, including the +addition of the virtual environment's Python executable from the beginning of +the PATH variable. + +.Parameter NonDestructive +If present, do not remove this function from the global namespace for the +session. + +#> +function global:deactivate ([switch]$NonDestructive) { + # Revert to original values + + # The prior prompt: + if (Test-Path -Path Function:_OLD_VIRTUAL_PROMPT) { + Copy-Item -Path Function:_OLD_VIRTUAL_PROMPT -Destination Function:prompt + Remove-Item -Path Function:_OLD_VIRTUAL_PROMPT + } + + # The prior PYTHONHOME: + if (Test-Path -Path Env:_OLD_VIRTUAL_PYTHONHOME) { + Copy-Item -Path Env:_OLD_VIRTUAL_PYTHONHOME -Destination Env:PYTHONHOME + Remove-Item -Path Env:_OLD_VIRTUAL_PYTHONHOME + } + + # The prior PATH: + if (Test-Path -Path Env:_OLD_VIRTUAL_PATH) { + Copy-Item -Path Env:_OLD_VIRTUAL_PATH -Destination Env:PATH + Remove-Item -Path Env:_OLD_VIRTUAL_PATH + } + + # Just remove the VIRTUAL_ENV altogether: + if (Test-Path -Path Env:VIRTUAL_ENV) { + Remove-Item -Path env:VIRTUAL_ENV + } + + # Just remove VIRTUAL_ENV_PROMPT altogether. + if (Test-Path -Path Env:VIRTUAL_ENV_PROMPT) { + Remove-Item -Path env:VIRTUAL_ENV_PROMPT + } + + # Just remove the _PYTHON_VENV_PROMPT_PREFIX altogether: + if (Get-Variable -Name "_PYTHON_VENV_PROMPT_PREFIX" -ErrorAction SilentlyContinue) { + Remove-Variable -Name _PYTHON_VENV_PROMPT_PREFIX -Scope Global -Force + } + + # Leave deactivate function in the global namespace if requested: + if (-not $NonDestructive) { + Remove-Item -Path function:deactivate + } +} + +<# +.Description +Get-PyVenvConfig parses the values from the pyvenv.cfg file located in the +given folder, and returns them in a map. + +For each line in the pyvenv.cfg file, if that line can be parsed into exactly +two strings separated by `=` (with any amount of whitespace surrounding the =) +then it is considered a `key = value` line. The left hand string is the key, +the right hand is the value. + +If the value starts with a `'` or a `"` then the first and last character is +stripped from the value before being captured. + +.Parameter ConfigDir +Path to the directory that contains the `pyvenv.cfg` file. +#> +function Get-PyVenvConfig( + [String] + $ConfigDir +) { + Write-Verbose "Given ConfigDir=$ConfigDir, obtain values in pyvenv.cfg" + + # Ensure the file exists, and issue a warning if it doesn't (but still allow the function to continue). + $pyvenvConfigPath = Join-Path -Resolve -Path $ConfigDir -ChildPath 'pyvenv.cfg' -ErrorAction Continue + + # An empty map will be returned if no config file is found. + $pyvenvConfig = @{ } + + if ($pyvenvConfigPath) { + + Write-Verbose "File exists, parse `key = value` lines" + $pyvenvConfigContent = Get-Content -Path $pyvenvConfigPath + + $pyvenvConfigContent | ForEach-Object { + $keyval = $PSItem -split "\s*=\s*", 2 + if ($keyval[0] -and $keyval[1]) { + $val = $keyval[1] + + # Remove extraneous quotations around a string value. + if ("'""".Contains($val.Substring(0, 1))) { + $val = $val.Substring(1, $val.Length - 2) + } + + $pyvenvConfig[$keyval[0]] = $val + Write-Verbose "Adding Key: '$($keyval[0])'='$val'" + } + } + } + return $pyvenvConfig +} + + +<# Begin Activate script --------------------------------------------------- #> + +# Determine the containing directory of this script +$VenvExecPath = Split-Path -Parent $MyInvocation.MyCommand.Definition +$VenvExecDir = Get-Item -Path $VenvExecPath + +Write-Verbose "Activation script is located in path: '$VenvExecPath'" +Write-Verbose "VenvExecDir Fullname: '$($VenvExecDir.FullName)" +Write-Verbose "VenvExecDir Name: '$($VenvExecDir.Name)" + +# Set values required in priority: CmdLine, ConfigFile, Default +# First, get the location of the virtual environment, it might not be +# VenvExecDir if specified on the command line. +if ($VenvDir) { + Write-Verbose "VenvDir given as parameter, using '$VenvDir' to determine values" +} +else { + Write-Verbose "VenvDir not given as a parameter, using parent directory name as VenvDir." + $VenvDir = $VenvExecDir.Parent.FullName.TrimEnd("\\/") + Write-Verbose "VenvDir=$VenvDir" +} + +# Next, read the `pyvenv.cfg` file to determine any required value such +# as `prompt`. +$pyvenvCfg = Get-PyVenvConfig -ConfigDir $VenvDir + +# Next, set the prompt from the command line, or the config file, or +# just use the name of the virtual environment folder. +if ($Prompt) { + Write-Verbose "Prompt specified as argument, using '$Prompt'" +} +else { + Write-Verbose "Prompt not specified as argument to script, checking pyvenv.cfg value" + if ($pyvenvCfg -and $pyvenvCfg['prompt']) { + Write-Verbose " Setting based on value in pyvenv.cfg='$($pyvenvCfg['prompt'])'" + $Prompt = $pyvenvCfg['prompt']; + } + else { + Write-Verbose " Setting prompt based on parent's directory's name. (Is the directory name passed to venv module when creating the virtual environment)" + Write-Verbose " Got leaf-name of $VenvDir='$(Split-Path -Path $venvDir -Leaf)'" + $Prompt = Split-Path -Path $venvDir -Leaf + } +} + +Write-Verbose "Prompt = '$Prompt'" +Write-Verbose "VenvDir='$VenvDir'" + +# Deactivate any currently active virtual environment, but leave the +# deactivate function in place. +deactivate -nondestructive + +# Now set the environment variable VIRTUAL_ENV, used by many tools to determine +# that there is an activated venv. +$env:VIRTUAL_ENV = $VenvDir + +if (-not $Env:VIRTUAL_ENV_DISABLE_PROMPT) { + + Write-Verbose "Setting prompt to '$Prompt'" + + # Set the prompt to include the env name + # Make sure _OLD_VIRTUAL_PROMPT is global + function global:_OLD_VIRTUAL_PROMPT { "" } + Copy-Item -Path function:prompt -Destination function:_OLD_VIRTUAL_PROMPT + New-Variable -Name _PYTHON_VENV_PROMPT_PREFIX -Description "Python virtual environment prompt prefix" -Scope Global -Option ReadOnly -Visibility Public -Value $Prompt + + function global:prompt { + Write-Host -NoNewline -ForegroundColor Green "($_PYTHON_VENV_PROMPT_PREFIX) " + _OLD_VIRTUAL_PROMPT + } + $env:VIRTUAL_ENV_PROMPT = $Prompt +} + +# Clear PYTHONHOME +if (Test-Path -Path Env:PYTHONHOME) { + Copy-Item -Path Env:PYTHONHOME -Destination Env:_OLD_VIRTUAL_PYTHONHOME + Remove-Item -Path Env:PYTHONHOME +} + +# Add the venv to the PATH +Copy-Item -Path Env:PATH -Destination Env:_OLD_VIRTUAL_PATH +$Env:PATH = "$VenvExecDir$([System.IO.Path]::PathSeparator)$Env:PATH" diff --git a/myenv/bin/activate b/myenv/bin/activate new file mode 100644 index 0000000..8439835 --- /dev/null +++ b/myenv/bin/activate @@ -0,0 +1,69 @@ +# This file must be used with "source bin/activate" *from bash* +# you cannot run it directly + +deactivate () { + # reset old environment variables + if [ -n "${_OLD_VIRTUAL_PATH:-}" ] ; then + PATH="${_OLD_VIRTUAL_PATH:-}" + export PATH + unset _OLD_VIRTUAL_PATH + fi + if [ -n "${_OLD_VIRTUAL_PYTHONHOME:-}" ] ; then + PYTHONHOME="${_OLD_VIRTUAL_PYTHONHOME:-}" + export PYTHONHOME + unset _OLD_VIRTUAL_PYTHONHOME + fi + + # This should detect bash and zsh, which have a hash command that must + # be called to get it to forget past commands. Without forgetting + # past commands the $PATH changes we made may not be respected + if [ -n "${BASH:-}" -o -n "${ZSH_VERSION:-}" ] ; then + hash -r 2> /dev/null + fi + + if [ -n "${_OLD_VIRTUAL_PS1:-}" ] ; then + PS1="${_OLD_VIRTUAL_PS1:-}" + export PS1 + unset _OLD_VIRTUAL_PS1 + fi + + unset VIRTUAL_ENV + unset VIRTUAL_ENV_PROMPT + if [ ! "${1:-}" = "nondestructive" ] ; then + # Self destruct! + unset -f deactivate + fi +} + +# unset irrelevant variables +deactivate nondestructive + +VIRTUAL_ENV="/Users/sahithikasim/guac-alytics/myenv" +export VIRTUAL_ENV + +_OLD_VIRTUAL_PATH="$PATH" +PATH="$VIRTUAL_ENV/bin:$PATH" +export PATH + +# unset PYTHONHOME if set +# this will fail if PYTHONHOME is set to the empty string (which is bad anyway) +# could use `if (set -u; : $PYTHONHOME) ;` in bash +if [ -n "${PYTHONHOME:-}" ] ; then + _OLD_VIRTUAL_PYTHONHOME="${PYTHONHOME:-}" + unset PYTHONHOME +fi + +if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT:-}" ] ; then + _OLD_VIRTUAL_PS1="${PS1:-}" + PS1="(myenv) ${PS1:-}" + export PS1 + VIRTUAL_ENV_PROMPT="(myenv) " + export VIRTUAL_ENV_PROMPT +fi + +# This should detect bash and zsh, which have a hash command that must +# be called to get it to forget past commands. Without forgetting +# past commands the $PATH changes we made may not be respected +if [ -n "${BASH:-}" -o -n "${ZSH_VERSION:-}" ] ; then + hash -r 2> /dev/null +fi diff --git a/myenv/bin/activate.csh b/myenv/bin/activate.csh new file mode 100644 index 0000000..8e4476b --- /dev/null +++ b/myenv/bin/activate.csh @@ -0,0 +1,26 @@ +# This file must be used with "source bin/activate.csh" *from csh*. +# You cannot run it directly. +# Created by Davide Di Blasi . +# Ported to Python 3.3 venv by Andrew Svetlov + +alias deactivate 'test $?_OLD_VIRTUAL_PATH != 0 && setenv PATH "$_OLD_VIRTUAL_PATH" && unset _OLD_VIRTUAL_PATH; rehash; test $?_OLD_VIRTUAL_PROMPT != 0 && set prompt="$_OLD_VIRTUAL_PROMPT" && unset _OLD_VIRTUAL_PROMPT; unsetenv VIRTUAL_ENV; unsetenv VIRTUAL_ENV_PROMPT; test "\!:*" != "nondestructive" && unalias deactivate' + +# Unset irrelevant variables. +deactivate nondestructive + +setenv VIRTUAL_ENV "/Users/sahithikasim/guac-alytics/myenv" + +set _OLD_VIRTUAL_PATH="$PATH" +setenv PATH "$VIRTUAL_ENV/bin:$PATH" + + +set _OLD_VIRTUAL_PROMPT="$prompt" + +if (! "$?VIRTUAL_ENV_DISABLE_PROMPT") then + set prompt = "(myenv) $prompt" + setenv VIRTUAL_ENV_PROMPT "(myenv) " +endif + +alias pydoc python -m pydoc + +rehash diff --git a/myenv/bin/activate.fish b/myenv/bin/activate.fish new file mode 100644 index 0000000..2aa114d --- /dev/null +++ b/myenv/bin/activate.fish @@ -0,0 +1,69 @@ +# This file must be used with "source /bin/activate.fish" *from fish* +# (https://fishshell.com/); you cannot run it directly. + +function deactivate -d "Exit virtual environment and return to normal shell environment" + # reset old environment variables + if test -n "$_OLD_VIRTUAL_PATH" + set -gx PATH $_OLD_VIRTUAL_PATH + set -e _OLD_VIRTUAL_PATH + end + if test -n "$_OLD_VIRTUAL_PYTHONHOME" + set -gx PYTHONHOME $_OLD_VIRTUAL_PYTHONHOME + set -e _OLD_VIRTUAL_PYTHONHOME + end + + if test -n "$_OLD_FISH_PROMPT_OVERRIDE" + set -e _OLD_FISH_PROMPT_OVERRIDE + # prevents error when using nested fish instances (Issue #93858) + if functions -q _old_fish_prompt + functions -e fish_prompt + functions -c _old_fish_prompt fish_prompt + functions -e _old_fish_prompt + end + end + + set -e VIRTUAL_ENV + set -e VIRTUAL_ENV_PROMPT + if test "$argv[1]" != "nondestructive" + # Self-destruct! + functions -e deactivate + end +end + +# Unset irrelevant variables. +deactivate nondestructive + +set -gx VIRTUAL_ENV "/Users/sahithikasim/guac-alytics/myenv" + +set -gx _OLD_VIRTUAL_PATH $PATH +set -gx PATH "$VIRTUAL_ENV/bin" $PATH + +# Unset PYTHONHOME if set. +if set -q PYTHONHOME + set -gx _OLD_VIRTUAL_PYTHONHOME $PYTHONHOME + set -e PYTHONHOME +end + +if test -z "$VIRTUAL_ENV_DISABLE_PROMPT" + # fish uses a function instead of an env var to generate the prompt. + + # Save the current fish_prompt function as the function _old_fish_prompt. + functions -c fish_prompt _old_fish_prompt + + # With the original prompt function renamed, we can override with our own. + function fish_prompt + # Save the return status of the last command. + set -l old_status $status + + # Output the venv prompt; color taken from the blue of the Python logo. + printf "%s%s%s" (set_color 4B8BBE) "(myenv) " (set_color normal) + + # Restore the return status of the previous command. + echo "exit $old_status" | . + # Output the original/"old" prompt. + _old_fish_prompt + end + + set -gx _OLD_FISH_PROMPT_OVERRIDE "$VIRTUAL_ENV" + set -gx VIRTUAL_ENV_PROMPT "(myenv) " +end diff --git a/myenv/bin/python b/myenv/bin/python new file mode 120000 index 0000000..b8a0adb --- /dev/null +++ b/myenv/bin/python @@ -0,0 +1 @@ +python3 \ No newline at end of file diff --git a/myenv/bin/python3 b/myenv/bin/python3 new file mode 120000 index 0000000..79ab74b --- /dev/null +++ b/myenv/bin/python3 @@ -0,0 +1 @@ +/usr/local/bin/python3 \ No newline at end of file diff --git a/myenv/bin/python3.10 b/myenv/bin/python3.10 new file mode 120000 index 0000000..b8a0adb --- /dev/null +++ b/myenv/bin/python3.10 @@ -0,0 +1 @@ +python3 \ No newline at end of file diff --git a/myenv/pyvenv.cfg b/myenv/pyvenv.cfg new file mode 100644 index 0000000..3e2efc5 --- /dev/null +++ b/myenv/pyvenv.cfg @@ -0,0 +1,3 @@ +home = /usr/local/bin +include-system-site-packages = false +version = 3.10.10 diff --git a/scripts/ingestion/constants.py b/scripts/ingestion/constants.py index a25d068..bf5c30b 100644 --- a/scripts/ingestion/constants.py +++ b/scripts/ingestion/constants.py @@ -1,11 +1,13 @@ #!/usr/bin/env python # encoding: utf-8 -DB_LOC = '/data/yellow/vineet/database/bi_multi_tables.db' +DB_LOC = './bi_multi_tables.db' MAINTAINER_INST_LOC = "https://popcon.debian.org/maint/by_inst" MAINTAINER_TEXT_FILE = '/data/yellow/vineet/python_files/new_scripts/database_creation/maintainer.txt' MAINTAINER_CSV_FILE = '/data/yellow/vineet/python_files/new_scripts/database_creation/maintainer.csv' REGEX = "(\ )+" INST_LOC = 'https://popcon.debian.org/by_inst' POPCON_CSV = '/data/yellow/vineet/python_files/new_scripts/data_pre_processing/today.csv' -POPCON_TEXT = '/data/yellow/vineet/python_files/new_scripts/data_pre_processing/today.txt' \ No newline at end of file +POPCON_TEXT = '/data/yellow/vineet/python_files/new_scripts/data_pre_processing/today.txt' +VULNERABILITY_URL = 'https://security-tracker.debian.org/tracker/data/json' +NIST_URL = 'https://nvd.nist.gov/vuln/detail/' \ No newline at end of file diff --git a/scripts/ingestion/database/vulnerability_db_init.py b/scripts/ingestion/database/vulnerability_db_init.py new file mode 100644 index 0000000..b46be93 --- /dev/null +++ b/scripts/ingestion/database/vulnerability_db_init.py @@ -0,0 +1,19 @@ +#!/usr/bin/env python +import sqlite3 + +def db_init(location): + conn = sqlite3.connect(location) + cursor = conn.cursor() + cursor.execute("""CREATE TABLE IF NOT EXISTS vulnerability_table( + source_id integer, + package varchar, + vulnerability varchar, + description varchar, + published_date datetime, + last_modified_date datetime, + debianbug integer, + scope varchar, + releases varchar, + FOREIGN KEY(source_id) references source_table(source_id) )""") + + return conn,cursor \ No newline at end of file diff --git a/scripts/ingestion/parsers/vulnerability_compare.py b/scripts/ingestion/parsers/vulnerability_compare.py new file mode 100644 index 0000000..ee2b211 --- /dev/null +++ b/scripts/ingestion/parsers/vulnerability_compare.py @@ -0,0 +1,106 @@ +import requests +import json +from bs4 import BeautifulSoup +from datetime import date +import sqlite3 +import sys +import os +# Get the absolute path of the parent directory +parent_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), '..')) +# Add the parent directory to the Python path +sys.path.append(parent_dir) +from constants import VULNERABILITY_DATA, NIST_URL, DB_LOC, VULNERABILITY_URL + +# Function to fetch data from the URL +def fetch_data(): + response = requests.get(VULNERABILITY_URL) + if response.status_code == 200: + return response.json() + else: + print("Failed to fetch data from the URL.") + return None + +# Function to save data to a JSON file +def save_to_json(data): + year=date.today().year + mon=date.today().month + day=date.today().day + f=VULNERABILITY_DATA.format(year,mon,day) + with open(f, "w") as file: + json.dump(data, file) + file.close() + return file + +def parse_vulnerability_data(vulnerability_id): + url = f"{NIST_URL}{vulnerability_id}" + + response = requests.get(url) + if response.status_code == 200: + soup = BeautifulSoup(response.content, 'html.parser') + + # Find and extract the published date + published_date_element = soup.find('span', {'data-testid': 'vuln-published-on'}) + published_date = published_date_element.text.strip() if published_date_element else None + + # Find and extract the last modified date + last_modified_date_element = soup.find('span', {'data-testid': 'vuln-last-modified-on'}) + last_modified_date = last_modified_date_element.text.strip() if last_modified_date_element else None + + return published_date, last_modified_date + + return None, None + +def insert_records(cur,name,cve,description,published_date, last_modified_date,debianbug,scope,releases): + INSERT_RECS = '''INSERT OR IGNORE INTO vulnerability_table (source_id,package,vulnerability,description,published_date, last_modified_date,debianbug,scope,releases) VALUES (?,?,?,?,?,?,?,?,?)''' + cur.execute('''delete from vulnerability_table''') + cur.execute('''select source_id,source_name from source_table where source_name ='{}' limit 1'''.format(name)) + id = cur.fetchone() + if id: + cur.execute(INSERT_RECS, (id[0],name,cve,description,published_date, last_modified_date,debianbug,scope,str(releases))) + else: + cur.execute(INSERT_RECS, ('Null',name,cve,description,published_date, last_modified_date,debianbug,scope,str(releases))) + + +def parser(file): + con = sqlite3.connect(DB_LOC) + # Check if the file exists locally, if not, download it + with open (file, 'r') as f: + parsed_data = json.load(f) + + for name, vulnerability in parsed_data.items(): + for cve, info in vulnerability.items(): + if 'description' in info: + description = info['description'] + else: + description="None" + if 'debianbug' in info: + debianbug = info['debianbug'] + else: + debianbug="None" + if 'scope' in info: + scope = info['scope'] + else: + scope="None" + releases = info['releases'] + # print(f"{name}|{cve}|{description}|{debianbug}|{scope}|{releases}") + cur = con.cursor() + # to insert records into the table + published_date, last_modified_date = parse_vulnerability_data(cve) + insert_records(cur,name,cve,description,published_date, last_modified_date,debianbug,scope,releases) + con.commit() + + con.close() + +# Main function to run the process +def main(): + data = fetch_data() + if data: + print(date.today()) + file_obj = save_to_json(data) + file = file_obj.name + parser(file) + print("Data Updated.") + +# Run the process +if __name__ == "__main__": + main() diff --git a/scripts/ingestion/parsers/vulnerability_parser.py b/scripts/ingestion/parsers/vulnerability_parser.py new file mode 100644 index 0000000..055f136 --- /dev/null +++ b/scripts/ingestion/parsers/vulnerability_parser.py @@ -0,0 +1,59 @@ +#!/usr/bin/env python +import requests +from bs4 import BeautifulSoup +import sys +import os +# Get the absolute path of the parent directory +parent_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), '..')) +# Add the parent directory to the Python path +sys.path.append(parent_dir) +from constants import NIST_URL, VULNERABILITY_URL + +def parse_vulnerability_data(vulnerability_id): + url = f"{NIST_URL}{vulnerability_id}" + + response = requests.get(url) + if response.status_code == 200: + soup = BeautifulSoup(response.content, 'html.parser') + + # Find and extract the published date + published_date_element = soup.find('span', {'data-testid': 'vuln-published-on'}) + published_date = published_date_element.text.strip() if published_date_element else None + + # Find and extract the last modified date + last_modified_date_element = soup.find('span', {'data-testid': 'vuln-last-modified-on'}) + last_modified_date = last_modified_date_element.text.strip() if last_modified_date_element else None + + return published_date, last_modified_date + + return None, None + +# Define the main parser function +def parser(): + # Check if the file exists locally, if not, download it + response = requests.get(VULNERABILITY_URL) + + # Parse the data as JSON + parsed_data = response.json() + results = [] + for name, vulnerability in parsed_data.items(): + for cve, info in vulnerability.items(): + if 'description' in info: + description = info['description'] + else: + description="None" + if 'debianbug' in info: + debianbug = info['debianbug'] + else: + debianbug="None" + if 'scope' in info: + scope = info['scope'] + else: + scope="None" + releases = info['releases'] + # print(f"{name}|{cve}|{description}|{debianbug}|{scope}|{releases}") + # to insert records into the table + published_date, last_modified_date = parse_vulnerability_data(cve) + result = (name, cve, description,published_date, last_modified_date, debianbug, scope, releases) + results.append(result) + return results \ No newline at end of file diff --git a/scripts/popcon_main.py b/scripts/popcon_main.py index 6b9fbe5..a71c016 100644 --- a/scripts/popcon_main.py +++ b/scripts/popcon_main.py @@ -1,9 +1,9 @@ import csv from datetime import datetime import sqlite3 -from database.popcon_db_init import db_init,conn,cursor -from constants import POPCON_CSV -from parsers.popcon_parser import parser +from ingestion.database.popcon_db_init import db_init,conn,cursor +from ingestion.constants import POPCON_CSV +from ingestion.parsers.popcon_parser import parser no_records = 0 def popcon(): diff --git a/scripts/vulnerability_main.py b/scripts/vulnerability_main.py new file mode 100644 index 0000000..40fb448 --- /dev/null +++ b/scripts/vulnerability_main.py @@ -0,0 +1,23 @@ +#!/usr/bin/env python +from ingestion.database.vulnerability_db_init import db_init +from ingestion.parsers.vulnerability_parser import parser +from ingestion.constants import DB_LOC + +def insert_records(results): + for name, cve, description, published_date, last_modified_date, debianbug, scope, releases in results: + cursor.execute('''select source_id,source_name from source_table where source_name ='{}' limit 1'''.format(name)) + id = cursor.fetchone() + if id: + cursor.execute(INSERT_RECS, (id[0],name,cve,description,published_date, last_modified_date,debianbug,scope,str(releases))) + else: + cursor.execute(INSERT_RECS, ('Null',name,cve,description,published_date, last_modified_date,debianbug,scope,str(releases))) + + conn.commit() + + +if __name__ == "__main__": + conn,cursor = db_init(DB_LOC) # Initialize the database + INSERT_RECS = '''INSERT OR IGNORE INTO vulnerability_table (source_id,package,vulnerability,description,published_date, last_modified_date,debianbug,scope,releases) VALUES (?,?,?,?,?,?,?,?,?)''' + results = parser() # Parse the data + insert_records(results) + conn.close() \ No newline at end of file