From 80ed305c6112b3296a2ad0bf57ab8b0c75a31966 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maciek=20Go=C5=82aszewski?= Date: Thu, 15 Aug 2024 14:57:26 +0200 Subject: [PATCH 1/6] Update send-scan.py with gh meta and verbosity Added: * GitHub metadata to request - usefull when having multiple repos * Verbosity flag printing request body and response code KU-1194 --- scripts/cve-reports/send-scan.py | 37 ++++++++++++++++++++++++-------- 1 file changed, 28 insertions(+), 9 deletions(-) diff --git a/scripts/cve-reports/send-scan.py b/scripts/cve-reports/send-scan.py index f1f260b..7f71a5c 100755 --- a/scripts/cve-reports/send-scan.py +++ b/scripts/cve-reports/send-scan.py @@ -1,5 +1,5 @@ #!/usr/bin/python3 -# Copyright 2023 Canonical Ltd. +# Copyright 2024 Canonical Ltd. # See LICENSE file for licensing details. # @@ -19,6 +19,16 @@ } +def get_github_meta(): + """Get GitHub Metadata""" + return { + "github_server_url": os.getenv("GITHUB_SERVER_URL"), + "github_run_id": os.getenv("GITHUB_RUN_ID"), + "github_sha": os.getenv("GITHUB_SHA"), + "github_repository": os.getenv("GITHUB_REPOSITORY"), + } + + def parse_json(filename): """Parse JSON file""" record_list = [] @@ -55,10 +65,11 @@ def parse_json(filename): "description": vuln["Description"], "references": "\n".join(vuln["References"]), "primary_url": vuln["PrimaryURL"], - "priority": severity_to_priority_map.get(vuln["Severity"], "Lowest"), + "priority": severity_to_priority_map.get( + vuln["Severity"], "Lowest" + ), } ) - return record_list @@ -91,7 +102,9 @@ def parse_sarif(filename): "severity": severity, "cve_id": result["ruleId"], "package_name": pkg_name, - "installed_version": record_message[1].replace("Installed Version: ", ""), + "installed_version": record_message[1].replace( + "Installed Version: ", "" + ), "fixed_version": record_message[4].replace("Fixed Version: ", ""), "title": record_rule["shortDescription"]["text"], "description": record_rule["help"]["text"], @@ -100,13 +113,12 @@ def parse_sarif(filename): "priority": severity_to_priority_map.get(severity, "Lowest"), } ) - return record_list -def main(report_path, jira_url): +def main(report_path, jira_url, gh_meta=False, verbose=False): input_path = Path(report_path) - + gh_metadata = get_github_meta() if gh_meta else None file_list = [] if input_path.is_dir(): # directory is supplied, retrieve list of files @@ -133,12 +145,19 @@ def main(report_path, jira_url): # send records for record in records: - requests.post(jira_url, json=record) + if gh_metadata is not None: + record = {**record, **gh_metadata} + res = requests.post(jira_url, json=record) + if verbose: + print(record) + print(res) if __name__ == "__main__": parser = argparse.ArgumentParser() parser.add_argument("--report-path") parser.add_argument("--jira-url") + parser.add_argument("--add-github-meta", action="store_true") + parser.add_argument("--verbose", action="store_true") args = parser.parse_args() - main(args.report_path, args.jira_url) + main(args.report_path, args.jira_url, args.add_github_meta, args.verbose) From 82e5e85318c1f70bf1bc6bb9eddb6c0f544cbef9 Mon Sep 17 00:00:00 2001 From: Maciek Golaszewski Date: Fri, 16 Aug 2024 13:48:01 +0200 Subject: [PATCH 2/6] update linting --- scripts/cve-reports/send-scan.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/scripts/cve-reports/send-scan.py b/scripts/cve-reports/send-scan.py index 7f71a5c..6de84da 100755 --- a/scripts/cve-reports/send-scan.py +++ b/scripts/cve-reports/send-scan.py @@ -65,9 +65,7 @@ def parse_json(filename): "description": vuln["Description"], "references": "\n".join(vuln["References"]), "primary_url": vuln["PrimaryURL"], - "priority": severity_to_priority_map.get( - vuln["Severity"], "Lowest" - ), + "priority": severity_to_priority_map.get(vuln["Severity"], "Lowest"), } ) return record_list @@ -102,9 +100,7 @@ def parse_sarif(filename): "severity": severity, "cve_id": result["ruleId"], "package_name": pkg_name, - "installed_version": record_message[1].replace( - "Installed Version: ", "" - ), + "installed_version": record_message[1].replace("Installed Version: ", ""), "fixed_version": record_message[4].replace("Fixed Version: ", ""), "title": record_rule["shortDescription"]["text"], "description": record_rule["help"]["text"], From 609f390d7bece78df9f78bd1ad2b27153108b561 Mon Sep 17 00:00:00 2001 From: Maciek Golaszewski Date: Fri, 16 Aug 2024 13:48:01 +0200 Subject: [PATCH 3/6] updated README.md --- scripts/cve-reports/README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/scripts/cve-reports/README.md b/scripts/cve-reports/README.md index c2bc3de..ea0d840 100644 --- a/scripts/cve-reports/README.md +++ b/scripts/cve-reports/README.md @@ -14,6 +14,8 @@ send-scan.py --report-path --jira-url - `--report-path` - Specifies location of report(s). If it is single file, it will be parsed on its own. If it is a directory all files in the directory will be parsed one by one. Only JSON and SARIF formats are supported. - `--jira-url` - Specifies URL of Jira automation to which reports should be sent. +- `--add-github-meta` - Adds GitHub metadata to sent request. +- `--verbose` - Prints out sent body and return code of request. ## Testing From 9c5f74a04c6d048cbcd924b196ac7a1df63f7f14 Mon Sep 17 00:00:00 2001 From: Maciek Golaszewski Date: Fri, 16 Aug 2024 21:06:27 +0200 Subject: [PATCH 4/6] review fixes --- scripts/cve-reports/README.md | 4 +- scripts/cve-reports/send-scan.py | 88 ++++++++++++++++++++++++-------- 2 files changed, 68 insertions(+), 24 deletions(-) diff --git a/scripts/cve-reports/README.md b/scripts/cve-reports/README.md index ea0d840..947ab72 100644 --- a/scripts/cve-reports/README.md +++ b/scripts/cve-reports/README.md @@ -14,8 +14,8 @@ send-scan.py --report-path --jira-url - `--report-path` - Specifies location of report(s). If it is single file, it will be parsed on its own. If it is a directory all files in the directory will be parsed one by one. Only JSON and SARIF formats are supported. - `--jira-url` - Specifies URL of Jira automation to which reports should be sent. -- `--add-github-meta` - Adds GitHub metadata to sent request. -- `--verbose` - Prints out sent body and return code of request. +- `--add-github-meta` - If set, GitHub metadata is added to the sent request. This metadata will be generated automatically from the runner's environment variables: GITHUB_SERVER_URL, GITHUB_RUN_ID, GITHUB_SHA, GITHUB_REPOSITORY. The default behaviour is to not send the metadata. +- `--verbose` - If set, the body and return code of request will be printed. Defaults to non-verbose. ## Testing diff --git a/scripts/cve-reports/send-scan.py b/scripts/cve-reports/send-scan.py index 6de84da..b3cef7a 100755 --- a/scripts/cve-reports/send-scan.py +++ b/scripts/cve-reports/send-scan.py @@ -7,11 +7,18 @@ import argparse import json +import logging import os +import sys from pathlib import Path +from typing import Dict, List import requests +logger = logging.getLogger(__name__) +logging.basicConfig(level=logging.INFO, stream=sys.stdout, format="%(message)s") + + severity_to_priority_map = { "CRITICAL": "Highest", "HIGH": "High", @@ -19,8 +26,8 @@ } -def get_github_meta(): - """Get GitHub Metadata""" +def get_github_meta() -> Dict[str, str]: + """Return a dictionary with GitHub metadata.""" return { "github_server_url": os.getenv("GITHUB_SERVER_URL"), "github_run_id": os.getenv("GITHUB_RUN_ID"), @@ -29,8 +36,15 @@ def get_github_meta(): } -def parse_json(filename): - """Parse JSON file""" +def parse_json(filename: Path) -> List[Dict[str, str]]: + """Parse JSON file + + Args: + filename(Path): path of file to parse. + Returns: + List[Dict[str, str]]: List of CVE dictionary. + """ + record_list = [] with open(filename, "r") as json_file: data = json.load(json_file) @@ -71,14 +85,20 @@ def parse_json(filename): return record_list -def parse_sarif(filename): - """Parse SARIF file""" +def parse_sarif(filename: Path) -> List[Dict[str, str]]: + """Parse SARIF file + + Args: + filename(Path): path of file to parse. + Returns: + List[Dict[str, str]]: List of CVE dictionary. + """ record_list = [] with open(filename, "r") as json_file: data = json.load(json_file) if "runs" not in data and "tool" not in data["runs"][0]: # no scan results found, skip this report - print(f"No results in report {filename}") + logger.warning(f"No results in report {filename}") return [] rules = data["runs"][0]["tool"]["driver"]["rules"] @@ -112,9 +132,41 @@ def parse_sarif(filename): return record_list -def main(report_path, jira_url, gh_meta=False, verbose=False): +def send_request_with_records( + records: List[Dict[str, str]], + jira_url: str, + gh_metadata: Dict[str, str] = {}, + verbose: bool = False, +) -> None: + """Send the request with records to Jira. + + Args: + records(List[Dict[str, str]]): a list of records of CVE's. + jira_url(str): the url to send the request to. + gh_metadata(Dict[str, str]): a dictionary containing the GitHub metadata to attach to the request, defaults to {}. + verbose(bool): if True body of request and response code will be printed. + """ + + for record in records: + record = {**record, **gh_metadata} + res = requests.post(jira_url, json=record) + if verbose: + logger.info(record) + logger.info(res) + + +def main(report_path: str, jira_url: str, gh_meta: bool, verbose: bool) -> None: + """Main function for processing CVE's files. + + Args: + report_path(str): path where report is stored + jira_url(str): the url to send the request to. + gh_meta(bool): if True GitHub metadata will be attached to the request. + verbose(bool): if True GitHub metadata will be attached to the request. + """ + input_path = Path(report_path) - gh_metadata = get_github_meta() if gh_meta else None + gh_metadata = get_github_meta() if gh_meta else {} file_list = [] if input_path.is_dir(): # directory is supplied, retrieve list of files @@ -122,31 +174,23 @@ def main(report_path, jira_url, gh_meta=False, verbose=False): elif input_path.is_file(): file_list.append(input_path) else: - print(f"Invalid input {report_path} supplied") + logger.error(f"Invalid input {report_path} supplied") return if not file_list: - print(f"Failed to retrieve list of files from {report_path}") + logger.error(f"Failed to retrieve list of files from {report_path}") return for file in file_list: - print(f"Processing report in: {file}") + logger.info(f"Processing report in: {file}") if file.suffix == ".json": records = parse_json(file) elif file.suffix == ".sarif": records = parse_sarif(file) else: - print(f"Unsupported file type: {file}. Skip it.") + logger.warning(f"Unsupported file type: {file}. Skip it.") continue - - # send records - for record in records: - if gh_metadata is not None: - record = {**record, **gh_metadata} - res = requests.post(jira_url, json=record) - if verbose: - print(record) - print(res) + send_request_with_records(records, jira_url, gh_metadata, verbose) if __name__ == "__main__": From 73e0a229b30f89cc9d93efa489fce6fb1d4ce213 Mon Sep 17 00:00:00 2001 From: Maciek Golaszewski Date: Fri, 16 Aug 2024 21:09:22 +0200 Subject: [PATCH 5/6] fmt fixes --- scripts/cve-reports/send-scan.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/cve-reports/send-scan.py b/scripts/cve-reports/send-scan.py index b3cef7a..5bc87c4 100755 --- a/scripts/cve-reports/send-scan.py +++ b/scripts/cve-reports/send-scan.py @@ -143,7 +143,8 @@ def send_request_with_records( Args: records(List[Dict[str, str]]): a list of records of CVE's. jira_url(str): the url to send the request to. - gh_metadata(Dict[str, str]): a dictionary containing the GitHub metadata to attach to the request, defaults to {}. + gh_metadata(Dict[str, str]): a dictionary containing the GitHub metadata + to attach to the request, defaults to {}. verbose(bool): if True body of request and response code will be printed. """ From 425191a1ff98998ebbc50190fa463ecd44338cb7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maciek=20Go=C5=82aszewski?= Date: Tue, 20 Aug 2024 10:13:51 +0200 Subject: [PATCH 6/6] Apply suggestions from code review Co-authored-by: Daniela Plascencia --- scripts/cve-reports/send-scan.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/cve-reports/send-scan.py b/scripts/cve-reports/send-scan.py index 5bc87c4..fbc3f74 100755 --- a/scripts/cve-reports/send-scan.py +++ b/scripts/cve-reports/send-scan.py @@ -37,7 +37,7 @@ def get_github_meta() -> Dict[str, str]: def parse_json(filename: Path) -> List[Dict[str, str]]: - """Parse JSON file + """Parse JSON file. Args: filename(Path): path of file to parse. @@ -86,7 +86,7 @@ def parse_json(filename: Path) -> List[Dict[str, str]]: def parse_sarif(filename: Path) -> List[Dict[str, str]]: - """Parse SARIF file + """Parse SARIF file. Args: filename(Path): path of file to parse.