From c6f35d479d2efb657ab7d9a35c5bc0df6857a53d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn=20Ricks?= Date: Tue, 17 Oct 2023 16:46:09 +0200 Subject: [PATCH] Add: Extend GitHub code scanning API for handling SARIF data Implement the two missing APIs for the GitHub code scanning API handling SARIF data. --- pontos/github/api/code_scanning.py | 120 ++++++++++++++++++++++ pontos/github/models/code_scanning.py | 29 ++++++ tests/github/api/test_code_scanning.py | 110 ++++++++++++++++++++ tests/github/models/test_code_scanning.py | 20 ++++ 4 files changed, 279 insertions(+) diff --git a/pontos/github/api/code_scanning.py b/pontos/github/api/code_scanning.py index 8287b376f..c7b4f404d 100644 --- a/pontos/github/api/code_scanning.py +++ b/pontos/github/api/code_scanning.py @@ -2,9 +2,13 @@ # # SPDX-License-Identifier: GPL-3.0-or-later +import base64 +import gzip +from datetime import datetime from typing import AsyncIterator, Iterable, Optional, Union from pontos.github.api.client import GitHubAsyncREST +from pontos.github.api.helper import JSON_OBJECT from pontos.github.models.base import SortOrder from pontos.github.models.code_scanning import ( AlertSort, @@ -18,6 +22,7 @@ Instance, Language, QuerySuite, + SarifUploadInformation, Severity, ) from pontos.helper import enum_or_value @@ -630,3 +635,118 @@ async def update_default_setup( response = await self._client.patch(api, data=data) response.raise_for_status() return response.json() + + async def upload_sarif_data( + self, + repo: str, + commit_sha: str, + ref: str, + sarif: bytes, + *, + checkout_uri: Optional[str] = None, + started_at: Optional[datetime] = None, + tool_name: Optional[str] = None, + validate: Optional[bool] = None, + ) -> dict[str, str]: + """ + Upload SARIF data containing the results of a code scanning analysis to + make the results available in a repository + + https://docs.github.com/en/rest/code-scanning/code-scanning#upload-an-analysis-as-sarif-data + + Args: + repo: GitHub repository (owner/name) + commit_sha: The SHA of the commit to which the analysis you are + uploading relates + ref: The full Git reference, formatted as refs/heads/, + refs/pull//merge, or refs/pull//head + sarif: + checkout_uri: The base directory used in the analysis, as it appears + in the SARIF file + started_at: The time that the analysis run began + tool_name: The name of the tool used to generate the code scanning + analysis + validate: Whether the SARIF file will be validated according to the + code scanning specifications + + Raises: + HTTPStatusError: A httpx.HTTPStatusError is raised if the request + failed. + + Returns: + See the GitHub documentation for the response object + + Example: + .. code-block:: python + + from pathlib import Path + from pontos.github.api import GitHubAsyncRESTApi + + async with GitHubAsyncRESTApi(token) as api: + json = await api.code_scanning.upload_sarif_data( + "org/repo", + commit_sha="4b6472266afd7b471e86085a6659e8c7f2b119da", + ref="refs/heads/main", + sarif=Path("/path/to/sarif.file").read_bytes(), + ) + print(json["id"]) + """ + api = f"/repos/{repo}/code-scanning/sarifs" + data: JSON_OBJECT = { + "commit_sha": commit_sha, + "ref": ref, + } + if checkout_uri: + data["checkout_uri"] = checkout_uri + if started_at: + data["started_at"] = started_at.isoformat(timespec="seconds") + if tool_name: + data["tool_name"] = tool_name + if validate is not None: + data["validate"] = validate + + compressed = gzip.compress(sarif, mtime=0) + encoded = base64.b64encode(compressed).decode(encoding="ascii") + + data["sarif"] = encoded + + response = await self._client.post(api, data=data) + response.raise_for_status() + return response.json() + + async def sarif(self, repo: str, sarif_id: str) -> SarifUploadInformation: + """ + Gets information about a SARIF upload, including the status and the URL + of the analysis that was uploaded so that you can retrieve details of + the analysis + + https://docs.github.com/en/rest/code-scanning/code-scanning#get-information-about-a-sarif-upload + + Args: + repo: GitHub repository (owner/name) + sarif_id: The SARIF ID obtained after uploading + + Raises: + HTTPStatusError: A httpx.HTTPStatusError is raised if the request + failed. + + Returns: + Information about the SARIF upload + + Example: + .. code-block:: python + + from pontos.github.api import GitHubAsyncRESTApi + + async with GitHubAsyncRESTApi(token) as api: + sarif = await api.code_scanning.sarif( + "org/repo", + "47177e22-5596-11eb-80a1-c1e54ef945c6", + ) + print(sarif) + """ + api = f"/repos/{repo}/code-scanning/sarifs/{sarif_id}" + + response = await self._client.get(api) + response.raise_for_status() + return SarifUploadInformation.from_dict(response.json()) diff --git a/pontos/github/models/code_scanning.py b/pontos/github/models/code_scanning.py index 571c32deb..3dfbcbfe2 100644 --- a/pontos/github/models/code_scanning.py +++ b/pontos/github/models/code_scanning.py @@ -368,3 +368,32 @@ class DefaultSetup(GitHubModel): query_suite: QuerySuite updated_at: Optional[datetime] = None schedule: Optional[str] = None + + +class SarifProcessingStatus(Enum): + """ + `pending` files have not yet been processed, while `complete` means results + from the SARIF have been stored. `failed` files have either not been + processed at all, or could only be partially processed + """ + + PENDING = "pending" + COMPLETE = "complete" + FAILED = "failed" + + +@dataclass +class SarifUploadInformation(GitHubModel): + """ + Information about the SARIF upload + + Attributes: + processing_status: Status of the SARIF processing + analyses_url: The REST API URL for getting the analyses associated with + the upload + errors: Any errors that ocurred during processing of the delivery + """ + + processing_status: SarifProcessingStatus + analyses_url: Optional[str] = None + errors: Optional[list[str]] = None diff --git a/tests/github/api/test_code_scanning.py b/tests/github/api/test_code_scanning.py index 47567c675..bba69d1fd 100644 --- a/tests/github/api/test_code_scanning.py +++ b/tests/github/api/test_code_scanning.py @@ -4,6 +4,8 @@ # ruff: noqa:E501 +import json + from pontos.github.api.code_scanning import GitHubAsyncRESTCodeScanning from pontos.github.models.base import SortOrder from pontos.github.models.code_scanning import ( @@ -13,6 +15,7 @@ DismissedReason, Language, QuerySuite, + SarifProcessingStatus, Severity, ) from tests import AsyncIteratorMock, aiter, anext @@ -1251,3 +1254,110 @@ async def test_update_default_setup(self): resp["run_url"], "https://api.github.com/repos/octoorg/octocat/actions/runs/42", ) + + async def test_upload_sarif_data(self): + sarif = { + "version": "2.1.0", + "$schema": "http://json.schemastore.org/sarif-2.1.0-rtm.4", + "runs": [ + { + "tool": { + "driver": { + "name": "ESLint", + "informationUri": "https://eslint.org", + "rules": [ + { + "id": "no-unused-vars", + "shortDescription": { + "text": "disallow unused variables" + }, + "helpUri": "https://eslint.org/docs/rules/no-unused-vars", + "properties": {"category": "Variables"}, + } + ], + } + }, + "artifacts": [ + { + "location": { + "uri": "file:///C:/dev/sarif/sarif-tutorials/samples/Introduction/simple-example.js" + } + } + ], + "results": [ + { + "level": "error", + "message": { + "text": "'x' is assigned a value but never used." + }, + "locations": [ + { + "physicalLocation": { + "artifactLocation": { + "uri": "file:///C:/dev/sarif/sarif-tutorials/samples/Introduction/simple-example.js", + "index": 0, + }, + "region": { + "startLine": 1, + "startColumn": 5, + }, + } + } + ], + "ruleId": "no-unused-vars", + "ruleIndex": 0, + } + ], + } + ], + } + + response = create_response() + response.json.return_value = { + "id": "47177e22-5596-11eb-80a1-c1e54ef945c6", + "url": "https://api.github.com/repos/octocat/hello-world/code-scanning/sarifs/47177e22-5596-11eb-80a1-c1e54ef945c6", + } + self.client.post.return_value = response + + resp = await self.api.upload_sarif_data( + "foo/bar", + commit_sha="4b6472266afd7b471e86085a6659e8c7f2b119da", + ref="refs/heads/master", + sarif=json.dumps(sarif).encode(), + ) + self.client.post.assert_awaited_once_with( + "/repos/foo/bar/code-scanning/sarifs", + data={ + "commit_sha": "4b6472266afd7b471e86085a6659e8c7f2b119da", + "ref": "refs/heads/master", + "sarif": "H4sIAAAAAAACA7VSO2/cMAz+K4JQIEttJUW73Jp2CHBbkC5FBsXm2Qpk0SCl6wUH//eSsu/aoshYaNGD/F7i2R6BOGCyO2M/tXftrf1o7AfuRpi83o05zzvnXhlTu95yRoIWaXDsKRya2tVQntrP2kslsTT+ONuMGGV3tj0FYanb5CdQ2G+P+5Cy1od0QJp8Fg1PFC6ULJzAUWqUacWNsAGHXssSNiUVhr45emIt4REpfwXuKMx59SQq4JS1vA/sY8SfZm0y0hT8i2Iu0jpCnN+ldz127KoA9y/rTDgD5VDVnW3nMwxIbwr1/TfH8rwoj5fCg+/y5iRi569Ky8p/CBGE3t3vXA/HNeQt6lwk++Ajy3maVc5DyoR96RTEcdDLBk71sX2ttBodcIlXSjiCfosFIiQ1MAGzH+CvtG5ONyaw8cxhSJKWl7xiAfNSskmCQEYzaGt2FxMbwTy+ceh83P/p7eJ7/58N14Hq4SS4t0u1PlzYOIsImTo1eqfToud7jGXS9y/LlpX88sM781XfrujPsn4BtlGkUj8DAAA=", + }, + ) + + self.assertEqual(resp["id"], "47177e22-5596-11eb-80a1-c1e54ef945c6") + self.assertEqual( + resp["url"], + "https://api.github.com/repos/octocat/hello-world/code-scanning/sarifs/47177e22-5596-11eb-80a1-c1e54ef945c6", + ) + + async def test_sarif(self): + response = create_response() + response.json.return_value = { + "processing_status": "complete", + "analyses_url": "https://api.github.com/repos/octocat/hello-world/code-scanning/analyses?sarif_id=47177e22-5596-11eb-80a1-c1e54ef945c6", + } + self.client.get.return_value = response + + resp = await self.api.sarif( + "foo/bar", "47177e22-5596-11eb-80a1-c1e54ef945c6" + ) + self.client.get.assert_awaited_once_with( + "/repos/foo/bar/code-scanning/sarifs/47177e22-5596-11eb-80a1-c1e54ef945c6", + ) + + self.assertEqual(resp.processing_status, SarifProcessingStatus.COMPLETE) + self.assertEqual( + resp.analyses_url, + "https://api.github.com/repos/octocat/hello-world/code-scanning/analyses?sarif_id=47177e22-5596-11eb-80a1-c1e54ef945c6", + ) + self.assertIsNone(resp.errors) diff --git a/tests/github/models/test_code_scanning.py b/tests/github/models/test_code_scanning.py index 580e50cbf..9a6ef4b6b 100644 --- a/tests/github/models/test_code_scanning.py +++ b/tests/github/models/test_code_scanning.py @@ -19,6 +19,8 @@ Location, QuerySuite, Rule, + SarifProcessingStatus, + SarifUploadInformation, Severity, Tool, ) @@ -427,3 +429,21 @@ def test_from_dict(self): datetime(2023, 1, 19, 11, 21, 34, tzinfo=timezone.utc), ) self.assertEqual(setup.schedule, "weekly") + + +class SarifUploadInformationTestCase(unittest.TestCase): + def test_from_dict(self): + sarif = SarifUploadInformation.from_dict( + { + "processing_status": "complete", + "analyses_url": "https://api.github.com/repos/octocat/hello-world/code-scanning/analyses?sarif_id=47177e22-5596-11eb-80a1-c1e54ef945c6", + } + ) + self.assertEqual( + sarif.processing_status, SarifProcessingStatus.COMPLETE + ) + self.assertEqual( + sarif.analyses_url, + "https://api.github.com/repos/octocat/hello-world/code-scanning/analyses?sarif_id=47177e22-5596-11eb-80a1-c1e54ef945c6", + ) + self.assertIsNone(sarif.errors)