diff --git a/components/collector/src/source_collectors/dependency_track/base.py b/components/collector/src/source_collectors/dependency_track/base.py index 722942eecd..a72784eb9a 100644 --- a/components/collector/src/source_collectors/dependency_track/base.py +++ b/components/collector/src/source_collectors/dependency_track/base.py @@ -1,10 +1,25 @@ """Dependency-Track base collector.""" +from collections.abc import AsyncIterator +from typing import TypedDict, cast + from base_collectors import SourceCollector -from collector_utilities.type import URL +from collector_utilities.functions import match_string_or_regular_expression +from collector_utilities.type import URL, Response from model import SourceResponses +class DependencyTrackProject(TypedDict): + """Project as returned by Dependency-Track.""" + + # Last BOM import is a Unix timestamp, despite the Dependency-Tracker Swagger docs saying it's a datetime string + # See https://github.com/DependencyTrack/dependency-track/issues/840 + lastBomImport: int + name: str + uuid: str + version: str + + class DependencyTrackBase(SourceCollector): """Dependency-Track base class.""" @@ -40,9 +55,28 @@ async def _get_source_responses(self, *urls: URL) -> SourceResponses: async def _get_project_uuids(self) -> dict[str, str]: """Return a mapping of project UUIDs to project names.""" + return {project["uuid"]: project["name"] async for project in self._get_projects()} + + async def _get_projects(self) -> AsyncIterator[DependencyTrackProject]: + """Return the Dependency-Track projects.""" projects_api = URL(await self._api_url() + "/project") - project_uuids = {} for response in await DependencyTrackBase._get_source_responses(self, projects_api): - projects = await response.json(content_type=None) - project_uuids.update({project["uuid"]: project["name"] for project in projects}) - return project_uuids + # We need an async for-loop and yield projects one by one because Python has no `async yield from`, + # see https://peps.python.org/pep-0525/#asynchronous-yield-from + async for project in self._get_projects_from_response(response): + yield project + + async def _get_projects_from_response(self, response: Response) -> AsyncIterator[DependencyTrackProject]: + """Return the projects from the response that match the configured project names and versions.""" + project_names = cast(list, self._parameter("project_names")) + project_versions = cast(list, self._parameter("project_versions")) + for project in await response.json(content_type=None): + if self._project_matches(project, project_names, project_versions): + yield project + + @staticmethod + def _project_matches(project: DependencyTrackProject, names: list[str], versions: list[str]) -> bool: + """Return whether the project name matches the project names and versions.""" + project_matches_name = match_string_or_regular_expression(project["name"], names) if names else True + project_matches_version = match_string_or_regular_expression(project["version"], versions) if versions else True + return project_matches_name and project_matches_version diff --git a/components/collector/src/source_collectors/dependency_track/dependencies.py b/components/collector/src/source_collectors/dependency_track/dependencies.py index 180035d282..d7be9451ad 100644 --- a/components/collector/src/source_collectors/dependency_track/dependencies.py +++ b/components/collector/src/source_collectors/dependency_track/dependencies.py @@ -5,14 +5,7 @@ from collector_utilities.type import URL from model import Entities, Entity, SourceResponses -from .base import DependencyTrackBase - - -class DependencyTrackProject(TypedDict): - """Project as returned by Dependency-Track.""" - - name: str - uuid: str +from .base import DependencyTrackBase, DependencyTrackProject class DependencyTrackRepositoryMetaData(TypedDict): diff --git a/components/collector/src/source_collectors/dependency_track/source_up_to_dateness.py b/components/collector/src/source_collectors/dependency_track/source_up_to_dateness.py index bb4711fccd..e57e413115 100644 --- a/components/collector/src/source_collectors/dependency_track/source_up_to_dateness.py +++ b/components/collector/src/source_collectors/dependency_track/source_up_to_dateness.py @@ -1,22 +1,14 @@ """Dependency-Track source up-to-dateness collector.""" from datetime import datetime -from typing import TypedDict from base_collectors import TimePassedCollector from collector_utilities.date_time import datetime_from_timestamp +from collector_utilities.exceptions import CollectorError from collector_utilities.type import URL, Response from model import Entities, Entity, SourceResponses -from .base import DependencyTrackBase - - -class DependencyTrackProject(TypedDict): - """Project as returned by Dependency-Track.""" - - lastBomImport: int # Timestamp - name: str - uuid: str +from .base import DependencyTrackBase, DependencyTrackProject class DependencyTrackSourceUpToDateness(DependencyTrackBase, TimePassedCollector): @@ -28,16 +20,19 @@ async def _api_url(self) -> URL: async def _parse_source_response_date_time(self, response: Response) -> datetime: """Override to parse the timestamp from the response.""" - projects = await response.json(content_type=None) - datetimes = [self._last_bom_import_datetime(project) for project in projects] - return self.minimum(datetimes) + if datetimes := [ + self._last_bom_import_datetime(project) async for project in self._get_projects_from_response(response) + ]: + return self.minimum(datetimes) + error_message = "No projects found" + raise CollectorError(error_message) async def _parse_entities(self, responses: SourceResponses) -> Entities: """Parse the entities from the responses.""" landing_url = str(self._parameter("landing_url")).strip("/") entities = Entities() for response in responses: - for project in await response.json(content_type=None): + async for project in self._get_projects_from_response(response): uuid = project["uuid"] entity = Entity( key=uuid, diff --git a/components/collector/tests/source_collectors/dependency_track/test_dependencies.py b/components/collector/tests/source_collectors/dependency_track/test_dependencies.py index 958c138f7e..3604d7db2b 100644 --- a/components/collector/tests/source_collectors/dependency_track/test_dependencies.py +++ b/components/collector/tests/source_collectors/dependency_track/test_dependencies.py @@ -1,5 +1,6 @@ """Unit tests for the Dependency-Track security warnings collector.""" +from source_collectors.dependency_track.base import DependencyTrackProject from source_collectors.dependency_track.dependencies import DependencyTrackComponent from tests.source_collectors.source_collector_test_case import SourceCollectorTestCase @@ -11,16 +12,15 @@ class DependencyTrackDependenciesTest(SourceCollectorTestCase): METRIC_TYPE = "dependencies" SOURCE_TYPE = "dependency_track" - def setUp(self) -> None: - """Extend to set up Dependency-Track projects.""" - super().setUp() - self.projects = [{"uuid": "project uuid", "name": "project name"}] + def projects(self) -> list[DependencyTrackProject]: + """Create the Dependency-Track projects fixture.""" + return [DependencyTrackProject(name="project name", uuid="project uuid", version="1.4", lastBomImport=0)] - def create_dependencies(self, latest_version: str) -> list[DependencyTrackComponent]: + def dependencies(self, latest_version: str) -> list[DependencyTrackComponent]: """Create a list of dependencies as returned by Dependency-Track.""" dependency: DependencyTrackComponent = { "name": "component name", - "project": {"name": "project name", "uuid": "project uuid"}, + "project": self.projects()[0], "uuid": "component-uuid", "version": "1.0", } @@ -28,7 +28,7 @@ def create_dependencies(self, latest_version: str) -> list[DependencyTrackCompon dependency["repositoryMeta"] = {"latestVersion": latest_version} return [dependency] - def create_entities(self, latest_version: str, latest_version_status: str) -> list[dict[str, str]]: + def entities(self, latest_version: str, latest_version_status: str) -> list[dict[str, str]]: """Create a list of expected entities.""" return [ { @@ -50,26 +50,51 @@ async def test_no_projects(self): async def test_no_vulnerabilities(self): """Test one project without dependencies.""" - response = await self.collect(get_request_json_side_effect=[self.projects, []]) + response = await self.collect(get_request_json_side_effect=[self.projects(), []]) self.assert_measurement(response, value="0", entities=[]) async def test_updateable_dependencies(self): """Test one project with a component that can be updated.""" - response = await self.collect(get_request_json_side_effect=[self.projects, self.create_dependencies("1.1")]) - self.assert_measurement(response, value="1", entities=self.create_entities("1.1", "update possible")) + response = await self.collect(get_request_json_side_effect=[self.projects(), self.dependencies("1.1")]) + self.assert_measurement(response, value="1", entities=self.entities("1.1", "update possible")) async def test_up_to_date_dependencies(self): """Test one project with a component that is up to date.""" - response = await self.collect(get_request_json_side_effect=[self.projects, self.create_dependencies("1.0")]) - self.assert_measurement(response, value="1", entities=self.create_entities("1.0", "up-to-date")) + response = await self.collect(get_request_json_side_effect=[self.projects(), self.dependencies("1.0")]) + self.assert_measurement(response, value="1", entities=self.entities("1.0", "up-to-date")) async def test_unknown_latest_version(self): """Test one project with a component whose latest version is unknown.""" - response = await self.collect(get_request_json_side_effect=[self.projects, self.create_dependencies("")]) - self.assert_measurement(response, value="1", entities=self.create_entities("unknown", "unknown")) + response = await self.collect(get_request_json_side_effect=[self.projects(), self.dependencies("")]) + self.assert_measurement(response, value="1", entities=self.entities("unknown", "unknown")) async def test_filter_by_latest_version_status(self): """Test that components can be filtered by latest version status.""" self.set_source_parameter("latest_version_status", ["update possible"]) - response = await self.collect(get_request_json_side_effect=[self.projects, self.create_dependencies("1.0")]) + response = await self.collect(get_request_json_side_effect=[self.projects(), self.dependencies("1.0")]) self.assert_measurement(response, value="0", entities=[]) + + async def test_filter_by_project_name(self): + """Test filtering projects by name.""" + self.set_source_parameter("project_names", ["other project"]) + response = await self.collect(get_request_json_return_value=self.projects()) + self.assert_measurement(response, value="0", entities=[]) + + async def test_filter_by_project_regular_expression(self): + """Test filtering projects by regular expression.""" + self.set_source_parameter("project_names", ["project .*"]) + response = await self.collect(get_request_json_side_effect=[self.projects(), self.dependencies("1.0")]) + self.assert_measurement(response, value="1", entities=self.entities("1.0", "up-to-date")) + + async def test_filter_by_project_version(self): + """Test filtering projects by version.""" + self.set_source_parameter("project_versions", ["1.2", "1.3"]) + response = await self.collect(get_request_json_return_value=self.projects()) + self.assert_measurement(response, value="0", entities=[]) + + async def test_filter_by_project_name_and_version(self): + """Test filtering projects by name and version.""" + self.set_source_parameter("project_names", ["project .*"]) + self.set_source_parameter("project_versions", ["1.3", "1.4"]) + response = await self.collect(get_request_json_side_effect=[self.projects(), self.dependencies("1.0")]) + self.assert_measurement(response, value="1", entities=self.entities("1.0", "up-to-date")) diff --git a/components/collector/tests/source_collectors/dependency_track/test_security_warnings.py b/components/collector/tests/source_collectors/dependency_track/test_security_warnings.py index 1ecd3d7c34..4feed254e1 100644 --- a/components/collector/tests/source_collectors/dependency_track/test_security_warnings.py +++ b/components/collector/tests/source_collectors/dependency_track/test_security_warnings.py @@ -2,6 +2,13 @@ from aiohttp import BasicAuth +from source_collectors.dependency_track.base import DependencyTrackProject +from source_collectors.dependency_track.security_warnings import ( + DependencyTrackComponent, + DependencyTrackFinding, + DependencyTrackVulnerability, +) + from tests.source_collectors.source_collector_test_case import SourceCollectorTestCase @@ -11,22 +18,29 @@ class DependencyTrackSecurityWarningsTest(SourceCollectorTestCase): METRIC_TYPE = "security_warnings" SOURCE_TYPE = "dependency_track" - def setUp(self) -> None: - """Extend to set up Dependency-Track responses and expected measurement entities.""" - super().setUp() - self.projects = [{"uuid": "project uuid", "name": "project name"}] - self.vulnerabilities = [ - { - "component": {"name": "component name", "project": "project uuid", "uuid": "component-uuid"}, - "matrix": "matrix", - "vulnerability": { - "description": "vulnerability description", - "severity": "UNASSIGNED", - "vulnId": "CVE-123", - }, - }, + def projects(self) -> list[DependencyTrackProject]: + """Create the Dependency-Track projects fixture.""" + return [DependencyTrackProject(name="project name", uuid="project uuid", version="1.4", lastBomImport=0)] + + def findings(self) -> list[DependencyTrackFinding]: + """Create the Dependency-Track findings fixture.""" + return [ + DependencyTrackFinding( + component=DependencyTrackComponent( + name="component name", project="project uuid", uuid="component-uuid" + ), + matrix="matrix", + vulnerability=DependencyTrackVulnerability( + description="vulnerability description", + severity="UNASSIGNED", + vulnId="CVE-123", + ), + ), ] - self.entities = [ + + def entities(self) -> list[dict[str, str]]: + """Create the expected entities.""" + return [ { "component": "component name", "component_landing_url": "/components/component-uuid", @@ -46,20 +60,45 @@ async def test_no_projects(self): async def test_one_project_without_vulnerabilities(self): """Test one project without vulnerabilities.""" - response = await self.collect(get_request_json_side_effect=[self.projects, []]) + response = await self.collect(get_request_json_side_effect=[self.projects(), []]) self.assert_measurement(response, value="0", entities=[]) async def test_one_project_with_vulnerabilities(self): """Test one project with vulnerabilities.""" - response = await self.collect(get_request_json_side_effect=[self.projects, self.vulnerabilities]) - self.assert_measurement(response, value="1", entities=self.entities) + response = await self.collect(get_request_json_side_effect=[self.projects(), self.findings()]) + self.assert_measurement(response, value="1", entities=self.entities()) - async def test_filter_vulnerabilities(self): + async def test_filter_by_severity(self): """Test that vulnerabilities can be filtered.""" self.set_source_parameter("severities", ["High", "Critical"]) - response = await self.collect(get_request_json_side_effect=[self.projects, self.vulnerabilities]) + response = await self.collect(get_request_json_side_effect=[self.projects(), self.findings()]) self.assert_measurement(response, value="0", entities=[]) + async def test_filter_by_project_name(self): + """Test filtering projects by name.""" + self.set_source_parameter("project_names", ["other project"]) + response = await self.collect(get_request_json_side_effect=[self.projects(), self.findings()]) + self.assert_measurement(response, value="0", entities=[]) + + async def test_filter_by_project_regular_expression(self): + """Test filtering projects by regular expression.""" + self.set_source_parameter("project_names", ["project .*"]) + response = await self.collect(get_request_json_side_effect=[self.projects(), self.findings()]) + self.assert_measurement(response, value="1", entities=self.entities()) + + async def test_filter_by_project_version(self): + """Test filtering projects by version.""" + self.set_source_parameter("project_versions", ["1.2", "1.3"]) + response = await self.collect(get_request_json_side_effect=[self.projects(), self.findings()]) + self.assert_measurement(response, value="0", entities=[]) + + async def test_filter_by_project_name_and_version(self): + """Test filtering projects by name and version.""" + self.set_source_parameter("project_names", ["project .*"]) + self.set_source_parameter("project_versions", ["1.3", "1.4"]) + response = await self.collect(get_request_json_side_effect=[self.projects(), self.findings()]) + self.assert_measurement(response, value="1", entities=self.entities()) + async def test_api_key(self): """Test that the API key is passed as header.""" self.set_source_parameter("private_token", "API key") diff --git a/components/collector/tests/source_collectors/dependency_track/test_source_up_to_dateness.py b/components/collector/tests/source_collectors/dependency_track/test_source_up_to_dateness.py index 1d8eb8abf4..f25113c094 100644 --- a/components/collector/tests/source_collectors/dependency_track/test_source_up_to_dateness.py +++ b/components/collector/tests/source_collectors/dependency_track/test_source_up_to_dateness.py @@ -4,6 +4,8 @@ from dateutil.tz import tzlocal +from source_collectors.dependency_track.base import DependencyTrackProject + from tests.source_collectors.source_collector_test_case import SourceCollectorTestCase @@ -13,35 +15,63 @@ class DependencyTrackSourceUpToDatenessVersionTest(SourceCollectorTestCase): METRIC_ADDITION = "min" METRIC_TYPE = "source_up_to_dateness" SOURCE_TYPE = "dependency_track" + LANDING_URL = "https://dependency_track" - async def test_source_up_to_dateness(self): - """Test that the source up-to-dateness can be measured.""" + def projects(self) -> list[DependencyTrackProject]: + """Create Dependency-Track projects fixture.""" now = datetime.now(tz=tzlocal()).replace(microsecond=0) - yesterday = now - timedelta(days=1) - yesterday_timestamp = yesterday.timestamp() * 1000 # To be prepared, test a timestamp that is a float... - last_week = now - timedelta(days=7) - last_week_timestamp = str(int(last_week.timestamp() * 1000)) # ... and test a string containing an integer - projects = [ - {"name": "Project 1", "lastBomImport": yesterday_timestamp, "uuid": "p1"}, - {"name": "Project 2", "lastBomImport": str(int(last_week_timestamp)), "uuid": "p2"}, + self.yesterday = now - timedelta(days=1) + yesterday_timestamp = round(self.yesterday.timestamp() * 1000) + self.last_week = now - timedelta(days=7) + last_week_timestamp = round(self.last_week.timestamp() * 1000) + return [ + DependencyTrackProject(name="Project 1", version="1.1", uuid="p1", lastBomImport=yesterday_timestamp), + DependencyTrackProject(name="Project 2", version="1.2", uuid="p2", lastBomImport=last_week_timestamp), ] - response = await self.collect(get_request_json_return_value=projects) - self.assert_measurement( - response, - value="7", - landing_url="https://dependency_track", - entities=[ - { - "key": "p1", - "last_bom_import": yesterday.isoformat(), - "project": "Project 1", - "project_landing_url": "/projects/p1", - }, - { - "key": "p2", - "last_bom_import": last_week.isoformat(), - "project": "Project 2", - "project_landing_url": "/projects/p2", - }, - ], - ) + + def entities(self) -> list[dict[str, str]]: + """Create the expected entities.""" + return [ + { + "key": "p1", + "last_bom_import": self.yesterday.isoformat(), + "project": "Project 1", + "project_landing_url": "/projects/p1", + }, + { + "key": "p2", + "last_bom_import": self.last_week.isoformat(), + "project": "Project 2", + "project_landing_url": "/projects/p2", + }, + ] + + async def test_source_up_to_dateness(self): + """Test that the source up-to-dateness can be measured.""" + response = await self.collect(get_request_json_return_value=self.projects()) + self.assert_measurement(response, value="7", landing_url=self.LANDING_URL, entities=self.entities()) + + async def test_filter_by_project_name(self): + """Test that projects can be filtered by name.""" + self.set_source_parameter("project_names", ["other project"]) + response = await self.collect(get_request_json_return_value=self.projects()) + self.assert_measurement(response, parse_error="No projects found") + + async def test_filter_by_regular_expression(self): + """Test that projects can be filtered by regular expression.""" + self.set_source_parameter("project_names", ["Project .*"]) + response = await self.collect(get_request_json_return_value=self.projects()) + self.assert_measurement(response, value="7", landing_url=self.LANDING_URL, entities=self.entities()) + + async def test_filter_by_project_version(self): + """Test filtering projects by version.""" + self.set_source_parameter("project_versions", ["1.2", "1.3"]) + response = await self.collect(get_request_json_return_value=self.projects()) + self.assert_measurement(response, value="7", landing_url=self.LANDING_URL, entities=self.entities()[1:]) + + async def test_filter_by_project_name_and_version(self): + """Test filtering projects by name and version.""" + self.set_source_parameter("project_names", ["Project .*"]) + self.set_source_parameter("project_versions", ["1.3", "1.4"]) + response = await self.collect(get_request_json_return_value=self.projects()) + self.assert_measurement(response, parse_error="No projects found") diff --git a/components/shared_code/src/shared_data_model/sources/dependency_track.py b/components/shared_code/src/shared_data_model/sources/dependency_track.py index d3a59df91a..91f3b63c29 100644 --- a/components/shared_code/src/shared_data_model/sources/dependency_track.py +++ b/components/shared_code/src/shared_data_model/sources/dependency_track.py @@ -4,7 +4,14 @@ from shared_data_model.meta.entity import Color, Entity, EntityAttribute, EntityAttributeType from shared_data_model.meta.source import Source -from shared_data_model.parameters import URL, LandingURL, MultipleChoiceParameter, PrivateToken, Severities +from shared_data_model.parameters import ( + URL, + LandingURL, + MultipleChoiceParameter, + MultipleChoiceWithAdditionParameter, + PrivateToken, + Severities, +) ALL_DEPENDENCY_TRACK_METRICS = ["dependencies", "security_warnings", "source_up_to_dateness", "source_version"] DEPENDENCY_TRACK_URL = HttpUrl("https://dependencytrack.org") @@ -37,6 +44,18 @@ help_url=HttpUrl("https://docs.dependencytrack.org/integrations/rest-api/"), metrics=["dependencies", "source_up_to_dateness", "security_warnings"], ), + "project_names": MultipleChoiceWithAdditionParameter( + name="Project names (regular expressions or project names)", + placeholder="all project names", + short_name="project names", + metrics=["dependencies", "security_warnings", "source_up_to_dateness"], + ), + "project_versions": MultipleChoiceWithAdditionParameter( + name="Project versions (regular expressions or versions)", + placeholder="all project versions", + short_name="project versions", + metrics=["dependencies", "security_warnings", "source_up_to_dateness"], + ), "latest_version_status": MultipleChoiceParameter( name="Latest version statuses", short_name="statuses", diff --git a/docs/src/changelog.md b/docs/src/changelog.md index aaca9ad700..8942523cdb 100644 --- a/docs/src/changelog.md +++ b/docs/src/changelog.md @@ -20,6 +20,10 @@ If your currently installed *Quality-time* version is v4.10.0 or older, please r - Hiding metrics without issues would not hide metrics with deleted issues. Fixes [#8699](https://github.com/ICTU/quality-time/issues/8699). - The spinner indicating that the latest measurement of a metric is not up-to-date with the latest source configuration would not disappear if the measurement value made with the latest source configuration was equal to the measurement value made with the previous source configuration. Fixes [#8702](https://github.com/ICTU/quality-time/issues/8702). +### Added + +- When using Dependency-Track as source for dependencies, security warnings, or source-up-to-dateness, allow for filtering by project name and version. Closes [#8686](https://github.com/ICTU/quality-time/issues/8686). + ## v5.12.0 - 2024-05-17 ### Deployment notes