-
Notifications
You must be signed in to change notification settings - Fork 16
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Add a new metric 'test suites' that can be used to count test suites (or test scenario's in Robot Framework parlance). Test suites can be filtered by test result. Supporting sources are Robot Framework, JUnit, and TestNG. Closes #10078.
- Loading branch information
Showing
27 changed files
with
710 additions
and
97 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
58 changes: 58 additions & 0 deletions
58
components/collector/src/source_collectors/junit/test_suites.py
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,58 @@ | ||
| """JUnit test suites collector.""" | ||
|
|
||
| from typing import cast | ||
| from xml.etree.ElementTree import Element # nosec # Element is not available from defusedxml, but only used as type | ||
|
|
||
| from base_collectors import XMLFileSourceCollector | ||
| from collector_utilities.functions import parse_source_response_xml | ||
| from model import Entities, Entity, SourceMeasurement, SourceResponses | ||
|
|
||
|
|
||
| class JUnitTestSuites(XMLFileSourceCollector): | ||
| """Collector for JUnit test suites.""" | ||
|
|
||
| async def _parse_source_responses(self, responses: SourceResponses) -> SourceMeasurement: | ||
| """Override to parse the test suites from the JUnit XML.""" | ||
| entities = Entities() | ||
| total = 0 | ||
| for response in responses: | ||
| tree = await parse_source_response_xml(response) | ||
| test_suites = [tree] if tree.tag == "testsuite" else tree.findall(".//testsuite[testcase]") | ||
| for test_suite in test_suites: | ||
| parsed_entity = self.__entity(test_suite) | ||
| if self._include_entity(parsed_entity): | ||
| entities.append(parsed_entity) | ||
| total += 1 | ||
| return SourceMeasurement(entities=entities, total=str(total)) | ||
|
|
||
| def _include_entity(self, entity: Entity) -> bool: | ||
| """Return whether to include the entity in the measurement.""" | ||
| results_to_count = cast(list[str], self._parameter("test_result")) | ||
| return entity["suite_result"] in results_to_count | ||
|
|
||
| @staticmethod | ||
| def __entity(suite: Element) -> Entity: | ||
| """Transform a test case into a test suite entity.""" | ||
| name = suite.get("name", "unknown") | ||
| tests = len(suite.findall("testcase")) | ||
| skipped = int(suite.get("skipped", 0)) | ||
| failed = int(suite.get("failures", 0)) | ||
| errored = int(suite.get("errors", 0)) | ||
| passed = tests - (errored + failed + skipped) | ||
| suite_result = "passed" | ||
| if errored: | ||
| suite_result = "errored" | ||
| elif failed: | ||
| suite_result = "failed" | ||
| elif skipped: | ||
| suite_result = "skipped" | ||
| return Entity( | ||
| key=name, | ||
| suite_name=name, | ||
| suite_result=suite_result, | ||
| tests=str(tests), | ||
| passed=str(passed), | ||
| errored=str(errored), | ||
| failed=str(failed), | ||
| skipped=str(skipped), | ||
| ) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
71 changes: 71 additions & 0 deletions
71
components/collector/src/source_collectors/robot_framework/test_suites.py
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,71 @@ | ||
| """Robot Framework test suites collector.""" | ||
|
|
||
| from typing import cast | ||
| from xml.etree.ElementTree import Element # nosec # Element is not available from defusedxml, but only used as type | ||
|
|
||
| from collector_utilities.functions import parse_source_response_xml | ||
| from collector_utilities.type import Response | ||
| from model import Entities, Entity, SourceMeasurement, SourceResponses | ||
|
|
||
| from .base import RobotFrameworkBaseClass | ||
|
|
||
|
|
||
| class RobotFrameworkTestSuites(RobotFrameworkBaseClass): | ||
| """Collector for Robot Framework test suites.""" | ||
|
|
||
| async def _parse_source_responses(self, responses: SourceResponses) -> SourceMeasurement: | ||
| """Override to parse the tests from the Robot Framework XML.""" | ||
| nr_of_suites, total_nr_of_suites, suite_entities = 0, 0, Entities() | ||
| test_results = cast(list[str], self._parameter("test_result")) | ||
| for response in responses: | ||
| count, total, entities = await self._parse_source_response(response, test_results) | ||
| nr_of_suites += count | ||
| total_nr_of_suites += total | ||
| suite_entities.extend(entities) | ||
| return SourceMeasurement(value=str(nr_of_suites), total=str(total_nr_of_suites), entities=suite_entities) | ||
|
|
||
| @classmethod | ||
| async def _parse_source_response(cls, response: Response, test_results: list[str]) -> tuple[int, int, Entities]: | ||
| """Parse a Robot Framework XML.""" | ||
| nr_of_suites, total_nr_of_suites, entities = 0, 0, Entities() | ||
| tree = await parse_source_response_xml(response) | ||
| for suite in tree.findall(".//suite[test]"): | ||
| total_nr_of_suites += 1 | ||
| suite_status = suite.find("status") | ||
| if suite_status is None: | ||
| continue | ||
| suite_result = suite_status.get("status", "").lower() | ||
| if suite_result not in test_results: | ||
| continue | ||
| nr_of_suites += 1 | ||
| entities.append(cls._create_entity(suite, suite_result)) | ||
| return nr_of_suites, total_nr_of_suites, entities | ||
|
|
||
| @classmethod | ||
| def _create_entity(cls, suite: Element, suite_result: str) -> Entity: | ||
| """Create a measurement entity from the test suite element.""" | ||
| suite_id = suite.get("id", "") | ||
| suite_name = suite.get("name", "unknown") | ||
| doc = suite.find("doc") | ||
| suite_documentation = "" if doc is None else doc.text | ||
| tests = len(suite.findall("test")) | ||
| failed = cls._nr_tests(suite, "FAIL") | ||
| errored = cls._nr_tests(suite, "ERROR") | ||
| skipped = cls._nr_tests(suite, "SKIP") | ||
| passed = tests - (failed + errored + skipped) | ||
| return Entity( | ||
| key=suite_id, | ||
| suite_name=suite_name, | ||
| suite_documentation=suite_documentation, | ||
| suite_result=suite_result, | ||
| tests=str(tests), | ||
| passed=str(passed), | ||
| failed=str(failed), | ||
| errored=str(errored), | ||
| skipped=str(skipped), | ||
| ) | ||
|
|
||
| @staticmethod | ||
| def _nr_tests(suite: Element, status: str) -> int: | ||
| """Return the number of tests in the suite with the given status.""" | ||
| return len(suite.findall(f"test/status[@status='{status}']")) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
62 changes: 62 additions & 0 deletions
62
components/collector/src/source_collectors/testng/test_suites.py
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,62 @@ | ||
| """TestNG test suites collector.""" | ||
|
|
||
| from typing import cast | ||
| from xml.etree.ElementTree import Element # nosec # Element is not available from defusedxml, but only used as type | ||
|
|
||
| from base_collectors import XMLFileSourceCollector | ||
| from collector_utilities.functions import parse_source_response_xml | ||
| from model import Entities, Entity, SourceMeasurement, SourceResponses | ||
|
|
||
|
|
||
| class TestNGTestSuites(XMLFileSourceCollector): | ||
| """Collector for TestNG test suites.""" | ||
|
|
||
| async def _parse_source_responses(self, responses: SourceResponses) -> SourceMeasurement: | ||
| """Override to parse the suites for the TestNG XML.""" | ||
| entities = Entities() | ||
| total = 0 | ||
| for response in responses: | ||
| tree = await parse_source_response_xml(response) | ||
| for test_suite in tree.findall(".//suite[test]"): | ||
| parsed_entity = self.__entity(test_suite) | ||
| if self._include_entity(parsed_entity): | ||
| entities.append(parsed_entity) | ||
| total += 1 | ||
| return SourceMeasurement(entities=entities, total=str(total)) | ||
|
|
||
| def _include_entity(self, entity: Entity) -> bool: | ||
| """Return whether to include the entity in the measurement.""" | ||
| results_to_count = cast(list[str], self._parameter("test_result")) | ||
| return entity["suite_result"] in results_to_count | ||
|
|
||
| @classmethod | ||
| def __entity(cls, suite: Element) -> Entity: | ||
| """Transform a test suite element into a test suite entity.""" | ||
| name = suite.get("name", "unknown") | ||
| tests = len(suite.findall(".//test-method")) | ||
| skipped = cls._nr_tests(suite, "SKIP") | ||
| failed = cls._nr_tests(suite, "FAIL") | ||
| passed = cls._nr_tests(suite, "PASS") | ||
| ignored = tests - (skipped + failed + passed) | ||
| suite_result = "passed" | ||
| if failed: | ||
| suite_result = "failed" | ||
| elif skipped: | ||
| suite_result = "skipped" | ||
| elif ignored: | ||
| suite_result = "ignored" | ||
| return Entity( | ||
| key=name, | ||
| suite_name=name, | ||
| suite_result=suite_result, | ||
| tests=str(tests), | ||
| passed=str(passed), | ||
| ignored=str(ignored), | ||
| failed=str(failed), | ||
| skipped=str(skipped), | ||
| ) | ||
|
|
||
| @staticmethod | ||
| def _nr_tests(suite: Element, status: str) -> int: | ||
| """Return the number of tests in the suite with the given status.""" | ||
| return len(suite.findall(f".//test-method[@status='{status}']")) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.