10083 bitbucket collector

components/collector/src/source_collectors/__init__.py

@@ -22,6 +22,7 @@
 from .azure_devops.user_story_points import AzureDevopsUserStoryPoints
 from .bandit.security_warnings import BanditSecurityWarnings
 from .bandit.source_up_to_dateness import BanditSourceUpToDateness
+from .bitbucket.inactive_branches import BitbucketInactiveBranches
 from .calendar.source_up_to_dateness import CalendarSourceUpToDateness
 from .calendar.time_remaining import CalendarTimeRemaining
 from .cargo_audit.security_warnings import CargoAuditSecurityWarnings

components/collector/src/source_collectors/bitbucket/base.py

@@ -0,0 +1,49 @@
+"""Bitbucket collector base classes."""
+
+from abc import ABC
+
+from base_collectors import SourceCollector
+from collector_utilities.functions import add_query
+from collector_utilities.type import URL
+from model import SourceResponses
+
+
+class BitbucketBase(SourceCollector, ABC):
+    """Base class for Bitbucket collectors."""
+
+    async def _get_source_responses(self, *urls: URL) -> SourceResponses:
+        """Extend to follow Bitbucket pagination links, if necessary."""
+        all_responses = responses = await super()._get_source_responses(*urls)
+        while next_urls := await self._next_urls(responses):
+            # Retrieving consecutive big responses without reading the response hangs the client, see
+            # https://github.com/aio-libs/aiohttp/issues/2217
+            for response in responses:
+                await response.read()
+            all_responses.extend(responses := await super()._get_source_responses(*next_urls))
+        return all_responses
+
+    def _basic_auth_credentials(self) -> tuple[str, str] | None:
+        """Override to return None, as the private token is passed as header."""
+        return None
+
+    def _headers(self) -> dict[str, str]:
+        """Extend to add the private token, if any, to the headers."""
+        headers = super()._headers()
+        if private_token := self._parameter("private_token"):
+            headers["Authorization"] = "Bearer " + str(private_token)
+        return headers
+
+    async def _next_urls(self, responses: SourceResponses) -> list[URL]:
+        """Return the next (pagination) links from the responses."""
+        return [URL(next_url) for response in responses if (next_url := response.links.get("next", {}).get("url"))]
+
+
+class BitbucketProjectBase(BitbucketBase, ABC):
+    """Base class for Bitbucket collectors for a specific project."""
+
+    async def _bitbucket_api_url(self, api: str) -> URL:
+        """Return a Bitbucket API url for a project, if present in the parameters."""
+        url = await super()._api_url()
+        project = self._parameter("project")
+        api_url = URL(f"{url}/rest/api/1.0/projects/{project}" + (f"/{api}" if api else ""))
+        return add_query(api_url, "pagelen=100")

components/collector/src/source_collectors/bitbucket/inactive_branches.py

@@ -0,0 +1,56 @@
+"""Bitbucket inactive branches collector."""
+
+from datetime import datetime
+from typing import cast
+
+from base_collectors import BranchType, InactiveBranchesSourceCollector
+from collector_utilities.date_time import parse_datetime
+from collector_utilities.type import URL
+from model import SourceResponses
+
+from .base import BitbucketProjectBase
+
+
+class BitbucketBranchType(BranchType):
+    """Bitbucket branch information as returned by the API."""
+
+    "Contains commit(ed)-date"
+    commit: dict[str, str]
+    default: bool
+    merged: bool
+    web_url: str
+
+
+class BitbucketInactiveBranches[Branch: BitbucketBranchType](BitbucketProjectBase, InactiveBranchesSourceCollector):
+    """Collector for inactive branches."""
+
+    async def _api_url(self) -> URL:
+        """Override to return the branches API."""
+        return await self._bitbucket_api_url("branches")
+
+    async def _landing_url(self, responses: SourceResponses) -> URL:
+        """Extend to add the project branches."""
+        return URL(f"{await super()._landing_url(responses)}/{self._parameter('project')}/-/branches")
+
+    async def _branches(self, responses: SourceResponses) -> list[Branch]:
+        """Return a list of branches from the responses."""
+        branches = []
+        for response in responses:
+            branches.extend(await response.json())
+        return branches
+
+    def _is_default_branch(self, branch: Branch) -> bool:
+        """Return whether the branch is the default branch."""
+        return branch["default"]
+
+    def _is_branch_merged(self, branch: Branch) -> bool:
+        """Return whether the branch has been merged with the default branch."""
+        return branch["merged"]
+
+    def _commit_datetime(self, branch: Branch) -> datetime:
+        """Override to parse the commit date from the branch."""
+        return parse_datetime(branch["commit"]["committed_date"])
+
+    def _branch_landing_url(self, branch: Branch) -> URL:
+        """Override to get the landing URL from the branch."""
+        return cast(URL, branch.get("web_url") or "")

components/collector/tests/source_collectors/bitbucket/base.py

@@ -0,0 +1,18 @@
+"""Bitbucket unit test base classes."""
+
+from tests.source_collectors.source_collector_test_case import SourceCollectorTestCase
+
+
+class BitbucketTestCase(SourceCollectorTestCase):
+    """Base class for testing Bitbucket collectors."""
+
+    SOURCE_TYPE = "bitbucket"
+    LOOKBACK_DAYS = "100000"
+
+    def setUp(self):
+        """Extend to add generic test fixtures."""
+        super().setUp()
+        self.set_source_parameter("branch", "branch")
+        self.set_source_parameter("file_path", "file")
+        self.set_source_parameter("lookback_days", self.LOOKBACK_DAYS)
+        self.set_source_parameter("project", "namespace/project")

components/collector/tests/source_collectors/bitbucket/test_inactive_branches.py

@@ -0,0 +1,79 @@
+"""Unit tests for the Bitbucket inactive branches collector."""
+
+from datetime import datetime
+
+from dateutil.tz import tzutc
+
+from .base import BitbucketTestCase
+
+
+class BitbucketInactiveBranchesTest(BitbucketTestCase):
+    """Unit tests for the inactive branches metric."""
+
+    METRIC_TYPE = "inactive_branches"
+    WEB_URL = "https://bitbucket/namespace/project/-/tree/branch"
+
+    def setUp(self):
+        """Extend to setup fixtures."""
+        super().setUp()
+        self.set_source_parameter("branches_to_ignore", ["ignored_.*"])
+        main = self.create_branch("main", default=True)
+        unmerged = self.create_branch("unmerged_branch")
+        ignored = self.create_branch("ignored_branch")
+        active_unmerged = self.create_branch("active_unmerged_branch", active=True)
+        recently_merged = self.create_branch("merged_branch", merged=True, active=True)
+        inactive_merged = self.create_branch("merged_branch", merged=True)
+        self.branches = [main, unmerged, ignored, active_unmerged, recently_merged, inactive_merged]
+        self.unmerged_branch_entity = self.create_entity("unmerged_branch", merged=False)
+        self.merged_branch_entity = self.create_entity("merged_branch", merged=True)
+        self.entities = [self.unmerged_branch_entity, self.merged_branch_entity]
+        self.landing_url = "https://bitbucket/namespace/project/-/branches"
+
+    def create_branch(
+        self, name: str, *, default: bool = False, merged: bool = False, active: bool = False
+    ) -> dict[str, str | bool | dict[str, str]]:
+        """Create a Bitbucket branch."""
+        commit_date = datetime.now(tz=tzutc()).isoformat() if active else "2019-04-02T11:33:04.000+02:00"
+        return {
+            "name": name,
+            "default": default,
+            "merged": merged,
+            "web_url": self.WEB_URL,
+            "commit": {"committed_date": commit_date},
+        }
+
+    def create_entity(self, name: str, *, merged: bool) -> dict[str, str]:
+        """Create an entity."""
+        return {
+            "key": name,
+            "name": name,
+            "commit_date": "2019-04-02",
+            "merge_status": "merged" if merged else "unmerged",
+            "url": self.WEB_URL,
+        }
+
+    async def test_inactive_branches(self):
+        """Test that the number of inactive branches can be measured."""
+        response = await self.collect(get_request_json_return_value=self.branches)
+        self.assert_measurement(response, value="2", entities=self.entities, landing_url=self.landing_url)
+
+    async def test_unmerged_inactive_branches(self):
+        """Test that the number of unmerged inactive branches can be measured."""
+        self.set_source_parameter("branch_merge_status", ["unmerged"])
+        response = await self.collect(get_request_json_return_value=self.branches)
+        self.assert_measurement(
+            response, value="1", entities=[self.unmerged_branch_entity], landing_url=self.landing_url
+        )
+
+    async def test_merged_inactive_branches(self):
+        """Test that the number of merged inactive branches can be measured."""
+        self.set_source_parameter("branch_merge_status", ["merged"])
+        response = await self.collect(get_request_json_return_value=self.branches)
+        self.assert_measurement(response, value="1", entities=[self.merged_branch_entity], landing_url=self.landing_url)
+
+    async def test_pagination(self):
+        """Test that pagination works."""
+        branches = [self.branches[:3], self.branches[3:]]
+        links = {"next": {"url": "https://bitbucket/next_page"}}
+        response = await self.collect(get_request_json_side_effect=branches, get_request_links=links)
+        self.assert_measurement(response, value="2", entities=self.entities, landing_url=self.landing_url)

components/shared_code/src/shared_data_model/metrics.py

@@ -135,7 +135,7 @@
     change-your-default-branch).""",
         unit=Unit.BRANCHES,
         near_target="5",
-        sources=["azure_devops", "gitlab", "manual_number"],
+        sources=["azure_devops", "bitbucket", "gitlab", "manual_number"],
         tags=[Tag.CI],
     ),
     "issues": Metric(

components/shared_code/src/shared_data_model/sources/__init__.py

@@ -4,6 +4,7 @@
 from .axe import AXE_CORE, AXE_CSV, AXE_HTML_REPORTER
 from .azure_devops import AZURE_DEVOPS
 from .bandit import BANDIT
+from .bitbucket import BITBUCKET
 from .calendar import CALENDAR
 from .cargo_audit import CARGO_AUDIT
 from .cloc import CLOC
@@ -49,6 +50,7 @@
     "axecsv": AXE_CSV,
     "azure_devops": AZURE_DEVOPS,
     "bandit": BANDIT,
+    "bitbucket": BITBUCKET,
     "calendar": CALENDAR,
     "cargo_audit": CARGO_AUDIT,
     "cloc": CLOC,

components/shared_code/src/shared_data_model/sources/bitbucket.py

@@ -0,0 +1,69 @@
+"""Bitbucket source."""
+
+from pydantic import HttpUrl
+
+from shared_data_model.meta.entity import Entity, EntityAttribute, EntityAttributeType
+from shared_data_model.meta.source import Source
+from shared_data_model.parameters import (
+    URL,
+    BranchesToIgnore,
+    BranchMergeStatus,
+    Days,
+    PrivateToken,
+    StringParameter,
+)
+
+BITBUCKET_BRANCH_HELP_URL = HttpUrl("https://confluence.atlassian.com/bitbucketserver/branches-776639968.html")
+
+BITBUCKET = Source(
+    name="Bitbucket",
+    description="Bitbucket is a Git-based source code hosting and collaboration tool, built for teams.",
+    url=HttpUrl("https://bitbucket.org/product/guides/getting-started/overview#a-brief-overview-of-bitbucket/"),
+    documentation={},
+    parameters={
+        "url": URL(
+            name="Bitbucket instance URL",
+            help="URL of the Bitbucket instance, with port if necessary, but without path. For example, "
+            "'https://bitbucket.org'.",
+            validate_on=["private_token"],
+            metrics=["inactive_branches"],
+        ),
+        "project": StringParameter(
+            name="Project (name with namespace or id)",
+            short_name="project",
+            mandatory=True,
+            help_url=HttpUrl("https://support.atlassian.com/bitbucket-cloud/docs/create-a-project/"),
+            metrics=[
+                "inactive_branches",
+            ],
+        ),
+        "private_token": PrivateToken(
+            name="Private token (with read_api scope)",
+            help_url=HttpUrl("https://support.atlassian.com/bitbucket-cloud/docs/create-a-repository-access-token/"),
+            metrics=["inactive_branches"],
+        ),
+        "branches_to_ignore": BranchesToIgnore(help_url=BITBUCKET_BRANCH_HELP_URL),
+        "branch_merge_status": BranchMergeStatus(),
+        "inactive_days": Days(
+            name="Number of days since last commit after which to consider branches inactive",
+            short_name="number of days since last commit",
+            default_value="7",
+            metrics=["inactive_branches"],
+        ),
+    },
+    entities={
+        "inactive_branches": Entity(
+            name="branch",
+            name_plural="branches",
+            attributes=[
+                EntityAttribute(name="Branch name", key="name", url="url"),
+                EntityAttribute(
+                    name="Date of most recent commit",
+                    key="commit_date",
+                    type=EntityAttributeType.DATE,
+                ),
+                EntityAttribute(name="Merge status"),
+            ],
+        )
+    },
+)

components/shared_code/src/shared_data_model/sources/quality_time.py

@@ -172,6 +172,7 @@
                 "Axe-core",
                 "Azure DevOps Server",
                 "Bandit",
+                "Bitbucket",
                 "Calendar date",
                 "Cargo Audit",
                 "Checkmarx CxSAST",
@@ -223,6 +224,7 @@
                 "Axe-core": "axe_core",
                 "Azure DevOps Server": "azure_devops",
                 "Bandit": "bandit",
+                "Bitbucket": "bitbucket",
                 "Calendar date": "calendar",
                 "Cargo Audit": "cargo_audit",
                 "Checkmarx CxSAST": "cxsast",