feat: minor adjustments to gh backfill task (#376)

* feat: minor adjustments to gh backfill task * rename function for better description
codecov · Apr 11, 2024 · c7d4f45 · c7d4f45
1 parent 6d77be2
commit c7d4f45
Show file tree

Hide file tree

Showing 4 changed files with 70 additions and 29 deletions.
diff --git a/requirements.in b/requirements.in
@@ -1,4 +1,4 @@
-https://github.com/codecov/shared/archive/b48aec853deed52c19b0c3a44549787009861691.tar.gz#egg=shared
+https://github.com/codecov/shared/archive/b653541785c677d6bc2ba438c628e1e645d521b9.tar.gz#egg=shared
 https://github.com/codecov/opentelem-python/archive/refs/tags/v0.0.4a1.tar.gz#egg=codecovopentelem
 https://github.com/codecov/test-results-parser/archive/5515e960d5d38881036e9127f86320efca649f13.tar.gz#egg=test-results-parser
 boto3

diff --git a/requirements.txt b/requirements.txt
@@ -1,5 +1,5 @@
 #
-# This file is autogenerated by pip-compile with Python 3.11
+# This file is autogenerated by pip-compile with Python 3.9
 # by the following command:
 #
 #    pip-compile requirements.in
@@ -116,6 +116,8 @@ ecdsa==0.18.0
     # via tlslite-ng
 excel-base==1.0.4
     # via django-excel-response2
+exceptiongroup==1.2.0
+    # via pytest
 factory-boy==3.2.0
     # via -r requirements.in
 faker==8.8.2
@@ -355,7 +357,9 @@ requests==2.31.0
 respx==0.20.2
     # via -r requirements.in
 rfc3986[idna2008]==1.4.0
-    # via httpx
+    # via
+    #   httpx
+    #   rfc3986
 rsa==4.7.2
     # via google-auth
 s3transfer==0.3.4
@@ -366,7 +370,7 @@ sentry-sdk==1.40.0
     # via
     #   -r requirements.in
     #   shared
-shared @ https://github.com/codecov/shared/archive/b48aec853deed52c19b0c3a44549787009861691.tar.gz
+shared @ https://github.com/codecov/shared/archive/b653541785c677d6bc2ba438c628e1e645d521b9.tar.gz
     # via -r requirements.in
 six==1.15.0
     # via
@@ -410,12 +414,16 @@ timestring @ https://github.com/codecov/timestring/archive/d37ceacc5954dff3b5bd2
     # via -r requirements.in
 tlslite-ng==0.8.0b1
     # via shared
+tomli==2.0.1
+    # via pytest
 tqdm==4.66.1
     # via openai
 typing==3.7.4.3
     # via shared
 typing-extensions==4.6.3
     # via
+    #   asgiref
+    #   kombu
     #   openai
     #   opentelemetry-sdk
     #   pydantic

diff --git a/tasks/backfill_gh_app_installations.py b/tasks/backfill_gh_app_installations.py
@@ -3,11 +3,17 @@
 
 import shared.torngit as torngit
 from asgiref.sync import async_to_sync
+from shared.config import get_config
 from sqlalchemy.orm.session import Session
 
 from app import celery_app
 from celery_config import backfill_gh_app_installations_name
-from database.models.core import GithubAppInstallation, Owner, Repository
+from database.models.core import (
+    GITHUB_APP_INSTALLATION_DEFAULT_NAME,
+    GithubAppInstallation,
+    Owner,
+    Repository,
+)
 from services.owner import get_owner_provider_service
 from tasks.base import BaseCodecovTask
 
@@ -54,6 +60,28 @@ def add_repos_service_ids_from_provider(
             gh_app_installation.repository_service_ids = new_repo_service_ids
             db_session.commit()
 
+    def maybe_set_installation_to_all_repos(
+        self,
+        db_session: Session,
+        owner_service,
+        gh_app_installation: GithubAppInstallation,
+    ):
+        remote_gh_app_installation = async_to_sync(
+            owner_service.get_gh_app_installation
+        )(installation_id=gh_app_installation.installation_id)
+        repository_selection = remote_gh_app_installation.get(
+            "repository_selection", ""
+        )
+        if repository_selection == "all":
+            gh_app_installation.repository_service_ids = None
+            db_session.commit()
+            log.info(
+                "Selection is set to all, no installation is needed",
+                extra=dict(ownerid=gh_app_installation.ownerid),
+            )
+            return True
+        return False
+
     def backfill_existing_gh_apps(
         self, db_session: Session, owner_ids: List[int] = None
     ):
@@ -84,29 +112,19 @@ def backfill_existing_gh_apps(
 
         for gh_app_installation in gh_app_installations:
             # Check if gh app has 'all' repositories selected
-            installation_id = gh_app_installation.installation_id
-
             owner = owners_dict[gh_app_installation.ownerid]
             ownerid = owner.ownerid
 
             owner_service = get_owner_provider_service(
                 owner=owner, using_integration=True
             )
-
-            remote_gh_app_installation = async_to_sync(
-                owner_service.get_gh_app_installation
-            )(installation_id=installation_id)
-            repository_selection = remote_gh_app_installation.get(
-                "repository_selection", ""
+            is_selection_all = self.maybe_set_installation_to_all_repos(
+                db_session=db_session,
+                owner_service=owner_service,
+                gh_app_installation=gh_app_installation,
             )
-            if repository_selection == "all":
-                gh_app_installation.repository_service_ids = None
-                db_session.commit()
-                log.info(
-                    "Selection is set to all, no installation is needed",
-                    extra=dict(ownerid=ownerid),
-                )
-            else:
+
+            if not is_selection_all:
                 # Find and add all repos the gh app has access to
                 self.add_repos_service_ids_from_provider(
                     db_session=db_session,
@@ -153,18 +171,28 @@ def backfill_owners_with_integration_without_gh_app(
                 extra=dict(ownerid=ownerid),
             )
             gh_app_installation = GithubAppInstallation(
-                owner=owner, installation_id=owner.integration_id
+                owner=owner,
+                installation_id=owner.integration_id,
+                app_id=get_config("github", "integration", "id"),
+                name=GITHUB_APP_INSTALLATION_DEFAULT_NAME,
             )
             db_session.add(gh_app_installation)
             db_session.commit()
 
-            # Find and add all repos the gh app has access to
-            self.add_repos_service_ids_from_provider(
+            is_selection_all = self.maybe_set_installation_to_all_repos(
                 db_session=db_session,
-                ownerid=ownerid,
                 owner_service=owner_service,
                 gh_app_installation=gh_app_installation,
             )
+
+            if not is_selection_all:
+                # Find and add all repos the gh app has access to
+                self.add_repos_service_ids_from_provider(
+                    db_session=db_session,
+                    ownerid=ownerid,
+                    owner_service=owner_service,
+                    gh_app_installation=gh_app_installation,
+                )
             log.info("Successful backfill", extra=dict(ownerid=ownerid))
 
     def run_impl(
@@ -174,6 +202,10 @@ def run_impl(
         *args,
         **kwargs
     ):
+        log.info(
+            "Starting GH App backfill task",
+        )
+
         # Backfill gh apps we already have
         self.backfill_existing_gh_apps(db_session=db_session, owner_ids=owner_ids)
 

diff --git a/tasks/tests/unit/test_backfill_gh_app_installations.py b/tasks/tests/unit/test_backfill_gh_app_installations.py
@@ -312,10 +312,11 @@ def test_backfill_with_both_types_of_owners(
         dbsession.add_all([owner, gh_app_installation, owner_without_app])
         dbsession.commit()
 
-        # Mock fn return values
-        mock_repo_provider.get_gh_app_installation.return_value = {
-            "repository_selection": "all"
-        }
+        # Mock fn return values; 1st call "all", 2nd call "selected"
+        mock_repo_provider.get_gh_app_installation.side_effect = [
+            {"repository_selection": "all"},
+            {"repository_selection": "selected"},
+        ]
         mock_repo_provider.list_repos_using_installation.return_value = mock_repos
         mocker.patch(
             f"tasks.backfill_gh_app_installations.get_owner_provider_service",