change: improve update mechanism for new installations, check mechani…

…sm for blueprints (#353)

CHANGELOG.md

@@ -16,6 +16,8 @@
 
 ### Changed
 
+- Improved the check mechanism for blueprints by only checking a certain number each run and by taking the last check time into account.
+- Improved the update mechanism when installing a new GitHub organization to only update the newly added organization. ([#349](https://github.com/eclipse-csi/otterdog/issues/349))
 - Integrated existing logging with standard python logging facility.
 - Utilized `rich` console formatting instead of low-level colorama styles.
 - Improved processing when archiving repositories to process all other requested changes before archiving them. ([#134](https://github.com/eclipse-csi/otterdog/issues/134))

otterdog/app.py

@@ -5,6 +5,7 @@
 #  which is available at http://www.eclipse.org/legal/epl-v20.html
 #  SPDX-License-Identifier: EPL-2.0
 #  *******************************************************************************
+
 import logging
 import os
 from sys import exit

otterdog/providers/github/rest/app_client.py

@@ -35,7 +35,15 @@ async def get_app_installations(self) -> list[dict[str, Any]]:
         try:
             return await self.requester.request_paged_json("GET", "/app/installations")
         except GitHubException as ex:
-            raise RuntimeError(f"failed retrieving authenticated app:\n{ex}") from ex
+            raise RuntimeError(f"failed retrieving app installations:\n{ex}") from ex
+
+    async def get_app_installation(self, installation_id: int) -> list[dict[str, Any]]:
+        _logger.debug("retrieving app installation for id '%d'", installation_id)
+
+        try:
+            return await self.requester.request_paged_json("GET", f"/app/installations/{installation_id}")
+        except GitHubException as ex:
+            raise RuntimeError(f"failed retrieving app installation:\n{ex}") from ex
 
     async def create_installation_access_token(self, installation_id: str) -> tuple[str, datetime]:
         _logger.debug("creating an installation access token for installation '%s'", installation_id)

otterdog/webapp/blueprints/__init__.py

@@ -42,7 +42,7 @@ class Blueprint(ABC, BaseModel):
 
     @cached_property
     def logger(self) -> Logger:
-        return getLogger(type(self).__name__)
+        return getLogger(__name__)
 
     @property
     @abstractmethod

otterdog/webapp/db/models.py

@@ -170,6 +170,7 @@ class BlueprintModel(Model):
     name: Optional[str] = None
     description: Optional[str] = None
     recheck_needed: bool = True
+    last_checked: Optional[datetime] = Field(index=True, default=None)
     config: dict
 
 

otterdog/webapp/db/service.py

@@ -68,7 +68,7 @@ async def update_installation_status(installation_id: int, action: str) -> None:
         case "created":
             policies = await refresh_global_policies()
             blueprints = await refresh_global_blueprints()
-            await update_app_installations(policies, blueprints)
+            await add_app_installation(installation_id, policies, blueprints)
 
         case "deleted":
             installation = await mongo.odm.find_one(
@@ -214,6 +214,35 @@ async def update_app_installations(
         await update_policies_and_blueprints_for_installation(installation, global_policies, global_blueprints)
 
 
+async def add_app_installation(
+    installation_id: int,
+    global_policies: list[Policy],
+    global_blueprints: list[Blueprint],
+):
+    logger.info("adding app installation for id '%d'", installation_id)
+
+    rest_api = get_rest_api_for_app()
+    app_installation = await rest_api.app.get_app_installation(installation_id)
+
+    async with mongo.odm.session() as session:
+        installation_id = app_installation["id"]
+        github_id = app_installation["account"]["login"]
+        suspended_at = app_installation["suspended_at"]
+        installation_status = InstallationStatus.INSTALLED if suspended_at is None else InstallationStatus.SUSPENDED
+
+        installation_model = await get_installation_by_github_id(github_id)
+        if installation_model is not None:
+            installation_model.installation_id = int(installation_id)
+            installation_model.installation_status = installation_status
+
+            await session.save(installation_model)
+        else:
+            return
+
+    await update_data_for_installation(installation_model)
+    await update_policies_and_blueprints_for_installation(installation_model, global_policies, global_blueprints)
+
+
 async def update_data_for_installation(installation: InstallationModel) -> None:
     from otterdog.webapp.tasks.fetch_all_pull_requests import FetchAllPullRequestsTask
     from otterdog.webapp.tasks.fetch_config import FetchConfigTask
@@ -752,6 +781,14 @@ async def get_blueprints(owner: str) -> list[BlueprintModel]:
     )
 
 
+async def get_blueprints_by_last_checked_time(limit: int) -> list[BlueprintModel]:
+    return await mongo.odm.find(
+        BlueprintModel,
+        limit=limit,
+        sort=query.asc(BlueprintModel.last_checked),
+    )
+
+
 async def find_blueprint(owner: str, blueprint_id: str) -> BlueprintModel | None:
     return await mongo.odm.find_one(
         BlueprintModel,
@@ -760,7 +797,7 @@ async def find_blueprint(owner: str, blueprint_id: str) -> BlueprintModel | None
     )
 
 
-async def update_or_create_blueprint(owner: str, blueprint: Blueprint, recheck_needed: bool | None = None) -> None:
+async def update_or_create_blueprint(owner: str, blueprint: Blueprint) -> bool:
     blueprint_model = await find_blueprint(owner, blueprint.id)
     if blueprint_model is None:
         blueprint_model = BlueprintModel(
@@ -771,15 +808,24 @@ async def update_or_create_blueprint(owner: str, blueprint: Blueprint, recheck_n
             config=blueprint.config,
         )
     else:
-        blueprint_model.path = blueprint.path
-        blueprint_model.name = blueprint.name
-        blueprint_model.description = blueprint.description
-        blueprint_model.config = blueprint.config
+        recheck = False
+
+        def update_if_changed(obj: BlueprintModel, attr: str, value: Any) -> bool:
+            if obj.__getattribute__(attr) != value:
+                obj.__setattr__(attr, value)
+                return True
+            else:
+                return False
+
+        recheck = recheck or update_if_changed(blueprint_model, "path", blueprint.path)
+        recheck = recheck or update_if_changed(blueprint_model, "name", blueprint.name)
+        recheck = recheck or update_if_changed(blueprint_model, "description", blueprint.description)
+        recheck = recheck or update_if_changed(blueprint_model, "config", blueprint.config)
 
-    if recheck_needed is not None:
-        blueprint_model.recheck_needed = recheck_needed
+        blueprint_model.recheck_needed = recheck
 
     await save_blueprint(blueprint_model)
+    return blueprint_model.recheck_needed
 
 
 async def save_blueprint(blueprint_model: BlueprintModel) -> None:

otterdog/webapp/internal/routes.py

@@ -6,17 +6,23 @@
 #  SPDX-License-Identifier: EPL-2.0
 #  *******************************************************************************
 
-
 from otterdog.webapp.blueprints import create_blueprint_from_model
 from otterdog.webapp.db.service import (
     get_active_installations,
-    get_blueprints,
+    get_blueprints_by_last_checked_time,
+    get_installation_by_github_id,
     logger,
     save_blueprint,
     update_data_for_installation,
     update_installations_from_config,
 )
-from otterdog.webapp.utils import refresh_global_blueprints, refresh_global_policies, refresh_otterdog_config
+from otterdog.webapp.utils import (
+    current_utc_time,
+    has_minimum_timedelta_elapsed,
+    refresh_global_blueprints,
+    refresh_global_policies,
+    refresh_otterdog_config,
+)
 
 from . import blueprint
 
@@ -39,22 +45,44 @@ async def init():
     return {}, 200
 
 
-@blueprint.route("/check")
-async def check():
-    logger.debug("checking blueprints...")
+@blueprint.route("/check", defaults={"limit": 50})
+@blueprint.route("/check/<int:limit>")
+async def check(limit: int):
+    from datetime import timedelta
 
-    for installation in await get_active_installations():
-        org_id = installation.github_id
-        logger.debug(f"checking org '{org_id}'")
+    logger.info("checking blueprints...")
+
+    for blueprint_model in await get_blueprints_by_last_checked_time(limit=limit):
+        org_id = blueprint_model.id.org_id
+
+        if blueprint_model.last_checked is not None and not has_minimum_timedelta_elapsed(
+            blueprint_model.last_checked, timedelta(hours=1)
+        ):
+            logger.debug(
+                "skipping blueprint with id '%s' for org '%s', last checked at '%s'",
+                blueprint_model.id.blueprint_id,
+                org_id,
+                blueprint_model.last_checked.strftime("%d/%m/%Y %H:%M:%S"),
+            )
+            continue
+
+        installation = await get_installation_by_github_id(org_id)
+        if installation is None:
+            logger.error("no installation model found for org '%s'", org_id)
+            continue
+
+        logger.debug("checking blueprint with id '%s' for org '%s'...", blueprint_model.id.blueprint_id, org_id)
+
+        blueprint_instance = create_blueprint_from_model(blueprint_model)
+        await blueprint_instance.evaluate(installation.installation_id, org_id, blueprint_model.recheck_needed)
+
+        blueprint_model.last_checked = current_utc_time()
 
-        for blueprint_model in await get_blueprints(org_id):
-            blueprint_instance = create_blueprint_from_model(blueprint_model)
-            await blueprint_instance.evaluate(installation.installation_id, org_id, blueprint_model.recheck_needed)
+        # if we were forced to do a recheck, reset it afterward
+        if blueprint_model.recheck_needed is True:
+            blueprint_model.recheck_needed = False
 
-            # if we were forced to do a recheck, reset it afterward
-            if blueprint_model.recheck_needed is True:
-                blueprint_model.recheck_needed = False
-                await save_blueprint(blueprint_model)
+        await save_blueprint(blueprint_model)
 
     return {}, 200
 

otterdog/webapp/tasks/__init__.py

@@ -47,7 +47,7 @@
 class Task(ABC, Generic[T]):
     @cached_property
     def logger(self) -> Logger:
-        return getLogger(type(self).__name__)
+        return getLogger(__name__)
 
     def create_task_model(self) -> TaskModel | None:
         return None

otterdog/webapp/tasks/fetch_blueprints.py

@@ -51,7 +51,16 @@ async def _execute(self) -> None:
             await cleanup_blueprints_status_of_owner(self.org_id, list(blueprints))
 
             for blueprint in list(blueprints.values()):
-                await update_or_create_blueprint(self.org_id, blueprint, recheck_needed=True)
+                recheck = await update_or_create_blueprint(self.org_id, blueprint)
+                self.logger.debug(
+                    "updating blueprint with id '%s' for repo '%s/%s', recheck = %s",
+                    blueprint.id,
+                    self.org_id,
+                    self.repo_name,
+                    str(recheck),
+                )
+
+        self.logger.info("done fetching blueprints for repo '%s/%s'", self.org_id, self.repo_name)
 
     async def _fetch_blueprints(self, rest_api: RestApi, repo: str) -> dict[str, Blueprint]:
         config_file_path = BLUEPRINT_PATH

otterdog/webapp/utils.py

@@ -306,6 +306,15 @@ def escape_for_github(text: str) -> str:
     return "\n".join(output)
 
 
+def epoch_utc_time():
+    if sys.version_info < (3, 12):
+        return datetime.utcfromtimestamp(0)
+    else:
+        from datetime import UTC
+
+        return datetime.fromtimestamp(0, UTC)
+
+
 def current_utc_time() -> datetime:
     if sys.version_info < (3, 12):
         return datetime.utcnow()
@@ -340,6 +349,14 @@ async def backoff_if_needed(last_event: datetime, required_timeout: timedelta) -
         await asyncio.sleep(remaining_backoff_seconds)
 
 
+def has_minimum_timedelta_elapsed(last_event: datetime, minimum_timedelta: timedelta) -> bool:
+    last_event = make_aware_utc(last_event)
+    now = make_aware_utc(current_utc_time())
+
+    current_timedelta = now - last_event
+    return current_timedelta >= minimum_timedelta
+
+
 def is_cache_control_enabled() -> bool:
     return bool(current_app.config["CACHE_CONTROL"]) is True