diff --git a/CHANGELOG.md b/CHANGELOG.md index f4e638a8..3321bb94 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,8 @@ ### Changed +- Improved the check mechanism for blueprints by only checking a certain number each run and by taking the last check time into account. +- Improved the update mechanism when installing a new GitHub organization to only update the newly added organization. ([#349](https://github.com/eclipse-csi/otterdog/issues/349)) - Integrated existing logging with standard python logging facility. - Utilized `rich` console formatting instead of low-level colorama styles. - Improved processing when archiving repositories to process all other requested changes before archiving them. ([#134](https://github.com/eclipse-csi/otterdog/issues/134)) diff --git a/otterdog/app.py b/otterdog/app.py index 2cc10e2c..ae83db8a 100644 --- a/otterdog/app.py +++ b/otterdog/app.py @@ -5,6 +5,7 @@ # which is available at http://www.eclipse.org/legal/epl-v20.html # SPDX-License-Identifier: EPL-2.0 # ******************************************************************************* + import logging import os from sys import exit diff --git a/otterdog/providers/github/rest/app_client.py b/otterdog/providers/github/rest/app_client.py index f6c559a5..07c65ded 100644 --- a/otterdog/providers/github/rest/app_client.py +++ b/otterdog/providers/github/rest/app_client.py @@ -35,7 +35,15 @@ async def get_app_installations(self) -> list[dict[str, Any]]: try: return await self.requester.request_paged_json("GET", "/app/installations") except GitHubException as ex: - raise RuntimeError(f"failed retrieving authenticated app:\n{ex}") from ex + raise RuntimeError(f"failed retrieving app installations:\n{ex}") from ex + + async def get_app_installation(self, installation_id: int) -> list[dict[str, Any]]: + _logger.debug("retrieving app installation for id '%d'", installation_id) + + try: + return await self.requester.request_paged_json("GET", f"/app/installations/{installation_id}") + except GitHubException as ex: + raise RuntimeError(f"failed retrieving app installation:\n{ex}") from ex async def create_installation_access_token(self, installation_id: str) -> tuple[str, datetime]: _logger.debug("creating an installation access token for installation '%s'", installation_id) diff --git a/otterdog/webapp/blueprints/__init__.py b/otterdog/webapp/blueprints/__init__.py index 0cc8d28a..19525484 100644 --- a/otterdog/webapp/blueprints/__init__.py +++ b/otterdog/webapp/blueprints/__init__.py @@ -42,7 +42,7 @@ class Blueprint(ABC, BaseModel): @cached_property def logger(self) -> Logger: - return getLogger(type(self).__name__) + return getLogger(__name__) @property @abstractmethod diff --git a/otterdog/webapp/db/models.py b/otterdog/webapp/db/models.py index 3080a5a9..cac44f57 100644 --- a/otterdog/webapp/db/models.py +++ b/otterdog/webapp/db/models.py @@ -170,6 +170,7 @@ class BlueprintModel(Model): name: Optional[str] = None description: Optional[str] = None recheck_needed: bool = True + last_checked: Optional[datetime] = Field(index=True, default=None) config: dict diff --git a/otterdog/webapp/db/service.py b/otterdog/webapp/db/service.py index 5ec71e9f..c141d003 100644 --- a/otterdog/webapp/db/service.py +++ b/otterdog/webapp/db/service.py @@ -68,7 +68,7 @@ async def update_installation_status(installation_id: int, action: str) -> None: case "created": policies = await refresh_global_policies() blueprints = await refresh_global_blueprints() - await update_app_installations(policies, blueprints) + await add_app_installation(installation_id, policies, blueprints) case "deleted": installation = await mongo.odm.find_one( @@ -214,6 +214,35 @@ async def update_app_installations( await update_policies_and_blueprints_for_installation(installation, global_policies, global_blueprints) +async def add_app_installation( + installation_id: int, + global_policies: list[Policy], + global_blueprints: list[Blueprint], +): + logger.info("adding app installation for id '%d'", installation_id) + + rest_api = get_rest_api_for_app() + app_installation = await rest_api.app.get_app_installation(installation_id) + + async with mongo.odm.session() as session: + installation_id = app_installation["id"] + github_id = app_installation["account"]["login"] + suspended_at = app_installation["suspended_at"] + installation_status = InstallationStatus.INSTALLED if suspended_at is None else InstallationStatus.SUSPENDED + + installation_model = await get_installation_by_github_id(github_id) + if installation_model is not None: + installation_model.installation_id = int(installation_id) + installation_model.installation_status = installation_status + + await session.save(installation_model) + else: + return + + await update_data_for_installation(installation_model) + await update_policies_and_blueprints_for_installation(installation_model, global_policies, global_blueprints) + + async def update_data_for_installation(installation: InstallationModel) -> None: from otterdog.webapp.tasks.fetch_all_pull_requests import FetchAllPullRequestsTask from otterdog.webapp.tasks.fetch_config import FetchConfigTask @@ -752,6 +781,14 @@ async def get_blueprints(owner: str) -> list[BlueprintModel]: ) +async def get_blueprints_by_last_checked_time(limit: int) -> list[BlueprintModel]: + return await mongo.odm.find( + BlueprintModel, + limit=limit, + sort=query.asc(BlueprintModel.last_checked), + ) + + async def find_blueprint(owner: str, blueprint_id: str) -> BlueprintModel | None: return await mongo.odm.find_one( BlueprintModel, @@ -760,7 +797,7 @@ async def find_blueprint(owner: str, blueprint_id: str) -> BlueprintModel | None ) -async def update_or_create_blueprint(owner: str, blueprint: Blueprint, recheck_needed: bool | None = None) -> None: +async def update_or_create_blueprint(owner: str, blueprint: Blueprint) -> bool: blueprint_model = await find_blueprint(owner, blueprint.id) if blueprint_model is None: blueprint_model = BlueprintModel( @@ -771,15 +808,24 @@ async def update_or_create_blueprint(owner: str, blueprint: Blueprint, recheck_n config=blueprint.config, ) else: - blueprint_model.path = blueprint.path - blueprint_model.name = blueprint.name - blueprint_model.description = blueprint.description - blueprint_model.config = blueprint.config + recheck = False + + def update_if_changed(obj: BlueprintModel, attr: str, value: Any) -> bool: + if obj.__getattribute__(attr) != value: + obj.__setattr__(attr, value) + return True + else: + return False + + recheck = recheck or update_if_changed(blueprint_model, "path", blueprint.path) + recheck = recheck or update_if_changed(blueprint_model, "name", blueprint.name) + recheck = recheck or update_if_changed(blueprint_model, "description", blueprint.description) + recheck = recheck or update_if_changed(blueprint_model, "config", blueprint.config) - if recheck_needed is not None: - blueprint_model.recheck_needed = recheck_needed + blueprint_model.recheck_needed = recheck await save_blueprint(blueprint_model) + return blueprint_model.recheck_needed async def save_blueprint(blueprint_model: BlueprintModel) -> None: diff --git a/otterdog/webapp/internal/routes.py b/otterdog/webapp/internal/routes.py index 4cb6e6a6..bf49b4c8 100644 --- a/otterdog/webapp/internal/routes.py +++ b/otterdog/webapp/internal/routes.py @@ -6,17 +6,23 @@ # SPDX-License-Identifier: EPL-2.0 # ******************************************************************************* - from otterdog.webapp.blueprints import create_blueprint_from_model from otterdog.webapp.db.service import ( get_active_installations, - get_blueprints, + get_blueprints_by_last_checked_time, + get_installation_by_github_id, logger, save_blueprint, update_data_for_installation, update_installations_from_config, ) -from otterdog.webapp.utils import refresh_global_blueprints, refresh_global_policies, refresh_otterdog_config +from otterdog.webapp.utils import ( + current_utc_time, + has_minimum_timedelta_elapsed, + refresh_global_blueprints, + refresh_global_policies, + refresh_otterdog_config, +) from . import blueprint @@ -39,22 +45,44 @@ async def init(): return {}, 200 -@blueprint.route("/check") -async def check(): - logger.debug("checking blueprints...") +@blueprint.route("/check", defaults={"limit": 50}) +@blueprint.route("/check/") +async def check(limit: int): + from datetime import timedelta - for installation in await get_active_installations(): - org_id = installation.github_id - logger.debug(f"checking org '{org_id}'") + logger.info("checking blueprints...") + + for blueprint_model in await get_blueprints_by_last_checked_time(limit=limit): + org_id = blueprint_model.id.org_id + + if blueprint_model.last_checked is not None and not has_minimum_timedelta_elapsed( + blueprint_model.last_checked, timedelta(hours=1) + ): + logger.debug( + "skipping blueprint with id '%s' for org '%s', last checked at '%s'", + blueprint_model.id.blueprint_id, + org_id, + blueprint_model.last_checked.strftime("%d/%m/%Y %H:%M:%S"), + ) + continue + + installation = await get_installation_by_github_id(org_id) + if installation is None: + logger.error("no installation model found for org '%s'", org_id) + continue + + logger.debug("checking blueprint with id '%s' for org '%s'...", blueprint_model.id.blueprint_id, org_id) + + blueprint_instance = create_blueprint_from_model(blueprint_model) + await blueprint_instance.evaluate(installation.installation_id, org_id, blueprint_model.recheck_needed) + + blueprint_model.last_checked = current_utc_time() - for blueprint_model in await get_blueprints(org_id): - blueprint_instance = create_blueprint_from_model(blueprint_model) - await blueprint_instance.evaluate(installation.installation_id, org_id, blueprint_model.recheck_needed) + # if we were forced to do a recheck, reset it afterward + if blueprint_model.recheck_needed is True: + blueprint_model.recheck_needed = False - # if we were forced to do a recheck, reset it afterward - if blueprint_model.recheck_needed is True: - blueprint_model.recheck_needed = False - await save_blueprint(blueprint_model) + await save_blueprint(blueprint_model) return {}, 200 diff --git a/otterdog/webapp/tasks/__init__.py b/otterdog/webapp/tasks/__init__.py index 5e648237..635faa66 100644 --- a/otterdog/webapp/tasks/__init__.py +++ b/otterdog/webapp/tasks/__init__.py @@ -47,7 +47,7 @@ class Task(ABC, Generic[T]): @cached_property def logger(self) -> Logger: - return getLogger(type(self).__name__) + return getLogger(__name__) def create_task_model(self) -> TaskModel | None: return None diff --git a/otterdog/webapp/tasks/fetch_blueprints.py b/otterdog/webapp/tasks/fetch_blueprints.py index 55e72c0e..8b404f73 100644 --- a/otterdog/webapp/tasks/fetch_blueprints.py +++ b/otterdog/webapp/tasks/fetch_blueprints.py @@ -51,7 +51,16 @@ async def _execute(self) -> None: await cleanup_blueprints_status_of_owner(self.org_id, list(blueprints)) for blueprint in list(blueprints.values()): - await update_or_create_blueprint(self.org_id, blueprint, recheck_needed=True) + recheck = await update_or_create_blueprint(self.org_id, blueprint) + self.logger.debug( + "updating blueprint with id '%s' for repo '%s/%s', recheck = %s", + blueprint.id, + self.org_id, + self.repo_name, + str(recheck), + ) + + self.logger.info("done fetching blueprints for repo '%s/%s'", self.org_id, self.repo_name) async def _fetch_blueprints(self, rest_api: RestApi, repo: str) -> dict[str, Blueprint]: config_file_path = BLUEPRINT_PATH diff --git a/otterdog/webapp/utils.py b/otterdog/webapp/utils.py index c9956217..d02f9862 100644 --- a/otterdog/webapp/utils.py +++ b/otterdog/webapp/utils.py @@ -306,6 +306,15 @@ def escape_for_github(text: str) -> str: return "\n".join(output) +def epoch_utc_time(): + if sys.version_info < (3, 12): + return datetime.utcfromtimestamp(0) + else: + from datetime import UTC + + return datetime.fromtimestamp(0, UTC) + + def current_utc_time() -> datetime: if sys.version_info < (3, 12): return datetime.utcnow() @@ -340,6 +349,14 @@ async def backoff_if_needed(last_event: datetime, required_timeout: timedelta) - await asyncio.sleep(remaining_backoff_seconds) +def has_minimum_timedelta_elapsed(last_event: datetime, minimum_timedelta: timedelta) -> bool: + last_event = make_aware_utc(last_event) + now = make_aware_utc(current_utc_time()) + + current_timedelta = now - last_event + return current_timedelta >= minimum_timedelta + + def is_cache_control_enabled() -> bool: return bool(current_app.config["CACHE_CONTROL"]) is True