From 110a4f984135b4774b8d2fbff4440b2cfcaf9895 Mon Sep 17 00:00:00 2001 From: Laura Barcziova Date: Thu, 21 Nov 2024 14:13:35 +0100 Subject: [PATCH] Rework handler for Koji scratch build reporting So that it can be used for both upstream and downstream reporting. --- packit_service/worker/checker/koji.py | 7 ++ packit_service/worker/handlers/abstract.py | 1 + packit_service/worker/handlers/koji.py | 135 +++++++++++++++------ packit_service/worker/tasks.py | 11 ++ tests/conftest.py | 58 +++++++++ tests/integration/test_listen_to_fedmsg.py | 59 +++++++++ 6 files changed, 232 insertions(+), 39 deletions(-) diff --git a/packit_service/worker/checker/koji.py b/packit_service/worker/checker/koji.py index e10a0a570..0a106bb3c 100644 --- a/packit_service/worker/checker/koji.py +++ b/packit_service/worker/checker/koji.py @@ -3,6 +3,8 @@ import logging +from ogr.services.pagure import PagureProject + from packit_service.constants import ( KOJI_PRODUCTION_BUILDS_ISSUE, PERMISSIONS_ERROR_WRITE_OR_ADMIN, @@ -25,6 +27,11 @@ def pre_check(self) -> bool: return self.koji_build_helper.is_job_config_trigger_matching(self.job_config) +class IsUpstreamKojiScratchBuild(Checker, GetKojiBuildJobHelperMixin): + def pre_check(self) -> bool: + return not isinstance(self.koji_build_helper.project, PagureProject) + + class PermissionOnKoji(Checker, GetKojiBuildJobHelperMixin): def pre_check(self) -> bool: if ( diff --git a/packit_service/worker/handlers/abstract.py b/packit_service/worker/handlers/abstract.py index 73b90e583..d47ddf8ed 100644 --- a/packit_service/worker/handlers/abstract.py +++ b/packit_service/worker/handlers/abstract.py @@ -218,6 +218,7 @@ class TaskName(str, enum.Enum): openscanhub_task_finished = "task.openscanhub_task_finished" openscanhub_task_started = "task.openscanhub_task_started" downstream_koji_scratch_build = "task.run_downstream_koji_scratch_build_handler" + downstream_koji_scratch_build_report = "task.run_downstream_koji_scratch_build_report_handler" class Handler(PackitAPIProtocol, Config): diff --git a/packit_service/worker/handlers/koji.py b/packit_service/worker/handlers/koji.py index fd95fb24f..84cd6d878 100644 --- a/packit_service/worker/handlers/koji.py +++ b/packit_service/worker/handlers/koji.py @@ -6,6 +6,7 @@ """ import logging +from abc import ABC, abstractmethod from datetime import datetime from os import getenv from typing import Optional @@ -40,6 +41,7 @@ from packit_service.worker.checker.abstract import Checker from packit_service.worker.checker.koji import ( IsJobConfigTriggerMatching, + IsUpstreamKojiScratchBuild, PermissionOnKoji, SidetagExists, ) @@ -62,6 +64,7 @@ TaskName, configured_as, reacts_to, + reacts_to_as_fedora_ci, run_for_check_rerun, run_for_comment, ) @@ -69,6 +72,7 @@ from packit_service.worker.handlers.distgit import DownstreamKojiBuildHandler from packit_service.worker.handlers.mixin import GetKojiBuildJobHelperMixin from packit_service.worker.helpers.build.koji_build import KojiBuildJobHelper +from packit_service.worker.helpers.fedora_ci import FedoraCIHelper from packit_service.worker.helpers.sidetag import SidetagHelper from packit_service.worker.mixin import ( ConfigFromEventMixin, @@ -130,16 +134,9 @@ def run(self) -> TaskResults: return self.koji_build_helper.run_koji_build() -@configured_as(job_type=JobType.production_build) -@configured_as(job_type=JobType.upstream_koji_build) -@reacts_to(event=KojiTaskEvent) -class KojiTaskReportHandler( - JobHandler, - PackitAPIWithDownstreamMixin, - ConfigFromEventMixin, +class AbstractKojiTaskReportHandler( + ABC, JobHandler, PackitAPIWithDownstreamMixin, ConfigFromEventMixin ): - task_name = TaskName.upstream_koji_build_report - def __init__( self, package_config: PackageConfig, @@ -156,6 +153,14 @@ def __init__( self._db_project_event: Optional[ProjectEventModel] = None self._build: Optional[KojiBuildTargetModel] = None + @abstractmethod + def report(self, description: str, commit_status: BaseCommitStatus, url: str): ... + + @abstractmethod + def notify_about_failure_if_configured( + self, packit_dashboard_url: str, external_dashboard_url: str, logs_url: str + ): ... + @property def build(self) -> Optional[KojiBuildTargetModel]: if not self._build: @@ -171,21 +176,17 @@ def db_project_event(self) -> Optional[ProjectEventModel]: return self._db_project_event def run(self): - build = KojiBuildTargetModel.get_by_task_id( - task_id=str(self.koji_task_event.task_id), - ) - - if not build: + if not self.build: msg = f"Koji task {self.koji_task_event.task_id} not found in the database." logger.warning(msg) return TaskResults(success=False, details={"msg": msg}) logger.debug( - f"Build on {build.target} in koji changed state " + f"Build on {self.build.target} in Koji changed state " f"from {self.koji_task_event.old_state} to {self.koji_task_event.state}.", ) - build.set_build_start_time( + self.build.set_build_start_time( ( datetime.utcfromtimestamp(self.koji_task_event.start_time) if self.koji_task_event.start_time @@ -193,7 +194,7 @@ def run(self): ), ) - build.set_build_finished_time( + self.build.set_build_finished_time( ( datetime.utcfromtimestamp(self.koji_task_event.completion_time) if self.koji_task_event.completion_time @@ -201,15 +202,7 @@ def run(self): ), ) - url = get_koji_build_info_url(build.id) - build_job_helper = KojiBuildJobHelper( - service_config=self.service_config, - package_config=self.package_config, - project=self.project, - metadata=self.data, - db_project_event=self.db_project_event, - job_config=self.job_config, - ) + url = get_koji_build_info_url(self.build.id) new_commit_status = { KojiTaskState.free: BaseCommitStatus.pending, @@ -233,7 +226,7 @@ def run(self): logger.debug( f"We don't react to this koji build state change: {self.koji_task_event.state}", ) - elif new_commit_status.value == build.status: + elif new_commit_status.value == self.build.status: logger.debug( "Status was already processed (status in the DB is the " "same as the one about to report)", @@ -244,38 +237,102 @@ def run(self): ) else: - build.set_status(new_commit_status.value) - build_job_helper.report_status_to_all_for_chroot( - description=description, - state=new_commit_status, - url=url, - chroot=build.target, - ) + self.build.set_status(new_commit_status.value) + self.report(description, new_commit_status, url) koji_build_logs = self.koji_task_event.get_koji_build_rpm_tasks_logs_urls( self.service_config.koji_logs_url, ) - build.set_build_logs_urls(koji_build_logs) + self.build.set_build_logs_urls(koji_build_logs) koji_rpm_task_web_url = KojiTaskEvent.get_koji_rpm_build_web_url( - rpm_build_task_id=int(build.task_id), + rpm_build_task_id=int(self.build.task_id), koji_web_url=self.service_config.koji_web_url, ) - build.set_web_url(koji_rpm_task_web_url) + self.build.set_web_url(koji_rpm_task_web_url) if self.koji_task_event.state == KojiTaskState.failed: - build_job_helper.notify_about_failure_if_configured( + self.notify_about_failure_if_configured( packit_dashboard_url=url, external_dashboard_url=koji_rpm_task_web_url, logs_url=koji_build_logs, ) msg = ( - f"Build on {build.target} in koji changed state " + f"Build on {self.build.target} in koji changed state " f"from {self.koji_task_event.old_state} to {self.koji_task_event.state}." ) return TaskResults(success=True, details={"msg": msg}) +@configured_as(job_type=JobType.production_build) +@configured_as(job_type=JobType.upstream_koji_build) +@reacts_to(event=KojiTaskEvent) +class KojiTaskReportHandler(AbstractKojiTaskReportHandler): + task_name = TaskName.upstream_koji_build_report + _helper: Optional[KojiBuildJobHelper] = None + + @property + def helper(self): + if not self._helper: + self._helper = KojiBuildJobHelper( + service_config=self.service_config, + package_config=self.package_config, + project=self.project, + metadata=self.data, + db_project_event=self.db_project_event, + job_config=self.job_config, + ) + return self._helper + + @staticmethod + def get_checkers() -> tuple[type[Checker], ...]: + return (IsUpstreamKojiScratchBuild,) + + def report(self, description: str, commit_status: BaseCommitStatus, url: str): + self.helper.report_status_to_all_for_chroot( + description=description, + state=commit_status, + url=url, + chroot=self.build.target, + ) + + def notify_about_failure_if_configured( + self, packit_dashboard_url: str, external_dashboard_url: str, logs_url: str + ): + self.helper.notify_about_failure_if_configured( + packit_dashboard_url=packit_dashboard_url, + external_dashboard_url=external_dashboard_url, + logs_url=logs_url, + ) + + +@reacts_to_as_fedora_ci(event=KojiTaskEvent) +class KojiTaskReportDownstreamHandler(AbstractKojiTaskReportHandler): + task_name = TaskName.downstream_koji_build_report + _helper: Optional[FedoraCIHelper] = None + + @property + def helper(self): + if not self._helper: + self._helper = FedoraCIHelper( + project=self.project, + metadata=self.data, + ) + return self._helper + + def report(self, description: str, commit_status: BaseCommitStatus, url: str): + self.helper.report( + state=commit_status, + description=description, + url=url, + ) + + def notify_about_failure_if_configured( + self, packit_dashboard_url: str, external_dashboard_url: str, logs_url: str + ): + pass + + @configured_as(job_type=JobType.koji_build) @configured_as(job_type=JobType.bodhi_update) @reacts_to(event=KojiBuildEvent) diff --git a/packit_service/worker/tasks.py b/packit_service/worker/tasks.py index cdfad5c6e..8411d0450 100644 --- a/packit_service/worker/tasks.py +++ b/packit_service/worker/tasks.py @@ -80,6 +80,7 @@ from packit_service.worker.handlers.koji import ( KojiBuildReportHandler, KojiBuildTagHandler, + KojiTaskReportDownstreamHandler, ) from packit_service.worker.handlers.usage import check_onboarded_projects from packit_service.worker.helpers.build.babysit import ( @@ -396,6 +397,16 @@ def run_koji_build_report_handler(event: dict, package_config: dict, job_config: return get_handlers_task_results(handler.run_job(), event) +@celery_app.task(name=TaskName.downstream_koji_scratch_build_report, base=TaskWithRetry) +def run_downstream_koji_scratch_build_report_handler( + event: dict, package_config: dict, job_config: dict +): + handler = KojiTaskReportDownstreamHandler( + package_config=load_package_config(package_config), + job_config=load_job_config(job_config), + event=event, + ) + return get_handlers_task_results(handler.run_job(), event) @celery_app.task(bind=True, name=TaskName.downstream_koji_scratch_build, base=TaskWithRetry) diff --git a/tests/conftest.py b/tests/conftest.py index c6a041586..878afb60a 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -332,6 +332,64 @@ def koji_build_pr(): return koji_build_model +@pytest.fixture() +def koji_build_pr_downstream(): + project_model = flexmock( + repo_name="packit", + namespace="rpms", + project_url="https://src.fedoraproject.org/rpms/packit", + ) + pr_model = flexmock( + id=1, + pr_id=123, + project=project_model, + job_config_trigger_type=JobConfigTriggerType.pull_request, + project_event_model_type=ProjectEventModelType.pull_request, + commit_sha="0011223344", + ) + project_event_model = flexmock( + id=2, + type=ProjectEventModelType.pull_request, + event_id=1, + get_project_event_object=lambda: pr_model, + ) + runs = [] + srpm_build = flexmock(logs="asdsdf", url=None, runs=runs) + koji_build_model = flexmock( + id=1, + task_id="1", + commit_sha="0011223344", + project_name="some-project", + owner="some-owner", + web_url="https://some-url", + target="some-target", + status="some-status", + runs=runs, + ) + koji_build_model._srpm_build_for_mocking = srpm_build + koji_build_model.get_project_event_object = lambda: pr_model + koji_build_model.get_srpm_build = lambda: srpm_build + koji_build_model.should_receive("get_project_event_model").and_return( + project_event_model, + ) + + flexmock(ProjectEventModel).should_receive("get_or_create").with_args( + type=pr_model.project_event_model_type, + event_id=pr_model.id, + commit_sha="0011223344", + ).and_return(project_event_model) + + run_model = flexmock( + id=3, + job_project_event=project_event_model, + srpm_build=srpm_build, + copr_build=koji_build_model, + ) + runs.append(run_model) + + return koji_build_model + + @pytest.fixture() def add_pull_request_event_with_sha_123456(): db_project_object = flexmock( diff --git a/tests/integration/test_listen_to_fedmsg.py b/tests/integration/test_listen_to_fedmsg.py index a773cfc2b..09fe42d2e 100644 --- a/tests/integration/test_listen_to_fedmsg.py +++ b/tests/integration/test_listen_to_fedmsg.py @@ -59,6 +59,7 @@ from packit_service.worker.tasks import ( run_copr_build_end_handler, run_copr_build_start_handler, + run_downstream_koji_scratch_build_report_handler, run_koji_build_report_handler, run_koji_build_tag_handler, run_testing_farm_handler, @@ -2567,6 +2568,64 @@ def test_koji_build_end(koji_build_scratch_end, pc_koji_build_pr, koji_build_pr) assert first_dict_value(results["job"])["success"] +def test_koji_build_end_downstream( + koji_build_scratch_end, pc_koji_build_pr, koji_build_pr_downstream +): + service_config = ( + flexmock( + enabled_projects_for_fedora_ci="https://src.fedoraproject.org/rpms/packit", + koji_logs_url="", + koji_web_url="", + ) + .should_receive("get_project") + .and_return(flexmock(namespace="rpms", repo="packit")) + .mock() + ) + flexmock(ServiceConfig).should_receive("get_service_config").and_return(service_config) + koji_build_pr_downstream.target = "rawhide" + flexmock(GithubProject).should_receive("is_private").and_return(False) + flexmock(KojiTaskEvent).should_receive("get_packages_config").and_return( + pc_koji_build_pr, + ) + + flexmock(KojiBuildTargetModel).should_receive("get_by_task_id").and_return( + koji_build_pr_downstream, + ) + url = get_koji_build_info_url(1) + flexmock(requests).should_receive("get").and_return(requests.Response()) + flexmock(requests.Response).should_receive("raise_for_status").and_return(None) + + koji_build_pr_downstream.should_receive("set_build_start_time").once() + koji_build_pr_downstream.should_receive("set_build_finished_time").once() + koji_build_pr_downstream.should_receive("set_status").with_args("success").once() + koji_build_pr_downstream.should_receive("set_build_logs_urls") + koji_build_pr_downstream.should_receive("set_web_url") + + # check if packit-service set correct PR status + flexmock(StatusReporter).should_receive("set_status").with_args( + state=BaseCommitStatus.success, + description="RPM build succeeded.", + url=url, + check_name="Packit - scratch build", + ).once() + flexmock(Signature).should_receive("apply_async").once() + flexmock(Pushgateway).should_receive("push").times(2).and_return() + + processing_results = SteveJobs().process_message(koji_build_scratch_end) + event_dict, job, job_config, package_config = get_parameters_from_results( + processing_results, + ) + assert json.dumps(event_dict) + + results = run_downstream_koji_scratch_build_report_handler( + package_config=package_config, + event=event_dict, + job_config=job_config, + ) + + assert first_dict_value(results["job"])["success"] + + def test_koji_build_tag( koji_build_tagged, pc_koji_build_tag_specfile,