From c773f9ab90644daf4485421da2eab24ba637ebaf Mon Sep 17 00:00:00 2001 From: Vitor Guidi Date: Mon, 9 Dec 2024 05:59:28 +0000 Subject: [PATCH] Accounting for when utask main fails but does not throw --- .../_internal/bot/tasks/utasks/__init__.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/src/clusterfuzz/_internal/bot/tasks/utasks/__init__.py b/src/clusterfuzz/_internal/bot/tasks/utasks/__init__.py index 4288a70a0a..ae232ca5bb 100644 --- a/src/clusterfuzz/_internal/bot/tasks/utasks/__init__.py +++ b/src/clusterfuzz/_internal/bot/tasks/utasks/__init__.py @@ -25,6 +25,7 @@ from clusterfuzz._internal.base.tasks import task_utils from clusterfuzz._internal.bot.tasks.utasks import uworker_io from clusterfuzz._internal.bot.webserver import http_server +from clusterfuzz._internal.protos import uworker_msg_pb2 from clusterfuzz._internal.metrics import logs from clusterfuzz._internal.metrics import monitoring_metrics from clusterfuzz._internal.system import environment @@ -80,6 +81,7 @@ def __init__(self, subtask: _Subtask): self.start_time_ns = time.time_ns() self._subtask = subtask self._labels = None + self.saw_failure = False if subtask == _Subtask.PREPROCESS: self._preprocess_start_time_ns = self.start_time_ns @@ -138,7 +140,10 @@ def __exit__(self, _exc_type, _exc_value, _traceback): monitoring_metrics.UTASK_SUBTASK_E2E_DURATION_SECS.add( e2e_duration_secs, self._labels) - outcome = 'error' if _exc_type else 'success' + # The only case where a task might fail without throwing, is in + # utask_main, by returning an ErrorType proto which indicates + # failure. + outcome = 'error' if _exc_type or self.saw_failure else 'success' monitoring_metrics.TASK_OUTCOME_COUNT.increment({ **self._labels, 'outcome': outcome }) @@ -231,6 +236,8 @@ def uworker_main_no_io(utask_module, serialized_uworker_input): return None # NOTE: Keep this in sync with `uworker_main()`. + if uworker_output.error_type != uworker_msg_pb2.ErrorType.NO_ERROR: + recorder.saw_failure = True uworker_output.bot_name = environment.get_value('BOT_NAME', '') uworker_output.platform_id = environment.get_platform_id() @@ -311,6 +318,9 @@ def uworker_main(input_download_url) -> None: logs.info('Starting utask_main: %s.' % utask_module) uworker_output = utask_module.utask_main(uworker_input) + if uworker_output.error_type != uworker_msg_pb2.ErrorType.NO_ERROR: + recorder.saw_failure = True + # NOTE: Keep this in sync with `uworker_main_no_io()`. uworker_output.bot_name = environment.get_value('BOT_NAME', '') uworker_output.platform_id = environment.get_platform_id()