From 3b9397f26d7bb147bb7560549f1f38ec98e49927 Mon Sep 17 00:00:00 2001
From: Vitor Guidi <vitorguidi@google.com>
Date: Wed, 27 Nov 2024 18:04:06 +0000
Subject: [PATCH 01/13] Adding task outcome metric definition

---
 .../_internal/metrics/monitoring_metrics.py           | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/src/clusterfuzz/_internal/metrics/monitoring_metrics.py b/src/clusterfuzz/_internal/metrics/monitoring_metrics.py
index 58489eae45..1900780b54 100644
--- a/src/clusterfuzz/_internal/metrics/monitoring_metrics.py
+++ b/src/clusterfuzz/_internal/metrics/monitoring_metrics.py
@@ -228,6 +228,7 @@
     field_spec=[
         monitor.StringField('task'),
         monitor.StringField('job'),
+        monitor.StringField('outcome'),
     ],
 )
 
@@ -241,6 +242,7 @@
         monitor.StringField('job'),
     ],
 )
+
 TASK_RATE_LIMIT_COUNT = monitor.CounterMetric(
     'task/rate_limit',
     description=('Counter for rate limit events.'),
@@ -250,6 +252,15 @@
         monitor.StringField('argument'),
     ])
 
+TASK_OUTCOME_COUNT = monitor.CounterMetric(
+    'task/outcome',
+    description=('Counter metric for task outcome (success/failure).'),
+    field_spec=[
+        monitor.StringField('task'),
+        monitor.StringField('job'),
+    ]
+)
+
 UTASK_SUBTASK_E2E_DURATION_SECS = monitor.CumulativeDistributionMetric(
     'utask/subtask_e2e_duration_secs',
     description=(

From 7c07642488914c4d65ed4079a05ba61036b59006 Mon Sep 17 00:00:00 2001
From: Vitor Guidi <vitorguidi@google.com>
Date: Wed, 27 Nov 2024 18:08:41 +0000
Subject: [PATCH 02/13] Emiting metric outcome metric

---
 src/clusterfuzz/_internal/bot/tasks/commands.py | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/src/clusterfuzz/_internal/bot/tasks/commands.py b/src/clusterfuzz/_internal/bot/tasks/commands.py
index d33a15a0c6..18d468ef3f 100644
--- a/src/clusterfuzz/_internal/bot/tasks/commands.py
+++ b/src/clusterfuzz/_internal/bot/tasks/commands.py
@@ -208,6 +208,12 @@ def get_command_object(task_name):
   # Force remote execution.
   return task_types.UTask(_COMMAND_MODULE_MAP[task_name])
 
+def _emit_task_outcome_metric(task, job, outcome):
+  monitoring_metrics.TASK_OUTCOME_COUNT.increment(labels={
+    'job': job,
+    'task': task,
+    'outcome': outcome,
+  })
 
 def run_command(task_name, task_argument, job_name, uworker_env):
   """Runs the command."""
@@ -249,14 +255,17 @@ def run_command(task_name, task_argument, job_name, uworker_env):
     # where a test case is reloaded from the datastore, just abort the task.
     logs.warning('Test case %s no longer exists.' % task_argument)
     rate_limiter.record_task(success=False)
+    _emit_task_outcome_metric(task_name, job_name, 'failure')
   except BaseException:
     # On any other exceptions, update state to reflect error and re-raise.
     rate_limiter.record_task(success=False)
+    _emit_task_outcome_metric(task_name, job_name, 'failure')
     if should_update_task_status(task_name):
       data_handler.update_task_status(task_state_name,
                                       data_types.TaskState.ERROR)
     raise
   else:
+    _emit_task_outcome_metric(task_name, job_name, 'success')
     rate_limiter.record_task(success=True)
 
   # Task completed successfully.

From dc8920f15dba61d4fe8b0d8f7a4d22f155a8dd18 Mon Sep 17 00:00:00 2001
From: Vitor Guidi <vitorguidi@google.com>
Date: Wed, 27 Nov 2024 18:11:25 +0000
Subject: [PATCH 03/13] Fix lint

---
 src/clusterfuzz/_internal/bot/tasks/commands.py         | 8 +++++---
 src/clusterfuzz/_internal/metrics/monitoring_metrics.py | 3 +--
 2 files changed, 6 insertions(+), 5 deletions(-)

diff --git a/src/clusterfuzz/_internal/bot/tasks/commands.py b/src/clusterfuzz/_internal/bot/tasks/commands.py
index 18d468ef3f..615189a866 100644
--- a/src/clusterfuzz/_internal/bot/tasks/commands.py
+++ b/src/clusterfuzz/_internal/bot/tasks/commands.py
@@ -208,13 +208,15 @@ def get_command_object(task_name):
   # Force remote execution.
   return task_types.UTask(_COMMAND_MODULE_MAP[task_name])
 
+
 def _emit_task_outcome_metric(task, job, outcome):
   monitoring_metrics.TASK_OUTCOME_COUNT.increment(labels={
-    'job': job,
-    'task': task,
-    'outcome': outcome,
+      'job': job,
+      'task': task,
+      'outcome': outcome,
   })
 
+
 def run_command(task_name, task_argument, job_name, uworker_env):
   """Runs the command."""
   task = get_command_object(task_name)
diff --git a/src/clusterfuzz/_internal/metrics/monitoring_metrics.py b/src/clusterfuzz/_internal/metrics/monitoring_metrics.py
index 1900780b54..b8c80b1dc2 100644
--- a/src/clusterfuzz/_internal/metrics/monitoring_metrics.py
+++ b/src/clusterfuzz/_internal/metrics/monitoring_metrics.py
@@ -258,8 +258,7 @@
     field_spec=[
         monitor.StringField('task'),
         monitor.StringField('job'),
-    ]
-)
+    ])
 
 UTASK_SUBTASK_E2E_DURATION_SECS = monitor.CumulativeDistributionMetric(
     'utask/subtask_e2e_duration_secs',

From 53cdc9e7c20a0e6ebec20b73673da25e59383fc3 Mon Sep 17 00:00:00 2001
From: Vitor Guidi <vitorguidi@google.com>
Date: Wed, 27 Nov 2024 18:38:51 +0000
Subject: [PATCH 04/13] Moving task outcome to correct position

---
 src/clusterfuzz/_internal/metrics/monitoring_metrics.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/clusterfuzz/_internal/metrics/monitoring_metrics.py b/src/clusterfuzz/_internal/metrics/monitoring_metrics.py
index b8c80b1dc2..daeeb505cd 100644
--- a/src/clusterfuzz/_internal/metrics/monitoring_metrics.py
+++ b/src/clusterfuzz/_internal/metrics/monitoring_metrics.py
@@ -228,7 +228,6 @@
     field_spec=[
         monitor.StringField('task'),
         monitor.StringField('job'),
-        monitor.StringField('outcome'),
     ],
 )
 
@@ -258,6 +257,7 @@
     field_spec=[
         monitor.StringField('task'),
         monitor.StringField('job'),
+        monitor.StringField('outcome'),
     ])
 
 UTASK_SUBTASK_E2E_DURATION_SECS = monitor.CumulativeDistributionMetric(

From b9f8aa4471970c028aa64c8fc70e111fdef3a3ba Mon Sep 17 00:00:00 2001
From: Vitor Guidi <vitorguidi@google.com>
Date: Mon, 9 Dec 2024 05:06:35 +0000
Subject: [PATCH 05/13] Move task outcome metric emission to the utasks module

---
 src/clusterfuzz/_internal/bot/tasks/commands.py       | 11 -----------
 .../_internal/bot/tasks/utasks/__init__.py            |  8 ++++++++
 2 files changed, 8 insertions(+), 11 deletions(-)

diff --git a/src/clusterfuzz/_internal/bot/tasks/commands.py b/src/clusterfuzz/_internal/bot/tasks/commands.py
index 615189a866..d33a15a0c6 100644
--- a/src/clusterfuzz/_internal/bot/tasks/commands.py
+++ b/src/clusterfuzz/_internal/bot/tasks/commands.py
@@ -209,14 +209,6 @@ def get_command_object(task_name):
   return task_types.UTask(_COMMAND_MODULE_MAP[task_name])
 
 
-def _emit_task_outcome_metric(task, job, outcome):
-  monitoring_metrics.TASK_OUTCOME_COUNT.increment(labels={
-      'job': job,
-      'task': task,
-      'outcome': outcome,
-  })
-
-
 def run_command(task_name, task_argument, job_name, uworker_env):
   """Runs the command."""
   task = get_command_object(task_name)
@@ -257,17 +249,14 @@ def run_command(task_name, task_argument, job_name, uworker_env):
     # where a test case is reloaded from the datastore, just abort the task.
     logs.warning('Test case %s no longer exists.' % task_argument)
     rate_limiter.record_task(success=False)
-    _emit_task_outcome_metric(task_name, job_name, 'failure')
   except BaseException:
     # On any other exceptions, update state to reflect error and re-raise.
     rate_limiter.record_task(success=False)
-    _emit_task_outcome_metric(task_name, job_name, 'failure')
     if should_update_task_status(task_name):
       data_handler.update_task_status(task_state_name,
                                       data_types.TaskState.ERROR)
     raise
   else:
-    _emit_task_outcome_metric(task_name, job_name, 'success')
     rate_limiter.record_task(success=True)
 
   # Task completed successfully.
diff --git a/src/clusterfuzz/_internal/bot/tasks/utasks/__init__.py b/src/clusterfuzz/_internal/bot/tasks/utasks/__init__.py
index 005e9f50e8..0623f8934d 100644
--- a/src/clusterfuzz/_internal/bot/tasks/utasks/__init__.py
+++ b/src/clusterfuzz/_internal/bot/tasks/utasks/__init__.py
@@ -33,6 +33,14 @@
 Timestamp = timestamp_pb2.Timestamp  # pylint: disable=no-member
 
 
+def _emit_task_outcome_metric(task, job, outcome):
+  monitoring_metrics.TASK_OUTCOME_COUNT.increment(labels={
+      'job': job,
+      'task': task,
+      'outcome': outcome,
+  })
+
+
 class Mode(enum.Enum):
   """The execution mode of `uworker_main` tasks in a bot process."""
 

From 49844d66d372f8d79bdaca02705bc7bf976dbeb9 Mon Sep 17 00:00:00 2001
From: Vitor Guidi <vitorguidi@google.com>
Date: Mon, 9 Dec 2024 05:19:13 +0000
Subject: [PATCH 06/13] Reusing _MetricRecorder to emit the task outcome metric

---
 src/clusterfuzz/_internal/bot/tasks/utasks/__init__.py  | 3 +++
 src/clusterfuzz/_internal/metrics/monitoring_metrics.py | 3 +++
 2 files changed, 6 insertions(+)

diff --git a/src/clusterfuzz/_internal/bot/tasks/utasks/__init__.py b/src/clusterfuzz/_internal/bot/tasks/utasks/__init__.py
index 0623f8934d..4a4b92ca96 100644
--- a/src/clusterfuzz/_internal/bot/tasks/utasks/__init__.py
+++ b/src/clusterfuzz/_internal/bot/tasks/utasks/__init__.py
@@ -146,6 +146,9 @@ def __exit__(self, _exc_type, _exc_value, _traceback):
     monitoring_metrics.UTASK_SUBTASK_E2E_DURATION_SECS.add(
         e2e_duration_secs, self._labels)
 
+    outcome = 'error' if _exc_type else 'success'
+    monitoring_metrics.TASK_OUTCOME_COUNT.increment({**self._labels, 'outcome': outcome})
+
 
 def ensure_uworker_env_type_safety(uworker_env):
   """Converts all values in |uworker_env| to str types.
diff --git a/src/clusterfuzz/_internal/metrics/monitoring_metrics.py b/src/clusterfuzz/_internal/metrics/monitoring_metrics.py
index daeeb505cd..02d6c5c164 100644
--- a/src/clusterfuzz/_internal/metrics/monitoring_metrics.py
+++ b/src/clusterfuzz/_internal/metrics/monitoring_metrics.py
@@ -257,6 +257,9 @@
     field_spec=[
         monitor.StringField('task'),
         monitor.StringField('job'),
+        monitor.StringField('subtask'),
+        monitor.StringField('mode'),
+        monitor.StringField('platform'),
         monitor.StringField('outcome'),
     ])
 

From 0ae1f4378a10c3a7cc424a9a56f4ead2cdeb65dd Mon Sep 17 00:00:00 2001
From: Vitor Guidi <vitorguidi@google.com>
Date: Mon, 9 Dec 2024 05:28:50 +0000
Subject: [PATCH 07/13] Emiting metric in place, fixing lint

---
 .../_internal/bot/tasks/utasks/__init__.py           | 12 +++---------
 1 file changed, 3 insertions(+), 9 deletions(-)

diff --git a/src/clusterfuzz/_internal/bot/tasks/utasks/__init__.py b/src/clusterfuzz/_internal/bot/tasks/utasks/__init__.py
index 4a4b92ca96..4288a70a0a 100644
--- a/src/clusterfuzz/_internal/bot/tasks/utasks/__init__.py
+++ b/src/clusterfuzz/_internal/bot/tasks/utasks/__init__.py
@@ -33,14 +33,6 @@
 Timestamp = timestamp_pb2.Timestamp  # pylint: disable=no-member
 
 
-def _emit_task_outcome_metric(task, job, outcome):
-  monitoring_metrics.TASK_OUTCOME_COUNT.increment(labels={
-      'job': job,
-      'task': task,
-      'outcome': outcome,
-  })
-
-
 class Mode(enum.Enum):
   """The execution mode of `uworker_main` tasks in a bot process."""
 
@@ -147,7 +139,9 @@ def __exit__(self, _exc_type, _exc_value, _traceback):
         e2e_duration_secs, self._labels)
 
     outcome = 'error' if _exc_type else 'success'
-    monitoring_metrics.TASK_OUTCOME_COUNT.increment({**self._labels, 'outcome': outcome})
+    monitoring_metrics.TASK_OUTCOME_COUNT.increment({
+        **self._labels, 'outcome': outcome
+    })
 
 
 def ensure_uworker_env_type_safety(uworker_env):

From c773f9ab90644daf4485421da2eab24ba637ebaf Mon Sep 17 00:00:00 2001
From: Vitor Guidi <vitorguidi@google.com>
Date: Mon, 9 Dec 2024 05:59:28 +0000
Subject: [PATCH 08/13] Accounting for when utask main fails but does not throw

---
 .../_internal/bot/tasks/utasks/__init__.py           | 12 +++++++++++-
 1 file changed, 11 insertions(+), 1 deletion(-)

diff --git a/src/clusterfuzz/_internal/bot/tasks/utasks/__init__.py b/src/clusterfuzz/_internal/bot/tasks/utasks/__init__.py
index 4288a70a0a..ae232ca5bb 100644
--- a/src/clusterfuzz/_internal/bot/tasks/utasks/__init__.py
+++ b/src/clusterfuzz/_internal/bot/tasks/utasks/__init__.py
@@ -25,6 +25,7 @@
 from clusterfuzz._internal.base.tasks import task_utils
 from clusterfuzz._internal.bot.tasks.utasks import uworker_io
 from clusterfuzz._internal.bot.webserver import http_server
+from clusterfuzz._internal.protos import uworker_msg_pb2
 from clusterfuzz._internal.metrics import logs
 from clusterfuzz._internal.metrics import monitoring_metrics
 from clusterfuzz._internal.system import environment
@@ -80,6 +81,7 @@ def __init__(self, subtask: _Subtask):
     self.start_time_ns = time.time_ns()
     self._subtask = subtask
     self._labels = None
+    self.saw_failure = False
 
     if subtask == _Subtask.PREPROCESS:
       self._preprocess_start_time_ns = self.start_time_ns
@@ -138,7 +140,10 @@ def __exit__(self, _exc_type, _exc_value, _traceback):
     monitoring_metrics.UTASK_SUBTASK_E2E_DURATION_SECS.add(
         e2e_duration_secs, self._labels)
 
-    outcome = 'error' if _exc_type else 'success'
+    # The only case where a task might fail without throwing, is in
+    # utask_main, by returning an ErrorType proto which indicates
+    # failure.
+    outcome = 'error' if _exc_type or self.saw_failure else 'success'
     monitoring_metrics.TASK_OUTCOME_COUNT.increment({
         **self._labels, 'outcome': outcome
     })
@@ -231,6 +236,8 @@ def uworker_main_no_io(utask_module, serialized_uworker_input):
       return None
 
     # NOTE: Keep this in sync with `uworker_main()`.
+    if uworker_output.error_type != uworker_msg_pb2.ErrorType.NO_ERROR:
+      recorder.saw_failure = True
     uworker_output.bot_name = environment.get_value('BOT_NAME', '')
     uworker_output.platform_id = environment.get_platform_id()
 
@@ -311,6 +318,9 @@ def uworker_main(input_download_url) -> None:
     logs.info('Starting utask_main: %s.' % utask_module)
     uworker_output = utask_module.utask_main(uworker_input)
 
+    if uworker_output.error_type != uworker_msg_pb2.ErrorType.NO_ERROR:
+      recorder.saw_failure = True
+
     # NOTE: Keep this in sync with `uworker_main_no_io()`.
     uworker_output.bot_name = environment.get_value('BOT_NAME', '')
     uworker_output.platform_id = environment.get_platform_id()

From f4ed2a3cb7a73d151509ab51b7b9b5d86c13feae Mon Sep 17 00:00:00 2001
From: Vitor Guidi <vitorguidi@google.com>
Date: Mon, 9 Dec 2024 06:02:28 +0000
Subject: [PATCH 09/13] Fix lint

---
 src/clusterfuzz/_internal/bot/tasks/utasks/__init__.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/clusterfuzz/_internal/bot/tasks/utasks/__init__.py b/src/clusterfuzz/_internal/bot/tasks/utasks/__init__.py
index ae232ca5bb..abb43b4dbb 100644
--- a/src/clusterfuzz/_internal/bot/tasks/utasks/__init__.py
+++ b/src/clusterfuzz/_internal/bot/tasks/utasks/__init__.py
@@ -25,9 +25,9 @@
 from clusterfuzz._internal.base.tasks import task_utils
 from clusterfuzz._internal.bot.tasks.utasks import uworker_io
 from clusterfuzz._internal.bot.webserver import http_server
-from clusterfuzz._internal.protos import uworker_msg_pb2
 from clusterfuzz._internal.metrics import logs
 from clusterfuzz._internal.metrics import monitoring_metrics
+from clusterfuzz._internal.protos import uworker_msg_pb2
 from clusterfuzz._internal.system import environment
 
 # Define an alias to appease pylint.
@@ -236,7 +236,7 @@ def uworker_main_no_io(utask_module, serialized_uworker_input):
       return None
 
     # NOTE: Keep this in sync with `uworker_main()`.
-    if uworker_output.error_type != uworker_msg_pb2.ErrorType.NO_ERROR:
+    if uworker_output.error_type != uworker_msg_pb2.ErrorType.NO_ERROR:  # pylint: disable=no-member
       recorder.saw_failure = True
     uworker_output.bot_name = environment.get_value('BOT_NAME', '')
     uworker_output.platform_id = environment.get_platform_id()
@@ -318,7 +318,7 @@ def uworker_main(input_download_url) -> None:
     logs.info('Starting utask_main: %s.' % utask_module)
     uworker_output = utask_module.utask_main(uworker_input)
 
-    if uworker_output.error_type != uworker_msg_pb2.ErrorType.NO_ERROR:
+    if uworker_output.error_type != uworker_msg_pb2.ErrorType.NO_ERROR:  # pylint: disable=no-member
       recorder.saw_failure = True
 
     # NOTE: Keep this in sync with `uworker_main_no_io()`.

From 390ec9f16bebc2e3f1072717c896fdfff54c721a Mon Sep 17 00:00:00 2001
From: Vitor Guidi <vitorguidi@google.com>
Date: Mon, 9 Dec 2024 23:35:19 +0000
Subject: [PATCH 10/13] Conform to python style guide

---
 src/clusterfuzz/_internal/bot/tasks/utasks/__init__.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/clusterfuzz/_internal/bot/tasks/utasks/__init__.py b/src/clusterfuzz/_internal/bot/tasks/utasks/__init__.py
index abb43b4dbb..c25a8ac3e9 100644
--- a/src/clusterfuzz/_internal/bot/tasks/utasks/__init__.py
+++ b/src/clusterfuzz/_internal/bot/tasks/utasks/__init__.py
@@ -75,6 +75,7 @@ class _MetricRecorder(contextlib.AbstractContextManager):
   Members:
     start_time_ns (int): The time at which this recorder was constructed, in
       nanoseconds since the Unix epoch.
+    saw_failure: indicates if utask_main returned an error code as a value.
   """
 
   def __init__(self, subtask: _Subtask):

From ce4fb9364a3393685ef631da7a81fd5c7a1e1761 Mon Sep 17 00:00:00 2001
From: Vitor Guidi <vitorguidi@google.com>
Date: Tue, 10 Dec 2024 00:10:56 +0000
Subject: [PATCH 11/13] Add TASK_COUNT_BY_ERROR_TYPE metric, so we can drill
 down by utask main outcome

---
 .../_internal/bot/tasks/utasks/__init__.py    | 26 +++++++++++++++----
 .../_internal/metrics/monitoring_metrics.py   | 12 +++++++++
 2 files changed, 33 insertions(+), 5 deletions(-)

diff --git a/src/clusterfuzz/_internal/bot/tasks/utasks/__init__.py b/src/clusterfuzz/_internal/bot/tasks/utasks/__init__.py
index c25a8ac3e9..cded185f67 100644
--- a/src/clusterfuzz/_internal/bot/tasks/utasks/__init__.py
+++ b/src/clusterfuzz/_internal/bot/tasks/utasks/__init__.py
@@ -75,14 +75,15 @@ class _MetricRecorder(contextlib.AbstractContextManager):
   Members:
     start_time_ns (int): The time at which this recorder was constructed, in
       nanoseconds since the Unix epoch.
-    saw_failure: indicates if utask_main returned an error code as a value.
+    utask_main_failure: this class stores the uworker_output.ErrorType object returned
+      by utask_main, and uses it to emmit a metric.
   """
 
   def __init__(self, subtask: _Subtask):
     self.start_time_ns = time.time_ns()
     self._subtask = subtask
     self._labels = None
-    self.saw_failure = False
+    self.utask_main_failure = None
 
     if subtask == _Subtask.PREPROCESS:
       self._preprocess_start_time_ns = self.start_time_ns
@@ -144,10 +145,25 @@ def __exit__(self, _exc_type, _exc_value, _traceback):
     # The only case where a task might fail without throwing, is in
     # utask_main, by returning an ErrorType proto which indicates
     # failure.
-    outcome = 'error' if _exc_type or self.saw_failure else 'success'
+    outcome = 'error' if _exc_type or self.utask_main_failure else 'success'
     monitoring_metrics.TASK_OUTCOME_COUNT.increment({
         **self._labels, 'outcome': outcome
     })
+    if outcome == "success":
+      error_condition = 'N/A'
+    elif _exc_type:
+      error_condition = 'UNHANDLED_EXCEPTION'
+    else:
+      error_condition = uworker_msg_pb2.ErrorType.Name(self.utask_main_failure)
+    # Get rid of job as a label, so we can have another metric to make 
+    # error conditions more explicit, respecting the 30k distinct
+    # labels limit recommended by gcp.
+    trimmed_labels = self._labels
+    del trimmed_labels['job']
+    trimmed_labels['outcome'] = outcome
+    trimmed_labels['error_condition'] = error_condition
+    monitoring_metrics.TASK_OUTCOME_COUNT_BY_ERROR_TYPE.increment(trimmed_labels)
+
 
 
 def ensure_uworker_env_type_safety(uworker_env):
@@ -238,7 +254,7 @@ def uworker_main_no_io(utask_module, serialized_uworker_input):
 
     # NOTE: Keep this in sync with `uworker_main()`.
     if uworker_output.error_type != uworker_msg_pb2.ErrorType.NO_ERROR:  # pylint: disable=no-member
-      recorder.saw_failure = True
+      recorder.utask_main_failure = uworker_output.error_type
     uworker_output.bot_name = environment.get_value('BOT_NAME', '')
     uworker_output.platform_id = environment.get_platform_id()
 
@@ -320,7 +336,7 @@ def uworker_main(input_download_url) -> None:
     uworker_output = utask_module.utask_main(uworker_input)
 
     if uworker_output.error_type != uworker_msg_pb2.ErrorType.NO_ERROR:  # pylint: disable=no-member
-      recorder.saw_failure = True
+      recorder.utask_main_failure = uworker_output.error_type
 
     # NOTE: Keep this in sync with `uworker_main_no_io()`.
     uworker_output.bot_name = environment.get_value('BOT_NAME', '')
diff --git a/src/clusterfuzz/_internal/metrics/monitoring_metrics.py b/src/clusterfuzz/_internal/metrics/monitoring_metrics.py
index 02d6c5c164..362c76cdee 100644
--- a/src/clusterfuzz/_internal/metrics/monitoring_metrics.py
+++ b/src/clusterfuzz/_internal/metrics/monitoring_metrics.py
@@ -263,6 +263,18 @@
         monitor.StringField('outcome'),
     ])
 
+TASK_OUTCOME_COUNT_BY_ERROR_TYPE = monitor.CounterMetric(
+    'task/outcome_by_error_type',
+    description=('Counter metric for task outcome, with error type.'),
+    field_spec=[
+        monitor.StringField('task'),
+        monitor.StringField('subtask'),
+        monitor.StringField('mode'),
+        monitor.StringField('platform'),
+        monitor.StringField('outcome'),
+        monitor.StringField('error_condition'),
+    ])
+
 UTASK_SUBTASK_E2E_DURATION_SECS = monitor.CumulativeDistributionMetric(
     'utask/subtask_e2e_duration_secs',
     description=(

From 30b67a47877774742156c15c87037c1777c0c623 Mon Sep 17 00:00:00 2001
From: Vitor Guidi <vitorguidi@google.com>
Date: Tue, 10 Dec 2024 00:18:11 +0000
Subject: [PATCH 12/13] Fix lint

---
 src/clusterfuzz/_internal/bot/tasks/utasks/__init__.py | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/src/clusterfuzz/_internal/bot/tasks/utasks/__init__.py b/src/clusterfuzz/_internal/bot/tasks/utasks/__init__.py
index cded185f67..3d395b2866 100644
--- a/src/clusterfuzz/_internal/bot/tasks/utasks/__init__.py
+++ b/src/clusterfuzz/_internal/bot/tasks/utasks/__init__.py
@@ -75,8 +75,8 @@ class _MetricRecorder(contextlib.AbstractContextManager):
   Members:
     start_time_ns (int): The time at which this recorder was constructed, in
       nanoseconds since the Unix epoch.
-    utask_main_failure: this class stores the uworker_output.ErrorType object returned
-      by utask_main, and uses it to emmit a metric.
+    utask_main_failure: this class stores the uworker_output.ErrorType 
+      object returned by utask_main, and uses it to emmit a metric.
   """
 
   def __init__(self, subtask: _Subtask):
@@ -154,7 +154,8 @@ def __exit__(self, _exc_type, _exc_value, _traceback):
     elif _exc_type:
       error_condition = 'UNHANDLED_EXCEPTION'
     else:
-      error_condition = uworker_msg_pb2.ErrorType.Name(self.utask_main_failure)
+      error_condition = uworker_msg_pb2.ErrorType.Name(  # pylint: disable=no-member
+          self.utask_main_failure)
     # Get rid of job as a label, so we can have another metric to make 
     # error conditions more explicit, respecting the 30k distinct
     # labels limit recommended by gcp.

From b83e12f80d5026ab82a4adea111fa3f18b488d1d Mon Sep 17 00:00:00 2001
From: Vitor Guidi <vitorguidi@google.com>
Date: Tue, 10 Dec 2024 00:24:13 +0000
Subject: [PATCH 13/13] Fix lint

---
 src/clusterfuzz/_internal/bot/tasks/utasks/__init__.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/clusterfuzz/_internal/bot/tasks/utasks/__init__.py b/src/clusterfuzz/_internal/bot/tasks/utasks/__init__.py
index 3d395b2866..863e5cd734 100644
--- a/src/clusterfuzz/_internal/bot/tasks/utasks/__init__.py
+++ b/src/clusterfuzz/_internal/bot/tasks/utasks/__init__.py
@@ -156,15 +156,15 @@ def __exit__(self, _exc_type, _exc_value, _traceback):
     else:
       error_condition = uworker_msg_pb2.ErrorType.Name(  # pylint: disable=no-member
           self.utask_main_failure)
-    # Get rid of job as a label, so we can have another metric to make 
+    # Get rid of job as a label, so we can have another metric to make
     # error conditions more explicit, respecting the 30k distinct
     # labels limit recommended by gcp.
     trimmed_labels = self._labels
     del trimmed_labels['job']
     trimmed_labels['outcome'] = outcome
     trimmed_labels['error_condition'] = error_condition
-    monitoring_metrics.TASK_OUTCOME_COUNT_BY_ERROR_TYPE.increment(trimmed_labels)
-
+    monitoring_metrics.TASK_OUTCOME_COUNT_BY_ERROR_TYPE.increment(
+        trimmed_labels)
 
 
 def ensure_uworker_env_type_safety(uworker_env):