From d535b452e73cbb78f4b31f5b47eb194e15567b00 Mon Sep 17 00:00:00 2001 From: iback Date: Fri, 9 Aug 2024 14:03:45 +0000 Subject: [PATCH 1/6] added test --- unit_tests/test_panoptic_evaluator.py | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/unit_tests/test_panoptic_evaluator.py b/unit_tests/test_panoptic_evaluator.py index a3009ae..5ab23a8 100644 --- a/unit_tests/test_panoptic_evaluator.py +++ b/unit_tests/test_panoptic_evaluator.py @@ -40,6 +40,28 @@ def test_simple_evaluation(self): self.assertEqual(result.fp, 0) self.assertEqual(result.sq, 0.75) self.assertEqual(result.pq, 0.75) + self.assertAlmostEqual(result.global_bin_dsc, 0.8571428571428571) + + def test_simple_evaluation_instance_multiclass(self): + a = np.zeros([50, 50], dtype=np.uint16) + b = a.copy().astype(a.dtype) + a[20:30, 10:20] = 1 + a[30:40, 10:20] = 3 + b[20:35, 10:20] = 2 + + evaluator = Panoptica_Evaluator( + expected_input=InputType.UNMATCHED_INSTANCE, + instance_matcher=NaiveThresholdMatching(), + ) + + result, debug_data = evaluator.evaluate(b, a)["ungrouped"] + print(result) + self.assertAlmostEqual(result.global_bin_dsc, 0.8571428571428571) + self.assertEqual(result.tp, 1) + self.assertEqual(result.fp, 0) + self.assertEqual(result.fn, 1) + self.assertAlmostEqual(result.sq, 0.6666666666666666) + self.assertAlmostEqual(result.pq, 0.4444444444444444) def test_simple_evaluation_DSC(self): a = np.zeros([50, 50], dtype=np.uint16) From 6ee331ef337d8cd3c0ac2c4655cb2a7ccd3727e3 Mon Sep 17 00:00:00 2001 From: iback Date: Fri, 9 Aug 2024 14:04:57 +0000 Subject: [PATCH 2/6] fixed global metrics calculation --- panoptica/panoptica_result.py | 71 +++++++++++++++++++++-------------- 1 file changed, 42 insertions(+), 29 deletions(-) diff --git a/panoptica/panoptica_result.py b/panoptica/panoptica_result.py index ef631f9..5e5419a 100644 --- a/panoptica/panoptica_result.py +++ b/panoptica/panoptica_result.py @@ -11,10 +11,6 @@ MetricCouldNotBeComputedException, MetricMode, MetricType, - _compute_centerline_dice_coefficient, - _compute_dice_coefficient, - _average_symmetric_surface_distance, - _compute_relative_volume_difference, ) from panoptica.utils import EdgeCaseHandler @@ -44,8 +40,6 @@ def __init__( """ self._edge_case_handler = edge_case_handler empty_list_std = self._edge_case_handler.handle_empty_list_std().value - self._prediction_arr = prediction_arr - self._reference_arr = reference_arr self._global_metrics: list[Metric] = global_metrics ###################### # Evaluation Metrics # @@ -249,17 +243,39 @@ def __init__( num_pred_instances=self.num_pred_instances, num_ref_instances=self.num_ref_instances, ) - self._list_metrics[m] = Evaluation_List_Metric( - m, empty_list_std, list_metrics[m], is_edge_case, edge_case_result - ) + self._list_metrics[m] = Evaluation_List_Metric(m, empty_list_std, list_metrics[m], is_edge_case, edge_case_result) # even if not available, set the global vars + default_value = None + was_calculated = False + if m in self._global_metrics: + default_value = self._calc_global_bin_metric(m, prediction_arr, reference_arr) + was_calculated = True + self._add_metric( f"global_bin_{m.name.lower()}", MetricType.GLOBAL, - _build_global_bin_metric_function(m), + lambda x: MetricCouldNotBeComputedException(f"Global Metric {m} not set"), long_name="Global Binary " + m.value.long_name, + default_value=default_value, + was_calculated=was_calculated, ) + def _calc_global_bin_metric(self, metric: Metric, prediction_arr, reference_arr): + if metric not in self._global_metrics: + raise MetricCouldNotBeComputedException(f"Global Metric {metric} not set") + if self.tp == 0: + is_edgecase, result = self._edge_case_handler.handle_zero_tp(metric, self.tp, res.num_pred_instances, res.num_ref_instances) + if is_edgecase: + return result + pred_binary = prediction_arr + ref_binary = reference_arr + pred_binary[pred_binary != 0] = 1 + ref_binary[ref_binary != 0] = 1 + return metric( + reference_arr=ref_binary, + prediction_arr=pred_binary, + ) + def _add_metric( self, name_id: str, @@ -292,6 +308,7 @@ def calculate_all(self, print_errors: bool = False): print_errors (bool, optional): If true, will print every metric that could not be computed and its reason. Defaults to False. """ metric_errors: dict[str, Exception] = {} + for k, v in self._evaluation_metrics.items(): try: v = getattr(self, k) @@ -302,6 +319,13 @@ def calculate_all(self, print_errors: bool = False): for k, v in metric_errors.items(): print(f"Metric {k}: {v}") + def _calc(self, k, v): + try: + v = getattr(self, k) + return False, v + except Exception as e: + return True, e + def __str__(self) -> str: text = "" for metric_type in MetricType: @@ -329,19 +353,13 @@ def __str__(self) -> str: return text def to_dict(self) -> dict: - return { - k: getattr(self, v.id) - for k, v in self._evaluation_metrics.items() - if (v._error == False and v._was_calculated) - } + return {k: getattr(self, v.id) for k, v in self._evaluation_metrics.items() if (v._error == False and v._was_calculated)} def get_list_metric(self, metric: Metric, mode: MetricMode): if metric in self._list_metrics: return self._list_metrics[metric][mode] else: - raise MetricCouldNotBeComputedException( - f"{metric} could not be found, have you set it in eval_metrics during evaluation?" - ) + raise MetricCouldNotBeComputedException(f"{metric} could not be found, have you set it in eval_metrics during evaluation?") def _calc_metric(self, metric_name: str, supress_error: bool = False): if metric_name in self._evaluation_metrics: @@ -357,24 +375,22 @@ def _calc_metric(self, metric_name: str, supress_error: bool = False): self._evaluation_metrics[metric_name]._was_calculated = True return value else: - raise MetricCouldNotBeComputedException( - f"could not find metric with name {metric_name}" - ) + raise MetricCouldNotBeComputedException(f"could not find metric with name {metric_name}") def __getattribute__(self, __name: str) -> Any: attr = None try: attr = object.__getattribute__(self, __name) except AttributeError as e: + if __name == "_evaluation_metrics": + raise e if __name in self._evaluation_metrics.keys(): pass else: raise e if attr is None: if self._evaluation_metrics[__name]._error: - raise MetricCouldNotBeComputedException( - f"Requested metric {__name} that could not be computed" - ) + raise MetricCouldNotBeComputedException(f"Requested metric {__name} that could not be computed") elif not self._evaluation_metrics[__name]._was_calculated: value = self._calc_metric(__name) setattr(self, __name, value) @@ -500,9 +516,7 @@ def function_template(res: PanopticaResult): if metric not in res._global_metrics: raise MetricCouldNotBeComputedException(f"Global Metric {metric} not set") if res.tp == 0: - is_edgecase, result = res._edge_case_handler.handle_zero_tp( - metric, res.tp, res.num_pred_instances, res.num_ref_instances - ) + is_edgecase, result = res._edge_case_handler.handle_zero_tp(metric, res.tp, res.num_pred_instances, res.num_ref_instances) if is_edgecase: return result pred_binary = res._prediction_arr.copy() @@ -514,12 +528,11 @@ def function_template(res: PanopticaResult): prediction_arr=res._prediction_arr, ) - return function_template + return lambda x: function_template(x) # endregion - if __name__ == "__main__": c = PanopticaResult( reference_arr=np.zeros([5, 5, 5]), From d83da44931857f4284f0010fe268ca898fcb9fa5 Mon Sep 17 00:00:00 2001 From: iback Date: Fri, 9 Aug 2024 14:05:29 +0000 Subject: [PATCH 3/6] removed import --- panoptica/_functionals.py | 1 - 1 file changed, 1 deletion(-) diff --git a/panoptica/_functionals.py b/panoptica/_functionals.py index eee9a49..5344d20 100644 --- a/panoptica/_functionals.py +++ b/panoptica/_functionals.py @@ -3,7 +3,6 @@ import numpy as np -from panoptica.metrics.iou import _compute_instance_iou from panoptica.utils.constants import CCABackend from panoptica.utils.numpy_utils import _get_bbox_nd From 8b733af0362d88eb257f340012a7c21e7c095388 Mon Sep 17 00:00:00 2001 From: "brainless-bot[bot]" <153751247+brainless-bot[bot]@users.noreply.github.com> Date: Fri, 9 Aug 2024 14:07:00 +0000 Subject: [PATCH 4/6] Autoformat with black --- panoptica/panoptica_result.py | 38 ++++++++++++++++++++++++++--------- 1 file changed, 29 insertions(+), 9 deletions(-) diff --git a/panoptica/panoptica_result.py b/panoptica/panoptica_result.py index 5e5419a..086fc0e 100644 --- a/panoptica/panoptica_result.py +++ b/panoptica/panoptica_result.py @@ -243,18 +243,24 @@ def __init__( num_pred_instances=self.num_pred_instances, num_ref_instances=self.num_ref_instances, ) - self._list_metrics[m] = Evaluation_List_Metric(m, empty_list_std, list_metrics[m], is_edge_case, edge_case_result) + self._list_metrics[m] = Evaluation_List_Metric( + m, empty_list_std, list_metrics[m], is_edge_case, edge_case_result + ) # even if not available, set the global vars default_value = None was_calculated = False if m in self._global_metrics: - default_value = self._calc_global_bin_metric(m, prediction_arr, reference_arr) + default_value = self._calc_global_bin_metric( + m, prediction_arr, reference_arr + ) was_calculated = True self._add_metric( f"global_bin_{m.name.lower()}", MetricType.GLOBAL, - lambda x: MetricCouldNotBeComputedException(f"Global Metric {m} not set"), + lambda x: MetricCouldNotBeComputedException( + f"Global Metric {m} not set" + ), long_name="Global Binary " + m.value.long_name, default_value=default_value, was_calculated=was_calculated, @@ -264,7 +270,9 @@ def _calc_global_bin_metric(self, metric: Metric, prediction_arr, reference_arr) if metric not in self._global_metrics: raise MetricCouldNotBeComputedException(f"Global Metric {metric} not set") if self.tp == 0: - is_edgecase, result = self._edge_case_handler.handle_zero_tp(metric, self.tp, res.num_pred_instances, res.num_ref_instances) + is_edgecase, result = self._edge_case_handler.handle_zero_tp( + metric, self.tp, res.num_pred_instances, res.num_ref_instances + ) if is_edgecase: return result pred_binary = prediction_arr @@ -353,13 +361,19 @@ def __str__(self) -> str: return text def to_dict(self) -> dict: - return {k: getattr(self, v.id) for k, v in self._evaluation_metrics.items() if (v._error == False and v._was_calculated)} + return { + k: getattr(self, v.id) + for k, v in self._evaluation_metrics.items() + if (v._error == False and v._was_calculated) + } def get_list_metric(self, metric: Metric, mode: MetricMode): if metric in self._list_metrics: return self._list_metrics[metric][mode] else: - raise MetricCouldNotBeComputedException(f"{metric} could not be found, have you set it in eval_metrics during evaluation?") + raise MetricCouldNotBeComputedException( + f"{metric} could not be found, have you set it in eval_metrics during evaluation?" + ) def _calc_metric(self, metric_name: str, supress_error: bool = False): if metric_name in self._evaluation_metrics: @@ -375,7 +389,9 @@ def _calc_metric(self, metric_name: str, supress_error: bool = False): self._evaluation_metrics[metric_name]._was_calculated = True return value else: - raise MetricCouldNotBeComputedException(f"could not find metric with name {metric_name}") + raise MetricCouldNotBeComputedException( + f"could not find metric with name {metric_name}" + ) def __getattribute__(self, __name: str) -> Any: attr = None @@ -390,7 +406,9 @@ def __getattribute__(self, __name: str) -> Any: raise e if attr is None: if self._evaluation_metrics[__name]._error: - raise MetricCouldNotBeComputedException(f"Requested metric {__name} that could not be computed") + raise MetricCouldNotBeComputedException( + f"Requested metric {__name} that could not be computed" + ) elif not self._evaluation_metrics[__name]._was_calculated: value = self._calc_metric(__name) setattr(self, __name, value) @@ -516,7 +534,9 @@ def function_template(res: PanopticaResult): if metric not in res._global_metrics: raise MetricCouldNotBeComputedException(f"Global Metric {metric} not set") if res.tp == 0: - is_edgecase, result = res._edge_case_handler.handle_zero_tp(metric, res.tp, res.num_pred_instances, res.num_ref_instances) + is_edgecase, result = res._edge_case_handler.handle_zero_tp( + metric, res.tp, res.num_pred_instances, res.num_ref_instances + ) if is_edgecase: return result pred_binary = res._prediction_arr.copy() From d4e67c5d3011e176cc8a2fffab9a34abda5e4d45 Mon Sep 17 00:00:00 2001 From: iback Date: Fri, 9 Aug 2024 14:08:01 +0000 Subject: [PATCH 5/6] fixed typo --- panoptica/panoptica_result.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/panoptica/panoptica_result.py b/panoptica/panoptica_result.py index 5e5419a..24a446e 100644 --- a/panoptica/panoptica_result.py +++ b/panoptica/panoptica_result.py @@ -264,7 +264,7 @@ def _calc_global_bin_metric(self, metric: Metric, prediction_arr, reference_arr) if metric not in self._global_metrics: raise MetricCouldNotBeComputedException(f"Global Metric {metric} not set") if self.tp == 0: - is_edgecase, result = self._edge_case_handler.handle_zero_tp(metric, self.tp, res.num_pred_instances, res.num_ref_instances) + is_edgecase, result = self._edge_case_handler.handle_zero_tp(metric, self.tp, self.num_pred_instances, self.num_ref_instances) if is_edgecase: return result pred_binary = prediction_arr From 98de64b4857610636fb068393c8bc2b54c2baece Mon Sep 17 00:00:00 2001 From: "brainless-bot[bot]" <153751247+brainless-bot[bot]@users.noreply.github.com> Date: Fri, 9 Aug 2024 14:11:28 +0000 Subject: [PATCH 6/6] Autoformat with black --- panoptica/panoptica_result.py | 38 ++++++++++++++++++++++++++--------- 1 file changed, 29 insertions(+), 9 deletions(-) diff --git a/panoptica/panoptica_result.py b/panoptica/panoptica_result.py index 24a446e..60b0e43 100644 --- a/panoptica/panoptica_result.py +++ b/panoptica/panoptica_result.py @@ -243,18 +243,24 @@ def __init__( num_pred_instances=self.num_pred_instances, num_ref_instances=self.num_ref_instances, ) - self._list_metrics[m] = Evaluation_List_Metric(m, empty_list_std, list_metrics[m], is_edge_case, edge_case_result) + self._list_metrics[m] = Evaluation_List_Metric( + m, empty_list_std, list_metrics[m], is_edge_case, edge_case_result + ) # even if not available, set the global vars default_value = None was_calculated = False if m in self._global_metrics: - default_value = self._calc_global_bin_metric(m, prediction_arr, reference_arr) + default_value = self._calc_global_bin_metric( + m, prediction_arr, reference_arr + ) was_calculated = True self._add_metric( f"global_bin_{m.name.lower()}", MetricType.GLOBAL, - lambda x: MetricCouldNotBeComputedException(f"Global Metric {m} not set"), + lambda x: MetricCouldNotBeComputedException( + f"Global Metric {m} not set" + ), long_name="Global Binary " + m.value.long_name, default_value=default_value, was_calculated=was_calculated, @@ -264,7 +270,9 @@ def _calc_global_bin_metric(self, metric: Metric, prediction_arr, reference_arr) if metric not in self._global_metrics: raise MetricCouldNotBeComputedException(f"Global Metric {metric} not set") if self.tp == 0: - is_edgecase, result = self._edge_case_handler.handle_zero_tp(metric, self.tp, self.num_pred_instances, self.num_ref_instances) + is_edgecase, result = self._edge_case_handler.handle_zero_tp( + metric, self.tp, self.num_pred_instances, self.num_ref_instances + ) if is_edgecase: return result pred_binary = prediction_arr @@ -353,13 +361,19 @@ def __str__(self) -> str: return text def to_dict(self) -> dict: - return {k: getattr(self, v.id) for k, v in self._evaluation_metrics.items() if (v._error == False and v._was_calculated)} + return { + k: getattr(self, v.id) + for k, v in self._evaluation_metrics.items() + if (v._error == False and v._was_calculated) + } def get_list_metric(self, metric: Metric, mode: MetricMode): if metric in self._list_metrics: return self._list_metrics[metric][mode] else: - raise MetricCouldNotBeComputedException(f"{metric} could not be found, have you set it in eval_metrics during evaluation?") + raise MetricCouldNotBeComputedException( + f"{metric} could not be found, have you set it in eval_metrics during evaluation?" + ) def _calc_metric(self, metric_name: str, supress_error: bool = False): if metric_name in self._evaluation_metrics: @@ -375,7 +389,9 @@ def _calc_metric(self, metric_name: str, supress_error: bool = False): self._evaluation_metrics[metric_name]._was_calculated = True return value else: - raise MetricCouldNotBeComputedException(f"could not find metric with name {metric_name}") + raise MetricCouldNotBeComputedException( + f"could not find metric with name {metric_name}" + ) def __getattribute__(self, __name: str) -> Any: attr = None @@ -390,7 +406,9 @@ def __getattribute__(self, __name: str) -> Any: raise e if attr is None: if self._evaluation_metrics[__name]._error: - raise MetricCouldNotBeComputedException(f"Requested metric {__name} that could not be computed") + raise MetricCouldNotBeComputedException( + f"Requested metric {__name} that could not be computed" + ) elif not self._evaluation_metrics[__name]._was_calculated: value = self._calc_metric(__name) setattr(self, __name, value) @@ -516,7 +534,9 @@ def function_template(res: PanopticaResult): if metric not in res._global_metrics: raise MetricCouldNotBeComputedException(f"Global Metric {metric} not set") if res.tp == 0: - is_edgecase, result = res._edge_case_handler.handle_zero_tp(metric, res.tp, res.num_pred_instances, res.num_ref_instances) + is_edgecase, result = res._edge_case_handler.handle_zero_tp( + metric, res.tp, res.num_pred_instances, res.num_ref_instances + ) if is_edgecase: return result pred_binary = res._prediction_arr.copy()