From 73d83a760e7248fa03f29f7cf46648bf9e8cc13d Mon Sep 17 00:00:00 2001 From: iback Date: Thu, 25 Jan 2024 13:19:55 +0000 Subject: [PATCH 01/29] added MetricType to the mix, sorting upon print the metrics into different categories. move large sections from result.py into the metrics.py --- examples/example_spine_instance.py | 2 +- panoptica/metrics/__init__.py | 28 ++-- panoptica/metrics/metrics.py | 158 +++++++++++++++++++--- panoptica/panoptic_result.py | 209 +++++++++-------------------- 4 files changed, 213 insertions(+), 184 deletions(-) diff --git a/examples/example_spine_instance.py b/examples/example_spine_instance.py index 61720c3..dc2b639 100644 --- a/examples/example_spine_instance.py +++ b/examples/example_spine_instance.py @@ -4,7 +4,7 @@ from auxiliary.turbopath import turbopath from panoptica import MatchedInstancePair, Panoptic_Evaluator -from panoptica.metrics import Metric, Metric, MetricMode +from panoptica.metrics import Metric, MetricMode directory = turbopath(__file__).parent diff --git a/panoptica/metrics/__init__.py b/panoptica/metrics/__init__.py index f636541..f12ed71 100644 --- a/panoptica/metrics/__init__.py +++ b/panoptica/metrics/__init__.py @@ -1,17 +1,13 @@ -from panoptica.metrics.assd import ( - _average_surface_distance, - _average_symmetric_surface_distance, +from panoptica.metrics.assd import _average_surface_distance, _average_symmetric_surface_distance +from panoptica.metrics.cldice import _compute_centerline_dice, _compute_centerline_dice_coefficient +from panoptica.metrics.dice import _compute_dice_coefficient, _compute_instance_volumetric_dice +from panoptica.metrics.iou import _compute_instance_iou, _compute_iou +from panoptica.metrics.metrics import ( + Evaluation_List_Metric, + Evaluation_Metric, + Metric, + MetricCouldNotBeComputedException, + MetricMode, + MetricType, + _Metric, ) -from panoptica.metrics.dice import ( - _compute_dice_coefficient, - _compute_instance_volumetric_dice, -) -from panoptica.metrics.iou import ( - _compute_instance_iou, - _compute_iou, -) -from panoptica.metrics.cldice import ( - _compute_centerline_dice, - _compute_centerline_dice_coefficient, -) -from panoptica.metrics.metrics import Metric, _Metric, MetricMode diff --git a/panoptica/metrics/metrics.py b/panoptica/metrics/metrics.py index 3b2b46f..758a4a2 100644 --- a/panoptica/metrics/metrics.py +++ b/panoptica/metrics/metrics.py @@ -1,17 +1,20 @@ from dataclasses import dataclass -from enum import EnumMeta, Enum -from typing import Any, Callable +from enum import EnumMeta +from typing import TYPE_CHECKING, Any, Callable import numpy as np from panoptica.metrics import ( _average_symmetric_surface_distance, + _compute_centerline_dice_coefficient, _compute_dice_coefficient, _compute_iou, - _compute_centerline_dice_coefficient, ) from panoptica.utils.constants import _Enum_Compare, auto +if TYPE_CHECKING: + from panoptic_result import PanopticaResult + @dataclass class _Metric: @@ -34,9 +37,7 @@ def __call__( reference_arr = reference_arr.copy() == ref_instance_idx if isinstance(pred_instance_idx, int): pred_instance_idx = [pred_instance_idx] - prediction_arr = np.isin( - prediction_arr.copy(), pred_instance_idx - ) # type:ignore + prediction_arr = np.isin(prediction_arr.copy(), pred_instance_idx) # type:ignore return self._metric_function(reference_arr, prediction_arr, *args, **kwargs) def __eq__(self, __value: object) -> bool: @@ -60,12 +61,8 @@ def __hash__(self) -> int: def increasing(self): return not self.decreasing - def score_beats_threshold( - self, matching_score: float, matching_threshold: float - ) -> bool: - return (self.increasing and matching_score >= matching_threshold) or ( - self.decreasing and matching_score <= matching_threshold - ) + def score_beats_threshold(self, matching_score: float, matching_threshold: float) -> bool: + return (self.increasing and matching_score >= matching_threshold) or (self.decreasing and matching_score <= matching_threshold) class DirectValueMeta(EnumMeta): @@ -120,9 +117,7 @@ def __call__( **kwargs, ) - def score_beats_threshold( - self, matching_score: float, matching_threshold: float - ) -> bool: + def score_beats_threshold(self, matching_score: float, matching_threshold: float) -> bool: """Calculates whether a score beats a specified threshold Args: @@ -132,9 +127,7 @@ def score_beats_threshold( Returns: bool: True if the matching_score beats the threshold, False otherwise. """ - return (self.increasing and matching_score >= matching_threshold) or ( - self.decreasing and matching_score <= matching_threshold - ) + return (self.increasing and matching_score >= matching_threshold) or (self.decreasing and matching_score <= matching_threshold) @property def name(self): @@ -165,6 +158,134 @@ class MetricMode(_Enum_Compare): STD = auto() +class MetricType(_Enum_Compare): + """Different type of metrics + + Args: + _Enum_Compare (_type_): _description_ + """ + + MATCHING = auto() + GLOBAL = auto() + INSTANCE = auto() + + +class MetricCouldNotBeComputedException(Exception): + """Exception for when a Metric cannot be computed""" + + def __init__(self, *args: object) -> None: + super().__init__(*args) + + +class Evaluation_Metric: + def __init__( + self, + name_id: str, + metric_type: MetricType, + calc_func: Callable | None, + long_name: str | None = None, + was_calculated: bool = False, + error: bool = False, + ): + """This represents a metric in the evaluation derived from other metrics or list metrics (no circular dependancies!) + + Args: + name_id (str): code-name of this metric, must be same as the member variable of PanopticResult + calc_func (Callable): the function to calculate this metric based on the PanopticResult object + long_name (str | None, optional): A longer descriptive name for printing/logging purposes. Defaults to None. + was_calculated (bool, optional): Whether this metric has been calculated or not. Defaults to False. + error (bool, optional): If true, means the metric could not have been calculated (because dependancies do not exist or have this flag set to True). Defaults to False. + """ + self.id = name_id + self.metric_type = metric_type + self._calc_func = calc_func + self.long_name = long_name + self._was_calculated = was_calculated + self._value = None + self._error = error + self._error_obj: MetricCouldNotBeComputedException | None = None + + def __call__(self, result_obj: "PanopticaResult") -> Any: + """If called, needs to return its way, raise error or calculate it + + Args: + result_obj (PanopticaResult): _description_ + + Raises: + MetricCouldNotBeComputedException: _description_ + self._error_obj: _description_ + + Returns: + Any: _description_ + """ + # ERROR + if self._error: + if self._error_obj is None: + self._error_obj = MetricCouldNotBeComputedException(f"Metric {self.id} requested, but could not be computed") + raise self._error_obj + # Already calculated? + if self._was_calculated: + return self._value + + # Calculate it + try: + assert not self._was_calculated, f"Metric {self.id} was called to compute, but is set to have been already calculated" + assert self._calc_func is not None, f"Metric {self.id} was called to compute, but has no calculation function set" + value = self._calc_func(result_obj) + except MetricCouldNotBeComputedException as e: + value = e + self._error = True + self._error_obj = e + self._was_calculated = True + + self._value = value + return self._value + + def __str__(self) -> str: + if self.long_name is not None: + return self.long_name + f" ({self.id})" + else: + return self.id + + +class Evaluation_List_Metric: + def __init__( + self, + name_id: Metric, + empty_list_std: float | None, + value_list: list[float] | None, # None stands for not calculated + is_edge_case: bool = False, + edge_case_result: float | None = None, + ): + """This represents the metrics resulting from a Metric calculated between paired instances (IoU, ASSD, Dice, ...) + + Args: + name_id (Metric): code-name of this metric + empty_list_std (float): Value for the standard deviation if the list of values is empty + value_list (list[float] | None): List of values of that metric (only the TPs) + """ + self.id = name_id + self.error = value_list is None + self.ALL: list[float] | None = value_list + if is_edge_case: + self.AVG: float | None = edge_case_result + self.SUM: None | float = edge_case_result + else: + self.AVG = None if self.ALL is None else np.average(self.ALL) + self.SUM = None if self.ALL is None else np.sum(self.ALL) + self.STD = None if self.ALL is None else empty_list_std if len(self.ALL) == 0 else np.std(self.ALL) + + def __getitem__(self, mode: MetricMode | str): + if self.error: + raise MetricCouldNotBeComputedException(f"Metric {self.id} has not been calculated, add it to your eval_metrics") + if isinstance(mode, MetricMode): + mode = mode.name + if hasattr(self, mode): + return getattr(self, mode) + else: + raise MetricCouldNotBeComputedException(f"List_Metric {self.id} does not contain {mode} member") + + if __name__ == "__main__": print(Metric.DSC) # print(MatchingMetric.DSC.name) @@ -175,3 +296,4 @@ class MetricMode(_Enum_Compare): # print(Metric.DSC == Metric.IOU) print(Metric.DSC == "IOU") + print(Metric.DSC == "IOU") diff --git a/panoptica/panoptic_result.py b/panoptica/panoptic_result.py index 38b002a..2e01da1 100644 --- a/panoptica/panoptic_result.py +++ b/panoptica/panoptic_result.py @@ -1,123 +1,20 @@ from __future__ import annotations from typing import Any, Callable + import numpy as np -from panoptica.metrics import MetricMode, Metric + from panoptica.metrics import ( - _compute_dice_coefficient, + Evaluation_List_Metric, + Evaluation_Metric, + Metric, + MetricCouldNotBeComputedException, + MetricMode, + MetricType, _compute_centerline_dice_coefficient, + _compute_dice_coefficient, ) from panoptica.utils import EdgeCaseHandler -from panoptica.utils.processing_pair import MatchedInstancePair - - -class MetricCouldNotBeComputedException(Exception): - """Exception for when a Metric cannot be computed""" - - def __init__(self, *args: object) -> None: - super().__init__(*args) - - -class Evaluation_Metric: - def __init__( - self, - name_id: str, - calc_func: Callable | None, - long_name: str | None = None, - was_calculated: bool = False, - error: bool = False, - ): - """This represents a metric in the evaluation derived from other metrics or list metrics (no circular dependancies!) - - Args: - name_id (str): code-name of this metric, must be same as the member variable of PanopticResult - calc_func (Callable): the function to calculate this metric based on the PanopticResult object - long_name (str | None, optional): A longer descriptive name for printing/logging purposes. Defaults to None. - was_calculated (bool, optional): Whether this metric has been calculated or not. Defaults to False. - error (bool, optional): If true, means the metric could not have been calculated (because dependancies do not exist or have this flag set to True). Defaults to False. - """ - self.id = name_id - self.calc_func = calc_func - self.long_name = long_name - self.was_calculated = was_calculated - self.error = error - self.error_obj: MetricCouldNotBeComputedException | None = None - - def __call__(self, result_obj: PanopticaResult) -> Any: - if self.error: - if self.error_obj is None: - raise MetricCouldNotBeComputedException( - f"Metric {self.id} requested, but could not be computed" - ) - else: - raise self.error_obj - assert ( - not self.was_calculated - ), f"Metric {self.id} was called to compute, but is set to have been already calculated" - assert ( - self.calc_func is not None - ), f"Metric {self.id} was called to compute, but has no calculation function set" - try: - value = self.calc_func(result_obj) - except MetricCouldNotBeComputedException as e: - value = e - self.error = True - self.error_obj = e - return value - - def __str__(self) -> str: - if self.long_name is not None: - return self.long_name + f" ({self.id})" - else: - return self.id - - -class Evaluation_List_Metric: - def __init__( - self, - name_id: Metric, - empty_list_std: float | None, - value_list: list[float] | None, # None stands for not calculated - is_edge_case: bool = False, - edge_case_result: float | None = None, - ): - """This represents the metrics resulting from a Metric calculated between paired instances (IoU, ASSD, Dice, ...) - - Args: - name_id (Metric): code-name of this metric - empty_list_std (float): Value for the standard deviation if the list of values is empty - value_list (list[float] | None): List of values of that metric (only the TPs) - """ - self.id = name_id - self.error = value_list is None - self.ALL: list[float] | None = value_list - if is_edge_case: - self.AVG: float | None = edge_case_result - self.SUM: None | float = edge_case_result - else: - self.AVG = None if self.ALL is None else np.average(self.ALL) - self.SUM = None if self.ALL is None else np.sum(self.ALL) - self.STD = ( - None - if self.ALL is None - else empty_list_std - if len(self.ALL) == 0 - else np.std(self.ALL) - ) - - def __getitem__(self, mode: MetricMode | str): - if self.error: - raise MetricCouldNotBeComputedException( - f"Metric {self.id} has not been calculated, add it to your eval_metrics" - ) - if isinstance(mode, MetricMode): - mode = mode.name - if hasattr(self, mode): - return getattr(self, mode) - else: - raise MetricCouldNotBeComputedException( - f"List_Metric {self.id} does not contain {mode} member" - ) class PanopticaResult(object): @@ -156,6 +53,7 @@ def __init__( self.num_ref_instances: int self._add_metric( "num_ref_instances", + MetricType.MATCHING, None, long_name="Number of instances in reference", default_value=num_ref_instances, @@ -164,6 +62,7 @@ def __init__( self.num_pred_instances: int self._add_metric( "num_pred_instances", + MetricType.MATCHING, None, long_name="Number of instances in prediction", default_value=num_pred_instances, @@ -172,6 +71,7 @@ def __init__( self.tp: int self._add_metric( "tp", + MetricType.MATCHING, None, long_name="True Positives", default_value=tp, @@ -183,18 +83,21 @@ def __init__( self.fp: int self._add_metric( "fp", + MetricType.MATCHING, fp, long_name="False Positives", ) self.fn: int self._add_metric( "fn", + MetricType.MATCHING, fn, long_name="False Negatives", ) self.rq: float self._add_metric( "rq", + MetricType.MATCHING, rq, long_name="Recognition Quality", ) @@ -204,6 +107,7 @@ def __init__( self.global_bin_dsc: int self._add_metric( "global_bin_dsc", + MetricType.GLOBAL, global_bin_dsc, long_name="Global Binary Dice", ) @@ -211,6 +115,7 @@ def __init__( self.global_bin_cldsc: int self._add_metric( "global_bin_cldsc", + MetricType.GLOBAL, global_bin_cldsc, long_name="Global Binary Centerline Dice", ) @@ -220,18 +125,21 @@ def __init__( self.sq: float self._add_metric( "sq", + MetricType.INSTANCE, sq, long_name="Segmentation Quality IoU", ) self.sq_std: float self._add_metric( "sq_std", + MetricType.INSTANCE, sq_std, long_name="Segmentation Quality IoU Standard Deviation", ) self.pq: float self._add_metric( "pq", + MetricType.INSTANCE, pq, long_name="Panoptic Quality IoU", ) @@ -241,18 +149,21 @@ def __init__( self.sq_dsc: float self._add_metric( "sq_dsc", + MetricType.INSTANCE, sq_dsc, long_name="Segmentation Quality Dsc", ) self.sq_dsc_std: float self._add_metric( "sq_dsc_std", + MetricType.INSTANCE, sq_dsc_std, long_name="Segmentation Quality Dsc Standard Deviation", ) self.pq_dsc: float self._add_metric( "pq_dsc", + MetricType.INSTANCE, pq_dsc, long_name="Panoptic Quality Dsc", ) @@ -262,18 +173,21 @@ def __init__( self.sq_cldsc: float self._add_metric( "sq_cldsc", + MetricType.INSTANCE, sq_cldsc, long_name="Segmentation Quality Centerline Dsc", ) self.sq_cldsc_std: float self._add_metric( "sq_cldsc_std", + MetricType.INSTANCE, sq_cldsc_std, long_name="Segmentation Quality Centerline Dsc Standard Deviation", ) self.pq_cldsc: float self._add_metric( "pq_cldsc", + MetricType.INSTANCE, pq_cldsc, long_name="Panoptic Quality Centerline Dsc", ) @@ -283,12 +197,14 @@ def __init__( self.sq_assd: float self._add_metric( "sq_assd", + MetricType.INSTANCE, sq_assd, long_name="Segmentation Quality Assd", ) self.sq_assd_std: float self._add_metric( "sq_assd_std", + MetricType.INSTANCE, sq_assd_std, long_name="Segmentation Quality Assd Standard Deviation", ) @@ -305,13 +221,12 @@ def __init__( num_pred_instances=self.num_pred_instances, num_ref_instances=self.num_ref_instances, ) - self._list_metrics[k] = Evaluation_List_Metric( - k, empty_list_std, v, is_edge_case, edge_case_result - ) + self._list_metrics[k] = Evaluation_List_Metric(k, empty_list_std, v, is_edge_case, edge_case_result) def _add_metric( self, name_id: str, + metric_type: MetricType, calc_func: Callable | None, long_name: str | None = None, default_value=None, @@ -323,7 +238,9 @@ def _add_metric( assert ( was_calculated ), "Tried to add a metric without a calc_function but that hasn't been calculated yet, how did you think this could works?" - eval_metric = Evaluation_Metric(name_id, calc_func, long_name, was_calculated) + eval_metric = Evaluation_Metric( + name_id, metric_type=metric_type, calc_func=calc_func, long_name=long_name, was_calculated=was_calculated + ) self._evaluation_metrics[name_id] = eval_metric return default_value @@ -346,36 +263,34 @@ def calculate_all(self, print_errors: bool = False): def __str__(self) -> str: text = "" - for k, v in self._evaluation_metrics.items(): - if k.endswith("_std"): - continue - if v.was_calculated and not v.error: - # is there standard deviation for this? - text += f"{v}: {self.__getattribute__(k)}" - k_std = k + "_std" - if ( - k_std in self._evaluation_metrics - and self._evaluation_metrics[k_std].was_calculated - and not self._evaluation_metrics[k_std].error - ): - text += f" +- {self.__getattribute__(k_std)}" - text += "\n" + for metric_type in MetricType: + text += f"\n+++ {metric_type.name} +++\n" + for k, v in self._evaluation_metrics.items(): + if v.metric_type != metric_type: + continue + if k.endswith("_std"): + continue + if v._was_calculated and not v._error: + # is there standard deviation for this? + text += f"{v}: {self.__getattribute__(k)}" + k_std = k + "_std" + if ( + k_std in self._evaluation_metrics + and self._evaluation_metrics[k_std]._was_calculated + and not self._evaluation_metrics[k_std]._error + ): + text += f" +- {self.__getattribute__(k_std)}" + text += "\n" return text def to_dict(self) -> dict: - return { - k: getattr(self, v.id) - for k, v in self._evaluation_metrics.items() - if (v.error == False and v.was_calculated) - } + return {k: getattr(self, v.id) for k, v in self._evaluation_metrics.items() if (v._error == False and v._was_calculated)} def get_list_metric(self, metric: Metric, mode: MetricMode): if metric in self._list_metrics: return self._list_metrics[metric][mode] else: - raise MetricCouldNotBeComputedException( - f"{metric} could not be found, have you set it in eval_metrics during evaluation?" - ) + raise MetricCouldNotBeComputedException(f"{metric} could not be found, have you set it in eval_metrics during evaluation?") def _calc_metric(self, metric_name: str, supress_error: bool = False): if metric_name in self._evaluation_metrics: @@ -384,16 +299,14 @@ def _calc_metric(self, metric_name: str, supress_error: bool = False): except MetricCouldNotBeComputedException as e: value = e if isinstance(value, MetricCouldNotBeComputedException): - self._evaluation_metrics[metric_name].error = True - self._evaluation_metrics[metric_name].was_calculated = True + self._evaluation_metrics[metric_name]._error = True + self._evaluation_metrics[metric_name]._was_calculated = True if not supress_error: raise value - self._evaluation_metrics[metric_name].was_calculated = True + self._evaluation_metrics[metric_name]._was_calculated = True return value else: - raise MetricCouldNotBeComputedException( - f"could not find metric with name {metric_name}" - ) + raise MetricCouldNotBeComputedException(f"could not find metric with name {metric_name}") def __getattribute__(self, __name: str) -> Any: attr = None @@ -405,11 +318,9 @@ def __getattribute__(self, __name: str) -> Any: else: raise e if attr is None: - if self._evaluation_metrics[__name].error: - raise MetricCouldNotBeComputedException( - f"Requested metric {__name} that could not be computed" - ) - elif not self._evaluation_metrics[__name].was_calculated: + if self._evaluation_metrics[__name]._error: + raise MetricCouldNotBeComputedException(f"Requested metric {__name} that could not be computed") + elif not self._evaluation_metrics[__name]._was_calculated: value = self._calc_metric(__name) setattr(self, __name, value) if isinstance(value, MetricCouldNotBeComputedException): From dbccf1cf784dee978fc5a73c28ad2c4d0e125173 Mon Sep 17 00:00:00 2001 From: "brainless-bot[bot]" <153751247+brainless-bot[bot]@users.noreply.github.com> Date: Thu, 25 Jan 2024 13:21:39 +0000 Subject: [PATCH 02/29] Autoformat with black --- panoptica/metrics/__init__.py | 15 ++++++++--- panoptica/metrics/metrics.py | 48 +++++++++++++++++++++++++++-------- panoptica/panoptic_result.py | 28 +++++++++++++++----- 3 files changed, 71 insertions(+), 20 deletions(-) diff --git a/panoptica/metrics/__init__.py b/panoptica/metrics/__init__.py index f12ed71..f0e515f 100644 --- a/panoptica/metrics/__init__.py +++ b/panoptica/metrics/__init__.py @@ -1,6 +1,15 @@ -from panoptica.metrics.assd import _average_surface_distance, _average_symmetric_surface_distance -from panoptica.metrics.cldice import _compute_centerline_dice, _compute_centerline_dice_coefficient -from panoptica.metrics.dice import _compute_dice_coefficient, _compute_instance_volumetric_dice +from panoptica.metrics.assd import ( + _average_surface_distance, + _average_symmetric_surface_distance, +) +from panoptica.metrics.cldice import ( + _compute_centerline_dice, + _compute_centerline_dice_coefficient, +) +from panoptica.metrics.dice import ( + _compute_dice_coefficient, + _compute_instance_volumetric_dice, +) from panoptica.metrics.iou import _compute_instance_iou, _compute_iou from panoptica.metrics.metrics import ( Evaluation_List_Metric, diff --git a/panoptica/metrics/metrics.py b/panoptica/metrics/metrics.py index 758a4a2..c7ce019 100644 --- a/panoptica/metrics/metrics.py +++ b/panoptica/metrics/metrics.py @@ -37,7 +37,9 @@ def __call__( reference_arr = reference_arr.copy() == ref_instance_idx if isinstance(pred_instance_idx, int): pred_instance_idx = [pred_instance_idx] - prediction_arr = np.isin(prediction_arr.copy(), pred_instance_idx) # type:ignore + prediction_arr = np.isin( + prediction_arr.copy(), pred_instance_idx + ) # type:ignore return self._metric_function(reference_arr, prediction_arr, *args, **kwargs) def __eq__(self, __value: object) -> bool: @@ -61,8 +63,12 @@ def __hash__(self) -> int: def increasing(self): return not self.decreasing - def score_beats_threshold(self, matching_score: float, matching_threshold: float) -> bool: - return (self.increasing and matching_score >= matching_threshold) or (self.decreasing and matching_score <= matching_threshold) + def score_beats_threshold( + self, matching_score: float, matching_threshold: float + ) -> bool: + return (self.increasing and matching_score >= matching_threshold) or ( + self.decreasing and matching_score <= matching_threshold + ) class DirectValueMeta(EnumMeta): @@ -117,7 +123,9 @@ def __call__( **kwargs, ) - def score_beats_threshold(self, matching_score: float, matching_threshold: float) -> bool: + def score_beats_threshold( + self, matching_score: float, matching_threshold: float + ) -> bool: """Calculates whether a score beats a specified threshold Args: @@ -127,7 +135,9 @@ def score_beats_threshold(self, matching_score: float, matching_threshold: float Returns: bool: True if the matching_score beats the threshold, False otherwise. """ - return (self.increasing and matching_score >= matching_threshold) or (self.decreasing and matching_score <= matching_threshold) + return (self.increasing and matching_score >= matching_threshold) or ( + self.decreasing and matching_score <= matching_threshold + ) @property def name(self): @@ -221,7 +231,9 @@ def __call__(self, result_obj: "PanopticaResult") -> Any: # ERROR if self._error: if self._error_obj is None: - self._error_obj = MetricCouldNotBeComputedException(f"Metric {self.id} requested, but could not be computed") + self._error_obj = MetricCouldNotBeComputedException( + f"Metric {self.id} requested, but could not be computed" + ) raise self._error_obj # Already calculated? if self._was_calculated: @@ -229,8 +241,12 @@ def __call__(self, result_obj: "PanopticaResult") -> Any: # Calculate it try: - assert not self._was_calculated, f"Metric {self.id} was called to compute, but is set to have been already calculated" - assert self._calc_func is not None, f"Metric {self.id} was called to compute, but has no calculation function set" + assert ( + not self._was_calculated + ), f"Metric {self.id} was called to compute, but is set to have been already calculated" + assert ( + self._calc_func is not None + ), f"Metric {self.id} was called to compute, but has no calculation function set" value = self._calc_func(result_obj) except MetricCouldNotBeComputedException as e: value = e @@ -273,17 +289,27 @@ def __init__( else: self.AVG = None if self.ALL is None else np.average(self.ALL) self.SUM = None if self.ALL is None else np.sum(self.ALL) - self.STD = None if self.ALL is None else empty_list_std if len(self.ALL) == 0 else np.std(self.ALL) + self.STD = ( + None + if self.ALL is None + else empty_list_std + if len(self.ALL) == 0 + else np.std(self.ALL) + ) def __getitem__(self, mode: MetricMode | str): if self.error: - raise MetricCouldNotBeComputedException(f"Metric {self.id} has not been calculated, add it to your eval_metrics") + raise MetricCouldNotBeComputedException( + f"Metric {self.id} has not been calculated, add it to your eval_metrics" + ) if isinstance(mode, MetricMode): mode = mode.name if hasattr(self, mode): return getattr(self, mode) else: - raise MetricCouldNotBeComputedException(f"List_Metric {self.id} does not contain {mode} member") + raise MetricCouldNotBeComputedException( + f"List_Metric {self.id} does not contain {mode} member" + ) if __name__ == "__main__": diff --git a/panoptica/panoptic_result.py b/panoptica/panoptic_result.py index 2e01da1..fce3ace 100644 --- a/panoptica/panoptic_result.py +++ b/panoptica/panoptic_result.py @@ -221,7 +221,9 @@ def __init__( num_pred_instances=self.num_pred_instances, num_ref_instances=self.num_ref_instances, ) - self._list_metrics[k] = Evaluation_List_Metric(k, empty_list_std, v, is_edge_case, edge_case_result) + self._list_metrics[k] = Evaluation_List_Metric( + k, empty_list_std, v, is_edge_case, edge_case_result + ) def _add_metric( self, @@ -239,7 +241,11 @@ def _add_metric( was_calculated ), "Tried to add a metric without a calc_function but that hasn't been calculated yet, how did you think this could works?" eval_metric = Evaluation_Metric( - name_id, metric_type=metric_type, calc_func=calc_func, long_name=long_name, was_calculated=was_calculated + name_id, + metric_type=metric_type, + calc_func=calc_func, + long_name=long_name, + was_calculated=was_calculated, ) self._evaluation_metrics[name_id] = eval_metric return default_value @@ -284,13 +290,19 @@ def __str__(self) -> str: return text def to_dict(self) -> dict: - return {k: getattr(self, v.id) for k, v in self._evaluation_metrics.items() if (v._error == False and v._was_calculated)} + return { + k: getattr(self, v.id) + for k, v in self._evaluation_metrics.items() + if (v._error == False and v._was_calculated) + } def get_list_metric(self, metric: Metric, mode: MetricMode): if metric in self._list_metrics: return self._list_metrics[metric][mode] else: - raise MetricCouldNotBeComputedException(f"{metric} could not be found, have you set it in eval_metrics during evaluation?") + raise MetricCouldNotBeComputedException( + f"{metric} could not be found, have you set it in eval_metrics during evaluation?" + ) def _calc_metric(self, metric_name: str, supress_error: bool = False): if metric_name in self._evaluation_metrics: @@ -306,7 +318,9 @@ def _calc_metric(self, metric_name: str, supress_error: bool = False): self._evaluation_metrics[metric_name]._was_calculated = True return value else: - raise MetricCouldNotBeComputedException(f"could not find metric with name {metric_name}") + raise MetricCouldNotBeComputedException( + f"could not find metric with name {metric_name}" + ) def __getattribute__(self, __name: str) -> Any: attr = None @@ -319,7 +333,9 @@ def __getattribute__(self, __name: str) -> Any: raise e if attr is None: if self._evaluation_metrics[__name]._error: - raise MetricCouldNotBeComputedException(f"Requested metric {__name} that could not be computed") + raise MetricCouldNotBeComputedException( + f"Requested metric {__name} that could not be computed" + ) elif not self._evaluation_metrics[__name]._was_calculated: value = self._calc_metric(__name) setattr(self, __name, value) From 6b7223f101b2edbe8fd48ab497401a5d3621fa45 Mon Sep 17 00:00:00 2001 From: neuronflow Date: Thu, 1 Feb 2024 09:59:44 +0100 Subject: [PATCH 03/29] add F1 score description Signed-off-by: neuronflow --- panoptica/panoptic_result.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/panoptica/panoptic_result.py b/panoptica/panoptic_result.py index fce3ace..2b4b79b 100644 --- a/panoptica/panoptic_result.py +++ b/panoptica/panoptic_result.py @@ -99,7 +99,7 @@ def __init__( "rq", MetricType.MATCHING, rq, - long_name="Recognition Quality", + long_name="Recognition Quality / F1-Score", ) # endregion # From dfda69217861e55323ae8f3c382f772b0bfb8d9c Mon Sep 17 00:00:00 2001 From: neuronflow Date: Thu, 1 Feb 2024 10:00:20 +0100 Subject: [PATCH 04/29] sort imports Signed-off-by: neuronflow --- unit_tests/test_panoptic_evaluator.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/unit_tests/test_panoptic_evaluator.py b/unit_tests/test_panoptic_evaluator.py index f732701..e69b2b3 100644 --- a/unit_tests/test_panoptic_evaluator.py +++ b/unit_tests/test_panoptic_evaluator.py @@ -2,16 +2,17 @@ # coverage run -m unittest # coverage report # coverage html -import unittest import os +import unittest + import numpy as np -from panoptica.panoptic_evaluator import Panoptic_Evaluator from panoptica.instance_approximator import ConnectedComponentsInstanceApproximator -from panoptica.instance_matcher import NaiveThresholdMatching, MaximizeMergeMatching -from panoptica.metrics import _Metric, Metric +from panoptica.instance_matcher import MaximizeMergeMatching, NaiveThresholdMatching +from panoptica.metrics import Metric, _Metric +from panoptica.panoptic_evaluator import Panoptic_Evaluator +from panoptica.panoptic_result import MetricCouldNotBeComputedException, PanopticaResult from panoptica.utils.processing_pair import SemanticPair -from panoptica.panoptic_result import PanopticaResult, MetricCouldNotBeComputedException class Test_Panoptic_Evaluator(unittest.TestCase): From e1f7ba88c4a244e7c99f3e197cafc1652e6ecd2c Mon Sep 17 00:00:00 2001 From: neuronflow Date: Thu, 1 Feb 2024 10:00:40 +0100 Subject: [PATCH 05/29] sorted imports Signed-off-by: neuronflow --- unit_tests/test_datatype.py | 11 ++++++----- unit_tests/test_panoptic_result.py | 11 ++++++----- 2 files changed, 12 insertions(+), 10 deletions(-) diff --git a/unit_tests/test_datatype.py b/unit_tests/test_datatype.py index faffc56..7941989 100644 --- a/unit_tests/test_datatype.py +++ b/unit_tests/test_datatype.py @@ -2,15 +2,16 @@ # coverage run -m unittest # coverage report # coverage html -import unittest import os +import unittest + import numpy as np -from panoptica.panoptic_evaluator import Panoptic_Evaluator from panoptica.instance_approximator import ConnectedComponentsInstanceApproximator -from panoptica.instance_matcher import NaiveThresholdMatching, MaximizeMergeMatching -from panoptica.panoptic_result import PanopticaResult, MetricCouldNotBeComputedException -from panoptica.metrics import _Metric, Metric, Metric, MetricMode +from panoptica.instance_matcher import MaximizeMergeMatching, NaiveThresholdMatching +from panoptica.metrics import Metric, MetricMode, _Metric +from panoptica.panoptic_evaluator import Panoptic_Evaluator +from panoptica.panoptic_result import MetricCouldNotBeComputedException, PanopticaResult from panoptica.utils.edge_case_handling import EdgeCaseHandler, EdgeCaseResult from panoptica.utils.processing_pair import SemanticPair diff --git a/unit_tests/test_panoptic_result.py b/unit_tests/test_panoptic_result.py index 4a87b1d..0491111 100644 --- a/unit_tests/test_panoptic_result.py +++ b/unit_tests/test_panoptic_result.py @@ -2,15 +2,16 @@ # coverage run -m unittest # coverage report # coverage html -import unittest import os +import unittest + import numpy as np -from panoptica.panoptic_evaluator import Panoptic_Evaluator from panoptica.instance_approximator import ConnectedComponentsInstanceApproximator -from panoptica.instance_matcher import NaiveThresholdMatching, MaximizeMergeMatching -from panoptica.panoptic_result import PanopticaResult, MetricCouldNotBeComputedException -from panoptica.metrics import _Metric, Metric, Metric, MetricMode +from panoptica.instance_matcher import MaximizeMergeMatching, NaiveThresholdMatching +from panoptica.metrics import Metric, MetricMode, _Metric +from panoptica.panoptic_evaluator import Panoptic_Evaluator +from panoptica.panoptic_result import MetricCouldNotBeComputedException, PanopticaResult from panoptica.utils.edge_case_handling import EdgeCaseHandler, EdgeCaseResult from panoptica.utils.processing_pair import SemanticPair From 5b2caf19ce96f122463f59ca0bd642d893dd8c3d Mon Sep 17 00:00:00 2001 From: neuronflow Date: Thu, 1 Feb 2024 10:01:44 +0100 Subject: [PATCH 06/29] sorting imports Signed-off-by: neuronflow --- panoptica/_functionals.py | 2 +- panoptica/instance_approximator.py | 14 ++++++++------ panoptica/instance_evaluator.py | 3 ++- panoptica/metrics/cldice.py | 2 +- panoptica/panoptic_evaluator.py | 4 ++-- panoptica/utils/__init__.py | 14 +++++++------- panoptica/utils/edge_case_handling.py | 2 +- panoptica/utils/processing_pair.py | 2 +- 8 files changed, 23 insertions(+), 20 deletions(-) diff --git a/panoptica/_functionals.py b/panoptica/_functionals.py index 071af97..2775c37 100644 --- a/panoptica/_functionals.py +++ b/panoptica/_functionals.py @@ -2,7 +2,7 @@ import numpy as np -from panoptica.metrics import _compute_instance_iou, Metric +from panoptica.metrics import Metric, _compute_instance_iou from panoptica.utils.constants import CCABackend from panoptica.utils.numpy_utils import _get_bbox_nd diff --git a/panoptica/instance_approximator.py b/panoptica/instance_approximator.py index 0a905b5..36845b6 100644 --- a/panoptica/instance_approximator.py +++ b/panoptica/instance_approximator.py @@ -1,13 +1,15 @@ -from abc import abstractmethod, ABC +from abc import ABC, abstractmethod + +import numpy as np + +from panoptica._functionals import CCABackend, _connected_components +from panoptica.timing import measure_time +from panoptica.utils.numpy_utils import _get_smallest_fitting_uint from panoptica.utils.processing_pair import ( + MatchedInstancePair, SemanticPair, UnmatchedInstancePair, - MatchedInstancePair, ) -from panoptica._functionals import _connected_components, CCABackend -from panoptica.utils.numpy_utils import _get_smallest_fitting_uint -from panoptica.timing import measure_time -import numpy as np class InstanceApproximator(ABC): diff --git a/panoptica/instance_evaluator.py b/panoptica/instance_evaluator.py index 028e19f..e405f43 100644 --- a/panoptica/instance_evaluator.py +++ b/panoptica/instance_evaluator.py @@ -1,10 +1,11 @@ from multiprocessing import Pool + import numpy as np +from panoptica.metrics import Metric from panoptica.panoptic_result import PanopticaResult from panoptica.utils import EdgeCaseHandler from panoptica.utils.processing_pair import MatchedInstancePair -from panoptica.metrics import Metric def evaluate_matched_instance( diff --git a/panoptica/metrics/cldice.py b/panoptica/metrics/cldice.py index 3924751..bdbe1fc 100644 --- a/panoptica/metrics/cldice.py +++ b/panoptica/metrics/cldice.py @@ -1,5 +1,5 @@ -from skimage.morphology import skeletonize, skeletonize_3d import numpy as np +from skimage.morphology import skeletonize, skeletonize_3d def cl_score(volume: np.ndarray, skeleton: np.ndarray): diff --git a/panoptica/panoptic_evaluator.py b/panoptica/panoptic_evaluator.py index 1a10b3c..225930e 100644 --- a/panoptica/panoptic_evaluator.py +++ b/panoptica/panoptic_evaluator.py @@ -5,17 +5,17 @@ from panoptica.instance_approximator import InstanceApproximator from panoptica.instance_evaluator import evaluate_matched_instance from panoptica.instance_matcher import InstanceMatchingAlgorithm -from panoptica.metrics import Metric, _Metric, Metric +from panoptica.metrics import Metric, _Metric from panoptica.panoptic_result import PanopticaResult from panoptica.timing import measure_time from panoptica.utils import EdgeCaseHandler +from panoptica.utils.citation_reminder import citation_reminder from panoptica.utils.processing_pair import ( MatchedInstancePair, SemanticPair, UnmatchedInstancePair, _ProcessingPair, ) -from panoptica.utils.citation_reminder import citation_reminder class Panoptic_Evaluator: diff --git a/panoptica/utils/__init__.py b/panoptica/utils/__init__.py index b5b9927..8abe770 100644 --- a/panoptica/utils/__init__.py +++ b/panoptica/utils/__init__.py @@ -1,17 +1,17 @@ +from panoptica.utils.edge_case_handling import ( + EdgeCaseHandler, + EdgeCaseResult, + EdgeCaseZeroTP, +) from panoptica.utils.numpy_utils import ( _count_unique_without_zeros, _unique_without_zeros, ) from panoptica.utils.processing_pair import ( + InstanceLabelMap, + MatchedInstancePair, SemanticPair, UnmatchedInstancePair, - MatchedInstancePair, - InstanceLabelMap, -) -from panoptica.utils.edge_case_handling import ( - EdgeCaseHandler, - EdgeCaseResult, - EdgeCaseZeroTP, ) # from utils.constants import diff --git a/panoptica/utils/edge_case_handling.py b/panoptica/utils/edge_case_handling.py index 33b4c29..c8eec76 100644 --- a/panoptica/utils/edge_case_handling.py +++ b/panoptica/utils/edge_case_handling.py @@ -2,7 +2,7 @@ import numpy as np -from panoptica.metrics import Metric, Metric +from panoptica.metrics import Metric from panoptica.utils.constants import _Enum_Compare, auto diff --git a/panoptica/utils/processing_pair.py b/panoptica/utils/processing_pair.py index bde4a58..df1c659 100644 --- a/panoptica/utils/processing_pair.py +++ b/panoptica/utils/processing_pair.py @@ -3,8 +3,8 @@ import numpy as np from numpy import dtype -from panoptica.utils import _count_unique_without_zeros, _unique_without_zeros from panoptica._functionals import _get_paired_crop +from panoptica.utils import _count_unique_without_zeros, _unique_without_zeros uint_type: type = np.unsignedinteger int_type: type = np.integer From dbf9d0b1c8c9acd0c42fa4c08e7c110c668a0f91 Mon Sep 17 00:00:00 2001 From: "brainless-bot[bot]" <153751247+brainless-bot[bot]@users.noreply.github.com> Date: Thu, 1 Feb 2024 09:03:51 +0000 Subject: [PATCH 07/29] Autoformat with black --- panoptica/metrics/metrics.py | 4 +--- panoptica/panoptic_evaluator.py | 20 +++++++++----------- panoptica/utils/processing_pair.py | 20 ++++++++++++++------ 3 files changed, 24 insertions(+), 20 deletions(-) diff --git a/panoptica/metrics/metrics.py b/panoptica/metrics/metrics.py index c7ce019..57432d7 100644 --- a/panoptica/metrics/metrics.py +++ b/panoptica/metrics/metrics.py @@ -292,9 +292,7 @@ def __init__( self.STD = ( None if self.ALL is None - else empty_list_std - if len(self.ALL) == 0 - else np.std(self.ALL) + else empty_list_std if len(self.ALL) == 0 else np.std(self.ALL) ) def __getitem__(self, mode: MetricMode | str): diff --git a/panoptica/panoptic_evaluator.py b/panoptica/panoptic_evaluator.py index 225930e..2534ad5 100644 --- a/panoptica/panoptic_evaluator.py +++ b/panoptica/panoptic_evaluator.py @@ -21,9 +21,9 @@ class Panoptic_Evaluator: def __init__( self, - expected_input: Type[SemanticPair] - | Type[UnmatchedInstancePair] - | Type[MatchedInstancePair] = MatchedInstancePair, + expected_input: ( + Type[SemanticPair] | Type[UnmatchedInstancePair] | Type[MatchedInstancePair] + ) = MatchedInstancePair, instance_approximator: InstanceApproximator | None = None, instance_matcher: InstanceMatchingAlgorithm | None = None, edge_case_handler: EdgeCaseHandler | None = None, @@ -64,10 +64,9 @@ def __init__( @measure_time def evaluate( self, - processing_pair: SemanticPair - | UnmatchedInstancePair - | MatchedInstancePair - | PanopticaResult, + processing_pair: ( + SemanticPair | UnmatchedInstancePair | MatchedInstancePair | PanopticaResult + ), result_all: bool = True, verbose: bool | None = None, ) -> tuple[PanopticaResult, dict[str, _ProcessingPair]]: @@ -89,10 +88,9 @@ def evaluate( def panoptic_evaluate( - processing_pair: SemanticPair - | UnmatchedInstancePair - | MatchedInstancePair - | PanopticaResult, + processing_pair: ( + SemanticPair | UnmatchedInstancePair | MatchedInstancePair | PanopticaResult + ), instance_approximator: InstanceApproximator | None = None, instance_matcher: InstanceMatchingAlgorithm | None = None, eval_metrics: list[Metric] = [Metric.DSC, Metric.IOU, Metric.ASSD], diff --git a/panoptica/utils/processing_pair.py b/panoptica/utils/processing_pair.py index df1c659..100ff7a 100644 --- a/panoptica/utils/processing_pair.py +++ b/panoptica/utils/processing_pair.py @@ -60,9 +60,13 @@ def crop_data(self, verbose: bool = False): self._prediction_arr = self._prediction_arr[self.crop] self._reference_arr = self._reference_arr[self.crop] - print( - f"-- Cropped from {self.uncropped_shape} to {self._prediction_arr.shape}" - ) if verbose else None + ( + print( + f"-- Cropped from {self.uncropped_shape} to {self._prediction_arr.shape}" + ) + if verbose + else None + ) self.is_cropped = True def uncrop_data(self, verbose: bool = False): @@ -77,9 +81,13 @@ def uncrop_data(self, verbose: bool = False): reference_arr = np.zeros(self.uncropped_shape) reference_arr[self.crop] = self._reference_arr - print( - f"-- Uncropped from {self._reference_arr.shape} to {self.uncropped_shape}" - ) if verbose else None + ( + print( + f"-- Uncropped from {self._reference_arr.shape} to {self.uncropped_shape}" + ) + if verbose + else None + ) self._reference_arr = reference_arr self.is_cropped = False From a1951c476e8e3076ea279231050b86e7b3266689 Mon Sep 17 00:00:00 2001 From: neuronflow Date: Thu, 1 Feb 2024 10:10:54 +0100 Subject: [PATCH 08/29] clean unused imports Signed-off-by: neuronflow --- unit_tests/test_panoptic_evaluator.py | 6 +++--- unit_tests/test_panoptic_result.py | 6 +----- 2 files changed, 4 insertions(+), 8 deletions(-) diff --git a/unit_tests/test_panoptic_evaluator.py b/unit_tests/test_panoptic_evaluator.py index e69b2b3..b44a6cd 100644 --- a/unit_tests/test_panoptic_evaluator.py +++ b/unit_tests/test_panoptic_evaluator.py @@ -9,9 +9,9 @@ from panoptica.instance_approximator import ConnectedComponentsInstanceApproximator from panoptica.instance_matcher import MaximizeMergeMatching, NaiveThresholdMatching -from panoptica.metrics import Metric, _Metric +from panoptica.metrics import Metric from panoptica.panoptic_evaluator import Panoptic_Evaluator -from panoptica.panoptic_result import MetricCouldNotBeComputedException, PanopticaResult +from panoptica.panoptic_result import MetricCouldNotBeComputedException from panoptica.utils.processing_pair import SemanticPair @@ -74,7 +74,7 @@ def test_simple_evaluation_DSC_partial(self): expected_input=SemanticPair, instance_approximator=ConnectedComponentsInstanceApproximator(), instance_matcher=NaiveThresholdMatching(matching_metric=Metric.DSC), - eval_metrics=[Metric.DSC], + eval_metrics=[Metric.ASSD], ) result, debug_data = evaluator.evaluate(sample) diff --git a/unit_tests/test_panoptic_result.py b/unit_tests/test_panoptic_result.py index 0491111..96c5c64 100644 --- a/unit_tests/test_panoptic_result.py +++ b/unit_tests/test_panoptic_result.py @@ -7,13 +7,9 @@ import numpy as np -from panoptica.instance_approximator import ConnectedComponentsInstanceApproximator -from panoptica.instance_matcher import MaximizeMergeMatching, NaiveThresholdMatching -from panoptica.metrics import Metric, MetricMode, _Metric -from panoptica.panoptic_evaluator import Panoptic_Evaluator +from panoptica.metrics import Metric from panoptica.panoptic_result import MetricCouldNotBeComputedException, PanopticaResult from panoptica.utils.edge_case_handling import EdgeCaseHandler, EdgeCaseResult -from panoptica.utils.processing_pair import SemanticPair class Test_Panoptic_Evaluator(unittest.TestCase): From 24591442e943c54f237eee8dd5b970052cdd818e Mon Sep 17 00:00:00 2001 From: neuronflow Date: Thu, 1 Feb 2024 10:11:29 +0100 Subject: [PATCH 09/29] cleaning unused imports Signed-off-by: neuronflow --- unit_tests/test_datatype.py | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/unit_tests/test_datatype.py b/unit_tests/test_datatype.py index 7941989..d9f3bc3 100644 --- a/unit_tests/test_datatype.py +++ b/unit_tests/test_datatype.py @@ -5,15 +5,7 @@ import os import unittest -import numpy as np - -from panoptica.instance_approximator import ConnectedComponentsInstanceApproximator -from panoptica.instance_matcher import MaximizeMergeMatching, NaiveThresholdMatching -from panoptica.metrics import Metric, MetricMode, _Metric -from panoptica.panoptic_evaluator import Panoptic_Evaluator -from panoptica.panoptic_result import MetricCouldNotBeComputedException, PanopticaResult -from panoptica.utils.edge_case_handling import EdgeCaseHandler, EdgeCaseResult -from panoptica.utils.processing_pair import SemanticPair +from panoptica.metrics import Metric class Test_Panoptic_Evaluator(unittest.TestCase): From abbfbbef5faa13d12ffd630c7f7d917afdf5fd84 Mon Sep 17 00:00:00 2001 From: neuronflow Date: Thu, 1 Feb 2024 10:12:11 +0100 Subject: [PATCH 10/29] back to DSC Signed-off-by: neuronflow --- unit_tests/test_panoptic_evaluator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/unit_tests/test_panoptic_evaluator.py b/unit_tests/test_panoptic_evaluator.py index b44a6cd..00245c0 100644 --- a/unit_tests/test_panoptic_evaluator.py +++ b/unit_tests/test_panoptic_evaluator.py @@ -74,7 +74,7 @@ def test_simple_evaluation_DSC_partial(self): expected_input=SemanticPair, instance_approximator=ConnectedComponentsInstanceApproximator(), instance_matcher=NaiveThresholdMatching(matching_metric=Metric.DSC), - eval_metrics=[Metric.ASSD], + eval_metrics=[Metric.DSC], ) result, debug_data = evaluator.evaluate(sample) From 2b22ce0ea3e4e22fea73aa061ad0dbd9b19a9f2e Mon Sep 17 00:00:00 2001 From: neuronflow Date: Thu, 1 Feb 2024 10:13:45 +0100 Subject: [PATCH 11/29] clean unused imports Signed-off-by: neuronflow --- examples/example_spine_instance.py | 2 +- examples/example_spine_semantic.py | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/examples/example_spine_instance.py b/examples/example_spine_instance.py index dc2b639..241407e 100644 --- a/examples/example_spine_instance.py +++ b/examples/example_spine_instance.py @@ -4,7 +4,7 @@ from auxiliary.turbopath import turbopath from panoptica import MatchedInstancePair, Panoptic_Evaluator -from panoptica.metrics import Metric, MetricMode +from panoptica.metrics import Metric directory = turbopath(__file__).parent diff --git a/examples/example_spine_semantic.py b/examples/example_spine_semantic.py index 19ae99b..8bf993d 100644 --- a/examples/example_spine_semantic.py +++ b/examples/example_spine_semantic.py @@ -9,7 +9,6 @@ Panoptic_Evaluator, SemanticPair, ) -from panoptica.metrics import Metric directory = turbopath(__file__).parent From 99a7811884823ef49e65ec0e6e4990553bf90a93 Mon Sep 17 00:00:00 2001 From: neuronflow Date: Thu, 1 Feb 2024 10:16:04 +0100 Subject: [PATCH 12/29] fix: scipy import warnings Signed-off-by: neuronflow --- panoptica/metrics/assd.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/panoptica/metrics/assd.py b/panoptica/metrics/assd.py index a08bba2..7119185 100644 --- a/panoptica/metrics/assd.py +++ b/panoptica/metrics/assd.py @@ -1,7 +1,6 @@ import numpy as np -from scipy.ndimage import _ni_support +from scipy.ndimage import _ni_support, binary_erosion, generate_binary_structure from scipy.ndimage._nd_image import euclidean_feature_transform -from scipy.ndimage.morphology import binary_erosion, generate_binary_structure def _average_symmetric_surface_distance( From 21c7829fb3828c454200f47b53fd59736a60b70a Mon Sep 17 00:00:00 2001 From: neuronflow Date: Thu, 1 Feb 2024 10:16:55 +0100 Subject: [PATCH 13/29] clean Signed-off-by: neuronflow --- panoptica/instance_approximator.py | 1 - 1 file changed, 1 deletion(-) diff --git a/panoptica/instance_approximator.py b/panoptica/instance_approximator.py index 36845b6..9471654 100644 --- a/panoptica/instance_approximator.py +++ b/panoptica/instance_approximator.py @@ -3,7 +3,6 @@ import numpy as np from panoptica._functionals import CCABackend, _connected_components -from panoptica.timing import measure_time from panoptica.utils.numpy_utils import _get_smallest_fitting_uint from panoptica.utils.processing_pair import ( MatchedInstancePair, From 3dffd94e609c4cd18a22ae1c0ec9bad56d87dd6f Mon Sep 17 00:00:00 2001 From: neuronflow Date: Thu, 1 Feb 2024 15:14:49 +0100 Subject: [PATCH 14/29] stop accessing protected variables Signed-off-by: neuronflow --- panoptica/instance_evaluator.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/panoptica/instance_evaluator.py b/panoptica/instance_evaluator.py index e405f43..a379bf3 100644 --- a/panoptica/instance_evaluator.py +++ b/panoptica/instance_evaluator.py @@ -43,8 +43,8 @@ def evaluate_matched_instance( score_dict: dict[Metric, list[float]] = {m: [] for m in eval_metrics} reference_arr, prediction_arr = ( - matched_instance_pair._reference_arr, - matched_instance_pair._prediction_arr, + matched_instance_pair.reference_arr, + matched_instance_pair.prediction_arr, ) ref_matched_labels = matched_instance_pair.matched_instances From 495f729d82eee92aa2f8f475e295447abf2a779a Mon Sep 17 00:00:00 2001 From: neuronflow Date: Thu, 1 Feb 2024 15:15:58 +0100 Subject: [PATCH 15/29] clean Signed-off-by: neuronflow --- panoptica/utils/edge_case_handling.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/panoptica/utils/edge_case_handling.py b/panoptica/utils/edge_case_handling.py index c8eec76..c1f93a2 100644 --- a/panoptica/utils/edge_case_handling.py +++ b/panoptica/utils/edge_case_handling.py @@ -1,5 +1,3 @@ -from typing import Any - import numpy as np from panoptica.metrics import Metric From 1f8b7f1c7cc1c7b261b683248bba19687f11cbbb Mon Sep 17 00:00:00 2001 From: neuronflow Date: Thu, 1 Feb 2024 15:22:02 +0100 Subject: [PATCH 16/29] add todo Signed-off-by: neuronflow --- panoptica/_functionals.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/panoptica/_functionals.py b/panoptica/_functionals.py index 2775c37..c062f58 100644 --- a/panoptica/_functionals.py +++ b/panoptica/_functionals.py @@ -62,6 +62,7 @@ def _calc_matching_metric_of_overlapping_labels( ) ] with Pool() as pool: + # TODO check this protected access mm_values = pool.starmap(matching_metric.value._metric_function, instance_pairs) mm_pairs = [ @@ -79,7 +80,7 @@ def _calc_iou_of_overlapping_labels( prediction_arr: np.ndarray, reference_arr: np.ndarray, ref_labels: tuple[int, ...], - pred_labels: tuple[int, ...], + **kwargs, ) -> list[tuple[float, tuple[int, int]]]: """Calculates the IOU for all overlapping labels (fast!) @@ -156,7 +157,10 @@ def _calc_iou_matrix( return iou_matrix -def _map_labels(arr: np.ndarray, label_map: dict[np.integer, np.integer]) -> np.ndarray: +def _map_labels( + arr: np.ndarray, + label_map: dict[np.integer, np.integer], +) -> np.ndarray: """ Maps labels in the given array according to the label_map dictionary. @@ -212,7 +216,9 @@ def _connected_components( def _get_paired_crop( - prediction_arr: np.ndarray, reference_arr: np.ndarray, px_pad: int = 2 + prediction_arr: np.ndarray, + reference_arr: np.ndarray, + px_pad: int = 2, ): assert prediction_arr.shape == reference_arr.shape From f92655832cfbdc92079774a5eece03e4c521325d Mon Sep 17 00:00:00 2001 From: neuronflow Date: Thu, 1 Feb 2024 15:22:09 +0100 Subject: [PATCH 17/29] format Signed-off-by: neuronflow --- panoptica/metrics/metrics.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/panoptica/metrics/metrics.py b/panoptica/metrics/metrics.py index 57432d7..c7ce019 100644 --- a/panoptica/metrics/metrics.py +++ b/panoptica/metrics/metrics.py @@ -292,7 +292,9 @@ def __init__( self.STD = ( None if self.ALL is None - else empty_list_std if len(self.ALL) == 0 else np.std(self.ALL) + else empty_list_std + if len(self.ALL) == 0 + else np.std(self.ALL) ) def __getitem__(self, mode: MetricMode | str): From 9a90686cf4f15dd82ecf3e90b810a3e4e7958254 Mon Sep 17 00:00:00 2001 From: "brainless-bot[bot]" <153751247+brainless-bot[bot]@users.noreply.github.com> Date: Thu, 1 Feb 2024 14:24:45 +0000 Subject: [PATCH 18/29] Autoformat with black --- panoptica/metrics/metrics.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/panoptica/metrics/metrics.py b/panoptica/metrics/metrics.py index c7ce019..57432d7 100644 --- a/panoptica/metrics/metrics.py +++ b/panoptica/metrics/metrics.py @@ -292,9 +292,7 @@ def __init__( self.STD = ( None if self.ALL is None - else empty_list_std - if len(self.ALL) == 0 - else np.std(self.ALL) + else empty_list_std if len(self.ALL) == 0 else np.std(self.ALL) ) def __getitem__(self, mode: MetricMode | str): From bb7d42254e944033bc23f94a66a7edf380e76998 Mon Sep 17 00:00:00 2001 From: neuronflow Date: Thu, 1 Feb 2024 15:28:25 +0100 Subject: [PATCH 19/29] more todos Signed-off-by: neuronflow --- panoptica/instance_matcher.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/panoptica/instance_matcher.py b/panoptica/instance_matcher.py index 32a927c..0647448 100644 --- a/panoptica/instance_matcher.py +++ b/panoptica/instance_matcher.py @@ -6,7 +6,7 @@ _calc_matching_metric_of_overlapping_labels, _map_labels, ) -from panoptica.metrics import Metric, _Metric +from panoptica.metrics import Metric from panoptica.utils.processing_pair import ( InstanceLabelMap, MatchedInstancePair, @@ -98,6 +98,7 @@ def map_instance_labels( >>> labelmap = [([1, 2], [3, 4]), ([5], [6])] >>> result = map_instance_labels(unmatched_instance_pair, labelmap) """ + # TODO check this protected accesses prediction_arr = processing_pair._prediction_arr ref_labels = processing_pair._ref_labels @@ -185,12 +186,14 @@ def _match_instances( Returns: Instance_Label_Map: The result of the instance matching. """ + # TODO check this protected accesses ref_labels = unmatched_instance_pair._ref_labels # Initialize variables for True Positives (tp) and False Positives (fp) labelmap = InstanceLabelMap() pred_arr, ref_arr = ( + # TODO check this protected accesses unmatched_instance_pair._prediction_arr, unmatched_instance_pair._reference_arr, ) @@ -259,6 +262,7 @@ def _match_instances( Returns: Instance_Label_Map: The result of the instance matching. """ + # TODO check this protected accesses ref_labels = unmatched_instance_pair._ref_labels # pred_labels = unmatched_instance_pair._pred_labels @@ -267,11 +271,15 @@ def _match_instances( score_ref: dict[int, float] = {} pred_arr, ref_arr = ( + # TODO check this protected accesses unmatched_instance_pair._prediction_arr, unmatched_instance_pair._reference_arr, ) mm_pairs = _calc_matching_metric_of_overlapping_labels( - pred_arr, ref_arr, ref_labels, matching_metric=self.matching_metric + prediction_arr=pred_arr, + reference_arr=ref_arr, + ref_labels=ref_labels, + matching_metric=self.matching_metric, ) # Loop through matched instances to compute PQ components From ac55a58549a3346448399acb87f711bd3c596c00 Mon Sep 17 00:00:00 2001 From: neuronflow Date: Thu, 1 Feb 2024 15:29:00 +0100 Subject: [PATCH 20/29] clean Signed-off-by: neuronflow --- panoptica/panoptic_evaluator.py | 1 - 1 file changed, 1 deletion(-) diff --git a/panoptica/panoptic_evaluator.py b/panoptica/panoptic_evaluator.py index 2534ad5..0119fbc 100644 --- a/panoptica/panoptic_evaluator.py +++ b/panoptica/panoptic_evaluator.py @@ -1,4 +1,3 @@ -from abc import ABC, abstractmethod from time import perf_counter from typing import Type From 3f55da067107053cffdbd236a9374012c1b3383d Mon Sep 17 00:00:00 2001 From: neuronflow Date: Thu, 1 Feb 2024 15:30:09 +0100 Subject: [PATCH 21/29] add todo Signed-off-by: neuronflow --- panoptica/panoptic_evaluator.py | 1 + 1 file changed, 1 insertion(+) diff --git a/panoptica/panoptic_evaluator.py b/panoptica/panoptic_evaluator.py index 0119fbc..26b95b3 100644 --- a/panoptica/panoptic_evaluator.py +++ b/panoptica/panoptic_evaluator.py @@ -244,6 +244,7 @@ def _handle_zero_instances_cases( is_edge_case = True elif n_prediction_instance == 0: # All predictions are missing, only false negatives + # TODO what is going on here! n_reference_instance = n_reference_instance n_prediction_instance = 0 is_edge_case = True From db229c22e9374f7527c126bcd3c4b78a288593e5 Mon Sep 17 00:00:00 2001 From: neuronflow Date: Thu, 1 Feb 2024 15:30:55 +0100 Subject: [PATCH 22/29] add docstring Signed-off-by: neuronflow --- panoptica/timing.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/panoptica/timing.py b/panoptica/timing.py index 3ed8698..dbee75a 100644 --- a/panoptica/timing.py +++ b/panoptica/timing.py @@ -2,6 +2,8 @@ def measure_time(func): + """Decorator to measure the time it takes to execute a function.""" + def wrapper(*args, **kwargs): start_time = time.time() result = func(*args, **kwargs) From fec57344adf4dd288b93f40c4d402a0949e576f0 Mon Sep 17 00:00:00 2001 From: neuronflow Date: Thu, 1 Feb 2024 15:36:05 +0100 Subject: [PATCH 23/29] add todo Signed-off-by: neuronflow --- panoptica/metrics/assd.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/panoptica/metrics/assd.py b/panoptica/metrics/assd.py index 7119185..a7608f5 100644 --- a/panoptica/metrics/assd.py +++ b/panoptica/metrics/assd.py @@ -13,10 +13,17 @@ def _average_symmetric_surface_distance( assd = np.mean( ( _average_surface_distance( - prediction, reference, voxelspacing, connectivity + # TODO is this intended? + reference=prediction, + prediction=reference, + voxelspacing=voxelspacing, + connectivity=connectivity, ), _average_surface_distance( - reference, prediction, voxelspacing, connectivity + reference=reference, + prediction=prediction, + voxelspacing=voxelspacing, + connectivity=connectivity, ), ) ) @@ -37,6 +44,7 @@ def __surface_distances(reference, prediction, voxelspacing=None, connectivity=1 prediction = np.atleast_1d(prediction.astype(bool)) reference = np.atleast_1d(reference.astype(bool)) if voxelspacing is not None: + # TODO check protected access voxelspacing = _ni_support._normalize_sequence(voxelspacing, prediction.ndim) voxelspacing = np.asarray(voxelspacing, dtype=np.float64) if not voxelspacing.flags.contiguous: From bd0bd94f6d470bccd06bfbaead9532a51ec9c792 Mon Sep 17 00:00:00 2001 From: neuronflow Date: Thu, 1 Feb 2024 15:36:10 +0100 Subject: [PATCH 24/29] docs Signed-off-by: neuronflow --- panoptica/utils/citation_reminder.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/panoptica/utils/citation_reminder.py b/panoptica/utils/citation_reminder.py index f4fedb7..92cd262 100644 --- a/panoptica/utils/citation_reminder.py +++ b/panoptica/utils/citation_reminder.py @@ -6,6 +6,8 @@ def citation_reminder(func): + """Decorator to remind users to cite panoptica.""" + def wrapper(*args, **kwargs): if os.environ.get("PANOPTICA_CITATION_REMINDER", "true").lower() == "true": console = Console() From b7710cb80b7e3f7f5c6087918f1e901fb0e55c0e Mon Sep 17 00:00:00 2001 From: neuronflow Date: Thu, 1 Feb 2024 15:38:01 +0100 Subject: [PATCH 25/29] clean Signed-off-by: neuronflow --- panoptica/utils/processing_pair.py | 1 - 1 file changed, 1 deletion(-) diff --git a/panoptica/utils/processing_pair.py b/panoptica/utils/processing_pair.py index 100ff7a..5ed1a7c 100644 --- a/panoptica/utils/processing_pair.py +++ b/panoptica/utils/processing_pair.py @@ -1,7 +1,6 @@ from abc import ABC import numpy as np -from numpy import dtype from panoptica._functionals import _get_paired_crop from panoptica.utils import _count_unique_without_zeros, _unique_without_zeros From 23c416f54d6230c9a5ddaef2e22b84775e58f504 Mon Sep 17 00:00:00 2001 From: neuronflow Date: Thu, 1 Feb 2024 16:53:00 +0100 Subject: [PATCH 26/29] docs Signed-off-by: neuronflow --- panoptica/metrics/assd.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/panoptica/metrics/assd.py b/panoptica/metrics/assd.py index a7608f5..4e6aa26 100644 --- a/panoptica/metrics/assd.py +++ b/panoptica/metrics/assd.py @@ -10,10 +10,10 @@ def _average_symmetric_surface_distance( connectivity=1, *args, ) -> float: + """ASSD is computed by computing the average of the bidrectionally computed ASD.""" assd = np.mean( ( _average_surface_distance( - # TODO is this intended? reference=prediction, prediction=reference, voxelspacing=voxelspacing, From 0b7f27b4d2f80a6ac7e5caffe32ffe39ad8a7207 Mon Sep 17 00:00:00 2001 From: neuronflow Date: Thu, 1 Feb 2024 16:53:19 +0100 Subject: [PATCH 27/29] rename to iarray to avoid name collision Signed-off-by: neuronflow --- panoptica/metrics/assd.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/panoptica/metrics/assd.py b/panoptica/metrics/assd.py index 4e6aa26..1e8e18f 100644 --- a/panoptica/metrics/assd.py +++ b/panoptica/metrics/assd.py @@ -77,7 +77,7 @@ def __surface_distances(reference, prediction, voxelspacing=None, connectivity=1 def _distance_transform_edt( - input: np.ndarray, + input_array: np.ndarray, sampling=None, return_distances=True, return_indices=False, @@ -90,12 +90,12 @@ def _distance_transform_edt( # if not sampling.flags.contiguous: # sampling = sampling.copy() - ft = np.zeros((input.ndim,) + input.shape, dtype=np.int32) + ft = np.zeros((input_array.ndim,) + input_array.shape, dtype=np.int32) - euclidean_feature_transform(input, sampling, ft) + euclidean_feature_transform(input_array, sampling, ft) # if requested, calculate the distance transform if return_distances: - dt = ft - np.indices(input.shape, dtype=ft.dtype) + dt = ft - np.indices(input_array.shape, dtype=ft.dtype) dt = dt.astype(np.float64) # if sampling is not None: # for ii in range(len(sampling)): From 3c7e2f9fe0eceed34b5f071665960c445b5d5508 Mon Sep 17 00:00:00 2001 From: iback Date: Fri, 9 Feb 2024 13:37:51 +0000 Subject: [PATCH 28/29] fixed cyclic import, a lot of deprecated protected accesses --- panoptica/_functionals.py | 39 ++++++++-------------- panoptica/instance_approximator.py | 33 +++++------------- panoptica/instance_evaluator.py | 18 +++------- panoptica/instance_matcher.py | 48 +++++++++------------------ panoptica/metrics/assd.py | 10 ++---- panoptica/metrics/metrics.py | 46 ++++++------------------- panoptica/panoptic_evaluator.py | 33 +++++------------- panoptica/utils/__init__.py | 10 +++--- panoptica/utils/edge_case_handling.py | 37 +++++---------------- unit_tests/test_datatype.py | 41 +++++++++++++++++++++-- 10 files changed, 115 insertions(+), 200 deletions(-) diff --git a/panoptica/_functionals.py b/panoptica/_functionals.py index c062f58..44639e6 100644 --- a/panoptica/_functionals.py +++ b/panoptica/_functionals.py @@ -1,11 +1,15 @@ +from typing import TYPE_CHECKING from multiprocessing import Pool import numpy as np -from panoptica.metrics import Metric, _compute_instance_iou +from panoptica.metrics.iou import _compute_instance_iou from panoptica.utils.constants import CCABackend from panoptica.utils.numpy_utils import _get_bbox_nd +if TYPE_CHECKING: + from panoptica.metrics import Metric + def _calc_overlapping_labels( prediction_arr: np.ndarray, @@ -30,18 +34,14 @@ def _calc_overlapping_labels( # instance_pairs = [(reference_arr, prediction_arr, i, j) for i, j in overlapping_indices] # (ref, pred) - return [ - (int(i % (max_ref)), int(i // (max_ref))) - for i in np.unique(overlap_arr) - if i > max_ref - ] + return [(int(i % (max_ref)), int(i // (max_ref))) for i in np.unique(overlap_arr) if i > max_ref] def _calc_matching_metric_of_overlapping_labels( prediction_arr: np.ndarray, reference_arr: np.ndarray, ref_labels: tuple[int, ...], - matching_metric: Metric, + matching_metric: "Metric", ) -> list[tuple[float, tuple[int, int]]]: """Calculates the MatchingMetric for all overlapping labels (fast!) @@ -54,7 +54,7 @@ def _calc_matching_metric_of_overlapping_labels( list[tuple[float, tuple[int, int]]]: List of pairs in style: (iou, (ref_label, pred_label)) """ instance_pairs = [ - (reference_arr == i[0], prediction_arr == i[1], i[0], i[1]) + (reference_arr, prediction_arr, i[0], i[1]) for i in _calc_overlapping_labels( prediction_arr=prediction_arr, reference_arr=reference_arr, @@ -62,16 +62,10 @@ def _calc_matching_metric_of_overlapping_labels( ) ] with Pool() as pool: - # TODO check this protected access - mm_values = pool.starmap(matching_metric.value._metric_function, instance_pairs) + mm_values = pool.starmap(matching_metric.value, instance_pairs) - mm_pairs = [ - (i, (instance_pairs[idx][2], instance_pairs[idx][3])) - for idx, i in enumerate(mm_values) - ] - mm_pairs = sorted( - mm_pairs, key=lambda x: x[0], reverse=not matching_metric.decreasing - ) + mm_pairs = [(i, (instance_pairs[idx][2], instance_pairs[idx][3])) for idx, i in enumerate(mm_values)] + mm_pairs = sorted(mm_pairs, key=lambda x: x[0], reverse=not matching_metric.decreasing) return mm_pairs @@ -104,10 +98,7 @@ def _calc_iou_of_overlapping_labels( with Pool() as pool: iou_values = pool.starmap(_compute_instance_iou, instance_pairs) - iou_pairs = [ - (i, (instance_pairs[idx][2], instance_pairs[idx][3])) - for idx, i in enumerate(iou_values) - ] + iou_pairs = [(i, (instance_pairs[idx][2], instance_pairs[idx][3])) for idx, i in enumerate(iou_values)] iou_pairs = sorted(iou_pairs, key=lambda x: x[0], reverse=True) return iou_pairs @@ -143,11 +134,7 @@ def _calc_iou_matrix( # Create a pool of worker processes to parallelize the computation with Pool() as pool: # # Generate all possible pairs of instance indices for IoU computation - instance_pairs = [ - (reference_arr, prediction_arr, ref_idx, pred_idx) - for ref_idx in ref_labels - for pred_idx in pred_labels - ] + instance_pairs = [(reference_arr, prediction_arr, ref_idx, pred_idx) for ref_idx in ref_labels for pred_idx in pred_labels] # Calculate IoU for all instance pairs in parallel using starmap iou_values = pool.starmap(_compute_instance_iou, instance_pairs) diff --git a/panoptica/instance_approximator.py b/panoptica/instance_approximator.py index 9471654..b5de46c 100644 --- a/panoptica/instance_approximator.py +++ b/panoptica/instance_approximator.py @@ -2,7 +2,8 @@ import numpy as np -from panoptica._functionals import CCABackend, _connected_components +from panoptica.utils.constants import CCABackend +from panoptica._functionals import _connected_components from panoptica.utils.numpy_utils import _get_smallest_fitting_uint from panoptica.utils.processing_pair import ( MatchedInstancePair, @@ -40,9 +41,7 @@ class InstanceApproximator(ABC): """ @abstractmethod - def _approximate_instances( - self, semantic_pair: SemanticPair, **kwargs - ) -> UnmatchedInstancePair | MatchedInstancePair: + def _approximate_instances(self, semantic_pair: SemanticPair, **kwargs) -> UnmatchedInstancePair | MatchedInstancePair: """ Abstract method to be implemented by subclasses for instance approximation. @@ -73,19 +72,11 @@ def approximate_instances( """ # Check validity pred_labels, ref_labels = semantic_pair._pred_labels, semantic_pair._ref_labels - pred_label_range = ( - (np.min(pred_labels), np.max(pred_labels)) - if len(pred_labels) > 0 - else (0, 0) - ) - ref_label_range = ( - (np.min(ref_labels), np.max(ref_labels)) if len(ref_labels) > 0 else (0, 0) - ) + pred_label_range = (np.min(pred_labels), np.max(pred_labels)) if len(pred_labels) > 0 else (0, 0) + ref_label_range = (np.min(ref_labels), np.max(ref_labels)) if len(ref_labels) > 0 else (0, 0) # min_value = min(np.min(pred_label_range[0]), np.min(ref_label_range[0])) - assert ( - min_value >= 0 - ), "There are negative values in the semantic maps. This is not allowed!" + assert min_value >= 0, "There are negative values in the semantic maps. This is not allowed!" # Set dtype to smalles fitting uint max_value = max(np.max(pred_label_range[1]), np.max(ref_label_range[1])) dtype = _get_smallest_fitting_uint(max_value) @@ -125,9 +116,7 @@ def __init__(self, cca_backend: CCABackend | None = None) -> None: """ self.cca_backend = cca_backend - def _approximate_instances( - self, semantic_pair: SemanticPair, **kwargs - ) -> UnmatchedInstancePair: + def _approximate_instances(self, semantic_pair: SemanticPair, **kwargs) -> UnmatchedInstancePair: """ Approximate instances using the connected components algorithm. @@ -140,9 +129,7 @@ def _approximate_instances( """ cca_backend = self.cca_backend if self.cca_backend is None: - cca_backend = ( - CCABackend.cc3d if semantic_pair.n_dim >= 3 else CCABackend.scipy - ) + cca_backend = CCABackend.cc3d if semantic_pair.n_dim >= 3 else CCABackend.scipy assert cca_backend is not None empty_prediction = len(semantic_pair._pred_labels) == 0 @@ -153,9 +140,7 @@ def _approximate_instances( else (semantic_pair._prediction_arr, 0) ) reference_arr, n_reference_instance = ( - _connected_components(semantic_pair._reference_arr, cca_backend) - if not empty_reference - else (semantic_pair._reference_arr, 0) + _connected_components(semantic_pair._reference_arr, cca_backend) if not empty_reference else (semantic_pair._reference_arr, 0) ) return UnmatchedInstancePair( prediction_arr=prediction_arr, diff --git a/panoptica/instance_evaluator.py b/panoptica/instance_evaluator.py index a379bf3..c385b2b 100644 --- a/panoptica/instance_evaluator.py +++ b/panoptica/instance_evaluator.py @@ -34,9 +34,7 @@ def evaluate_matched_instance( if edge_case_handler is None: edge_case_handler = EdgeCaseHandler() if decision_metric is not None: - assert decision_metric.name in [ - v.name for v in eval_metrics - ], "decision metric not contained in eval_metrics" + assert decision_metric.name in [v.name for v in eval_metrics], "decision metric not contained in eval_metrics" assert decision_threshold is not None, "decision metric set but no threshold" # Initialize variables for True Positives (tp) tp = len(matched_instance_pair.matched_instances) @@ -48,21 +46,13 @@ def evaluate_matched_instance( ) ref_matched_labels = matched_instance_pair.matched_instances - instance_pairs = [ - (reference_arr, prediction_arr, ref_idx, eval_metrics) - for ref_idx in ref_matched_labels - ] + instance_pairs = [(reference_arr, prediction_arr, ref_idx, eval_metrics) for ref_idx in ref_matched_labels] with Pool() as pool: - metric_dicts: list[dict[Metric, float]] = pool.starmap( - _evaluate_instance, instance_pairs - ) + metric_dicts: list[dict[Metric, float]] = pool.starmap(_evaluate_instance, instance_pairs) for metric_dict in metric_dicts: if decision_metric is None or ( - decision_threshold is not None - and decision_metric.score_beats_threshold( - metric_dict[decision_metric], decision_threshold - ) + decision_threshold is not None and decision_metric.score_beats_threshold(metric_dict[decision_metric], decision_threshold) ): for k, v in metric_dict.items(): score_dict[k].append(v) diff --git a/panoptica/instance_matcher.py b/panoptica/instance_matcher.py index 0647448..4273fd5 100644 --- a/panoptica/instance_matcher.py +++ b/panoptica/instance_matcher.py @@ -80,9 +80,7 @@ def match_instances( return map_instance_labels(unmatched_instance_pair.copy(), instance_labelmap) -def map_instance_labels( - processing_pair: UnmatchedInstancePair, labelmap: InstanceLabelMap -) -> MatchedInstancePair: +def map_instance_labels(processing_pair: UnmatchedInstancePair, labelmap: InstanceLabelMap) -> MatchedInstancePair: """ Map instance labels based on the provided labelmap and create a MatchedInstancePair. @@ -98,11 +96,10 @@ def map_instance_labels( >>> labelmap = [([1, 2], [3, 4]), ([5], [6])] >>> result = map_instance_labels(unmatched_instance_pair, labelmap) """ - # TODO check this protected accesses - prediction_arr = processing_pair._prediction_arr + prediction_arr = processing_pair.prediction_arr - ref_labels = processing_pair._ref_labels - pred_labels = processing_pair._pred_labels + ref_labels = processing_pair.ref_labels + pred_labels = processing_pair.pred_labels ref_matched_labels = [] label_counter = int(max(ref_labels) + 1) @@ -186,31 +183,22 @@ def _match_instances( Returns: Instance_Label_Map: The result of the instance matching. """ - # TODO check this protected accesses - ref_labels = unmatched_instance_pair._ref_labels + ref_labels = unmatched_instance_pair.ref_labels # Initialize variables for True Positives (tp) and False Positives (fp) labelmap = InstanceLabelMap() pred_arr, ref_arr = ( - # TODO check this protected accesses - unmatched_instance_pair._prediction_arr, - unmatched_instance_pair._reference_arr, - ) - mm_pairs = _calc_matching_metric_of_overlapping_labels( - pred_arr, ref_arr, ref_labels, matching_metric=self.matching_metric + unmatched_instance_pair.prediction_arr, + unmatched_instance_pair.reference_arr, ) + mm_pairs = _calc_matching_metric_of_overlapping_labels(pred_arr, ref_arr, ref_labels, matching_metric=self.matching_metric) # Loop through matched instances to compute PQ components for matching_score, (ref_label, pred_label) in mm_pairs: - if ( - labelmap.contains_or(pred_label, ref_label) - and not self.allow_many_to_one - ): + if labelmap.contains_or(pred_label, ref_label) and not self.allow_many_to_one: continue # -> doesnt make speed difference - if self.matching_metric.score_beats_threshold( - matching_score, self.matching_threshold - ): + if self.matching_metric.score_beats_threshold(matching_score, self.matching_threshold): # Match found, increment true positive count and collect IoU and Dice values labelmap.add_labelmap_entry(pred_label, ref_label) # map label ref_idx to pred_idx @@ -262,8 +250,7 @@ def _match_instances( Returns: Instance_Label_Map: The result of the instance matching. """ - # TODO check this protected accesses - ref_labels = unmatched_instance_pair._ref_labels + ref_labels = unmatched_instance_pair.ref_labels # pred_labels = unmatched_instance_pair._pred_labels # Initialize variables for True Positives (tp) and False Positives (fp) @@ -271,9 +258,8 @@ def _match_instances( score_ref: dict[int, float] = {} pred_arr, ref_arr = ( - # TODO check this protected accesses - unmatched_instance_pair._prediction_arr, - unmatched_instance_pair._reference_arr, + unmatched_instance_pair.prediction_arr, + unmatched_instance_pair.reference_arr, ) mm_pairs = _calc_matching_metric_of_overlapping_labels( prediction_arr=pred_arr, @@ -289,15 +275,11 @@ def _match_instances( continue if labelmap.contains_ref(ref_label): pred_labels_ = labelmap.get_pred_labels_matched_to_ref(ref_label) - new_score = self.new_combination_score( - pred_labels_, pred_label, ref_label, unmatched_instance_pair - ) + new_score = self.new_combination_score(pred_labels_, pred_label, ref_label, unmatched_instance_pair) if new_score > score_ref[ref_label]: labelmap.add_labelmap_entry(pred_label, ref_label) score_ref[ref_label] = new_score - elif self.matching_metric.score_beats_threshold( - matching_score, self.matching_threshold - ): + elif self.matching_metric.score_beats_threshold(matching_score, self.matching_threshold): # Match found, increment true positive count and collect IoU and Dice values labelmap.add_labelmap_entry(pred_label, ref_label) score_ref[ref_label] = matching_score diff --git a/panoptica/metrics/assd.py b/panoptica/metrics/assd.py index 1e8e18f..a73a644 100644 --- a/panoptica/metrics/assd.py +++ b/panoptica/metrics/assd.py @@ -44,7 +44,7 @@ def __surface_distances(reference, prediction, voxelspacing=None, connectivity=1 prediction = np.atleast_1d(prediction.astype(bool)) reference = np.atleast_1d(reference.astype(bool)) if voxelspacing is not None: - # TODO check protected access + # Protected access presented by Scipy voxelspacing = _ni_support._normalize_sequence(voxelspacing, prediction.ndim) voxelspacing = np.asarray(voxelspacing, dtype=np.float64) if not voxelspacing.flags.contiguous: @@ -60,12 +60,8 @@ def __surface_distances(reference, prediction, voxelspacing=None, connectivity=1 # raise RuntimeError("The second supplied array does not contain any binary object.") # extract only 1-pixel border line of objects - result_border = prediction ^ binary_erosion( - prediction, structure=footprint, iterations=1 - ) - reference_border = reference ^ binary_erosion( - reference, structure=footprint, iterations=1 - ) + result_border = prediction ^ binary_erosion(prediction, structure=footprint, iterations=1) + reference_border = reference ^ binary_erosion(reference, structure=footprint, iterations=1) # compute average surface distance # Note: scipys distance transform is calculated only inside the borders of the diff --git a/panoptica/metrics/metrics.py b/panoptica/metrics/metrics.py index 57432d7..758a4a2 100644 --- a/panoptica/metrics/metrics.py +++ b/panoptica/metrics/metrics.py @@ -37,9 +37,7 @@ def __call__( reference_arr = reference_arr.copy() == ref_instance_idx if isinstance(pred_instance_idx, int): pred_instance_idx = [pred_instance_idx] - prediction_arr = np.isin( - prediction_arr.copy(), pred_instance_idx - ) # type:ignore + prediction_arr = np.isin(prediction_arr.copy(), pred_instance_idx) # type:ignore return self._metric_function(reference_arr, prediction_arr, *args, **kwargs) def __eq__(self, __value: object) -> bool: @@ -63,12 +61,8 @@ def __hash__(self) -> int: def increasing(self): return not self.decreasing - def score_beats_threshold( - self, matching_score: float, matching_threshold: float - ) -> bool: - return (self.increasing and matching_score >= matching_threshold) or ( - self.decreasing and matching_score <= matching_threshold - ) + def score_beats_threshold(self, matching_score: float, matching_threshold: float) -> bool: + return (self.increasing and matching_score >= matching_threshold) or (self.decreasing and matching_score <= matching_threshold) class DirectValueMeta(EnumMeta): @@ -123,9 +117,7 @@ def __call__( **kwargs, ) - def score_beats_threshold( - self, matching_score: float, matching_threshold: float - ) -> bool: + def score_beats_threshold(self, matching_score: float, matching_threshold: float) -> bool: """Calculates whether a score beats a specified threshold Args: @@ -135,9 +127,7 @@ def score_beats_threshold( Returns: bool: True if the matching_score beats the threshold, False otherwise. """ - return (self.increasing and matching_score >= matching_threshold) or ( - self.decreasing and matching_score <= matching_threshold - ) + return (self.increasing and matching_score >= matching_threshold) or (self.decreasing and matching_score <= matching_threshold) @property def name(self): @@ -231,9 +221,7 @@ def __call__(self, result_obj: "PanopticaResult") -> Any: # ERROR if self._error: if self._error_obj is None: - self._error_obj = MetricCouldNotBeComputedException( - f"Metric {self.id} requested, but could not be computed" - ) + self._error_obj = MetricCouldNotBeComputedException(f"Metric {self.id} requested, but could not be computed") raise self._error_obj # Already calculated? if self._was_calculated: @@ -241,12 +229,8 @@ def __call__(self, result_obj: "PanopticaResult") -> Any: # Calculate it try: - assert ( - not self._was_calculated - ), f"Metric {self.id} was called to compute, but is set to have been already calculated" - assert ( - self._calc_func is not None - ), f"Metric {self.id} was called to compute, but has no calculation function set" + assert not self._was_calculated, f"Metric {self.id} was called to compute, but is set to have been already calculated" + assert self._calc_func is not None, f"Metric {self.id} was called to compute, but has no calculation function set" value = self._calc_func(result_obj) except MetricCouldNotBeComputedException as e: value = e @@ -289,25 +273,17 @@ def __init__( else: self.AVG = None if self.ALL is None else np.average(self.ALL) self.SUM = None if self.ALL is None else np.sum(self.ALL) - self.STD = ( - None - if self.ALL is None - else empty_list_std if len(self.ALL) == 0 else np.std(self.ALL) - ) + self.STD = None if self.ALL is None else empty_list_std if len(self.ALL) == 0 else np.std(self.ALL) def __getitem__(self, mode: MetricMode | str): if self.error: - raise MetricCouldNotBeComputedException( - f"Metric {self.id} has not been calculated, add it to your eval_metrics" - ) + raise MetricCouldNotBeComputedException(f"Metric {self.id} has not been calculated, add it to your eval_metrics") if isinstance(mode, MetricMode): mode = mode.name if hasattr(self, mode): return getattr(self, mode) else: - raise MetricCouldNotBeComputedException( - f"List_Metric {self.id} does not contain {mode} member" - ) + raise MetricCouldNotBeComputedException(f"List_Metric {self.id} does not contain {mode} member") if __name__ == "__main__": diff --git a/panoptica/panoptic_evaluator.py b/panoptica/panoptic_evaluator.py index 26b95b3..32d600f 100644 --- a/panoptica/panoptic_evaluator.py +++ b/panoptica/panoptic_evaluator.py @@ -20,9 +20,7 @@ class Panoptic_Evaluator: def __init__( self, - expected_input: ( - Type[SemanticPair] | Type[UnmatchedInstancePair] | Type[MatchedInstancePair] - ) = MatchedInstancePair, + expected_input: Type[SemanticPair] | Type[UnmatchedInstancePair] | Type[MatchedInstancePair] = MatchedInstancePair, instance_approximator: InstanceApproximator | None = None, instance_matcher: InstanceMatchingAlgorithm | None = None, edge_case_handler: EdgeCaseHandler | None = None, @@ -48,13 +46,9 @@ def __init__( self.__decision_metric = decision_metric self.__decision_threshold = decision_threshold - self.__edge_case_handler = ( - edge_case_handler if edge_case_handler is not None else EdgeCaseHandler() - ) + self.__edge_case_handler = edge_case_handler if edge_case_handler is not None else EdgeCaseHandler() if self.__decision_metric is not None: - assert ( - self.__decision_threshold is not None - ), "decision metric set but no decision threshold for it" + assert self.__decision_threshold is not None, "decision metric set but no decision threshold for it" # self.__log_times = log_times self.__verbose = verbose @@ -63,15 +57,11 @@ def __init__( @measure_time def evaluate( self, - processing_pair: ( - SemanticPair | UnmatchedInstancePair | MatchedInstancePair | PanopticaResult - ), + processing_pair: SemanticPair | UnmatchedInstancePair | MatchedInstancePair | PanopticaResult, result_all: bool = True, verbose: bool | None = None, ) -> tuple[PanopticaResult, dict[str, _ProcessingPair]]: - assert ( - type(processing_pair) == self.__expected_input - ), f"input not of expected type {self.__expected_input}" + assert type(processing_pair) == self.__expected_input, f"input not of expected type {self.__expected_input}" return panoptic_evaluate( processing_pair=processing_pair, edge_case_handler=self.__edge_case_handler, @@ -87,9 +77,7 @@ def evaluate( def panoptic_evaluate( - processing_pair: ( - SemanticPair | UnmatchedInstancePair | MatchedInstancePair | PanopticaResult - ), + processing_pair: SemanticPair | UnmatchedInstancePair | MatchedInstancePair | PanopticaResult, instance_approximator: InstanceApproximator | None = None, instance_matcher: InstanceMatchingAlgorithm | None = None, eval_metrics: list[Metric] = [Metric.DSC, Metric.IOU, Metric.ASSD], @@ -142,9 +130,7 @@ def panoptic_evaluate( processing_pair.crop_data() if isinstance(processing_pair, SemanticPair): - assert ( - instance_approximator is not None - ), "Got SemanticPair but not InstanceApproximator" + assert instance_approximator is not None, "Got SemanticPair but not InstanceApproximator" print("-- Got SemanticPair, will approximate instances") processing_pair = instance_approximator.approximate_instances(processing_pair) start = perf_counter() @@ -163,9 +149,7 @@ def panoptic_evaluate( if isinstance(processing_pair, UnmatchedInstancePair): print("-- Got UnmatchedInstancePair, will match instances") - assert ( - instance_matcher is not None - ), "Got UnmatchedInstancePair but not InstanceMatchingAlgorithm" + assert instance_matcher is not None, "Got UnmatchedInstancePair but not InstanceMatchingAlgorithm" start = perf_counter() processing_pair = instance_matcher.match_instances( processing_pair, @@ -244,7 +228,6 @@ def _handle_zero_instances_cases( is_edge_case = True elif n_prediction_instance == 0: # All predictions are missing, only false negatives - # TODO what is going on here! n_reference_instance = n_reference_instance n_prediction_instance = 0 is_edge_case = True diff --git a/panoptica/utils/__init__.py b/panoptica/utils/__init__.py index 8abe770..4a4392f 100644 --- a/panoptica/utils/__init__.py +++ b/panoptica/utils/__init__.py @@ -1,8 +1,3 @@ -from panoptica.utils.edge_case_handling import ( - EdgeCaseHandler, - EdgeCaseResult, - EdgeCaseZeroTP, -) from panoptica.utils.numpy_utils import ( _count_unique_without_zeros, _unique_without_zeros, @@ -13,5 +8,10 @@ SemanticPair, UnmatchedInstancePair, ) +from panoptica.utils.edge_case_handling import ( + EdgeCaseHandler, + EdgeCaseResult, + EdgeCaseZeroTP, +) # from utils.constants import diff --git a/panoptica/utils/edge_case_handling.py b/panoptica/utils/edge_case_handling.py index c1f93a2..31b74f1 100644 --- a/panoptica/utils/edge_case_handling.py +++ b/panoptica/utils/edge_case_handling.py @@ -1,4 +1,5 @@ import numpy as np +from typing import TYPE_CHECKING from panoptica.metrics import Metric from panoptica.utils.constants import _Enum_Compare, auto @@ -32,26 +33,12 @@ def __init__( normal: EdgeCaseResult | None = None, ) -> None: self.edgecase_dict: dict[EdgeCaseZeroTP, EdgeCaseResult] = {} - self.edgecase_dict[EdgeCaseZeroTP.EMPTY_PRED] = ( - empty_prediction_result - if empty_prediction_result is not None - else default_result - ) - self.edgecase_dict[EdgeCaseZeroTP.EMPTY_REF] = ( - empty_reference_result - if empty_reference_result is not None - else default_result - ) - self.edgecase_dict[EdgeCaseZeroTP.NO_INSTANCES] = ( - no_instances_result if no_instances_result is not None else default_result - ) - self.edgecase_dict[EdgeCaseZeroTP.NORMAL] = ( - normal if normal is not None else default_result - ) + self.edgecase_dict[EdgeCaseZeroTP.EMPTY_PRED] = empty_prediction_result if empty_prediction_result is not None else default_result + self.edgecase_dict[EdgeCaseZeroTP.EMPTY_REF] = empty_reference_result if empty_reference_result is not None else default_result + self.edgecase_dict[EdgeCaseZeroTP.NO_INSTANCES] = no_instances_result if no_instances_result is not None else default_result + self.edgecase_dict[EdgeCaseZeroTP.NORMAL] = normal if normal is not None else default_result - def __call__( - self, tp: int, num_pred_instances, num_ref_instances - ) -> tuple[bool, float | None]: + def __call__(self, tp: int, num_pred_instances, num_ref_instances) -> tuple[bool, float | None]: if tp != 0: return False, EdgeCaseResult.NONE.value # @@ -96,9 +83,7 @@ def __init__( }, empty_list_std: EdgeCaseResult = EdgeCaseResult.NAN, ) -> None: - self.__listmetric_zeroTP_handling: dict[ - Metric, MetricZeroTPEdgeCaseHandling - ] = listmetric_zeroTP_handling + self.__listmetric_zeroTP_handling: dict[Metric, MetricZeroTPEdgeCaseHandling] = listmetric_zeroTP_handling self.__empty_list_std: EdgeCaseResult = empty_list_std def handle_zero_tp( @@ -111,9 +96,7 @@ def handle_zero_tp( if tp != 0: return False, EdgeCaseResult.NONE.value if metric not in self.__listmetric_zeroTP_handling: - raise NotImplementedError( - f"Metric {metric} encountered zero TP, but no edge handling available" - ) + raise NotImplementedError(f"Metric {metric} encountered zero TP, but no edge handling available") return self.__listmetric_zeroTP_handling[metric]( tp=tp, @@ -139,9 +122,7 @@ def __str__(self) -> str: print() # print(handler.get_metric_zero_tp_handle(ListMetric.IOU)) - r = handler.handle_zero_tp( - Metric.IOU, tp=0, num_pred_instances=1, num_ref_instances=1 - ) + r = handler.handle_zero_tp(Metric.IOU, tp=0, num_pred_instances=1, num_ref_instances=1) print(r) iou_test = MetricZeroTPEdgeCaseHandling( diff --git a/unit_tests/test_datatype.py b/unit_tests/test_datatype.py index d9f3bc3..00dcc7a 100644 --- a/unit_tests/test_datatype.py +++ b/unit_tests/test_datatype.py @@ -5,10 +5,10 @@ import os import unittest -from panoptica.metrics import Metric +from panoptica.metrics import Metric, Evaluation_List_Metric, MetricMode, MetricCouldNotBeComputedException -class Test_Panoptic_Evaluator(unittest.TestCase): +class Test_Datatypes(unittest.TestCase): def setUp(self) -> None: os.environ["PANOPTICA_CITATION_REMINDER"] = "False" return super().setUp() @@ -35,4 +35,39 @@ def test_matching_metric(self): self.assertFalse(assd_metric.score_beats_threshold(0.55, 0.5)) self.assertTrue(assd_metric.score_beats_threshold(0.5, 0.55)) - # TODO listmetric + Mode (STD and so on) + def test_listmetric(self): + lmetric = Evaluation_List_Metric( + name_id="Test", + empty_list_std=None, + value_list=[1, 3, 5], + ) + + self.assertEqual(lmetric[MetricMode.ALL], [1, 3, 5]) + self.assertTrue(lmetric[MetricMode.AVG] == 3) + self.assertTrue(lmetric[MetricMode.SUM] == 9) + + def test_listmetric_edgecase(self): + lmetric = Evaluation_List_Metric( + name_id="Test", + empty_list_std=None, + value_list=[1, 3, 5], + is_edge_case=True, + edge_case_result=50, + ) + + self.assertEqual(lmetric[MetricMode.ALL], [1, 3, 5]) + self.assertTrue(lmetric[MetricMode.AVG] == 50) + self.assertTrue(lmetric[MetricMode.SUM] == 50) + + def test_listmetric_emptylist(self): + lmetric = Evaluation_List_Metric( + name_id="Test", + empty_list_std=None, + value_list=None, + is_edge_case=True, + edge_case_result=50, + ) + + for mode in MetricMode: + with self.assertRaises(MetricCouldNotBeComputedException): + lmetric[mode] From 2c072a2e8e738c095c9efaf408fe8afcaec3b5a0 Mon Sep 17 00:00:00 2001 From: "brainless-bot[bot]" <153751247+brainless-bot[bot]@users.noreply.github.com> Date: Fri, 9 Feb 2024 13:39:12 +0000 Subject: [PATCH 29/29] Autoformat with black --- panoptica/_functionals.py | 26 ++++++++++++--- panoptica/instance_approximator.py | 30 +++++++++++++---- panoptica/instance_evaluator.py | 18 ++++++++--- panoptica/instance_matcher.py | 25 +++++++++++---- panoptica/metrics/assd.py | 8 +++-- panoptica/metrics/metrics.py | 46 ++++++++++++++++++++------- panoptica/panoptic_evaluator.py | 32 ++++++++++++++----- panoptica/utils/edge_case_handling.py | 36 ++++++++++++++++----- unit_tests/test_datatype.py | 7 +++- 9 files changed, 176 insertions(+), 52 deletions(-) diff --git a/panoptica/_functionals.py b/panoptica/_functionals.py index 44639e6..ebcf108 100644 --- a/panoptica/_functionals.py +++ b/panoptica/_functionals.py @@ -34,7 +34,11 @@ def _calc_overlapping_labels( # instance_pairs = [(reference_arr, prediction_arr, i, j) for i, j in overlapping_indices] # (ref, pred) - return [(int(i % (max_ref)), int(i // (max_ref))) for i in np.unique(overlap_arr) if i > max_ref] + return [ + (int(i % (max_ref)), int(i // (max_ref))) + for i in np.unique(overlap_arr) + if i > max_ref + ] def _calc_matching_metric_of_overlapping_labels( @@ -64,8 +68,13 @@ def _calc_matching_metric_of_overlapping_labels( with Pool() as pool: mm_values = pool.starmap(matching_metric.value, instance_pairs) - mm_pairs = [(i, (instance_pairs[idx][2], instance_pairs[idx][3])) for idx, i in enumerate(mm_values)] - mm_pairs = sorted(mm_pairs, key=lambda x: x[0], reverse=not matching_metric.decreasing) + mm_pairs = [ + (i, (instance_pairs[idx][2], instance_pairs[idx][3])) + for idx, i in enumerate(mm_values) + ] + mm_pairs = sorted( + mm_pairs, key=lambda x: x[0], reverse=not matching_metric.decreasing + ) return mm_pairs @@ -98,7 +107,10 @@ def _calc_iou_of_overlapping_labels( with Pool() as pool: iou_values = pool.starmap(_compute_instance_iou, instance_pairs) - iou_pairs = [(i, (instance_pairs[idx][2], instance_pairs[idx][3])) for idx, i in enumerate(iou_values)] + iou_pairs = [ + (i, (instance_pairs[idx][2], instance_pairs[idx][3])) + for idx, i in enumerate(iou_values) + ] iou_pairs = sorted(iou_pairs, key=lambda x: x[0], reverse=True) return iou_pairs @@ -134,7 +146,11 @@ def _calc_iou_matrix( # Create a pool of worker processes to parallelize the computation with Pool() as pool: # # Generate all possible pairs of instance indices for IoU computation - instance_pairs = [(reference_arr, prediction_arr, ref_idx, pred_idx) for ref_idx in ref_labels for pred_idx in pred_labels] + instance_pairs = [ + (reference_arr, prediction_arr, ref_idx, pred_idx) + for ref_idx in ref_labels + for pred_idx in pred_labels + ] # Calculate IoU for all instance pairs in parallel using starmap iou_values = pool.starmap(_compute_instance_iou, instance_pairs) diff --git a/panoptica/instance_approximator.py b/panoptica/instance_approximator.py index b5de46c..e73f843 100644 --- a/panoptica/instance_approximator.py +++ b/panoptica/instance_approximator.py @@ -41,7 +41,9 @@ class InstanceApproximator(ABC): """ @abstractmethod - def _approximate_instances(self, semantic_pair: SemanticPair, **kwargs) -> UnmatchedInstancePair | MatchedInstancePair: + def _approximate_instances( + self, semantic_pair: SemanticPair, **kwargs + ) -> UnmatchedInstancePair | MatchedInstancePair: """ Abstract method to be implemented by subclasses for instance approximation. @@ -72,11 +74,19 @@ def approximate_instances( """ # Check validity pred_labels, ref_labels = semantic_pair._pred_labels, semantic_pair._ref_labels - pred_label_range = (np.min(pred_labels), np.max(pred_labels)) if len(pred_labels) > 0 else (0, 0) - ref_label_range = (np.min(ref_labels), np.max(ref_labels)) if len(ref_labels) > 0 else (0, 0) + pred_label_range = ( + (np.min(pred_labels), np.max(pred_labels)) + if len(pred_labels) > 0 + else (0, 0) + ) + ref_label_range = ( + (np.min(ref_labels), np.max(ref_labels)) if len(ref_labels) > 0 else (0, 0) + ) # min_value = min(np.min(pred_label_range[0]), np.min(ref_label_range[0])) - assert min_value >= 0, "There are negative values in the semantic maps. This is not allowed!" + assert ( + min_value >= 0 + ), "There are negative values in the semantic maps. This is not allowed!" # Set dtype to smalles fitting uint max_value = max(np.max(pred_label_range[1]), np.max(ref_label_range[1])) dtype = _get_smallest_fitting_uint(max_value) @@ -116,7 +126,9 @@ def __init__(self, cca_backend: CCABackend | None = None) -> None: """ self.cca_backend = cca_backend - def _approximate_instances(self, semantic_pair: SemanticPair, **kwargs) -> UnmatchedInstancePair: + def _approximate_instances( + self, semantic_pair: SemanticPair, **kwargs + ) -> UnmatchedInstancePair: """ Approximate instances using the connected components algorithm. @@ -129,7 +141,9 @@ def _approximate_instances(self, semantic_pair: SemanticPair, **kwargs) -> Unmat """ cca_backend = self.cca_backend if self.cca_backend is None: - cca_backend = CCABackend.cc3d if semantic_pair.n_dim >= 3 else CCABackend.scipy + cca_backend = ( + CCABackend.cc3d if semantic_pair.n_dim >= 3 else CCABackend.scipy + ) assert cca_backend is not None empty_prediction = len(semantic_pair._pred_labels) == 0 @@ -140,7 +154,9 @@ def _approximate_instances(self, semantic_pair: SemanticPair, **kwargs) -> Unmat else (semantic_pair._prediction_arr, 0) ) reference_arr, n_reference_instance = ( - _connected_components(semantic_pair._reference_arr, cca_backend) if not empty_reference else (semantic_pair._reference_arr, 0) + _connected_components(semantic_pair._reference_arr, cca_backend) + if not empty_reference + else (semantic_pair._reference_arr, 0) ) return UnmatchedInstancePair( prediction_arr=prediction_arr, diff --git a/panoptica/instance_evaluator.py b/panoptica/instance_evaluator.py index c385b2b..a379bf3 100644 --- a/panoptica/instance_evaluator.py +++ b/panoptica/instance_evaluator.py @@ -34,7 +34,9 @@ def evaluate_matched_instance( if edge_case_handler is None: edge_case_handler = EdgeCaseHandler() if decision_metric is not None: - assert decision_metric.name in [v.name for v in eval_metrics], "decision metric not contained in eval_metrics" + assert decision_metric.name in [ + v.name for v in eval_metrics + ], "decision metric not contained in eval_metrics" assert decision_threshold is not None, "decision metric set but no threshold" # Initialize variables for True Positives (tp) tp = len(matched_instance_pair.matched_instances) @@ -46,13 +48,21 @@ def evaluate_matched_instance( ) ref_matched_labels = matched_instance_pair.matched_instances - instance_pairs = [(reference_arr, prediction_arr, ref_idx, eval_metrics) for ref_idx in ref_matched_labels] + instance_pairs = [ + (reference_arr, prediction_arr, ref_idx, eval_metrics) + for ref_idx in ref_matched_labels + ] with Pool() as pool: - metric_dicts: list[dict[Metric, float]] = pool.starmap(_evaluate_instance, instance_pairs) + metric_dicts: list[dict[Metric, float]] = pool.starmap( + _evaluate_instance, instance_pairs + ) for metric_dict in metric_dicts: if decision_metric is None or ( - decision_threshold is not None and decision_metric.score_beats_threshold(metric_dict[decision_metric], decision_threshold) + decision_threshold is not None + and decision_metric.score_beats_threshold( + metric_dict[decision_metric], decision_threshold + ) ): for k, v in metric_dict.items(): score_dict[k].append(v) diff --git a/panoptica/instance_matcher.py b/panoptica/instance_matcher.py index 4273fd5..3b32e63 100644 --- a/panoptica/instance_matcher.py +++ b/panoptica/instance_matcher.py @@ -80,7 +80,9 @@ def match_instances( return map_instance_labels(unmatched_instance_pair.copy(), instance_labelmap) -def map_instance_labels(processing_pair: UnmatchedInstancePair, labelmap: InstanceLabelMap) -> MatchedInstancePair: +def map_instance_labels( + processing_pair: UnmatchedInstancePair, labelmap: InstanceLabelMap +) -> MatchedInstancePair: """ Map instance labels based on the provided labelmap and create a MatchedInstancePair. @@ -192,13 +194,20 @@ def _match_instances( unmatched_instance_pair.prediction_arr, unmatched_instance_pair.reference_arr, ) - mm_pairs = _calc_matching_metric_of_overlapping_labels(pred_arr, ref_arr, ref_labels, matching_metric=self.matching_metric) + mm_pairs = _calc_matching_metric_of_overlapping_labels( + pred_arr, ref_arr, ref_labels, matching_metric=self.matching_metric + ) # Loop through matched instances to compute PQ components for matching_score, (ref_label, pred_label) in mm_pairs: - if labelmap.contains_or(pred_label, ref_label) and not self.allow_many_to_one: + if ( + labelmap.contains_or(pred_label, ref_label) + and not self.allow_many_to_one + ): continue # -> doesnt make speed difference - if self.matching_metric.score_beats_threshold(matching_score, self.matching_threshold): + if self.matching_metric.score_beats_threshold( + matching_score, self.matching_threshold + ): # Match found, increment true positive count and collect IoU and Dice values labelmap.add_labelmap_entry(pred_label, ref_label) # map label ref_idx to pred_idx @@ -275,11 +284,15 @@ def _match_instances( continue if labelmap.contains_ref(ref_label): pred_labels_ = labelmap.get_pred_labels_matched_to_ref(ref_label) - new_score = self.new_combination_score(pred_labels_, pred_label, ref_label, unmatched_instance_pair) + new_score = self.new_combination_score( + pred_labels_, pred_label, ref_label, unmatched_instance_pair + ) if new_score > score_ref[ref_label]: labelmap.add_labelmap_entry(pred_label, ref_label) score_ref[ref_label] = new_score - elif self.matching_metric.score_beats_threshold(matching_score, self.matching_threshold): + elif self.matching_metric.score_beats_threshold( + matching_score, self.matching_threshold + ): # Match found, increment true positive count and collect IoU and Dice values labelmap.add_labelmap_entry(pred_label, ref_label) score_ref[ref_label] = matching_score diff --git a/panoptica/metrics/assd.py b/panoptica/metrics/assd.py index a73a644..7f030e3 100644 --- a/panoptica/metrics/assd.py +++ b/panoptica/metrics/assd.py @@ -60,8 +60,12 @@ def __surface_distances(reference, prediction, voxelspacing=None, connectivity=1 # raise RuntimeError("The second supplied array does not contain any binary object.") # extract only 1-pixel border line of objects - result_border = prediction ^ binary_erosion(prediction, structure=footprint, iterations=1) - reference_border = reference ^ binary_erosion(reference, structure=footprint, iterations=1) + result_border = prediction ^ binary_erosion( + prediction, structure=footprint, iterations=1 + ) + reference_border = reference ^ binary_erosion( + reference, structure=footprint, iterations=1 + ) # compute average surface distance # Note: scipys distance transform is calculated only inside the borders of the diff --git a/panoptica/metrics/metrics.py b/panoptica/metrics/metrics.py index 758a4a2..57432d7 100644 --- a/panoptica/metrics/metrics.py +++ b/panoptica/metrics/metrics.py @@ -37,7 +37,9 @@ def __call__( reference_arr = reference_arr.copy() == ref_instance_idx if isinstance(pred_instance_idx, int): pred_instance_idx = [pred_instance_idx] - prediction_arr = np.isin(prediction_arr.copy(), pred_instance_idx) # type:ignore + prediction_arr = np.isin( + prediction_arr.copy(), pred_instance_idx + ) # type:ignore return self._metric_function(reference_arr, prediction_arr, *args, **kwargs) def __eq__(self, __value: object) -> bool: @@ -61,8 +63,12 @@ def __hash__(self) -> int: def increasing(self): return not self.decreasing - def score_beats_threshold(self, matching_score: float, matching_threshold: float) -> bool: - return (self.increasing and matching_score >= matching_threshold) or (self.decreasing and matching_score <= matching_threshold) + def score_beats_threshold( + self, matching_score: float, matching_threshold: float + ) -> bool: + return (self.increasing and matching_score >= matching_threshold) or ( + self.decreasing and matching_score <= matching_threshold + ) class DirectValueMeta(EnumMeta): @@ -117,7 +123,9 @@ def __call__( **kwargs, ) - def score_beats_threshold(self, matching_score: float, matching_threshold: float) -> bool: + def score_beats_threshold( + self, matching_score: float, matching_threshold: float + ) -> bool: """Calculates whether a score beats a specified threshold Args: @@ -127,7 +135,9 @@ def score_beats_threshold(self, matching_score: float, matching_threshold: float Returns: bool: True if the matching_score beats the threshold, False otherwise. """ - return (self.increasing and matching_score >= matching_threshold) or (self.decreasing and matching_score <= matching_threshold) + return (self.increasing and matching_score >= matching_threshold) or ( + self.decreasing and matching_score <= matching_threshold + ) @property def name(self): @@ -221,7 +231,9 @@ def __call__(self, result_obj: "PanopticaResult") -> Any: # ERROR if self._error: if self._error_obj is None: - self._error_obj = MetricCouldNotBeComputedException(f"Metric {self.id} requested, but could not be computed") + self._error_obj = MetricCouldNotBeComputedException( + f"Metric {self.id} requested, but could not be computed" + ) raise self._error_obj # Already calculated? if self._was_calculated: @@ -229,8 +241,12 @@ def __call__(self, result_obj: "PanopticaResult") -> Any: # Calculate it try: - assert not self._was_calculated, f"Metric {self.id} was called to compute, but is set to have been already calculated" - assert self._calc_func is not None, f"Metric {self.id} was called to compute, but has no calculation function set" + assert ( + not self._was_calculated + ), f"Metric {self.id} was called to compute, but is set to have been already calculated" + assert ( + self._calc_func is not None + ), f"Metric {self.id} was called to compute, but has no calculation function set" value = self._calc_func(result_obj) except MetricCouldNotBeComputedException as e: value = e @@ -273,17 +289,25 @@ def __init__( else: self.AVG = None if self.ALL is None else np.average(self.ALL) self.SUM = None if self.ALL is None else np.sum(self.ALL) - self.STD = None if self.ALL is None else empty_list_std if len(self.ALL) == 0 else np.std(self.ALL) + self.STD = ( + None + if self.ALL is None + else empty_list_std if len(self.ALL) == 0 else np.std(self.ALL) + ) def __getitem__(self, mode: MetricMode | str): if self.error: - raise MetricCouldNotBeComputedException(f"Metric {self.id} has not been calculated, add it to your eval_metrics") + raise MetricCouldNotBeComputedException( + f"Metric {self.id} has not been calculated, add it to your eval_metrics" + ) if isinstance(mode, MetricMode): mode = mode.name if hasattr(self, mode): return getattr(self, mode) else: - raise MetricCouldNotBeComputedException(f"List_Metric {self.id} does not contain {mode} member") + raise MetricCouldNotBeComputedException( + f"List_Metric {self.id} does not contain {mode} member" + ) if __name__ == "__main__": diff --git a/panoptica/panoptic_evaluator.py b/panoptica/panoptic_evaluator.py index 32d600f..0119fbc 100644 --- a/panoptica/panoptic_evaluator.py +++ b/panoptica/panoptic_evaluator.py @@ -20,7 +20,9 @@ class Panoptic_Evaluator: def __init__( self, - expected_input: Type[SemanticPair] | Type[UnmatchedInstancePair] | Type[MatchedInstancePair] = MatchedInstancePair, + expected_input: ( + Type[SemanticPair] | Type[UnmatchedInstancePair] | Type[MatchedInstancePair] + ) = MatchedInstancePair, instance_approximator: InstanceApproximator | None = None, instance_matcher: InstanceMatchingAlgorithm | None = None, edge_case_handler: EdgeCaseHandler | None = None, @@ -46,9 +48,13 @@ def __init__( self.__decision_metric = decision_metric self.__decision_threshold = decision_threshold - self.__edge_case_handler = edge_case_handler if edge_case_handler is not None else EdgeCaseHandler() + self.__edge_case_handler = ( + edge_case_handler if edge_case_handler is not None else EdgeCaseHandler() + ) if self.__decision_metric is not None: - assert self.__decision_threshold is not None, "decision metric set but no decision threshold for it" + assert ( + self.__decision_threshold is not None + ), "decision metric set but no decision threshold for it" # self.__log_times = log_times self.__verbose = verbose @@ -57,11 +63,15 @@ def __init__( @measure_time def evaluate( self, - processing_pair: SemanticPair | UnmatchedInstancePair | MatchedInstancePair | PanopticaResult, + processing_pair: ( + SemanticPair | UnmatchedInstancePair | MatchedInstancePair | PanopticaResult + ), result_all: bool = True, verbose: bool | None = None, ) -> tuple[PanopticaResult, dict[str, _ProcessingPair]]: - assert type(processing_pair) == self.__expected_input, f"input not of expected type {self.__expected_input}" + assert ( + type(processing_pair) == self.__expected_input + ), f"input not of expected type {self.__expected_input}" return panoptic_evaluate( processing_pair=processing_pair, edge_case_handler=self.__edge_case_handler, @@ -77,7 +87,9 @@ def evaluate( def panoptic_evaluate( - processing_pair: SemanticPair | UnmatchedInstancePair | MatchedInstancePair | PanopticaResult, + processing_pair: ( + SemanticPair | UnmatchedInstancePair | MatchedInstancePair | PanopticaResult + ), instance_approximator: InstanceApproximator | None = None, instance_matcher: InstanceMatchingAlgorithm | None = None, eval_metrics: list[Metric] = [Metric.DSC, Metric.IOU, Metric.ASSD], @@ -130,7 +142,9 @@ def panoptic_evaluate( processing_pair.crop_data() if isinstance(processing_pair, SemanticPair): - assert instance_approximator is not None, "Got SemanticPair but not InstanceApproximator" + assert ( + instance_approximator is not None + ), "Got SemanticPair but not InstanceApproximator" print("-- Got SemanticPair, will approximate instances") processing_pair = instance_approximator.approximate_instances(processing_pair) start = perf_counter() @@ -149,7 +163,9 @@ def panoptic_evaluate( if isinstance(processing_pair, UnmatchedInstancePair): print("-- Got UnmatchedInstancePair, will match instances") - assert instance_matcher is not None, "Got UnmatchedInstancePair but not InstanceMatchingAlgorithm" + assert ( + instance_matcher is not None + ), "Got UnmatchedInstancePair but not InstanceMatchingAlgorithm" start = perf_counter() processing_pair = instance_matcher.match_instances( processing_pair, diff --git a/panoptica/utils/edge_case_handling.py b/panoptica/utils/edge_case_handling.py index 31b74f1..c474557 100644 --- a/panoptica/utils/edge_case_handling.py +++ b/panoptica/utils/edge_case_handling.py @@ -33,12 +33,26 @@ def __init__( normal: EdgeCaseResult | None = None, ) -> None: self.edgecase_dict: dict[EdgeCaseZeroTP, EdgeCaseResult] = {} - self.edgecase_dict[EdgeCaseZeroTP.EMPTY_PRED] = empty_prediction_result if empty_prediction_result is not None else default_result - self.edgecase_dict[EdgeCaseZeroTP.EMPTY_REF] = empty_reference_result if empty_reference_result is not None else default_result - self.edgecase_dict[EdgeCaseZeroTP.NO_INSTANCES] = no_instances_result if no_instances_result is not None else default_result - self.edgecase_dict[EdgeCaseZeroTP.NORMAL] = normal if normal is not None else default_result + self.edgecase_dict[EdgeCaseZeroTP.EMPTY_PRED] = ( + empty_prediction_result + if empty_prediction_result is not None + else default_result + ) + self.edgecase_dict[EdgeCaseZeroTP.EMPTY_REF] = ( + empty_reference_result + if empty_reference_result is not None + else default_result + ) + self.edgecase_dict[EdgeCaseZeroTP.NO_INSTANCES] = ( + no_instances_result if no_instances_result is not None else default_result + ) + self.edgecase_dict[EdgeCaseZeroTP.NORMAL] = ( + normal if normal is not None else default_result + ) - def __call__(self, tp: int, num_pred_instances, num_ref_instances) -> tuple[bool, float | None]: + def __call__( + self, tp: int, num_pred_instances, num_ref_instances + ) -> tuple[bool, float | None]: if tp != 0: return False, EdgeCaseResult.NONE.value # @@ -83,7 +97,9 @@ def __init__( }, empty_list_std: EdgeCaseResult = EdgeCaseResult.NAN, ) -> None: - self.__listmetric_zeroTP_handling: dict[Metric, MetricZeroTPEdgeCaseHandling] = listmetric_zeroTP_handling + self.__listmetric_zeroTP_handling: dict[ + Metric, MetricZeroTPEdgeCaseHandling + ] = listmetric_zeroTP_handling self.__empty_list_std: EdgeCaseResult = empty_list_std def handle_zero_tp( @@ -96,7 +112,9 @@ def handle_zero_tp( if tp != 0: return False, EdgeCaseResult.NONE.value if metric not in self.__listmetric_zeroTP_handling: - raise NotImplementedError(f"Metric {metric} encountered zero TP, but no edge handling available") + raise NotImplementedError( + f"Metric {metric} encountered zero TP, but no edge handling available" + ) return self.__listmetric_zeroTP_handling[metric]( tp=tp, @@ -122,7 +140,9 @@ def __str__(self) -> str: print() # print(handler.get_metric_zero_tp_handle(ListMetric.IOU)) - r = handler.handle_zero_tp(Metric.IOU, tp=0, num_pred_instances=1, num_ref_instances=1) + r = handler.handle_zero_tp( + Metric.IOU, tp=0, num_pred_instances=1, num_ref_instances=1 + ) print(r) iou_test = MetricZeroTPEdgeCaseHandling( diff --git a/unit_tests/test_datatype.py b/unit_tests/test_datatype.py index 00dcc7a..4d6b287 100644 --- a/unit_tests/test_datatype.py +++ b/unit_tests/test_datatype.py @@ -5,7 +5,12 @@ import os import unittest -from panoptica.metrics import Metric, Evaluation_List_Metric, MetricMode, MetricCouldNotBeComputedException +from panoptica.metrics import ( + Metric, + Evaluation_List_Metric, + MetricMode, + MetricCouldNotBeComputedException, +) class Test_Datatypes(unittest.TestCase):