From 9f2fbb8ce08bb1526ac1dd0a20d6482891cdc738 Mon Sep 17 00:00:00 2001 From: Gagan Bhullar Date: Thu, 17 Oct 2024 04:22:24 -0600 Subject: [PATCH 001/103] IsValidJSON() descriptor (#1325) * isvalidjson descriptor * test cases * review comments * doc * review comments * review comments * review comments --------- Co-authored-by: Emeli Dral --- docs/book/reference/all-metrics.md | 1 + src/evidently/descriptors/__init__.py | 2 ++ src/evidently/descriptors/_registry.py | 3 ++ .../descriptors/is_valid_json_descriptor.py | 11 ++++++++ src/evidently/features/_registry.py | 3 ++ .../features/is_valid_json_feature.py | 28 +++++++++++++++++++ tests/features/test_is_valid_json_feature.py | 23 +++++++++++++++ 7 files changed, 71 insertions(+) create mode 100644 src/evidently/descriptors/is_valid_json_descriptor.py create mode 100644 src/evidently/features/is_valid_json_feature.py create mode 100644 tests/features/test_is_valid_json_feature.py diff --git a/docs/book/reference/all-metrics.md b/docs/book/reference/all-metrics.md index 484440cd06..459fcc054f 100644 --- a/docs/book/reference/all-metrics.md +++ b/docs/book/reference/all-metrics.md @@ -274,6 +274,7 @@ Check for regular expression matches. | **ExcludesWords()** Example use:
`ExcludesWords(words_list=['buy', 'sell', 'bet']`| **Required:**
`words_list: List[str]`

**Optional:** | | **ItemMatch()** Example use:
`ItemMatch(with_column="expected")`| **Required:**
`with_column: str`

**Optional:** | | **ItemNoMatch()** Example use:
`ItemMatch(with_column="forbidden")`| **Required:**
`with_column: str`

**Optional:** | +| **IsValidJSON()** Example use:
`IsValidJSON(column_name='column_1')`| **Required:**
`column_name`

**Optional:** | | **JSONSchemaMatch()** Example use:
`JSONSchemaMatch(expected_schema={"name": str, "age": int}, exact_match=False, validate_types=True)`| **Required:**
`expected_schema: Dict[str, type]`

**Optional:** | ## Descriptors: Text stats diff --git a/src/evidently/descriptors/__init__.py b/src/evidently/descriptors/__init__.py index 173a1be486..c28a64460f 100644 --- a/src/evidently/descriptors/__init__.py +++ b/src/evidently/descriptors/__init__.py @@ -3,6 +3,7 @@ from .custom_descriptor import CustomPairColumnEval from .hf_descriptor import HuggingFaceModel from .hf_descriptor import HuggingFaceToxicityModel +from .is_valid_json_descriptor import IsValidJSON from .json_schema_match_descriptor import JSONSchemaMatch from .llm_judges import BiasLLMEval from .llm_judges import ContextQualityLLMEval @@ -60,6 +61,7 @@ "SentenceCount", "Sentiment", "RegExp", + "IsValidJSON", "JSONSchemaMatch", "_registry", ] diff --git a/src/evidently/descriptors/_registry.py b/src/evidently/descriptors/_registry.py index 5ac97efc42..84389f7d07 100644 --- a/src/evidently/descriptors/_registry.py +++ b/src/evidently/descriptors/_registry.py @@ -111,3 +111,6 @@ "evidently.descriptors.custom_descriptor.CustomPairColumnEval", "evidently:descriptor:CustomPairColumnEval", ) +register_type_alias( + FeatureDescriptor, "evidently.descriptors.is_valid_json_descriptor.IsValidJSON", "evidently:descriptor:IsValidJSON" +) diff --git a/src/evidently/descriptors/is_valid_json_descriptor.py b/src/evidently/descriptors/is_valid_json_descriptor.py new file mode 100644 index 0000000000..e09202577c --- /dev/null +++ b/src/evidently/descriptors/is_valid_json_descriptor.py @@ -0,0 +1,11 @@ +from evidently.features import is_valid_json_feature +from evidently.features.generated_features import FeatureDescriptor +from evidently.features.generated_features import GeneratedFeature + + +class IsValidJSON(FeatureDescriptor): + class Config: + type_alias = "evidently:descriptor:IsValidJSON" + + def feature(self, column_name: str) -> GeneratedFeature: + return is_valid_json_feature.IsValidJSON(column_name, self.display_name) diff --git a/src/evidently/features/_registry.py b/src/evidently/features/_registry.py index b3f579a981..e7524768d7 100644 --- a/src/evidently/features/_registry.py +++ b/src/evidently/features/_registry.py @@ -85,3 +85,6 @@ register_type_alias( GeneratedFeatures, "evidently.features.words_feature.WordsPresence", "evidently:feature:WordsPresence" ) +register_type_alias( + GeneratedFeatures, "evidently.features.is_valid_json_feature.IsValidJSON", "evidently:feature:IsValidJSON" +) diff --git a/src/evidently/features/is_valid_json_feature.py b/src/evidently/features/is_valid_json_feature.py new file mode 100644 index 0000000000..b8f4289415 --- /dev/null +++ b/src/evidently/features/is_valid_json_feature.py @@ -0,0 +1,28 @@ +import json +from typing import Any +from typing import ClassVar +from typing import Optional + +from evidently import ColumnType +from evidently.features.generated_features import ApplyColumnGeneratedFeature + + +class IsValidJSON(ApplyColumnGeneratedFeature): + class Config: + type_alias = "evidently:feature:IsValidJSON" + + __feature_type__: ClassVar = ColumnType.Categorical + display_name_template: ClassVar = "JSON valid for {column_name}" + column_name: str + + def __init__(self, column_name: str, display_name: Optional[str] = None): + self.column_name = column_name + self.display_name = display_name + super().__init__() + + def apply(self, value: Any): + try: + json.loads(value) + except ValueError: + return False + return True diff --git a/tests/features/test_is_valid_json_feature.py b/tests/features/test_is_valid_json_feature.py new file mode 100644 index 0000000000..e3078cbcee --- /dev/null +++ b/tests/features/test_is_valid_json_feature.py @@ -0,0 +1,23 @@ +import pandas as pd +import pytest + +from evidently.features.is_valid_json_feature import IsValidJSON +from evidently.pipeline.column_mapping import ColumnMapping +from evidently.utils.data_preprocessing import create_data_definition + + +@pytest.mark.parametrize( + ("item", "expected"), + [ + ('{"test": "abc"}', True), + ("not json", False), + ], +) +def test_is_valid_json_feature(item: str, expected: bool): + feature_generator = IsValidJSON("column_1") + data = pd.DataFrame(dict(column_1=[item])) + result = feature_generator.generate_feature( + data=data, + data_definition=create_data_definition(None, data, ColumnMapping()), + ) + assert result.equals(pd.DataFrame(dict(column_1=[expected]))) From aaf64238232971639d6cf55f2a77bcddeb7998a8 Mon Sep 17 00:00:00 2001 From: Gagan Bhullar Date: Thu, 17 Oct 2024 06:50:57 -0600 Subject: [PATCH 002/103] ExactMatch descriptor (#1326) --- docs/book/reference/all-metrics.md | 1 + src/evidently/descriptors/__init__.py | 2 ++ src/evidently/descriptors/_registry.py | 3 ++ .../descriptors/exact_match_descriptor.py | 13 +++++++++ src/evidently/features/_registry.py | 3 ++ src/evidently/features/exact_match_feature.py | 29 +++++++++++++++++++ tests/features/test_exact_feature.py | 23 +++++++++++++++ 7 files changed, 74 insertions(+) create mode 100644 src/evidently/descriptors/exact_match_descriptor.py create mode 100644 src/evidently/features/exact_match_feature.py create mode 100644 tests/features/test_exact_feature.py diff --git a/docs/book/reference/all-metrics.md b/docs/book/reference/all-metrics.md index 459fcc054f..ba78b2f570 100644 --- a/docs/book/reference/all-metrics.md +++ b/docs/book/reference/all-metrics.md @@ -274,6 +274,7 @@ Check for regular expression matches. | **ExcludesWords()**
  • Checks if the text excludes all specified words.
  • Considers only vocabulary words (from NLTK vocabulary).
  • By default, considers inflected and variant forms of the same word.
  • Returns True/False for every input.
Example use:
`ExcludesWords(words_list=['buy', 'sell', 'bet']`| **Required:**
`words_list: List[str]`

**Optional:**
  • `display_name`
  • `mode = 'all'` or `'any'`
  • `lemmatize = True` or `False`
| | **ItemMatch()**
  • Checks whether the text contains **any** (default) or **all** specified items that are specific to each row (represented as tuples)
  • Returns True/False for each row.
Example use:
`ItemMatch(with_column="expected")`| **Required:**
`with_column: str`

**Optional:**
  • `display_name`
  • `mode = 'all'` or `'any'`
  • `case_sensitive = True` or `False`
| | **ItemNoMatch()**
  • Checks whether the text excludes **any** (default) or **all** specified items that are specific to each row (represented as tuples)
  • Returns True/False for each row.
Example use:
`ItemMatch(with_column="forbidden")`| **Required:**
`with_column: str`

**Optional:**
  • `display_name`
  • `mode = 'all'` or `'any'`
  • `case_sensitive = True` or `False`
| +| **ExactMatch()**
  • Checks if the text matches between two columns.
  • Returns True/False for every input.
Example use:
`ExactMatch(column_name='column_1')`| **Required:**
`with_column`

**Optional:**
  • `display_name`
| | **IsValidJSON()**
  • Checks if the text in a specified column is a valid JSON.
  • Returns True/False for every input.
Example use:
`IsValidJSON(column_name='column_1')`| **Required:**
`column_name`

**Optional:**
  • `display_name`
| | **JSONSchemaMatch()**
  • Checks if the text contains a JSON object matching the **expected_schema**. Supports exact (**exact=True**) or minimal (**exact=False**) matching, with optional strict type validation (**validate_types=True**).
  • Returns True/False for each row.
Example use:
`JSONSchemaMatch(expected_schema={"name": str, "age": int}, exact_match=False, validate_types=True)`| **Required:**
`expected_schema: Dict[str, type]`

**Optional:**
  • `exact_match = True` or `False`
  • `validate_types = True` or `False`
| diff --git a/src/evidently/descriptors/__init__.py b/src/evidently/descriptors/__init__.py index c28a64460f..d1b5bd46c5 100644 --- a/src/evidently/descriptors/__init__.py +++ b/src/evidently/descriptors/__init__.py @@ -1,6 +1,7 @@ from . import _registry from .custom_descriptor import CustomColumnEval from .custom_descriptor import CustomPairColumnEval +from .exact_match_descriptor import ExactMatch from .hf_descriptor import HuggingFaceModel from .hf_descriptor import HuggingFaceToxicityModel from .is_valid_json_descriptor import IsValidJSON @@ -60,6 +61,7 @@ "SemanticSimilarity", "SentenceCount", "Sentiment", + "ExactMatch", "RegExp", "IsValidJSON", "JSONSchemaMatch", diff --git a/src/evidently/descriptors/_registry.py b/src/evidently/descriptors/_registry.py index 84389f7d07..be0504b4db 100644 --- a/src/evidently/descriptors/_registry.py +++ b/src/evidently/descriptors/_registry.py @@ -111,6 +111,9 @@ "evidently.descriptors.custom_descriptor.CustomPairColumnEval", "evidently:descriptor:CustomPairColumnEval", ) +register_type_alias( + FeatureDescriptor, "evidently.descriptors.exact_match_descriptor.ExactMatch", "evidently:descriptor:ExactMatch" +) register_type_alias( FeatureDescriptor, "evidently.descriptors.is_valid_json_descriptor.IsValidJSON", "evidently:descriptor:IsValidJSON" ) diff --git a/src/evidently/descriptors/exact_match_descriptor.py b/src/evidently/descriptors/exact_match_descriptor.py new file mode 100644 index 0000000000..7c03cf948f --- /dev/null +++ b/src/evidently/descriptors/exact_match_descriptor.py @@ -0,0 +1,13 @@ +from evidently.features.exact_match_feature import ExactMatchFeature +from evidently.features.generated_features import FeatureDescriptor +from evidently.features.generated_features import GeneratedFeatures + + +class ExactMatch(FeatureDescriptor): + class Config: + type_alias = "evidently:descriptor:ExactMatch" + + with_column: str + + def feature(self, column_name: str) -> GeneratedFeatures: + return ExactMatchFeature(columns=[column_name, self.with_column], display_name=self.display_name) diff --git a/src/evidently/features/_registry.py b/src/evidently/features/_registry.py index e7524768d7..b8286ee34f 100644 --- a/src/evidently/features/_registry.py +++ b/src/evidently/features/_registry.py @@ -85,6 +85,9 @@ register_type_alias( GeneratedFeatures, "evidently.features.words_feature.WordsPresence", "evidently:feature:WordsPresence" ) +register_type_alias( + GeneratedFeatures, "evidently.features.exact_match_feature.ExactMatchFeature", "evidently:feature:ExactMatchFeature" +) register_type_alias( GeneratedFeatures, "evidently.features.is_valid_json_feature.IsValidJSON", "evidently:feature:IsValidJSON" ) diff --git a/src/evidently/features/exact_match_feature.py b/src/evidently/features/exact_match_feature.py new file mode 100644 index 0000000000..4f46d43802 --- /dev/null +++ b/src/evidently/features/exact_match_feature.py @@ -0,0 +1,29 @@ +from typing import ClassVar +from typing import List + +import pandas as pd + +from evidently.base_metric import ColumnName +from evidently.core import ColumnType +from evidently.features.generated_features import GeneratedFeature +from evidently.utils.data_preprocessing import DataDefinition + + +class ExactMatchFeature(GeneratedFeature): + class Config: + type_alias = "evidently:feature:ExactMatchFeature" + + __feature_type__: ClassVar = ColumnType.Categorical + columns: List[str] + + def generate_feature(self, data: pd.DataFrame, data_definition: DataDefinition) -> pd.DataFrame: + return pd.DataFrame({self._feature_name(): data[self.columns[0]] == data[self.columns[1]]}) + + def _feature_name(self): + return "|".join(self.columns) + + def _as_column(self) -> "ColumnName": + return self._create_column( + self._feature_name(), + default_display_name=f"Exact Match for {' '.join(self.columns)}.", + ) diff --git a/tests/features/test_exact_feature.py b/tests/features/test_exact_feature.py new file mode 100644 index 0000000000..cac6f5f0d0 --- /dev/null +++ b/tests/features/test_exact_feature.py @@ -0,0 +1,23 @@ +import pandas as pd +import pytest + +from evidently.features.exact_match_feature import ExactMatchFeature +from evidently.pipeline.column_mapping import ColumnMapping +from evidently.utils.data_preprocessing import create_data_definition + + +@pytest.mark.parametrize( + ("value1", "value2", "expected"), + [ + ("this is same", "this is same", True), + ("this is same", "this is different", False), + ], +) +def test_exact_match_feature(value1: str, value2: str, expected: bool): + feature_generator = ExactMatchFeature(columns=["column_1", "column_2"]) + data = pd.DataFrame(dict(column_1=[value1], column_2=[value2])) + result = feature_generator.generate_feature( + data=data, data_definition=create_data_definition(None, data, ColumnMapping()) + ) + expected_df = pd.DataFrame([[expected]], columns=["column_1|column_2"]) + pd.testing.assert_frame_equal(result, expected_df) From 0d02c6241fb4b9c5a9647105762da04ec1f53dec Mon Sep 17 00:00:00 2001 From: Jonathan Bown Date: Thu, 17 Oct 2024 11:49:55 -0600 Subject: [PATCH 003/103] Add WordMatch + WordNoMatch descriptor (#1334) * initial implementation * Initial unit tests * Update test_words_feature.py * clean up * Remove extra input data * Initial implementation * Update type alias * Add WordNoMatch to init * add display name parameter * Add test case from original issue * Fix incorrect aliases * update default display name * add column check to tests * fix failing code analysis * fix double quote syntax issue --------- Co-authored-by: Emeli Dral --- docs/book/reference/all-metrics.md | 2 + src/evidently/descriptors/__init__.py | 4 + src/evidently/descriptors/_registry.py | 6 ++ src/evidently/descriptors/words_descriptor.py | 34 ++++++++ src/evidently/features/_registry.py | 5 ++ src/evidently/features/words_feature.py | 84 +++++++++++++++++++ tests/features/test_words_feature.py | 82 ++++++++++++++++++ 7 files changed, 217 insertions(+) diff --git a/docs/book/reference/all-metrics.md b/docs/book/reference/all-metrics.md index ba78b2f570..266090b422 100644 --- a/docs/book/reference/all-metrics.md +++ b/docs/book/reference/all-metrics.md @@ -274,6 +274,8 @@ Check for regular expression matches. | **ExcludesWords()**
  • Checks if the text excludes all specified words.
  • Considers only vocabulary words (from NLTK vocabulary).
  • By default, considers inflected and variant forms of the same word.
  • Returns True/False for every input.
Example use:
`ExcludesWords(words_list=['buy', 'sell', 'bet']`| **Required:**
`words_list: List[str]`

**Optional:**
  • `display_name`
  • `mode = 'all'` or `'any'`
  • `lemmatize = True` or `False`
| | **ItemMatch()**
  • Checks whether the text contains **any** (default) or **all** specified items that are specific to each row (represented as tuples)
  • Returns True/False for each row.
Example use:
`ItemMatch(with_column="expected")`| **Required:**
`with_column: str`

**Optional:**
  • `display_name`
  • `mode = 'all'` or `'any'`
  • `case_sensitive = True` or `False`
| | **ItemNoMatch()**
  • Checks whether the text excludes **any** (default) or **all** specified items that are specific to each row (represented as tuples)
  • Returns True/False for each row.
Example use:
`ItemMatch(with_column="forbidden")`| **Required:**
`with_column: str`

**Optional:**
  • `display_name`
  • `mode = 'all'` or `'any'`
  • `case_sensitive = True` or `False`
| +| **WordMatch()**
  • Checks whether the text includes **any** (default) or **all** specified words for each row (represented as tuples).
  • Considers only vocabulary words (from NLTK vocabulary).
  • By default, considers inflected and variant forms of the same word.
  • Returns True/False for every input.
Example use:
`WordMatch(with_column="expected"` | **Required:**
`with_column: str`

**Optional:**
  • `display_name`
  • `mode = 'any'` or `'all'`
  • `lemmatize = True` or `False`
| +| **WordNoMatch()**
  • Checks whether the text excludes **any** (default) or **all** specified words for each row (represented as tuples).
  • Considers only vocabulary words (from NLTK vocabulary).
  • By default, considers inflected and variant forms of the same word.
  • Returns True/False for every input.
Example use:
`WordMatch(with_column="forbidden"` | **Required:**
`with_column: str`

**Optional:**
  • `display_name`
  • `mode = 'any'` or `'all'`
  • `lemmatize = True` or `False`
| | **ExactMatch()**
  • Checks if the text matches between two columns.
  • Returns True/False for every input.
Example use:
`ExactMatch(column_name='column_1')`| **Required:**
`with_column`

**Optional:**
  • `display_name`
| | **IsValidJSON()**
  • Checks if the text in a specified column is a valid JSON.
  • Returns True/False for every input.
Example use:
`IsValidJSON(column_name='column_1')`| **Required:**
`column_name`

**Optional:**
  • `display_name`
| | **JSONSchemaMatch()**
  • Checks if the text contains a JSON object matching the **expected_schema**. Supports exact (**exact=True**) or minimal (**exact=False**) matching, with optional strict type validation (**validate_types=True**).
  • Returns True/False for each row.
Example use:
`JSONSchemaMatch(expected_schema={"name": str, "age": int}, exact_match=False, validate_types=True)`| **Required:**
`expected_schema: Dict[str, type]`

**Optional:**
  • `exact_match = True` or `False`
  • `validate_types = True` or `False`
| diff --git a/src/evidently/descriptors/__init__.py b/src/evidently/descriptors/__init__.py index d1b5bd46c5..3f78f60f84 100644 --- a/src/evidently/descriptors/__init__.py +++ b/src/evidently/descriptors/__init__.py @@ -31,6 +31,8 @@ from .word_count_descriptor import WordCount from .words_descriptor import ExcludesWords from .words_descriptor import IncludesWords +from .words_descriptor import WordMatch +from .words_descriptor import WordNoMatch __all__ = [ "CustomColumnEval", @@ -63,6 +65,8 @@ "Sentiment", "ExactMatch", "RegExp", + "WordMatch", + "WordNoMatch", "IsValidJSON", "JSONSchemaMatch", "_registry", diff --git a/src/evidently/descriptors/_registry.py b/src/evidently/descriptors/_registry.py index be0504b4db..d47c1b5777 100644 --- a/src/evidently/descriptors/_registry.py +++ b/src/evidently/descriptors/_registry.py @@ -106,6 +106,12 @@ register_type_alias( FeatureDescriptor, "evidently.descriptors.words_descriptor.IncludesWords", "evidently:descriptor:IncludesWords" ) +register_type_alias( + FeatureDescriptor, "evidently.descriptors.words_descriptor.WordMatch", "evidently:descriptor:WordMatch" +) +register_type_alias( + FeatureDescriptor, "evidently.descriptors.words_descriptor.WordNoMatch", "evidently:descriptor:WordNoMatch" +) register_type_alias( GeneralDescriptor, "evidently.descriptors.custom_descriptor.CustomPairColumnEval", diff --git a/src/evidently/descriptors/words_descriptor.py b/src/evidently/descriptors/words_descriptor.py index 14e30060e5..1b96164505 100644 --- a/src/evidently/descriptors/words_descriptor.py +++ b/src/evidently/descriptors/words_descriptor.py @@ -39,3 +39,37 @@ def feature(self, column_name: str) -> GeneratedFeature: self.lemmatize, self.display_name, ) + + +class WordMatch(FeatureDescriptor): + class Config: + type_alias = "evidently:descriptor:WordMatch" + + with_column: str + mode: str = "any" + lemmatize: bool = True + + def feature(self, column_name: str) -> GeneratedFeature: + return words_feature.WordMatch( + columns=[column_name, self.with_column], + mode=self.mode, + lemmatize=self.lemmatize, + display_name=self.display_name, + ) + + +class WordNoMatch(FeatureDescriptor): + class Config: + type_alias = "evidently:descriptor:WordNoMatch" + + with_column: str + mode: str = "any" + lemmatize: bool = True + + def feature(self, column_name: str) -> GeneratedFeature: + return words_feature.WordNoMatch( + columns=[column_name, self.with_column], + mode=self.mode, + lemmatize=self.lemmatize, + display_name=self.display_name, + ) diff --git a/src/evidently/features/_registry.py b/src/evidently/features/_registry.py index b8286ee34f..0540174f24 100644 --- a/src/evidently/features/_registry.py +++ b/src/evidently/features/_registry.py @@ -82,6 +82,11 @@ register_type_alias( GeneratedFeatures, "evidently.features.words_feature.IncludesWords", "evidently:feature:IncludesWords" ) +register_type_alias( + GeneratedFeatures, "evidently.features.words_feature.RowWordPresence", "evidently:feature:RowWordPresence" +) +register_type_alias(GeneratedFeatures, "evidently.features.words_feature.WordMatch", "evidently:feature:WordMatch") +register_type_alias(GeneratedFeatures, "evidently.features.words_feature.WordNoMatch", "evidently:feature:WordNoMatch") register_type_alias( GeneratedFeatures, "evidently.features.words_feature.WordsPresence", "evidently:feature:WordsPresence" ) diff --git a/src/evidently/features/words_feature.py b/src/evidently/features/words_feature.py index a85131585c..f76e04150a 100644 --- a/src/evidently/features/words_feature.py +++ b/src/evidently/features/words_feature.py @@ -5,11 +5,15 @@ from typing import Optional import numpy as np +import pandas as pd from nltk.stem.wordnet import WordNetLemmatizer from evidently._pydantic_compat import PrivateAttr +from evidently.base_metric import ColumnName from evidently.core import ColumnType from evidently.features.generated_features import ApplyColumnGeneratedFeature +from evidently.features.generated_features import GeneratedFeature +from evidently.utils.data_preprocessing import DataDefinition def _listed_words_present( @@ -127,3 +131,83 @@ def _feature_display_name(self): f"Text Excludes {self.mode} words [{self.words_list}]," f" lemmatize: {self.lemmatize}] for {self.column_name}" ) + + +class RowWordPresence(GeneratedFeature): + class Config: + type_alias = "evidently:feature:RowWordPresence" + + __feature_type__: ClassVar = ColumnType.Categorical + columns: List[str] + mode: str = "any" + lemmatize: bool = True + _lem: Optional[WordNetLemmatizer] = PrivateAttr(None) + + def __init__(self, columns: List[str], mode: str, lemmatize: bool, display_name: Optional[str] = None): + self.columns = columns + self.mode = mode + self.lemmatize = lemmatize + self.display_name = display_name + super().__init__() + + def generate_feature(self, data: pd.DataFrame, data_definition: DataDefinition) -> pd.DataFrame: + generated_col = data[self.columns[0]].fillna("") + expected_col = data[self.columns[1]].fillna("") + + return pd.DataFrame( + { + self._feature_name(): pd.Series( + [ + _listed_words_present( + in_str=generated_str, + mode=self.mode, + lemmatize=self.lemmatize, + words_list=expected_words, + lem=self.lem, + ) + for generated_str, expected_words in zip(generated_col, expected_col) + ], + index=data.index, + ) + } + ) + + @property + def lem(self): + if self._lem is None: + import nltk + + nltk.download("wordnet", quiet=True) + self._lem = WordNetLemmatizer() + return self._lem + + def _feature_name(self): + return "_".join(["RowWordPresence", self.columns[0], self.columns[1], str(self.lemmatize), str(self.mode)]) + + +class WordMatch(RowWordPresence): + class Config: + type_alias = "evidently:feature:WordMatch" + + def __init__(self, columns: List[str], mode: str, lemmatize: bool, display_name: Optional[str] = None): + super().__init__(columns=columns, mode="includes_" + mode, lemmatize=lemmatize, display_name=display_name) + + def _as_column(self) -> "ColumnName": + return self._create_column( + self._feature_name(), + default_display_name=f"Text contains {self.mode.split('_')[1]} defined words", + ) + + +class WordNoMatch(RowWordPresence): + class Config: + type_alias = "evidently:feature:WordNoMatch" + + def __init__(self, columns: List[str], mode: str, lemmatize: bool, display_name: Optional[str] = None): + super().__init__(columns=columns, mode="excludes_" + mode, lemmatize=lemmatize, display_name=display_name) + + def _as_column(self) -> "ColumnName": + return self._create_column( + self._feature_name(), + default_display_name=f"Text does not contain {self.mode.split('_')[1]} defined words", + ) diff --git a/tests/features/test_words_feature.py b/tests/features/test_words_feature.py index 96b5617d0c..d58344e723 100644 --- a/tests/features/test_words_feature.py +++ b/tests/features/test_words_feature.py @@ -5,6 +5,8 @@ from evidently.features.words_feature import ExcludesWords from evidently.features.words_feature import IncludesWords +from evidently.features.words_feature import WordMatch +from evidently.features.words_feature import WordNoMatch from evidently.pipeline.column_mapping import ColumnMapping from evidently.utils.data_preprocessing import create_data_definition @@ -51,3 +53,83 @@ def test_excludes_words(words: List[str], mode: str, lemmatize: bool, expected: data_definition=create_data_definition(None, data, ColumnMapping()), ) assert result.equals(pd.DataFrame(dict([(feature_generator._feature_column_name(), expected)]))) + + +@pytest.mark.parametrize( + ["mode", "lemmatize", "expected"], + [ + ("any", False, [True, True, False, False, False, True]), + ("all", False, [False, True, False, False, False, True]), + ("any", True, [False, False, True, True, False, True]), + ("all", True, [False, False, True, False, False, True]), + ], +) +def test_word_match(mode: str, lemmatize: bool, expected: List[bool]): + data = { + "generated": [ + "I love eating apples and grapes.", + "I eat apples, grapes, and oranges", + "Grapes, oranges, apples.", + "Oranges are more sour than grapes.", + "This test doesn't have the words.", + "You are allowed to cancel at any time, and we guarantee that you will receive a refund.", + ], + "expected": [ + ["apples", "grapes", "oranges"], + ["grapes", "apples", "oranges"], + ["apple", "orange", "grape"], + ["orange", "sweet", "grape"], + ["none", "of", "these"], + ["guarantee", "allowed", "refund"], + ], + } + df = pd.DataFrame(data) + df["expected"] = df["expected"].apply(tuple) + feature_generator = WordMatch(columns=["generated", "expected"], mode=mode, lemmatize=lemmatize) + result = feature_generator.generate_feature( + data=df, + data_definition=create_data_definition(None, df, ColumnMapping()), + ) + assert result.equals(pd.DataFrame(dict([(feature_generator._feature_name(), expected)]))) + column_obj = feature_generator._as_column() + assert column_obj.display_name == f"Text contains {mode} defined words" + + +@pytest.mark.parametrize( + ["mode", "lemmatize", "expected"], + [ + ("any", False, [True, False, True, True, True, False]), + ("all", False, [False, False, True, True, True, False]), + ("any", True, [True, True, False, True, True, False]), + ("all", True, [True, True, False, False, True, False]), + ], +) +def test_word_no_match(mode: str, lemmatize: bool, expected: List[bool]): + data = { + "generated": [ + "I love eating apples and grapes.", + "I eat apples, grapes, and oranges", + "Grapes, oranges, apples.", + "Oranges are more sour than grapes.", + "This test doesn't have the words.", + "You are allowed to cancel at any time, and we guarantee that you will receive a refund.", + ], + "forbidden": [ + ["apples", "grapes", "oranges"], + ["grapes", "apples", "oranges"], + ["apple", "orange", "grape"], + ["orange", "sweet", "grape"], + ["none", "of", "these"], + ["guarantee", "allowed", "refund"], + ], + } + df = pd.DataFrame(data) + df["forbidden"] = df["forbidden"].apply(tuple) + feature_generator = WordNoMatch(columns=["generated", "forbidden"], mode=mode, lemmatize=lemmatize) + result = feature_generator.generate_feature( + data=df, + data_definition=create_data_definition(None, df, ColumnMapping()), + ) + assert result.equals(pd.DataFrame(dict([(feature_generator._feature_name(), expected)]))) + column_obj = feature_generator._as_column() + assert column_obj.display_name == f"Text does not contain {mode} defined words" From 5f8716897d31036d3424e6a9d61e7b1662aebf5c Mon Sep 17 00:00:00 2001 From: Mikhail Sveshnikov Date: Fri, 18 Oct 2024 16:50:39 +0400 Subject: [PATCH 004/103] use new wrapper/prompt api (#1346) * use new wrapper/prompt api * wip * mypy * mypy * oops * test run report with judge --- src/evidently/features/llm_judge.py | 210 +++++----------------------- src/evidently/ui/base.py | 38 +---- src/evidently/ui/workspace/view.py | 2 +- src/evidently/utils/llm/base.py | 2 +- src/evidently/utils/llm/wrapper.py | 2 +- src/evidently/utils/sync.py | 37 +++++ tests/features/test_llm_judge.py | 58 +++++++- 7 files changed, 126 insertions(+), 223 deletions(-) create mode 100644 src/evidently/utils/sync.py diff --git a/src/evidently/features/llm_judge.py b/src/evidently/features/llm_judge.py index 5a193c0853..bd69cb8b72 100644 --- a/src/evidently/features/llm_judge.py +++ b/src/evidently/features/llm_judge.py @@ -1,99 +1,42 @@ -import json -from abc import ABC from abc import abstractmethod from enum import Enum -from typing import Callable from typing import ClassVar from typing import Dict from typing import Iterator from typing import List from typing import Optional +from typing import Sequence from typing import Tuple -from typing import Type -from typing import Union import pandas as pd from evidently import ColumnType from evidently._pydantic_compat import Field from evidently._pydantic_compat import PrivateAttr -from evidently._pydantic_compat import SecretStr from evidently.base_metric import ColumnName -from evidently.errors import EvidentlyError from evidently.features.generated_features import GeneratedFeatures from evidently.options.base import Options -from evidently.options.option import Option from evidently.pydantic_utils import EnumValueMixin -from evidently.pydantic_utils import EvidentlyBaseModel from evidently.pydantic_utils import autoregister from evidently.utils.data_preprocessing import DataDefinition +from evidently.utils.llm.base import LLMMessage +from evidently.utils.llm.prompts import PromptBlock +from evidently.utils.llm.prompts import PromptTemplate +from evidently.utils.llm.wrapper import LLMRequest +from evidently.utils.llm.wrapper import LLMWrapper +from evidently.utils.llm.wrapper import get_llm_wrapper -LLMMessage = Tuple[str, str] -LLMResponse = Dict[str, Union[str, float]] - -class EvidentlyLLMError(EvidentlyError): - pass - - -class LLMResponseParseError(EvidentlyLLMError): - pass - - -class LLMRequestError(EvidentlyLLMError): - pass - - -class LLMWrapper(ABC): - __used_options__: ClassVar[List[Type[Option]]] = [] - - @abstractmethod - def complete(self, messages: List[LLMMessage]) -> str: - raise NotImplementedError - - def get_used_options(self) -> List[Type[Option]]: - return self.__used_options__ - - -LLMProvider = str -LLMModel = str -LLMWrapperProvider = Callable[[LLMModel, Options], LLMWrapper] -_wrappers: Dict[Tuple[LLMProvider, Optional[LLMModel]], LLMWrapperProvider] = {} - - -def llm_provider(name: LLMProvider, model: Optional[LLMModel]): - def dec(f: LLMWrapperProvider): - _wrappers[(name, model)] = f - return f - - return dec - - -def get_llm_wrapper(provider: LLMProvider, model: LLMModel, options: Options) -> LLMWrapper: - key: Tuple[str, Optional[str]] = (provider, model) - if key in _wrappers: - return _wrappers[key](model, options) - key = (provider, None) - if key in _wrappers: - return _wrappers[key](model, options) - raise ValueError(f"LLM wrapper for provider {provider} model {model} not found") - - -class BaseLLMPromptTemplate(EvidentlyBaseModel, ABC): +class BaseLLMPromptTemplate(PromptTemplate): class Config: is_base_type = True - @abstractmethod - def iterate_messages(self, data: pd.DataFrame, input_columns: Dict[str, str]) -> Iterator[LLMMessage]: - raise NotImplementedError - - @abstractmethod - def get_system_prompts(self) -> List[LLMMessage]: - raise NotImplementedError - - @abstractmethod - def parse_response(self, response: str) -> LLMResponse: - raise NotImplementedError + def iterate_messages(self, data: pd.DataFrame, input_columns: Dict[str, str]) -> Iterator[LLMRequest[dict]]: + template = self.get_template() + for _, column_values in data[list(input_columns)].rename(columns=input_columns).iterrows(): + yield LLMRequest( + messages=self.get_messages(column_values, template), response_parser=self.parse, response_type=dict + ) @abstractmethod def list_output_columns(self) -> List[str]: @@ -103,10 +46,6 @@ def list_output_columns(self) -> List[str]: def get_type(self, subcolumn: Optional[str]) -> ColumnType: raise NotImplementedError - @abstractmethod - def get_prompt_template(self) -> str: - raise NotImplementedError - class Uncertainty(str, Enum): UNKNOWN = "unknown" @@ -119,9 +58,6 @@ class BinaryClassificationPromptTemplate(BaseLLMPromptTemplate, EnumValueMixin): class Config: type_alias = "evidently:prompt_template:BinaryClassificationPromptTemplate" - template: str = ( - """{__criteria__}\n{__task__}\n\n{__as__}\n{{input}}\n{__ae__}\n\n{__instructions__}\n\n{__output_format__}""" - ) criteria: str = "" instructions_template: str = ( "Use the following categories for classification:\n{__categories__}\n{__scoring__}\nThink step by step." @@ -146,32 +82,6 @@ class Config: pre_messages: List[LLMMessage] = Field(default_factory=list) - def iterate_messages(self, data: pd.DataFrame, input_columns: Dict[str, str]) -> Iterator[LLMMessage]: - prompt_template = self.get_prompt_template() - for _, column_values in data[list(input_columns)].rename(columns=input_columns).iterrows(): - yield "user", prompt_template.format(**dict(column_values)) - - def get_prompt_template(self) -> str: - values = { - "__criteria__": self._criteria(), - "__task__": self._task(), - "__instructions__": self._instructions(), - "__output_format__": self._output_format(), - "__as__": self.anchor_start, - "__ae__": self.anchor_end, - **self.placeholders, - } - return self.template.format(**values) - - def _task(self): - return ( - f"Classify text between {self.anchor_start} and {self.anchor_end} " - f"into two categories: {self.target_category} and {self.non_target_category}." - ) - - def _criteria(self): - return self.criteria - def _instructions(self): categories = ( ( @@ -203,30 +113,30 @@ def _uncertainty_class(self): return self.target_category raise ValueError(f"Unknown uncertainty value: {self.uncertainty}") - def _output_format(self): - values = [] - columns = {} + def get_blocks(self) -> Sequence[PromptBlock]: + fields = {} if self.include_category: cat = f"{self.target_category} or {self.non_target_category}" if self.uncertainty == Uncertainty.UNKNOWN: cat += " or UNKNOWN" - columns[self.output_column] = f'"{cat}"' - values.append("category") + fields["category"] = (cat, self.output_column) if self.include_score: - columns[self.output_score_column] = "" - values.append("score") + fields["score"] = ("", self.output_score_column) if self.include_reasoning: - columns[self.output_reasoning_column] = '""' - values.append("reasoning") - - keys = "\n".join(f'"{k}": {v}' for k, v in columns.items()) - return f"Return {', '.join(values)} formatted as json without formatting as follows:\n{{{{\n{keys}\n}}}}" + fields["reasoning"] = ('""', self.output_reasoning_column) + return [ + PromptBlock.simple(self.criteria), + PromptBlock.simple( + f"Classify text between {self.anchor_start} and {self.anchor_end} " + f"into two categories: {self.target_category} and {self.non_target_category}." + ), + PromptBlock.input().anchored(self.anchor_start, self.anchor_end), + PromptBlock.simple(self._instructions()), + PromptBlock.json_output(**fields), + ] - def parse_response(self, response: str) -> LLMResponse: - try: - return json.loads(response) - except json.JSONDecodeError as e: - raise LLMResponseParseError(f"Failed to parse response '{response}' as json") from e + def get_messages(self, values, template: Optional[str] = None) -> List[LLMMessage]: + return [*self.pre_messages, *super().get_messages(values)] def list_output_columns(self) -> List[str]: result = [] @@ -247,9 +157,6 @@ def get_type(self, subcolumn: Optional[str]) -> ColumnType: return ColumnType.Categorical raise ValueError(f"Unknown subcolumn {subcolumn}") - def get_system_prompts(self) -> List[LLMMessage]: - return self.pre_messages - class LLMJudge(GeneratedFeatures): class Config: @@ -281,12 +188,10 @@ def get_input_columns(self): return {self.input_column: self.DEFAULT_INPUT_COLUMN} def generate_features(self, data: pd.DataFrame, data_definition: DataDefinition, options: Options) -> pd.DataFrame: - result: List[Dict[str, Union[str, float]]] = [] + result = self.get_llm_wrapper(options).run_batch_sync( + requests=self.template.iterate_messages(data, self.get_input_columns()) + ) - for message in self.template.iterate_messages(data, self.get_input_columns()): - messages: List[LLMMessage] = [*self.template.get_system_prompts(), message] - response = self.get_llm_wrapper(options).complete(messages) - result.append(self.template.parse_response(response)) return pd.DataFrame(result) def list_columns(self) -> List["ColumnName"]: @@ -300,50 +205,3 @@ def get_type(self, subcolumn: Optional[str] = None) -> ColumnType: subcolumn = self._extract_subcolumn_name(subcolumn) return self.template.get_type(subcolumn) - - -class OpenAIKey(Option): - api_key: Optional[SecretStr] = None - - def __init__(self, api_key: Optional[str] = None): - self.api_key = SecretStr(api_key) if api_key is not None else None - super().__init__() - - def get_value(self) -> Optional[str]: - if self.api_key is None: - return None - return self.api_key.get_secret_value() - - -@llm_provider("openai", None) -class OpenAIWrapper(LLMWrapper): - __used_options__: ClassVar = [OpenAIKey] - - def __init__(self, model: str, options: Options): - import openai - - self.model = model - self.client = openai.OpenAI(api_key=options.get(OpenAIKey).get_value()) - - def complete(self, messages: List[LLMMessage]) -> str: - import openai - - messages = [{"role": user, "content": msg} for user, msg in messages] - try: - response = self.client.chat.completions.create(model=self.model, messages=messages) # type: ignore[arg-type] - except openai.OpenAIError as e: - raise LLMRequestError("Failed to call OpenAI complete API") from e - content = response.choices[0].message.content - assert content is not None # todo: better error - return content - - -@llm_provider("litellm", None) -class LiteLLMWrapper(LLMWrapper): - def __init__(self, model: str): - self.model = model - - def complete(self, messages: List[LLMMessage]) -> str: - from litellm import completion - - return completion(model=self.model, messages=messages).choices[0].message.content diff --git a/src/evidently/ui/base.py b/src/evidently/ui/base.py index abf8c54183..f0ed0796d5 100644 --- a/src/evidently/ui/base.py +++ b/src/evidently/ui/base.py @@ -1,16 +1,11 @@ -import asyncio import contextlib import datetime import json -import threading from abc import ABC from abc import abstractmethod from enum import Enum -from functools import wraps from typing import IO from typing import Any -from typing import Awaitable -from typing import Callable from typing import ClassVar from typing import Dict from typing import Iterator @@ -59,30 +54,7 @@ from evidently.utils import NumpyEncoder from evidently.utils.dashboard import TemplateParams from evidently.utils.dashboard import inline_iframe_html_template - -_loop = asyncio.new_event_loop() - -_thr = threading.Thread(target=_loop.run_forever, name="Async Runner", daemon=True) - - -TA = TypeVar("TA") - - -def async_to_sync(awaitable: Awaitable[TA]) -> TA: - try: - asyncio.get_running_loop() - # we are in sync context but inside a running loop - if not _thr.is_alive(): - _thr.start() - future = asyncio.run_coroutine_threadsafe(awaitable, _loop) - return future.result() - except RuntimeError: - new_loop = asyncio.new_event_loop() - asyncio.set_event_loop(new_loop) - try: - return new_loop.run_until_complete(awaitable) - finally: - new_loop.close() +from evidently.utils.sync import sync_api class BlobMetadata(BaseModel): @@ -188,14 +160,6 @@ def _default_dashboard(): return DashboardConfig(name="", panels=[]) -def sync_api(f: Callable[..., Awaitable[TA]]) -> Callable[..., TA]: - @wraps(f) - def sync_call(*args, **kwargs): - return async_to_sync(f(*args, **kwargs)) - - return sync_call - - class Project(Entity): entity_type: ClassVar[EntityType] = EntityType.Project diff --git a/src/evidently/ui/workspace/view.py b/src/evidently/ui/workspace/view.py index faa5a02d76..fcacb4c224 100644 --- a/src/evidently/ui/workspace/view.py +++ b/src/evidently/ui/workspace/view.py @@ -9,7 +9,6 @@ from evidently.suite.base_suite import Snapshot from evidently.ui.base import Project from evidently.ui.base import ProjectManager -from evidently.ui.base import async_to_sync from evidently.ui.type_aliases import STR_UUID from evidently.ui.type_aliases import ZERO_UUID from evidently.ui.type_aliases import DatasetID @@ -17,6 +16,7 @@ from evidently.ui.type_aliases import TeamID from evidently.ui.type_aliases import UserID from evidently.ui.workspace.base import WorkspaceBase +from evidently.utils.sync import async_to_sync class WorkspaceView(WorkspaceBase): diff --git a/src/evidently/utils/llm/base.py b/src/evidently/utils/llm/base.py index 2abf77b571..b7852062a0 100644 --- a/src/evidently/utils/llm/base.py +++ b/src/evidently/utils/llm/base.py @@ -3,7 +3,7 @@ from typing import Dict -@dataclasses.dataclass +@dataclasses.dataclass(unsafe_hash=True, frozen=True) class LLMMessage: role: str content: str diff --git a/src/evidently/utils/llm/wrapper.py b/src/evidently/utils/llm/wrapper.py index ef26cdb68d..8f6a123a43 100644 --- a/src/evidently/utils/llm/wrapper.py +++ b/src/evidently/utils/llm/wrapper.py @@ -20,9 +20,9 @@ from evidently._pydantic_compat import SecretStr from evidently.options.base import Options from evidently.options.option import Option -from evidently.ui.base import sync_api from evidently.utils.llm.base import LLMMessage from evidently.utils.llm.errors import LLMRequestError +from evidently.utils.sync import sync_api TResult = TypeVar("TResult") diff --git a/src/evidently/utils/sync.py b/src/evidently/utils/sync.py new file mode 100644 index 0000000000..81efbb5f8e --- /dev/null +++ b/src/evidently/utils/sync.py @@ -0,0 +1,37 @@ +import asyncio +import threading +from functools import wraps +from typing import Awaitable +from typing import Callable +from typing import TypeVar + +_loop = asyncio.new_event_loop() + +_thr = threading.Thread(target=_loop.run_forever, name="Async Runner", daemon=True) + +TA = TypeVar("TA") + + +def async_to_sync(awaitable: Awaitable[TA]) -> TA: + try: + asyncio.get_running_loop() + # we are in sync context but inside a running loop + if not _thr.is_alive(): + _thr.start() + future = asyncio.run_coroutine_threadsafe(awaitable, _loop) + return future.result() + except RuntimeError: + new_loop = asyncio.new_event_loop() + asyncio.set_event_loop(new_loop) + try: + return new_loop.run_until_complete(awaitable) + finally: + new_loop.close() + + +def sync_api(f: Callable[..., Awaitable[TA]]) -> Callable[..., TA]: + @wraps(f) + def sync_call(*args, **kwargs): + return async_to_sync(f(*args, **kwargs)) + + return sync_call diff --git a/tests/features/test_llm_judge.py b/tests/features/test_llm_judge.py index 030891285e..96104912ff 100644 --- a/tests/features/test_llm_judge.py +++ b/tests/features/test_llm_judge.py @@ -9,14 +9,17 @@ import pandas as pd import pytest +from evidently.descriptors import NegativityLLMEval from evidently.features.llm_judge import BinaryClassificationPromptTemplate from evidently.features.llm_judge import LLMJudge from evidently.features.llm_judge import LLMMessage -from evidently.features.llm_judge import LLMResponseParseError from evidently.features.llm_judge import LLMWrapper -from evidently.features.llm_judge import llm_provider +from evidently.metric_preset import TextEvals from evidently.options.base import Options +from evidently.report import Report from evidently.utils.data_preprocessing import DataDefinition +from evidently.utils.llm.errors import LLMResponseParseError +from evidently.utils.llm.wrapper import llm_provider def _LLMPromptTemplate( @@ -74,14 +77,15 @@ def _LLMPromptTemplate( ], ) def test_parse_response( - template: _LLMPromptTemplate, results: Dict[str, Union[LLMResponseParseError, Dict[str, Union[str, float]]]] + template: BinaryClassificationPromptTemplate, + results: Dict[str, Union[LLMResponseParseError, Dict[str, Union[str, float]]]], ): for response, expected_result in results.items(): if isinstance(expected_result, LLMResponseParseError): with pytest.raises(expected_result.__class__): - template.parse_response(response) + template.parse(response) else: - assert template.parse_response(response) == expected_result + assert template.parse(response) == expected_result @llm_provider("mock", None) @@ -89,12 +93,13 @@ class MockLLMWrapper(LLMWrapper): def __init__(self, model: str, options: Options): self.model = model - def complete(self, messages: List[LLMMessage]) -> str: - text = messages[-1][1] + async def complete(self, messages: List[LLMMessage]) -> str: + text = messages[-1].content cat = re.findall("___text_starts_here___\n(.*)\n___text_ends_here___", text)[0][0] return json.dumps({"category": cat}) +@pytest.mark.asyncio def test_llm_judge(): llm_judge = LLMJudge( input_column="text", @@ -110,6 +115,7 @@ def test_llm_judge(): pd.testing.assert_frame_equal(fts, pd.DataFrame({"category": ["A", "B"]})) +@pytest.mark.asyncio def test_multicol_llm_judge(): llm_judge = LLMJudge( input_columns={"text": "input", "text2": "input2"}, @@ -123,3 +129,41 @@ def test_multicol_llm_judge(): dd = DataDefinition(columns={}, reference_present=False) fts = llm_judge.generate_features(data, dd, Options()) pd.testing.assert_frame_equal(fts, pd.DataFrame({"category": ["A", "B"]})) + + +def test_run_snapshot_with_llm_judge(): + data = pd.DataFrame({"text": ["A", "B"], "text2": ["C", "D"]}) + neg_eval = NegativityLLMEval( + input_columns={"text": "input", "text2": "input2"}, + provider="mock", + model="", + template=BinaryClassificationPromptTemplate(target_category="A", non_target_category="B"), + ) + report = Report(metrics=[TextEvals("text", descriptors=[neg_eval])]) + + report.run(current_data=data, reference_data=None) + report._inner_suite.raise_for_error() + assert report.as_dict() == { + "metrics": [ + { + "metric": "ColumnSummaryMetric", + "result": { + "column_name": "Negativity category", + "column_type": "cat", + "current_characteristics": { + "count": 2, + "missing": 0, + "missing_percentage": 0.0, + "most_common": "A", + "most_common_percentage": 50.0, + "new_in_current_values_count": None, + "number_of_rows": 2, + "unique": 2, + "unique_percentage": 100.0, + "unused_in_current_values_count": None, + }, + "reference_characteristics": None, + }, + } + ] + } From b61d96f42d98359dfda60cf7f8023ffe0dba0ee1 Mon Sep 17 00:00:00 2001 From: Sifr'un <36736908+Sifr-un@users.noreply.github.com> Date: Tue, 22 Oct 2024 16:25:03 +0200 Subject: [PATCH 005/103] Feat/add contains link descriptor (#1333) --- docs/book/reference/all-metrics.md | 1 + src/evidently/descriptors/__init__.py | 2 ++ src/evidently/descriptors/_registry.py | 5 +++ .../descriptors/contains_link_descriptor.py | 11 ++++++ src/evidently/features/_registry.py | 3 ++ .../features/contains_link_feature.py | 36 +++++++++++++++++++ tests/features/test_contains_link_feature.py | 35 ++++++++++++++++++ 7 files changed, 93 insertions(+) create mode 100644 src/evidently/descriptors/contains_link_descriptor.py create mode 100644 src/evidently/features/contains_link_feature.py create mode 100644 tests/features/test_contains_link_feature.py diff --git a/docs/book/reference/all-metrics.md b/docs/book/reference/all-metrics.md index 266090b422..71a233fcdf 100644 --- a/docs/book/reference/all-metrics.md +++ b/docs/book/reference/all-metrics.md @@ -279,6 +279,7 @@ Check for regular expression matches. | **ExactMatch()**
  • Checks if the text matches between two columns.
  • Returns True/False for every input.
Example use:
`ExactMatch(column_name='column_1')`| **Required:**
`with_column`

**Optional:**
  • `display_name`
| | **IsValidJSON()**
  • Checks if the text in a specified column is a valid JSON.
  • Returns True/False for every input.
Example use:
`IsValidJSON(column_name='column_1')`| **Required:**
`column_name`

**Optional:**
  • `display_name`
| | **JSONSchemaMatch()**
  • Checks if the text contains a JSON object matching the **expected_schema**. Supports exact (**exact=True**) or minimal (**exact=False**) matching, with optional strict type validation (**validate_types=True**).
  • Returns True/False for each row.
Example use:
`JSONSchemaMatch(expected_schema={"name": str, "age": int}, exact_match=False, validate_types=True)`| **Required:**
`expected_schema: Dict[str, type]`

**Optional:**
  • `exact_match = True` or `False`
  • `validate_types = True` or `False`
| +| **ContainsLink()**
  • Checks if the text contains at least one valid URL.
  • Returns True/False for each row.
Example use:
`ContainsLink(column_name='column_1')`| **Required:**
`column_name: str`

**Optional:**
  • `display_name`
| ## Descriptors: Text stats diff --git a/src/evidently/descriptors/__init__.py b/src/evidently/descriptors/__init__.py index 3f78f60f84..f5c80aea45 100644 --- a/src/evidently/descriptors/__init__.py +++ b/src/evidently/descriptors/__init__.py @@ -1,4 +1,5 @@ from . import _registry +from .contains_link_descriptor import ContainsLink from .custom_descriptor import CustomColumnEval from .custom_descriptor import CustomPairColumnEval from .exact_match_descriptor import ExactMatch @@ -65,6 +66,7 @@ "Sentiment", "ExactMatch", "RegExp", + "ContainsLink", "WordMatch", "WordNoMatch", "IsValidJSON", diff --git a/src/evidently/descriptors/_registry.py b/src/evidently/descriptors/_registry.py index d47c1b5777..d476a2cee5 100644 --- a/src/evidently/descriptors/_registry.py +++ b/src/evidently/descriptors/_registry.py @@ -117,6 +117,11 @@ "evidently.descriptors.custom_descriptor.CustomPairColumnEval", "evidently:descriptor:CustomPairColumnEval", ) +register_type_alias( + FeatureDescriptor, + "evidently.descriptors.contains_link_descriptor.ContainsLink", + "evidently:descriptor:ContainsLink", +) register_type_alias( FeatureDescriptor, "evidently.descriptors.exact_match_descriptor.ExactMatch", "evidently:descriptor:ExactMatch" ) diff --git a/src/evidently/descriptors/contains_link_descriptor.py b/src/evidently/descriptors/contains_link_descriptor.py new file mode 100644 index 0000000000..dfc78f7e6a --- /dev/null +++ b/src/evidently/descriptors/contains_link_descriptor.py @@ -0,0 +1,11 @@ +from evidently.features import contains_link_feature +from evidently.features.generated_features import FeatureDescriptor +from evidently.features.generated_features import GeneratedFeature + + +class ContainsLink(FeatureDescriptor): + class Config: + type_alias = "evidently:descriptor:ContainsLink" + + def feature(self, column_name: str) -> GeneratedFeature: + return contains_link_feature.ContainsLink(column_name, self.display_name) diff --git a/src/evidently/features/_registry.py b/src/evidently/features/_registry.py index 0540174f24..8566150913 100644 --- a/src/evidently/features/_registry.py +++ b/src/evidently/features/_registry.py @@ -90,6 +90,9 @@ register_type_alias( GeneratedFeatures, "evidently.features.words_feature.WordsPresence", "evidently:feature:WordsPresence" ) +register_type_alias( + GeneratedFeatures, "evidently.features.contains_link_feature.ContainsLink", "evidently:feature:ContainsLink" +) register_type_alias( GeneratedFeatures, "evidently.features.exact_match_feature.ExactMatchFeature", "evidently:feature:ExactMatchFeature" ) diff --git a/src/evidently/features/contains_link_feature.py b/src/evidently/features/contains_link_feature.py new file mode 100644 index 0000000000..3d1978363a --- /dev/null +++ b/src/evidently/features/contains_link_feature.py @@ -0,0 +1,36 @@ +from typing import Any +from typing import ClassVar +from typing import Optional +from urllib.parse import urlparse + +import numpy as np + +from evidently import ColumnType +from evidently.features.generated_features import ApplyColumnGeneratedFeature + + +class ContainsLink(ApplyColumnGeneratedFeature): + class Config: + type_alias = "evidently:feature:ContainsLink" + + __feature_type__: ClassVar = ColumnType.Categorical + display_name_template: ClassVar = "{column_name} contains link" + column_name: str + + def __init__(self, column_name: str, display_name: Optional[str] = None): + self.column_name = column_name + self.display_name = display_name + super().__init__() + + def apply(self, value: Any): + if value is None or (isinstance(value, float) and np.isnan(value)): + return 0 + # Split the text into words + words = str(value).split() + + # Check if any word is a valid URL using urlparse + for word in words: + parsed = urlparse(word) + if parsed.scheme and parsed.netloc: + return True + return False diff --git a/tests/features/test_contains_link_feature.py b/tests/features/test_contains_link_feature.py new file mode 100644 index 0000000000..ab95f05d3a --- /dev/null +++ b/tests/features/test_contains_link_feature.py @@ -0,0 +1,35 @@ +import pandas as pd + +from evidently.features.contains_link_feature import ContainsLink +from evidently.pipeline.column_mapping import ColumnMapping +from evidently.utils.data_preprocessing import create_data_definition + + +def test_contains_link_feature(): + # Initialize the ContainsLink feature generator for column_1 + feature_generator = ContainsLink("column_1") + + # Sample data with varying texts that contain or don't contain links + data = pd.DataFrame( + dict( + column_1=[ + "Check out https://example.com for more info", # Contains a valid link + "Visit our website at http://www.test.com.", # Contains a valid link + "No link here, just plain text", # No link + "Another string without a link", # No link + "Here is a malformed link: www.test.com", # Invalid link (missing scheme) + ] + ) + ) + + # Generate the feature + result = feature_generator.generate_feature( + data=data, + data_definition=create_data_definition(None, data, ColumnMapping()), + ) + + # Expected result: True for valid links, False otherwise + expected_result = pd.DataFrame(dict(column_1=[True, True, False, False, False])) + + # Assert that the generated result matches the expected result + assert result.equals(expected_result) From d3e21fb657118f82e5e388223347e57e313c800e Mon Sep 17 00:00:00 2001 From: Vyacheslav Morov Date: Tue, 22 Oct 2024 17:18:26 +0200 Subject: [PATCH 006/103] Fix embeddings test presets error (#1347) --- src/evidently/test_preset/data_drift.py | 11 +++--- .../test_preset/no_target_performance.py | 15 ++++---- tests/test_preset/test_data_drift_preset.py | 25 ++++++++++++ .../test_no_target_performance_preset.py | 38 +++++++++++++++++++ 4 files changed, 77 insertions(+), 12 deletions(-) create mode 100644 tests/test_preset/test_data_drift_preset.py create mode 100644 tests/test_preset/test_no_target_performance_preset.py diff --git a/src/evidently/test_preset/data_drift.py b/src/evidently/test_preset/data_drift.py index dfec627696..29373e9166 100644 --- a/src/evidently/test_preset/data_drift.py +++ b/src/evidently/test_preset/data_drift.py @@ -85,21 +85,22 @@ def generate_tests( self, data_definition: DataDefinition, additional_data: Optional[Dict[str, Any]] ) -> List[AnyTest]: embeddings_data = data_definition.embeddings + columns = self.columns if embeddings_data is not None: embs = list(set(v for values in embeddings_data.values() for v in values)) - if self.columns is None: - self.columns = list( + if columns is None: + columns = list( np.setdiff1d( [column.column_name for column in data_definition.get_columns(features_only=True)], embs, ) ) else: - self.columns = list(np.setdiff1d(self.columns, embs)) + columns = list(np.setdiff1d(columns, embs)) preset_tests: list = [ TestShareOfDriftedColumns( - columns=self.columns, + columns=columns, lt=0.3 if self.drift_share is None else self.drift_share, stattest=self.stattest, cat_stattest=self.cat_stattest, @@ -164,7 +165,7 @@ def generate_tests( preset_tests.append( TestAllFeaturesValueDrift( - self.columns, + columns, self.stattest, self.cat_stattest, self.num_stattest, diff --git a/src/evidently/test_preset/no_target_performance.py b/src/evidently/test_preset/no_target_performance.py index 92cc6888a1..96bccd0fc8 100644 --- a/src/evidently/test_preset/no_target_performance.py +++ b/src/evidently/test_preset/no_target_performance.py @@ -97,17 +97,18 @@ def generate_tests( self, data_definition: DataDefinition, additional_data: Optional[Dict[str, Any]] ) -> List[AnyTest]: embeddings_data = data_definition.embeddings + columns = self.columns if embeddings_data is not None: embs = list(set(v for values in embeddings_data.values() for v in values)) - if self.columns is None: - self.columns = list( + if columns is None: + columns = list( np.setdiff1d( [column.column_name for column in data_definition.get_columns(features_only=True)], embs, ) ) else: - self.columns = list(np.setdiff1d(self.columns, embs)) + columns = list(np.setdiff1d(columns, embs)) preset_tests: List = [] @@ -150,10 +151,10 @@ def generate_tests( ) ) preset_tests.append(TestColumnsType()) - preset_tests.append(TestAllColumnsShareOfMissingValues(columns=self.columns)) - preset_tests.append(TestNumColumnsOutOfRangeValues(columns=self.columns)) - preset_tests.append(TestCatColumnsOutOfListValues(columns=self.columns)) - preset_tests.append(TestNumColumnsMeanInNSigmas(columns=self.columns)) + preset_tests.append(TestAllColumnsShareOfMissingValues(columns=columns)) + preset_tests.append(TestNumColumnsOutOfRangeValues(columns=columns)) + preset_tests.append(TestCatColumnsOutOfListValues(columns=columns)) + preset_tests.append(TestNumColumnsMeanInNSigmas(columns=columns)) if embeddings_data is None: return preset_tests diff --git a/tests/test_preset/test_data_drift_preset.py b/tests/test_preset/test_data_drift_preset.py new file mode 100644 index 0000000000..dfbbd05241 --- /dev/null +++ b/tests/test_preset/test_data_drift_preset.py @@ -0,0 +1,25 @@ +from evidently import ColumnType +from evidently.test_preset import DataDriftTestPreset +from evidently.tests import TestAllFeaturesValueDrift +from evidently.tests import TestEmbeddingsDrift +from evidently.tests import TestShareOfDriftedColumns +from evidently.utils.data_preprocessing import ColumnDefinition +from evidently.utils.data_preprocessing import DataDefinition + + +def test_embeddings_data_drift_preset(): + data_definition = DataDefinition( + columns={ + "target": ColumnDefinition("target", ColumnType.Numerical), + }, + embeddings={ + "small_set": ["col_1", "col_2"], + "big_set": ["col_3", "col_4"], + }, + reference_present=True, + ) + preset = DataDriftTestPreset(embeddings=["small_set", "big_set"]) + tests = preset.generate_tests(data_definition=data_definition, additional_data=None) + assert len(tests) == 4 + expected_tests = [TestShareOfDriftedColumns, TestAllFeaturesValueDrift, TestEmbeddingsDrift, TestEmbeddingsDrift] + assert expected_tests == [type(test) for test in tests] diff --git a/tests/test_preset/test_no_target_performance_preset.py b/tests/test_preset/test_no_target_performance_preset.py new file mode 100644 index 0000000000..1ef1ecd025 --- /dev/null +++ b/tests/test_preset/test_no_target_performance_preset.py @@ -0,0 +1,38 @@ +from evidently import ColumnType +from evidently.test_preset import NoTargetPerformanceTestPreset +from evidently.tests import TestAllColumnsShareOfMissingValues +from evidently.tests import TestCatColumnsOutOfListValues +from evidently.tests import TestColumnsType +from evidently.tests import TestEmbeddingsDrift +from evidently.tests import TestNumColumnsMeanInNSigmas +from evidently.tests import TestNumColumnsOutOfRangeValues +from evidently.tests import TestShareOfDriftedColumns +from evidently.utils.data_preprocessing import ColumnDefinition +from evidently.utils.data_preprocessing import DataDefinition + + +def test_embeddings_data_drift_preset(): + data_definition = DataDefinition( + columns={ + "target": ColumnDefinition("target", ColumnType.Numerical), + }, + embeddings={ + "small_set": ["col_1", "col_2"], + "big_set": ["col_3", "col_4"], + }, + reference_present=True, + ) + preset = NoTargetPerformanceTestPreset(embeddings=["small_set", "big_set"]) + tests = preset.generate_tests(data_definition=data_definition, additional_data=None) + assert len(tests) == 8 + expected_tests = [ + TestShareOfDriftedColumns, + TestColumnsType, + TestAllColumnsShareOfMissingValues, + TestNumColumnsOutOfRangeValues, + TestCatColumnsOutOfListValues, + TestNumColumnsMeanInNSigmas, + TestEmbeddingsDrift, + TestEmbeddingsDrift, + ] + assert expected_tests == [type(test) for test in tests] From 3106c3b7ad66826bfe18afa27b14cfa894c12dae Mon Sep 17 00:00:00 2001 From: Vyacheslav Morov Date: Tue, 22 Oct 2024 17:59:54 +0200 Subject: [PATCH 007/103] Add missing type aliases in custom metrics in how_to_make_custom_metric_and_test example. (#1349) --- .../how_to_make_custom_metric_and_test.ipynb | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/examples/how_to_questions/how_to_make_custom_metric_and_test.ipynb b/examples/how_to_questions/how_to_make_custom_metric_and_test.ipynb index cb15262531..1822d3dec2 100644 --- a/examples/how_to_questions/how_to_make_custom_metric_and_test.ipynb +++ b/examples/how_to_questions/how_to_make_custom_metric_and_test.ipynb @@ -98,6 +98,8 @@ "outputs": [], "source": [ "class MyMetricResult(MetricResult):\n", + " class Config:\n", + " type_alias = \"evidently:metric_result:MyMetricResult\"\n", " sum_value: float" ] }, @@ -119,6 +121,8 @@ "outputs": [], "source": [ "class MyMetric(Metric[MyMetricResult]):\n", + " class Config:\n", + " type_alias = \"evidently:metric:MyMetric\"\n", " column_name: str\n", "\n", " def __init__(self, column_name: str):\n", @@ -235,6 +239,8 @@ "\n", "\n", "class MyMetricResult(MetricResult):\n", + " class Config:\n", + " type_alias = \"evidently:metric_result:MyMetricResult\"\n", " feature_name: str\n", " current_sum_value: float\n", " x_values_for_hist: list\n", @@ -243,6 +249,8 @@ "\n", "\n", "class MyMetric(Metric[MyMetricResult]):\n", + " class Config:\n", + " type_alias = \"evidently:metric:MyMetric\"\n", " column_name: str\n", "\n", " def __init__(self, column_name: str) -> None:\n", From 082f131d7314ffaf26096b3bb5f929753642a563 Mon Sep 17 00:00:00 2001 From: Vyacheslav Morov Date: Tue, 22 Oct 2024 18:00:13 +0200 Subject: [PATCH 008/103] Remove TextOverviewPresets references from examples and docs (#1348) --- ...w_to_run_calculations_over_text_data.ipynb | 47 ------------------- 1 file changed, 47 deletions(-) diff --git a/examples/how_to_questions/how_to_run_calculations_over_text_data.ipynb b/examples/how_to_questions/how_to_run_calculations_over_text_data.ipynb index 5a0bcefae9..7e0ae5c011 100644 --- a/examples/how_to_questions/how_to_run_calculations_over_text_data.ipynb +++ b/examples/how_to_questions/how_to_run_calculations_over_text_data.ipynb @@ -44,7 +44,6 @@ "from evidently.metric_preset import RegressionPreset\n", "from evidently.metric_preset import ClassificationPreset\n", "from evidently.metric_preset import TargetDriftPreset\n", - "from evidently.metric_preset import TextOverviewPreset\n", "\n", "from evidently.metrics import *\n", "\n", @@ -360,52 +359,6 @@ "classification_report.json()" ] }, - { - "cell_type": "markdown", - "metadata": { - "id": "IAe0g1SWlV4L" - }, - "source": [ - "# Text Overview Preset" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/", - "height": 1000 - }, - "id": "9H8xHPN-tYY8", - "outputId": "644abbf8-d717-484f-8125-902b99288c59" - }, - "outputs": [], - "source": [ - "text_overview_report = Report(metrics=[\n", - " TextOverviewPreset(column_name=\"Review_Text\")\n", - "])\n", - "\n", - "text_overview_report.run(reference_data=reviews_ref, current_data=reviews_cur, column_mapping=column_mapping)\n", - "text_overview_report" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/", - "height": 104 - }, - "id": "msjgy3j-f-5i", - "outputId": "0e86becd-b75d-42f9-d115-72de002f8786" - }, - "outputs": [], - "source": [ - "text_overview_report.json()" - ] - }, { "cell_type": "markdown", "metadata": { From edfa07a5f0f3ebe8714f3428c668cc95df105a94 Mon Sep 17 00:00:00 2001 From: Vyacheslav Morov Date: Tue, 22 Oct 2024 22:03:46 +0200 Subject: [PATCH 009/103] Fix llm judge example and fix ignore for llm examples. (#1350) --- .github/workflows/examples.yml | 2 +- example_test.py | 4 ++-- .../how_to_use_llm_judge_template.ipynb | 11 +++++------ 3 files changed, 8 insertions(+), 9 deletions(-) diff --git a/.github/workflows/examples.yml b/.github/workflows/examples.yml index b36aa7e80e..6f4062502b 100644 --- a/.github/workflows/examples.yml +++ b/.github/workflows/examples.yml @@ -42,7 +42,7 @@ jobs: if: matrix.minimal run: pip install -r requirements.min.txt - name: Prepare examples dependencies - run: pip install catboost sentence-transformers + run: pip install catboost sentence-transformers openai - name: Export examples run: jupyter nbconvert --to python examples/*/*.ipynb --output-dir example_scripts - name: Run examples diff --git a/example_test.py b/example_test.py index 9132656429..ae4c71ffe4 100644 --- a/example_test.py +++ b/example_test.py @@ -12,8 +12,8 @@ "comparing_custom_statest_with_classic_distributions.py", "how_to_evaluate_llm_with_text_descriptors.py", "how_to_run_drift_report_for_text_data.py", # too slow & torch version conflict? - "llm_evaluation_tutorial.ipynb", # cloud usage - "llm_tracing_tutorial.ipynb", # cloud usage + "llm_evaluation_tutorial.py", # cloud usage + "llm_tracing_tutorial.py", # cloud usage ] diff --git a/examples/how_to_questions/how_to_use_llm_judge_template.ipynb b/examples/how_to_questions/how_to_use_llm_judge_template.ipynb index d6f57a6ea5..3623818fa7 100644 --- a/examples/how_to_questions/how_to_use_llm_judge_template.ipynb +++ b/examples/how_to_questions/how_to_use_llm_judge_template.ipynb @@ -191,9 +191,7 @@ "id": "204d90a4-694e-406b-949a-f7ba3b601eac", "metadata": {}, "outputs": [], - "source": [ - "print(ToxicityLLMEval().get_template().get_prompt_template())" - ] + "source": "print(ToxicityLLMEval().get_template().get_template())" }, { "cell_type": "code", @@ -308,7 +306,7 @@ "source": [ "#that's how you can see the prompt\n", "\n", - "print(ContextQualityLLMEval(question=\"question\").get_template().get_prompt_template())" + "print(ContextQualityLLMEval(question=\"question\").get_template().get_template())" ] }, { @@ -414,12 +412,13 @@ " pre_messages=[(\"system\", \"You are a judge which evaluates text.\")],\n", " ),\n", " provider = \"openai\",\n", - " model = \"gpt-4o-mini\"\n", + " model = \"gpt-4o-mini\",\n", + " display_name=\"test\"\n", ")\n", "\n", "report = Report(metrics=[\n", " TextEvals(column_name=\"response\", descriptors=[\n", - " custom_judge(display_name=\"test\")\n", + " custom_judge\n", " ])\n", "])\n", "\n", From 3c3b9fbcdfa7f162cb1f286d2bbc2d6bc34e053a Mon Sep 17 00:00:00 2001 From: Vyacheslav Morov Date: Tue, 15 Oct 2024 14:28:31 +0200 Subject: [PATCH 010/103] Add default value for options in base Metric class. --- src/evidently/base_metric.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/evidently/base_metric.py b/src/evidently/base_metric.py index 29d559078b..7852708db8 100644 --- a/src/evidently/base_metric.py +++ b/src/evidently/base_metric.py @@ -18,6 +18,7 @@ import pandas as pd import typing_inspect +from evidently._pydantic_compat import Field from evidently._pydantic_compat import ModelMetaclass from evidently._pydantic_compat import PrivateAttr from evidently.core import BaseResult @@ -236,7 +237,7 @@ class Config: _context: Optional["Context"] = None - options: Options + options: Options = Field(default_factory=Options) fields: ClassVar[FieldsDescriptor] = FieldsDescriptor() # resulting options will be determined via From e36174012042d5de1b45a1a3de73ae731da5db35 Mon Sep 17 00:00:00 2001 From: Vyacheslav Morov Date: Tue, 15 Oct 2024 14:29:21 +0200 Subject: [PATCH 011/103] Update mypy version and rules. --- requirements.dev.txt | 2 +- setup.cfg | 8 ++------ setup.py | 2 +- 3 files changed, 4 insertions(+), 8 deletions(-) diff --git a/requirements.dev.txt b/requirements.dev.txt index 7f8701e3ca..b0da653b4a 100644 --- a/requirements.dev.txt +++ b/requirements.dev.txt @@ -3,7 +3,7 @@ wheel==0.38.1 setuptools==65.5.1; python_version < '3.12' setuptools==68.2.2; python_version >= '3.12' jupyter==1.0.0 -mypy==0.981 +mypy==1.1.1 pytest==7.4.4 pytest-asyncio==0.23.7 types-PyYAML==6.0.1 diff --git a/setup.cfg b/setup.cfg index 231d1f6f6c..2bd67531fc 100644 --- a/setup.cfg +++ b/setup.cfg @@ -26,6 +26,8 @@ files = src/evidently python_version = 3.8 disable_error_code = misc namespace_packages = true +no_implicit_optional = False +plugins = pydantic.mypy,pydantic.v1.mypy,numpy.typing.mypy_plugin [mypy-nltk.*] ignore_missing_imports = True @@ -46,12 +48,6 @@ ignore_missing_imports = True [mypy-sentence_transformers.*] ignore_missing_imports = True -[mypy-pandas.*] -ignore_missing_imports = True - -[mypy-numpy.*] -ignore_missing_imports = True - [mypy-scipy.*] ignore_missing_imports = True diff --git a/setup.py b/setup.py index df67f1a052..bcab894123 100644 --- a/setup.py +++ b/setup.py @@ -84,7 +84,7 @@ "setuptools==65.5.1; python_version < '3.12'", "setuptools==68.2.2; python_version >= '3.12'", "jupyter==1.0.0", - "mypy==0.981", + "mypy==1.1.1", "pytest==7.4.4", "types-PyYAML==6.0.1", "types-requests==2.26.0", From 899c783f613bd1142e7f750d25051e027b8b688a Mon Sep 17 00:00:00 2001 From: Vyacheslav Morov Date: Tue, 15 Oct 2024 14:29:57 +0200 Subject: [PATCH 012/103] Ignore signature missmatch due to skiprows. --- src/evidently/runner/loader.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/evidently/runner/loader.py b/src/evidently/runner/loader.py index 50ca1a737e..82ea0d9868 100644 --- a/src/evidently/runner/loader.py +++ b/src/evidently/runner/loader.py @@ -64,7 +64,7 @@ def __init__(self): def load(self, filename: str, data_options: DataOptions, sampling_options: SamplingOptions = None): sampling_opts = SamplingOptions("none", 0, 0) if sampling_options is None else sampling_options parse_dates = [data_options.date_column] if data_options.date_column else False - return pd.read_csv( + return pd.read_csv( # type: ignore[call-overload] filename, header=0 if data_options.header else None, sep=data_options.separator, From 5628c8b3c30bebcab4e6a8263de1f3d774a784e8 Mon Sep 17 00:00:00 2001 From: Vyacheslav Morov Date: Tue, 15 Oct 2024 14:30:33 +0200 Subject: [PATCH 013/103] Add named argument label for tests TestRecallByClass, TestPrecisionByClass. --- src/evidently/test_preset/classification_multiclass.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/evidently/test_preset/classification_multiclass.py b/src/evidently/test_preset/classification_multiclass.py index 85d9a26410..55fecde560 100644 --- a/src/evidently/test_preset/classification_multiclass.py +++ b/src/evidently/test_preset/classification_multiclass.py @@ -70,8 +70,8 @@ def generate_tests( tests: List[AnyTest] = [ TestAccuracyScore(), TestF1Score(), - *[TestPrecisionByClass(label) for label in labels], - *[TestRecallByClass(label) for label in labels], + *[TestPrecisionByClass(label=label) for label in labels], + *[TestRecallByClass(label=label) for label in labels], TestNumberOfRows(), TestColumnDrift( column_name=target.column_name, From 53d3b7b50535d69e099a4ff8d7de6095998456c9 Mon Sep 17 00:00:00 2001 From: Vyacheslav Morov Date: Tue, 15 Oct 2024 14:32:57 +0200 Subject: [PATCH 014/103] Make TestRenderer to be generics. --- src/evidently/renderers/base_renderer.py | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/src/evidently/renderers/base_renderer.py b/src/evidently/renderers/base_renderer.py index ff7237092c..cf15778641 100644 --- a/src/evidently/renderers/base_renderer.py +++ b/src/evidently/renderers/base_renderer.py @@ -2,8 +2,10 @@ import warnings from typing import TYPE_CHECKING from typing import Dict +from typing import Generic from typing import List from typing import Optional +from typing import TypeVar from typing import Union import pandas as pd @@ -78,14 +80,17 @@ def with_details(self, title: str, info: BaseWidgetInfo): return self -class TestRenderer(BaseRenderer): - def html_description(self, obj: "Test"): +TTest = TypeVar("TTest", bound="Test") + + +class TestRenderer(Generic[TTest], BaseRenderer): + def html_description(self, obj: TTest): return obj.get_result().description - def json_description(self, obj: "Test"): + def json_description(self, obj: TTest): return obj.get_result().description - def render_html(self, obj: "Test") -> TestHtmlInfo: + def render_html(self, obj: TTest) -> TestHtmlInfo: result = obj.get_result() return TestHtmlInfo( name=result.name, @@ -97,7 +102,7 @@ def render_html(self, obj: "Test") -> TestHtmlInfo: def render_json( self, - obj: "Test", + obj: TTest, include_render: bool = False, include: "IncludeOptions" = None, exclude: "IncludeOptions" = None, From 46a71b3a23e24d8d818608adf8ba9635de7161c3 Mon Sep 17 00:00:00 2001 From: Vyacheslav Morov Date: Tue, 15 Oct 2024 14:43:55 +0200 Subject: [PATCH 015/103] Make MetricRenderer generic. --- src/evidently/renderers/base_renderer.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/src/evidently/renderers/base_renderer.py b/src/evidently/renderers/base_renderer.py index cf15778641..a03c93f0a3 100644 --- a/src/evidently/renderers/base_renderer.py +++ b/src/evidently/renderers/base_renderer.py @@ -18,7 +18,6 @@ if TYPE_CHECKING: from evidently.base_metric import Metric - from evidently.base_metric import TResult from evidently.core import IncludeOptions from evidently.tests.base_test import Test @@ -36,8 +35,11 @@ def __init__(self, color_options: Optional[ColorOptions] = None) -> None: self.color_options = color_options -class MetricRenderer(BaseRenderer): - def render_pandas(self, obj: "Metric[TResult]") -> pd.DataFrame: +TMetric = TypeVar("TMetric", bound="Metric") + + +class MetricRenderer(Generic[TMetric], BaseRenderer): + def render_pandas(self, obj: TMetric) -> pd.DataFrame: result = obj.get_result() if not result.__config__.pd_include: warnings.warn( @@ -48,7 +50,7 @@ def render_pandas(self, obj: "Metric[TResult]") -> pd.DataFrame: def render_json( self, - obj: "Metric[TResult]", + obj: TMetric, include_render: bool = False, include: "IncludeOptions" = None, exclude: "IncludeOptions" = None, @@ -56,7 +58,7 @@ def render_json( result = obj.get_result() return result.get_dict(include_render=include_render, include=include, exclude=exclude) - def render_html(self, obj) -> List[BaseWidgetInfo]: + def render_html(self, obj: TMetric) -> List[BaseWidgetInfo]: raise NotImplementedError() From 9ca452573b028c17794159ca27ce5e5a018f9cee Mon Sep 17 00:00:00 2001 From: Vyacheslav Morov Date: Tue, 15 Oct 2024 14:44:50 +0200 Subject: [PATCH 016/103] Ignore assignment for different versions of pydantic. --- src/evidently/_pydantic_compat.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/evidently/_pydantic_compat.py b/src/evidently/_pydantic_compat.py index 5b0cf2ba0d..3cef08bff5 100644 --- a/src/evidently/_pydantic_compat.py +++ b/src/evidently/_pydantic_compat.py @@ -30,13 +30,13 @@ from pydantic.v1.typing import DictStrAny else: - from pydantic import BaseConfig - from pydantic import BaseModel - from pydantic import Extra - from pydantic import Field + from pydantic import BaseConfig # type: ignore[assignment] + from pydantic import BaseModel # type: ignore[assignment] + from pydantic import Extra # type: ignore[assignment] + from pydantic import Field # type: ignore[assignment] from pydantic import PrivateAttr - from pydantic import SecretStr - from pydantic import ValidationError + from pydantic import SecretStr # type: ignore[assignment] + from pydantic import ValidationError # type: ignore[assignment] from pydantic import parse_obj_as from pydantic import validator from pydantic.fields import SHAPE_DICT # type: ignore[attr-defined,no-redef] From 284ff4a56c30ac0b2f5f2c7ef4c9a5f8b4554d96 Mon Sep 17 00:00:00 2001 From: Vyacheslav Morov Date: Tue, 15 Oct 2024 14:45:51 +0200 Subject: [PATCH 017/103] Add missing return for method get_condition_from_reference --- src/evidently/tests/data_quality_tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/evidently/tests/data_quality_tests.py b/src/evidently/tests/data_quality_tests.py index a57eb7c127..a78c50495c 100644 --- a/src/evidently/tests/data_quality_tests.py +++ b/src/evidently/tests/data_quality_tests.py @@ -388,7 +388,7 @@ def __init__( ) def get_condition_from_reference(self, reference: Optional[DatasetCorrelation]) -> TestValueCondition: - pass + raise NotImplementedError() def get_condition(self) -> TestValueCondition: if self.condition.has_condition(): From 483e7439ebfc503c63f133bf561e642317155ba9 Mon Sep 17 00:00:00 2001 From: Vyacheslav Morov Date: Wed, 16 Oct 2024 10:02:51 +0200 Subject: [PATCH 018/103] Fix types in visualizations functions. --- src/evidently/utils/visualizations.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/evidently/utils/visualizations.py b/src/evidently/utils/visualizations.py index f32180aa6c..e91219b275 100644 --- a/src/evidently/utils/visualizations.py +++ b/src/evidently/utils/visualizations.py @@ -1389,10 +1389,10 @@ def plot_metric_k(curr_data: pd.Series, ref_data: Optional[pd.Series], yaxis_nam def plot_bias( - curr: Distribution, - curr_train: Distribution, - ref: Optional[Distribution], - ref_train: Optional[Distribution], + curr: HistogramData, + curr_train: HistogramData, + ref: Optional[HistogramData], + ref_train: Optional[HistogramData], xaxis_name: str, ): color_options = ColorOptions() @@ -1444,10 +1444,10 @@ def plot_bias( def plot_4_distr( - curr_1: Distribution, - curr_2: Optional[Distribution], - ref_1: Optional[Distribution], - ref_2: Optional[Distribution], + curr_1: HistogramData, + curr_2: Optional[HistogramData], + ref_1: Optional[HistogramData], + ref_2: Optional[HistogramData], name_1: str, name_2: str, xaxis_name: str, From e40e9205dad7820f09f12ed64b9d1721afc3db8e Mon Sep 17 00:00:00 2001 From: Vyacheslav Morov Date: Wed, 16 Oct 2024 10:03:30 +0200 Subject: [PATCH 019/103] Fix types in html_widgets get_lift_plot_data function. --- src/evidently/renderers/html_widgets.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/evidently/renderers/html_widgets.py b/src/evidently/renderers/html_widgets.py index 36f632e772..4d84a43cb0 100644 --- a/src/evidently/renderers/html_widgets.py +++ b/src/evidently/renderers/html_widgets.py @@ -731,7 +731,7 @@ def get_pr_rec_plot_data( def get_lift_plot_data( current_lift_curve: LiftCurve, - reference_lift_curve: Optional[PRCurve], + reference_lift_curve: Optional[LiftCurve], color_options: ColorOptions, ) -> List[Tuple[str, BaseWidgetInfo]]: """ From 7b052627aa1b806d83420c29235e8b0a0b501221 Mon Sep 17 00:00:00 2001 From: Vyacheslav Morov Date: Wed, 16 Oct 2024 10:34:37 +0200 Subject: [PATCH 020/103] Replace columns list assignment with explicit pd.Index --- src/evidently/calculations/recommender_systems.py | 2 +- src/evidently/calculations/utils.py | 2 +- .../class_separation_metric.py | 2 +- .../quality_by_feature_table.py | 4 ++-- .../metrics/data_drift/target_by_features_table.py | 4 ++-- .../data_drift/text_descriptors_drift_metric.py | 4 ++-- .../metrics/data_integrity/column_summary_metric.py | 12 ++++++------ .../metrics/data_quality/column_category_metric.py | 4 ++-- .../data_quality/dataset_correlations_metric.py | 4 ++-- .../text_descriptors_correlation_metric.py | 4 ++-- .../regression_performance/error_bias_table.py | 4 ++-- src/evidently/tests/data_integrity_tests.py | 4 ++-- src/evidently/tests/data_quality_tests.py | 4 ++-- 13 files changed, 27 insertions(+), 27 deletions(-) diff --git a/src/evidently/calculations/recommender_systems.py b/src/evidently/calculations/recommender_systems.py index c0666fcb06..4b5fdb7bf4 100644 --- a/src/evidently/calculations/recommender_systems.py +++ b/src/evidently/calculations/recommender_systems.py @@ -16,7 +16,7 @@ def collect_dataset( bin_data: bool, ): df = pd.concat([users, target, preds], axis=1) - df.columns = ["users", "target", "preds"] + df.columns = pd.Index(["users", "target", "preds"]) if min_rel_score: df["target"] = (df["target"] >= min_rel_score).astype(int) if recommendations_type == RecomType.SCORE: diff --git a/src/evidently/calculations/utils.py b/src/evidently/calculations/utils.py index ed7c22d328..ee4a6a2e5f 100644 --- a/src/evidently/calculations/utils.py +++ b/src/evidently/calculations/utils.py @@ -125,7 +125,7 @@ def prepare_box_data( for df, name in zip(dfs, names): data = df.groupby(cat_feature_name, observed=False)[num_feature_name] df_for_plot = data.quantile([0, 0.25, 0.5, 0.75, 1]).reset_index() - df_for_plot.columns = [cat_feature_name, "q", num_feature_name] + df_for_plot.columns = pd.Index([cat_feature_name, "q", num_feature_name]) res_df = {} values = df_for_plot[cat_feature_name].unique() diff --git a/src/evidently/metrics/classification_performance/class_separation_metric.py b/src/evidently/metrics/classification_performance/class_separation_metric.py index 485989bd0d..2c0d024244 100644 --- a/src/evidently/metrics/classification_performance/class_separation_metric.py +++ b/src/evidently/metrics/classification_performance/class_separation_metric.py @@ -54,7 +54,7 @@ def prepare_box_data(df: pd.DataFrame, target_name: str, prediction_names: List[ df_name = df.copy() df_name[target_name] = (df_name[target_name] == name).astype(int) df_for_plot = df_name.groupby(target_name)[name].quantile([0, 0.25, 0.5, 0.75, 1]).reset_index() - df_for_plot.columns = [target_name, "q", name] + df_for_plot.columns = pd.Index([target_name, "q", name]) res_df = pd.DataFrame() values = df_for_plot[target_name].unique() diff --git a/src/evidently/metrics/classification_performance/quality_by_feature_table.py b/src/evidently/metrics/classification_performance/quality_by_feature_table.py index c89235cff4..edf400caca 100644 --- a/src/evidently/metrics/classification_performance/quality_by_feature_table.py +++ b/src/evidently/metrics/classification_performance/quality_by_feature_table.py @@ -143,7 +143,7 @@ def calculate(self, data: InputData) -> ClassificationQualityByFeatureTableResul columns.remove(column) columns += list(features.keys()) curr_text_df = pd.concat([data.get_current_column(x.as_column()) for x in features.values()], axis=1) - curr_text_df.columns = list(features.keys()) + curr_text_df.columns = pd.Index(list(features.keys())) curr_df = pd.concat([curr_df.reset_index(drop=True), curr_text_df.reset_index(drop=True)], axis=1) if ref_df is not None: @@ -151,7 +151,7 @@ def calculate(self, data: InputData) -> ClassificationQualityByFeatureTableResul [data.get_reference_column(x.as_column()) for x in features.values()], axis=1, ) - ref_text_df.columns = list(features.keys()) + ref_text_df.columns = pd.Index(list(features.keys())) ref_df = pd.concat([ref_df.reset_index(drop=True), ref_text_df.reset_index(drop=True)], axis=1) table_columns = set(columns + [target_name]) diff --git a/src/evidently/metrics/data_drift/target_by_features_table.py b/src/evidently/metrics/data_drift/target_by_features_table.py index af8d7cd9c4..bbba4010dc 100644 --- a/src/evidently/metrics/data_drift/target_by_features_table.py +++ b/src/evidently/metrics/data_drift/target_by_features_table.py @@ -167,7 +167,7 @@ def calculate(self, data: InputData) -> TargetByFeaturesTableResults: [data.get_current_column(x.as_column()) for x in list(self._text_features_gen[col].values())], axis=1, ) - curr_text_df.columns = list(self._text_features_gen[col].keys()) + curr_text_df.columns = pd.Index(list(self._text_features_gen[col].keys())) curr_df = pd.concat( [ curr_df.reset_index(drop=True), @@ -181,7 +181,7 @@ def calculate(self, data: InputData) -> TargetByFeaturesTableResults: [data.get_reference_column(x.as_column()) for x in list(self._text_features_gen[col].values())], axis=1, ) - ref_text_df.columns = list(self._text_features_gen[col].keys()) + ref_text_df.columns = pd.Index(list(self._text_features_gen[col].keys())) ref_df = pd.concat( [ ref_df.reset_index(drop=True), diff --git a/src/evidently/metrics/data_drift/text_descriptors_drift_metric.py b/src/evidently/metrics/data_drift/text_descriptors_drift_metric.py index 5bd2c38135..2c8d01c4ac 100644 --- a/src/evidently/metrics/data_drift/text_descriptors_drift_metric.py +++ b/src/evidently/metrics/data_drift/text_descriptors_drift_metric.py @@ -115,13 +115,13 @@ def calculate(self, data: InputData) -> TextDescriptorsDriftMetricResults: [data.get_current_column(x.as_column()) for x in list(self.generated_text_features.values())], axis=1, ) - curr_text_df.columns = list(self.generated_text_features.keys()) + curr_text_df.columns = pd.Index(list(self.generated_text_features.keys())) ref_text_df = pd.concat( [data.get_reference_column(x.as_column()) for x in list(self.generated_text_features.values())], axis=1, ) - ref_text_df.columns = list(self.generated_text_features.keys()) + ref_text_df.columns = pd.Index(list(self.generated_text_features.keys())) # text_dataset_columns = DatasetColumns(num_feature_names=curr_text_df.columns) text_dataset_columns = process_columns(ref_text_df, ColumnMapping(numerical_features=ref_text_df.columns)) diff --git a/src/evidently/metrics/data_integrity/column_summary_metric.py b/src/evidently/metrics/data_integrity/column_summary_metric.py index eaf286be8f..9aeb75cb47 100644 --- a/src/evidently/metrics/data_integrity/column_summary_metric.py +++ b/src/evidently/metrics/data_integrity/column_summary_metric.py @@ -204,20 +204,20 @@ def plot_data( elif column_type == ColumnType.Datetime: prefix, freq = choose_agg_period(current_data, reference_data) curr_data = current_data.dt.to_period(freq=freq).value_counts().reset_index() - curr_data.columns = ["x", "number_of_items"] + curr_data.columns = pd.Index(["x", "number_of_items"]) curr_data["x"] = curr_data["x"].dt.to_timestamp() reference = None if reference_data is not None: ref_data = reference_data.dt.to_period(freq=freq).value_counts().reset_index() - ref_data.columns = ["x", "number_of_items"] + ref_data.columns = pd.Index(["x", "number_of_items"]) ref_data["x"] = ref_data["x"].dt.to_timestamp() max_ref_date = ref_data["x"].max() min_curr_date = curr_data["x"].min() if max_ref_date == min_curr_date: curr_data, ref_data = _split_periods(curr_data, ref_data, "x") reference = ref_data - reference.columns = ["x", "count"] - curr_data.columns = ["x", "count"] + reference.columns = pd.Index(["x", "count"]) + curr_data.columns = pd.Index(["x", "count"]) data_hist = Histogram( current=HistogramData.from_df(curr_data), reference=HistogramData.from_df(reference) if reference is not None else None, @@ -458,11 +458,11 @@ def calculate(self, data: InputData) -> ColumnSummaryResult: if column_type in [ColumnType.Categorical, ColumnType.Numerical]: counts_of_values = {} current_counts = column_current_data.value_counts(dropna=False).reset_index() - current_counts.columns = ["x", "count"] + current_counts.columns = pd.Index(["x", "count"]) counts_of_values["current"] = current_counts.head(10) if column_reference_data is not None: reference_counts = column_reference_data.value_counts(dropna=False).reset_index() - reference_counts.columns = ["x", "count"] + reference_counts.columns = pd.Index(["x", "count"]) counts_of_values["reference"] = reference_counts.head(10) return ColumnSummaryResult( diff --git a/src/evidently/metrics/data_quality/column_category_metric.py b/src/evidently/metrics/data_quality/column_category_metric.py index 380db52257..6d94b97aa6 100644 --- a/src/evidently/metrics/data_quality/column_category_metric.py +++ b/src/evidently/metrics/data_quality/column_category_metric.py @@ -105,12 +105,12 @@ def calculate(self, data: InputData) -> ColumnCategoryMetricResult: counts_of_values = {} current_counts = current_column.value_counts(dropna=False).reset_index() - current_counts.columns = ["x", "count"] + current_counts.columns = pd.Index(["x", "count"]) counts_of_values["current"] = current_counts.head(10) counts_of_values["current"].index = counts_of_values["current"].index.astype("str") if reference_column is not None: reference_counts = reference_column.value_counts(dropna=False).reset_index() - reference_counts.columns = ["x", "count"] + reference_counts.columns = pd.Index(["x", "count"]) counts_of_values["reference"] = reference_counts.head(10) counts_of_values["reference"].index = counts_of_values["reference"].index.astype("str") diff --git a/src/evidently/metrics/data_quality/dataset_correlations_metric.py b/src/evidently/metrics/data_quality/dataset_correlations_metric.py index ffaa3f5038..529c747d34 100644 --- a/src/evidently/metrics/data_quality/dataset_correlations_metric.py +++ b/src/evidently/metrics/data_quality/dataset_correlations_metric.py @@ -261,7 +261,7 @@ def calculate(self, data: InputData) -> DatasetCorrelationsMetricResult: [data.get_current_column(x.as_column()) for x in list(self._text_features_gen[col].values())], axis=1, ) - curr_text_df.columns = list(self._text_features_gen[col].keys()) + curr_text_df.columns = pd.Index(list(self._text_features_gen[col].keys())) text_columns.append(list(curr_text_df.columns)) curr_df = pd.concat( [ @@ -276,7 +276,7 @@ def calculate(self, data: InputData) -> DatasetCorrelationsMetricResult: [data.get_reference_column(x.as_column()) for x in list(self._text_features_gen[col].values())], axis=1, ) - ref_text_df.columns = list(self._text_features_gen[col].keys()) + ref_text_df.columns = pd.Index(list(self._text_features_gen[col].keys())) ref_df = pd.concat( [ ref_df.copy().reset_index(drop=True), diff --git a/src/evidently/metrics/data_quality/text_descriptors_correlation_metric.py b/src/evidently/metrics/data_quality/text_descriptors_correlation_metric.py index ce8b7bf785..22ebfa7c58 100644 --- a/src/evidently/metrics/data_quality/text_descriptors_correlation_metric.py +++ b/src/evidently/metrics/data_quality/text_descriptors_correlation_metric.py @@ -96,14 +96,14 @@ def calculate(self, data: InputData) -> TextDescriptorsCorrelationMetricResult: [data.get_current_column(x.as_column()) for x in list(self.generated_text_features.values())], axis=1, ) - curr_text_df.columns = list(self.generated_text_features.keys()) + curr_text_df.columns = pd.Index(list(self.generated_text_features.keys())) ref_df = None if data.reference_data is not None: ref_text_df = pd.concat( [data.get_reference_column(x.as_column()) for x in list(self.generated_text_features.values())], axis=1, ) - ref_text_df.columns = list(self.generated_text_features.keys()) + ref_text_df.columns = pd.Index(list(self.generated_text_features.keys())) ref_df = pd.concat( [ data.reference_data.copy().reset_index(drop=True), diff --git a/src/evidently/metrics/regression_performance/error_bias_table.py b/src/evidently/metrics/regression_performance/error_bias_table.py index d8875a026d..1472af241f 100644 --- a/src/evidently/metrics/regression_performance/error_bias_table.py +++ b/src/evidently/metrics/regression_performance/error_bias_table.py @@ -182,7 +182,7 @@ def calculate(self, data: InputData) -> RegressionErrorBiasTableResults: num_feature_names += list(features.keys()) columns += list(features.keys()) curr_text_df = pd.concat([data.get_current_column(x.as_column()) for x in features.values()], axis=1) - curr_text_df.columns = list(features.keys()) + curr_text_df.columns = pd.Index(list(features.keys())) curr_df = pd.concat([curr_df.reset_index(drop=True), curr_text_df.reset_index(drop=True)], axis=1) if ref_df is not None: @@ -190,7 +190,7 @@ def calculate(self, data: InputData) -> RegressionErrorBiasTableResults: [data.get_reference_column(x.as_column()) for x in features.values()], axis=1, ) - ref_text_df.columns = list(features.keys()) + ref_text_df.columns = pd.Index(list(features.keys())) ref_df = pd.concat([ref_df.reset_index(drop=True), ref_text_df.reset_index(drop=True)], axis=1) columns_ext = np.union1d(columns, [target_name, prediction_name]) diff --git a/src/evidently/tests/data_integrity_tests.py b/src/evidently/tests/data_integrity_tests.py index 301785d9c5..214e6a6fa2 100644 --- a/src/evidently/tests/data_integrity_tests.py +++ b/src/evidently/tests/data_integrity_tests.py @@ -1146,7 +1146,7 @@ def render_html(self, obj: TestColumnRegExp) -> TestHtmlInfo: if metric_result.current.table_of_not_matched: curr_df = pd.DataFrame(metric_result.current.table_of_not_matched.items()) - curr_df.columns = ["x", "count"] + curr_df.columns = pd.Index(["x", "count"]) else: curr_df = pd.DataFrame(columns=["x", "count"]) @@ -1155,7 +1155,7 @@ def render_html(self, obj: TestColumnRegExp) -> TestHtmlInfo: if metric_result.reference is not None and metric_result.reference.table_of_not_matched: ref_df = pd.DataFrame(metric_result.reference.table_of_not_matched.items()) - ref_df.columns = ["x", "count"] + ref_df.columns = pd.Index(["x", "count"]) additional_plots = plot_value_counts_tables_ref_curr( column_name, curr_df, ref_df, f"{column_name}_ColumnValueRegExp" diff --git a/src/evidently/tests/data_quality_tests.py b/src/evidently/tests/data_quality_tests.py index a78c50495c..b04962af70 100644 --- a/src/evidently/tests/data_quality_tests.py +++ b/src/evidently/tests/data_quality_tests.py @@ -1727,7 +1727,7 @@ def get_value_counts_table_with_percents( ref_df = ref_df.copy() replace.append(("reference value counts", n_ref)) df = curr_df.merge(ref_df, on="x", how="outer") - df.columns = ["value", "current value counts", "reference value counts"] + df.columns = pd.Index(["value", "current value counts", "reference value counts"]) df[["current value counts", "reference value counts"]] = df[ ["current value counts", "reference value counts"] ].fillna(0.0) @@ -1735,7 +1735,7 @@ def get_value_counts_table_with_percents( else: df = curr_df - df.columns = ["value", "current value counts"] + df.columns = pd.Index(["value", "current value counts"]) df.sort_values("current value counts", ascending=False, inplace=True) for col, n in replace: df[col] = self._get_number_and_percents(df[col].fillna(0), n) From 9035eb24d066334eeab611061c03403790e247bc Mon Sep 17 00:00:00 2001 From: Vyacheslav Morov Date: Wed, 16 Oct 2024 10:36:09 +0200 Subject: [PATCH 021/103] Explicit check for LocalStorage in app as we require it. --- src/evidently/ui/app.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/evidently/ui/app.py b/src/evidently/ui/app.py index 8dd42872f7..4a6ce949ad 100644 --- a/src/evidently/ui/app.py +++ b/src/evidently/ui/app.py @@ -4,6 +4,7 @@ from evidently._pydantic_compat import SecretStr from evidently.ui.components.base import AppBuilder +from evidently.ui.components.storage import LocalStorageComponent from evidently.ui.config import AppConfig from evidently.ui.config import load_config from evidently.ui.config import settings @@ -37,6 +38,8 @@ def get_config( config = load_config(LocalConfig, settings) config.service.host = host config.service.port = port + if not isinstance(config.storage, LocalStorageComponent): + raise ValueError("Storage component is not a LocalStorageComponent") config.storage.path = workspace secret = secret or os.environ.get(EVIDENTLY_SECRET_ENV) From 35fbad688ee13dd07cc9866d8693e8e512de59c6 Mon Sep 17 00:00:00 2001 From: Vyacheslav Morov Date: Wed, 16 Oct 2024 10:57:48 +0200 Subject: [PATCH 022/103] Fix problem with missing reference in TestCorrelationChanges. --- src/evidently/tests/data_quality_tests.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/evidently/tests/data_quality_tests.py b/src/evidently/tests/data_quality_tests.py index b04962af70..36c54bead2 100644 --- a/src/evidently/tests/data_quality_tests.py +++ b/src/evidently/tests/data_quality_tests.py @@ -411,9 +411,10 @@ def calculate_value_for_test(self) -> Optional[Numeric]: return (diff.abs() > self.corr_diff).sum().sum() / 2 current_correlations = current_correlations[self.column_name.display_name] - if reference_correlations is not None: - reference_correlations = reference_correlations[self.column_name.display_name] - diff = reference_correlations - current_correlations + if reference_correlations is None: + raise ValueError("Reference is required for test") + reference_correlations_data = reference_correlations[self.column_name.display_name] + diff = reference_correlations_data - current_correlations return (diff.abs() > self.corr_diff).sum() def get_description(self, value: Numeric) -> str: @@ -1708,7 +1709,7 @@ def get_parameters(self) -> CheckValueParameters: @default_renderer(wrap_type=TestCategoryShare) class TestCategoryRenderer(TestRenderer): @staticmethod - def _get_number_and_percents(s: pd.Series, num: int) -> pd.DataFrame: + def _get_number_and_percents(s: pd.Series, num: int) -> pd.Series: """Get a string with missing values numbers and percents from info for results table""" return s.astype(str) + " (" + (s / num * 100).round(2).astype(str) + "%)" From 4611385ec7f28d319349233d311821473a4a1632 Mon Sep 17 00:00:00 2001 From: Vyacheslav Morov Date: Wed, 16 Oct 2024 11:08:35 +0200 Subject: [PATCH 023/103] Ignore type error in get_component. --- src/evidently/ui/config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/evidently/ui/config.py b/src/evidently/ui/config.py index 53a35f00a0..f63f6695a8 100644 --- a/src/evidently/ui/config.py +++ b/src/evidently/ui/config.py @@ -37,7 +37,7 @@ def __init__(self, config: "Config", components_mapping: Dict[Type[Component], C def get_component(self, type_: Type[T]) -> T: for cls in self.components_mapping: if issubclass(cls, type_): - return self.components_mapping[cls] + return self.components_mapping[cls] # type: ignore[return-value] raise ValueError(f"Component of type {type_.__name__} not found") @property From a845a3aef2edafc672bdecbb99772fa88be3dacd Mon Sep 17 00:00:00 2001 From: Vyacheslav Morov Date: Wed, 16 Oct 2024 11:09:04 +0200 Subject: [PATCH 024/103] Make Recsys tests generic. --- src/evidently/tests/recsys_tests.py | 27 ++++++++++++++++----------- 1 file changed, 16 insertions(+), 11 deletions(-) diff --git a/src/evidently/tests/recsys_tests.py b/src/evidently/tests/recsys_tests.py index bc9b27671f..02c2568e21 100644 --- a/src/evidently/tests/recsys_tests.py +++ b/src/evidently/tests/recsys_tests.py @@ -1,7 +1,9 @@ import abc from typing import ClassVar +from typing import Generic from typing import List from typing import Optional +from typing import TypeVar from typing import Union from evidently.metric_results import HistogramData @@ -267,13 +269,16 @@ class TestMRRKRenderer(BaseTopkRecsysRenderer): ] -class BaseNotRankRecsysTest(BaseCheckValueTest, abc.ABC): +TBaseNotRankRecsysType = TypeVar("TBaseNotRankRecsysType") + + +class BaseNotRankRecsysTest(Generic[TBaseNotRankRecsysType], BaseCheckValueTest, abc.ABC): group: ClassVar = RECSYS_GROUP.id header: str k: int min_rel_score: Optional[int] item_features: Optional[List[str]] - _metric: BaseNotRankRecsysType + _metric: TBaseNotRankRecsysType def __init__( self, @@ -323,7 +328,7 @@ def get_description(self, value: Numeric) -> str: return f"{self.header}@{self.k} is {value:.3}. The test threshold is {self.get_condition()}" @abc.abstractmethod - def get_metric(self, k, min_rel_score, item_features) -> BaseTopKRecsysType: + def get_metric(self, k, min_rel_score, item_features) -> TBaseNotRankRecsysType: raise NotImplementedError() @property @@ -353,14 +358,14 @@ def render_html(self, obj: BaseNotRankRecsysTest) -> TestHtmlInfo: return info -class TestNovelty(BaseNotRankRecsysTest): +class TestNovelty(BaseNotRankRecsysTest[NoveltyMetric]): class Config: type_alias = "evidently:test:TestNovelty" name: ClassVar = "Novelty (top-k)" header: str = "Novelty" - def get_metric(self, k, min_rel_score, item_features) -> BaseNotRankRecsysType: + def get_metric(self, k, min_rel_score, item_features) -> NoveltyMetric: return NoveltyMetric(k=k) @@ -369,14 +374,14 @@ class TestNoveltyRenderer(BaseNotRankRecsysTestRenderer): xaxis_name = "novelty by user" -class TestDiversity(BaseNotRankRecsysTest): +class TestDiversity(BaseNotRankRecsysTest[DiversityMetric]): class Config: type_alias = "evidently:test:TestDiversity" name: ClassVar = "Diversity (top-k)" header: str = "Diversity" - def get_metric(self, k, min_rel_score, item_features) -> BaseNotRankRecsysType: + def get_metric(self, k, min_rel_score, item_features) -> DiversityMetric: return DiversityMetric(k=k, item_features=item_features) @@ -385,14 +390,14 @@ class TestDiversityRenderer(BaseNotRankRecsysTestRenderer): xaxis_name = "intra list diversity by user" -class TestSerendipity(BaseNotRankRecsysTest): +class TestSerendipity(BaseNotRankRecsysTest[SerendipityMetric]): class Config: type_alias = "evidently:test:TestSerendipity" name: ClassVar = "Serendipity (top-k)" header: str = "Serendipity" - def get_metric(self, k, min_rel_score, item_features) -> BaseNotRankRecsysType: + def get_metric(self, k, min_rel_score, item_features) -> SerendipityMetric: return SerendipityMetric(k=k, min_rel_score=min_rel_score, item_features=item_features) @@ -401,14 +406,14 @@ class TestSerendipityRenderer(BaseNotRankRecsysTestRenderer): xaxis_name = "serendipity by user" -class TestPersonalization(BaseNotRankRecsysTest): +class TestPersonalization(BaseNotRankRecsysTest[PersonalizationMetric]): class Config: type_alias = "evidently:test:TestPersonalization" name: ClassVar = "Personalization (top-k)" header: str = "Personalization" - def get_metric(self, k, min_rel_score, item_features) -> BaseNotRankRecsysType: + def get_metric(self, k, min_rel_score, item_features) -> PersonalizationMetric: return PersonalizationMetric(k=k) From 06a1d26f897e9337912f3f953431319f52f61ec6 Mon Sep 17 00:00:00 2001 From: Vyacheslav Morov Date: Wed, 16 Oct 2024 11:14:08 +0200 Subject: [PATCH 025/103] Add types in _feature_render_html. --- src/evidently/tests/data_quality_tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/evidently/tests/data_quality_tests.py b/src/evidently/tests/data_quality_tests.py index 36c54bead2..7b39feb5e6 100644 --- a/src/evidently/tests/data_quality_tests.py +++ b/src/evidently/tests/data_quality_tests.py @@ -630,7 +630,7 @@ def render_html(self, obj: BaseFeatureDataQualityMetricsTest) -> TestHtmlInfo: info.with_details(f"{obj.name} {column_name}", plotly_figure(title="", figure=fig)) return info - def _feature_render_html(self, obj): + def _feature_render_html(self, obj: BaseFeatureDataQualityMetricsTest): column_name = obj.column_name info = super().render_html(obj) metric_result: ColumnSummaryResult = obj.metric.get_result() From ef71527fc0b25b2d01dc561f6dc19cd7858fc383 Mon Sep 17 00:00:00 2001 From: Vyacheslav Morov Date: Wed, 16 Oct 2024 11:21:05 +0200 Subject: [PATCH 026/103] Fix types errors in text_descriptors_drift_metric.py --- .../data_drift/text_descriptors_drift_metric.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/src/evidently/metrics/data_drift/text_descriptors_drift_metric.py b/src/evidently/metrics/data_drift/text_descriptors_drift_metric.py index 2c8d01c4ac..78ae9321c3 100644 --- a/src/evidently/metrics/data_drift/text_descriptors_drift_metric.py +++ b/src/evidently/metrics/data_drift/text_descriptors_drift_metric.py @@ -20,6 +20,7 @@ from evidently.features.generated_features import GeneratedFeature from evidently.metric_results import DatasetColumns from evidently.metric_results import HistogramData +from evidently.metric_results import ScatterField from evidently.model.widget import BaseWidgetInfo from evidently.options.base import AnyOptions from evidently.options.data_drift import DataDriftOptions @@ -123,7 +124,10 @@ def calculate(self, data: InputData) -> TextDescriptorsDriftMetricResults: ) ref_text_df.columns = pd.Index(list(self.generated_text_features.keys())) # text_dataset_columns = DatasetColumns(num_feature_names=curr_text_df.columns) - text_dataset_columns = process_columns(ref_text_df, ColumnMapping(numerical_features=ref_text_df.columns)) + text_dataset_columns = process_columns( + ref_text_df, + ColumnMapping(numerical_features=ref_text_df.columns.tolist()), + ) drift_by_columns: Dict[str, ColumnDataDriftMetrics] = {} for col in curr_text_df.columns: @@ -173,9 +177,11 @@ def _generate_column_params( data_drift = "Detected" if data.drift_detected else "Not Detected" if data.column_type == "num" and data.scatter is not None: if not agg_data: + if not isinstance(data.scatter, ScatterField): + raise ValueError(f"TypeMismatch, data.scatter({type(data.scatter)}) expected to be ScatterField ") scatter_fig = plot_scatter_for_data_drift( - curr_y=data.scatter.scatter[data.column_name], - curr_x=data.scatter.scatter[data.scatter.x_name], + curr_y=data.scatter.scatter[data.column_name].tolist(), + curr_x=data.scatter.scatter[data.scatter.x_name].tolist(), y0=data.scatter.plot_shape["y0"], y1=data.scatter.plot_shape["y1"], y_name=data.column_name, From fa90a1fbee0214ef70db43606b1cb54c91a65176 Mon Sep 17 00:00:00 2001 From: Vyacheslav Morov Date: Wed, 16 Oct 2024 11:30:07 +0200 Subject: [PATCH 027/103] Add Optional to column argument in calculate_numerical_correlation. --- src/evidently/calculations/data_quality.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/evidently/calculations/data_quality.py b/src/evidently/calculations/data_quality.py index 3575ab7489..ef9eff60fe 100644 --- a/src/evidently/calculations/data_quality.py +++ b/src/evidently/calculations/data_quality.py @@ -449,10 +449,10 @@ def calculate_category_correlation( def calculate_numerical_correlation( column_display_name: str, - column: pd.Series, + column: Optional[pd.Series], features: pd.DataFrame, ) -> List[ColumnCorrelations]: - if column.empty or features.empty: + if column is None or column.empty or features.empty: return [] result = [] From c0f75c9fc94c0f6231248e11bd9bceeb0513e5bd Mon Sep 17 00:00:00 2001 From: Vyacheslav Morov Date: Wed, 16 Oct 2024 13:05:59 +0200 Subject: [PATCH 028/103] Fix typing errors in classification_dummy_metric. --- .../classification_dummy_metric.py | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/src/evidently/metrics/classification_performance/classification_dummy_metric.py b/src/evidently/metrics/classification_performance/classification_dummy_metric.py index a8ca44f0cb..cdf9452b1a 100644 --- a/src/evidently/metrics/classification_performance/classification_dummy_metric.py +++ b/src/evidently/metrics/classification_performance/classification_dummy_metric.py @@ -76,8 +76,8 @@ def calculate(self, data: InputData) -> ClassificationDummyMetricResults: # dummy by current labels_ratio = data.current_data[target_name].value_counts(normalize=True) np.random.seed(0) - dummy_preds = np.random.choice(labels_ratio.index, data.current_data.shape[0], p=labels_ratio) - dummy_preds = pd.Series(dummy_preds) + dummy_preds_choices = np.random.choice(labels_ratio.index, data.current_data.shape[0], p=labels_ratio) + dummy_preds = pd.Series(dummy_preds_choices) prediction: Optional[PredictionData] = None if prediction_name is not None: @@ -147,7 +147,7 @@ def calculate(self, data: InputData) -> ClassificationDummyMetricResults: } if prediction is not None and prediction.prediction_probas is not None: # dummy log_loss and roc_auc - binaraized_target = (target.astype(str).values.reshape(-1, 1) == list(labels)).astype(int) + binaraized_target = (target.astype(str).to_numpy().reshape(-1, 1) == list(labels)).astype(int) dummy_prediction = np.full( prediction.prediction_probas.shape, 1 / prediction.prediction_probas.shape[1], @@ -161,8 +161,8 @@ def calculate(self, data: InputData) -> ClassificationDummyMetricResults: if data.reference_data is not None: labels_ratio = data.reference_data[target_name].value_counts(normalize=True) np.random.seed(1) - dummy_preds = np.random.choice(labels_ratio.index, data.current_data.shape[0], p=labels_ratio) - dummy_preds = pd.Series(dummy_preds) + dummy_preds_choices = np.random.choice(labels_ratio.index, data.current_data.shape[0], p=labels_ratio) + dummy_preds = pd.Series(dummy_preds_choices) if prediction_name is not None: target, prediction = self.get_target_prediction_data(data.current_data, data.column_mapping) @@ -192,7 +192,7 @@ def calculate(self, data: InputData) -> ClassificationDummyMetricResults: ) if prediction is not None and prediction.prediction_probas is not None: # dummy log_loss and roc_auc - binaraized_target = (target.astype(str).values.reshape(-1, 1) == list(labels)).astype(int) + binaraized_target = (target.astype(str).to_numpy().reshape(-1, 1) == list(labels)).astype(int) dummy_prediction = np.full( prediction.prediction_probas.shape, 1 / prediction.prediction_probas.shape[1], @@ -298,5 +298,9 @@ def render_html(self, obj: ClassificationDummyMetric) -> List[BaseWidgetInfo]: return [ header_text(label="Dummy Classification Quality"), - table_data(column_names=columns, data=np.around(in_table_data, 3).values, title=""), + table_data( + column_names=columns, + data=np.around(in_table_data, 3).values, # type: ignore[attr-defined] + title="", + ), ] From 267793a3819043cee08266cdb399f5d0b8d591fd Mon Sep 17 00:00:00 2001 From: Vyacheslav Morov Date: Wed, 16 Oct 2024 14:33:02 +0200 Subject: [PATCH 029/103] Fix typing errors in dataset_correlations_metric. --- .../dataset_correlations_metric.py | 22 ++++++++++++++----- 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/src/evidently/metrics/data_quality/dataset_correlations_metric.py b/src/evidently/metrics/data_quality/dataset_correlations_metric.py index 529c747d34..8c3eb7359e 100644 --- a/src/evidently/metrics/data_quality/dataset_correlations_metric.py +++ b/src/evidently/metrics/data_quality/dataset_correlations_metric.py @@ -140,8 +140,14 @@ def _get_correlations_stats(correlation: pd.DataFrame, data_definition: DataDefi # fill diagonal with 1 values for getting abs max values np.fill_diagonal(correlation_matrix.values, 0) - if prediction_name in correlation_matrix and target_name in correlation_matrix: - target_prediction_correlation = correlation_matrix.loc[prediction_name, target_name] + target_prediction_correlation: Optional[pd.DataFrame] + if ( + prediction_name is not None + and target_name is not None + and prediction_name in correlation_matrix + and target_name in correlation_matrix + ): + target_prediction_correlation = correlation_matrix.loc[prediction_name, target_name] # type: ignore[assignment] if pd.isnull(target_prediction_correlation): target_prediction_correlation = None @@ -149,8 +155,10 @@ def _get_correlations_stats(correlation: pd.DataFrame, data_definition: DataDefi else: target_prediction_correlation = None - if target_name in correlation_matrix: - abs_max_target_features_correlation = correlation_matrix.loc[target_name, columns_corr].abs().max() + abs_max_target_features_correlation: Optional[pd.Series] + if target_name is not None and target_name in correlation_matrix: + cols = [target_name] + columns_corr + abs_max_target_features_correlation = correlation_matrix.loc[cols].abs().max() if pd.isnull(abs_max_target_features_correlation): abs_max_target_features_correlation = None @@ -158,8 +166,10 @@ def _get_correlations_stats(correlation: pd.DataFrame, data_definition: DataDefi else: abs_max_target_features_correlation = None - if prediction_name in correlation_matrix: - abs_max_prediction_features_correlation = correlation_matrix.loc[prediction_name, columns_corr].abs().max() + abs_max_prediction_features_correlation: Optional[pd.Series] + if prediction_name is not None and prediction_name in correlation_matrix: + cols = [prediction_name] + columns_corr + abs_max_prediction_features_correlation = correlation_matrix.loc[cols].abs().max() if pd.isnull(abs_max_prediction_features_correlation): abs_max_prediction_features_correlation = None From e84fd3c2c5fc6bb3e00526aaa6acc4fb2b2adfd0 Mon Sep 17 00:00:00 2001 From: Vyacheslav Morov Date: Thu, 17 Oct 2024 10:03:48 +0200 Subject: [PATCH 030/103] Ignore call overloads in visualizations. --- src/evidently/utils/visualizations.py | 29 ++++++++++++++++++--------- 1 file changed, 19 insertions(+), 10 deletions(-) diff --git a/src/evidently/utils/visualizations.py b/src/evidently/utils/visualizations.py index e91219b275..e10a269c89 100644 --- a/src/evidently/utils/visualizations.py +++ b/src/evidently/utils/visualizations.py @@ -684,20 +684,28 @@ def plot_num_num_rel( def make_hist_for_cat_plot(curr: pd.Series, ref: pd.Series = None, normalize: bool = False, dropna=False) -> Histogram: - hist_df = curr.astype(str).value_counts(normalize=normalize, dropna=dropna).reset_index() + hist_df = ( + curr.astype(str) + .value_counts(normalize=normalize, dropna=dropna) # type: ignore[call-overload] + .reset_index() + ) hist_df.columns = ["x", "count"] current = HistogramData.from_df(hist_df) reference = None if ref is not None: - hist_df = ref.astype(str).value_counts(normalize=normalize, dropna=dropna).reset_index() + hist_df = ( + ref.astype(str) + .value_counts(normalize=normalize, dropna=dropna) # type: ignore[call-overload] + .reset_index() + ) hist_df.columns = ["x", "count"] reference = HistogramData.from_df(hist_df) return Histogram(current=current, reference=reference) def get_distribution_for_category_column(column: pd.Series, normalize: bool = False) -> Distribution: - value_counts = column.value_counts(normalize=normalize, dropna=False) + value_counts = column.value_counts(normalize=normalize, dropna=False) # type: ignore[call-overload] # filter out na values if it amount == 0 new_values = [(k, v) for k, v in value_counts.items() if (not pd.isna(k) or v > 0)] @@ -1217,13 +1225,14 @@ def prepare_df_for_time_index_plot( if datetime_name is not None: if prefix is None and freq is None: prefix, freq = choose_agg_period(df[datetime_name], None) - plot_df = df.copy() - plot_df["per"] = plot_df[datetime_name].dt.to_period(freq=freq) - plot_df = plot_df.groupby("per")[column_name].agg(["mean", "std"]).reset_index() - plot_df["per"] = plot_df["per"].dt.to_timestamp() - return plot_df, prefix - plot_df = df[column_name].reset_index().sort_values(index_name) - plot_df["per"] = pd.cut(plot_df[index_name], OPTIMAL_POINTS if bins is None else bins, labels=False) + dt_plot_df: pd.DataFrame = df.copy() + dt_plot_df["per"] = dt_plot_df[datetime_name].dt.to_period(freq=freq) + dt_plot_df = dt_plot_df.groupby("per")[column_name].agg(["mean", "std"]).reset_index() + dt_plot_df["per"] = dt_plot_df["per"].dt.to_timestamp() + return dt_plot_df, prefix + plot_df: pd.DataFrame = df[column_name].reset_index().sort_values(index_name) + new_bins = OPTIMAL_POINTS if bins is None else bins + plot_df["per"] = pd.cut(plot_df[index_name], bins=new_bins, labels=False) # type: ignore[call-overload] plot_df = plot_df.groupby("per")[column_name].agg(["mean", "std"]).reset_index() return plot_df, None From bf9426b24e9c9396aeb478fddb2c0a9202424612 Mon Sep 17 00:00:00 2001 From: Vyacheslav Morov Date: Thu, 17 Oct 2024 11:14:51 +0200 Subject: [PATCH 031/103] Fix typing errors in column_drift_metric. --- .../metrics/data_drift/column_drift_metric.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/src/evidently/metrics/data_drift/column_drift_metric.py b/src/evidently/metrics/data_drift/column_drift_metric.py index 097ad41cfc..53788873c8 100644 --- a/src/evidently/metrics/data_drift/column_drift_metric.py +++ b/src/evidently/metrics/data_drift/column_drift_metric.py @@ -144,12 +144,12 @@ def get_one_column_drift( column.name: current_feature_data.values, "Timestamp": None if datetime_data is None else datetime_data.values, }, - index=index_data.values, + index=index_data.values, # type: ignore[arg-type] ), column.name, datetime_name, ) - current_scatter["current (mean)"] = df + current_scatter["current (mean)"] = df # type: ignore[assignment] if prefix is None: x_name = "Index binned" else: @@ -281,7 +281,8 @@ def calculate(self, data: InputData) -> ColumnDataDriftMetrics: reference_feature_data = data.get_reference_column(self.column_name) except ColumnNotFound as ex: raise ValueError(f"Cannot find column '{ex.column_name}' in reference dataset") - + if reference_feature_data is None: + raise ValueError(f"Cannot find column '{self.column_name.display_name}' in reference dataset") column_type = ColumnType.Numerical if self.column_name.is_main_dataset(): column_type = data.data_definition.get_column(self.column_name.name).column_type @@ -299,7 +300,7 @@ def calculate(self, data: InputData) -> ColumnDataDriftMetrics: current_feature_data=current_feature_data, reference_feature_data=reference_feature_data, column=self.column_name, - index_data=data.current_data.index, + index_data=data.current_data.index.to_series(), column_type=column_type, datetime_data=data.current_data[datetime_column.column_name] if datetime_column else None, data_definition=data.data_definition, @@ -338,9 +339,11 @@ def render_html(self, obj: ColumnDriftMetric) -> List[BaseWidgetInfo]: # fig_json = fig.to_plotly_json() if result.scatter is not None: if obj.get_options().render_options.raw_data: + if not isinstance(result.scatter, ScatterField): + raise ValueError("Result have incompatible type") scatter_fig = plot_scatter_for_data_drift( - curr_y=result.scatter.scatter[result.column_name], - curr_x=result.scatter.scatter[result.scatter.x_name], + curr_y=result.scatter.scatter[result.column_name].tolist(), + curr_x=result.scatter.scatter[result.scatter.x_name].tolist(), y0=result.scatter.plot_shape["y0"], y1=result.scatter.plot_shape["y1"], y_name=result.column_name, From 71a24787784d1f6159c8d068b0a5d3ec1bcd4a0e Mon Sep 17 00:00:00 2001 From: Vyacheslav Morov Date: Thu, 17 Oct 2024 11:18:34 +0200 Subject: [PATCH 032/103] Fix typing errors in data_drift_table. --- src/evidently/metrics/data_drift/data_drift_table.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/evidently/metrics/data_drift/data_drift_table.py b/src/evidently/metrics/data_drift/data_drift_table.py index f611bdebc7..37ded23e94 100644 --- a/src/evidently/metrics/data_drift/data_drift_table.py +++ b/src/evidently/metrics/data_drift/data_drift_table.py @@ -13,6 +13,7 @@ from evidently.core import IncludeTags from evidently.metric_results import DatasetColumns from evidently.metric_results import HistogramData +from evidently.metric_results import ScatterData from evidently.metrics.data_drift.base import WithDriftOptions from evidently.metrics.data_drift.feature_importance import FeatureImportanceMetric from evidently.model.widget import BaseWidgetInfo @@ -221,9 +222,11 @@ def _generate_column_params( data_drift = "Detected" if data.drift_detected else "Not Detected" if data.column_type == "num" and data.scatter is not None: if not agg_data: + if not isinstance(data.scatter, ScatterData): + raise ValueError("data.scatter has incompatible type") scatter_fig = plot_scatter_for_data_drift( - curr_y=data.scatter.scatter[data.column_name], - curr_x=data.scatter.scatter[data.scatter.x_name], + curr_y=data.scatter.scatter[data.column_name].tolist(), + curr_x=data.scatter.scatter[data.scatter.x_name].tolist(), y0=data.scatter.plot_shape["y0"], y1=data.scatter.plot_shape["y1"], y_name=data.column_name, From c9478087086e9a72b181cf99630d3f7a5b806b07 Mon Sep 17 00:00:00 2001 From: Vyacheslav Morov Date: Thu, 17 Oct 2024 11:27:17 +0200 Subject: [PATCH 033/103] Ignore numpy typing errors in regression_dummy_metric. --- .../regression_dummy_metric.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/src/evidently/metrics/regression_performance/regression_dummy_metric.py b/src/evidently/metrics/regression_performance/regression_dummy_metric.py index e7355e37e9..781a581f4c 100644 --- a/src/evidently/metrics/regression_performance/regression_dummy_metric.py +++ b/src/evidently/metrics/regression_performance/regression_dummy_metric.py @@ -84,8 +84,8 @@ def calculate(self, data: InputData) -> RegressionDummyMetricResults: # mape default values # optimal constant for mape s = data.current_data[target_name] - inv_y = 1 / s[s != 0].values - w = inv_y / sum(inv_y) + inv_y = 1.0 / s[s != 0].values # type: ignore[operator] + w = inv_y / sum(inv_y) # type: ignore[operator,arg-type] idxs = np.argsort(w) sorted_w = w[idxs] sorted_w_cumsum = np.cumsum(sorted_w) @@ -126,8 +126,8 @@ def calculate(self, data: InputData) -> RegressionDummyMetricResults: # mape default values # optimal constant for mape s = data.reference_data[target_name] - inv_y = 1 / s[s != 0].values - w = inv_y / sum(inv_y) + inv_y = 1.0 / s[s != 0].values # type: ignore[operator] + w = inv_y / sum(inv_y) # type: ignore[operator,arg-type] idxs = np.argsort(w) sorted_w = w[idxs] sorted_w_cumsum = np.cumsum(sorted_w) @@ -210,5 +210,9 @@ def render_html(self, obj: RegressionDummyMetric) -> List[BaseWidgetInfo]: return [ header_text(label="Dummy Regression Quality"), - table_data(column_names=columns, data=np.around(in_table_data, 3).values, title=""), + table_data( + column_names=columns, + data=np.around(in_table_data, 3).values, # type: ignore[attr-defined] + title="", + ), ] From 6d5e8770bc1756b744ca8ce608678c97327d2982 Mon Sep 17 00:00:00 2001 From: Vyacheslav Morov Date: Thu, 17 Oct 2024 12:48:35 +0200 Subject: [PATCH 034/103] Add optional to ref argument in plot_cat_cat_rel. --- src/evidently/calculations/data_drift.py | 14 ++++++++++---- src/evidently/utils/visualizations.py | 6 +++++- 2 files changed, 15 insertions(+), 5 deletions(-) diff --git a/src/evidently/calculations/data_drift.py b/src/evidently/calculations/data_drift.py index 21384d3969..a53dc62849 100644 --- a/src/evidently/calculations/data_drift.py +++ b/src/evidently/calculations/data_drift.py @@ -210,7 +210,7 @@ def get_one_column_drift( current_scatter["Timestamp"] = current_data[datetime_column_name] x_name = "Timestamp" else: - current_scatter["Index"] = current_data.index + current_scatter["Index"] = current_data.index.to_series() x_name = "Index" else: current_scatter = {} @@ -225,7 +225,8 @@ def get_one_column_drift( column_name, datetime_column_name, ) - current_scatter["current (mean)"] = df + # TODO: assignment DataFrame to Series + current_scatter["current (mean)"] = df # type: ignore[assignment] if prefix is None: x_name = "Index binned" else: @@ -295,8 +296,13 @@ def get_one_column_drift( if len(new_values) > 0: raise ValueError(f"Values {new_values} not presented in 'target_names'") else: - current_column = current_column.map(dataset_columns.target_names) - reference_column = reference_column.map(dataset_columns.target_names) + target_names_mapping = ( + dataset_columns.target_names + if isinstance(dataset_columns.target_names, dict) + else {idx: value for (idx, value) in enumerate(dataset_columns.target_names)} + ) + current_column = current_column.map(target_names_mapping) + reference_column = reference_column.map(target_names_mapping) current_distribution, reference_distribution = get_distribution_for_column( column_type=column_type.value, current=current_column, diff --git a/src/evidently/utils/visualizations.py b/src/evidently/utils/visualizations.py index e10a269c89..91c552d36f 100644 --- a/src/evidently/utils/visualizations.py +++ b/src/evidently/utils/visualizations.py @@ -579,7 +579,11 @@ def make_hist_for_num_plot(curr: pd.Series, ref: Optional[pd.Series] = None, cal def plot_cat_cat_rel( - curr: pd.DataFrame, ref: pd.DataFrame, target_name: str, feature_name: str, color_options: ColorOptions + curr: pd.DataFrame, + ref: Optional[pd.DataFrame], + target_name: str, + feature_name: str, + color_options: ColorOptions, ): """ Accepts current and reference data as pandas dataframes with two columns: feature_name and "count_objects". From 012cc7f7ea36b7ab4c969c843c3adbe768f6ecc1 Mon Sep 17 00:00:00 2001 From: Vyacheslav Morov Date: Thu, 17 Oct 2024 12:52:36 +0200 Subject: [PATCH 035/103] Check that reference data is present if required in column_summary_metric. --- .../metrics/data_integrity/column_summary_metric.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/evidently/metrics/data_integrity/column_summary_metric.py b/src/evidently/metrics/data_integrity/column_summary_metric.py index 9aeb75cb47..73b9ad8ad1 100644 --- a/src/evidently/metrics/data_integrity/column_summary_metric.py +++ b/src/evidently/metrics/data_integrity/column_summary_metric.py @@ -540,9 +540,12 @@ def get_text_stats( oov = data.get_current_column(generated_text_features["oov"].as_column()) non_letter_char = data.get_current_column(generated_text_features["non_letter_char"].as_column()) else: - text_length = data.get_reference_column(generated_text_features["text_length"].as_column()) - oov = data.get_reference_column(generated_text_features["oov"].as_column()) - non_letter_char = data.get_reference_column(generated_text_features["non_letter_char"].as_column()) + text_length_ref = data.get_reference_column(generated_text_features["text_length"].as_column()) + oov_ref = data.get_reference_column(generated_text_features["oov"].as_column()) + non_letter_char_ref = data.get_reference_column(generated_text_features["non_letter_char"].as_column()) + if text_length_ref is None or oov_ref is None or non_letter_char_ref is None: + raise ValueError("Reference required but not present in data") + (text_length, oov, non_letter_char) = (text_length_ref, oov_ref, non_letter_char_ref) return TextCharacteristics( number_of_rows=number_of_rows, From c521d2703ece77f2eaadaabf038ba432c4b14027 Mon Sep 17 00:00:00 2001 From: Vyacheslav Morov Date: Thu, 17 Oct 2024 12:56:14 +0200 Subject: [PATCH 036/103] Add explicit check of reference data is not None before using it in column_correlations_metric. --- .../metrics/data_quality/column_correlations_metric.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/evidently/metrics/data_quality/column_correlations_metric.py b/src/evidently/metrics/data_quality/column_correlations_metric.py index 31ce9a9c10..8f8d6fa968 100644 --- a/src/evidently/metrics/data_quality/column_correlations_metric.py +++ b/src/evidently/metrics/data_quality/column_correlations_metric.py @@ -110,6 +110,8 @@ def calculate(self, data: InputData) -> ColumnCorrelationsMetricResult: reference_correlations = None if reference_data is not None: + if data.reference_data is None: + raise ValueError("data.reference_data was not set but part of it available") reference_correlations = self._calculate_correlation( self.column_name, reference_data, From a86085f8809cc48f46fc4c1619d58746d5b3703a Mon Sep 17 00:00:00 2001 From: Vyacheslav Morov Date: Thu, 17 Oct 2024 13:00:50 +0200 Subject: [PATCH 037/103] Fix type hints in ndcg_k. --- src/evidently/metrics/recsys/ndcg_k.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/evidently/metrics/recsys/ndcg_k.py b/src/evidently/metrics/recsys/ndcg_k.py index e4d0cc940e..20b9b254c8 100644 --- a/src/evidently/metrics/recsys/ndcg_k.py +++ b/src/evidently/metrics/recsys/ndcg_k.py @@ -31,7 +31,7 @@ def __init__( def calculate(self, data: InputData) -> TopKMetricResult: curr, ref = get_curr_and_ref_df(data, self.min_rel_score, self.no_feedback_users, False) current = self.calculate_ndcg(curr, self.k) - reference: Optional[dict] = None + reference: Optional[pd.Series] = None if ref is not None: reference = self.calculate_ndcg(ref, self.k) From 3dbfae019fdb56c37396830faec0afcedf949139 Mon Sep 17 00:00:00 2001 From: Vyacheslav Morov Date: Thu, 17 Oct 2024 13:06:32 +0200 Subject: [PATCH 038/103] Add explicit reference_data check and ignore numpy typing errors in regression_quality. --- .../metrics/regression_performance/regression_quality.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/evidently/metrics/regression_performance/regression_quality.py b/src/evidently/metrics/regression_performance/regression_quality.py index 992d80973c..7c36a8c7a2 100644 --- a/src/evidently/metrics/regression_performance/regression_quality.py +++ b/src/evidently/metrics/regression_performance/regression_quality.py @@ -145,8 +145,9 @@ def calculate(self, data: InputData) -> RegressionQualityMetricResults: # mape default values # optimal constant for mape s = data.current_data[target_name] - inv_y = 1 / s[s != 0].values - w = inv_y / sum(inv_y) + # TODO: fix typing + inv_y = 1 / s[s != 0].values # type: ignore[operator] + w = inv_y / sum(inv_y) # type: ignore[operator,arg-type] idxs = np.argsort(w) sorted_w = w[idxs] sorted_w_cumsum = np.cumsum(sorted_w) @@ -219,7 +220,7 @@ def calculate(self, data: InputData) -> RegressionQualityMetricResults: err_curr = data.current_data[prediction_name] - data.current_data[target_name] err_ref = None - if is_ref_data: + if data.reference_data is not None: err_ref = data.reference_data[prediction_name] - data.reference_data[target_name] me_hist_for_plot = make_hist_for_num_plot(err_curr, err_ref) From 47259998d5e29c46681f3d096b3025f76d1a2f68 Mon Sep 17 00:00:00 2001 From: Vyacheslav Morov Date: Thu, 17 Oct 2024 13:11:07 +0200 Subject: [PATCH 039/103] Fix variable reuse in spark/visualizations. --- src/evidently/spark/visualizations.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/evidently/spark/visualizations.py b/src/evidently/spark/visualizations.py index a8d283ed49..05043a11d9 100644 --- a/src/evidently/spark/visualizations.py +++ b/src/evidently/spark/visualizations.py @@ -57,13 +57,13 @@ def prepare_df_for_time_index_plot( sf.floor(sf.col("_2") / (ptp / (OPTIMAL_POINTS - 1))).alias(PERIOD_COL), ) ) - plot_df = ( + plot_df_pandas = ( plot_df.groupby(PERIOD_COL) .agg(sf.mean(column_name).alias("mean"), sf.stddev(column_name).alias("std")) .toPandas() .sort_values(PERIOD_COL) # type: ignore[attr-defined] ) - return plot_df, None + return plot_df_pandas, None def choose_agg_period( From adac0d341fd7d8c9558d6602ddd0d03100b90514 Mon Sep 17 00:00:00 2001 From: Vyacheslav Morov Date: Thu, 17 Oct 2024 13:31:35 +0200 Subject: [PATCH 040/103] Ignore numpy typing errors and fix typing for correlations call in data_quality. --- src/evidently/calculations/data_quality.py | 24 ++++++++++++++-------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/src/evidently/calculations/data_quality.py b/src/evidently/calculations/data_quality.py index ef9eff60fe..a3620f9d40 100644 --- a/src/evidently/calculations/data_quality.py +++ b/src/evidently/calculations/data_quality.py @@ -4,6 +4,7 @@ from typing import Callable from typing import Dict from typing import List +from typing import Literal from typing import Optional from typing import Tuple from typing import Union @@ -187,7 +188,8 @@ def get_percentage_from_all_values(value: Union[int, float]) -> float: if feature_type == ColumnType.Numerical: # round most common feature value for numeric features to 1e-5 - if not np.issubdtype(feature, np.number): + # TODO: Fix this check + if not np.issubdtype(feature, np.number): # type: ignore[arg-type] feature = feature.astype(float) if isinstance(result.most_common_value, float): result.most_common_value = np.round(result.most_common_value, 5) @@ -197,11 +199,12 @@ def get_percentage_from_all_values(value: Union[int, float]) -> float: result.min = np.round(feature.min(), 2) common_stats = dict(feature.describe()) std = common_stats["std"] - result.std = np.round(std, 2) - result.mean = np.round(common_stats["mean"], 2) - result.percentile_25 = np.round(common_stats["25%"], 2) - result.percentile_50 = np.round(common_stats["50%"], 2) - result.percentile_75 = np.round(common_stats["75%"], 2) + # TODO: Fix assignment types + result.std = np.round(std, 2) # type: ignore[assignment] + result.mean = np.round(common_stats["mean"], 2) # type: ignore[assignment] + result.percentile_25 = np.round(common_stats["25%"], 2) # type: ignore[assignment] + result.percentile_50 = np.round(common_stats["50%"], 2) # type: ignore[assignment] + result.percentile_75 = np.round(common_stats["75%"], 2) # type: ignore[assignment] if feature_type == ColumnType.Datetime: # cast datetime value to str for datetime features @@ -457,7 +460,9 @@ def calculate_numerical_correlation( result = [] - for kind in ["pearson", "spearman", "kendall"]: + kind: Literal["pearson", "spearman", "kendall"] + kinds: List[Literal["pearson", "spearman", "kendall"]] = ["pearson", "spearman", "kendall"] + for kind in kinds: correlations_columns = [] correlations_values = [] @@ -465,7 +470,8 @@ def calculate_numerical_correlation( correlations_columns.append(other_column_name) correlations_values.append( column.replace([np.inf, -np.inf], np.nan).corr( - features[other_column_name].replace([np.inf, -np.inf], np.nan), method=kind + features[other_column_name].replace([np.inf, -np.inf], np.nan), + method=kind, ) ) @@ -473,7 +479,7 @@ def calculate_numerical_correlation( ColumnCorrelations( column_name=column_display_name, kind=kind, - values=Distribution(x=correlations_columns, y=correlations_values), + values=DistributionIncluded(x=correlations_columns, y=correlations_values), ) ) From 0efd6a79a8a8aaf6b3597a546494600fa07cd11b Mon Sep 17 00:00:00 2001 From: Vyacheslav Morov Date: Thu, 17 Oct 2024 13:34:44 +0200 Subject: [PATCH 041/103] Replace values with to_numpy in quality_by_class_metric. --- .../classification_performance/quality_by_class_metric.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/evidently/metrics/classification_performance/quality_by_class_metric.py b/src/evidently/metrics/classification_performance/quality_by_class_metric.py index 9bf73c2fe2..70eb5bd99a 100644 --- a/src/evidently/metrics/classification_performance/quality_by_class_metric.py +++ b/src/evidently/metrics/classification_performance/quality_by_class_metric.py @@ -90,7 +90,9 @@ def calculate(self, data: InputData) -> ClassificationQualityByClassResult: current_roc_aucs = None if prediction.prediction_probas is not None: - binaraized_target = (target.values.reshape(-1, 1) == list(prediction.prediction_probas.columns)).astype(int) + binaraized_target = (target.to_numpy().reshape(-1, 1) == list(prediction.prediction_probas.columns)).astype( + int + ) current_roc_aucs = sklearn.metrics.roc_auc_score( binaraized_target, prediction.prediction_probas, average=None ).tolist() @@ -108,7 +110,7 @@ def calculate(self, data: InputData) -> ClassificationQualityByClassResult: ).classes if ref_prediction.prediction_probas is not None: binaraized_target = ( - ref_target.values.reshape(-1, 1) == list(ref_prediction.prediction_probas.columns) + ref_target.to_numpy().reshape(-1, 1) == list(ref_prediction.prediction_probas.columns) ).astype(int) reference_roc_aucs = sklearn.metrics.roc_auc_score( binaraized_target, ref_prediction.prediction_probas, average=None From 00bca860001a17c22c26b6df4bb1298bfc6967f6 Mon Sep 17 00:00:00 2001 From: Vyacheslav Morov Date: Thu, 17 Oct 2024 13:54:10 +0200 Subject: [PATCH 042/103] Explicit check for reference data and ignore numpy errors in regression_performance_metrics. --- .../regression_performance_metrics.py | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) diff --git a/src/evidently/metrics/regression_performance/regression_performance_metrics.py b/src/evidently/metrics/regression_performance/regression_performance_metrics.py index 0a33bcdf3e..5f6b2d7c70 100644 --- a/src/evidently/metrics/regression_performance/regression_performance_metrics.py +++ b/src/evidently/metrics/regression_performance/regression_performance_metrics.py @@ -152,8 +152,9 @@ def calculate(self, data: InputData) -> RegressionPerformanceMetricsResults: # mape default values # optimal constant for mape s = data.current_data[data.column_mapping.target] - inv_y = 1 / s[s != 0].values - w = inv_y / sum(inv_y) + # TODO: Fix assignments + inv_y = 1 / s[s != 0].values # type: ignore[operator] + w = inv_y / sum(inv_y) # type: ignore[operator,arg-type] idxs = np.argsort(w) sorted_w = w[idxs] sorted_w_cumsum = np.cumsum(sorted_w) @@ -206,12 +207,6 @@ def calculate(self, data: InputData) -> RegressionPerformanceMetricsResults: vals_for_plots: Dict[str, RegressionMetricScatter] = {} - if data.reference_data is not None: - is_ref_data = True - - else: - is_ref_data = False - for name, func in zip( ["r2_score", "rmse", "mean_abs_error", "mean_abs_perc_error"], [ @@ -226,14 +221,14 @@ def calculate(self, data: InputData) -> RegressionPerformanceMetricsResults: func, data.column_mapping.target, data.column_mapping.prediction, - is_ref_data, + data.reference_data is not None, ) # me plot err_curr = data.current_data[data.column_mapping.prediction] - data.current_data[data.column_mapping.target] err_ref = None - if is_ref_data: + if data.reference_data is not None: err_ref = ( data.reference_data[data.column_mapping.prediction] - data.reference_data[data.column_mapping.target] ) From e6b83ffbddab9506662dad1f5155d461e023853e Mon Sep 17 00:00:00 2001 From: Vyacheslav Morov Date: Thu, 17 Oct 2024 14:00:39 +0200 Subject: [PATCH 043/103] Fix typing for _make_df_for_plot in top_er --- .../metrics/regression_performance/top_error.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/src/evidently/metrics/regression_performance/top_error.py b/src/evidently/metrics/regression_performance/top_error.py index 243ef1c566..cfeeae05d7 100644 --- a/src/evidently/metrics/regression_performance/top_error.py +++ b/src/evidently/metrics/regression_performance/top_error.py @@ -130,7 +130,13 @@ def calculate(self, data: InputData) -> RegressionTopErrorMetricResults: agg_data=True, ) - def _make_df_for_plot(self, df, target_name: str, prediction_name: str, datetime_column_name: Optional[str]): + def _make_df_for_plot( + self, + df: pd.DataFrame, + target_name: str, + prediction_name: str, + datetime_column_name: Optional[str], + ) -> pd.DataFrame: result = df.replace([np.inf, -np.inf], np.nan) if datetime_column_name is not None: result.dropna( @@ -139,9 +145,11 @@ def _make_df_for_plot(self, df, target_name: str, prediction_name: str, datetime inplace=True, subset=[target_name, prediction_name, datetime_column_name], ) - return result.sort_values(datetime_column_name) + result.sort_values(datetime_column_name, inplace=True) + return result result.dropna(axis=0, how="any", inplace=True, subset=[target_name, prediction_name]) - return result.sort_index() + result.sort_index(inplace=True) + return result @staticmethod def _get_data_for_scatter(df: pd.DataFrame, target_name: str, prediction_name: str) -> RegressionScatter: From aa016e3b32693486d228001a95e7a3d5ca4cf1a5 Mon Sep 17 00:00:00 2001 From: Vyacheslav Morov Date: Thu, 17 Oct 2024 14:09:47 +0200 Subject: [PATCH 044/103] Replace values with to_numpy in cramer_von_mises_stattest. --- .../calculations/stattests/cramer_von_mises_stattest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/evidently/calculations/stattests/cramer_von_mises_stattest.py b/src/evidently/calculations/stattests/cramer_von_mises_stattest.py index 63015adb52..d749675e94 100644 --- a/src/evidently/calculations/stattests/cramer_von_mises_stattest.py +++ b/src/evidently/calculations/stattests/cramer_von_mises_stattest.py @@ -208,7 +208,7 @@ def _cramer_von_mises( p_value: p-value test_result: whether the drift is detected """ - res = _cvm_2samp(reference_data.values, current_data.values) + res = _cvm_2samp(reference_data.to_numpy(), current_data.to_numpy()) return res.pvalue, res.pvalue <= threshold From f26fc2aa63c4d14e05491b86dfc1818235b52f8a Mon Sep 17 00:00:00 2001 From: Vyacheslav Morov Date: Thu, 17 Oct 2024 14:10:14 +0200 Subject: [PATCH 045/103] Add type hint in histogram. --- src/evidently/spark/calculations/histogram.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/evidently/spark/calculations/histogram.py b/src/evidently/spark/calculations/histogram.py index cfad2b1acb..cc9a70cf13 100644 --- a/src/evidently/spark/calculations/histogram.py +++ b/src/evidently/spark/calculations/histogram.py @@ -39,7 +39,7 @@ def get_histogram( bin_edges = np.array([min_val + step * i for i in range(nbinsx + 1)]) if density: - db = np.array(np.diff(bin_edges), float) + db: np.ndarray = np.array(np.diff(bin_edges), float) return (n / db / n.sum()).tolist(), bin_edges From c6269b0979329f644c8e6f594a46c2ac79726631 Mon Sep 17 00:00:00 2001 From: Vyacheslav Morov Date: Thu, 17 Oct 2024 14:11:45 +0200 Subject: [PATCH 046/103] Replace values with to_numpy in fisher_exact_stattest. --- src/evidently/calculations/stattests/fisher_exact_stattest.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/evidently/calculations/stattests/fisher_exact_stattest.py b/src/evidently/calculations/stattests/fisher_exact_stattest.py index 81ccf4a992..56f0af51aa 100644 --- a/src/evidently/calculations/stattests/fisher_exact_stattest.py +++ b/src/evidently/calculations/stattests/fisher_exact_stattest.py @@ -54,8 +54,8 @@ def _fisher_exact_stattest( """ if ( - (reference_data.isnull().values.any()) - or (current_data.isnull().values.any()) + (reference_data.isnull().to_numpy().any()) + or (current_data.isnull().to_numpy().any()) or (reference_data.isin([np.inf, -np.inf]).any()) or (current_data.isin([np.inf, -np.inf]).any()) ): From 45b4a188b87df5faa5b2bddba2bf8c876560c347 Mon Sep 17 00:00:00 2001 From: Vyacheslav Morov Date: Thu, 17 Oct 2024 14:21:36 +0200 Subject: [PATCH 047/103] Replace values with to_numpy in hellinger_distance. --- .../calculations/stattests/hellinger_distance.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/evidently/calculations/stattests/hellinger_distance.py b/src/evidently/calculations/stattests/hellinger_distance.py index 57e110fef1..64ea09c580 100644 --- a/src/evidently/calculations/stattests/hellinger_distance.py +++ b/src/evidently/calculations/stattests/hellinger_distance.py @@ -59,8 +59,8 @@ def _hellinger_distance( if feature_type == ColumnType.Categorical: dd: DefaultDict[int, int] = defaultdict(int) - ref = (reference_data.value_counts() / len(reference_data)).to_dict(dd) - curr = (current_data.value_counts() / len(current_data)).to_dict(dd) + ref = (reference_data.value_counts() / len(reference_data)).to_dict(into=dd) + curr = (current_data.value_counts() / len(current_data)).to_dict(into=dd) hellinger_distance = 0.0 for key in keys: @@ -73,8 +73,8 @@ def _hellinger_distance( else: bins = np.histogram_bin_edges(keys, bins="sturges") - h1 = np.histogram(reference_data.values, bins=bins, density=True)[0] - h2 = np.histogram(current_data.values, bins=bins, density=True)[0] + h1 = np.histogram(reference_data.to_numpy(), bins=bins, density=True)[0] + h2 = np.histogram(current_data.to_numpy(), bins=bins, density=True)[0] bin_width = (max(bins) - min(bins)) / (len(bins) - 1) From c2328c11e26b9ff2f0fc91c062fb3cf179db118b Mon Sep 17 00:00:00 2001 From: Vyacheslav Morov Date: Thu, 17 Oct 2024 14:23:55 +0200 Subject: [PATCH 048/103] Replace values with to_numpy in mmd_stattest. Fix reuse of variables. --- src/evidently/calculations/stattests/mmd_stattest.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/evidently/calculations/stattests/mmd_stattest.py b/src/evidently/calculations/stattests/mmd_stattest.py index da5e463883..99f23ab900 100644 --- a/src/evidently/calculations/stattests/mmd_stattest.py +++ b/src/evidently/calculations/stattests/mmd_stattest.py @@ -136,10 +136,10 @@ def _mmd_stattest( p_value: p-value test_result: whether the drift is detected """ - reference_data = reference_data.values.reshape(-1, 1) - current_data = current_data.values.reshape(-1, 1) + transformed_ref = reference_data.to_numpy().reshape(-1, 1) + transformed_curr = current_data.to_numpy().reshape(-1, 1) - p_value, mmd = mmd_pval(reference_data, current_data) + p_value, mmd = mmd_pval(transformed_ref, transformed_curr) return p_value, p_value < threshold From 50c51d291aa787aa8185cf0b90bbf123d291948a Mon Sep 17 00:00:00 2001 From: Vyacheslav Morov Date: Thu, 17 Oct 2024 14:27:52 +0200 Subject: [PATCH 049/103] Fix type hints in z_stattest. --- src/evidently/calculations/stattests/z_stattest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/evidently/calculations/stattests/z_stattest.py b/src/evidently/calculations/stattests/z_stattest.py index cdfe050a89..9ed4d1c421 100644 --- a/src/evidently/calculations/stattests/z_stattest.py +++ b/src/evidently/calculations/stattests/z_stattest.py @@ -35,7 +35,7 @@ from evidently.core import ColumnType -def proportions_diff_z_stat_ind(ref: pd.DataFrame, curr: pd.DataFrame): +def proportions_diff_z_stat_ind(ref: pd.Series, curr: pd.Series): # pylint: disable=invalid-name n1 = len(ref) n2 = len(curr) From 0f357af215232fd2c45c2e3821740d983e61e396 Mon Sep 17 00:00:00 2001 From: Vyacheslav Morov Date: Thu, 17 Oct 2024 15:06:56 +0200 Subject: [PATCH 050/103] Fix reuse variables in utils. --- src/evidently/calculations/utils.py | 42 ++++++++++++++++++++--------- 1 file changed, 29 insertions(+), 13 deletions(-) diff --git a/src/evidently/calculations/utils.py b/src/evidently/calculations/utils.py index ee4a6a2e5f..9676c3941d 100644 --- a/src/evidently/calculations/utils.py +++ b/src/evidently/calculations/utils.py @@ -42,13 +42,27 @@ def make_hist_for_num_plot(curr: pd.Series, ref: pd.Series = None): return result -def make_hist_for_cat_plot(curr: pd.Series, ref: pd.Series = None, normalize: bool = False, dropna=False): +def make_hist_for_cat_plot(curr: pd.Series, ref: pd.Series = None, normalize: bool = False, dropna: bool = False): result = {} - hist_df = curr.astype(str).value_counts(normalize=normalize, dropna=dropna).reset_index() + hist_df = ( + curr.astype(str) + .value_counts( # type: ignore[call-overload] + normalize=normalize, + dropna=dropna, + ) + .reset_index() + ) hist_df.columns = ["x", "count"] result["current"] = hist_df if ref is not None: - hist_df = ref.astype(str).value_counts(normalize=normalize, dropna=dropna).reset_index() + hist_df = ( + ref.astype(str) + .value_counts( # type: ignore[call-overload] + normalize=normalize, + dropna=dropna, + ) + .reset_index() + ) hist_df.columns = ["x", "count"] result["reference"] = hist_df return result @@ -56,10 +70,10 @@ def make_hist_for_cat_plot(curr: pd.Series, ref: pd.Series = None, normalize: bo def get_count_values(col1: pd.Series, col2: pd.Series, col1_name: str, col2_name: str): df = pd.DataFrame({col2_name: col2, col1_name: col1}) - df = df.groupby([col2_name, col1_name], observed=False).size() - df.name = "count_objects" - df = df.reset_index() - return df[df["count_objects"] > 0] + grouped = df.groupby([col2_name, col1_name], observed=False).size() + grouped.name = "count_objects" + grouped.reset_index(inplace=True) + return grouped[grouped["count_objects"] > 0] def get_data_for_cat_cat_plot( @@ -124,7 +138,7 @@ def prepare_box_data( res = {} for df, name in zip(dfs, names): data = df.groupby(cat_feature_name, observed=False)[num_feature_name] - df_for_plot = data.quantile([0, 0.25, 0.5, 0.75, 1]).reset_index() + df_for_plot = data.quantile(np.array([0, 0.25, 0.5, 0.75, 1])).reset_index() df_for_plot.columns = pd.Index([cat_feature_name, "q", num_feature_name]) res_df = {} values = df_for_plot[cat_feature_name].unique() @@ -231,11 +245,13 @@ def transform_df_to_time_count_view( column_data: pd.Series, ): df = pd.DataFrame({"period": period_data, datetime_column_name: datetime_data, data_column_name: column_data}) - df = df.groupby(["period", data_column_name]).size() - df.name = "num" - df = df.reset_index() - df[datetime_column_name] = df["period"].dt.to_timestamp() - return df[df["num"] > 0] + grouped = df.groupby(["period", data_column_name]).size() + if not isinstance(grouped, pd.Series): + raise ValueError("grouped has incorrect type") + grouped.name = "num" + grouped.reset_index(inplace=True) + grouped[datetime_column_name] = grouped["period"].dt.to_timestamp() + return grouped[grouped["num"] > 0] def prepare_data_for_date_cat(date_curr, date_ref, datetime_name, cat_name, cat_curr, cat_ref): From 855f9218b62ef69a932fa3e7ae3f96d72fee1311 Mon Sep 17 00:00:00 2001 From: Vyacheslav Morov Date: Thu, 17 Oct 2024 15:08:30 +0200 Subject: [PATCH 051/103] Convert to proper types in class_separation_metric. --- .../classification_performance/class_separation_metric.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/evidently/metrics/classification_performance/class_separation_metric.py b/src/evidently/metrics/classification_performance/class_separation_metric.py index 2c0d024244..d19653be5c 100644 --- a/src/evidently/metrics/classification_performance/class_separation_metric.py +++ b/src/evidently/metrics/classification_performance/class_separation_metric.py @@ -53,7 +53,7 @@ def prepare_box_data(df: pd.DataFrame, target_name: str, prediction_names: List[ for name in prediction_names: df_name = df.copy() df_name[target_name] = (df_name[target_name] == name).astype(int) - df_for_plot = df_name.groupby(target_name)[name].quantile([0, 0.25, 0.5, 0.75, 1]).reset_index() + df_for_plot = df_name.groupby(target_name)[name].quantile(np.array([0, 0.25, 0.5, 0.75, 1])).reset_index() df_for_plot.columns = pd.Index([target_name, "q", name]) res_df = pd.DataFrame() values = df_for_plot[target_name].unique() @@ -108,9 +108,9 @@ def calculate(self, data: InputData) -> ClassificationClassSeparationPlotResults reference=column_scatter_from_df(reference_plot, True), target_name=target_name, ) - current_plot = prepare_box_data(current_plot, target_name, prediction_names) + current_plot = prepare_box_data(current_plot, target_name, prediction_names.tolist()) if reference_plot is not None: - reference_plot = prepare_box_data(reference_plot, target_name, prediction_names) + reference_plot = prepare_box_data(reference_plot, target_name, prediction_names.tolist()) return ClassificationClassSeparationPlotResults( current=current_plot, reference=reference_plot, From cc371a1ad9ce5966284e724c54ba46800bdafb09 Mon Sep 17 00:00:00 2001 From: Vyacheslav Morov Date: Thu, 17 Oct 2024 15:18:34 +0200 Subject: [PATCH 052/103] Add type hints in lift_curve_metric. --- .../classification_performance/lift_curve_metric.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/evidently/metrics/classification_performance/lift_curve_metric.py b/src/evidently/metrics/classification_performance/lift_curve_metric.py index 6c17a3a136..e5bb02915d 100644 --- a/src/evidently/metrics/classification_performance/lift_curve_metric.py +++ b/src/evidently/metrics/classification_performance/lift_curve_metric.py @@ -1,3 +1,5 @@ +from typing import Any +from typing import Dict from typing import List from typing import Optional @@ -9,6 +11,7 @@ from evidently.calculations.classification_performance import calculate_lift_table from evidently.calculations.classification_performance import get_prediction_data from evidently.core import IncludeTags +from evidently.metric_results import Label from evidently.metric_results import LiftCurve from evidently.metric_results import LiftCurveData from evidently.metric_results import PredictionData @@ -62,9 +65,9 @@ def calculate_metrics(self, target_data: pd.Series, prediction: PredictionData) labels = prediction.labels if prediction.prediction_probas is None: raise ValueError("Lift Curve can be calculated only " "on binary probabilistic predictions") - binaraized_target = (target_data.values.reshape(-1, 1) == labels).astype(int) - lift_curve = {} - lift_table = {} + binaraized_target = (target_data.to_numpy().reshape(-1, 1) == labels).astype(int) + lift_curve: LiftCurve = {} + lift_table: Dict[Label, Any] = {} if len(labels) <= 2: binaraized_target = pd.DataFrame(binaraized_target[:, 0]) binaraized_target.columns = ["target"] @@ -90,7 +93,6 @@ def calculate_metrics(self, target_data: pd.Series, prediction: PredictionData) max_lift=[i[9] for i in lift_table[prediction.prediction_probas.columns[0]]], relative_lift=[i[10] for i in lift_table[prediction.prediction_probas.columns[0]]], percent=[i[11] for i in lift_table[prediction.prediction_probas.columns[0]]], - # percent = lift_table[prediction.prediction_probas.columns[0]][0][11], ) else: binaraized_target = pd.DataFrame(binaraized_target) @@ -106,7 +108,6 @@ def calculate_metrics(self, target_data: pd.Series, prediction: PredictionData) lift_table[label] = calculate_lift_table(binded) for label in labels: - # lift_curve[int(prediction.prediction_probas.columns[0])] = LiftCurveData( lift_curve[label] = LiftCurveData( lift=[i[8] for i in lift_table[prediction.prediction_probas.columns[0]]], top=[i[0] for i in lift_table[prediction.prediction_probas.columns[0]]], @@ -120,7 +121,6 @@ def calculate_metrics(self, target_data: pd.Series, prediction: PredictionData) max_lift=[i[9] for i in lift_table[prediction.prediction_probas.columns[0]]], relative_lift=[i[10] for i in lift_table[prediction.prediction_probas.columns[0]]], percent=[i[11] for i in lift_table[prediction.prediction_probas.columns[0]]], - # percent = lift_table[prediction.prediction_probas.columns[0]][0][11], ) return lift_curve From 2fbc3adb08f0ed1ff7e0754e61c59042cf039357 Mon Sep 17 00:00:00 2001 From: Vyacheslav Morov Date: Thu, 17 Oct 2024 15:19:10 +0200 Subject: [PATCH 053/103] Replace values with to_numpy in lift_table_metric. --- .../metrics/classification_performance/lift_table_metric.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/evidently/metrics/classification_performance/lift_table_metric.py b/src/evidently/metrics/classification_performance/lift_table_metric.py index e9dc86391c..8a7edb3f64 100644 --- a/src/evidently/metrics/classification_performance/lift_table_metric.py +++ b/src/evidently/metrics/classification_performance/lift_table_metric.py @@ -105,7 +105,7 @@ def calculate_metrics(self, target_data: pd.Series, prediction: PredictionData): labels = prediction.labels if prediction.prediction_probas is None: raise ValueError("Lift Table can be calculated only on " "binary probabilistic predictions") - binaraized_target = (target_data.values.reshape(-1, 1) == labels).astype(int) + binaraized_target = (target_data.to_numpy().reshape(-1, 1) == labels).astype(int) lift_table = {} if len(labels) <= 2: binaraized_target = pd.DataFrame(binaraized_target[:, 0]) From 8973a311d74d6169daa4d0a510fe205705477998 Mon Sep 17 00:00:00 2001 From: Vyacheslav Morov Date: Thu, 17 Oct 2024 15:21:29 +0200 Subject: [PATCH 054/103] Replace values with to_numpy and add type hints in pr_curve_metric.py. --- .../metrics/classification_performance/pr_curve_metric.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/evidently/metrics/classification_performance/pr_curve_metric.py b/src/evidently/metrics/classification_performance/pr_curve_metric.py index b2537bc6fe..ad86e57161 100644 --- a/src/evidently/metrics/classification_performance/pr_curve_metric.py +++ b/src/evidently/metrics/classification_performance/pr_curve_metric.py @@ -58,8 +58,8 @@ def calculate_metrics(self, target_data: pd.Series, prediction: PredictionData) labels = prediction.labels if prediction.prediction_probas is None: raise ValueError("PR Curve can be calculated only on binary probabilistic predictions") - binaraized_target = (target_data.values.reshape(-1, 1) == labels).astype(int) - pr_curve = {} + binaraized_target = (target_data.to_numpy().reshape(-1, 1) == labels).astype(int) + pr_curve: PRCurve = {} if len(labels) <= 2: binaraized_target = pd.DataFrame(binaraized_target[:, 0]) binaraized_target.columns = ["target"] From e611e3bbfaff51f207c2effc218c4c4dd07c1624 Mon Sep 17 00:00:00 2001 From: Vyacheslav Morov Date: Thu, 17 Oct 2024 15:23:03 +0200 Subject: [PATCH 055/103] Replace values with to_numpy and add type hints in pr_table_metric. --- .../metrics/classification_performance/pr_table_metric.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/evidently/metrics/classification_performance/pr_table_metric.py b/src/evidently/metrics/classification_performance/pr_table_metric.py index a12e26eed0..c9cee3b4ce 100644 --- a/src/evidently/metrics/classification_performance/pr_table_metric.py +++ b/src/evidently/metrics/classification_performance/pr_table_metric.py @@ -74,12 +74,12 @@ def calculate(self, data: InputData) -> ClassificationPRTableResults: reference=ref_pr_table, ) - def calculate_metrics(self, target_data: pd.Series, prediction: PredictionData): + def calculate_metrics(self, target_data: pd.Series, prediction: PredictionData) -> PRTable: labels = prediction.labels if prediction.prediction_probas is None: raise ValueError("PR Table can be calculated only on binary probabilistic predictions") - binaraized_target = (target_data.values.reshape(-1, 1) == labels).astype(int) - pr_table = {} + binaraized_target = (target_data.to_numpy().reshape(-1, 1) == labels).astype(int) + pr_table: PRTable = {} if len(labels) <= 2: binaraized_target = pd.DataFrame(binaraized_target[:, 0]) binaraized_target.columns = ["target"] From a7e04728efa109694fff76f4e114b834465bfe27 Mon Sep 17 00:00:00 2001 From: Vyacheslav Morov Date: Thu, 17 Oct 2024 15:24:32 +0200 Subject: [PATCH 056/103] Use proper variable in quality_by_feature_table. --- .../classification_performance/quality_by_feature_table.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/evidently/metrics/classification_performance/quality_by_feature_table.py b/src/evidently/metrics/classification_performance/quality_by_feature_table.py index edf400caca..fbbab8bf88 100644 --- a/src/evidently/metrics/classification_performance/quality_by_feature_table.py +++ b/src/evidently/metrics/classification_performance/quality_by_feature_table.py @@ -117,7 +117,7 @@ def calculate(self, data: InputData) -> ClassificationQualityByFeatureTableResul curr_predictions = get_prediction_data(data.current_data, dataset_columns, data.column_mapping.pos_label) ref_predictions = None if ref_df is not None: - ref_predictions = get_prediction_data(data.reference_data, dataset_columns, data.column_mapping.pos_label) + ref_predictions = get_prediction_data(ref_df, dataset_columns, data.column_mapping.pos_label) if self.columns is None: columns = ( dataset_columns.num_feature_names From 7a7fd3138c6db8ef19f626bde12e69ad1e0d919b Mon Sep 17 00:00:00 2001 From: Vyacheslav Morov Date: Thu, 17 Oct 2024 16:24:33 +0200 Subject: [PATCH 057/103] Fix default options value in metrics. --- src/evidently/base_metric.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/evidently/base_metric.py b/src/evidently/base_metric.py index 7852708db8..6999d5fa69 100644 --- a/src/evidently/base_metric.py +++ b/src/evidently/base_metric.py @@ -237,7 +237,7 @@ class Config: _context: Optional["Context"] = None - options: Options = Field(default_factory=Options) + options: Optional[Options] = Field(default=None) fields: ClassVar[FieldsDescriptor] = FieldsDescriptor() # resulting options will be determined via From a4e2e8a7419888ce6d32509d8df52ca973f635e3 Mon Sep 17 00:00:00 2001 From: Vyacheslav Morov Date: Thu, 17 Oct 2024 20:21:17 +0200 Subject: [PATCH 058/103] Replace values to to_numpy in roc_curve_metric. --- .../metrics/classification_performance/roc_curve_metric.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/evidently/metrics/classification_performance/roc_curve_metric.py b/src/evidently/metrics/classification_performance/roc_curve_metric.py index 4850a7aae7..cb8000e5c2 100644 --- a/src/evidently/metrics/classification_performance/roc_curve_metric.py +++ b/src/evidently/metrics/classification_performance/roc_curve_metric.py @@ -60,8 +60,8 @@ def calculate_metrics(self, target_data: pd.Series, prediction: PredictionData) labels = prediction.labels if prediction.prediction_probas is None: raise ValueError("Roc Curve can be calculated only on binary probabilistic predictions") - binaraized_target = (target_data.values.reshape(-1, 1) == labels).astype(int) - roc_curve = {} + binaraized_target = (target_data.to_numpy().reshape(-1, 1) == labels).astype(int) + roc_curve: ROCCurve = {} if len(labels) <= 2: binaraized_target = pd.DataFrame(binaraized_target[:, 0]) binaraized_target.columns = ["target"] From c3c904403984822e76714ffe82b26f9ccda80a23 Mon Sep 17 00:00:00 2001 From: Vyacheslav Morov Date: Thu, 17 Oct 2024 20:23:21 +0200 Subject: [PATCH 059/103] Fix typing errors in error_in_time. --- .../regression_performance/error_in_time.py | 28 +++++++++++++------ 1 file changed, 20 insertions(+), 8 deletions(-) diff --git a/src/evidently/metrics/regression_performance/error_in_time.py b/src/evidently/metrics/regression_performance/error_in_time.py index dbea78ce57..50f5dc325e 100644 --- a/src/evidently/metrics/regression_performance/error_in_time.py +++ b/src/evidently/metrics/regression_performance/error_in_time.py @@ -1,3 +1,4 @@ +from typing import Dict from typing import List from typing import Optional from typing import Union @@ -9,7 +10,6 @@ from evidently.base_metric import Metric from evidently.base_metric import UsesRawDataMixin from evidently.metric_results import ColumnAggScatterResult -from evidently.metric_results import ColumnScatter from evidently.metric_results import ColumnScatterResult from evidently.model.widget import BaseWidgetInfo from evidently.options.base import AnyOptions @@ -46,8 +46,8 @@ def calculate(self, data: InputData) -> ColumnScatterResult: if ref_df is not None: ref_df = self._make_df_for_plot(ref_df.copy(), target_name, prediction_name, datetime_column_name) ref_error = ref_df[prediction_name] - ref_df[target_name] - current_scatter = {} - reference_scatter: Optional[Union[dict, ColumnScatter]] = None + current_scatter: Dict[str, Union[pd.Series, pd.DataFrame]] = {} + reference_scatter: Optional[Dict[str, Union[pd.Series, pd.DataFrame]]] = None raw_data = self.get_options().render_options.raw_data if raw_data: current_scatter["Predicted - Actual"] = curr_error @@ -55,13 +55,17 @@ def calculate(self, data: InputData) -> ColumnScatterResult: current_scatter["x"] = curr_df[datetime_column_name] x_name = "Timestamp" else: - current_scatter["x"] = curr_df.index + current_scatter["x"] = curr_df.index.to_series() x_name = "Index" if ref_df is not None: + if ref_error is None: + raise ValueError("ref_error is None but required") reference_scatter = {} reference_scatter["Predicted - Actual"] = ref_error - reference_scatter["x"] = ref_df[datetime_column_name] if datetime_column_name else ref_df.index + reference_scatter["x"] = ( + ref_df[datetime_column_name] if datetime_column_name else ref_df.index.to_series() + ) return ColumnScatterResult( current=current_scatter, @@ -96,7 +100,13 @@ def calculate(self, data: InputData) -> ColumnScatterResult: x_name_ref=x_name_ref, ) - def _make_df_for_plot(self, df, target_name: str, prediction_name: str, datetime_column_name: Optional[str]): + def _make_df_for_plot( + self, + df: pd.DataFrame, + target_name: str, + prediction_name: str, + datetime_column_name: Optional[str], + ) -> pd.DataFrame: result = df.replace([np.inf, -np.inf], np.nan) if datetime_column_name is not None: result.dropna( @@ -105,9 +115,11 @@ def _make_df_for_plot(self, df, target_name: str, prediction_name: str, datetime inplace=True, subset=[target_name, prediction_name, datetime_column_name], ) - return result.sort_values(datetime_column_name) + result.sort_values(datetime_column_name, inplace=True) + return result result.dropna(axis=0, how="any", inplace=True, subset=[target_name, prediction_name]) - return result.sort_index() + result.sort_index(inplace=True) + return result @default_renderer(wrap_type=RegressionErrorPlot) From 9ce518118a2e95cee13fcd9bfb20cf253d5582db Mon Sep 17 00:00:00 2001 From: Vyacheslav Morov Date: Fri, 18 Oct 2024 10:22:57 +0200 Subject: [PATCH 060/103] Fix typing for results in abs_perc_error_in_time. --- .../abs_perc_error_in_time.py | 42 ++++++++++++------- 1 file changed, 26 insertions(+), 16 deletions(-) diff --git a/src/evidently/metrics/regression_performance/abs_perc_error_in_time.py b/src/evidently/metrics/regression_performance/abs_perc_error_in_time.py index 0a3922ec49..117f09bc99 100644 --- a/src/evidently/metrics/regression_performance/abs_perc_error_in_time.py +++ b/src/evidently/metrics/regression_performance/abs_perc_error_in_time.py @@ -3,10 +3,12 @@ from typing import Union import numpy as np +import pandas as pd from evidently.base_metric import InputData from evidently.base_metric import Metric from evidently.base_metric import UsesRawDataMixin +from evidently.metric_results import ColumnAggScatter from evidently.metric_results import ColumnAggScatterResult from evidently.metric_results import ColumnScatter from evidently.metric_results import ColumnScatterResult @@ -53,35 +55,38 @@ def calculate(self, data: InputData) -> ColumnScatterResult: reference_scatter: Optional[Union[ColumnScatter, dict]] = None raw_data = self.get_options().render_options.raw_data if raw_data: - current_scatter = {} + current_scatter: ColumnScatter = {} current_scatter["Absolute Percentage Error"] = curr_df["Absolute Percentage Error"] if datetime_column_name is not None: current_scatter["x"] = curr_df[datetime_column_name] x_name = "Timestamp" else: - current_scatter["x"] = curr_df.index + current_scatter["x"] = curr_df.index.to_series() x_name = "Index" if ref_df is not None: reference_scatter = {} reference_scatter["Absolute Percentage Error"] = ref_df["Absolute Percentage Error"] - reference_scatter["x"] = ref_df[datetime_column_name] if datetime_column_name else ref_df.index + reference_scatter["x"] = ( + ref_df[datetime_column_name] if datetime_column_name else ref_df.index.to_series() + ) return ColumnScatterResult( current=current_scatter, reference=reference_scatter, x_name=x_name, ) - current_scatter = {} + agg_current_scatter: ColumnAggScatter = {} + agg_reference_scatter: Optional[ColumnAggScatter] = None plot_df, prefix = prepare_df_for_time_index_plot(curr_df, "Absolute Percentage Error", datetime_column_name) - current_scatter["Absolute Percentage Error"] = plot_df + agg_current_scatter["Absolute Percentage Error"] = plot_df x_name_ref: Optional[str] = None if ref_df is not None: - reference_scatter = {} + agg_reference_scatter = {} plot_df, prefix_ref = prepare_df_for_time_index_plot( ref_df, "Absolute Percentage Error", datetime_column_name ) - reference_scatter["Absolute Percentage Error"] = plot_df + agg_reference_scatter["Absolute Percentage Error"] = plot_df if datetime_column_name is None: x_name_ref = "Index binned" else: @@ -90,18 +95,21 @@ def calculate(self, data: InputData) -> ColumnScatterResult: x_name = "Index binned" else: x_name = datetime_column_name + f" ({prefix})" - cls = ColumnScatterResult - if not raw_data: - cls = ColumnAggScatterResult - return cls( - current=current_scatter, - reference=reference_scatter, + return ColumnAggScatterResult( + current=agg_current_scatter, + reference=agg_reference_scatter, x_name=x_name, x_name_ref=x_name_ref, ) - def _make_df_for_plot(self, df, target_name: str, prediction_name: str, datetime_column_name: Optional[str]): + def _make_df_for_plot( + self, + df: pd.DataFrame, + target_name: str, + prediction_name: str, + datetime_column_name: Optional[str], + ) -> pd.DataFrame: result = df.replace([np.inf, -np.inf], np.nan) if datetime_column_name is not None: result.dropna( @@ -110,9 +118,11 @@ def _make_df_for_plot(self, df, target_name: str, prediction_name: str, datetime inplace=True, subset=[target_name, prediction_name, datetime_column_name], ) - return result.sort_values(datetime_column_name) + result.sort_values(datetime_column_name, inplace=True) + return result result.dropna(axis=0, how="any", inplace=True, subset=[target_name, prediction_name]) - return result.sort_index() + result.sort_index(inplace=True) + return result @default_renderer(wrap_type=RegressionAbsPercentageErrorPlot) From ec733417b1c7edc11da1776fc69ad707d5387619 Mon Sep 17 00:00:00 2001 From: Vyacheslav Morov Date: Fri, 18 Oct 2024 10:26:12 +0200 Subject: [PATCH 061/103] Fix typing for DataFrame processing in error_distribution. --- .../regression_performance/error_distribution.py | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/src/evidently/metrics/regression_performance/error_distribution.py b/src/evidently/metrics/regression_performance/error_distribution.py index 1c3caa0332..dea248c614 100644 --- a/src/evidently/metrics/regression_performance/error_distribution.py +++ b/src/evidently/metrics/regression_performance/error_distribution.py @@ -2,6 +2,7 @@ from typing import Optional import numpy as np +import pandas as pd from evidently.base_metric import InputData from evidently.base_metric import Metric @@ -56,7 +57,13 @@ def calculate(self, data: InputData) -> RegressionErrorDistributionResults: return RegressionErrorDistributionResults(current_bins=current_bins, reference_bins=reference_bins) - def _make_df_for_plot(self, df, target_name: str, prediction_name: str, datetime_column_name: Optional[str]): + def _make_df_for_plot( + self, + df: pd.DataFrame, + target_name: str, + prediction_name: str, + datetime_column_name: Optional[str], + ) -> pd.DataFrame: result = df.replace([np.inf, -np.inf], np.nan) if datetime_column_name is not None: result.dropna( @@ -65,9 +72,11 @@ def _make_df_for_plot(self, df, target_name: str, prediction_name: str, datetime inplace=True, subset=[target_name, prediction_name, datetime_column_name], ) - return result.sort_values(datetime_column_name) + result.sort_values(datetime_column_name, inplace=True) + return result result.dropna(axis=0, how="any", inplace=True, subset=[target_name, prediction_name]) - return result.sort_index() + result.sort_index(inplace=True) + return result @default_renderer(wrap_type=RegressionErrorDistribution) From 49f551e8daa99535b07733dcda0cbd74cf911676 Mon Sep 17 00:00:00 2001 From: Vyacheslav Morov Date: Fri, 18 Oct 2024 10:30:58 +0200 Subject: [PATCH 062/103] Fix typing errors in error_normality. --- .../regression_performance/error_normality.py | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/src/evidently/metrics/regression_performance/error_normality.py b/src/evidently/metrics/regression_performance/error_normality.py index cd2a713624..199086461a 100644 --- a/src/evidently/metrics/regression_performance/error_normality.py +++ b/src/evidently/metrics/regression_performance/error_normality.py @@ -82,7 +82,13 @@ def calculate(self, data: InputData) -> RegressionErrorNormalityResults: reference_theoretical=reference_theoretical, ) - def _make_df_for_plot(self, df, target_name: str, prediction_name: str, datetime_column_name: Optional[str]): + def _make_df_for_plot( + self, + df: pd.DataFrame, + target_name: str, + prediction_name: str, + datetime_column_name: Optional[str], + ) -> pd.DataFrame: result = df.replace([np.inf, -np.inf], np.nan) if datetime_column_name is not None: result.dropna( @@ -91,9 +97,11 @@ def _make_df_for_plot(self, df, target_name: str, prediction_name: str, datetime inplace=True, subset=[target_name, prediction_name, datetime_column_name], ) - return result.sort_values(datetime_column_name) + result.sort_values(datetime_column_name, inplace=True) + return result result.dropna(axis=0, how="any", inplace=True, subset=[target_name, prediction_name]) - return result.sort_index() + result.sort_index(inplace=True) + return result def _get_theoretical_line(self, res: Any): x = [res[0][0][0], res[0][0][-1]] @@ -104,7 +112,7 @@ def _get_plot_data(self, res: Any, err_data: pd.Series, agg_data: bool): df = pd.DataFrame({"x": res[0][0], "y": res[0][1]}) if not agg_data: return df - df["bin"] = pd.cut(err_data.sort_values().values, bins=10, labels=False, retbins=False) + df["bin"] = pd.cut(err_data.sort_values().to_numpy(), bins=10, labels=False, retbins=False) return ( df.groupby("bin", group_keys=False) .apply(lambda x: x.sample(n=min(100, x.shape[0]), random_state=0)) From be6a3ad0c335b2281c6d515f2be44753fadbd2fe Mon Sep 17 00:00:00 2001 From: Vyacheslav Morov Date: Fri, 18 Oct 2024 10:36:16 +0200 Subject: [PATCH 063/103] Fix typing errors in predicted_and_actual_in_time. --- .../predicted_and_actual_in_time.py | 30 +++++++++++-------- 1 file changed, 17 insertions(+), 13 deletions(-) diff --git a/src/evidently/metrics/regression_performance/predicted_and_actual_in_time.py b/src/evidently/metrics/regression_performance/predicted_and_actual_in_time.py index d3772f3fda..03679aacd4 100644 --- a/src/evidently/metrics/regression_performance/predicted_and_actual_in_time.py +++ b/src/evidently/metrics/regression_performance/predicted_and_actual_in_time.py @@ -52,28 +52,35 @@ def calculate(self, data: InputData) -> ColumnScatterResult: current_scatter["x"] = curr_df[datetime_column_name] x_name = "Timestamp" else: - current_scatter["x"] = curr_df.index + current_scatter["x"] = curr_df.index.to_series() x_name = "Index" if ref_df is not None: reference_scatter = {} reference_scatter["Predicted"] = ref_df[prediction_name] reference_scatter["Actual"] = ref_df[target_name] - reference_scatter["x"] = ref_df[datetime_column_name] if datetime_column_name else ref_df.index + reference_scatter["x"] = ( + ref_df[datetime_column_name] if datetime_column_name else ref_df.index.to_series() + ) return ColumnScatterResult( current=current_scatter, reference=reference_scatter, x_name=x_name, ) - current_scatter = {} + agg_current_scatter = {} + agg_reference_scatter = None plot_df, prefix = prepare_df_for_time_index_plot(curr_df, prediction_name, datetime_column_name) - current_scatter["Predicted"] = plot_df - current_scatter["Actual"], _ = prepare_df_for_time_index_plot(curr_df, target_name, datetime_column_name) + agg_current_scatter["Predicted"] = plot_df + agg_current_scatter["Actual"], _ = prepare_df_for_time_index_plot(curr_df, target_name, datetime_column_name) x_name_ref: Optional[str] = None if ref_df is not None: - reference_scatter = {} + agg_reference_scatter = {} plot_df, prefix_ref = prepare_df_for_time_index_plot(ref_df, prediction_name, datetime_column_name) - reference_scatter["Predicted"] = plot_df - reference_scatter["Actual"], _ = prepare_df_for_time_index_plot(ref_df, target_name, datetime_column_name) + agg_reference_scatter["Predicted"] = plot_df + agg_reference_scatter["Actual"], _ = prepare_df_for_time_index_plot( + ref_df, + target_name, + datetime_column_name, + ) if datetime_column_name is None: x_name_ref = "Index binned" else: @@ -82,11 +89,8 @@ def calculate(self, data: InputData) -> ColumnScatterResult: x_name = "Index binned" else: x_name = datetime_column_name + f" ({prefix})" - cls = ColumnScatterResult - if not raw_data: - cls = ColumnAggScatterResult - return cls( - current=current_scatter, + return ColumnAggScatterResult( + current=agg_current_scatter, reference=reference_scatter, x_name=x_name, x_name_ref=x_name_ref, From bfe497db775516d3ddc66b7199698948d048d221 Mon Sep 17 00:00:00 2001 From: Vyacheslav Morov Date: Fri, 18 Oct 2024 10:47:34 +0200 Subject: [PATCH 064/103] Add explicit check for reference data in target_by_features_table. --- .../metrics/data_drift/target_by_features_table.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/evidently/metrics/data_drift/target_by_features_table.py b/src/evidently/metrics/data_drift/target_by_features_table.py index bbba4010dc..9049fe3c3f 100644 --- a/src/evidently/metrics/data_drift/target_by_features_table.py +++ b/src/evidently/metrics/data_drift/target_by_features_table.py @@ -221,10 +221,12 @@ def render_html(self, obj: TargetByFeaturesTable) -> List[BaseWidgetInfo]: current_data = result.current.plot_data # todo: better typing assert current_data is not None - reference_data = result.reference.plot_data if result.reference is not None else None + if result.reference is None: + raise ValueError("reference is not set but required") + reference_data = result.reference.plot_data target_name = result.target_name curr_predictions = result.current.predictions - ref_predictions = result.reference.predictions if result.reference is not None else None + ref_predictions = result.reference.predictions columns = result.columns task = result.task if curr_predictions is not None and ref_predictions is not None: From 062641771a087e8b9d5f2a060436ae4df9ca1536 Mon Sep 17 00:00:00 2001 From: Vyacheslav Morov Date: Fri, 18 Oct 2024 10:48:11 +0200 Subject: [PATCH 065/103] Ignore import of infer_dtype_from_object in data_integrity_tests. --- src/evidently/tests/data_integrity_tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/evidently/tests/data_integrity_tests.py b/src/evidently/tests/data_integrity_tests.py index 214e6a6fa2..8c2e503528 100644 --- a/src/evidently/tests/data_integrity_tests.py +++ b/src/evidently/tests/data_integrity_tests.py @@ -7,7 +7,7 @@ import numpy as np import pandas as pd -from pandas.core.dtypes.common import infer_dtype_from_object +from pandas.core.dtypes.common import infer_dtype_from_object # type: ignore[attr-defined] from evidently.base_metric import ColumnName from evidently.metrics import ColumnRegExpMetric From 7297b6a86a1fc431a168e8e10568771ba127ed21 Mon Sep 17 00:00:00 2001 From: Vyacheslav Morov Date: Fri, 18 Oct 2024 10:56:40 +0200 Subject: [PATCH 066/103] Fix types in utils. --- src/evidently/tests/utils.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/evidently/tests/utils.py b/src/evidently/tests/utils.py index 4dad885b62..0cadbb5460 100644 --- a/src/evidently/tests/utils.py +++ b/src/evidently/tests/utils.py @@ -1,4 +1,5 @@ from typing import List +from typing import Literal from typing import Optional from typing import Tuple @@ -193,7 +194,7 @@ def dataframes_to_table( columns: List[str], table_id: str, sort_by: str = "curr", - na_position: str = "first", + na_position: Literal["first", "last"] = "first", asc: bool = False, ): display_columns = ["display"] From 131305135a56b8e2785eb494c69392a7d2822fb7 Mon Sep 17 00:00:00 2001 From: Vyacheslav Morov Date: Fri, 18 Oct 2024 10:59:13 +0200 Subject: [PATCH 067/103] User proper types in pydantic_utils. --- src/evidently/pydantic_utils.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/evidently/pydantic_utils.py b/src/evidently/pydantic_utils.py index 983330107d..971935959a 100644 --- a/src/evidently/pydantic_utils.py +++ b/src/evidently/pydantic_utils.py @@ -28,6 +28,7 @@ from typing_inspect import is_union_type from evidently._pydantic_compat import SHAPE_DICT +from evidently._pydantic_compat import BaseConfig from evidently._pydantic_compat import BaseModel from evidently._pydantic_compat import Field from evidently._pydantic_compat import ModelMetaclass @@ -189,7 +190,7 @@ def is_not_abstract(cls): class PolymorphicModel(BaseModel): - class Config(BaseModel.Config): + class Config(BaseConfig): # value to put into "type" field type_alias: ClassVar[Optional[str]] = None # flag to mark alias required. If not required, classpath is used by default @@ -200,7 +201,7 @@ class Config(BaseModel.Config): # flag to mark type as base. This means it will be possible to parse all subclasses of it as this type is_base_type: ClassVar[bool] = False - __config__: ClassVar[Config] + __config__: ClassVar[Type[Config]] = Config @classmethod def __get_type__(cls): @@ -272,7 +273,7 @@ def validate(cls: Type[TPM], value: Any) -> TPM: subcls = import_string(classpath) except ImportError as e: raise ValueError(f"Error importing subclass from '{classpath}'") from e - return subcls.validate(value) + return subcls.validate(value) # type: ignore[return-value] return super().validate(value) # type: ignore[misc] From a3f5593a813a3bcb11e576b5e9561d9ae3eab09c Mon Sep 17 00:00:00 2001 From: Vyacheslav Morov Date: Fri, 18 Oct 2024 11:23:08 +0200 Subject: [PATCH 068/103] Fix types in test_suites. --- src/evidently/ui/dashboards/test_suites.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/evidently/ui/dashboards/test_suites.py b/src/evidently/ui/dashboards/test_suites.py index 673f557c53..d7eb70efa3 100644 --- a/src/evidently/ui/dashboards/test_suites.py +++ b/src/evidently/ui/dashboards/test_suites.py @@ -186,7 +186,7 @@ def get_color(test, date) -> Optional[str]: def to_period(time_agg: Optional[str], timestamp: datetime.datetime) -> datetime.datetime: if time_agg is None: return timestamp - return pd.Series([timestamp], name="dt").dt.to_period(time_agg)[0] + return pd.Series([timestamp], name="dt").dt.to_period(time_agg)[0].to_timestamp() @autoregister From 99cfd151945457bab62cb0d17b446b46b3c3da8d Mon Sep 17 00:00:00 2001 From: Vyacheslav Morov Date: Fri, 18 Oct 2024 13:22:57 +0200 Subject: [PATCH 069/103] Ignore typing error with generic operation in reports. --- src/evidently/ui/dashboards/reports.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/evidently/ui/dashboards/reports.py b/src/evidently/ui/dashboards/reports.py index 8548cd0573..832b60f073 100644 --- a/src/evidently/ui/dashboards/reports.py +++ b/src/evidently/ui/dashboards/reports.py @@ -156,7 +156,7 @@ async def build( timestamp_start: Optional[datetime.datetime], timestamp_end: Optional[datetime.datetime], ) -> BaseWidgetInfo: - bins_for_hists: Dict[Metric, List[Tuple[datetime.datetime, Union[HistogramData, Distribution]]]] = ( + bins_for_hists: Dict[Metric, List[Tuple[datetime.datetime, Union[HistogramData, Distribution]]]] = ( # type: ignore[assignment] await data_storage.load_points_as_type( Union[HistogramData, Distribution], # type: ignore[arg-type] project_id, From aef8a8cf20dc44250c9d106408ccb775b83bf8e0 Mon Sep 17 00:00:00 2001 From: Vyacheslav Morov Date: Fri, 18 Oct 2024 13:46:43 +0200 Subject: [PATCH 070/103] Fix typing in utils. --- src/evidently/calculations/stattests/utils.py | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/src/evidently/calculations/stattests/utils.py b/src/evidently/calculations/stattests/utils.py index 39a1bce86d..ebd6f8d1f6 100644 --- a/src/evidently/calculations/stattests/utils.py +++ b/src/evidently/calculations/stattests/utils.py @@ -100,17 +100,16 @@ def generate_fisher2x2_contingency_table(reference_data: pd.Series, current_data if len(unique_categories) != 2: unique_categories.add("placeholder") - unique_categories = list(unique_categories) # type: ignore - unique_categories = dict(zip(unique_categories, [0, 1])) # type: ignore + unique_categories_mapping = dict(zip(list(unique_categories), [0, 1])) - reference_data = reference_data.map(unique_categories).values - current_data = current_data.map(unique_categories).values + ref_data = reference_data.map(unique_categories_mapping).to_numpy() + curr_data = current_data.map(unique_categories_mapping).to_numpy() - zero_ref = reference_data.size - np.count_nonzero(reference_data) - one_ref = np.count_nonzero(reference_data) + zero_ref = ref_data.size - np.count_nonzero(ref_data) + one_ref = np.count_nonzero(ref_data) - zero_cur = current_data.size - np.count_nonzero(current_data) - one_cur = np.count_nonzero(current_data) + zero_cur = curr_data.size - np.count_nonzero(curr_data) + one_cur = np.count_nonzero(curr_data) contingency_table = np.array([[one_cur, zero_cur], [one_ref, zero_ref]]) From 62cd03fef7a931c559f538ba19247ee5b600f79c Mon Sep 17 00:00:00 2001 From: Vyacheslav Morov Date: Fri, 18 Oct 2024 13:54:57 +0200 Subject: [PATCH 071/103] Ignore typing error in classification_performance. --- src/evidently/calculations/classification_performance.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/evidently/calculations/classification_performance.py b/src/evidently/calculations/classification_performance.py index 66bf140c54..92465d15fe 100644 --- a/src/evidently/calculations/classification_performance.py +++ b/src/evidently/calculations/classification_performance.py @@ -225,7 +225,7 @@ def get_prediction_data( return PredictionData( predictions=data[prediction], prediction_probas=None, - labels=data[prediction].unique().tolist(), + labels=data[prediction].unique().tolist(), # type: ignore[operator] ) @@ -376,7 +376,7 @@ def calculate_metrics( f1 = metrics.f1_score(target, prediction.predictions, average="macro") if prediction.prediction_probas is not None: binaraized_target = ( - target.astype(str).values.reshape(-1, 1) == list(prediction.prediction_probas.columns.astype(str)) + target.astype(str).to_numpy().reshape(-1, 1) == list(prediction.prediction_probas.columns.astype(str)) ).astype(int) prediction_probas_array = prediction.prediction_probas.to_numpy() roc_auc = metrics.roc_auc_score(binaraized_target, prediction_probas_array, average="macro") From ac265551d8c84563d6c05d9bca04a873f5c4d3bb Mon Sep 17 00:00:00 2001 From: Vyacheslav Morov Date: Fri, 18 Oct 2024 13:59:00 +0200 Subject: [PATCH 072/103] Fix type conversion in regression_performance. --- src/evidently/calculations/regression_performance.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/evidently/calculations/regression_performance.py b/src/evidently/calculations/regression_performance.py index 7c77217387..40f77868f7 100644 --- a/src/evidently/calculations/regression_performance.py +++ b/src/evidently/calculations/regression_performance.py @@ -146,7 +146,7 @@ def _error_num_feature_bias(dataset, feature_name, err_quantiles: ErrorWithQuant def _stable_value_counts(series: pd.Series): - return series.value_counts().reindex(pd.unique(series)) + return series.value_counts().reindex(pd.unique(series.to_numpy())) def _error_cat_feature_bias(dataset, feature_name, err_quantiles: ErrorWithQuantiles) -> FeatureBias: From 071e68c2601f35b9b4973a9a0a2824205b82e3f4 Mon Sep 17 00:00:00 2001 From: Vyacheslav Morov Date: Fri, 18 Oct 2024 14:11:57 +0200 Subject: [PATCH 073/103] Explicitly provide column_name to super class in OOV_words_percentage_feature. --- src/evidently/features/OOV_words_percentage_feature.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/evidently/features/OOV_words_percentage_feature.py b/src/evidently/features/OOV_words_percentage_feature.py index 6ab72e6d9a..209c41e6cd 100644 --- a/src/evidently/features/OOV_words_percentage_feature.py +++ b/src/evidently/features/OOV_words_percentage_feature.py @@ -26,10 +26,9 @@ class Config: _eng_words: Set def __init__(self, column_name: str, ignore_words=(), display_name: Optional[str] = None): - self.column_name = column_name self.ignore_words = ignore_words self.display_name = display_name - super().__init__() + super().__init__(column_name=column_name) def apply(self, value: Any): if value is None or (isinstance(value, float) and np.isnan(value)): From b498e29941d8a6b9037d94785c184dafbee0f49f Mon Sep 17 00:00:00 2001 From: Vyacheslav Morov Date: Fri, 18 Oct 2024 14:16:26 +0200 Subject: [PATCH 074/103] Make proper calls to super init in hf_feature. --- src/evidently/features/hf_feature.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/evidently/features/hf_feature.py b/src/evidently/features/hf_feature.py index e002015458..5c517ceb93 100644 --- a/src/evidently/features/hf_feature.py +++ b/src/evidently/features/hf_feature.py @@ -27,8 +27,7 @@ def __init__(self, *, column_name: str, model: str, params: dict, display_name: self.model = model self.params = params self.display_name = display_name - self.feature_type = _model_type(model) - super().__init__() + super().__init__(feature_type=_model_type(model)) def generate_data(self, data: pd.DataFrame, data_definition: DataDefinition) -> pd.Series: val = _models.get(self.model) @@ -60,10 +59,9 @@ def __init__( toxic_label: Optional[str] = None, ): self.column_name = column_name - self.display_name = display_name self.model = model self.toxic_label = toxic_label - super().__init__() + super().__init__(display_name=display_name) def generate_data(self, data: pd.DataFrame, data_definition: DataDefinition) -> pd.Series: return _toxicity(self.model, self.toxic_label, data[self.column_name]) From e4c6b4542b34db6a75df75d3f26eec42ebb2a78d Mon Sep 17 00:00:00 2001 From: Vyacheslav Morov Date: Fri, 18 Oct 2024 14:28:19 +0200 Subject: [PATCH 075/103] Make proper return type in __iter__ methods in base_suite. --- src/evidently/suite/base_suite.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/evidently/suite/base_suite.py b/src/evidently/suite/base_suite.py index 2e109afeaf..fa6225255d 100644 --- a/src/evidently/suite/base_suite.py +++ b/src/evidently/suite/base_suite.py @@ -7,7 +7,7 @@ from typing import IO from typing import Any from typing import Dict -from typing import Iterator +from typing import Generator from typing import List from typing import Optional from typing import Tuple @@ -90,7 +90,7 @@ def find_metric_renderer(obj, renderers: RenderersDefinitions) -> MetricRenderer raise KeyError(f"No renderer found for {obj}") -def _discover_dependencies(test: Union[Metric, Test]) -> Iterator[Tuple[str, Union[Metric, Test]]]: +def _discover_dependencies(test: Union[Metric, Test]) -> Generator[Tuple[str, Union[Metric, Test]], None, None]: if hasattr(test, "__evidently_dependencies__"): yield from test.__evidently_dependencies__() # type: ignore[union-attr] return @@ -480,7 +480,7 @@ class DatasetLinks(BaseModel): current: Optional[DatasetID] = None additional: Dict[str, DatasetID] = {} - def __iter__(self) -> Iterator[Tuple[str, DatasetID]]: + def __iter__(self) -> Generator[Tuple[str, DatasetID], None, None]: if self.reference is not None: yield "reference", self.reference if self.current is not None: @@ -492,9 +492,9 @@ class DatasetInputOutputLinks(BaseModel): input: DatasetLinks = DatasetLinks() output: DatasetLinks = DatasetLinks() - def __iter__(self) -> Iterator[Tuple[str, str, DatasetID]]: - yield from (("input", subtype, dataset_id) for subtype, dataset_id in self.input) - yield from (("output", subtype, dataset_id) for subtype, dataset_id in self.output) + def __iter__(self) -> Generator[Tuple[str, Tuple[str, DatasetID]], None, None]: + yield from (("input", (subtype, dataset_id)) for subtype, dataset_id in self.input) + yield from (("output", (subtype, dataset_id)) for subtype, dataset_id in self.output) class SnapshotLinks(BaseModel): From a8aabd48eab7a2e33f2c8cecef947cb0d16b0174 Mon Sep 17 00:00:00 2001 From: Vyacheslav Morov Date: Fri, 18 Oct 2024 14:29:09 +0200 Subject: [PATCH 076/103] Move column_name to super.__init__ call in non_letter_character_percentage_feature. --- .../features/non_letter_character_percentage_feature.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/evidently/features/non_letter_character_percentage_feature.py b/src/evidently/features/non_letter_character_percentage_feature.py index 4ce38ced52..8b0d1aa8de 100644 --- a/src/evidently/features/non_letter_character_percentage_feature.py +++ b/src/evidently/features/non_letter_character_percentage_feature.py @@ -17,9 +17,8 @@ class Config: column_name: str def __init__(self, column_name: str, display_name: Optional[str] = None): - self.column_name = column_name self.display_name = display_name - super().__init__() + super().__init__(column_name=column_name) def apply(self, value: Any): """counts share of characters that are not letters or spaces""" From 0a8898b8baca9f17dac1a48c99fce4417588f1b7 Mon Sep 17 00:00:00 2001 From: Vyacheslav Morov Date: Fri, 18 Oct 2024 14:30:24 +0200 Subject: [PATCH 077/103] Move column_name to super.__init__ call in features. --- src/evidently/features/sentence_count_feature.py | 3 +-- src/evidently/features/sentiment_feature.py | 3 +-- src/evidently/features/text_length_feature.py | 3 +-- src/evidently/features/trigger_words_presence_feature.py | 3 +-- src/evidently/features/word_count_feature.py | 3 +-- src/evidently/features/words_feature.py | 3 +-- 6 files changed, 6 insertions(+), 12 deletions(-) diff --git a/src/evidently/features/sentence_count_feature.py b/src/evidently/features/sentence_count_feature.py index 6b4215cae8..870657b5d8 100644 --- a/src/evidently/features/sentence_count_feature.py +++ b/src/evidently/features/sentence_count_feature.py @@ -19,9 +19,8 @@ class Config: column_name: str def __init__(self, column_name: str, display_name: Optional[str] = None): - self.column_name = column_name self.display_name = display_name - super().__init__() + super().__init__(column_name=column_name) def apply(self, value: Any): if value is None or (isinstance(value, float) and np.isnan(value)): diff --git a/src/evidently/features/sentiment_feature.py b/src/evidently/features/sentiment_feature.py index 8226ac2e2f..f654c3fdfb 100644 --- a/src/evidently/features/sentiment_feature.py +++ b/src/evidently/features/sentiment_feature.py @@ -21,9 +21,8 @@ class Config: _sid: Optional[SentimentIntensityAnalyzer] = PrivateAttr(None) def __init__(self, column_name: str, display_name: Optional[str] = None): - self.column_name = column_name self.display_name = display_name - super().__init__() + super().__init__(column_name=column_name) @property def sid(self): diff --git a/src/evidently/features/text_length_feature.py b/src/evidently/features/text_length_feature.py index 68a5bb1056..d350a2d9c6 100644 --- a/src/evidently/features/text_length_feature.py +++ b/src/evidently/features/text_length_feature.py @@ -17,9 +17,8 @@ class Config: column_name: str def __init__(self, column_name: str, display_name: Optional[str] = None): - self.column_name = column_name self.display_name = display_name - super().__init__() + super().__init__(column_name=column_name) def apply(self, value: Any): if value is None or (isinstance(value, float) and np.isnan(value)): diff --git a/src/evidently/features/trigger_words_presence_feature.py b/src/evidently/features/trigger_words_presence_feature.py index 933cabcb55..675b7bd23c 100644 --- a/src/evidently/features/trigger_words_presence_feature.py +++ b/src/evidently/features/trigger_words_presence_feature.py @@ -29,11 +29,10 @@ def __init__( lemmatize: bool = True, display_name: Optional[str] = None, ): - self.column_name = column_name self.words_list = words_list self.lemmatize = lemmatize self.display_name = display_name - super().__init__() + super().__init__(column_name=column_name) @property def lem(self): diff --git a/src/evidently/features/word_count_feature.py b/src/evidently/features/word_count_feature.py index 9154412fed..a36b4bfe0f 100644 --- a/src/evidently/features/word_count_feature.py +++ b/src/evidently/features/word_count_feature.py @@ -19,9 +19,8 @@ class Config: column_name: str def __init__(self, column_name: str, display_name: Optional[str] = None): - self.column_name = column_name self.display_name = display_name - super().__init__() + super().__init__(column_name=column_name) def apply(self, value: Any): if value is None or (isinstance(value, float) and np.isnan(value)): diff --git a/src/evidently/features/words_feature.py b/src/evidently/features/words_feature.py index f76e04150a..72526abf9a 100644 --- a/src/evidently/features/words_feature.py +++ b/src/evidently/features/words_feature.py @@ -63,14 +63,13 @@ def __init__( lemmatize: bool = True, display_name: Optional[str] = None, ): - self.column_name = column_name self.words_list = words_list if mode not in ["includes_any", "includes_all", "excludes_any", "excludes_all"]: raise ValueError("mode must be either 'includes_any', 'includes_all', 'excludes_any' or 'excludes_all'") self.mode = mode self.lemmatize = lemmatize self.display_name = display_name - super().__init__() + super().__init__(column_name=column_name) @property def lem(self): From 0f9b6dc25029a9304dc830ed270c7c00a2f0e513 Mon Sep 17 00:00:00 2001 From: Vyacheslav Morov Date: Fri, 18 Oct 2024 14:34:30 +0200 Subject: [PATCH 078/103] Replace values with to_numpy in feature_importance. --- src/evidently/metrics/data_drift/feature_importance.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/evidently/metrics/data_drift/feature_importance.py b/src/evidently/metrics/data_drift/feature_importance.py index 028881fe99..2fd7545712 100644 --- a/src/evidently/metrics/data_drift/feature_importance.py +++ b/src/evidently/metrics/data_drift/feature_importance.py @@ -72,9 +72,9 @@ def get_feature_importance_from_samples( for col in [x.column_name for x in cat_cols]: enc = OrdinalEncoder(handle_unknown="use_encoded_value", unknown_value=np.nan) - curr_sampled_data[col] = enc.fit_transform(curr_sampled_data[col].astype(str).values.reshape(-1, 1)) + curr_sampled_data[col] = enc.fit_transform(curr_sampled_data[col].astype(str).to_numpy().reshape(-1, 1)) if ref_sampled_data is not None: - ref_sampled_data[col] = enc.fit_transform(ref_sampled_data[col].astype(str).values.reshape(-1, 1)) + ref_sampled_data[col] = enc.fit_transform(ref_sampled_data[col].astype(str).to_numpy().reshape(-1, 1)) task = data_definition.task target_column = data_definition.get_target_column() From c183b8cbb91510733f4de731be41c7daf605f01b Mon Sep 17 00:00:00 2001 From: Vyacheslav Morov Date: Fri, 18 Oct 2024 14:37:41 +0200 Subject: [PATCH 079/103] Ignore type error in data_preprocessing. --- src/evidently/utils/data_preprocessing.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/evidently/utils/data_preprocessing.py b/src/evidently/utils/data_preprocessing.py index 4c84f9d652..d7ec1cf31f 100644 --- a/src/evidently/utils/data_preprocessing.py +++ b/src/evidently/utils/data_preprocessing.py @@ -586,7 +586,8 @@ def _get_column_type( f" Returning type from reference" ) cur_type = ref_type - if not np.can_cast(cur_type, ref_type) and not np.can_cast(ref_type, cur_type): + # TODO: add proper type check + if not np.can_cast(cur_type, ref_type) and not np.can_cast(ref_type, cur_type): # type: ignore[arg-type] logging.warning( f"Column {column_name} have different types in reference {ref_type} and current {cur_type}." f" Returning type from reference" From 814ca33de5a6c0387a40cdaef214785445c50842 Mon Sep 17 00:00:00 2001 From: Vyacheslav Morov Date: Fri, 18 Oct 2024 14:49:13 +0200 Subject: [PATCH 080/103] Explicit check for exclude argument in options/base. --- src/evidently/options/base.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/evidently/options/base.py b/src/evidently/options/base.py index a31787f075..c3c3e1cce4 100644 --- a/src/evidently/options/base.py +++ b/src/evidently/options/base.py @@ -109,8 +109,10 @@ def dict( exclude = {"custom"} elif isinstance(exclude, set): exclude.add("custom") - else: + elif isinstance(exclude, dict): exclude["custom"] = False + else: + raise TypeError("exclude must be either a dict or a set") return super().dict( include=include, exclude=exclude, From f7d7e848ca62a79641508b2417875e1258c91b0c Mon Sep 17 00:00:00 2001 From: Vyacheslav Morov Date: Fri, 18 Oct 2024 14:56:29 +0200 Subject: [PATCH 081/103] Make a result_type classmethod but ignore typing error in base_metric. --- src/evidently/base_metric.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/evidently/base_metric.py b/src/evidently/base_metric.py index 6999d5fa69..cc5a9f6981 100644 --- a/src/evidently/base_metric.py +++ b/src/evidently/base_metric.py @@ -143,12 +143,14 @@ def get_datasets(self) -> Tuple[Optional[TEngineDataType], TEngineDataType]: class InputData(GenericInputData[pd.DataFrame]): @staticmethod - def _get_by_column_name(dataset: pd.DataFrame, additional: pd.DataFrame, column: ColumnName) -> pd.Series: + def _get_by_column_name(dataset: pd.DataFrame, additional: Optional[pd.DataFrame], column: ColumnName) -> pd.Series: if column.dataset == DatasetType.MAIN: if column.name not in dataset.columns: raise ColumnNotFound(column.name) return dataset[column.name] if column.dataset == DatasetType.ADDITIONAL: + if additional is None: + raise ValueError("no additional dataset is provided, but field requested") return additional[column.name] raise ValueError("unknown column data") @@ -220,8 +222,11 @@ def __get__(self, instance: Optional["Metric"], type: Type["Metric"]) -> FieldPa class WithResultFieldPathMetaclass(FrozenBaseMeta): + @classmethod def result_type(cls) -> Type[MetricResult]: - return typing_inspect.get_args(next(b for b in cls.__orig_bases__ if typing_inspect.is_generic_type(b)))[0] + return typing_inspect.get_args( + next(b for b in cls.__orig_bases__ if typing_inspect.is_generic_type(b)) # type: ignore[attr-defined] + )[0] class BasePreset(EvidentlyBaseModel): From a6a9e27cdaa1deb7e84aae37d149df8f628e4b9f Mon Sep 17 00:00:00 2001 From: Vyacheslav Morov Date: Fri, 18 Oct 2024 15:15:23 +0200 Subject: [PATCH 082/103] Fix typing and ignore type collision in subclass in metric_results. --- src/evidently/metric_results.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/evidently/metric_results.py b/src/evidently/metric_results.py index fa614e48ed..a89b856bb3 100644 --- a/src/evidently/metric_results.py +++ b/src/evidently/metric_results.py @@ -236,9 +236,9 @@ def df_from_column_scatter(value: ColumnScatter) -> pd.DataFrame: def column_scatter_from_df(df: Optional[pd.DataFrame], with_index: bool) -> Optional[ColumnScatter]: if df is None: return None - data = {column: df[column] for column in df.columns} + data: ColumnScatter = {column: df[column] for column in df.columns} if with_index: - data["index"] = df.index + data["index"] = df.index.to_series() return data @@ -291,8 +291,9 @@ class Config: type_alias = "evidently:metric_result:ColumnAggScatterResult" field_tags = {"current": {IncludeTags.Current}, "reference": {IncludeTags.Reference}} - current: ColumnAggScatter - reference: Optional[ColumnAggScatter] + # TODO: fix type collision with super type + current: ColumnAggScatter # type: ignore[assignment] + reference: Optional[ColumnAggScatter] # type: ignore[assignment] PlotData = List[float] From 58802dab01ace44e677e8884da8bffa13519b802 Mon Sep 17 00:00:00 2001 From: Vyacheslav Morov Date: Fri, 18 Oct 2024 16:06:29 +0200 Subject: [PATCH 083/103] Fix deduplication in get_additional_features. --- src/evidently/calculation_engine/engine.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/evidently/calculation_engine/engine.py b/src/evidently/calculation_engine/engine.py index c46e123829..131551d339 100644 --- a/src/evidently/calculation_engine/engine.py +++ b/src/evidently/calculation_engine/engine.py @@ -117,7 +117,7 @@ def get_additional_features(self, data_definition: DataDefinition) -> List[Gener continue for feature in required_features: fp = feature.get_fingerprint() - if fp in feature: + if fp in features: continue features[fp] = feature return list(features.values()) From 16b62a10b5db973738f5f0d19634fc5084a67a54 Mon Sep 17 00:00:00 2001 From: Vyacheslav Morov Date: Fri, 18 Oct 2024 16:27:54 +0200 Subject: [PATCH 084/103] Ignore type errors in python_engine. --- src/evidently/calculation_engine/python_engine.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/src/evidently/calculation_engine/python_engine.py b/src/evidently/calculation_engine/python_engine.py index 4cfed0e72e..3c17695865 100644 --- a/src/evidently/calculation_engine/python_engine.py +++ b/src/evidently/calculation_engine/python_engine.py @@ -71,8 +71,8 @@ def calculate_additional_features( def merge_additional_features( self, features: Dict[GeneratedFeatures, FeatureResult[pd.DataFrame]] ) -> EngineDatasets[pd.DataFrame]: - currents: List[pd.DataFrame] = [] - references: List[pd.DataFrame] = [] + currents = [] + references = [] for feature, result in features.items(): currents.append(result.current) @@ -84,15 +84,13 @@ def merge_additional_features( elif len(currents) == 1: current = currents[0] else: - cur, *currents = currents - current = cur.join(currents) + current = currents[0].join(currents[1:]) # type: ignore[arg-type] if len(references) == 0: return EngineDatasets(current=current, reference=None) if len(references) == 1: return EngineDatasets(current=current, reference=references[0]) - ref, *references = references - return EngineDatasets(current=current, reference=ref.join(references)) + return EngineDatasets(current=current, reference=references[0].join(references[1:])) # type: ignore[arg-type] def get_metric_implementation(self, metric): impl = super().get_metric_implementation(metric) From ff7f441e118cb92c805bd85273d386198d2173c3 Mon Sep 17 00:00:00 2001 From: Vyacheslav Morov Date: Fri, 18 Oct 2024 16:40:42 +0200 Subject: [PATCH 085/103] Add convert index to series in data_operations. --- src/evidently/utils/data_operations.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/evidently/utils/data_operations.py b/src/evidently/utils/data_operations.py index d46896ae86..c433ae7ce5 100644 --- a/src/evidently/utils/data_operations.py +++ b/src/evidently/utils/data_operations.py @@ -73,7 +73,7 @@ def process_columns(dataset: pd.DataFrame, column_mapping: ColumnMapping) -> Dat else: num_feature_names = [col for col in num_feature_names if col in dataset.columns] empty_cols = dataset[num_feature_names].isnull().mean() - empty_cols = empty_cols[empty_cols == 1.0].index + empty_cols = empty_cols[empty_cols == 1.0].index.to_series() num_feature_names = sorted( list(set(dataset[num_feature_names].select_dtypes([np.number]).columns).union(set(empty_cols))) ) @@ -82,7 +82,7 @@ def process_columns(dataset: pd.DataFrame, column_mapping: ColumnMapping) -> Dat datetime_feature_names = sorted(list(set(dataset.select_dtypes(["datetime"]).columns) - utility_columns_set)) else: empty_cols = dataset[datetime_feature_names].isnull().mean() - empty_cols = empty_cols[empty_cols == 1.0].index + empty_cols = empty_cols[empty_cols == 1.0].index.to_series() datetime_feature_names = sorted( list(set(dataset[datetime_feature_names].select_dtypes(["datetime"]).columns).union(set(empty_cols))) ) From 4db99ee19b6a9d4237698da5785c829f4916fcec Mon Sep 17 00:00:00 2001 From: Vyacheslav Morov Date: Fri, 18 Oct 2024 17:23:16 +0200 Subject: [PATCH 086/103] Fix variable reuse in text_part_feature. --- src/evidently/features/text_part_feature.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/evidently/features/text_part_feature.py b/src/evidently/features/text_part_feature.py index e3a86d5e68..dd03481b95 100644 --- a/src/evidently/features/text_part_feature.py +++ b/src/evidently/features/text_part_feature.py @@ -35,12 +35,12 @@ def _feature_column_name(self) -> str: return f"{self.column_name}.{self.prefix}.{self.case_sensitive}" def generate_feature(self, data: pd.DataFrame, data_definition: DataDefinition) -> pd.DataFrame: - data = data[self.column_name] + column_data = data[self.column_name] substr = self.prefix if not self.case_sensitive: - data = data.str.casefold() + column_data = column_data.str.casefold() substr = substr.casefold() - calculated = data.str.startswith(substr) + calculated = column_data.str.startswith(substr) return pd.DataFrame({self._feature_column_name(): calculated}) def _as_column(self) -> ColumnName: @@ -76,12 +76,12 @@ def _feature_column_name(self) -> str: return f"{self.column_name}.{self.suffix}.{self.case_sensitive}" def generate_feature(self, data: pd.DataFrame, data_definition: DataDefinition) -> pd.DataFrame: - data = data[self.column_name] + column_data = data[self.column_name] substr = self.suffix if not self.case_sensitive: - data = data.str.casefold() + column_data = column_data.str.casefold() substr = substr.casefold() - calculated = data.str.endswith(substr) + calculated = column_data.str.endswith(substr) return pd.DataFrame({self._feature_column_name(): calculated}) def _as_column(self) -> ColumnName: From fc9f15bfb811b38135450637b9ee47d082f9b71c Mon Sep 17 00:00:00 2001 From: Vyacheslav Morov Date: Fri, 18 Oct 2024 17:23:37 +0200 Subject: [PATCH 087/103] Add type hint and ignore typing error in html_widgets. --- src/evidently/renderers/html_widgets.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/evidently/renderers/html_widgets.py b/src/evidently/renderers/html_widgets.py index 4d84a43cb0..f8fb79e261 100644 --- a/src/evidently/renderers/html_widgets.py +++ b/src/evidently/renderers/html_widgets.py @@ -613,7 +613,7 @@ def get_heatmaps_widget( # show values if thw heatmap is small if len(columns) < 15: - heatmap_text = np.round(data, 2).astype(str) + heatmap_text: Optional[pd.DataFrame] = np.round(data, 2).astype(str) # type: ignore[assignment] heatmap_text_template: Optional[str] = "%{text}" else: From c9493846c5d36578ca4bd744a2594cf668892133 Mon Sep 17 00:00:00 2001 From: Vyacheslav Morov Date: Fri, 18 Oct 2024 17:25:17 +0200 Subject: [PATCH 088/103] Ignore type errors in column_value_list_metric. --- .../metrics/data_quality/column_value_list_metric.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/evidently/metrics/data_quality/column_value_list_metric.py b/src/evidently/metrics/data_quality/column_value_list_metric.py index 840934b4af..6bba50477d 100644 --- a/src/evidently/metrics/data_quality/column_value_list_metric.py +++ b/src/evidently/metrics/data_quality/column_value_list_metric.py @@ -115,14 +115,14 @@ def _calculate_stats(values: list, column: pd.Series) -> ValueListStat: else: values_not_in_list[value] = value_counts[value] - number_in_list = sum(values_in_list.values()) + number_in_list = sum(values_in_list.values()) # type: ignore[arg-type] share_in_list = number_in_list / rows_count number_not_in_list = rows_count - number_in_list share_not_in_list = number_not_in_list / rows_count # fill other values from list with zeroes for value in values: if value not in values_in_list: - values_in_list[value] = 0 + values_in_list[value] = 0 # type: ignore[assignment] return ValueListStat( number_in_list=number_in_list, From 287a2f356ef1d2bae22840c905cd18441e69858f Mon Sep 17 00:00:00 2001 From: Vyacheslav Morov Date: Fri, 18 Oct 2024 17:25:38 +0200 Subject: [PATCH 089/103] Move feature_type to super.__init__ in openai_feature. --- src/evidently/features/openai_feature.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/evidently/features/openai_feature.py b/src/evidently/features/openai_feature.py index 11100a4ac7..6e79436911 100644 --- a/src/evidently/features/openai_feature.py +++ b/src/evidently/features/openai_feature.py @@ -56,12 +56,11 @@ def __init__( self.context_replace_string = context_replace_string self.openai_params = openai_params or {} self.model = model - self.feature_type = ColumnType.Categorical if feature_type == "cat" else ColumnType.Numerical self.column_name = column_name self.display_name = display_name self.check_mode = check_mode self.possible_values = [v.lower() for v in possible_values] if possible_values else None - super().__init__() + super().__init__(feature_type=ColumnType.Categorical if feature_type == "cat" else ColumnType.Numerical) def generate_feature(self, data: pd.DataFrame, data_definition: DataDefinition) -> pd.DataFrame: from openai import OpenAI From 6ed31650e809389051b1ac0a9106bc7db08a3498 Mon Sep 17 00:00:00 2001 From: Vyacheslav Morov Date: Fri, 18 Oct 2024 17:32:55 +0200 Subject: [PATCH 090/103] Enable typechecking for report.py. --- src/evidently/renderers/base_renderer.py | 3 ++- src/evidently/report/report.py | 4 ++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/src/evidently/renderers/base_renderer.py b/src/evidently/renderers/base_renderer.py index a03c93f0a3..89901fdc89 100644 --- a/src/evidently/renderers/base_renderer.py +++ b/src/evidently/renderers/base_renderer.py @@ -1,6 +1,7 @@ import dataclasses import warnings from typing import TYPE_CHECKING +from typing import Any from typing import Dict from typing import Generic from typing import List @@ -65,7 +66,7 @@ def render_html(self, obj: TMetric) -> List[BaseWidgetInfo]: @dataclasses.dataclass class DetailsInfo: title: str - info: BaseWidgetInfo + info: Union[BaseWidgetInfo, Any] id: str = dataclasses.field(default_factory=lambda: str(uuid6.uuid7())) diff --git a/src/evidently/report/report.py b/src/evidently/report/report.py index 327e8e60e7..e5c55721e3 100644 --- a/src/evidently/report/report.py +++ b/src/evidently/report/report.py @@ -6,6 +6,7 @@ from typing import Dict from typing import List from typing import Optional +from typing import Tuple from typing import Type from typing import Union @@ -217,7 +218,7 @@ def as_dataframe(self, group: str = None) -> Union[Dict[str, pd.DataFrame], pd.D raise ValueError(f"Metric group {group} not found in this report") return result[group] - def _build_dashboard_info(self): + def _build_dashboard_info(self) -> Tuple[str, DashboardInfo, Dict[str, dict]]: metrics_results: List[BaseWidgetInfo] = [] additional_graphs = [] @@ -237,7 +238,6 @@ def _build_dashboard_info(self): for additional_graph in info_item.get_additional_graphs(): if isinstance(additional_graph, AdditionalGraphInfo): additional_graphs.append(DetailsInfo("", additional_graph.params, additional_graph.id)) - else: additional_graphs.append(DetailsInfo("", additional_graph, additional_graph.id)) From a3d639372f661ce97f7df2b91556c6693a804a23 Mon Sep 17 00:00:00 2001 From: Vyacheslav Morov Date: Fri, 18 Oct 2024 17:40:21 +0200 Subject: [PATCH 091/103] Enable typechecking in get_distr in scores_distribution. --- src/evidently/metrics/recsys/scores_distribution.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/evidently/metrics/recsys/scores_distribution.py b/src/evidently/metrics/recsys/scores_distribution.py index 90cdc2e267..66a1188216 100644 --- a/src/evidently/metrics/recsys/scores_distribution.py +++ b/src/evidently/metrics/recsys/scores_distribution.py @@ -1,5 +1,6 @@ from typing import List from typing import Optional +from typing import Tuple import pandas as pd from scipy.special import softmax @@ -57,11 +58,16 @@ def __init__(self, k: int, options: AnyOptions = None) -> None: self.k = k super().__init__(options=options) - def get_distr(self, df, user_id, prediction_name): + def get_distr( + self, + df: pd.DataFrame, + user_id: Optional[str], + prediction_name: str, + ) -> Tuple[Distribution, Optional[Distribution], float]: df["rank"] = df.groupby(user_id)[prediction_name].transform("rank", ascending=False) top_k = df.loc[df["rank"] <= self.k, prediction_name] if self.k == df["rank"].max: - other: pd.Series = None + other: Optional[pd.Series] = None else: other = df.loc[df["rank"] > self.k, prediction_name] top_k_distr, other_distr = get_distribution_for_column(column_type="num", current=top_k, reference=other) From f83eb15da475e30034c2db6a3ae0de68052d4373 Mon Sep 17 00:00:00 2001 From: Vyacheslav Morov Date: Fri, 18 Oct 2024 17:42:44 +0200 Subject: [PATCH 092/103] Add type hints in column_category_metric. --- src/evidently/metrics/data_quality/column_category_metric.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/evidently/metrics/data_quality/column_category_metric.py b/src/evidently/metrics/data_quality/column_category_metric.py index 6d94b97aa6..fa2af32b7a 100644 --- a/src/evidently/metrics/data_quality/column_category_metric.py +++ b/src/evidently/metrics/data_quality/column_category_metric.py @@ -50,7 +50,7 @@ class Config: } smart_union = True - def __init__(self, **data): + def __init__(self, **data) -> None: """for backward compatibility""" if "counts_of_values" in data: counts_of_values: Dict[str, pd.DataFrame] = data.pop("counts_of_values") From 6ee8f76fd6bb89468ff3c4915c8c9642f075774f Mon Sep 17 00:00:00 2001 From: Vyacheslav Morov Date: Sat, 19 Oct 2024 00:55:56 +0200 Subject: [PATCH 093/103] Fix tests. --- src/evidently/base_metric.py | 1 - src/evidently/calculations/utils.py | 4 ++-- src/evidently/metrics/data_drift/data_drift_table.py | 4 ++-- .../metrics/data_quality/dataset_correlations_metric.py | 4 ++-- 4 files changed, 6 insertions(+), 7 deletions(-) diff --git a/src/evidently/base_metric.py b/src/evidently/base_metric.py index cc5a9f6981..631446be84 100644 --- a/src/evidently/base_metric.py +++ b/src/evidently/base_metric.py @@ -222,7 +222,6 @@ def __get__(self, instance: Optional["Metric"], type: Type["Metric"]) -> FieldPa class WithResultFieldPathMetaclass(FrozenBaseMeta): - @classmethod def result_type(cls) -> Type[MetricResult]: return typing_inspect.get_args( next(b for b in cls.__orig_bases__ if typing_inspect.is_generic_type(b)) # type: ignore[attr-defined] diff --git a/src/evidently/calculations/utils.py b/src/evidently/calculations/utils.py index 9676c3941d..0794931cc1 100644 --- a/src/evidently/calculations/utils.py +++ b/src/evidently/calculations/utils.py @@ -72,7 +72,7 @@ def get_count_values(col1: pd.Series, col2: pd.Series, col1_name: str, col2_name df = pd.DataFrame({col2_name: col2, col1_name: col1}) grouped = df.groupby([col2_name, col1_name], observed=False).size() grouped.name = "count_objects" - grouped.reset_index(inplace=True) + grouped = grouped.reset_index() return grouped[grouped["count_objects"] > 0] @@ -249,7 +249,7 @@ def transform_df_to_time_count_view( if not isinstance(grouped, pd.Series): raise ValueError("grouped has incorrect type") grouped.name = "num" - grouped.reset_index(inplace=True) + grouped = grouped.reset_index() grouped[datetime_column_name] = grouped["period"].dt.to_timestamp() return grouped[grouped["num"] > 0] diff --git a/src/evidently/metrics/data_drift/data_drift_table.py b/src/evidently/metrics/data_drift/data_drift_table.py index 37ded23e94..819f464565 100644 --- a/src/evidently/metrics/data_drift/data_drift_table.py +++ b/src/evidently/metrics/data_drift/data_drift_table.py @@ -13,7 +13,7 @@ from evidently.core import IncludeTags from evidently.metric_results import DatasetColumns from evidently.metric_results import HistogramData -from evidently.metric_results import ScatterData +from evidently.metric_results import ScatterField from evidently.metrics.data_drift.base import WithDriftOptions from evidently.metrics.data_drift.feature_importance import FeatureImportanceMetric from evidently.model.widget import BaseWidgetInfo @@ -222,7 +222,7 @@ def _generate_column_params( data_drift = "Detected" if data.drift_detected else "Not Detected" if data.column_type == "num" and data.scatter is not None: if not agg_data: - if not isinstance(data.scatter, ScatterData): + if not isinstance(data.scatter, ScatterField): raise ValueError("data.scatter has incompatible type") scatter_fig = plot_scatter_for_data_drift( curr_y=data.scatter.scatter[data.column_name].tolist(), diff --git a/src/evidently/metrics/data_quality/dataset_correlations_metric.py b/src/evidently/metrics/data_quality/dataset_correlations_metric.py index 8c3eb7359e..c24528baf5 100644 --- a/src/evidently/metrics/data_quality/dataset_correlations_metric.py +++ b/src/evidently/metrics/data_quality/dataset_correlations_metric.py @@ -158,7 +158,7 @@ def _get_correlations_stats(correlation: pd.DataFrame, data_definition: DataDefi abs_max_target_features_correlation: Optional[pd.Series] if target_name is not None and target_name in correlation_matrix: cols = [target_name] + columns_corr - abs_max_target_features_correlation = correlation_matrix.loc[cols].abs().max() + abs_max_target_features_correlation = correlation_matrix.loc[cols].abs().max().max() if pd.isnull(abs_max_target_features_correlation): abs_max_target_features_correlation = None @@ -169,7 +169,7 @@ def _get_correlations_stats(correlation: pd.DataFrame, data_definition: DataDefi abs_max_prediction_features_correlation: Optional[pd.Series] if prediction_name is not None and prediction_name in correlation_matrix: cols = [prediction_name] + columns_corr - abs_max_prediction_features_correlation = correlation_matrix.loc[cols].abs().max() + abs_max_prediction_features_correlation = correlation_matrix.loc[cols].abs().max().max() if pd.isnull(abs_max_prediction_features_correlation): abs_max_prediction_features_correlation = None From 59ca9e17c9714b350c6ccf07c303d6f26b08530a Mon Sep 17 00:00:00 2001 From: Vyacheslav Morov Date: Sat, 19 Oct 2024 01:46:49 +0200 Subject: [PATCH 094/103] Set min pydantic to 2.2.1 --- requirements.min.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.min.txt b/requirements.min.txt index e7a5d12f28..605741f081 100644 --- a/requirements.min.txt +++ b/requirements.min.txt @@ -8,7 +8,7 @@ nltk==3.6.7 scipy==1.10.0 requests==2.32.0 PyYAML==5.4 -pydantic==1.10.13 +pydantic==2.2.1 litestar==2.8.3 typing-inspect==0.9.0 uvicorn==0.22.0 From 41a95e7d73ed30fb99f823f807798ccb0fee5d23 Mon Sep 17 00:00:00 2001 From: Vyacheslav Morov Date: Sat, 19 Oct 2024 01:53:50 +0200 Subject: [PATCH 095/103] Revert pedantic version change and configure for pydantic v1. --- requirements.min.txt | 2 +- setup.cfg | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements.min.txt b/requirements.min.txt index 605741f081..e7a5d12f28 100644 --- a/requirements.min.txt +++ b/requirements.min.txt @@ -8,7 +8,7 @@ nltk==3.6.7 scipy==1.10.0 requests==2.32.0 PyYAML==5.4 -pydantic==2.2.1 +pydantic==1.10.13 litestar==2.8.3 typing-inspect==0.9.0 uvicorn==0.22.0 diff --git a/setup.cfg b/setup.cfg index 2bd67531fc..c0e8eef2c4 100644 --- a/setup.cfg +++ b/setup.cfg @@ -27,7 +27,7 @@ python_version = 3.8 disable_error_code = misc namespace_packages = true no_implicit_optional = False -plugins = pydantic.mypy,pydantic.v1.mypy,numpy.typing.mypy_plugin +plugins = pydantic.mypy,numpy.typing.mypy_plugin [mypy-nltk.*] ignore_missing_imports = True From c27ffbda862c7074a3644472da9189c8c0861b31 Mon Sep 17 00:00:00 2001 From: Vyacheslav Morov Date: Sat, 19 Oct 2024 02:26:18 +0200 Subject: [PATCH 096/103] Add pandas-stubs dependency. --- requirements.dev.txt | 1 + setup.py | 1 + 2 files changed, 2 insertions(+) diff --git a/requirements.dev.txt b/requirements.dev.txt index b0da653b4a..dc0589f754 100644 --- a/requirements.dev.txt +++ b/requirements.dev.txt @@ -4,6 +4,7 @@ setuptools==65.5.1; python_version < '3.12' setuptools==68.2.2; python_version >= '3.12' jupyter==1.0.0 mypy==1.1.1 +pandas-stubs pytest==7.4.4 pytest-asyncio==0.23.7 types-PyYAML==6.0.1 diff --git a/setup.py b/setup.py index bcab894123..26b30c6224 100644 --- a/setup.py +++ b/setup.py @@ -85,6 +85,7 @@ "setuptools==68.2.2; python_version >= '3.12'", "jupyter==1.0.0", "mypy==1.1.1", + "pandas-stubs", "pytest==7.4.4", "types-PyYAML==6.0.1", "types-requests==2.26.0", From 22c21dabdd6309c5677d7de7191779d0432606c7 Mon Sep 17 00:00:00 2001 From: Vyacheslav Morov Date: Sat, 19 Oct 2024 02:55:44 +0200 Subject: [PATCH 097/103] Fix another typing errors. --- src/evidently/calculations/utils.py | 4 ++-- src/evidently/metrics/recsys/pairwise_distance.py | 4 ++-- src/evidently/utils/visualizations.py | 6 +++--- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/evidently/calculations/utils.py b/src/evidently/calculations/utils.py index 0794931cc1..2ff745a34c 100644 --- a/src/evidently/calculations/utils.py +++ b/src/evidently/calculations/utils.py @@ -52,7 +52,7 @@ def make_hist_for_cat_plot(curr: pd.Series, ref: pd.Series = None, normalize: bo ) .reset_index() ) - hist_df.columns = ["x", "count"] + hist_df.columns = pd.Index(["x", "count"]) result["current"] = hist_df if ref is not None: hist_df = ( @@ -63,7 +63,7 @@ def make_hist_for_cat_plot(curr: pd.Series, ref: pd.Series = None, normalize: bo ) .reset_index() ) - hist_df.columns = ["x", "count"] + hist_df.columns = pd.Index(["x", "count"]) result["reference"] = hist_df return result diff --git a/src/evidently/metrics/recsys/pairwise_distance.py b/src/evidently/metrics/recsys/pairwise_distance.py index 6b5d354608..1866c56082 100644 --- a/src/evidently/metrics/recsys/pairwise_distance.py +++ b/src/evidently/metrics/recsys/pairwise_distance.py @@ -62,11 +62,11 @@ def calculate(self, data: InputData) -> PairwiseDistanceResult: all_items = all_items[all_items[prediction_name] <= self.k + 1] all_items = all_items[[item_id.column_name] + self.item_features] if current_train_data is not None: - if not np.in1d(self.item_features, current_train_data.columns).all(): + if not np.in1d(self.item_features, current_train_data.columns).all(): # type: ignore[attr-defined] raise ValueError("current_train_data must contain item_features.") all_items = pd.concat([all_items, current_train_data[[item_id.column_name] + self.item_features]]) if reference_train_data is not None: - if not np.in1d(self.item_features, reference_train_data.columns).all(): + if not np.in1d(self.item_features, reference_train_data.columns).all(): # type: ignore[attr-defined] raise ValueError("reference_train_data must contain item_features.") all_items = pd.concat([all_items, reference_train_data[[item_id.column_name] + self.item_features]]) diff --git a/src/evidently/utils/visualizations.py b/src/evidently/utils/visualizations.py index 91c552d36f..64d7200e3f 100644 --- a/src/evidently/utils/visualizations.py +++ b/src/evidently/utils/visualizations.py @@ -693,7 +693,7 @@ def make_hist_for_cat_plot(curr: pd.Series, ref: pd.Series = None, normalize: bo .value_counts(normalize=normalize, dropna=dropna) # type: ignore[call-overload] .reset_index() ) - hist_df.columns = ["x", "count"] + hist_df.columns = pd.Index(["x", "count"]) current = HistogramData.from_df(hist_df) reference = None @@ -703,7 +703,7 @@ def make_hist_for_cat_plot(curr: pd.Series, ref: pd.Series = None, normalize: bo .value_counts(normalize=normalize, dropna=dropna) # type: ignore[call-overload] .reset_index() ) - hist_df.columns = ["x", "count"] + hist_df.columns = pd.Index(["x", "count"]) reference = HistogramData.from_df(hist_df) return Histogram(current=current, reference=reference) @@ -712,7 +712,7 @@ def get_distribution_for_category_column(column: pd.Series, normalize: bool = Fa value_counts = column.value_counts(normalize=normalize, dropna=False) # type: ignore[call-overload] # filter out na values if it amount == 0 - new_values = [(k, v) for k, v in value_counts.items() if (not pd.isna(k) or v > 0)] + new_values = [(k, v) for k, v in value_counts.items() if (not pd.isna(k) or v > 0)] # type: ignore[call-overload] return Distribution( x=[x[0] for x in new_values], From becb5f7dae992f1525f686dacd66c4d9c7669dda Mon Sep 17 00:00:00 2001 From: Vyacheslav Morov Date: Sat, 19 Oct 2024 03:00:14 +0200 Subject: [PATCH 098/103] Fix another typing errors. --- src/evidently/experimental/dataset_generators/llm/prompts.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/evidently/experimental/dataset_generators/llm/prompts.py b/src/evidently/experimental/dataset_generators/llm/prompts.py index bb38038f57..9070613036 100644 --- a/src/evidently/experimental/dataset_generators/llm/prompts.py +++ b/src/evidently/experimental/dataset_generators/llm/prompts.py @@ -31,7 +31,7 @@ class Config: ] @llm_call - def generate(self, seed_question: str, number: int) -> List[str]: ... + def generate(self, seed_question: str, number: int) -> List[str]: ... # type: ignore[empty-body] class QuestionsFromContextPromptTemplate(WithSystemPrompt, BlockPromptTemplate): @@ -41,7 +41,7 @@ class Config: system_prompt: str = "You are an assistant who generates questions based on provided context" @llm_call - def generate_questions(self, context: str, number: int) -> List[str]: ... + def generate_questions(self, context: str, number: int) -> List[str]: ... # type: ignore[empty-body] class NaiveQuestionsFromContextPromptTemplate(QuestionsFromContextPromptTemplate): From da060570da199d9089d1758009478f957b9d4508 Mon Sep 17 00:00:00 2001 From: Vyacheslav Morov Date: Tue, 22 Oct 2024 13:16:57 +0200 Subject: [PATCH 099/103] Add pandas-stubs version. --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 26b30c6224..d5df8514e0 100644 --- a/setup.py +++ b/setup.py @@ -85,7 +85,7 @@ "setuptools==68.2.2; python_version >= '3.12'", "jupyter==1.0.0", "mypy==1.1.1", - "pandas-stubs", + "pandas-stubs>=1.3.5", "pytest==7.4.4", "types-PyYAML==6.0.1", "types-requests==2.26.0", From 15bfe24d674dc02e4f9bc7f99fc69bb8e7b1d507 Mon Sep 17 00:00:00 2001 From: Vyacheslav Morov Date: Wed, 23 Oct 2024 14:14:47 +0200 Subject: [PATCH 100/103] Remove unnecessary check. --- src/evidently/calculation_engine/engine.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/evidently/calculation_engine/engine.py b/src/evidently/calculation_engine/engine.py index 131551d339..4f78aca315 100644 --- a/src/evidently/calculation_engine/engine.py +++ b/src/evidently/calculation_engine/engine.py @@ -117,8 +117,6 @@ def get_additional_features(self, data_definition: DataDefinition) -> List[Gener continue for feature in required_features: fp = feature.get_fingerprint() - if fp in features: - continue features[fp] = feature return list(features.values()) From 35038bf79cf57dd1933c8980013fb5e5c4ffb0f1 Mon Sep 17 00:00:00 2001 From: Svetlana <164897708+svetleo-evidently@users.noreply.github.com> Date: Thu, 24 Oct 2024 16:53:25 +0200 Subject: [PATCH 101/103] OpenAI error message (#1354) --- src/evidently/utils/llm/wrapper.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/evidently/utils/llm/wrapper.py b/src/evidently/utils/llm/wrapper.py index 8f6a123a43..c858ec9a1e 100644 --- a/src/evidently/utils/llm/wrapper.py +++ b/src/evidently/utils/llm/wrapper.py @@ -194,8 +194,8 @@ async def complete(self, messages: List[LLMMessage]) -> str: messages = [{"role": msg.role, "content": msg.content} for msg in messages] try: response = await self.client.chat.completions.create(model=self.model, messages=messages) # type: ignore[arg-type] - except openai.OpenAIError as e: - raise LLMRequestError("Failed to call OpenAI complete API") from e + except openai.APIError as e: + raise LLMRequestError(f"Failed to call OpenAI complete API: {e.message}") from e content = response.choices[0].message.content assert content is not None # todo: better error return content From 08d15026e3f6511c4b81c9cf3bd95d44cdb70488 Mon Sep 17 00:00:00 2001 From: Rama Chaitnaya Karanam <50360061+ramachaitanya0@users.noreply.github.com> Date: Sat, 26 Oct 2024 02:36:11 +0530 Subject: [PATCH 102/103] added JSONMatch Descriptor (#1337) --- docs/book/reference/all-metrics.md | 1 + src/evidently/descriptors/__init__.py | 2 + src/evidently/descriptors/_registry.py | 5 ++ .../descriptors/json_match_descriptor.py | 13 +++++ src/evidently/features/_registry.py | 1 + src/evidently/features/json_match_feature.py | 46 +++++++++++++++ tests/features/test_json_match.py | 57 +++++++++++++++++++ 7 files changed, 125 insertions(+) create mode 100644 src/evidently/descriptors/json_match_descriptor.py create mode 100644 src/evidently/features/json_match_feature.py create mode 100644 tests/features/test_json_match.py diff --git a/docs/book/reference/all-metrics.md b/docs/book/reference/all-metrics.md index 71a233fcdf..bcb2683ffc 100644 --- a/docs/book/reference/all-metrics.md +++ b/docs/book/reference/all-metrics.md @@ -279,6 +279,7 @@ Check for regular expression matches. | **ExactMatch()**
  • Checks if the text matches between two columns.
  • Returns True/False for every input.
Example use:
`ExactMatch(column_name='column_1')`| **Required:**
`with_column`

**Optional:**
  • `display_name`
| | **IsValidJSON()**
  • Checks if the text in a specified column is a valid JSON.
  • Returns True/False for every input.
Example use:
`IsValidJSON(column_name='column_1')`| **Required:**
`column_name`

**Optional:**
  • `display_name`
| | **JSONSchemaMatch()**
  • Checks if the text contains a JSON object matching the **expected_schema**. Supports exact (**exact=True**) or minimal (**exact=False**) matching, with optional strict type validation (**validate_types=True**).
  • Returns True/False for each row.
Example use:
`JSONSchemaMatch(expected_schema={"name": str, "age": int}, exact_match=False, validate_types=True)`| **Required:**
`expected_schema: Dict[str, type]`

**Optional:**
  • `exact_match = True` or `False`
  • `validate_types = True` or `False`
| +| **JSONMatch()**
  • Compares two columns of a dataframe and checks whether the two objects in each row of the dataframe are matching JSON's or not.
  • Returns True/False for every input.
Example use:
`JSONMatch(with_column="column_2")`| **Required:**
`with_column : str`

**Optional:**
  • `display_name`
    • | | **ContainsLink()**
    • Checks if the text contains at least one valid URL.
    • Returns True/False for each row.
    Example use:
    `ContainsLink(column_name='column_1')`| **Required:**
    `column_name: str`

    **Optional:**
    • `display_name`
    | ## Descriptors: Text stats diff --git a/src/evidently/descriptors/__init__.py b/src/evidently/descriptors/__init__.py index f5c80aea45..f8ea236035 100644 --- a/src/evidently/descriptors/__init__.py +++ b/src/evidently/descriptors/__init__.py @@ -6,6 +6,7 @@ from .hf_descriptor import HuggingFaceModel from .hf_descriptor import HuggingFaceToxicityModel from .is_valid_json_descriptor import IsValidJSON +from .json_match_descriptor import JSONMatch from .json_schema_match_descriptor import JSONSchemaMatch from .llm_judges import BiasLLMEval from .llm_judges import ContextQualityLLMEval @@ -72,4 +73,5 @@ "IsValidJSON", "JSONSchemaMatch", "_registry", + "JSONMatch", ] diff --git a/src/evidently/descriptors/_registry.py b/src/evidently/descriptors/_registry.py index d476a2cee5..c04caa034c 100644 --- a/src/evidently/descriptors/_registry.py +++ b/src/evidently/descriptors/_registry.py @@ -117,6 +117,11 @@ "evidently.descriptors.custom_descriptor.CustomPairColumnEval", "evidently:descriptor:CustomPairColumnEval", ) +register_type_alias( + FeatureDescriptor, + "evidently.descriptors.json_match_descriptor.JSONMatch", + "evidently:descriptor:JSONMatch", +) register_type_alias( FeatureDescriptor, "evidently.descriptors.contains_link_descriptor.ContainsLink", diff --git a/src/evidently/descriptors/json_match_descriptor.py b/src/evidently/descriptors/json_match_descriptor.py new file mode 100644 index 0000000000..3d961d9284 --- /dev/null +++ b/src/evidently/descriptors/json_match_descriptor.py @@ -0,0 +1,13 @@ +from evidently.features import json_match_feature +from evidently.features.generated_features import FeatureDescriptor +from evidently.features.generated_features import GeneratedFeature + + +class JSONMatch(FeatureDescriptor): + class Config: + type_alias = "evidently:descriptor:JSONMatch" + + with_column: str + + def feature(self, column_name: str) -> GeneratedFeature: + return json_match_feature.JSONMatch(first_column=column_name, second_column=self.with_column) diff --git a/src/evidently/features/_registry.py b/src/evidently/features/_registry.py index 8566150913..bc1f60f689 100644 --- a/src/evidently/features/_registry.py +++ b/src/evidently/features/_registry.py @@ -90,6 +90,7 @@ register_type_alias( GeneratedFeatures, "evidently.features.words_feature.WordsPresence", "evidently:feature:WordsPresence" ) +register_type_alias(GeneratedFeatures, "evidently.features.json_match_feature.JSONMatch", "evidently:feature:JSONMatch") register_type_alias( GeneratedFeatures, "evidently.features.contains_link_feature.ContainsLink", "evidently:feature:ContainsLink" ) diff --git a/src/evidently/features/json_match_feature.py b/src/evidently/features/json_match_feature.py new file mode 100644 index 0000000000..77d16dd014 --- /dev/null +++ b/src/evidently/features/json_match_feature.py @@ -0,0 +1,46 @@ +import json + +import pandas as pd + +from evidently import ColumnType +from evidently.base_metric import ColumnName +from evidently.features.generated_features import FeatureTypeFieldMixin +from evidently.features.generated_features import GeneratedFeature +from evidently.utils.data_preprocessing import DataDefinition + + +class JSONMatch(FeatureTypeFieldMixin, GeneratedFeature): + class Config: + type_alias = "evidently:feature:JSONMatch" + + first_column: str + second_column: str + feature_type: ColumnType = ColumnType.Categorical + + def generate_feature(self, data: pd.DataFrame, data_definition: DataDefinition) -> pd.DataFrame: + def compare_json_objects(first_json_object: str, second_json_object: str) -> bool: + try: + # Load both JSON strings into dictionaries + first_json = json.loads(first_json_object) + second_json = json.loads(second_json_object) + + # Compare dictionaries for equality, ignoring order of keys + return first_json == second_json + + except (ValueError, TypeError): + # Return False if either of the JSONs is invalid + return False + + data[self._feature_column_name()] = data.apply( + lambda x: compare_json_objects(x[self.first_column], x[self.second_column]), axis=1 + ) + return pd.DataFrame(data[self._feature_column_name()]) + + def _as_column(self) -> "ColumnName": + return self._create_column( + self._feature_column_name(), + default_display_name=f"JSON match for columns {self.first_column} and {self.second_column}", + ) + + def _feature_column_name(self): + return f"JSON match for {self.first_column} and {self.second_column}" diff --git a/tests/features/test_json_match.py b/tests/features/test_json_match.py new file mode 100644 index 0000000000..397713d87d --- /dev/null +++ b/tests/features/test_json_match.py @@ -0,0 +1,57 @@ +import pandas as pd + +from evidently.features.json_match_feature import JSONMatch +from evidently.pipeline.column_mapping import ColumnMapping +from evidently.utils.data_preprocessing import create_data_definition + + +def test_is_valid_sql_feature(): + feature_generator = JSONMatch( + first_column="col_1", second_column="col_2", display_name="Json Match", feature_type="num", name="is_json_match" + ) + + # Define JSON strings for each scenario + scenarios = [ + # Scenario 1 - Matching JSONs + ('{"name": "Alice", "age": 25, "city": "London"}', '{"city": "London", "age": 25, "name": "Alice"}'), + # Scenario 2 - Different whitespace (still matching) + ('{ "name" : "Bob" , "age" : 22 , "city" : "Paris" }', '{"city": "Paris", "name": "Bob", "age": 22}'), + # Scenario 3 - Invalid JSON in one column + ( + '{"name": "Eve", "age": 28, "city": "Berlin"}', + '{"city": "Berlin", "age": 28, "name": Eve}', + ), # Missing quotes around "Eve" + # Scenario 4 - Keys mismatch + ( + '{"name": "Charlie", "age": 30, "country": "USA"}', + '{"name": "Charlie", "age": 30, "city": "USA"}', + ), # 'country' vs 'city' + # Scenario 5 - Values mismatch + ( + '{"name": "David", "age": 35, "city": "Tokyo"}', + '{"city": "Tokyo", "age": 35, "name": "Daniel"}', + ), # 'David' vs 'Daniel' + ] + + # Create DataFrame + data = pd.DataFrame(scenarios, columns=["col_1", "col_2"]) + + result = feature_generator.generate_feature( + data=data, + data_definition=create_data_definition(None, data, ColumnMapping()), + ) + + expected_result = pd.DataFrame(dict(is_json_match=[True, True, False, False, False])) + + print(result) + + print(expected_result) + + try: + assert result.equals(expected_result) + return True + except AssertionError: + return False + + +print(test_is_valid_sql_feature()) From 421630f238369643494115bb086eb46aeb074777 Mon Sep 17 00:00:00 2001 From: Trey Capps <60116939+trey-capps@users.noreply.github.com> Date: Sun, 27 Oct 2024 05:50:16 -0400 Subject: [PATCH 103/103] Add IsValidPython descriptor (#1351) * Add feature and descriptor for IsValidPython * Add tests for IsValidPython descriptor feature * Update documentation to include IsValidPython descriptor --------- Co-authored-by: Emeli Dral --- docs/book/reference/all-metrics.md | 1 + src/evidently/descriptors/__init__.py | 2 + src/evidently/descriptors/_registry.py | 5 ++ .../descriptors/is_valid_python_descriptor.py | 11 ++++ src/evidently/features/_registry.py | 5 ++ .../features/is_valid_python_feature.py | 28 ++++++++++ .../features/test_is_valid_python_feature.py | 54 +++++++++++++++++++ 7 files changed, 106 insertions(+) create mode 100644 src/evidently/descriptors/is_valid_python_descriptor.py create mode 100644 src/evidently/features/is_valid_python_feature.py create mode 100644 tests/features/test_is_valid_python_feature.py diff --git a/docs/book/reference/all-metrics.md b/docs/book/reference/all-metrics.md index bcb2683ffc..29d5762a5b 100644 --- a/docs/book/reference/all-metrics.md +++ b/docs/book/reference/all-metrics.md @@ -281,6 +281,7 @@ Check for regular expression matches. | **JSONSchemaMatch()**
    • Checks if the text contains a JSON object matching the **expected_schema**. Supports exact (**exact=True**) or minimal (**exact=False**) matching, with optional strict type validation (**validate_types=True**).
    • Returns True/False for each row.
    Example use:
    `JSONSchemaMatch(expected_schema={"name": str, "age": int}, exact_match=False, validate_types=True)`| **Required:**
    `expected_schema: Dict[str, type]`

    **Optional:**
    • `exact_match = True` or `False`
    • `validate_types = True` or `False`
    | | **JSONMatch()**
    • Compares two columns of a dataframe and checks whether the two objects in each row of the dataframe are matching JSON's or not.
    • Returns True/False for every input.
    Example use:
    `JSONMatch(with_column="column_2")`| **Required:**
    `with_column : str`

    **Optional:**
    • `display_name`
      • | | **ContainsLink()**
      • Checks if the text contains at least one valid URL.
      • Returns True/False for each row.
      Example use:
      `ContainsLink(column_name='column_1')`| **Required:**
      `column_name: str`

      **Optional:**
      • `display_name`
      | +| **IsValidPython()**
      • Checks if the text is valid Python code without syntax errors.
      • Returns True/False for every input.
      Example use:
      `IsValidPython(column_name='column_1')`| **Required:**
      n/a

      **Optional:**
      • `display_name`
      | ## Descriptors: Text stats diff --git a/src/evidently/descriptors/__init__.py b/src/evidently/descriptors/__init__.py index f8ea236035..174c3a73f1 100644 --- a/src/evidently/descriptors/__init__.py +++ b/src/evidently/descriptors/__init__.py @@ -6,6 +6,7 @@ from .hf_descriptor import HuggingFaceModel from .hf_descriptor import HuggingFaceToxicityModel from .is_valid_json_descriptor import IsValidJSON +from .is_valid_python_descriptor import IsValidPython from .json_match_descriptor import JSONMatch from .json_schema_match_descriptor import JSONSchemaMatch from .llm_judges import BiasLLMEval @@ -72,6 +73,7 @@ "WordNoMatch", "IsValidJSON", "JSONSchemaMatch", + "IsValidPython", "_registry", "JSONMatch", ] diff --git a/src/evidently/descriptors/_registry.py b/src/evidently/descriptors/_registry.py index c04caa034c..cf385db060 100644 --- a/src/evidently/descriptors/_registry.py +++ b/src/evidently/descriptors/_registry.py @@ -15,6 +15,11 @@ "evidently.descriptors.hf_descriptor.HuggingFaceToxicityModel", "evidently:descriptor:HuggingFaceToxicityModel", ) +register_type_alias( + FeatureDescriptor, + "evidently.descriptors.is_valid_python_descriptor.IsValidPython", + "evidently:descriptor:IsValidPython", +) register_type_alias( FeatureDescriptor, "evidently.descriptors.json_schema_match_descriptor.JSONSchemaMatch", diff --git a/src/evidently/descriptors/is_valid_python_descriptor.py b/src/evidently/descriptors/is_valid_python_descriptor.py new file mode 100644 index 0000000000..48560174d8 --- /dev/null +++ b/src/evidently/descriptors/is_valid_python_descriptor.py @@ -0,0 +1,11 @@ +from evidently.features import is_valid_python_feature +from evidently.features.generated_features import FeatureDescriptor +from evidently.features.generated_features import GeneratedFeature + + +class IsValidPython(FeatureDescriptor): + class Config: + type_alias = "evidently:descriptor:IsValidPython" + + def feature(self, column_name: str) -> GeneratedFeature: + return is_valid_python_feature.IsValidPython(column_name, self.display_name) diff --git a/src/evidently/features/_registry.py b/src/evidently/features/_registry.py index bc1f60f689..e5c9199968 100644 --- a/src/evidently/features/_registry.py +++ b/src/evidently/features/_registry.py @@ -27,6 +27,11 @@ "evidently.features.hf_feature.HuggingFaceToxicityFeature", "evidently:feature:HuggingFaceToxicityFeature", ) +register_type_alias( + GeneratedFeatures, + "evidently.features.is_valid_python_feature.IsValidPython", + "evidently:feature:IsValidPython", +) register_type_alias( GeneratedFeatures, "evidently.features.json_schema_match_feature.JSONSchemaMatch", diff --git a/src/evidently/features/is_valid_python_feature.py b/src/evidently/features/is_valid_python_feature.py new file mode 100644 index 0000000000..1c7a18c9de --- /dev/null +++ b/src/evidently/features/is_valid_python_feature.py @@ -0,0 +1,28 @@ +import ast +from typing import Any +from typing import ClassVar +from typing import Optional + +from evidently import ColumnType +from evidently.features.generated_features import ApplyColumnGeneratedFeature + + +class IsValidPython(ApplyColumnGeneratedFeature): + class Config: + type_alias = "evidently:feature:IsValidPython" + + __feature_type__: ClassVar = ColumnType.Categorical + display_name_template: ClassVar = "Valid Python for {column_name}" + column_name: str + + def __init__(self, column_name: str, display_name: Optional[str] = None): + self.column_name = column_name + self.display_name = display_name + super().__init__() + + def apply(self, value: Any) -> bool: + try: + ast.parse(value) + return True + except (SyntaxError, TypeError): + return False diff --git a/tests/features/test_is_valid_python_feature.py b/tests/features/test_is_valid_python_feature.py new file mode 100644 index 0000000000..d95005e6b3 --- /dev/null +++ b/tests/features/test_is_valid_python_feature.py @@ -0,0 +1,54 @@ +from typing import Any + +import numpy as np +import pandas as pd +import pytest + +from evidently.features.is_valid_python_feature import IsValidPython + + +@pytest.mark.parametrize( + ("column_value", "expected"), + [ + ("print('Hello')", True), + ("x = 5 + 3", True), + ("def foo():\n return 'bar'", True), + ("for i in range(10): print(i)", True), + ("print('Hello'", False), + ("for i in range(5) print(i)", False), + ("def foo(\n return 'bar'", False), + ("if True print('yes')", False), + (None, False), + ("12", True), + ("Sorry I can't answer this", False), + ("{'name': 'test', 'age': 13}", True), + ], +) +def test_is_valid_python_apply(column_value: Any, expected: bool): + is_python = IsValidPython("TestColumnName") + actual = is_python.apply(column_value) + assert actual == expected + + +test_data = pd.DataFrame( + { + "TestColumnName": [ + "print('Hello')", + "def foo():\n return 'bar'", + "def foo(\n return 'bar'", + None, + "{'name': 'test', 'age': 13}", + np.nan, + ] + } +) + + +@pytest.mark.parametrize( + ("expected"), + [[True, True, False, False, True, False]], +) +def test_is_valid_python(expected: bool): + is_python = IsValidPython("TestColumnName") + actual = is_python.generate_feature(test_data, None) + assert actual[is_python._feature_column_name()].tolist() == expected