diff --git a/CHANGELOG.md b/CHANGELOG.md index c239c2082..b9e332b12 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -46,6 +46,7 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) - updating listing file with three v2 sparse model - by @dhrubo-os ([#412](https://github.com/opensearch-project/opensearch-py-ml/pull/412)) - Update model upload history - opensearch-project/opensearch-neural-sparse-encoding-doc-v2-mini (v.1.0.0)(TORCH_SCRIPT) by @dhrubo-os ([#417](https://github.com/opensearch-project/opensearch-py-ml/pull/417)) - Update model upload history - opensearch-project/opensearch-neural-sparse-encoding-v2-distill (v.1.0.0)(TORCH_SCRIPT) by @dhrubo-os ([#419](https://github.com/opensearch-project/opensearch-py-ml/pull/419)) +- Bump pandas from 1.5.3 to 2.0.3 by @yerzhaisang ([#422](https://github.com/opensearch-project/opensearch-py-ml/pull/422)) ### Fixed - Fix the wrong final zip file name in model_uploader workflow, now will name it by the upload_prefix alse.([#413](https://github.com/opensearch-project/opensearch-py-ml/pull/413/files)) diff --git a/docs/requirements-docs.txt b/docs/requirements-docs.txt index d4e8a521f..5ae7950de 100644 --- a/docs/requirements-docs.txt +++ b/docs/requirements-docs.txt @@ -1,5 +1,5 @@ opensearch-py>=2 -pandas>=1.5,<3 +pandas==2.0.3 matplotlib>=3.6.0,<4 nbval sphinx diff --git a/opensearch_py_ml/common.py b/opensearch_py_ml/common.py index add998305..ece2aad24 100644 --- a/opensearch_py_ml/common.py +++ b/opensearch_py_ml/common.py @@ -55,14 +55,33 @@ def build_pd_series( - data: Dict[str, Any], dtype: Optional["DTypeLike"] = None, **kwargs: Any + data: Dict[str, Any], + dtype: Optional["DTypeLike"] = None, + index_name: Optional[str] = None, + **kwargs: Any, ) -> pd.Series: - """Builds a pd.Series while squelching the warning - for unspecified dtype on empty series """ + Builds a pandas Series from a dictionary, optionally setting an index name. + + Parameters: + data : Dict[str, Any] + The data to build the Series from, with keys as the index. + dtype : Optional[DTypeLike] + The desired data type of the Series. If not specified, uses EMPTY_SERIES_DTYPE if data is empty. + index_name : Optional[str] + Name to assign to the Series index, similar to `index_name` in `value_counts`. + + Returns: + pd.Series + A pandas Series constructed from the given data, with the specified dtype and index name. + """ + dtype = dtype or (EMPTY_SERIES_DTYPE if not data else dtype) if dtype is not None: kwargs["dtype"] = dtype + if index_name is not None: + index = pd.Index(data.keys(), name=index_name) + kwargs["index"] = index return pd.Series(data, **kwargs) diff --git a/opensearch_py_ml/dataframe.py b/opensearch_py_ml/dataframe.py index 64772887f..47f89872e 100644 --- a/opensearch_py_ml/dataframe.py +++ b/opensearch_py_ml/dataframe.py @@ -47,7 +47,7 @@ from opensearch_py_ml.groupby import DataFrameGroupBy from opensearch_py_ml.ndframe import NDFrame from opensearch_py_ml.series import Series -from opensearch_py_ml.utils import is_valid_attr_name +from opensearch_py_ml.utils import is_valid_attr_name, to_list_if_needed if TYPE_CHECKING: from opensearchpy import OpenSearch @@ -424,9 +424,14 @@ def drop( axis = pd.DataFrame._get_axis_name(axis) axes = {axis: labels} elif index is not None or columns is not None: - axes, _ = pd.DataFrame()._construct_axes_from_arguments( - (index, columns), {} - ) + axes = { + "index": to_list_if_needed(index), + "columns": ( + pd.Index(to_list_if_needed(columns)) + if columns is not None + else None + ), + } else: raise ValueError( "Need to specify at least one of 'labels', 'index' or 'columns'" @@ -440,7 +445,7 @@ def drop( axes["index"] = [axes["index"]] if errors == "raise": # Check if axes['index'] values exists in index - count = self._query_compiler._index_matches_count(axes["index"]) + count = self._query_compiler._index_matches_count(list(axes["index"])) if count != len(axes["index"]): raise ValueError( f"number of labels {count}!={len(axes['index'])} not contained in axis" @@ -1326,7 +1331,7 @@ def to_csv( compression="infer", quoting=None, quotechar='"', - line_terminator=None, + lineterminator=None, chunksize=None, tupleize_cols=None, date_format=None, @@ -1355,7 +1360,7 @@ def to_csv( "compression": compression, "quoting": quoting, "quotechar": quotechar, - "line_terminator": line_terminator, + "lineterminator": lineterminator, "chunksize": chunksize, "date_format": date_format, "doublequote": doublequote, diff --git a/opensearch_py_ml/groupby.py b/opensearch_py_ml/groupby.py index e5c4561c3..ea2083485 100644 --- a/opensearch_py_ml/groupby.py +++ b/opensearch_py_ml/groupby.py @@ -26,6 +26,7 @@ from typing import TYPE_CHECKING, List, Optional, Union from opensearch_py_ml.query_compiler import QueryCompiler +from opensearch_py_ml.utils import MEAN_ABSOLUTE_DEVIATION, STANDARD_DEVIATION, VARIANCE if TYPE_CHECKING: import pandas as pd # type: ignore @@ -153,7 +154,7 @@ def var(self, numeric_only: bool = True) -> "pd.DataFrame": """ return self._query_compiler.aggs_groupby( by=self._by, - pd_aggs=["var"], + pd_aggs=[VARIANCE], dropna=self._dropna, numeric_only=numeric_only, ) @@ -206,7 +207,7 @@ def std(self, numeric_only: bool = True) -> "pd.DataFrame": """ return self._query_compiler.aggs_groupby( by=self._by, - pd_aggs=["std"], + pd_aggs=[STANDARD_DEVIATION], dropna=self._dropna, numeric_only=numeric_only, ) @@ -259,7 +260,7 @@ def mad(self, numeric_only: bool = True) -> "pd.DataFrame": """ return self._query_compiler.aggs_groupby( by=self._by, - pd_aggs=["mad"], + pd_aggs=[MEAN_ABSOLUTE_DEVIATION], dropna=self._dropna, numeric_only=numeric_only, ) diff --git a/opensearch_py_ml/operations.py b/opensearch_py_ml/operations.py index c3d01e9e6..0b130c36b 100644 --- a/opensearch_py_ml/operations.py +++ b/opensearch_py_ml/operations.py @@ -65,6 +65,7 @@ SizeTask, TailTask, ) +from opensearch_py_ml.utils import MEAN_ABSOLUTE_DEVIATION, STANDARD_DEVIATION, VARIANCE if TYPE_CHECKING: from numpy.typing import DTypeLike @@ -475,7 +476,7 @@ def _terms_aggs( except IndexError: name = None - return build_pd_series(results, name=name) + return build_pd_series(results, index_name=name, name="count") def _hist_aggs( self, query_compiler: "QueryCompiler", num_bins: int @@ -620,7 +621,7 @@ def _unpack_metric_aggs( values.append(field.nan_value) # Explicit condition for mad to add NaN because it doesn't support bool elif is_dataframe_agg and numeric_only: - if pd_agg == "mad": + if pd_agg == MEAN_ABSOLUTE_DEVIATION: values.append(field.nan_value) continue @@ -1097,7 +1098,14 @@ def _map_pd_aggs_to_os_aggs( """ # pd aggs that will be mapped to os aggs # that can use 'extended_stats'. - extended_stats_pd_aggs = {"mean", "min", "max", "sum", "var", "std"} + extended_stats_pd_aggs = { + "mean", + "min", + "max", + "sum", + VARIANCE, + STANDARD_DEVIATION, + } extended_stats_os_aggs = {"avg", "min", "max", "sum"} extended_stats_calls = 0 @@ -1117,15 +1125,15 @@ def _map_pd_aggs_to_os_aggs( os_aggs.append("avg") elif pd_agg == "sum": os_aggs.append("sum") - elif pd_agg == "std": + elif pd_agg == STANDARD_DEVIATION: os_aggs.append(("extended_stats", "std_deviation")) - elif pd_agg == "var": + elif pd_agg == VARIANCE: os_aggs.append(("extended_stats", "variance")) # Aggs that aren't 'extended_stats' compatible elif pd_agg == "nunique": os_aggs.append("cardinality") - elif pd_agg == "mad": + elif pd_agg == MEAN_ABSOLUTE_DEVIATION: os_aggs.append("median_absolute_deviation") elif pd_agg == "median": os_aggs.append(("percentiles", (50.0,))) @@ -1205,7 +1213,7 @@ def describe(self, query_compiler: "QueryCompiler") -> pd.DataFrame: df1 = self.aggs( query_compiler=query_compiler, - pd_aggs=["count", "mean", "std", "min", "max"], + pd_aggs=["count", "mean", "min", "max", STANDARD_DEVIATION], numeric_only=True, ) df2 = self.quantile( @@ -1219,8 +1227,22 @@ def describe(self, query_compiler: "QueryCompiler") -> pd.DataFrame: # Convert [.25,.5,.75] to ["25%", "50%", "75%"] df2 = df2.set_index([["25%", "50%", "75%"]]) - return pd.concat([df1, df2]).reindex( - ["count", "mean", "std", "min", "25%", "50%", "75%", "max"] + df = pd.concat([df1, df2]) + + # Note: In recent pandas versions, `describe()` returns a different index order + # for one-column DataFrames compared to multi-column DataFrames. + # We adjust the order manually to ensure consistency. + if df.shape[1] == 1: + # For single-column DataFrames, `describe()` typically outputs: + # ["count", "mean", "std", "min", "25%", "50%", "75%", "max"] + return df.reindex( + ["count", "mean", STANDARD_DEVIATION, "min", "25%", "50%", "75%", "max"] + ) + + # For multi-column DataFrames, `describe()` typically outputs: + # ["count", "mean", "min", "25%", "50%", "75%", "max", "std"] + return df.reindex( + ["count", "mean", "min", "25%", "50%", "75%", "max", STANDARD_DEVIATION] ) def to_pandas( diff --git a/opensearch_py_ml/query_compiler.py b/opensearch_py_ml/query_compiler.py index c10899671..735b0307c 100644 --- a/opensearch_py_ml/query_compiler.py +++ b/opensearch_py_ml/query_compiler.py @@ -45,6 +45,7 @@ from opensearch_py_ml.filter import BooleanFilter, QueryFilter from opensearch_py_ml.index import Index from opensearch_py_ml.operations import Operations +from opensearch_py_ml.utils import MEAN_ABSOLUTE_DEVIATION, STANDARD_DEVIATION, VARIANCE if TYPE_CHECKING: from opensearchpy import OpenSearch @@ -587,17 +588,17 @@ def mean(self, numeric_only: Optional[bool] = None) -> pd.Series: def var(self, numeric_only: Optional[bool] = None) -> pd.Series: return self._operations._metric_agg_series( - self, ["var"], numeric_only=numeric_only + self, [VARIANCE], numeric_only=numeric_only ) def std(self, numeric_only: Optional[bool] = None) -> pd.Series: return self._operations._metric_agg_series( - self, ["std"], numeric_only=numeric_only + self, [STANDARD_DEVIATION], numeric_only=numeric_only ) def mad(self, numeric_only: Optional[bool] = None) -> pd.Series: return self._operations._metric_agg_series( - self, ["mad"], numeric_only=numeric_only + self, [MEAN_ABSOLUTE_DEVIATION], numeric_only=numeric_only ) def median(self, numeric_only: Optional[bool] = None) -> pd.Series: diff --git a/opensearch_py_ml/series.py b/opensearch_py_ml/series.py index 772660b13..538d8a879 100644 --- a/opensearch_py_ml/series.py +++ b/opensearch_py_ml/series.py @@ -312,11 +312,12 @@ def value_counts(self, os_size: int = 10) -> pd.Series: >>> df = oml.DataFrame(OPENSEARCH_TEST_CLIENT, 'flights') >>> df['Carrier'].value_counts() + Carrier Logstash Airways 3331 JetBeats 3274 Kibana Airlines 3234 ES-Air 3220 - Name: Carrier, dtype: int64 + Name: count, dtype: int64 """ if not isinstance(os_size, int): raise TypeError("os_size must be a positive integer.") diff --git a/opensearch_py_ml/utils.py b/opensearch_py_ml/utils.py index 8f1763085..e850d2724 100644 --- a/opensearch_py_ml/utils.py +++ b/opensearch_py_ml/utils.py @@ -30,9 +30,14 @@ from typing import Any, Callable, Collection, Iterable, List, TypeVar, Union, cast import pandas as pd # type: ignore +from pandas.core.dtypes.common import is_list_like # type: ignore RT = TypeVar("RT") +MEAN_ABSOLUTE_DEVIATION = "mad" +VARIANCE = "var" +STANDARD_DEVIATION = "std" + def deprecated_api( replace_with: str, @@ -61,6 +66,29 @@ def is_valid_attr_name(s: str) -> bool: ) +def to_list_if_needed(value): + """ + Converts the input to a list if necessary. + + If the input is a pandas Index, it converts it to a list. + If the input is not list-like (e.g., a single value), it wraps it in a list. + If the input is None or already list-like, it returns it as is. + + Parameters: + value: The input to potentially convert to a list. + + Returns: + The input converted to a list if needed, or the original input if no conversion is necessary. + """ + if value is None: + return None + if isinstance(value, pd.Index): + return value.tolist() + if not is_list_like(value): + return [value] + return value + + def to_list(x: Union[Collection[Any], pd.Series]) -> List[Any]: if isinstance(x, ABCCollection): return list(x) @@ -77,3 +105,23 @@ def try_sort(iterable: Iterable[str]) -> Iterable[str]: return sorted(listed) except TypeError: return listed + + +class CustomFunctionDispatcher: + # Define custom functions in a dictionary + customFunctionMap = { + MEAN_ABSOLUTE_DEVIATION: lambda x: (x - x.median()).abs().mean(), + } + + @classmethod + def apply_custom_function(cls, func, data): + """ + Apply a custom function if available, else return None. + :param func: Function name as a string + :param data: Data on which function is applied + :return: Result of custom function or None if func not found + """ + custom_func = cls.customFunctionMap.get(func) + if custom_func: + return custom_func(data) + return None diff --git a/requirements-dev.txt b/requirements-dev.txt index e7b62bcf0..6bbc817b0 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,7 +1,7 @@ # # Basic requirements # -pandas>=1.5.2,<2 +pandas==2.0.3 matplotlib>=3.6.2,<4 numpy>=1.24.0,<2 opensearch-py>=2.2.0 diff --git a/requirements.txt b/requirements.txt index cddfe801c..8af3ac141 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,7 @@ # # Basic requirements # -pandas>=1.5.2,<2 +pandas==2.0.3 matplotlib>=3.6.2,<4 numpy>=1.24.0,<2 opensearch-py>=2.2.0 diff --git a/setup.py b/setup.py index 9146b2503..f98028f3d 100644 --- a/setup.py +++ b/setup.py @@ -84,7 +84,7 @@ }, install_requires=[ "opensearch-py>=2", - "pandas>=1.5,<3", + "pandas==2.0.3", "matplotlib>=3.6.0,<4", "numpy>=1.24.0,<2", "deprecated>=1.2.14,<2", diff --git a/tests/dataframe/test_aggs_pytest.py b/tests/dataframe/test_aggs_pytest.py index 62f6843a6..748a3dfd3 100644 --- a/tests/dataframe/test_aggs_pytest.py +++ b/tests/dataframe/test_aggs_pytest.py @@ -28,6 +28,7 @@ import pytest from pandas.testing import assert_frame_equal, assert_series_equal +from opensearch_py_ml.utils import STANDARD_DEVIATION, VARIANCE from tests.common import TestData @@ -47,10 +48,10 @@ def test_basic_aggs(self): assert_frame_equal(pd_sum_min, oml_sum_min, check_exact=False) pd_sum_min_std = pd_flights.select_dtypes(include=[np.number]).agg( - ["sum", "min", "std"] + ["sum", "min", STANDARD_DEVIATION] ) oml_sum_min_std = oml_flights.select_dtypes(include=[np.number]).agg( - ["sum", "min", "std"], numeric_only=True + ["sum", "min", STANDARD_DEVIATION], numeric_only=True ) print(pd_sum_min_std.dtypes) @@ -75,10 +76,10 @@ def test_terms_aggs(self): assert_frame_equal(pd_sum_min, oml_sum_min, check_exact=False) pd_sum_min_std = pd_flights.select_dtypes(include=[np.number]).agg( - ["sum", "min", "std"] + ["sum", "min", STANDARD_DEVIATION] ) oml_sum_min_std = oml_flights.select_dtypes(include=[np.number]).agg( - ["sum", "min", "std"], numeric_only=True + ["sum", "min", STANDARD_DEVIATION], numeric_only=True ) print(pd_sum_min_std.dtypes) @@ -94,10 +95,10 @@ def test_aggs_median_var(self): pd_aggs = pd_ecommerce[ ["taxful_total_price", "taxless_total_price", "total_quantity"] - ].agg(["median", "var"]) + ].agg(["median", VARIANCE]) oml_aggs = oml_ecommerce[ ["taxful_total_price", "taxless_total_price", "total_quantity"] - ].agg(["median", "var"], numeric_only=True) + ].agg(["median", VARIANCE], numeric_only=True) print(pd_aggs, pd_aggs.dtypes) print(oml_aggs, oml_aggs.dtypes) diff --git a/tests/dataframe/test_describe_pytest.py b/tests/dataframe/test_describe_pytest.py index 8d0344d26..e8c8ea2a6 100644 --- a/tests/dataframe/test_describe_pytest.py +++ b/tests/dataframe/test_describe_pytest.py @@ -34,7 +34,7 @@ def test_flights_describe(self): pd_flights = self.pd_flights() oml_flights = self.oml_flights() - pd_describe = pd_flights.describe() + pd_describe = pd_flights.describe().drop(["timestamp"], axis=1) # We remove bool columns to match pandas output oml_describe = oml_flights.describe().drop( ["Cancelled", "FlightDelay"], axis="columns" diff --git a/tests/dataframe/test_groupby_pytest.py b/tests/dataframe/test_groupby_pytest.py index 3ee3fa7bc..89938b110 100644 --- a/tests/dataframe/test_groupby_pytest.py +++ b/tests/dataframe/test_groupby_pytest.py @@ -28,6 +28,12 @@ import pytest from pandas.testing import assert_frame_equal, assert_index_equal, assert_series_equal +from opensearch_py_ml.utils import ( + MEAN_ABSOLUTE_DEVIATION, + STANDARD_DEVIATION, + VARIANCE, + CustomFunctionDispatcher, +) from tests.common import TestData @@ -100,13 +106,22 @@ def test_groupby_aggs_numeric_only_true(self, pd_agg, dropna): ) @pytest.mark.parametrize("dropna", [True, False]) - @pytest.mark.parametrize("pd_agg", ["mad", "var", "std"]) + @pytest.mark.parametrize( + "pd_agg", [MEAN_ABSOLUTE_DEVIATION, VARIANCE, STANDARD_DEVIATION] + ) def test_groupby_aggs_mad_var_std(self, pd_agg, dropna): # For these aggs pandas doesn't support numeric_only pd_flights = self.pd_flights().filter(self.filter_data) oml_flights = self.oml_flights().filter(self.filter_data) - pd_groupby = getattr(pd_flights.groupby("Cancelled", dropna=dropna), pd_agg)() + if pd_agg in CustomFunctionDispatcher.customFunctionMap: + pd_groupby = pd_flights.groupby("Cancelled", dropna=dropna).agg( + lambda x: CustomFunctionDispatcher.apply_custom_function(pd_agg, x) + ) + else: + pd_groupby = getattr( + pd_flights.groupby("Cancelled", dropna=dropna), pd_agg + )() oml_groupby = getattr(oml_flights.groupby("Cancelled", dropna=dropna), pd_agg)( numeric_only=True ) @@ -224,15 +239,44 @@ def test_groupby_dataframe_mad(self): pd_flights = self.pd_flights().filter(self.filter_data + ["DestCountry"]) oml_flights = self.oml_flights().filter(self.filter_data + ["DestCountry"]) - pd_mad = pd_flights.groupby("DestCountry").mad() + pd_mad = pd_flights.groupby("DestCountry").apply( + lambda group: group.select_dtypes(include="number").apply( + lambda x: CustomFunctionDispatcher.apply_custom_function( + MEAN_ABSOLUTE_DEVIATION, x + ) + ) + ) + + # Re-merge non-numeric columns back, with suffixes to avoid column overlap + non_numeric_columns = ( + pd_flights.select_dtypes(exclude="number").groupby("DestCountry").first() + ) + pd_mad = pd_mad.join( + non_numeric_columns, lsuffix="_numeric", rsuffix="_non_numeric" + )[self.filter_data] + if "Cancelled" in pd_mad.columns: + pd_mad["Cancelled"] = pd_mad["Cancelled"].astype(float) oml_mad = oml_flights.groupby("DestCountry").mad() assert_index_equal(pd_mad.columns, oml_mad.columns) assert_index_equal(pd_mad.index, oml_mad.index) assert_series_equal(pd_mad.dtypes, oml_mad.dtypes) - pd_min_mad = pd_flights.groupby("DestCountry").aggregate(["min", "mad"]) - oml_min_mad = oml_flights.groupby("DestCountry").aggregate(["min", "mad"]) + pd_min_mad = pd_flights.groupby("DestCountry").agg( + [ + "min", + lambda x: CustomFunctionDispatcher.apply_custom_function( + MEAN_ABSOLUTE_DEVIATION, x + ), + ] + ) + + pd_min_mad.columns = pd_min_mad.columns.set_levels( + ["min", MEAN_ABSOLUTE_DEVIATION], level=1 + ) + oml_min_mad = oml_flights.groupby("DestCountry").aggregate( + ["min", MEAN_ABSOLUTE_DEVIATION] + ) assert_index_equal(pd_min_mad.columns, oml_min_mad.columns) assert_index_equal(pd_min_mad.index, oml_min_mad.index) diff --git a/tests/dataframe/test_metrics_pytest.py b/tests/dataframe/test_metrics_pytest.py index f3055ea5a..e334643b3 100644 --- a/tests/dataframe/test_metrics_pytest.py +++ b/tests/dataframe/test_metrics_pytest.py @@ -24,17 +24,22 @@ import numpy as np import pandas as pd - -# File called _pytest for PyCharm compatibility import pytest from pandas.testing import assert_frame_equal, assert_series_equal +# File called _pytest for PyCharm compatibility +from opensearch_py_ml.utils import ( + MEAN_ABSOLUTE_DEVIATION, + STANDARD_DEVIATION, + VARIANCE, + CustomFunctionDispatcher, +) from tests.common import TestData, assert_almost_equal class TestDataFrameMetrics(TestData): funcs = ["max", "min", "mean", "sum"] - extended_funcs = ["median", "mad", "var", "std"] + extended_funcs = ["median", MEAN_ABSOLUTE_DEVIATION, VARIANCE, STANDARD_DEVIATION] filter_data = [ "AvgTicketPrice", "Cancelled", @@ -81,9 +86,12 @@ def test_flights_extended_metrics(self): logger.setLevel(logging.DEBUG) for func in self.extended_funcs: - pd_metric = getattr(pd_flights, func)( - **({"numeric_only": True} if func != "mad" else {}) - ) + if func in CustomFunctionDispatcher.customFunctionMap: + pd_metric = CustomFunctionDispatcher.apply_custom_function( + func, pd_flights + ) + else: + pd_metric = getattr(pd_flights, func)(**({"numeric_only": True})) oml_metric = getattr(oml_flights, func)(numeric_only=True) pd_value = pd_metric["AvgTicketPrice"] @@ -101,7 +109,12 @@ def test_flights_extended_metrics_nan(self): ] for func in self.extended_funcs: - pd_metric = getattr(pd_flights_1, func)() + if func in CustomFunctionDispatcher.customFunctionMap: + pd_metric = pd_flights_1.apply( + lambda x: CustomFunctionDispatcher.apply_custom_function(func, x) + ) + else: + pd_metric = getattr(pd_flights_1, func)() oml_metric = getattr(oml_flights_1, func)(numeric_only=False) assert_series_equal(pd_metric, oml_metric, check_exact=False) @@ -111,7 +124,12 @@ def test_flights_extended_metrics_nan(self): oml_flights_0 = oml_flights[oml_flights.FlightNum == "XXX"][["AvgTicketPrice"]] for func in self.extended_funcs: - pd_metric = getattr(pd_flights_0, func)() + if func in CustomFunctionDispatcher.customFunctionMap: + pd_metric = pd_flights_0.apply( + lambda x: CustomFunctionDispatcher.apply_custom_function(func, x) + ) + else: + pd_metric = getattr(pd_flights_0, func)() oml_metric = getattr(oml_flights_0, func)(numeric_only=False) assert_series_equal(pd_metric, oml_metric, check_exact=False) @@ -177,9 +195,9 @@ def test_flights_datetime_metrics_agg(self): "min": pd.Timestamp("2018-01-01 00:00:00"), "mean": pd.Timestamp("2018-01-21 19:20:45.564438232"), "sum": pd.NaT, - "mad": pd.NaT, - "var": pd.NaT, - "std": pd.NaT, + MEAN_ABSOLUTE_DEVIATION: pd.NaT, + VARIANCE: pd.NaT, + STANDARD_DEVIATION: pd.NaT, "nunique": 12236, } @@ -288,7 +306,7 @@ def test_flights_numeric_only(self): agg_data = oml_flights.agg(filtered_aggs, numeric_only=True).transpose() for agg in filtered_aggs: # Explicitly check for mad because it returns nan for bools - if agg == "mad": + if agg == MEAN_ABSOLUTE_DEVIATION: assert np.isnan(agg_data[agg]["Cancelled"]) else: assert_series_equal( @@ -304,7 +322,7 @@ def test_numeric_only_true_single_aggs(self): for agg in self.funcs + self.extended_funcs: result = getattr(oml_flights, agg)(numeric_only=True) assert result.dtype == np.dtype("float64") - assert result.shape == ((3,) if agg != "mad" else (2,)) + assert result.shape == ((3,) if agg != MEAN_ABSOLUTE_DEVIATION else (2,)) # check dtypes and shape of min, max and median for numeric_only=False | None @pytest.mark.parametrize("agg", ["min", "max", "median"]) @@ -498,7 +516,8 @@ def test_flights_agg_quantile(self, numeric_only): ["AvgTicketPrice", "FlightDelayMin", "dayOfWeek"] ) - pd_quantile = pd_flights.agg(["quantile", "min"], numeric_only=numeric_only) + pd_quantile = pd_flights.agg([lambda x: x.quantile(0.5), lambda x: x.min()]) + pd_quantile.index = ["quantile", "min"] oml_quantile = oml_flights.agg(["quantile", "min"], numeric_only=numeric_only) assert_frame_equal( diff --git a/tests/operations/test_map_pd_aggs_to_es_aggs_pytest.py b/tests/operations/test_map_pd_aggs_to_es_aggs_pytest.py index f5e4e8a96..c030eb890 100644 --- a/tests/operations/test_map_pd_aggs_to_es_aggs_pytest.py +++ b/tests/operations/test_map_pd_aggs_to_es_aggs_pytest.py @@ -23,6 +23,7 @@ # under the License. from opensearch_py_ml.operations import Operations +from opensearch_py_ml.utils import MEAN_ABSOLUTE_DEVIATION, STANDARD_DEVIATION, VARIANCE def test_all_aggs(): @@ -31,9 +32,9 @@ def test_all_aggs(): "min", "max", "mean", - "std", - "var", - "mad", + STANDARD_DEVIATION, + VARIANCE, + MEAN_ABSOLUTE_DEVIATION, "count", "nunique", "median", @@ -69,7 +70,7 @@ def test_extended_stats_optimization(): os_aggs = Operations._map_pd_aggs_to_os_aggs(["count", "nunique"]) assert os_aggs == ["value_count", "cardinality"] - for pd_agg in ["var", "std"]: + for pd_agg in [VARIANCE, STANDARD_DEVIATION]: extended_os_agg = Operations._map_pd_aggs_to_os_aggs([pd_agg])[0] os_aggs = Operations._map_pd_aggs_to_os_aggs([pd_agg, "nunique"]) diff --git a/tests/series/test_arithmetics_pytest.py b/tests/series/test_arithmetics_pytest.py index 18226675c..196e5d38d 100644 --- a/tests/series/test_arithmetics_pytest.py +++ b/tests/series/test_arithmetics_pytest.py @@ -80,9 +80,7 @@ def to_pandas(self): # "type cast" to modified class (inherits from ed.Series) that overrides the `to_pandas` function oml_series.__class__ = ModifiedOMLSeries - assert_pandas_opensearch_py_ml_series_equal( - pd_series, oml_series, check_less_precise=True - ) + assert_pandas_opensearch_py_ml_series_equal(pd_series, oml_series) def test_ecommerce_series_invalid_div(self): pd_df = self.pd_ecommerce() diff --git a/tests/series/test_describe_pytest.py b/tests/series/test_describe_pytest.py index b0ad65602..0841253d4 100644 --- a/tests/series/test_describe_pytest.py +++ b/tests/series/test_describe_pytest.py @@ -24,6 +24,7 @@ import pandas as pd +from opensearch_py_ml.utils import STANDARD_DEVIATION from tests.common import TestData, assert_series_equal @@ -42,7 +43,7 @@ def test_series_describe(self): # Percentiles calculations vary for Elasticsearch assert_series_equal( - oml_desc[["count", "mean", "std", "min", "max"]], - pd_desc[["count", "mean", "std", "min", "max"]], + oml_desc[["count", "mean", STANDARD_DEVIATION, "min", "max"]], + pd_desc[["count", "mean", STANDARD_DEVIATION, "min", "max"]], rtol=0.2, ) diff --git a/tests/series/test_metrics_pytest.py b/tests/series/test_metrics_pytest.py index 71037d4da..bc3330eb2 100644 --- a/tests/series/test_metrics_pytest.py +++ b/tests/series/test_metrics_pytest.py @@ -31,15 +31,30 @@ import pytest from pandas.testing import assert_series_equal +from opensearch_py_ml.utils import ( + MEAN_ABSOLUTE_DEVIATION, + STANDARD_DEVIATION, + VARIANCE, + CustomFunctionDispatcher, +) from tests.common import TestData, assert_almost_equal class TestSeriesMetrics(TestData): - all_funcs = ["max", "min", "mean", "sum", "nunique", "var", "std", "mad"] + all_funcs = [ + "max", + "min", + "mean", + "sum", + "nunique", + VARIANCE, + STANDARD_DEVIATION, + MEAN_ABSOLUTE_DEVIATION, + ] timestamp_funcs = ["max", "min", "mean", "nunique"] def assert_almost_equal_for_agg(self, func, pd_metric, oml_metric): - if func in ("nunique", "var", "mad"): + if func in ("nunique", VARIANCE, MEAN_ABSOLUTE_DEVIATION): np.testing.assert_almost_equal(pd_metric, oml_metric, decimal=-3) else: np.testing.assert_almost_equal(pd_metric, oml_metric, decimal=2) @@ -49,7 +64,12 @@ def test_flights_metrics(self): oml_flights = self.oml_flights()["AvgTicketPrice"] for func in self.all_funcs: - pd_metric = getattr(pd_flights, func)() + if func in CustomFunctionDispatcher.customFunctionMap: + pd_metric = pd_flights.agg( + lambda x: CustomFunctionDispatcher.apply_custom_function(func, x) + ) + else: + pd_metric = getattr(pd_flights, func)() oml_metric = getattr(oml_flights, func)() self.assert_almost_equal_for_agg(func, pd_metric, oml_metric) @@ -94,7 +114,14 @@ def test_ecommerce_selected_all_numeric_source_fields(self): oml_ecommerce = self.oml_ecommerce()[column] for func in self.all_funcs: - pd_metric = getattr(pd_ecommerce, func)() + if func in CustomFunctionDispatcher.customFunctionMap: + pd_metric = pd_ecommerce.agg( + lambda x: CustomFunctionDispatcher.apply_custom_function( + func, x + ) + ) + else: + pd_metric = getattr(pd_ecommerce, func)() oml_metric = getattr(oml_ecommerce, func)( **({"numeric_only": True} if (func != "nunique") else {}) )