Bump pandas from 1.5.3 to 2.0.3

CHANGELOG.md

@@ -46,6 +46,7 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
 - updating listing file with three v2 sparse model - by @dhrubo-os ([#412](https://github.com/opensearch-project/opensearch-py-ml/pull/412))
 - Update model upload history -  opensearch-project/opensearch-neural-sparse-encoding-doc-v2-mini (v.1.0.0)(TORCH_SCRIPT) by @dhrubo-os ([#417](https://github.com/opensearch-project/opensearch-py-ml/pull/417))
 - Update model upload history -  opensearch-project/opensearch-neural-sparse-encoding-v2-distill (v.1.0.0)(TORCH_SCRIPT) by @dhrubo-os ([#419](https://github.com/opensearch-project/opensearch-py-ml/pull/419))
+- Bump pandas from 1.5.3 to 2.0.3 by @yerzhaisang ([#422](https://github.com/opensearch-project/opensearch-py-ml/pull/422))
 
 ### Fixed
 - Fix the wrong final zip file name in model_uploader workflow, now will name it by the upload_prefix alse.([#413](https://github.com/opensearch-project/opensearch-py-ml/pull/413/files))

docs/requirements-docs.txt

@@ -1,5 +1,5 @@
 opensearch-py>=2
-pandas>=1.5,<3
+pandas==2.0.3
 matplotlib>=3.6.0,<4
 nbval
 sphinx

opensearch_py_ml/common.py

@@ -55,14 +55,33 @@
 
 
 def build_pd_series(
-    data: Dict[str, Any], dtype: Optional["DTypeLike"] = None, **kwargs: Any
+    data: Dict[str, Any],
+    dtype: Optional["DTypeLike"] = None,
+    index_name: Optional[str] = None,
+    **kwargs: Any,
 ) -> pd.Series:
-    """Builds a pd.Series while squelching the warning
-    for unspecified dtype on empty series
     """
+    Builds a pandas Series from a dictionary, optionally setting an index name.
+
+    Parameters:
+    data : Dict[str, Any]
+        The data to build the Series from, with keys as the index.
+    dtype : Optional[DTypeLike]
+        The desired data type of the Series. If not specified, uses EMPTY_SERIES_DTYPE if data is empty.
+    index_name : Optional[str]
+        Name to assign to the Series index, similar to `index_name` in `value_counts`.
+
+    Returns:
+    pd.Series
+        A pandas Series constructed from the given data, with the specified dtype and index name.
+    """
+
     dtype = dtype or (EMPTY_SERIES_DTYPE if not data else dtype)
     if dtype is not None:
         kwargs["dtype"] = dtype
+    if index_name is not None:
+        index = pd.Index(data.keys(), name=index_name)
+        kwargs["index"] = index
     return pd.Series(data, **kwargs)
 
 

opensearch_py_ml/dataframe.py

@@ -47,7 +47,7 @@
 from opensearch_py_ml.groupby import DataFrameGroupBy
 from opensearch_py_ml.ndframe import NDFrame
 from opensearch_py_ml.series import Series
-from opensearch_py_ml.utils import is_valid_attr_name
+from opensearch_py_ml.utils import is_valid_attr_name, to_list_if_needed
 
 if TYPE_CHECKING:
     from opensearchpy import OpenSearch
@@ -424,9 +424,14 @@ def drop(
             axis = pd.DataFrame._get_axis_name(axis)
             axes = {axis: labels}
         elif index is not None or columns is not None:
-            axes, _ = pd.DataFrame()._construct_axes_from_arguments(
-                (index, columns), {}
-            )
+            axes = {
+                "index": to_list_if_needed(index),
+                "columns": (
+                    pd.Index(to_list_if_needed(columns))
+                    if columns is not None
+                    else None
+                ),
+            }
         else:
             raise ValueError(
                 "Need to specify at least one of 'labels', 'index' or 'columns'"
@@ -440,7 +445,7 @@ def drop(
                 axes["index"] = [axes["index"]]
             if errors == "raise":
                 # Check if axes['index'] values exists in index
-                count = self._query_compiler._index_matches_count(axes["index"])
+                count = self._query_compiler._index_matches_count(list(axes["index"]))
                 if count != len(axes["index"]):
                     raise ValueError(
                         f"number of labels {count}!={len(axes['index'])} not contained in axis"
@@ -1326,7 +1331,7 @@ def to_csv(
         compression="infer",
         quoting=None,
         quotechar='"',
-        line_terminator=None,
+        lineterminator=None,
         chunksize=None,
         tupleize_cols=None,
         date_format=None,
@@ -1355,7 +1360,7 @@ def to_csv(
             "compression": compression,
             "quoting": quoting,
             "quotechar": quotechar,
-            "line_terminator": line_terminator,
+            "lineterminator": lineterminator,
             "chunksize": chunksize,
             "date_format": date_format,
             "doublequote": doublequote,

opensearch_py_ml/groupby.py

@@ -26,6 +26,7 @@
 from typing import TYPE_CHECKING, List, Optional, Union
 
 from opensearch_py_ml.query_compiler import QueryCompiler
+from opensearch_py_ml.utils import MEAN_ABSOLUTE_DEVIATION, STANDARD_DEVIATION, VARIANCE
 
 if TYPE_CHECKING:
     import pandas as pd  # type: ignore
@@ -153,7 +154,7 @@ def var(self, numeric_only: bool = True) -> "pd.DataFrame":
         """
         return self._query_compiler.aggs_groupby(
             by=self._by,
-            pd_aggs=["var"],
+            pd_aggs=[VARIANCE],
             dropna=self._dropna,
             numeric_only=numeric_only,
         )
@@ -206,7 +207,7 @@ def std(self, numeric_only: bool = True) -> "pd.DataFrame":
         """
         return self._query_compiler.aggs_groupby(
             by=self._by,
-            pd_aggs=["std"],
+            pd_aggs=[STANDARD_DEVIATION],
             dropna=self._dropna,
             numeric_only=numeric_only,
         )
@@ -259,7 +260,7 @@ def mad(self, numeric_only: bool = True) -> "pd.DataFrame":
         """
         return self._query_compiler.aggs_groupby(
             by=self._by,
-            pd_aggs=["mad"],
+            pd_aggs=[MEAN_ABSOLUTE_DEVIATION],
             dropna=self._dropna,
             numeric_only=numeric_only,
         )

opensearch_py_ml/operations.py

@@ -65,6 +65,7 @@
     SizeTask,
     TailTask,
 )
+from opensearch_py_ml.utils import MEAN_ABSOLUTE_DEVIATION, STANDARD_DEVIATION, VARIANCE
 
 if TYPE_CHECKING:
     from numpy.typing import DTypeLike
@@ -475,7 +476,7 @@ def _terms_aggs(
         except IndexError:
             name = None
 
-        return build_pd_series(results, name=name)
+        return build_pd_series(results, index_name=name, name="count")
 
     def _hist_aggs(
         self, query_compiler: "QueryCompiler", num_bins: int
@@ -620,7 +621,7 @@ def _unpack_metric_aggs(
                         values.append(field.nan_value)
                     # Explicit condition for mad to add NaN because it doesn't support bool
                     elif is_dataframe_agg and numeric_only:
-                        if pd_agg == "mad":
+                        if pd_agg == MEAN_ABSOLUTE_DEVIATION:
                             values.append(field.nan_value)
                     continue
 
@@ -1097,7 +1098,14 @@ def _map_pd_aggs_to_os_aggs(
         """
         # pd aggs that will be mapped to os aggs
         # that can use 'extended_stats'.
-        extended_stats_pd_aggs = {"mean", "min", "max", "sum", "var", "std"}
+        extended_stats_pd_aggs = {
+            "mean",
+            "min",
+            "max",
+            "sum",
+            VARIANCE,
+            STANDARD_DEVIATION,
+        }
         extended_stats_os_aggs = {"avg", "min", "max", "sum"}
         extended_stats_calls = 0
 
@@ -1117,15 +1125,15 @@ def _map_pd_aggs_to_os_aggs(
                 os_aggs.append("avg")
             elif pd_agg == "sum":
                 os_aggs.append("sum")
-            elif pd_agg == "std":
+            elif pd_agg == STANDARD_DEVIATION:
                 os_aggs.append(("extended_stats", "std_deviation"))
-            elif pd_agg == "var":
+            elif pd_agg == VARIANCE:
                 os_aggs.append(("extended_stats", "variance"))
 
             # Aggs that aren't 'extended_stats' compatible
             elif pd_agg == "nunique":
                 os_aggs.append("cardinality")
-            elif pd_agg == "mad":
+            elif pd_agg == MEAN_ABSOLUTE_DEVIATION:
                 os_aggs.append("median_absolute_deviation")
             elif pd_agg == "median":
                 os_aggs.append(("percentiles", (50.0,)))
@@ -1205,7 +1213,7 @@ def describe(self, query_compiler: "QueryCompiler") -> pd.DataFrame:
 
         df1 = self.aggs(
             query_compiler=query_compiler,
-            pd_aggs=["count", "mean", "std", "min", "max"],
+            pd_aggs=["count", "mean", "min", "max", STANDARD_DEVIATION],
             numeric_only=True,
         )
         df2 = self.quantile(
@@ -1219,8 +1227,22 @@ def describe(self, query_compiler: "QueryCompiler") -> pd.DataFrame:
         # Convert [.25,.5,.75] to ["25%", "50%", "75%"]
         df2 = df2.set_index([["25%", "50%", "75%"]])
 
-        return pd.concat([df1, df2]).reindex(
-            ["count", "mean", "std", "min", "25%", "50%", "75%", "max"]
+        df = pd.concat([df1, df2])
+
+        # Note: In recent pandas versions, `describe()` returns a different index order
+        # for one-column DataFrames compared to multi-column DataFrames.
+        # We adjust the order manually to ensure consistency.
+        if df.shape[1] == 1:
+            # For single-column DataFrames, `describe()` typically outputs:
+            # ["count", "mean", "std", "min", "25%", "50%", "75%", "max"]
+            return df.reindex(
+                ["count", "mean", STANDARD_DEVIATION, "min", "25%", "50%", "75%", "max"]
+            )
+
+        # For multi-column DataFrames, `describe()` typically outputs:
+        # ["count", "mean", "min", "25%", "50%", "75%", "max", "std"]
+        return df.reindex(
+            ["count", "mean", "min", "25%", "50%", "75%", "max", STANDARD_DEVIATION]
         )
 
     def to_pandas(

opensearch_py_ml/query_compiler.py

@@ -45,6 +45,7 @@
 from opensearch_py_ml.filter import BooleanFilter, QueryFilter
 from opensearch_py_ml.index import Index
 from opensearch_py_ml.operations import Operations
+from opensearch_py_ml.utils import MEAN_ABSOLUTE_DEVIATION, STANDARD_DEVIATION, VARIANCE
 
 if TYPE_CHECKING:
     from opensearchpy import OpenSearch
@@ -587,17 +588,17 @@ def mean(self, numeric_only: Optional[bool] = None) -> pd.Series:
 
     def var(self, numeric_only: Optional[bool] = None) -> pd.Series:
         return self._operations._metric_agg_series(
-            self, ["var"], numeric_only=numeric_only
+            self, [VARIANCE], numeric_only=numeric_only
         )
 
     def std(self, numeric_only: Optional[bool] = None) -> pd.Series:
         return self._operations._metric_agg_series(
-            self, ["std"], numeric_only=numeric_only
+            self, [STANDARD_DEVIATION], numeric_only=numeric_only
         )
 
     def mad(self, numeric_only: Optional[bool] = None) -> pd.Series:
         return self._operations._metric_agg_series(
-            self, ["mad"], numeric_only=numeric_only
+            self, [MEAN_ABSOLUTE_DEVIATION], numeric_only=numeric_only
         )
 
     def median(self, numeric_only: Optional[bool] = None) -> pd.Series:

opensearch_py_ml/series.py

@@ -312,11 +312,12 @@ def value_counts(self, os_size: int = 10) -> pd.Series:
 
         >>> df = oml.DataFrame(OPENSEARCH_TEST_CLIENT, 'flights')
         >>> df['Carrier'].value_counts()
+        Carrier
         Logstash Airways    3331
         JetBeats            3274
         Kibana Airlines     3234
         ES-Air              3220
-        Name: Carrier, dtype: int64
+        Name: count, dtype: int64
         """
         if not isinstance(os_size, int):
             raise TypeError("os_size must be a positive integer.")

opensearch_py_ml/utils.py

@@ -30,9 +30,14 @@
 from typing import Any, Callable, Collection, Iterable, List, TypeVar, Union, cast
 
 import pandas as pd  # type: ignore
+from pandas.core.dtypes.common import is_list_like  # type: ignore
 
 RT = TypeVar("RT")
 
+MEAN_ABSOLUTE_DEVIATION = "mad"
+VARIANCE = "var"
+STANDARD_DEVIATION = "std"
+
 
 def deprecated_api(
     replace_with: str,
@@ -61,6 +66,29 @@ def is_valid_attr_name(s: str) -> bool:
     )
 
 
+def to_list_if_needed(value):
+    """
+    Converts the input to a list if necessary.
+
+    If the input is a pandas Index, it converts it to a list.
+    If the input is not list-like (e.g., a single value), it wraps it in a list.
+    If the input is None or already list-like, it returns it as is.
+
+    Parameters:
+    value: The input to potentially convert to a list.
+
+    Returns:
+    The input converted to a list if needed, or the original input if no conversion is necessary.
+    """
+    if value is None:
+        return None
+    if isinstance(value, pd.Index):
+        return value.tolist()
+    if not is_list_like(value):
+        return [value]
+    return value
+
+
 def to_list(x: Union[Collection[Any], pd.Series]) -> List[Any]:
     if isinstance(x, ABCCollection):
         return list(x)
@@ -77,3 +105,23 @@ def try_sort(iterable: Iterable[str]) -> Iterable[str]:
         return sorted(listed)
     except TypeError:
         return listed
+
+
+class CustomFunctionDispatcher:
+    # Define custom functions in a dictionary
+    customFunctionMap = {
+        MEAN_ABSOLUTE_DEVIATION: lambda x: (x - x.median()).abs().mean(),
+    }
+
+    @classmethod
+    def apply_custom_function(cls, func, data):
+        """
+        Apply a custom function if available, else return None.
+        :param func: Function name as a string
+        :param data: Data on which function is applied
+        :return: Result of custom function or None if func not found
+        """
+        custom_func = cls.customFunctionMap.get(func)
+        if custom_func:
+            return custom_func(data)
+        return None

requirements-dev.txt

@@ -1,7 +1,7 @@
 #
 # Basic requirements
 #
-pandas>=1.5.2,<2
+pandas==2.0.3
 matplotlib>=3.6.2,<4
 numpy>=1.24.0,<2
 opensearch-py>=2.2.0

requirements.txt

@@ -1,7 +1,7 @@
 #
 # Basic requirements
 #
-pandas>=1.5.2,<2
+pandas==2.0.3
 matplotlib>=3.6.2,<4
 numpy>=1.24.0,<2
 opensearch-py>=2.2.0

setup.py

@@ -84,7 +84,7 @@
     },
     install_requires=[
         "opensearch-py>=2",
-        "pandas>=1.5,<3",
+        "pandas==2.0.3",
         "matplotlib>=3.6.0,<4",
         "numpy>=1.24.0,<2",
         "deprecated>=1.2.14,<2",