[SPARK-46858][PYTHON][PS][BUILD] Upgrade Pandas to 2.2.0

dev/infra/Dockerfile

@@ -91,10 +91,10 @@ RUN mkdir -p /usr/local/pypy/pypy3.8 && \
     ln -sf /usr/local/pypy/pypy3.8/bin/pypy /usr/local/bin/pypy3.8 && \
     ln -sf /usr/local/pypy/pypy3.8/bin/pypy /usr/local/bin/pypy3
 RUN curl -sS https://bootstrap.pypa.io/get-pip.py | pypy3
-RUN pypy3 -m pip install numpy 'six==1.16.0' 'pandas<=2.1.4' scipy coverage matplotlib lxml
+RUN pypy3 -m pip install numpy 'six==1.16.0' 'pandas<=2.2.0' scipy coverage matplotlib lxml
 
 
-ARG BASIC_PIP_PKGS="numpy pyarrow>=15.0.0 six==1.16.0 pandas<=2.1.4 scipy plotly>=4.8 mlflow>=2.8.1 coverage matplotlib openpyxl memory-profiler>=0.61.0 scikit-learn>=1.3.2"
+ARG BASIC_PIP_PKGS="numpy pyarrow>=15.0.0 six==1.16.0 pandas<=2.2.0 scipy plotly>=4.8 mlflow>=2.8.1 coverage matplotlib openpyxl memory-profiler>=0.61.0 scikit-learn>=1.3.2"
 # Python deps for Spark Connect
 ARG CONNECT_PIP_PKGS="grpcio==1.59.3 grpcio-status==1.59.3 protobuf==4.25.1 googleapis-common-protos==1.56.4"
 

python/pyspark/pandas/frame.py

@@ -10607,7 +10607,9 @@ def melt(
                     name_like_string(name) if name is not None else "variable_{}".format(i)
                     for i, name in enumerate(self._internal.column_label_names)
                 ]
-        elif isinstance(var_name, str):
+        elif is_list_like(var_name):
+            raise ValueError(f"{var_name=} must be a scalar.")
+        else:
             var_name = [var_name]
 
         pairs = F.explode(

python/pyspark/pandas/namespace.py

@@ -85,7 +85,6 @@
 from pyspark.pandas.frame import DataFrame, _reduce_spark_multi
 from pyspark.pandas.internal import (
     InternalFrame,
-    DEFAULT_SERIES_NAME,
     HIDDEN_COLUMNS,
     SPARK_INDEX_NAME_FORMAT,
 )
@@ -2554,7 +2553,7 @@ def resolve_func(psdf, this_column_labels, that_column_labels):
         if isinstance(obj, Series):
             num_series += 1
             series_names.add(obj.name)
-            new_objs.append(obj.to_frame(DEFAULT_SERIES_NAME))
+            new_objs.append(obj.to_frame())
         else:
             assert isinstance(obj, DataFrame)
             new_objs.append(obj)

python/pyspark/pandas/supported_api_gen.py

@@ -37,7 +37,7 @@
 MAX_MISSING_PARAMS_SIZE = 5
 COMMON_PARAMETER_SET = {"kwargs", "args", "cls"}
 MODULE_GROUP_MATCH = [(pd, ps), (pdw, psw), (pdg, psg)]
-PANDAS_LATEST_VERSION = "2.1.4"
+PANDAS_LATEST_VERSION = "2.2.0"
 
 RST_HEADER = """
 =====================

python/pyspark/pandas/tests/computation/test_melt.py

@@ -100,23 +100,16 @@ def test_melt(self):
             .sort_values(["variable_0", "variable_1", "value"])
             .rename(columns=name_like_string),
         )
-        self.assert_eq(
-            psdf.melt(
+        self.assertRaises(
+            ValueError,
+            lambda: psdf.melt(
                 id_vars=[(TEN, "A")],
                 value_vars=[(TEN, "B")],
                 var_name=["myV1", "myV2"],
                 value_name="myValname",
             )
             .sort_values(["myV1", "myV2", "myValname"])
             .reset_index(drop=True),
-            pdf.melt(
-                id_vars=[(TEN, "A")],
-                value_vars=[(TEN, "B")],
-                var_name=["myV1", "myV2"],
-                value_name="myValname",
-            )
-            .sort_values(["myV1", "myV2", "myValname"])
-            .rename(columns=name_like_string),
         )
 
         columns.names = ["v0", "v1"]

python/pyspark/pandas/tests/data_type_ops/test_num_arithmetic.py

@@ -44,33 +44,33 @@ def test_add(self):
         pdf, psdf = self.pdf, self.psdf
         for col in self.numeric_df_cols:
             pser, psser = pdf[col], psdf[col]
-            self.assert_eq(pser + pser, psser + psser)
-            self.assert_eq(pser + 1, psser + 1)
+            self.assert_eq(pser + pser, psser + psser, check_exact=False)
+            self.assert_eq(pser + 1, psser + 1, check_exact=False)
             # self.assert_eq(pser + 0.1, psser + 0.1)
-            self.assert_eq(pser + pser.astype(bool), psser + psser.astype(bool))
-            self.assert_eq(pser + True, psser + True)
-            self.assert_eq(pser + False, psser + False)
+            self.assert_eq(pser + pser.astype(bool), psser + psser.astype(bool), check_exact=False)
+            self.assert_eq(pser + True, psser + True, check_exact=False)
+            self.assert_eq(pser + False, psser + False, check_exact=False)
 
             for n_col in self.non_numeric_df_cols:
                 if n_col == "bool":
-                    self.assert_eq(pser + pdf[n_col], psser + psdf[n_col])
+                    self.assert_eq(pser + pdf[n_col], psser + psdf[n_col], check_exact=False)
                 else:
                     self.assertRaises(TypeError, lambda: psser + psdf[n_col])
 
     def test_sub(self):
         pdf, psdf = self.pdf, self.psdf
         for col in self.numeric_df_cols:
             pser, psser = pdf[col], psdf[col]
-            self.assert_eq(pser - pser, psser - psser)
-            self.assert_eq(pser - 1, psser - 1)
+            self.assert_eq(pser - pser, psser - psser, check_exact=False)
+            self.assert_eq(pser - 1, psser - 1, check_exact=False)
             # self.assert_eq(pser - 0.1, psser - 0.1)
-            self.assert_eq(pser - pser.astype(bool), psser - psser.astype(bool))
-            self.assert_eq(pser - True, psser - True)
-            self.assert_eq(pser - False, psser - False)
+            self.assert_eq(pser - pser.astype(bool), psser - psser.astype(bool), check_exact=False)
+            self.assert_eq(pser - True, psser - True, check_exact=False)
+            self.assert_eq(pser - False, psser - False, check_exact=False)
 
             for n_col in self.non_numeric_df_cols:
                 if n_col == "bool":
-                    self.assert_eq(pser - pdf[n_col], psser - psdf[n_col])
+                    self.assert_eq(pser - pdf[n_col], psser - psdf[n_col], check_exact=False)
                 else:
                     self.assertRaises(TypeError, lambda: psser - psdf[n_col])
 

python/pyspark/pandas/tests/indexes/test_conversion.py

@@ -195,8 +195,8 @@ def test_to_series(self):
         psidx = self.psdf.set_index("b", append=True).index
 
         with self.sql_conf({SPARK_CONF_ARROW_ENABLED: False}):
-            self.assert_eq(psidx.to_series(), pidx.to_series())
-            self.assert_eq(psidx.to_series(name="a"), pidx.to_series(name="a"))
+            self.assert_eq(psidx.to_series(), pidx.to_series(), check_exact=False)
+            self.assert_eq(psidx.to_series(name="a"), pidx.to_series(name="a"), check_exact=False)
 
         expected_error_message = "Series.name must be a hashable type"
         with self.assertRaisesRegex(TypeError, expected_error_message):

python/pyspark/pandas/tests/resample/test_error.py

@@ -52,12 +52,21 @@ def test_resample_error(self):
             psdf.A.resample("3W").sum()
 
         with self.assertRaisesRegex(ValueError, "rule offset must be positive"):
+            psdf.A.resample("0D").sum()
+
+        with self.assertRaisesRegex(ValueError, "rule code YE-DEC is not supported"):
             psdf.A.resample("0Y").sum()
 
         with self.assertRaisesRegex(ValueError, "invalid closed: 'middle'"):
+            psdf.A.resample("3D", closed="middle").sum()
+
+        with self.assertRaisesRegex(ValueError, "rule code YE-DEC is not supported"):
             psdf.A.resample("3Y", closed="middle").sum()
 
         with self.assertRaisesRegex(ValueError, "invalid label: 'both'"):
+            psdf.A.resample("3D", label="both").sum()
+
+        with self.assertRaisesRegex(ValueError, "rule code YE-DEC is not supported"):
             psdf.A.resample("3Y", label="both").sum()
 
         with self.assertRaisesRegex(

python/pyspark/sql/tests/connect/test_connect_basic.py

@@ -2244,6 +2244,7 @@ def test_stat_freq_items(self):
         self.assert_eq(
             self.connect.read.table(self.tbl_name2).stat.freqItems(["col1", "col3"]).toPandas(),
             self.spark.read.table(self.tbl_name2).stat.freqItems(["col1", "col3"]).toPandas(),
+            check_exact=False,
         )
 
         self.assert_eq(