Merge branch '2.1.x' into auto-backport-of-pr-55619-on-2.1.x

.github/workflows/unit-tests.yml

@@ -236,7 +236,7 @@ jobs:
           . ~/virtualenvs/pandas-dev/bin/activate
           python -m pip install --no-cache-dir -U pip wheel setuptools meson[ninja]==1.2.1 meson-python==0.13.1
           python -m pip install numpy --config-settings=setup-args="-Dallow-noblas=true"
-          python -m pip install --no-cache-dir versioneer[toml] cython python-dateutil pytz pytest>=7.3.2 pytest-xdist>=2.2.0 pytest-asyncio>=0.17 hypothesis>=6.46.1
+          python -m pip install --no-cache-dir versioneer[toml] cython python-dateutil pytz pytest>=7.3.2 pytest-xdist>=2.2.0 hypothesis>=6.46.1
           python -m pip install --no-cache-dir --no-build-isolation -e .
           python -m pip list --no-cache-dir
           export PANDAS_CI=1
@@ -274,7 +274,7 @@ jobs:
           /opt/python/cp311-cp311/bin/python -m venv ~/virtualenvs/pandas-dev
           . ~/virtualenvs/pandas-dev/bin/activate
           python -m pip install --no-cache-dir -U pip wheel setuptools meson-python==0.13.1 meson[ninja]==1.2.1
-          python -m pip install --no-cache-dir versioneer[toml] cython numpy python-dateutil pytz pytest>=7.3.2 pytest-xdist>=2.2.0 pytest-asyncio>=0.17 hypothesis>=6.46.1
+          python -m pip install --no-cache-dir versioneer[toml] cython numpy python-dateutil pytz pytest>=7.3.2 pytest-xdist>=2.2.0 hypothesis>=6.46.1
           python -m pip install --no-cache-dir --no-build-isolation -e .
           python -m pip list --no-cache-dir
 
@@ -347,7 +347,7 @@ jobs:
           python -m pip install --upgrade pip setuptools wheel meson[ninja]==1.2.1 meson-python==0.13.1
           python -m pip install --pre --extra-index-url https://pypi.anaconda.org/scientific-python-nightly-wheels/simple numpy
           python -m pip install versioneer[toml]
-          python -m pip install python-dateutil pytz tzdata cython hypothesis>=6.46.1 pytest>=7.3.2 pytest-xdist>=2.2.0 pytest-cov pytest-asyncio>=0.17
+          python -m pip install python-dateutil pytz tzdata cython hypothesis>=6.46.1 pytest>=7.3.2 pytest-xdist>=2.2.0 pytest-cov
           python -m pip install -ve . --no-build-isolation --no-index --no-deps
           python -m pip list
 

.github/workflows/wheels.yml

@@ -168,7 +168,7 @@ jobs:
         shell: pwsh
         run: |
           $TST_CMD = @"
-          python -m pip install hypothesis>=6.46.1 pytest>=7.3.2 pytest-xdist>=2.2.0 pytest-asyncio>=0.17;
+          python -m pip install hypothesis>=6.46.1 pytest>=7.3.2 pytest-xdist>=2.2.0;
           python -m pip install `$(Get-Item pandas\wheelhouse\*.whl);
           python -c `'import pandas as pd; pd.test(extra_args=[\"`\"--no-strict-data-files`\"\", \"`\"-m not clipboard and not single_cpu and not slow and not network and not db`\"\"])`';
           "@

ci/deps/actions-310.yaml

@@ -14,7 +14,6 @@ dependencies:
   - pytest>=7.3.2
   - pytest-cov
   - pytest-xdist>=2.2.0
-  - pytest-asyncio>=0.17.0
   - pytest-localserver>=0.7.1
   - boto3
 

ci/deps/actions-311-downstream_compat.yaml

@@ -15,7 +15,6 @@ dependencies:
   - pytest>=7.3.2
   - pytest-cov
   - pytest-xdist>=2.2.0
-  - pytest-asyncio>=0.17.0
   - pytest-localserver>=0.7.1
   - boto3
 

ci/deps/actions-311-numpydev.yaml

@@ -18,7 +18,6 @@ dependencies:
   # causes an InternalError within pytest
   - pytest-xdist>=2.2.0, <3
   - hypothesis>=6.46.1
-  - pytest-asyncio>=0.17.0
 
   # pandas dependencies
   - python-dateutil

ci/deps/actions-311-pyarrownightly.yaml

@@ -15,7 +15,6 @@ dependencies:
   - pytest-cov
   - pytest-xdist>=2.2.0
   - hypothesis>=6.46.1
-  - pytest-asyncio>=0.17.0
 
   # required dependencies
   - python-dateutil

ci/deps/actions-311.yaml

@@ -14,7 +14,6 @@ dependencies:
   - pytest>=7.3.2
   - pytest-cov
   - pytest-xdist>=2.2.0
-  - pytest-asyncio>=0.17.0
   - pytest-localserver>=0.7.1
   - boto3
 

ci/deps/actions-39-minimum_versions.yaml

@@ -16,7 +16,6 @@ dependencies:
   - pytest>=7.3.2
   - pytest-cov
   - pytest-xdist>=2.2.0
-  - pytest-asyncio>=0.17.0
   - pytest-localserver>=0.7.1
   - boto3
 

ci/deps/actions-39.yaml

@@ -14,7 +14,6 @@ dependencies:
   - pytest>=7.3.2
   - pytest-cov
   - pytest-xdist>=2.2.0
-  - pytest-asyncio>=0.17.0
   - pytest-localserver>=0.7.1
   - boto3
 

ci/deps/actions-pypy-39.yaml

@@ -16,7 +16,6 @@ dependencies:
   # test dependencies
   - pytest>=7.3.2
   - pytest-cov
-  - pytest-asyncio>=0.17.0
   - pytest-xdist>=2.2.0
   - hypothesis>=6.46.1
 

ci/deps/circle-310-arm64.yaml

@@ -14,7 +14,6 @@ dependencies:
   - pytest>=7.3.2
   - pytest-cov
   - pytest-xdist>=2.2.0
-  - pytest-asyncio>=0.17.0
   - pytest-localserver>=0.7.1
   - boto3
 

ci/meta.yaml

@@ -64,7 +64,6 @@ test:
   requires:
     - pip
     - pytest >=7.3.2
-    - pytest-asyncio >=0.17.0
     - pytest-xdist >=2.2.0
     - pytest-cov
     - hypothesis >=6.46.1

ci/run_tests.sh

@@ -10,7 +10,8 @@ echo PYTHONHASHSEED=$PYTHONHASHSEED
 
 COVERAGE="-s --cov=pandas --cov-report=xml --cov-append --cov-config=pyproject.toml"
 
-PYTEST_CMD="MESONPY_EDITABLE_VERBOSE=1 PYTHONDEVMODE=1 PYTHONWARNDEFAULTENCODING=1 pytest -r fEs -n $PYTEST_WORKERS --dist=loadfile $TEST_ARGS $COVERAGE $PYTEST_TARGET"
+# TODO: Support NEP 50 and remove NPY_PROMOTION_STATE
+PYTEST_CMD="NPY_PROMOTION_STATE=legacy MESONPY_EDITABLE_VERBOSE=1 PYTHONDEVMODE=1 PYTHONWARNDEFAULTENCODING=1 pytest -r fEs -n $PYTEST_WORKERS --dist=loadfile $TEST_ARGS $COVERAGE $PYTEST_TARGET"
 
 if [[ "$PATTERN" ]]; then
   PYTEST_CMD="$PYTEST_CMD -m \"$PATTERN\""

doc/source/user_guide/basics.rst

@@ -408,20 +408,6 @@ raise a ValueError:
 
     pd.Series(['foo', 'bar', 'baz']) == pd.Series(['foo'])
 
-Note that this is different from the NumPy behavior where a comparison can
-be broadcast:
-
-.. ipython:: python
-
-    np.array([1, 2, 3]) == np.array([2])
-
-or it can return False if broadcasting can not be done:
-
-.. ipython:: python
-   :okwarning:
-
-    np.array([1, 2, 3]) == np.array([1, 2])
-
 Combining overlapping data sets
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 

doc/source/whatsnew/index.rst

@@ -16,6 +16,8 @@ Version 2.1
 .. toctree::
    :maxdepth: 2
 
+   v2.1.4
+   v2.1.3
    v2.1.2
    v2.1.1
    v2.1.0

doc/source/whatsnew/v0.17.0.rst

@@ -726,10 +726,10 @@ be broadcast:
 
 or it can return False if broadcasting can not be done:
 
-.. ipython:: python
-   :okwarning:
+.. code-block:: ipython
 
-   np.array([1, 2, 3]) == np.array([1, 2])
+   In [11]: np.array([1, 2, 3]) == np.array([1, 2])
+   Out[11]: False
 
 Changes to boolean comparisons vs. None
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^

doc/source/whatsnew/v2.1.1.rst

@@ -52,4 +52,4 @@ Other
 Contributors
 ~~~~~~~~~~~~
 
-.. contributors:: v2.1.0..v2.1.1|HEAD
+.. contributors:: v2.1.0..v2.1.1

doc/source/whatsnew/v2.1.2.rst

@@ -1,6 +1,6 @@
 .. _whatsnew_212:
 
-What's new in 2.1.2 (October 25, 2023)
+What's new in 2.1.2 (October 26, 2023)
 ---------------------------------------
 
 These are the changes in pandas 2.1.2. See :ref:`release` for a full changelog
@@ -28,7 +28,9 @@ Fixed regressions
 - Fixed regression in :meth:`DataFrameGroupBy.agg` and :meth:`SeriesGroupBy.agg` where if the option ``compute.use_numba`` was set to True, groupby methods not supported by the numba engine would raise a ``TypeError`` (:issue:`55520`)
 - Fixed performance regression with wide DataFrames, typically involving methods where all columns were accessed individually (:issue:`55256`, :issue:`55245`)
 - Fixed regression in :func:`merge_asof` raising ``TypeError`` for ``by`` with datetime and timedelta dtypes (:issue:`55453`)
+- Fixed regression in :func:`read_parquet` when reading a file with a string column consisting of more than 2 GB of string data and using the ``"string"`` dtype (:issue:`55606`)
 - Fixed regression in :meth:`DataFrame.to_sql` not roundtripping datetime columns correctly for sqlite when using ``detect_types`` (:issue:`55554`)
+- Fixed regression in construction of certain DataFrame or Series subclasses (:issue:`54922`)
 
 .. ---------------------------------------------------------------------------
 .. _whatsnew_212.bug_fixes:
@@ -65,3 +67,5 @@ Other
 
 Contributors
 ~~~~~~~~~~~~
+
+.. contributors:: v2.1.1..v2.1.2

doc/source/whatsnew/v2.1.3.rst

@@ -0,0 +1,33 @@
+.. _whatsnew_213:
+
+What's new in 2.1.3 (November 10, 2023)
+---------------------------------------
+
+These are the changes in pandas 2.1.3. See :ref:`release` for a full changelog
+including other versions of pandas.
+
+{{ header }}
+
+.. ---------------------------------------------------------------------------
+.. _whatsnew_213.regressions:
+
+Fixed regressions
+~~~~~~~~~~~~~~~~~
+- Fixed infinite recursion from operations that return a new object on some DataFrame subclasses (:issue:`55763`)
+
+.. ---------------------------------------------------------------------------
+.. _whatsnew_213.bug_fixes:
+
+Bug fixes
+~~~~~~~~~
+- Bug in :meth:`DatetimeIndex.diff` raising ``TypeError`` (:issue:`55080`)
+- Bug in :meth:`Index.isin` raising for Arrow backed string and ``None`` value (:issue:`55821`)
+- Fix :func:`read_parquet` and :func:`read_feather` for `CVE-2023-47248 <https://www.cve.org/CVERecord?id=CVE-2023-47248>`__ (:issue:`55894`)
+
+.. ---------------------------------------------------------------------------
+.. _whatsnew_213.contributors:
+
+Contributors
+~~~~~~~~~~~~
+
+.. contributors:: v2.1.2..v2.1.3|HEAD

doc/source/whatsnew/v2.1.4.rst

@@ -0,0 +1,46 @@
+.. _whatsnew_214:
+
+What's new in 2.1.4 (December ??, 2023)
+---------------------------------------
+
+These are the changes in pandas 2.1.4. See :ref:`release` for a full changelog
+including other versions of pandas.
+
+{{ header }}
+
+.. ---------------------------------------------------------------------------
+.. _whatsnew_214.regressions:
+
+Fixed regressions
+~~~~~~~~~~~~~~~~~
+-
+-
+
+.. ---------------------------------------------------------------------------
+.. _whatsnew_214.bug_fixes:
+
+Bug fixes
+~~~~~~~~~
+- Bug in :meth:`DataFrame.apply` where passing ``raw=True`` ignored ``args`` passed to the applied function (:issue:`55753`)
+- Bug in :meth:`Index.__getitem__` returning wrong result for Arrow dtypes and negative stepsize (:issue:`55832`)
+- Fixed bug in :func:`to_numeric` converting to extension dtype for ``string[pyarrow_numpy]`` dtype (:issue:`56179`)
+- Fixed bug in :meth:`DataFrame.__setitem__` casting :class:`Index` with object-dtype to PyArrow backed strings when ``infer_string`` option is set (:issue:`55638`)
+- Fixed bug in :meth:`DataFrame.to_hdf` raising when columns have ``StringDtype`` (:issue:`55088`)
+- Fixed bug in :meth:`Index.insert` casting object-dtype to PyArrow backed strings when ``infer_string`` option is set (:issue:`55638`)
+- Fixed bug in :meth:`Series.str.translate` losing object dtype when string option is set (:issue:`56152`)
+
+.. ---------------------------------------------------------------------------
+.. _whatsnew_214.other:
+
+Other
+~~~~~
+-
+-
+
+.. ---------------------------------------------------------------------------
+.. _whatsnew_214.contributors:
+
+Contributors
+~~~~~~~~~~~~
+
+.. contributors:: v2.1.3..v2.1.4|HEAD

environment.yml

@@ -16,7 +16,6 @@ dependencies:
   - pytest>=7.3.2
   - pytest-cov
   - pytest-xdist>=2.2.0
-  - pytest-asyncio>=0.17.0
   - coverage
 
   # required dependencies

pandas/_libs/tslibs/tzconversion.pyx

@@ -332,13 +332,16 @@ timedelta-like}
     # Shift the delta_idx by if the UTC offset of
     # the target tz is greater than 0 and we're moving forward
     # or vice versa
-    first_delta = info.deltas[0]
-    if (shift_forward or shift_delta > 0) and first_delta > 0:
-        delta_idx_offset = 1
-    elif (shift_backward or shift_delta < 0) and first_delta < 0:
-        delta_idx_offset = 1
-    else:
-        delta_idx_offset = 0
+    # TODO: delta_idx_offset and info.deltas are needed for zoneinfo timezones,
+    # but are not applicable for all timezones. Setting the former to 0 and
+    # length checking the latter avoids UB, but this could use a larger refactor
+    delta_idx_offset = 0
+    if len(info.deltas):
+        first_delta = info.deltas[0]
+        if (shift_forward or shift_delta > 0) and first_delta > 0:
+            delta_idx_offset = 1
+        elif (shift_backward or shift_delta < 0) and first_delta < 0:
+            delta_idx_offset = 1
 
     for i in range(n):
         val = vals[i]

pandas/_testing/_warnings.py

@@ -13,6 +13,8 @@
 )
 import warnings
 
+from pandas.compat import PY311
+
 if TYPE_CHECKING:
     from collections.abc import (
         Generator,
@@ -179,6 +181,11 @@ def _assert_caught_no_extra_warnings(
                 # due to these open files.
                 if any("matplotlib" in mod for mod in sys.modules):
                     continue
+            if PY311 and actual_warning.category == EncodingWarning:
+                # EncodingWarnings are checked in the CI
+                # pyproject.toml errors on EncodingWarnings in pandas
+                # Ignore EncodingWarnings from other libraries
+                continue
             extra_warnings.append(
                 (
                     actual_warning.category.__name__,

pandas/compat/__init__.py

@@ -31,6 +31,7 @@
     pa_version_under11p0,
     pa_version_under13p0,
     pa_version_under14p0,
+    pa_version_under14p1,
 )
 
 if TYPE_CHECKING:
@@ -188,6 +189,7 @@ def get_bz2_file() -> type[pandas.compat.compressors.BZ2File]:
     "pa_version_under11p0",
     "pa_version_under13p0",
     "pa_version_under14p0",
+    "pa_version_under14p1",
     "IS64",
     "ISMUSL",
     "PY310",

pandas/compat/numpy/__init__.py

@@ -11,6 +11,7 @@
 np_version_gte1p24 = _nlv >= Version("1.24")
 np_version_gte1p24p3 = _nlv >= Version("1.24.3")
 np_version_gte1p25 = _nlv >= Version("1.25")
+np_version_gt2 = _nlv >= Version("2.0.0.dev0")
 is_numpy_dev = _nlv.dev is not None
 _min_numpy_ver = "1.22.4"
 
@@ -26,7 +27,7 @@
 np_long: type
 np_ulong: type
 
-if _nlv >= Version("2.0.0.dev0"):
+if np_version_gt2:
     try:
         with warnings.catch_warnings():
             warnings.filterwarnings(

pandas/compat/pyarrow.py

@@ -16,6 +16,8 @@
     pa_version_under12p0 = _palv < Version("12.0.0")
     pa_version_under13p0 = _palv < Version("13.0.0")
     pa_version_under14p0 = _palv < Version("14.0.0")
+    pa_version_under14p1 = _palv < Version("14.0.1")
+    pa_version_under15p0 = _palv < Version("15.0.0")
 except ImportError:
     pa_version_under7p0 = True
     pa_version_under8p0 = True
@@ -25,3 +27,5 @@
     pa_version_under12p0 = True
     pa_version_under13p0 = True
     pa_version_under14p0 = True
+    pa_version_under14p1 = True
+    pa_version_under15p0 = True

pandas/core/apply.py

@@ -923,7 +923,9 @@ def wrapper(*args, **kwargs):
 
             return wrapper
 
-        result = np.apply_along_axis(wrap_function(self.func), self.axis, self.values)
+        result = np.apply_along_axis(
+            wrap_function(self.func), self.axis, self.values, *self.args, **self.kwargs
+        )
 
         # TODO: mixed type case
         if result.ndim == 2: