Merge branch '2.1.x' into backport_rank

.github/actions/build_pandas/action.yml

@@ -25,8 +25,8 @@ runs:
     - name: Build Pandas
       run: |
         if [[ ${{ inputs.editable }} == "true" ]]; then
-          pip install -e . --no-build-isolation -v
+          pip install -e . --no-build-isolation -v --no-deps
         else
-          pip install . --no-build-isolation -v
+          pip install . --no-build-isolation -v --no-deps
         fi
       shell: bash -el {0}

.github/workflows/docbuild-and-upload.yml

@@ -67,7 +67,7 @@ jobs:
       run: cp doc/cheatsheet/Pandas_Cheat_Sheet* web/build/
 
     - name: Upload web
-      run: rsync -az --delete --exclude='pandas-docs' --exclude='docs' web/build/ web@${{ secrets.server_ip }}:/var/www/html
+      run: rsync -az --delete --exclude='pandas-docs' --exclude='docs' --exclude='benchmarks' web/build/ web@${{ secrets.server_ip }}:/var/www/html
       if: github.event_name == 'push' && github.ref == 'refs/heads/main'
 
     - name: Upload dev docs

.github/workflows/unit-tests.yml

@@ -236,7 +236,7 @@ jobs:
           . ~/virtualenvs/pandas-dev/bin/activate
           python -m pip install --no-cache-dir -U pip wheel setuptools meson[ninja]==1.2.1 meson-python==0.13.1
           python -m pip install numpy --config-settings=setup-args="-Dallow-noblas=true"
-          python -m pip install --no-cache-dir versioneer[toml] cython python-dateutil pytz pytest>=7.3.2 pytest-xdist>=2.2.0 pytest-asyncio>=0.17 hypothesis>=6.46.1
+          python -m pip install --no-cache-dir versioneer[toml] "cython<3.0.3" python-dateutil pytz pytest>=7.3.2 pytest-xdist>=2.2.0 pytest-asyncio>=0.17 hypothesis>=6.46.1
           python -m pip install --no-cache-dir --no-build-isolation -e .
           python -m pip list --no-cache-dir
           export PANDAS_CI=1
@@ -274,7 +274,7 @@ jobs:
           /opt/python/cp311-cp311/bin/python -m venv ~/virtualenvs/pandas-dev
           . ~/virtualenvs/pandas-dev/bin/activate
           python -m pip install --no-cache-dir -U pip wheel setuptools meson-python==0.13.1 meson[ninja]==1.2.1
-          python -m pip install --no-cache-dir versioneer[toml] cython numpy python-dateutil pytz pytest>=7.3.2 pytest-xdist>=2.2.0 pytest-asyncio>=0.17 hypothesis>=6.46.1
+          python -m pip install --no-cache-dir versioneer[toml] "cython<3.0.3" numpy python-dateutil pytz pytest>=7.3.2 pytest-xdist>=2.2.0 pytest-asyncio>=0.17 hypothesis>=6.46.1
           python -m pip install --no-cache-dir --no-build-isolation -e .
           python -m pip list --no-cache-dir
 
@@ -347,8 +347,8 @@ jobs:
           python -m pip install --upgrade pip setuptools wheel meson[ninja]==1.2.1 meson-python==0.13.1
           python -m pip install --pre --extra-index-url https://pypi.anaconda.org/scientific-python-nightly-wheels/simple numpy
           python -m pip install versioneer[toml]
-          python -m pip install python-dateutil pytz tzdata cython hypothesis>=6.46.1 pytest>=7.3.2 pytest-xdist>=2.2.0 pytest-cov pytest-asyncio>=0.17
-          python -m pip install -ve . --no-build-isolation --no-index
+          python -m pip install python-dateutil pytz tzdata "cython<3.0.3" hypothesis>=6.46.1 pytest>=7.3.2 pytest-xdist>=2.2.0 pytest-cov pytest-asyncio>=0.17
+          python -m pip install -ve . --no-build-isolation --no-index --no-deps
           python -m pip list
 
       - name: Run Tests

.github/workflows/wheels.yml

@@ -138,7 +138,7 @@ jobs:
         run: echo "sdist_name=$(cd ./dist && ls -d */)" >> "$GITHUB_ENV"
 
       - name: Build wheels
-        uses: pypa/[email protected].1
+        uses: pypa/[email protected].2
         with:
          package-dir: ./dist/${{ matrix.buildplat[1] == 'macosx_*' && env.sdist_name || needs.build_sdist.outputs.sdist_file }}
         env:

ci/deps/actions-310.yaml

@@ -6,7 +6,7 @@ dependencies:
 
   # build dependencies
   - versioneer[toml]
-  - cython>=0.29.33
+  - cython>=0.29.33, <3.0.3
   - meson[ninja]=1.2.1
   - meson-python=0.13.1
 
@@ -20,7 +20,7 @@ dependencies:
 
   # required dependencies
   - python-dateutil
-  - numpy
+  - numpy<2
   - pytz
 
   # optional dependencies

ci/deps/actions-311-downstream_compat.yaml

@@ -7,7 +7,7 @@ dependencies:
 
   # build dependencies
   - versioneer[toml]
-  - cython>=0.29.33
+  - cython>=0.29.33, <3.0.3
   - meson[ninja]=1.2.1
   - meson-python=0.13.1
 
@@ -21,7 +21,7 @@ dependencies:
 
   # required dependencies
   - python-dateutil
-  - numpy
+  - numpy<2
   - pytz
 
   # optional dependencies

ci/deps/actions-311-numpydev.yaml

@@ -8,7 +8,7 @@ dependencies:
   - versioneer[toml]
   - meson[ninja]=1.2.1
   - meson-python=0.13.1
-  - cython>=0.29.33
+  - cython>=0.29.33, <3.0.3
 
   # test dependencies
   - pytest>=7.3.2

ci/deps/actions-311-pyarrownightly.yaml

@@ -7,7 +7,7 @@ dependencies:
   # build dependencies
   - versioneer[toml]
   - meson[ninja]=1.2.1
-  - cython>=0.29.33
+  - cython>=0.29.33, <3.0.3
   - meson-python=0.13.1
 
   # test dependencies
@@ -19,7 +19,7 @@ dependencies:
 
   # required dependencies
   - python-dateutil
-  - numpy
+  - numpy<2
   - pytz
   - pip
 

ci/deps/actions-311.yaml

@@ -6,7 +6,7 @@ dependencies:
 
   # build dependencies
   - versioneer[toml]
-  - cython>=0.29.33
+  - cython>=0.29.33, <3.0.3
   - meson[ninja]=1.2.1
   - meson-python=0.13.1
 
@@ -20,7 +20,7 @@ dependencies:
 
   # required dependencies
   - python-dateutil
-  - numpy
+  - numpy<2
   - pytz
 
   # optional dependencies

ci/deps/actions-39-minimum_versions.yaml

@@ -8,7 +8,7 @@ dependencies:
 
   # build dependencies
   - versioneer[toml]
-  - cython>=0.29.33
+  - cython>=0.29.33, <3.0.3
   - meson[ninja]=1.2.1
   - meson-python=0.13.1
 
@@ -22,7 +22,7 @@ dependencies:
 
   # required dependencies
   - python-dateutil=2.8.2
-  - numpy=1.22.4
+  - numpy=1.22.4, <2
   - pytz=2020.1
 
   # optional dependencies

ci/deps/actions-39.yaml

@@ -6,7 +6,7 @@ dependencies:
 
   # build dependencies
   - versioneer[toml]
-  - cython>=0.29.33
+  - cython>=0.29.33, <3.0.3
   - meson[ninja]=1.2.1
   - meson-python=0.13.1
 
@@ -20,7 +20,7 @@ dependencies:
 
   # required dependencies
   - python-dateutil
-  - numpy
+  - numpy<2
   - pytz
 
   # optional dependencies

ci/deps/actions-pypy-39.yaml

@@ -9,7 +9,7 @@ dependencies:
 
   # build dependencies
   - versioneer[toml]
-  - cython>=0.29.33
+  - cython>=0.29.33, <3.0.3
   - meson[ninja]=1.2.1
   - meson-python=0.13.1
 
@@ -21,7 +21,7 @@ dependencies:
   - hypothesis>=6.46.1
 
   # required
-  - numpy
+  - numpy<2
   - python-dateutil
   - pytz
   - pip:

ci/deps/circle-310-arm64.yaml

@@ -6,7 +6,7 @@ dependencies:
 
   # build dependencies
   - versioneer[toml]
-  - cython>=0.29.33
+  - cython>=0.29.33, <3.0.3
   - meson[ninja]=1.2.1
   - meson-python=0.13.1
 
@@ -20,7 +20,7 @@ dependencies:
 
   # required dependencies
   - python-dateutil
-  - numpy
+  - numpy<2
   - pytz
 
   # optional dependencies

doc/source/whatsnew/v2.1.2.rst

@@ -26,6 +26,7 @@ Bug fixes
 - Fixed bug in :meth:`DataFrame.idxmin` and :meth:`DataFrame.idxmax` raising for arrow dtypes (:issue:`55368`)
 - Fixed bug in :meth:`DataFrame.interpolate` raising incorrect error message (:issue:`55347`)
 - Fixed bug in :meth:`Index.insert` raising when inserting ``None`` into :class:`Index` with ``dtype="string[pyarrow_numpy]"`` (:issue:`55365`)
+- Fixed bug in :meth:`Series.all`  and :meth:`Series.any` not treating missing values correctly for ``dtype="string[pyarrow_numpy]"`` (:issue:`55367`)
 - Fixed bug in :meth:`Series.rank` for ``string[pyarrow_numpy]`` dtype (:issue:`55362`)
 - Silence ``Period[B]`` warnings introduced by :issue:`53446` during normal plotting activity (:issue:`55138`)
 -

environment.yml

@@ -21,7 +21,7 @@ dependencies:
 
   # required dependencies
   - python-dateutil
-  - numpy
+  - numpy<2
   - pytz
 
   # optional dependencies

pandas/core/arrays/string_arrow.py

@@ -601,9 +601,11 @@ def _reduce(
         self, name: str, *, skipna: bool = True, keepdims: bool = False, **kwargs
     ):
         if name in ["any", "all"]:
-            arr = pc.and_kleene(
-                pc.invert(pc.is_null(self._pa_array)), pc.not_equal(self._pa_array, "")
-            )
+            if not skipna and name == "all":
+                nas = pc.invert(pc.is_null(self._pa_array))
+                arr = pc.and_kleene(nas, pc.not_equal(self._pa_array, ""))
+            else:
+                arr = pc.not_equal(self._pa_array, "")
             return ArrowExtensionArray(arr)._reduce(
                 name, skipna=skipna, keepdims=keepdims, **kwargs
             )

pandas/io/stata.py

@@ -977,7 +977,7 @@ def __init__(self) -> None:
         # with a label, but the underlying variable is -127 to 100
         # we're going to drop the label and cast to int
         self.DTYPE_MAP = dict(
-            list(zip(range(1, 245), [np.dtype("a" + str(i)) for i in range(1, 245)]))
+            [(i, np.dtype(f"S{i}")) for i in range(1, 245)]
             + [
                 (251, np.dtype(np.int8)),
                 (252, np.dtype(np.int16)),

pandas/tests/frame/constructors/test_from_records.py

@@ -281,7 +281,7 @@ def test_frame_from_records_utc(self):
 
     def test_from_records_to_records(self):
         # from numpy documentation
-        arr = np.zeros((2,), dtype=("i4,f4,a10"))
+        arr = np.zeros((2,), dtype=("i4,f4,S10"))
         arr[:] = [(1, 2.0, "Hello"), (2, 3.0, "World")]
 
         DataFrame.from_records(arr)

pandas/tests/frame/methods/test_select_dtypes.py

@@ -339,9 +339,7 @@ def test_select_dtypes_datetime_with_tz(self):
         expected = df3.reindex(columns=[])
         tm.assert_frame_equal(result, expected)
 
-    @pytest.mark.parametrize(
-        "dtype", [str, "str", np.bytes_, "S1", "unicode", np.str_, "U1"]
-    )
+    @pytest.mark.parametrize("dtype", [str, "str", np.bytes_, "S1", np.str_, "U1"])
     @pytest.mark.parametrize("arg", ["include", "exclude"])
     def test_select_dtypes_str_raises(self, dtype, arg):
         df = DataFrame(

pandas/tests/frame/methods/test_to_records.py

@@ -253,7 +253,7 @@ def test_to_records_with_categorical(self):
             ),
             # Pass in a dtype instance.
             (
-                {"column_dtypes": np.dtype("unicode")},
+                {"column_dtypes": np.dtype(np.str_)},
                 np.rec.array(
                     [("0", "1", "0.2", "a"), ("1", "2", "1.5", "bc")],
                     dtype=[

pandas/tests/indexes/base_class/test_setops.py

@@ -182,23 +182,23 @@ def test_symmetric_difference(self):
                 "intersection",
                 np.array(
                     [(1, "A"), (2, "A"), (1, "B"), (2, "B")],
-                    dtype=[("num", int), ("let", "a1")],
+                    dtype=[("num", int), ("let", "S1")],
                 ),
                 False,
             ),
             (
                 "intersection",
                 np.array(
                     [(1, "A"), (1, "B"), (2, "A"), (2, "B")],
-                    dtype=[("num", int), ("let", "a1")],
+                    dtype=[("num", int), ("let", "S1")],
                 ),
                 None,
             ),
             (
                 "union",
                 np.array(
                     [(1, "A"), (1, "B"), (1, "C"), (2, "A"), (2, "B"), (2, "C")],
-                    dtype=[("num", int), ("let", "a1")],
+                    dtype=[("num", int), ("let", "S1")],
                 ),
                 None,
             ),
@@ -208,13 +208,13 @@ def test_tuple_union_bug(self, method, expected, sort):
         index1 = Index(
             np.array(
                 [(1, "A"), (2, "A"), (1, "B"), (2, "B")],
-                dtype=[("num", int), ("let", "a1")],
+                dtype=[("num", int), ("let", "S1")],
             )
         )
         index2 = Index(
             np.array(
                 [(1, "A"), (2, "A"), (1, "B"), (2, "B"), (1, "C"), (2, "C")],
-                dtype=[("num", int), ("let", "a1")],
+                dtype=[("num", int), ("let", "S1")],
             )
         )
 

pandas/tests/io/json/test_pandas.py

@@ -599,7 +599,7 @@ def test_blocks_compat_GH9037(self):
         )
 
         # JSON deserialisation always creates unicode strings
-        df_mixed.columns = df_mixed.columns.astype("unicode")
+        df_mixed.columns = df_mixed.columns.astype(np.str_)
         data = StringIO(df_mixed.to_json(orient="split"))
         df_roundtrip = read_json(data, orient="split")
         tm.assert_frame_equal(

pandas/tests/reductions/test_reductions.py

@@ -1087,7 +1087,8 @@ def test_any_all_pyarrow_string(self):
 
         ser = Series([None, "a"], dtype="string[pyarrow_numpy]")
         assert ser.any()
-        assert not ser.all()
+        assert ser.all()
+        assert not ser.all(skipna=False)
 
         ser = Series([None, ""], dtype="string[pyarrow_numpy]")
         assert not ser.any()

pandas/tests/reshape/concat/test_append.py

@@ -91,10 +91,10 @@ def test_append_length0_frame(self, sort):
         tm.assert_frame_equal(df5, expected)
 
     def test_append_records(self):
-        arr1 = np.zeros((2,), dtype=("i4,f4,a10"))
+        arr1 = np.zeros((2,), dtype=("i4,f4,S10"))
         arr1[:] = [(1, 2.0, "Hello"), (2, 3.0, "World")]
 
-        arr2 = np.zeros((3,), dtype=("i4,f4,a10"))
+        arr2 = np.zeros((3,), dtype=("i4,f4,S10"))
         arr2[:] = [(3, 4.0, "foo"), (5, 6.0, "bar"), (7.0, 8.0, "baz")]
 
         df1 = DataFrame(arr1)

pandas/tests/series/methods/test_astype.py

@@ -403,12 +403,12 @@ def test_astype_unicode(self):
             #  bytes with obj.decode() instead of str(obj)
             item = "野菜食べないとやばい"
             ser = Series([item.encode()])
-            result = ser.astype("unicode")
+            result = ser.astype(np.str_)
             expected = Series([item])
             tm.assert_series_equal(result, expected)
 
         for ser in test_series:
-            res = ser.astype("unicode")
+            res = ser.astype(np.str_)
             expec = ser.map(str)
             tm.assert_series_equal(res, expec)
 

pyproject.toml

@@ -10,7 +10,7 @@ requires = [
     # we don't want to force users to compile with 1.25 though
     # (Ideally, in the future, though, oldest-supported-numpy can be dropped when our min numpy is 1.25.x)
     "oldest-supported-numpy>=2022.8.16; python_version<'3.12'",
-    "numpy>=1.26.0; python_version>='3.12'",
+    "numpy>=1.26.0,<2; python_version>='3.12'",
     "versioneer[toml]"
 ]
 
@@ -29,9 +29,9 @@ authors = [
 license = {file = 'LICENSE'}
 requires-python = '>=3.9'
 dependencies = [
-  "numpy>=1.22.4; python_version<'3.11'",
-  "numpy>=1.23.2; python_version=='3.11'",
-  "numpy>=1.26.0; python_version>='3.12'",
+  "numpy>=1.22.4,<2; python_version<'3.11'",
+  "numpy>=1.23.2,<2; python_version=='3.11'",
+  "numpy>=1.26.0,<2; python_version>='3.12'",
   "python-dateutil>=2.8.2",
   "pytz>=2020.1",
   "tzdata>=2022.1"