Merge remote-tracking branch 'upstream/main' into bitmask-backed

.github/workflows/package-checks.yml

@@ -24,7 +24,7 @@ jobs:
     runs-on: ubuntu-22.04
     strategy:
       matrix:
-        extra: ["test", "performance", "computation", "fss", "aws", "gcp", "excel", "parquet", "feather", "hdf5", "spss", "postgresql", "mysql", "sql-other", "html", "xml", "plot", "output_formatting", "clipboard", "compression", "consortium-standard", "all"]
+        extra: ["test", "performance", "computation", "fss", "aws", "gcp", "excel", "parquet", "feather", "hdf5", "spss", "postgresql", "mysql", "sql-other", "html", "xml", "plot", "output-formatting", "clipboard", "compression", "consortium-standard", "all"]
       fail-fast: false
     name: Install Extras - ${{ matrix.extra }}
     concurrency:

.github/workflows/unit-tests.yml

@@ -230,11 +230,13 @@ jobs:
               git -c user.email="[email protected]" merge --no-commit my_ref_name
           fi
       - name: Build environment and Run Tests
+        # https://github.com/numpy/numpy/issues/24703#issuecomment-1722379388
         run: |
           /opt/python/cp311-cp311/bin/python -m venv ~/virtualenvs/pandas-dev
           . ~/virtualenvs/pandas-dev/bin/activate
-          python -m pip install --no-cache-dir -U pip wheel setuptools meson[ninja]==1.0.1 meson-python==0.13.1
-          python -m pip install --no-cache-dir versioneer[toml] "cython<3.0.1" numpy python-dateutil pytz pytest>=7.3.2 pytest-xdist>=2.2.0 pytest-asyncio>=0.17 hypothesis>=6.46.1
+          python -m pip install --no-cache-dir -U pip wheel setuptools meson[ninja]==1.2.1 meson-python==0.13.1
+          python -m pip install numpy --config-settings=setup-args="-Dallow-noblas=true"
+          python -m pip install --no-cache-dir versioneer[toml] cython python-dateutil pytz pytest>=7.3.2 pytest-xdist>=2.2.0 pytest-asyncio>=0.17 hypothesis>=6.46.1
           python -m pip install --no-cache-dir --no-build-isolation -e .
           python -m pip list --no-cache-dir
           export PANDAS_CI=1
@@ -271,8 +273,8 @@ jobs:
         run: |
           /opt/python/cp311-cp311/bin/python -m venv ~/virtualenvs/pandas-dev
           . ~/virtualenvs/pandas-dev/bin/activate
-          python -m pip install --no-cache-dir -U pip wheel setuptools meson-python==0.13.1 meson[ninja]==1.0.1
-          python -m pip install --no-cache-dir versioneer[toml] "cython<3.0.1" numpy python-dateutil pytz pytest>=7.3.2 pytest-xdist>=2.2.0 pytest-asyncio>=0.17 hypothesis>=6.46.1
+          python -m pip install --no-cache-dir -U pip wheel setuptools meson-python==0.13.1 meson[ninja]==1.2.1
+          python -m pip install --no-cache-dir versioneer[toml] cython numpy python-dateutil pytz pytest>=7.3.2 pytest-xdist>=2.2.0 pytest-asyncio>=0.17 hypothesis>=6.46.1
           python -m pip install --no-cache-dir --no-build-isolation -e .
           python -m pip list --no-cache-dir
 
@@ -342,10 +344,10 @@ jobs:
       - name: Build Environment
         run: |
           python --version
-          python -m pip install --upgrade pip setuptools wheel meson[ninja]==1.0.1 meson-python==0.13.1
+          python -m pip install --upgrade pip setuptools wheel meson[ninja]==1.2.1 meson-python==0.13.1
           python -m pip install --pre --extra-index-url https://pypi.anaconda.org/scientific-python-nightly-wheels/simple numpy
           python -m pip install versioneer[toml]
-          python -m pip install python-dateutil pytz tzdata "cython<3.0.1" hypothesis>=6.46.1 pytest>=7.3.2 pytest-xdist>=2.2.0 pytest-cov pytest-asyncio>=0.17
+          python -m pip install python-dateutil pytz tzdata cython hypothesis>=6.46.1 pytest>=7.3.2 pytest-xdist>=2.2.0 pytest-cov pytest-asyncio>=0.17
           python -m pip install -ve . --no-build-isolation --no-index
           python -m pip list
 

.github/workflows/wheels.yml

@@ -138,7 +138,7 @@ jobs:
         run: echo "sdist_name=$(cd ./dist && ls -d */)" >> "$GITHUB_ENV"
 
       - name: Build wheels
-        uses: pypa/cibuildwheel@v2.15.0
+        uses: pypa/cibuildwheel@v2.16.1
         with:
          package-dir: ./dist/${{ matrix.buildplat[1] == 'macosx_*' && env.sdist_name || needs.build_sdist.outputs.sdist_file }}
         env:

.pre-commit-config.yaml

@@ -20,11 +20,11 @@ ci:
 repos:
 -   repo: https://github.com/hauntsaninja/black-pre-commit-mirror
     # black compiled with mypyc
-    rev: 23.7.0
+    rev: 23.9.1
     hooks:
       - id: black
 -   repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.0.287
+    rev: v0.0.291
     hooks:
     -   id: ruff
         args: [--exit-non-zero-on-fix]
@@ -92,7 +92,7 @@ repos:
             '--filter=-readability/casting,-runtime/int,-build/include_subdir,-readability/fn_size'
         ]
 -   repo: https://github.com/pylint-dev/pylint
-    rev: v3.0.0a7
+    rev: v3.0.0b0
     hooks:
     -   id: pylint
         stages: [manual]
@@ -115,7 +115,7 @@ repos:
     hooks:
     -   id: isort
 -   repo: https://github.com/asottile/pyupgrade
-    rev: v3.10.1
+    rev: v3.13.0
     hooks:
     -   id: pyupgrade
         args: [--py39-plus]

asv_bench/benchmarks/algorithms.py

@@ -1,6 +1,7 @@
 from importlib import import_module
 
 import numpy as np
+import pyarrow as pa
 
 import pandas as pd
 
@@ -72,7 +73,16 @@ class Duplicated:
     params = [
         [True, False],
         ["first", "last", False],
-        ["int", "uint", "float", "string", "datetime64[ns]", "datetime64[ns, tz]"],
+        [
+            "int",
+            "uint",
+            "float",
+            "string",
+            "datetime64[ns]",
+            "datetime64[ns, tz]",
+            "timestamp[ms][pyarrow]",
+            "duration[s][pyarrow]",
+        ],
     ]
     param_names = ["unique", "keep", "dtype"]
 
@@ -87,6 +97,12 @@ def setup(self, unique, keep, dtype):
             "datetime64[ns, tz]": pd.date_range(
                 "2011-01-01", freq="H", periods=N, tz="Asia/Tokyo"
             ),
+            "timestamp[ms][pyarrow]": pd.Index(
+                np.arange(N), dtype=pd.ArrowDtype(pa.timestamp("ms"))
+            ),
+            "duration[s][pyarrow]": pd.Index(
+                np.arange(N), dtype=pd.ArrowDtype(pa.duration("s"))
+            ),
         }[dtype]
         if not unique:
             data = data.repeat(5)

asv_bench/benchmarks/groupby.py

@@ -841,6 +841,23 @@ def time_groupby_sum_multiindex(self):
         self.df.groupby(level=[0, 1]).sum()
 
 
+class SumTimeDelta:
+    # GH 20660
+    def setup(self):
+        N = 10**4
+        self.df = DataFrame(
+            np.random.randint(1000, 100000, (N, 100)),
+            index=np.random.randint(200, size=(N,)),
+        ).astype("timedelta64[ns]")
+        self.df_int = self.df.copy().astype("int64")
+
+    def time_groupby_sum_timedelta(self):
+        self.df.groupby(lambda x: x).sum()
+
+    def time_groupby_sum_int(self):
+        self.df_int.groupby(lambda x: x).sum()
+
+
 class Transform:
     def setup(self):
         n1 = 400

ci/deps/actions-310.yaml

@@ -6,8 +6,8 @@ dependencies:
 
   # build dependencies
   - versioneer[toml]
-  - cython>=0.29.33, <3.0.1
-  - meson[ninja]=1.0.1
+  - cython>=0.29.33
+  - meson[ninja]=1.2.1
   - meson-python=0.13.1
 
   # test dependencies
@@ -34,7 +34,7 @@ dependencies:
   - gcsfs>=2022.05.0
   - jinja2>=3.1.2
   - lxml>=4.8.0
-  - matplotlib>=3.6.1
+  - matplotlib>=3.6.1, <3.8
   - numba>=0.55.2
   - numexpr>=2.8.0
   - odfpy>=1.4.1

ci/deps/actions-311-downstream_compat.yaml

@@ -7,8 +7,8 @@ dependencies:
 
   # build dependencies
   - versioneer[toml]
-  - cython>=0.29.33, <3.0.1
-  - meson[ninja]=1.0.1
+  - cython>=0.29.33
+  - meson[ninja]=1.2.1
   - meson-python=0.13.1
 
   # test dependencies
@@ -35,7 +35,7 @@ dependencies:
   - gcsfs>=2022.05.0
   - jinja2>=3.1.2
   - lxml>=4.8.0
-  - matplotlib>=3.6.1
+  - matplotlib>=3.6.1, <3.8
   - numba>=0.55.2
   - numexpr>=2.8.0
   - odfpy>=1.4.1

ci/deps/actions-311-numpydev.yaml

@@ -6,8 +6,9 @@ dependencies:
 
   # build dependencies
   - versioneer[toml]
-  - meson[ninja]=1.0.1
+  - meson[ninja]=1.2.1
   - meson-python=0.13.1
+  - cython>=0.29.33
 
   # test dependencies
   - pytest>=7.3.2
@@ -25,7 +26,6 @@ dependencies:
   - pip
 
   - pip:
-    - "cython<3.0.1"
     - "--extra-index-url https://pypi.anaconda.org/scientific-python-nightly-wheels/simple"
     - "--pre"
     - "numpy"

ci/deps/actions-311-pyarrownightly.yaml

@@ -6,8 +6,8 @@ dependencies:
 
   # build dependencies
   - versioneer[toml]
-  - meson[ninja]=1.0.1
-  - cython>=0.29.33, <3.0.1
+  - meson[ninja]=1.2.1
+  - cython>=0.29.33
   - meson-python=0.13.1
 
   # test dependencies

ci/deps/actions-311.yaml

@@ -6,8 +6,8 @@ dependencies:
 
   # build dependencies
   - versioneer[toml]
-  - cython>=0.29.33, <3.0.1
-  - meson[ninja]=1.0.1
+  - cython>=0.29.33
+  - meson[ninja]=1.2.1
   - meson-python=0.13.1
 
   # test dependencies
@@ -34,7 +34,7 @@ dependencies:
   - gcsfs>=2022.05.0
   - jinja2>=3.1.2
   - lxml>=4.8.0
-  - matplotlib>=3.6.1
+  - matplotlib>=3.6.1, <3.8
   - numba>=0.55.2
   - numexpr>=2.8.0
   - odfpy>=1.4.1

ci/deps/actions-39-minimum_versions.yaml

@@ -8,8 +8,8 @@ dependencies:
 
   # build dependencies
   - versioneer[toml]
-  - cython>=0.29.33, <3.0.1
-  - meson[ninja]=1.0.1
+  - cython>=0.29.33
+  - meson[ninja]=1.2.1
   - meson-python=0.13.1
 
   # test dependencies

ci/deps/actions-39.yaml

@@ -6,8 +6,8 @@ dependencies:
 
   # build dependencies
   - versioneer[toml]
-  - cython>=0.29.33, <3.0.1
-  - meson[ninja]=1.0.1
+  - cython>=0.29.33
+  - meson[ninja]=1.2.1
   - meson-python=0.13.1
 
   # test dependencies
@@ -34,7 +34,7 @@ dependencies:
   - gcsfs>=2022.05.0
   - jinja2>=3.1.2
   - lxml>=4.8.0
-  - matplotlib>=3.6.1
+  - matplotlib>=3.6.1, <3.8
   - numba>=0.55.2
   - numexpr>=2.8.0
   - odfpy>=1.4.1

ci/deps/actions-pypy-39.yaml

@@ -9,8 +9,8 @@ dependencies:
 
   # build dependencies
   - versioneer[toml]
-  - cython>=0.29.33, <3.0.1
-  - meson[ninja]=1.0.1
+  - cython>=0.29.33
+  - meson[ninja]=1.2.1
   - meson-python=0.13.1
 
   # test dependencies

ci/deps/circle-310-arm64.yaml

@@ -6,8 +6,8 @@ dependencies:
 
   # build dependencies
   - versioneer[toml]
-  - cython>=0.29.33, <3.0.1
-  - meson[ninja]=1.0.1
+  - cython>=0.29.33
+  - meson[ninja]=1.2.1
   - meson-python=0.13.1
 
   # test dependencies
@@ -34,7 +34,7 @@ dependencies:
   - gcsfs>=2022.05.0
   - jinja2>=3.1.2
   - lxml>=4.8.0
-  - matplotlib>=3.6.1
+  - matplotlib>=3.6.1, <3.8
   # test_numba_vs_cython segfaults with numba 0.57
   - numba>=0.55.2, <0.57.0
   - numexpr>=2.8.0

doc/make.py

@@ -123,14 +123,14 @@ def _sphinx_build(self, kind: str):
 
         Parameters
         ----------
-        kind : {'html', 'latex'}
+        kind : {'html', 'latex', 'linkcheck'}
 
         Examples
         --------
         >>> DocBuilder(num_jobs=4)._sphinx_build('html')
         """
-        if kind not in ("html", "latex"):
-            raise ValueError(f"kind must be html or latex, not {kind}")
+        if kind not in ("html", "latex", "linkcheck"):
+            raise ValueError(f"kind must be html, latex or linkcheck, not {kind}")
 
         cmd = ["sphinx-build", "-b", kind]
         if self.num_jobs:
@@ -288,6 +288,12 @@ def zip_html(self):
         os.chdir(dirname)
         self._run_os("zip", zip_fname, "-r", "-q", *fnames)
 
+    def linkcheck(self):
+        """
+        Check for broken links in the documentation.
+        """
+        return self._sphinx_build("linkcheck")
+
 
 def main():
     cmds = [method for method in dir(DocBuilder) if not method.startswith("_")]

doc/source/conf.py

@@ -162,7 +162,7 @@
 # General information about the project.
 project = "pandas"
 # We have our custom "pandas_footer.html" template, using copyright for the current year
-copyright = f"{datetime.now().year}"
+copyright = f"{datetime.now().year},"
 
 # The version info for the project you're documenting, acts as replacement for
 # |version| and |release|, also used in various other places throughout the

doc/source/getting_started/install.rst

@@ -247,14 +247,14 @@ Dependency                                            Minimum Version    pip ext
 Visualization
 ^^^^^^^^^^^^^
 
-Installable with ``pip install "pandas[plot, output_formatting]"``.
+Installable with ``pip install "pandas[plot, output-formatting]"``.
 
 ========================= ================== ================== =============================================================
 Dependency                Minimum Version    pip extra          Notes
 ========================= ================== ================== =============================================================
 matplotlib                3.6.1              plot               Plotting library
-Jinja2                    3.1.2              output_formatting  Conditional formatting with DataFrame.style
-tabulate                  0.8.10             output_formatting  Printing in Markdown-friendly format (see `tabulate`_)
+Jinja2                    3.1.2              output-formatting  Conditional formatting with DataFrame.style
+tabulate                  0.8.10             output-formatting  Printing in Markdown-friendly format (see `tabulate`_)
 ========================= ================== ================== =============================================================
 
 Computation

doc/source/getting_started/intro_tutorials/09_timeseries.rst

@@ -295,7 +295,7 @@ Aggregate the current hourly time series values to the monthly maximum value in
 
 .. ipython:: python
 
-    monthly_max = no_2.resample("M").max()
+    monthly_max = no_2.resample("ME").max()
     monthly_max
 
 A very powerful method on time series data with a datetime index, is the