Merge remote-tracking branch 'upstream/main' into hardcode-allowed-fr…

…eqs-for-hack

.github/workflows/package-checks.yml

@@ -24,7 +24,7 @@ jobs:
     runs-on: ubuntu-22.04
     strategy:
       matrix:
-        extra: ["test", "performance", "computation", "fss", "aws", "gcp", "excel", "parquet", "feather", "hdf5", "spss", "postgresql", "mysql", "sql-other", "html", "xml", "plot", "output_formatting", "clipboard", "compression", "consortium-standard", "all"]
+        extra: ["test", "performance", "computation", "fss", "aws", "gcp", "excel", "parquet", "feather", "hdf5", "spss", "postgresql", "mysql", "sql-other", "html", "xml", "plot", "output-formatting", "clipboard", "compression", "consortium-standard", "all"]
       fail-fast: false
     name: Install Extras - ${{ matrix.extra }}
     concurrency:

.github/workflows/wheels.yml

@@ -138,7 +138,7 @@ jobs:
         run: echo "sdist_name=$(cd ./dist && ls -d */)" >> "$GITHUB_ENV"
 
       - name: Build wheels
-        uses: pypa/cibuildwheel@v2.15.0
+        uses: pypa/cibuildwheel@v2.16.0
         with:
          package-dir: ./dist/${{ matrix.buildplat[1] == 'macosx_*' && env.sdist_name || needs.build_sdist.outputs.sdist_file }}
         env:

.pre-commit-config.yaml

@@ -20,11 +20,11 @@ ci:
 repos:
 -   repo: https://github.com/hauntsaninja/black-pre-commit-mirror
     # black compiled with mypyc
-    rev: 23.7.0
+    rev: 23.9.1
     hooks:
       - id: black
 -   repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.0.287
+    rev: v0.0.291
     hooks:
     -   id: ruff
         args: [--exit-non-zero-on-fix]
@@ -107,7 +107,7 @@ repos:
     hooks:
     -   id: isort
 -   repo: https://github.com/asottile/pyupgrade
-    rev: v3.10.1
+    rev: v3.13.0
     hooks:
     -   id: pyupgrade
         args: [--py39-plus]

asv_bench/benchmarks/groupby.py

@@ -841,6 +841,23 @@ def time_groupby_sum_multiindex(self):
         self.df.groupby(level=[0, 1]).sum()
 
 
+class SumTimeDelta:
+    # GH 20660
+    def setup(self):
+        N = 10**4
+        self.df = DataFrame(
+            np.random.randint(1000, 100000, (N, 100)),
+            index=np.random.randint(200, size=(N,)),
+        ).astype("timedelta64[ns]")
+        self.df_int = self.df.copy().astype("int64")
+
+    def time_groupby_sum_timedelta(self):
+        self.df.groupby(lambda x: x).sum()
+
+    def time_groupby_sum_int(self):
+        self.df_int.groupby(lambda x: x).sum()
+
+
 class Transform:
     def setup(self):
         n1 = 400

doc/source/getting_started/install.rst

@@ -247,14 +247,14 @@ Dependency                                            Minimum Version    pip ext
 Visualization
 ^^^^^^^^^^^^^
 
-Installable with ``pip install "pandas[plot, output_formatting]"``.
+Installable with ``pip install "pandas[plot, output-formatting]"``.
 
 ========================= ================== ================== =============================================================
 Dependency                Minimum Version    pip extra          Notes
 ========================= ================== ================== =============================================================
 matplotlib                3.6.1              plot               Plotting library
-Jinja2                    3.1.2              output_formatting  Conditional formatting with DataFrame.style
-tabulate                  0.8.10             output_formatting  Printing in Markdown-friendly format (see `tabulate`_)
+Jinja2                    3.1.2              output-formatting  Conditional formatting with DataFrame.style
+tabulate                  0.8.10             output-formatting  Printing in Markdown-friendly format (see `tabulate`_)
 ========================= ================== ================== =============================================================
 
 Computation

doc/source/whatsnew/v2.1.2.rst

@@ -29,7 +29,7 @@ Bug fixes
 
 Other
 ~~~~~
--
+- Fixed non-working installation of optional dependency group ``output_formatting``. Replacing underscore ``_`` with a dash ``-`` fixes broken dependency resolution. A correct way to use now is ``pip install pandas[output-formatting]``.
 -
 
 .. ---------------------------------------------------------------------------

doc/source/whatsnew/v2.2.0.rst

@@ -259,7 +259,7 @@ Categorical
 
 Datetimelike
 ^^^^^^^^^^^^
--
+- Bug in :meth:`DatetimeIndex.union` returning object dtype for tz-aware indexes with the same timezone but different units (:issue:`55238`)
 -
 
 Timedelta
@@ -292,6 +292,7 @@ Interval
 - Bug in :class:`Interval` ``__repr__`` not displaying UTC offsets for :class:`Timestamp` bounds. Additionally the hour, minute and second components will now be shown. (:issue:`55015`)
 - Bug in :meth:`IntervalIndex.get_indexer` with datetime or timedelta intervals incorrectly matching on integer targets (:issue:`47772`)
 - Bug in :meth:`IntervalIndex.get_indexer` with timezone-aware datetime intervals incorrectly matching on a sequence of timezone-naive targets (:issue:`47772`)
+- Bug in setting values on a :class:`Series` with an :class:`IntervalIndex` using a slice incorrectly raising (:issue:`54722`)
 -
 
 Indexing

pandas/core/dtypes/dtypes.py

@@ -928,6 +928,13 @@ def __setstate__(self, state) -> None:
         self._tz = state["tz"]
         self._unit = state["unit"]
 
+    def _get_common_dtype(self, dtypes: list[DtypeObj]) -> DtypeObj | None:
+        if all(isinstance(t, DatetimeTZDtype) and t.tz == self.tz for t in dtypes):
+            np_dtype = np.max([cast(DatetimeTZDtype, t).base for t in [self, *dtypes]])
+            unit = np.datetime_data(np_dtype)[0]
+            return type(self)(unit=unit, tz=self.tz)
+        return super()._get_common_dtype(dtypes)
+
     @cache_readonly
     def index_class(self) -> type_t[DatetimeIndex]:
         from pandas import DatetimeIndex

pandas/core/indexes/base.py

@@ -4204,15 +4204,9 @@ def _convert_slice_indexer(self, key: slice, kind: Literal["loc", "getitem"]):
                 self._validate_indexer("slice", key.step, "getitem")
                 return key
 
-        # convert the slice to an indexer here
-
-        # special case for interval_dtype bc we do not do partial-indexing
-        #  on integer Intervals when slicing
-        # TODO: write this in terms of e.g. should_partial_index?
-        ints_are_positional = self._should_fallback_to_positional or isinstance(
-            self.dtype, IntervalDtype
-        )
-        is_positional = is_index_slice and ints_are_positional
+        # convert the slice to an indexer here; checking that the user didn't
+        #  pass a positional slice to loc
+        is_positional = is_index_slice and self._should_fallback_to_positional
 
         # if we are mixed and have integers
         if is_positional:

pandas/tests/arithmetic/conftest.py

@@ -7,23 +7,8 @@
     RangeIndex,
 )
 import pandas._testing as tm
-from pandas.core.computation import expressions as expr
 
 
-@pytest.fixture(autouse=True, params=[0, 1000000], ids=["numexpr", "python"])
-def switch_numexpr_min_elements(request):
-    _MIN_ELEMENTS = expr._MIN_ELEMENTS
-    expr._MIN_ELEMENTS = request.param
-    yield request.param
-    expr._MIN_ELEMENTS = _MIN_ELEMENTS
-
-
-# ------------------------------------------------------------------
-
-
-# doctest with +SKIP for one fixture fails during setup with
-# 'DoctestItem' object has no attribute 'callspec'
-# due to switch_numexpr_min_elements fixture
 @pytest.fixture(params=[1, np.array(1, dtype=np.int64)])
 def one(request):
     """
@@ -58,9 +43,6 @@ def one(request):
 zeros.extend([0, 0.0, -0.0])
 
 
-# doctest with +SKIP for zero fixture fails during setup with
-# 'DoctestItem' object has no attribute 'callspec'
-# due to switch_numexpr_min_elements fixture
 @pytest.fixture(params=zeros)
 def zero(request):
     """

pandas/tests/arithmetic/test_numeric.py

@@ -29,6 +29,13 @@
 )
 
 
+@pytest.fixture(autouse=True, params=[0, 1000000], ids=["numexpr", "python"])
+def switch_numexpr_min_elements(request, monkeypatch):
+    with monkeypatch.context() as m:
+        m.setattr(expr, "_MIN_ELEMENTS", request.param)
+        yield request.param
+
+
 @pytest.fixture(params=[Index, Series, tm.to_array])
 def box_pandas_1d_array(request):
     """

pandas/tests/frame/test_arithmetic.py

@@ -33,11 +33,10 @@
 
 
 @pytest.fixture(autouse=True, params=[0, 1000000], ids=["numexpr", "python"])
-def switch_numexpr_min_elements(request):
-    _MIN_ELEMENTS = expr._MIN_ELEMENTS
-    expr._MIN_ELEMENTS = request.param
-    yield request.param
-    expr._MIN_ELEMENTS = _MIN_ELEMENTS
+def switch_numexpr_min_elements(request, monkeypatch):
+    with monkeypatch.context() as m:
+        m.setattr(expr, "_MIN_ELEMENTS", request.param)
+        yield request.param
 
 
 class DummyElement:
@@ -1074,7 +1073,7 @@ def test_frame_with_frame_reindex(self):
         ],
         ids=lambda x: x.__name__,
     )
-    def test_binop_other(self, op, value, dtype, switch_numexpr_min_elements, request):
+    def test_binop_other(self, op, value, dtype, switch_numexpr_min_elements):
         skip = {
             (operator.truediv, "bool"),
             (operator.pow, "bool"),

pandas/tests/indexes/datetimes/test_setops.py

@@ -189,6 +189,14 @@ def test_union_with_DatetimeIndex(self, sort):
         # Fails with "AttributeError: can't set attribute"
         i2.union(i1, sort=sort)
 
+    def test_union_same_timezone_different_units(self):
+        # GH 55238
+        idx1 = date_range("2000-01-01", periods=3, tz="UTC").as_unit("ms")
+        idx2 = date_range("2000-01-01", periods=3, tz="UTC").as_unit("us")
+        result = idx1.union(idx2)
+        expected = date_range("2000-01-01", periods=3, tz="UTC").as_unit("us")
+        tm.assert_index_equal(result, expected)
+
     # TODO: moved from test_datetimelike; de-duplicate with version below
     def test_intersection2(self):
         first = tm.makeDateIndex(10)

pandas/tests/indexing/interval/test_interval.py

@@ -134,6 +134,33 @@ def test_getitem_interval_with_nans(self, frame_or_series, indexer_sl):
 
         tm.assert_equal(result, expected)
 
+    def test_setitem_interval_with_slice(self):
+        # GH#54722
+        ii = IntervalIndex.from_breaks(range(4, 15))
+        ser = Series(range(10), index=ii)
+
+        orig = ser.copy()
+
+        # This should be a no-op (used to raise)
+        ser.loc[1:3] = 20
+        tm.assert_series_equal(ser, orig)
+
+        ser.loc[6:8] = 19
+        orig.iloc[1:4] = 19
+        tm.assert_series_equal(ser, orig)
+
+        ser2 = Series(range(5), index=ii[::2])
+        orig2 = ser2.copy()
+
+        # this used to raise
+        ser2.loc[6:8] = 22  # <- raises on main, sets on branch
+        orig2.iloc[1] = 22
+        tm.assert_series_equal(ser2, orig2)
+
+        ser2.loc[5:7] = 21
+        orig2.iloc[:2] = 21
+        tm.assert_series_equal(ser2, orig2)
+
 
 class TestIntervalIndexInsideMultiIndex:
     def test_mi_intervalindex_slicing_with_scalar(self):

pandas/tests/io/formats/test_format.py

@@ -22,8 +22,6 @@
 import pytest
 import pytz
 
-from pandas._config import config
-
 import pandas as pd
 from pandas import (
     DataFrame,
@@ -51,17 +49,6 @@ def get_local_am_pm():
     return am_local, pm_local
 
 
-@pytest.fixture(autouse=True)
-def clean_config():
-    curr_deprecated_options = config._deprecated_options.copy()
-    curr_registered_options = config._registered_options.copy()
-    curr_global_config = config._global_config.copy()
-    yield
-    config._deprecated_options = curr_deprecated_options
-    config._registered_options = curr_registered_options
-    config._global_config = curr_global_config
-
-
 @pytest.fixture(params=["string", "pathlike", "buffer"])
 def filepath_or_buffer_id(request):
     """
@@ -3604,7 +3591,7 @@ def test_repr_html_ipython_config(ip):
     df._repr_html_()
     """
     )
-    result = ip.run_cell(code)
+    result = ip.run_cell(code, silent=True)
     assert not result.error_in_exec