From 38e29ab7d049015b428a880ed0a160b10418fae6 Mon Sep 17 00:00:00 2001 From: jbrockmendel Date: Wed, 22 Nov 2023 12:51:04 -0800 Subject: [PATCH] BUG: DatetimeIndex.shift with non-nano (#56117) * BUG: DatetimeIndex.shift with non-nano * BUG: DatetimeIndex.shift with non-nano --- doc/source/whatsnew/v2.2.0.rst | 1 + pandas/core/indexes/datetimelike.py | 2 +- .../indexes/datetimes/methods/test_shift.py | 67 ++++++++++--------- 3 files changed, 39 insertions(+), 31 deletions(-) diff --git a/doc/source/whatsnew/v2.2.0.rst b/doc/source/whatsnew/v2.2.0.rst index 6b2c24c4e8c9f..f155fb27d0de5 100644 --- a/doc/source/whatsnew/v2.2.0.rst +++ b/doc/source/whatsnew/v2.2.0.rst @@ -447,6 +447,7 @@ Datetimelike - Bug in :func:`testing.assert_extension_array_equal` that could use the wrong unit when comparing resolutions (:issue:`55730`) - Bug in :func:`to_datetime` and :class:`DatetimeIndex` when passing a list of mixed-string-and-numeric types incorrectly raising (:issue:`55780`) - Bug in :func:`to_datetime` and :class:`DatetimeIndex` when passing mixed-type objects with a mix of timezones or mix of timezone-awareness failing to raise ``ValueError`` (:issue:`55693`) +- Bug in :meth:`DatetimeIndex.shift` with non-nanosecond resolution incorrectly returning with nanosecond resolution (:issue:`56117`) - Bug in :meth:`DatetimeIndex.union` returning object dtype for tz-aware indexes with the same timezone but different units (:issue:`55238`) - Bug in :meth:`Index.is_monotonic_increasing` and :meth:`Index.is_monotonic_decreasing` always caching :meth:`Index.is_unique` as ``True`` when first value in index is ``NaT`` (:issue:`55755`) - Bug in :meth:`Index.view` to a datetime64 dtype with non-supported resolution incorrectly raising (:issue:`55710`) diff --git a/pandas/core/indexes/datetimelike.py b/pandas/core/indexes/datetimelike.py index a3e6c50b21642..264ca8aa11495 100644 --- a/pandas/core/indexes/datetimelike.py +++ b/pandas/core/indexes/datetimelike.py @@ -518,7 +518,7 @@ def shift(self, periods: int = 1, freq=None) -> Self: # appropriate timezone from `start` and `end`, so tz does not need # to be passed explicitly. result = self._data._generate_range( - start=start, end=end, periods=None, freq=self.freq + start=start, end=end, periods=None, freq=self.freq, unit=self.unit ) return type(self)._simple_new(result, name=self.name) diff --git a/pandas/tests/indexes/datetimes/methods/test_shift.py b/pandas/tests/indexes/datetimes/methods/test_shift.py index c50f75894d810..d8bdcc2a17685 100644 --- a/pandas/tests/indexes/datetimes/methods/test_shift.py +++ b/pandas/tests/indexes/datetimes/methods/test_shift.py @@ -20,10 +20,10 @@ class TestDatetimeIndexShift: # ------------------------------------------------------------- # DatetimeIndex.shift is used in integer addition - def test_dti_shift_tzaware(self, tz_naive_fixture): + def test_dti_shift_tzaware(self, tz_naive_fixture, unit): # GH#9903 tz = tz_naive_fixture - idx = DatetimeIndex([], name="xxx", tz=tz) + idx = DatetimeIndex([], name="xxx", tz=tz).as_unit(unit) tm.assert_index_equal(idx.shift(0, freq="h"), idx) tm.assert_index_equal(idx.shift(3, freq="h"), idx) @@ -32,30 +32,31 @@ def test_dti_shift_tzaware(self, tz_naive_fixture): name="xxx", tz=tz, freq="h", - ) + ).as_unit(unit) tm.assert_index_equal(idx.shift(0, freq="h"), idx) exp = DatetimeIndex( ["2011-01-01 13:00", "2011-01-01 14:00", "2011-01-01 15:00"], name="xxx", tz=tz, freq="h", - ) + ).as_unit(unit) tm.assert_index_equal(idx.shift(3, freq="h"), exp) exp = DatetimeIndex( ["2011-01-01 07:00", "2011-01-01 08:00", "2011-01-01 09:00"], name="xxx", tz=tz, freq="h", - ) + ).as_unit(unit) tm.assert_index_equal(idx.shift(-3, freq="h"), exp) - def test_dti_shift_freqs(self): + def test_dti_shift_freqs(self, unit): # test shift for DatetimeIndex and non DatetimeIndex # GH#8083 - drange = date_range("20130101", periods=5) + drange = date_range("20130101", periods=5, unit=unit) result = drange.shift(1) expected = DatetimeIndex( ["2013-01-02", "2013-01-03", "2013-01-04", "2013-01-05", "2013-01-06"], + dtype=f"M8[{unit}]", freq="D", ) tm.assert_index_equal(result, expected) @@ -63,6 +64,7 @@ def test_dti_shift_freqs(self): result = drange.shift(-1) expected = DatetimeIndex( ["2012-12-31", "2013-01-01", "2013-01-02", "2013-01-03", "2013-01-04"], + dtype=f"M8[{unit}]", freq="D", ) tm.assert_index_equal(result, expected) @@ -70,12 +72,13 @@ def test_dti_shift_freqs(self): result = drange.shift(3, freq="2D") expected = DatetimeIndex( ["2013-01-07", "2013-01-08", "2013-01-09", "2013-01-10", "2013-01-11"], + dtype=f"M8[{unit}]", freq="D", ) tm.assert_index_equal(result, expected) - def test_dti_shift_int(self): - rng = date_range("1/1/2000", periods=20) + def test_dti_shift_int(self, unit): + rng = date_range("1/1/2000", periods=20, unit=unit) result = rng + 5 * rng.freq expected = rng.shift(5) @@ -85,25 +88,27 @@ def test_dti_shift_int(self): expected = rng.shift(-5) tm.assert_index_equal(result, expected) - def test_dti_shift_no_freq(self): + def test_dti_shift_no_freq(self, unit): # GH#19147 - dti = DatetimeIndex(["2011-01-01 10:00", "2011-01-01"], freq=None) + dti = DatetimeIndex(["2011-01-01 10:00", "2011-01-01"], freq=None).as_unit(unit) with pytest.raises(NullFrequencyError, match="Cannot shift with no freq"): dti.shift(2) @pytest.mark.parametrize("tzstr", ["US/Eastern", "dateutil/US/Eastern"]) - def test_dti_shift_localized(self, tzstr): - dr = date_range("2011/1/1", "2012/1/1", freq="W-FRI") + def test_dti_shift_localized(self, tzstr, unit): + dr = date_range("2011/1/1", "2012/1/1", freq="W-FRI", unit=unit) dr_tz = dr.tz_localize(tzstr) result = dr_tz.shift(1, "10min") assert result.tz == dr_tz.tz - def test_dti_shift_across_dst(self): + def test_dti_shift_across_dst(self, unit): # GH 8616 - idx = date_range("2013-11-03", tz="America/Chicago", periods=7, freq="h") - s = Series(index=idx[:-1], dtype=object) - result = s.shift(freq="h") + idx = date_range( + "2013-11-03", tz="America/Chicago", periods=7, freq="h", unit=unit + ) + ser = Series(index=idx[:-1], dtype=object) + result = ser.shift(freq="h") expected = Series(index=idx[1:], dtype=object) tm.assert_series_equal(result, expected) @@ -115,24 +120,26 @@ def test_dti_shift_across_dst(self): [1, "2014-11-14 01:00:00"], ], ) - def test_dti_shift_near_midnight(self, shift, result_time): + def test_dti_shift_near_midnight(self, shift, result_time, unit): # GH 8616 dt = datetime(2014, 11, 14, 0) dt_est = pytz.timezone("EST").localize(dt) - s = Series(data=[1], index=[dt_est]) - result = s.shift(shift, freq="h") - expected = Series(1, index=DatetimeIndex([result_time], tz="EST")) + idx = DatetimeIndex([dt_est]).as_unit(unit) + ser = Series(data=[1], index=idx) + result = ser.shift(shift, freq="h") + exp_index = DatetimeIndex([result_time], tz="EST").as_unit(unit) + expected = Series(1, index=exp_index) tm.assert_series_equal(result, expected) - def test_shift_periods(self): + def test_shift_periods(self, unit): # GH#22458 : argument 'n' was deprecated in favor of 'periods' - idx = date_range(start=START, end=END, periods=3) + idx = date_range(start=START, end=END, periods=3, unit=unit) tm.assert_index_equal(idx.shift(periods=0), idx) tm.assert_index_equal(idx.shift(0), idx) @pytest.mark.parametrize("freq", ["B", "C"]) - def test_shift_bday(self, freq): - rng = date_range(START, END, freq=freq) + def test_shift_bday(self, freq, unit): + rng = date_range(START, END, freq=freq, unit=unit) shifted = rng.shift(5) assert shifted[0] == rng[5] assert shifted.freq == rng.freq @@ -145,18 +152,18 @@ def test_shift_bday(self, freq): assert shifted[0] == rng[0] assert shifted.freq == rng.freq - def test_shift_bmonth(self): - rng = date_range(START, END, freq=pd.offsets.BMonthEnd()) + def test_shift_bmonth(self, unit): + rng = date_range(START, END, freq=pd.offsets.BMonthEnd(), unit=unit) shifted = rng.shift(1, freq=pd.offsets.BDay()) assert shifted[0] == rng[0] + pd.offsets.BDay() - rng = date_range(START, END, freq=pd.offsets.BMonthEnd()) + rng = date_range(START, END, freq=pd.offsets.BMonthEnd(), unit=unit) with tm.assert_produces_warning(pd.errors.PerformanceWarning): shifted = rng.shift(1, freq=pd.offsets.CDay()) assert shifted[0] == rng[0] + pd.offsets.CDay() - def test_shift_empty(self): + def test_shift_empty(self, unit): # GH#14811 - dti = date_range(start="2016-10-21", end="2016-10-21", freq="BME") + dti = date_range(start="2016-10-21", end="2016-10-21", freq="BME", unit=unit) result = dti.shift(1) tm.assert_index_equal(result, dti)