From fe178189e589711559a3064fcefa4d330b7f41fa Mon Sep 17 00:00:00 2001 From: Hadi Abdi Khojasteh Date: Thu, 26 Oct 2023 04:14:58 +0200 Subject: [PATCH] BUG: .rolling() returns incorrect values when ts index is not nano seconds (#55173) * Fix rolling microseconds for sum * - adding a test for rounding sum * Update pandas/tests/window/test_rolling.py Co-authored-by: Joris Van den Bossche * Update test_rolling.py the df variable name fixed in the test * Reverted version varialbe in doc/source/conf.py * Units generalised to us/ms/s and test parameterised * Rolling max tests added, related to #55026 * whatsnew note for 2.1.2 added * Update doc/source/whatsnew/v2.1.2.rst Co-authored-by: Matthew Roeschke <10647082+mroeschke@users.noreply.github.com> * UTC timezone for _index_array of rolling * Validating tz-aware data Data conversion removed Tests merged * Conversion replaced by Timedelta.as_unit * fixes for failing tests * update whatsnew * type checking --------- Co-authored-by: Joris Van den Bossche Co-authored-by: Matthew Roeschke <10647082+mroeschke@users.noreply.github.com> --- doc/source/whatsnew/v2.1.2.rst | 1 + pandas/core/window/rolling.py | 10 ++++++++- pandas/tests/window/test_rolling.py | 32 +++++++++++++++++++++++++++++ 3 files changed, 42 insertions(+), 1 deletion(-) diff --git a/doc/source/whatsnew/v2.1.2.rst b/doc/source/whatsnew/v2.1.2.rst index 8863bfa9f3f69..16f8e8d264268 100644 --- a/doc/source/whatsnew/v2.1.2.rst +++ b/doc/source/whatsnew/v2.1.2.rst @@ -14,6 +14,7 @@ including other versions of pandas. Fixed regressions ~~~~~~~~~~~~~~~~~ - Fixed regression in :meth:`DataFrame.join` where result has missing values and dtype is arrow backed string (:issue:`55348`) +- Fixed regression in :meth:`~DataFrame.rolling` where non-nanosecond index or ``on`` column would produce incorrect results (:issue:`55026`, :issue:`55106`, :issue:`55299`) - Fixed regression in :meth:`DataFrame.resample` which was extrapolating back to ``origin`` when ``origin`` was outside its bounds (:issue:`55064`) - Fixed regression in :meth:`DataFrame.sort_index` which was not sorting correctly when the index was a sliced :class:`MultiIndex` (:issue:`55379`) - Fixed regression in :meth:`DataFrameGroupBy.agg` and :meth:`SeriesGroupBy.agg` where if the option ``compute.use_numba`` was set to True, groupby methods not supported by the numba engine would raise a ``TypeError`` (:issue:`55520`) diff --git a/pandas/core/window/rolling.py b/pandas/core/window/rolling.py index 72e94d049a9de..f90863a8ea1ef 100644 --- a/pandas/core/window/rolling.py +++ b/pandas/core/window/rolling.py @@ -21,6 +21,7 @@ from pandas._libs.tslibs import ( BaseOffset, + Timedelta, to_offset, ) import pandas._libs.window.aggregations as window_aggregations @@ -112,6 +113,8 @@ from pandas.core.generic import NDFrame from pandas.core.groupby.ops import BaseGrouper +from pandas.core.arrays.datetimelike import dtype_to_unit + class BaseWindow(SelectionMixin): """Provides utilities for performing windowing operations.""" @@ -1887,7 +1890,12 @@ def _validate(self): self._on.freq.nanos / self._on.freq.n ) else: - self._win_freq_i8 = freq.nanos + try: + unit = dtype_to_unit(self._on.dtype) # type: ignore[arg-type] + except TypeError: + # if not a datetime dtype, eg for empty dataframes + unit = "ns" + self._win_freq_i8 = Timedelta(freq.nanos).as_unit(unit)._value # min_periods must be an integer if self.min_periods is None: diff --git a/pandas/tests/window/test_rolling.py b/pandas/tests/window/test_rolling.py index 3fe922539780d..b4f6fde6850a2 100644 --- a/pandas/tests/window/test_rolling.py +++ b/pandas/tests/window/test_rolling.py @@ -1950,3 +1950,35 @@ def test_numeric_only_corr_cov_series(kernel, use_arg, numeric_only, dtype): op2 = getattr(rolling2, kernel) expected = op2(*arg2, numeric_only=numeric_only) tm.assert_series_equal(result, expected) + + +@pytest.mark.parametrize("unit", ["s", "ms", "us", "ns"]) +@pytest.mark.parametrize("tz", [None, "UTC", "Europe/Prague"]) +def test_rolling_timedelta_window_non_nanoseconds(unit, tz): + # Test Sum, GH#55106 + df_time = DataFrame( + {"A": range(5)}, index=date_range("2013-01-01", freq="1s", periods=5, tz=tz) + ) + sum_in_nanosecs = df_time.rolling("1s").sum() + # microseconds / milliseconds should not break the correct rolling + df_time.index = df_time.index.as_unit(unit) + sum_in_microsecs = df_time.rolling("1s").sum() + sum_in_microsecs.index = sum_in_microsecs.index.as_unit("ns") + tm.assert_frame_equal(sum_in_nanosecs, sum_in_microsecs) + + # Test max, GH#55026 + ref_dates = date_range("2023-01-01", "2023-01-10", unit="ns", tz=tz) + ref_series = Series(0, index=ref_dates) + ref_series.iloc[0] = 1 + ref_max_series = ref_series.rolling(Timedelta(days=4)).max() + + dates = date_range("2023-01-01", "2023-01-10", unit=unit, tz=tz) + series = Series(0, index=dates) + series.iloc[0] = 1 + max_series = series.rolling(Timedelta(days=4)).max() + + ref_df = DataFrame(ref_max_series) + df = DataFrame(max_series) + df.index = df.index.as_unit("ns") + + tm.assert_frame_equal(ref_df, df)