From 2de134b62ba50ff44ef7909d525afeb8bb442f3d Mon Sep 17 00:00:00 2001 From: jrmylow Date: Sat, 25 Nov 2023 07:58:28 +1100 Subject: [PATCH 1/8] #56134 removed rollback from endpoint --- pandas/core/arrays/datetimes.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/pandas/core/arrays/datetimes.py b/pandas/core/arrays/datetimes.py index d12c2e7c4ae3b..45bd7429a2a19 100644 --- a/pandas/core/arrays/datetimes.py +++ b/pandas/core/arrays/datetimes.py @@ -2758,11 +2758,6 @@ def _generate_range( # variable has type "Optional[Timestamp]") start = offset.rollforward(start) # type: ignore[assignment] - elif end and not offset.is_on_offset(end): - # Incompatible types in assignment (expression has type "datetime", - # variable has type "Optional[Timestamp]") - end = offset.rollback(end) # type: ignore[assignment] - # Unsupported operand types for < ("Timestamp" and "None") if periods is None and end < start and offset.n >= 0: # type: ignore[operator] end = None From 8d0a2649839e4268163ba31a09d2646cd534249e Mon Sep 17 00:00:00 2001 From: jrmylow Date: Sat, 25 Nov 2023 08:20:05 +1100 Subject: [PATCH 2/8] #56134 test --- pandas/tests/indexes/datetimes/test_date_range.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/pandas/tests/indexes/datetimes/test_date_range.py b/pandas/tests/indexes/datetimes/test_date_range.py index dfad4d7ad01ea..4b4d1cc1afcec 100644 --- a/pandas/tests/indexes/datetimes/test_date_range.py +++ b/pandas/tests/indexes/datetimes/test_date_range.py @@ -1533,6 +1533,21 @@ def test_date_range_year_end(self, unit): ) tm.assert_index_equal(rng, exp) + def test_date_range_partial_day_year_end(self, unit): + # GH#56134 + rng = date_range( + start = "2021-12-31 00:00:01", + end = "2023-10-31 00:00:00", + freq="YE", + unit=unit, + ) + exp = DatetimeIndex( + ["2021-12-31 00:00:01", "2022-12-31 00:00:01"], + dtype=f"M8[{unit}]", + freq="YE", + ) + tm.assert_index_equal(rng, exp) + def test_date_range_negative_freq_year_end(self, unit): # GH#11018 rng = date_range("2011-12-31", freq="-2YE", periods=3, unit=unit) From 50386d81373672a23d82d9036a1d4862fa2096fe Mon Sep 17 00:00:00 2001 From: jrmylow Date: Sat, 25 Nov 2023 08:22:37 +1100 Subject: [PATCH 3/8] #56147 tests --- pandas/tests/indexes/datetimes/test_date_range.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/pandas/tests/indexes/datetimes/test_date_range.py b/pandas/tests/indexes/datetimes/test_date_range.py index 4b4d1cc1afcec..090384bd48292 100644 --- a/pandas/tests/indexes/datetimes/test_date_range.py +++ b/pandas/tests/indexes/datetimes/test_date_range.py @@ -1557,6 +1557,21 @@ def test_date_range_negative_freq_year_end(self, unit): tm.assert_index_equal(rng, exp) assert rng.freq == "-2YE" + def test_date_range_negative_freq_year_end_inbounds(self, unit): + # GH#56147 + rng = date_range( + start = "2023-10-31 00:00:00", + end = "2021-10-31 00:00:00", + freq="-1YE", + unit=unit, + ) + exp = DatetimeIndex( + ["2022-12-31 00:00:00", "2021-12-31 00:00:00"], + dtype=f"M8[{unit}]", + freq="-1YE", + ) + tm.assert_index_equal(rng, exp) + def test_date_range_business_year_end_year(self, unit): # see GH#9313 rng = date_range("1/1/2013", "7/1/2017", freq="BYE", unit=unit) From e96a9a0b9193e8a4502335944b58cf2376035801 Mon Sep 17 00:00:00 2001 From: jrmylow Date: Sat, 25 Nov 2023 08:24:45 +1100 Subject: [PATCH 4/8] #56147 negative offset and year end interaction --- pandas/core/arrays/datetimes.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/pandas/core/arrays/datetimes.py b/pandas/core/arrays/datetimes.py index 45bd7429a2a19..fc9ef255a2447 100644 --- a/pandas/core/arrays/datetimes.py +++ b/pandas/core/arrays/datetimes.py @@ -2756,7 +2756,10 @@ def _generate_range( if start and not offset.is_on_offset(start): # Incompatible types in assignment (expression has type "datetime", # variable has type "Optional[Timestamp]") - start = offset.rollforward(start) # type: ignore[assignment] + if offset.n >= 0: + start = offset.rollforward(start) # type: ignore[assignment] + else: + start = offset.rollback(start) # type: ignore[assignment] # Unsupported operand types for < ("Timestamp" and "None") if periods is None and end < start and offset.n >= 0: # type: ignore[operator] From 61ce977c6010bfff90adc31f6ecf4ecee01978aa Mon Sep 17 00:00:00 2001 From: jrmylow Date: Sat, 25 Nov 2023 08:27:35 +1100 Subject: [PATCH 5/8] commented to trace back to issue --- pandas/core/arrays/datetimes.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pandas/core/arrays/datetimes.py b/pandas/core/arrays/datetimes.py index fc9ef255a2447..3b678f2eabfaf 100644 --- a/pandas/core/arrays/datetimes.py +++ b/pandas/core/arrays/datetimes.py @@ -2756,6 +2756,8 @@ def _generate_range( if start and not offset.is_on_offset(start): # Incompatible types in assignment (expression has type "datetime", # variable has type "Optional[Timestamp]") + + # GH #56147 account for negative direction and range bounds if offset.n >= 0: start = offset.rollforward(start) # type: ignore[assignment] else: From 1d57c568ff03d6ff8dc584d801d02982d6d05d97 Mon Sep 17 00:00:00 2001 From: jrmylow Date: Sat, 25 Nov 2023 08:46:14 +1100 Subject: [PATCH 6/8] reformatted with black --- pandas/core/arrays/datetimes.py | 2 +- pandas/tests/indexes/datetimes/test_date_range.py | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/pandas/core/arrays/datetimes.py b/pandas/core/arrays/datetimes.py index 3b678f2eabfaf..36a73cccab6e4 100644 --- a/pandas/core/arrays/datetimes.py +++ b/pandas/core/arrays/datetimes.py @@ -2761,7 +2761,7 @@ def _generate_range( if offset.n >= 0: start = offset.rollforward(start) # type: ignore[assignment] else: - start = offset.rollback(start) # type: ignore[assignment] + start = offset.rollback(start) # type: ignore[assignment] # Unsupported operand types for < ("Timestamp" and "None") if periods is None and end < start and offset.n >= 0: # type: ignore[operator] diff --git a/pandas/tests/indexes/datetimes/test_date_range.py b/pandas/tests/indexes/datetimes/test_date_range.py index 090384bd48292..c3404bb5b9db6 100644 --- a/pandas/tests/indexes/datetimes/test_date_range.py +++ b/pandas/tests/indexes/datetimes/test_date_range.py @@ -1536,8 +1536,8 @@ def test_date_range_year_end(self, unit): def test_date_range_partial_day_year_end(self, unit): # GH#56134 rng = date_range( - start = "2021-12-31 00:00:01", - end = "2023-10-31 00:00:00", + start="2021-12-31 00:00:01", + end="2023-10-31 00:00:00", freq="YE", unit=unit, ) @@ -1560,8 +1560,8 @@ def test_date_range_negative_freq_year_end(self, unit): def test_date_range_negative_freq_year_end_inbounds(self, unit): # GH#56147 rng = date_range( - start = "2023-10-31 00:00:00", - end = "2021-10-31 00:00:00", + start="2023-10-31 00:00:00", + end="2021-10-31 00:00:00", freq="-1YE", unit=unit, ) @@ -1570,7 +1570,7 @@ def test_date_range_negative_freq_year_end_inbounds(self, unit): dtype=f"M8[{unit}]", freq="-1YE", ) - tm.assert_index_equal(rng, exp) + tm.assert_index_equal(rng, exp) def test_date_range_business_year_end_year(self, unit): # see GH#9313 From d528dfff07eec0a87f0c5f8866dafab6583c38bb Mon Sep 17 00:00:00 2001 From: jrmylow Date: Sat, 25 Nov 2023 21:45:00 +1100 Subject: [PATCH 7/8] documentation --- doc/source/whatsnew/v2.2.0.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/doc/source/whatsnew/v2.2.0.rst b/doc/source/whatsnew/v2.2.0.rst index 8893fe0ecd398..549f81fbcca66 100644 --- a/doc/source/whatsnew/v2.2.0.rst +++ b/doc/source/whatsnew/v2.2.0.rst @@ -592,6 +592,8 @@ Other - Bug in rendering ``inf`` values inside a a :class:`DataFrame` with the ``use_inf_as_na`` option enabled (:issue:`55483`) - Bug in rendering a :class:`Series` with a :class:`MultiIndex` when one of the index level's names is 0 not having that name displayed (:issue:`55415`) - Bug in the error message when assigning an empty dataframe to a column (:issue:`55956`) +- Bug in :func:`date_range` where the last valid timestamp would sometimes not be produced (:issue:`56134`) +- Bug in :func:`date_range` where a timestamp out of the valid range would be produced with a negative ``freq`` parameter (:issue:`56147`) .. ***DO NOT USE THIS SECTION*** From 6733db1dd53fbdd88b71a38c57270208747db7a7 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Sat, 25 Nov 2023 12:29:32 +0000 Subject: [PATCH 8/8] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- doc/source/whatsnew/v2.2.0.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/source/whatsnew/v2.2.0.rst b/doc/source/whatsnew/v2.2.0.rst index 549f81fbcca66..4024fa0e69a12 100644 --- a/doc/source/whatsnew/v2.2.0.rst +++ b/doc/source/whatsnew/v2.2.0.rst @@ -586,14 +586,14 @@ Other ^^^^^ - Bug in :func:`DataFrame.describe` when formatting percentiles in the resulting percentile 99.999% is rounded to 100% (:issue:`55765`) - Bug in :func:`cut` incorrectly allowing cutting of timezone-aware datetimes with timezone-naive bins (:issue:`54964`) +- Bug in :func:`date_range` where a timestamp out of the valid range would be produced with a negative ``freq`` parameter (:issue:`56147`) +- Bug in :func:`date_range` where the last valid timestamp would sometimes not be produced (:issue:`56134`) - Bug in :func:`infer_freq` and :meth:`DatetimeIndex.inferred_freq` with weekly frequencies and non-nanosecond resolutions (:issue:`55609`) - Bug in :meth:`DataFrame.apply` where passing ``raw=True`` ignored ``args`` passed to the applied function (:issue:`55009`) - Bug in :meth:`Dataframe.from_dict` which would always sort the rows of the created :class:`DataFrame`. (:issue:`55683`) - Bug in rendering ``inf`` values inside a a :class:`DataFrame` with the ``use_inf_as_na`` option enabled (:issue:`55483`) - Bug in rendering a :class:`Series` with a :class:`MultiIndex` when one of the index level's names is 0 not having that name displayed (:issue:`55415`) - Bug in the error message when assigning an empty dataframe to a column (:issue:`55956`) -- Bug in :func:`date_range` where the last valid timestamp would sometimes not be produced (:issue:`56134`) -- Bug in :func:`date_range` where a timestamp out of the valid range would be produced with a negative ``freq`` parameter (:issue:`56147`) .. ***DO NOT USE THIS SECTION***